From 80ad78e3722fab30182a8e0b5d2da3dbf19f154e Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 18 Apr 2019 14:29:02 -0700 Subject: [PATCH] imagetools: add create support Signed-off-by: Tonis Tiigi --- commands/imagetools/create.go | 201 ++++++++++++++++++++++++++++++++-- util/imagetools/create.go | 186 +++++++++++++++++++++++++++++++ util/imagetools/inspect.go | 29 ++++- 3 files changed, 403 insertions(+), 13 deletions(-) create mode 100644 util/imagetools/create.go diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index e028a6b1..d119192b 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -1,27 +1,214 @@ package commands import ( + "encoding/json" + "fmt" + "io/ioutil" + "strings" + "github.com/docker/cli/cli/command" + "github.com/docker/distribution/reference" + "github.com/moby/buildkit/util/appcontext" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/tonistiigi/buildx/util/imagetools" + "golang.org/x/sync/errgroup" ) type createOptions struct { - files []string - tags []string - dryrun bool - append bool + files []string + tags []string + dryrun bool + actionAppend bool } func runCreate(dockerCli command.Cli, in createOptions, args []string) error { - return errors.Errorf("not-implemented") + if len(args) == 0 && len(in.files) == 0 { + return errors.Errorf("no sources specified") + } + + if !in.dryrun && len(in.tags) == 0 { + return errors.Errorf("can't push with no tags specified, please set --tag or --dry-run") + } + + fileArgs := make([]string, len(in.files)) + for i, f := range in.files { + dt, err := ioutil.ReadFile(f) + if err != nil { + return err + } + fileArgs[i] = string(dt) + } + + args = append(fileArgs, args...) + + tags, err := parseRefs(in.tags) + if err != nil { + return err + } + + if in.actionAppend && len(in.tags) > 0 { + args = append([]string{in.tags[0]}, args...) + } + + srcs, err := parseSources(args) + if err != nil { + return err + } + + repos := map[string]struct{}{} + + for _, t := range tags { + repos[t.Name()] = struct{}{} + } + + sourceRefs := false + for _, s := range srcs { + if s.Ref != nil { + repos[s.Ref.Name()] = struct{}{} + sourceRefs = true + } + } + + if len(repos) == 0 { + return errors.Errorf("no repositories specified, please set a reference in tag or source") + } + if len(repos) > 1 { + return errors.Errorf("multiple repositories currently not supported, found %v", repos) + } + + var repo string + for r := range repos { + repo = r + } + + for i, s := range srcs { + if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" { + n, err := reference.ParseNormalizedNamed(repo) + if err != nil { + return err + } + r, err := reference.WithDigest(n, s.Desc.Digest) + if err != nil { + return err + } + srcs[i].Ref = r + sourceRefs = true + } + } + + ctx := appcontext.Context() + + r := imagetools.New(imagetools.Opt{ + Auth: dockerCli.ConfigFile(), + }) + + if sourceRefs { + eg, ctx2 := errgroup.WithContext(ctx) + for i, s := range srcs { + if s.Ref == nil { + continue + } + func(i int) { + eg.Go(func() error { + _, desc, err := r.Resolve(ctx2, srcs[i].Ref.String()) + if err != nil { + return err + } + srcs[i].Ref = nil + srcs[i].Desc = desc + return nil + }) + }(i) + } + if err := eg.Wait(); err != nil { + return err + } + } + + descs := make([]ocispec.Descriptor, len(srcs)) + for i := range descs { + descs[i] = srcs[i].Desc + } + + dt, desc, err := r.Combine(ctx, repo, descs) + if err != nil { + return err + } + _ = desc + + if in.dryrun { + fmt.Printf("%s\n", dt) + } + + return nil +} + +type src struct { + Desc ocispec.Descriptor + Ref reference.Named +} + +func parseSources(in []string) ([]*src, error) { + out := make([]*src, len(in)) + for i, in := range in { + s, err := parseSource(in) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, refereces and descriptors", in) + } + out[i] = s + } + return out, nil +} + +func parseRefs(in []string) ([]reference.Named, error) { + refs := make([]reference.Named, len(in)) + for i, in := range in { + n, err := reference.ParseNormalizedNamed(in) + if err != nil { + return nil, err + } + refs[i] = n + } + return refs, nil +} + +func parseSource(in string) (*src, error) { + // source can be a digest, reference or a descriptor JSON + dgst, err := digest.Parse(in) + if err == nil { + return &src{ + Desc: ocispec.Descriptor{ + Digest: dgst, + }, + }, nil + } else if strings.HasPrefix(in, "sha256") { + return nil, err + } + + ref, err := reference.ParseNormalizedNamed(in) + if err == nil { + return &src{ + Ref: ref, + }, nil + } else if !strings.HasPrefix(in, "{") { + return nil, err + } + + var s src + if err := json.Unmarshal([]byte(in), &s.Desc); err != nil { + return nil, errors.WithStack(err) + } + return &s, nil } func createCmd(dockerCli command.Cli) *cobra.Command { var options createOptions cmd := &cobra.Command{ - Use: "create [OPTIONS] [SOURCE...]", + Use: "create [OPTIONS] [SOURCE] [SOURCE...]", Short: "Create a new image based on source images", RunE: func(cmd *cobra.Command, args []string) error { return runCreate(dockerCli, options, args) @@ -33,7 +220,7 @@ func createCmd(dockerCli command.Cli) *cobra.Command { flags.StringArrayVarP(&options.files, "file", "f", []string{}, "Read source descriptor from file") flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image") flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing") - flags.BoolVar(&options.append, "append", false, "Append to existing manifest") + flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest") _ = flags diff --git a/util/imagetools/create.go b/util/imagetools/create.go new file mode 100644 index 00000000..2827c58e --- /dev/null +++ b/util/imagetools/create.go @@ -0,0 +1,186 @@ +package imagetools + +import ( + "context" + "encoding/json" + + "github.com/containerd/containerd/images" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) { + ref, err := parseRef(in) + if err != nil { + return nil, ocispec.Descriptor{}, err + } + + eg, ctx := errgroup.WithContext(ctx) + + dts := make([][]byte, len(descs)) + for i := range dts { + func(i int) { + eg.Go(func() error { + dt, err := r.GetDescriptor(ctx, ref.String(), descs[i]) + if err != nil { + return err + } + dts[i] = dt + + if descs[i].MediaType == "" { + mt, err := detectMediaType(dt) + if err != nil { + return err + } + descs[i].MediaType = mt + } + + mt := descs[i].MediaType + + switch mt { + case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + if descs[i].Platform == nil { + cfg, err := r.loadConfig(ctx, in, dt) + if err != nil { + return err + } + descs[i].Platform = &ocispec.Platform{ + OS: cfg.OS, + Architecture: cfg.Architecture, + } + } + case images.MediaTypeDockerSchema1Manifest: + return errors.Errorf("schema1 manifests are not allowed in manifest lists") + } + + return nil + }) + }(i) + } + + if err := eg.Wait(); err != nil { + return nil, ocispec.Descriptor{}, err + } + + // on single source, return original bytes + if len(descs) == 1 { + if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex { + return dts[0], descs[0], nil + } + } + + m := map[digest.Digest]int{} + newDescs := make([]ocispec.Descriptor, 0, len(descs)) + + addDesc := func(d ocispec.Descriptor) { + idx, ok := m[d.Digest] + if ok { + old := newDescs[idx] + if old.MediaType == "" { + old.MediaType = d.MediaType + } + if d.Platform != nil { + old.Platform = d.Platform + } + if old.Annotations == nil { + old.Annotations = map[string]string{} + } + for k, v := range d.Annotations { + old.Annotations[k] = v + } + newDescs[idx] = old + } else { + m[d.Digest] = len(newDescs) + newDescs = append(newDescs, d) + } + } + + for i, desc := range descs { + switch desc.MediaType { + case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: + var mfst ocispec.Index + if err := json.Unmarshal(dts[i], &mfst); err != nil { + return nil, ocispec.Descriptor{}, errors.WithStack(err) + } + for _, d := range mfst.Manifests { + addDesc(d) + } + default: + addDesc(desc) + } + } + + mt := images.MediaTypeDockerSchema2ManifestList //ocispec.MediaTypeImageIndex + idx := struct { + // MediaType is reserved in the OCI spec but + // excluded from go types. + MediaType string `json:"mediaType,omitempty"` + + ocispec.Index + }{ + MediaType: mt, + Index: ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Manifests: newDescs, + }, + } + + idxBytes, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return nil, ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal index") + } + + return idxBytes, ocispec.Descriptor{ + MediaType: mt, + Size: int64(len(idxBytes)), + Digest: digest.FromBytes(idxBytes), + }, nil +} + +func (r *Resolver) loadConfig(ctx context.Context, in string, dt []byte) (*ocispec.Image, error) { + var manifest ocispec.Manifest + if err := json.Unmarshal(dt, &manifest); err != nil { + return nil, errors.WithStack(err) + } + + dt, err := r.GetDescriptor(ctx, in, manifest.Config) + if err != nil { + return nil, err + } + + var img ocispec.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, errors.WithStack(err) + } + + return &img, nil +} + +func detectMediaType(dt []byte) (string, error) { + var mfst struct { + MediaType string `json:"mediaType"` + Config json.RawMessage `json:"config"` + FSLayers []string `json:"fsLayers"` + } + + if err := json.Unmarshal(dt, &mfst); err != nil { + return "", errors.WithStack(err) + } + + if mfst.MediaType != "" { + return mfst.MediaType, nil + } + if mfst.Config != nil { + return images.MediaTypeDockerSchema2Manifest, nil + } + if len(mfst.FSLayers) > 0 { + return images.MediaTypeDockerSchema1Manifest, nil + } + + return images.MediaTypeDockerSchema2ManifestList, nil +} diff --git a/util/imagetools/inspect.go b/util/imagetools/inspect.go index e73b9809..f09b3247 100644 --- a/util/imagetools/inspect.go +++ b/util/imagetools/inspect.go @@ -35,35 +35,52 @@ func New(opt Opt) *Resolver { } } -func (r *Resolver) Get(ctx context.Context, in string) ([]byte, ocispec.Descriptor, error) { +func (r *Resolver) Resolve(ctx context.Context, in string) (string, ocispec.Descriptor, error) { ref, err := parseRef(in) if err != nil { - return nil, ocispec.Descriptor{}, err + return "", ocispec.Descriptor{}, err } in, desc, err := r.r.Resolve(ctx, ref.String()) + if err != nil { + return "", ocispec.Descriptor{}, err + } + + return in, desc, nil +} + +func (r *Resolver) Get(ctx context.Context, in string) ([]byte, ocispec.Descriptor, error) { + in, desc, err := r.Resolve(ctx, in) if err != nil { return nil, ocispec.Descriptor{}, err } - fetcher, err := r.r.Fetcher(ctx, in) + dt, err := r.GetDescriptor(ctx, in, desc) if err != nil { return nil, ocispec.Descriptor{}, err } + return dt, desc, nil +} + +func (r *Resolver) GetDescriptor(ctx context.Context, in string, desc ocispec.Descriptor) ([]byte, error) { + fetcher, err := r.r.Fetcher(ctx, in) + if err != nil { + return nil, err + } rc, err := fetcher.Fetch(ctx, desc) if err != nil { - return nil, ocispec.Descriptor{}, err + return nil, err } buf := &bytes.Buffer{} _, err = io.Copy(buf, rc) rc.Close() if err != nil { - return nil, ocispec.Descriptor{}, err + return nil, err } - return buf.Bytes(), desc, nil + return buf.Bytes(), nil } func parseRef(s string) (reference.Named, error) {