From d3412f1039acd5f13e7891fa2d6b149fc7954d22 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 10 May 2022 15:23:26 +0100 Subject: [PATCH 1/4] imagetools: refactor combining repository logic This patch modifies the existing combining code in imagetools create to provide better support for multiple repositories down the road. Specifically, the code should no longer rely on a single repository being used for all sources and tags, and should resolve descriptors in their relevant repositories. Signed-off-by: Justin Chadwell --- build/build.go | 16 ++++++++++++- commands/imagetools/create.go | 39 ++++++++++++++------------------ util/imagetools/create.go | 42 +++++++++++++++++------------------ 3 files changed, 53 insertions(+), 44 deletions(-) diff --git a/build/build.go b/build/build.go index c3e3a5ac..aeacbab5 100644 --- a/build/build.go +++ b/build/build.go @@ -925,7 +925,21 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s itpull := imagetools.New(imageopt) - dt, desc, err := itpull.Combine(ctx, names[0], descs) + ref, err := reference.ParseNormalizedNamed(names[0]) + if err != nil { + return err + } + ref = reference.TagNameOnly(ref) + + srcs := make([]*imagetools.Source, len(descs)) + for i, desc := range descs { + srcs[i] = &imagetools.Source{ + Desc: desc, + Ref: ref, + } + } + + dt, desc, err := itpull.Combine(ctx, srcs) if err != nil { return err } diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 7a75d2c7..7e9db288 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -82,14 +82,20 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { return errors.Errorf("multiple repositories currently not supported, found %v", repos) } - var repo string - for r := range repos { - repo = r + var defaultRepo *string + if len(repos) == 1 { + for repo := range repos { + defaultRepo = &repo + } } for i, s := range srcs { if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" { - n, err := reference.ParseNormalizedNamed(repo) + if defaultRepo == nil { + return errors.Errorf("multiple repositories specified, cannot determine default from %v", repos) + } + + n, err := reference.ParseNormalizedNamed(*defaultRepo) if err != nil { return err } @@ -143,7 +149,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { if err != nil { return err } - srcs[i].Ref = nil if srcs[i].Desc.Digest == "" { srcs[i].Desc = desc } else { @@ -162,12 +167,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { } } - descs := make([]ocispec.Descriptor, len(srcs)) - for i := range descs { - descs[i] = srcs[i].Desc - } - - dt, desc, err := r.Combine(ctx, repo, descs) + dt, desc, err := r.Combine(ctx, srcs) if err != nil { return err } @@ -190,13 +190,8 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { return nil } -type src struct { - Desc ocispec.Descriptor - Ref reference.Named -} - -func parseSources(in []string) ([]*src, error) { - out := make([]*src, len(in)) +func parseSources(in []string) ([]*imagetools.Source, error) { + out := make([]*imagetools.Source, len(in)) for i, in := range in { s, err := parseSource(in) if err != nil { @@ -219,11 +214,11 @@ func parseRefs(in []string) ([]reference.Named, error) { return refs, nil } -func parseSource(in string) (*src, error) { +func parseSource(in string) (*imagetools.Source, error) { // source can be a digest, reference or a descriptor JSON dgst, err := digest.Parse(in) if err == nil { - return &src{ + return &imagetools.Source{ Desc: ocispec.Descriptor{ Digest: dgst, }, @@ -234,14 +229,14 @@ func parseSource(in string) (*src, error) { ref, err := reference.ParseNormalizedNamed(in) if err == nil { - return &src{ + return &imagetools.Source{ Ref: ref, }, nil } else if !strings.HasPrefix(in, "{") { return nil, err } - var s src + var s imagetools.Source if err := json.Unmarshal([]byte(in), &s.Desc); err != nil { return nil, errors.WithStack(err) } diff --git a/util/imagetools/create.go b/util/imagetools/create.go index bd7191f2..4b21e75c 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -17,46 +17,46 @@ import ( "golang.org/x/sync/errgroup" ) -func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) { - ref, err := parseRef(in) - if err != nil { - return nil, ocispec.Descriptor{}, err - } +type Source struct { + Desc ocispec.Descriptor + Ref reference.Named +} +func (r *Resolver) Combine(ctx context.Context, srcs []*Source) ([]byte, ocispec.Descriptor, error) { eg, ctx := errgroup.WithContext(ctx) - dts := make([][]byte, len(descs)) + dts := make([][]byte, len(srcs)) for i := range dts { func(i int) { eg.Go(func() error { - dt, err := r.GetDescriptor(ctx, ref.String(), descs[i]) + dt, err := r.GetDescriptor(ctx, srcs[i].Ref.String(), srcs[i].Desc) if err != nil { return err } dts[i] = dt - if descs[i].MediaType == "" { + if srcs[i].Desc.MediaType == "" { mt, err := detectMediaType(dt) if err != nil { return err } - descs[i].MediaType = mt + srcs[i].Desc.MediaType = mt } - mt := descs[i].MediaType + mt := srcs[i].Desc.MediaType switch mt { case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - p := descs[i].Platform - if descs[i].Platform == nil { + p := srcs[i].Desc.Platform + if srcs[i].Desc.Platform == nil { p = &ocispec.Platform{} } if p.OS == "" || p.Architecture == "" { - if err := r.loadPlatform(ctx, p, in, dt); err != nil { + if err := r.loadPlatform(ctx, p, srcs[i].Ref.String(), dt); err != nil { return err } } - descs[i].Platform = p + srcs[i].Desc.Platform = p case images.MediaTypeDockerSchema1Manifest: return errors.Errorf("schema1 manifests are not allowed in manifest lists") } @@ -71,14 +71,14 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr } // on single source, return original bytes - if len(descs) == 1 { - if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex { - return dts[0], descs[0], nil + if len(srcs) == 1 { + if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex { + return dts[0], srcs[0].Desc, nil } } m := map[digest.Digest]int{} - newDescs := make([]ocispec.Descriptor, 0, len(descs)) + newDescs := make([]ocispec.Descriptor, 0, len(srcs)) addDesc := func(d ocispec.Descriptor) { idx, ok := m[d.Digest] @@ -103,8 +103,8 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr } } - for i, desc := range descs { - switch desc.MediaType { + for i, src := range srcs { + switch src.Desc.MediaType { case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: var mfst ocispec.Index if err := json.Unmarshal(dts[i], &mfst); err != nil { @@ -114,7 +114,7 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr addDesc(d) } default: - addDesc(desc) + addDesc(src.Desc) } } From 37ca8631f94b29f78e3ad04d40dae3a333a77e85 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Wed, 18 May 2022 11:52:28 +0100 Subject: [PATCH 2/4] imagetools: copy manifests between repositories Signed-off-by: Justin Chadwell --- commands/imagetools/create.go | 9 +++++---- util/imagetools/create.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 7e9db288..7426cd04 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -78,9 +78,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { if len(repos) == 0 { return errors.Errorf("no repositories specified, please set a reference in tag or source") } - if len(repos) > 1 { - return errors.Errorf("multiple repositories currently not supported, found %v", repos) - } var defaultRepo *string if len(repos) == 1 { @@ -92,7 +89,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { for i, s := range srcs { if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" { if defaultRepo == nil { - return errors.Errorf("multiple repositories specified, cannot determine default from %v", repos) + return errors.Errorf("multiple repositories specified, cannot infer repository for %q", args[i]) } n, err := reference.ParseNormalizedNamed(*defaultRepo) @@ -181,6 +178,10 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { r = imagetools.New(imageopt) for _, t := range tags { + if err := r.Copy(ctx, srcs, t); err != nil { + return err + } + if err := r.Push(ctx, t, desc, dt); err != nil { return err } diff --git a/util/imagetools/create.go b/util/imagetools/create.go index 4b21e75c..2275636d 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -10,6 +10,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" + "github.com/moby/buildkit/util/contentutil" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -169,6 +170,33 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D return err } +func (r *Resolver) Copy(ctx context.Context, srcs []*Source, dest reference.Named) error { + dest = reference.TagNameOnly(dest) + p, err := r.resolver().Pusher(ctx, dest.String()) + if err != nil { + return err + } + + for _, src := range srcs { + if reference.Domain(src.Ref) == reference.Domain(dest) && reference.Path(src.Ref) == reference.Path(dest) { + continue + } + + srcRef := reference.TagNameOnly(src.Ref) + f, err := r.resolver().Fetcher(ctx, srcRef.String()) + if err != nil { + return err + } + + err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc) + if err != nil { + return err + } + } + + return nil +} + func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error { var manifest ocispec.Manifest if err := json.Unmarshal(dt, &manifest); err != nil { From 4ecca34a4277c3cbfe6ca178a4cac564fd0dc88d Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Mon, 23 May 2022 16:42:10 +0100 Subject: [PATCH 3/4] imagetools: give imagetools create a progress bar Signed-off-by: Justin Chadwell --- commands/imagetools/create.go | 47 ++++++++++++++++++---- docs/reference/buildx_imagetools_create.md | 1 + util/imagetools/create.go | 25 +++++------- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 7426cd04..1f67b974 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -1,6 +1,7 @@ package commands import ( + "context" "encoding/json" "fmt" "os" @@ -9,6 +10,7 @@ import ( "github.com/docker/buildx/store" "github.com/docker/buildx/store/storeutil" "github.com/docker/buildx/util/imagetools" + "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" "github.com/docker/distribution/reference" "github.com/moby/buildkit/util/appcontext" @@ -25,6 +27,7 @@ type createOptions struct { tags []string dryrun bool actionAppend bool + progress string } func runCreate(dockerCli command.Cli, in createOptions, args []string) error { @@ -177,18 +180,45 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { // new resolver cause need new auth r = imagetools.New(imageopt) + ctx2, cancel := context.WithCancel(context.TODO()) + defer cancel() + printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress) + + eg, _ := errgroup.WithContext(ctx) + pw := progress.WithPrefix(printer, "internal", true) + for _, t := range tags { - if err := r.Copy(ctx, srcs, t); err != nil { - return err - } + t := t + eg.Go(func() error { + return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error { + eg2, _ := errgroup.WithContext(ctx) + for _, s := range srcs { + if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) { + continue + } + s := s + eg2.Go(func() error { + sub.Log(1, []byte(fmt.Sprintf("copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String()))) + return r.Copy(ctx, s, t) + }) + } + + if err := eg2.Wait(); err != nil { + return err + } + sub.Log(1, []byte(fmt.Sprintf("pushing %s to %s\n", desc.Digest.String(), t.String()))) + return r.Push(ctx, t, desc, dt) + }) + }) + } - if err := r.Push(ctx, t, desc, dt); err != nil { - return err - } - fmt.Println(t.String()) + err = eg.Wait() + err1 := printer.Wait() + if err == nil { + err = err1 } - return nil + return err } func parseSources(in []string) ([]*imagetools.Source, error) { @@ -261,6 +291,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command { flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image") flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing") flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest") + flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`) return cmd } diff --git a/docs/reference/buildx_imagetools_create.md b/docs/reference/buildx_imagetools_create.md index 0ecb4eeb..146ac4c7 100644 --- a/docs/reference/buildx_imagetools_create.md +++ b/docs/reference/buildx_imagetools_create.md @@ -15,6 +15,7 @@ Create a new image based on source images | [`--builder`](#builder) | `string` | | Override the configured builder instance | | [`--dry-run`](#dry-run) | | | Show final image instead of pushing | | [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file | +| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output | | [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image | diff --git a/util/imagetools/create.go b/util/imagetools/create.go index 2275636d..22462041 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -170,30 +170,23 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D return err } -func (r *Resolver) Copy(ctx context.Context, srcs []*Source, dest reference.Named) error { +func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) error { dest = reference.TagNameOnly(dest) p, err := r.resolver().Pusher(ctx, dest.String()) if err != nil { return err } - for _, src := range srcs { - if reference.Domain(src.Ref) == reference.Domain(dest) && reference.Path(src.Ref) == reference.Path(dest) { - continue - } - - srcRef := reference.TagNameOnly(src.Ref) - f, err := r.resolver().Fetcher(ctx, srcRef.String()) - if err != nil { - return err - } - - err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc) - if err != nil { - return err - } + srcRef := reference.TagNameOnly(src.Ref) + f, err := r.resolver().Fetcher(ctx, srcRef.String()) + if err != nil { + return err } + err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc) + if err != nil { + return err + } return nil } From f1a9f913233551549ea51a7eef92e321e4f56200 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Wed, 25 May 2022 12:10:07 +0100 Subject: [PATCH 4/4] imagetools: support cross-repo mounting Signed-off-by: Justin Chadwell --- util/imagetools/create.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/util/imagetools/create.go b/util/imagetools/create.go index 22462041..2480ebb1 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "encoding/json" + "net/url" + "strings" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" @@ -183,6 +185,17 @@ func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) return err } + refspec := reference.TrimNamed(src.Ref).String() + u, err := url.Parse("dummy://" + refspec) + if err != nil { + return err + } + source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/") + if src.Desc.Annotations == nil { + src.Desc.Annotations = make(map[string]string) + } + src.Desc.Annotations["containerd.io/distribution.source."+source] = repo + err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc) if err != nil { return err