diff --git a/build/build.go b/build/build.go index 5f5ee8c7..27e5be15 100644 --- a/build/build.go +++ b/build/build.go @@ -925,7 +925,21 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s itpull := imagetools.New(imageopt) - dt, desc, err := itpull.Combine(ctx, names[0], descs) + ref, err := reference.ParseNormalizedNamed(names[0]) + if err != nil { + return err + } + ref = reference.TagNameOnly(ref) + + srcs := make([]*imagetools.Source, len(descs)) + for i, desc := range descs { + srcs[i] = &imagetools.Source{ + Desc: desc, + Ref: ref, + } + } + + dt, desc, err := itpull.Combine(ctx, srcs) if err != nil { return err } diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 7a75d2c7..1f67b974 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -1,6 +1,7 @@ package commands import ( + "context" "encoding/json" "fmt" "os" @@ -9,6 +10,7 @@ import ( "github.com/docker/buildx/store" "github.com/docker/buildx/store/storeutil" "github.com/docker/buildx/util/imagetools" + "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" "github.com/docker/distribution/reference" "github.com/moby/buildkit/util/appcontext" @@ -25,6 +27,7 @@ type createOptions struct { tags []string dryrun bool actionAppend bool + progress string } func runCreate(dockerCli command.Cli, in createOptions, args []string) error { @@ -78,18 +81,21 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { if len(repos) == 0 { return errors.Errorf("no repositories specified, please set a reference in tag or source") } - if len(repos) > 1 { - return errors.Errorf("multiple repositories currently not supported, found %v", repos) - } - var repo string - for r := range repos { - repo = r + var defaultRepo *string + if len(repos) == 1 { + for repo := range repos { + defaultRepo = &repo + } } for i, s := range srcs { if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" { - n, err := reference.ParseNormalizedNamed(repo) + if defaultRepo == nil { + return errors.Errorf("multiple repositories specified, cannot infer repository for %q", args[i]) + } + + n, err := reference.ParseNormalizedNamed(*defaultRepo) if err != nil { return err } @@ -143,7 +149,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { if err != nil { return err } - srcs[i].Ref = nil if srcs[i].Desc.Digest == "" { srcs[i].Desc = desc } else { @@ -162,12 +167,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { } } - descs := make([]ocispec.Descriptor, len(srcs)) - for i := range descs { - descs[i] = srcs[i].Desc - } - - dt, desc, err := r.Combine(ctx, repo, descs) + dt, desc, err := r.Combine(ctx, srcs) if err != nil { return err } @@ -180,23 +180,49 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { // new resolver cause need new auth r = imagetools.New(imageopt) + ctx2, cancel := context.WithCancel(context.TODO()) + defer cancel() + printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress) + + eg, _ := errgroup.WithContext(ctx) + pw := progress.WithPrefix(printer, "internal", true) + for _, t := range tags { - if err := r.Push(ctx, t, desc, dt); err != nil { - return err - } - fmt.Println(t.String()) + t := t + eg.Go(func() error { + return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error { + eg2, _ := errgroup.WithContext(ctx) + for _, s := range srcs { + if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) { + continue + } + s := s + eg2.Go(func() error { + sub.Log(1, []byte(fmt.Sprintf("copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String()))) + return r.Copy(ctx, s, t) + }) + } + + if err := eg2.Wait(); err != nil { + return err + } + sub.Log(1, []byte(fmt.Sprintf("pushing %s to %s\n", desc.Digest.String(), t.String()))) + return r.Push(ctx, t, desc, dt) + }) + }) } - return nil -} + err = eg.Wait() + err1 := printer.Wait() + if err == nil { + err = err1 + } -type src struct { - Desc ocispec.Descriptor - Ref reference.Named + return err } -func parseSources(in []string) ([]*src, error) { - out := make([]*src, len(in)) +func parseSources(in []string) ([]*imagetools.Source, error) { + out := make([]*imagetools.Source, len(in)) for i, in := range in { s, err := parseSource(in) if err != nil { @@ -219,11 +245,11 @@ func parseRefs(in []string) ([]reference.Named, error) { return refs, nil } -func parseSource(in string) (*src, error) { +func parseSource(in string) (*imagetools.Source, error) { // source can be a digest, reference or a descriptor JSON dgst, err := digest.Parse(in) if err == nil { - return &src{ + return &imagetools.Source{ Desc: ocispec.Descriptor{ Digest: dgst, }, @@ -234,14 +260,14 @@ func parseSource(in string) (*src, error) { ref, err := reference.ParseNormalizedNamed(in) if err == nil { - return &src{ + return &imagetools.Source{ Ref: ref, }, nil } else if !strings.HasPrefix(in, "{") { return nil, err } - var s src + var s imagetools.Source if err := json.Unmarshal([]byte(in), &s.Desc); err != nil { return nil, errors.WithStack(err) } @@ -265,6 +291,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command { flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image") flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing") flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest") + flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`) return cmd } diff --git a/docs/reference/buildx_imagetools_create.md b/docs/reference/buildx_imagetools_create.md index 0ecb4eeb..146ac4c7 100644 --- a/docs/reference/buildx_imagetools_create.md +++ b/docs/reference/buildx_imagetools_create.md @@ -15,6 +15,7 @@ Create a new image based on source images | [`--builder`](#builder) | `string` | | Override the configured builder instance | | [`--dry-run`](#dry-run) | | | Show final image instead of pushing | | [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file | +| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output | | [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image | diff --git a/util/imagetools/create.go b/util/imagetools/create.go index bd7191f2..2480ebb1 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -4,12 +4,15 @@ import ( "bytes" "context" "encoding/json" + "net/url" + "strings" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" + "github.com/moby/buildkit/util/contentutil" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -17,46 +20,46 @@ import ( "golang.org/x/sync/errgroup" ) -func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) { - ref, err := parseRef(in) - if err != nil { - return nil, ocispec.Descriptor{}, err - } +type Source struct { + Desc ocispec.Descriptor + Ref reference.Named +} +func (r *Resolver) Combine(ctx context.Context, srcs []*Source) ([]byte, ocispec.Descriptor, error) { eg, ctx := errgroup.WithContext(ctx) - dts := make([][]byte, len(descs)) + dts := make([][]byte, len(srcs)) for i := range dts { func(i int) { eg.Go(func() error { - dt, err := r.GetDescriptor(ctx, ref.String(), descs[i]) + dt, err := r.GetDescriptor(ctx, srcs[i].Ref.String(), srcs[i].Desc) if err != nil { return err } dts[i] = dt - if descs[i].MediaType == "" { + if srcs[i].Desc.MediaType == "" { mt, err := detectMediaType(dt) if err != nil { return err } - descs[i].MediaType = mt + srcs[i].Desc.MediaType = mt } - mt := descs[i].MediaType + mt := srcs[i].Desc.MediaType switch mt { case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: - p := descs[i].Platform - if descs[i].Platform == nil { + p := srcs[i].Desc.Platform + if srcs[i].Desc.Platform == nil { p = &ocispec.Platform{} } if p.OS == "" || p.Architecture == "" { - if err := r.loadPlatform(ctx, p, in, dt); err != nil { + if err := r.loadPlatform(ctx, p, srcs[i].Ref.String(), dt); err != nil { return err } } - descs[i].Platform = p + srcs[i].Desc.Platform = p case images.MediaTypeDockerSchema1Manifest: return errors.Errorf("schema1 manifests are not allowed in manifest lists") } @@ -71,14 +74,14 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr } // on single source, return original bytes - if len(descs) == 1 { - if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex { - return dts[0], descs[0], nil + if len(srcs) == 1 { + if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex { + return dts[0], srcs[0].Desc, nil } } m := map[digest.Digest]int{} - newDescs := make([]ocispec.Descriptor, 0, len(descs)) + newDescs := make([]ocispec.Descriptor, 0, len(srcs)) addDesc := func(d ocispec.Descriptor) { idx, ok := m[d.Digest] @@ -103,8 +106,8 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr } } - for i, desc := range descs { - switch desc.MediaType { + for i, src := range srcs { + switch src.Desc.MediaType { case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex: var mfst ocispec.Index if err := json.Unmarshal(dts[i], &mfst); err != nil { @@ -114,7 +117,7 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr addDesc(d) } default: - addDesc(desc) + addDesc(src.Desc) } } @@ -169,6 +172,37 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D return err } +func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) error { + dest = reference.TagNameOnly(dest) + p, err := r.resolver().Pusher(ctx, dest.String()) + if err != nil { + return err + } + + srcRef := reference.TagNameOnly(src.Ref) + f, err := r.resolver().Fetcher(ctx, srcRef.String()) + if err != nil { + return err + } + + refspec := reference.TrimNamed(src.Ref).String() + u, err := url.Parse("dummy://" + refspec) + if err != nil { + return err + } + source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/") + if src.Desc.Annotations == nil { + src.Desc.Annotations = make(map[string]string) + } + src.Desc.Annotations["containerd.io/distribution.source."+source] = repo + + err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc) + if err != nil { + return err + } + return nil +} + func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error { var manifest ocispec.Manifest if err := json.Unmarshal(dt, &manifest); err != nil {