From e4f7d92a94fa1e269906917e2f15fba11b55e539 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Thu, 24 Aug 2023 13:55:21 +0100 Subject: [PATCH] build: add --annotation shortcut flag This extracts the same logic for parsing annotations from the imagetools create command, and allows the same flags to be attached to the build command. These annotations are then merged into all provided exporters. Signed-off-by: Justin Chadwell --- commands/build.go | 13 ++++++++++ commands/imagetools/create.go | 19 +------------- docs/reference/buildx_build.md | 1 + tests/build.go | 45 +++++++++++++++++++++++++++++++++ util/buildflags/export.go | 46 ++++++++++++++++++++++++++++++++++ util/imagetools/create.go | 43 +++---------------------------- 6 files changed, 109 insertions(+), 58 deletions(-) diff --git a/commands/build.go b/commands/build.go index 70debb43..37465d0e 100644 --- a/commands/build.go +++ b/commands/build.go @@ -54,6 +54,7 @@ import ( type buildOptions struct { allow []string + annotations []string buildArgs []string cacheFrom []string cacheTo []string @@ -159,6 +160,16 @@ func (o *buildOptions) toControllerOptions() (*controllerapi.BuildOptions, error } } + annotations, err := buildflags.ParseAnnotations(o.annotations) + if err != nil { + return nil, err + } + for _, e := range opts.Exports { + for k, v := range annotations { + e.Attrs[k.String()] = v + } + } + opts.CacheFrom, err = buildflags.ParseCacheEntry(o.cacheFrom) if err != nil { return nil, err @@ -458,6 +469,8 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { flags.StringSliceVar(&options.allow, "allow", []string{}, `Allow extra privileged entitlement (e.g., "network.host", "security.insecure")`) + flags.StringArrayVarP(&options.annotations, "annotation", "", []string{}, "Add annotation to the image") + flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables") flags.StringArrayVar(&options.cacheFrom, "cache-from", []string{}, `External cache sources (e.g., "user/app:cache", "type=local,src=path/to/dir")`) diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 0e8e9d49..f0a22947 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -83,11 +83,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { return errors.Errorf("no repositories specified, please set a reference in tag or source") } - ann, err := parseAnnotations(in.annotations) - if err != nil { - return err - } - var defaultRepo *string if len(repos) == 1 { for repo := range repos { @@ -160,7 +155,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error { } } - dt, desc, err := r.Combine(ctx, srcs, ann) + dt, desc, err := r.Combine(ctx, srcs, in.annotations) if err != nil { return err } @@ -270,18 +265,6 @@ func parseSource(in string) (*imagetools.Source, error) { return &s, nil } -func parseAnnotations(in []string) (map[string]string, error) { - out := make(map[string]string) - for _, i := range in { - kv := strings.SplitN(i, "=", 2) - if len(kv) != 2 { - return nil, errors.Errorf("invalid annotation %q, expected key=value", in) - } - out[kv[0]] = kv[1] - } - return out, nil -} - func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command { var options createOptions diff --git a/docs/reference/buildx_build.md b/docs/reference/buildx_build.md index bb31cb82..4cb1577d 100644 --- a/docs/reference/buildx_build.md +++ b/docs/reference/buildx_build.md @@ -17,6 +17,7 @@ Start a build |:-------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------|:----------|:----------------------------------------------------------------------------------------------------| | [`--add-host`](https://docs.docker.com/engine/reference/commandline/build/#add-host) | `stringSlice` | | Add a custom host-to-IP mapping (format: `host:ip`) | | [`--allow`](#allow) | `stringSlice` | | Allow extra privileged entitlement (e.g., `network.host`, `security.insecure`) | +| `--annotation` | `stringArray` | | Add annotation to the image | | [`--attest`](#attest) | `stringArray` | | Attestation parameters (format: `type=sbom,generator=image`) | | [`--build-arg`](#build-arg) | `stringArray` | | Set build-time variables | | [`--build-context`](#build-context) | `stringArray` | | Additional build contexts (e.g., name=path) | diff --git a/tests/build.go b/tests/build.go index 34ad5920..8c621b58 100644 --- a/tests/build.go +++ b/tests/build.go @@ -20,6 +20,7 @@ import ( "github.com/moby/buildkit/util/testutil/integration" "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -40,6 +41,7 @@ var buildTests = []func(t *testing.T, sb integration.Sandbox){ testBuildMobyFromLocalImage, testBuildDetailsLink, testBuildProgress, + testBuildAnnotations, } func testBuild(t *testing.T, sb integration.Sandbox) { @@ -313,3 +315,46 @@ func testBuildProgress(t *testing.T, sb integration.Sandbox) { require.Contains(t, string(plainOutput), "[internal] load build definition from Dockerfile") require.Contains(t, string(plainOutput), "[base 1/3] FROM docker.io/library/busybox:latest") } + +func testBuildAnnotations(t *testing.T, sb integration.Sandbox) { + if sb.Name() == "docker" { + t.Skip("annotations not supported on docker worker") + } + + dir := createTestProject(t) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + target := registry + "/buildx/registry:latest" + + annotations := []string{ + "--annotation", "example1=www", + "--annotation", "index:example2=xxx", + "--annotation", "manifest:example3=yyy", + "--annotation", "manifest-descriptor[" + platforms.DefaultString() + "]:example4=zzz", + } + out, err := buildCmd(sb, withArgs(annotations...), withArgs(fmt.Sprintf("--output=type=image,name=%s,push=true", target), dir)) + require.NoError(t, err, string(out)) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + + pk := platforms.Format(platforms.Normalize(platforms.DefaultSpec())) + img := imgs.Find(pk) + require.NotNil(t, img) + + require.NotNil(t, imgs.Index) + assert.Equal(t, "xxx", imgs.Index.Annotations["example2"]) + + require.NotNil(t, img.Manifest) + assert.Equal(t, "www", img.Manifest.Annotations["example1"]) + assert.Equal(t, "yyy", img.Manifest.Annotations["example3"]) + + require.NotNil(t, img.Desc) + assert.Equal(t, "zzz", img.Desc.Annotations["example4"]) +} diff --git a/util/buildflags/export.go b/util/buildflags/export.go index 8f1b73cf..fb66d2a6 100644 --- a/util/buildflags/export.go +++ b/util/buildflags/export.go @@ -2,10 +2,14 @@ package buildflags import ( "encoding/csv" + "regexp" "strings" + "github.com/containerd/containerd/platforms" controllerapi "github.com/docker/buildx/controller/pb" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -74,3 +78,45 @@ func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) { } return outs, nil } + +func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) { + // TODO: use buildkit's annotation parser once it supports setting custom prefix and ":" separator + annotationRegexp := regexp.MustCompile(`^(?:([a-z-]+)(?:\[([A-Za-z0-9_/-]+)\])?:)?(\S+)$`) + annotations := make(map[exptypes.AnnotationKey]string) + for _, inp := range inp { + k, v, ok := strings.Cut(inp, "=") + if !ok { + return nil, errors.Errorf("invalid annotation %q, expected key=value", inp) + } + + groups := annotationRegexp.FindStringSubmatch(k) + if groups == nil { + return nil, errors.Errorf("invalid annotation format, expected :=, got %q", inp) + } + + typ, platform, key := groups[1], groups[2], groups[3] + switch typ { + case "": + case exptypes.AnnotationIndex, exptypes.AnnotationIndexDescriptor, exptypes.AnnotationManifest, exptypes.AnnotationManifestDescriptor: + default: + return nil, errors.Errorf("unknown annotation type %q", typ) + } + + var ociPlatform *ocispecs.Platform + if platform != "" { + p, err := platforms.Parse(platform) + if err != nil { + return nil, errors.Wrapf(err, "invalid platform %q", platform) + } + ociPlatform = &p + } + + ak := exptypes.AnnotationKey{ + Type: typ, + Platform: ociPlatform, + Key: key, + } + annotations[ak] = v + } + return annotations, nil +} diff --git a/util/imagetools/create.go b/util/imagetools/create.go index 0769fc5b..748b3e55 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "net/url" - "regexp" "strings" "github.com/containerd/containerd/content" @@ -13,6 +12,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" + "github.com/docker/buildx/util/buildflags" "github.com/docker/distribution/reference" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/util/contentutil" @@ -28,7 +28,7 @@ type Source struct { Ref reference.Named } -func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[string]string) ([]byte, ocispec.Descriptor, error) { +func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann []string) ([]byte, ocispec.Descriptor, error) { eg, ctx := errgroup.WithContext(ctx) dts := make([][]byte, len(srcs)) @@ -143,7 +143,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[string]s // annotations are only allowed on OCI indexes indexAnnotation := make(map[string]string) if mt == ocispec.MediaTypeImageIndex { - annotations, err := parseAnnotations(ann) + annotations, err := buildflags.ParseAnnotations(ann) if err != nil { return nil, ocispec.Descriptor{}, err } @@ -297,40 +297,3 @@ func detectMediaType(dt []byte) (string, error) { return images.MediaTypeDockerSchema2ManifestList, nil } - -func parseAnnotations(ann map[string]string) (map[exptypes.AnnotationKey]string, error) { - // TODO: use buildkit's annotation parser once it supports setting custom prefix and ":" separator - annotationRegexp := regexp.MustCompile(`^(?:([a-z-]+)(?:\[([A-Za-z0-9_/-]+)\])?:)?(\S+)$`) - annotations := make(map[exptypes.AnnotationKey]string) - for k, v := range ann { - groups := annotationRegexp.FindStringSubmatch(k) - if groups == nil { - return nil, errors.Errorf("invalid annotation format, expected :=, got %q", k) - } - - typ, platform, key := groups[1], groups[2], groups[3] - switch typ { - case "": - case exptypes.AnnotationIndex, exptypes.AnnotationIndexDescriptor, exptypes.AnnotationManifest, exptypes.AnnotationManifestDescriptor: - default: - return nil, errors.Errorf("unknown annotation type %q", typ) - } - - var ociPlatform *ocispec.Platform - if platform != "" { - p, err := platforms.Parse(platform) - if err != nil { - return nil, errors.Wrapf(err, "invalid platform %q", platform) - } - ociPlatform = &p - } - - ak := exptypes.AnnotationKey{ - Type: typ, - Platform: ociPlatform, - Key: key, - } - annotations[ak] = v - } - return annotations, nil -}