From d52c6230c0ec9cd033e95003a6ef585b96b90b45 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 5 Jul 2022 15:20:20 +0100 Subject: [PATCH] bake: add new --sync-output flag This patch introduces a new syncable output option, which ensures that all builds finish simultaneously in the solver, so that no outputs are completed independently of each other. This allows bake to easily express the notion that either all builds should succeed and output or none of them should. Signed-off-by: Justin Chadwell --- build/build.go | 40 ++++++++++++++++++++++++++++++++--- commands/bake.go | 10 +++++---- commands/build.go | 2 +- docs/reference/buildx_bake.md | 1 + 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/build/build.go b/build/build.go index e5d3b28d..fdea33bb 100644 --- a/build/build.go +++ b/build/build.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -782,11 +783,11 @@ func Invoke(ctx context.Context, cfg ContainerConfig) error { return err } -func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) { - return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, false) +func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, syncOutputs bool) (resp map[string]*client.SolveResponse, err error) { + return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, syncOutputs, false) } -func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { +func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), syncOutputs bool, allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { if len(drivers) == 0 { return nil, errors.Errorf("driver required for build") } @@ -854,9 +855,11 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s } } + msize := 0 for k, opt := range opt { multiDriver := len(m[k]) > 1 hasMobyDriver := false + msize += len(m[k]) for i, dp := range m[k] { di := drivers[dp.driverIndex] if di.Driver.IsMobyDriver() { @@ -928,6 +931,10 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s multiTarget := len(opt) > 1 + buildGrp := &sync.WaitGroup{} + buildGrp.Add(msize) + errCount := int64(0) + for k, opt := range opt { err := func(k string) error { opt := opt @@ -1147,6 +1154,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s res, err := c.Solve(ctx, req) if err != nil { if origErr != nil { + atomic.AddInt64(&errCount, 1) + buildGrp.Done() return nil, err } var reqErr *errdefs.UnsupportedSubrequestError @@ -1158,6 +1167,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s origErr = err continue } + atomic.AddInt64(&errCount, 1) + buildGrp.Done() return nil, err } // buildkit v0.8 vendored in Docker 20.10 does not support typed errors @@ -1167,6 +1178,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s continue } } + atomic.AddInt64(&errCount, 1) + buildGrp.Done() return nil, err } if opt.PrintFunc != nil { @@ -1176,6 +1189,27 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s if resultHandleFunc != nil { resultHandleFunc(dp.driverIndex, &ResultContext{cc, res}) } + + if syncOutputs { + err := res.EachRef(func(ref gateway.Reference) error { + return ref.Evaluate(ctx) + }) + if err != nil { + atomic.AddInt64(&errCount, 1) + buildGrp.Done() + return nil, err + } + } + buildGrp.Done() + if syncOutputs { + buildGrp.Wait() + if atomic.LoadInt64(&errCount) > 0 { + // wait until cancelled + <-ctx.Done() + return nil, ctx.Err() + } + } + return res, nil } }, ch) diff --git a/commands/bake.go b/commands/bake.go index 0629df5b..71cd5408 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -19,9 +19,10 @@ import ( ) type bakeOptions struct { - files []string - overrides []string - printOnly bool + files []string + overrides []string + printOnly bool + syncOutput bool commonOptions } @@ -146,7 +147,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error return nil } - resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer) + resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer, in.syncOutput) if err != nil { return wrapBuildError(err, true) } @@ -190,6 +191,7 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--set=*.output=type=docker"`) flags.BoolVar(&options.printOnly, "print", false, "Print the options without building") flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--set=*.output=type=registry"`) + flags.BoolVar(&options.syncOutput, "sync-output", false, "Ensure all builds complete before beginning output") flags.StringArrayVar(&options.overrides, "set", nil, `Override target value (e.g., "targetpattern.key=value")`) commonBuildFlags(&options.commonOptions, flags) diff --git a/commands/build.go b/commands/build.go index ddcdbb09..21b1f4bf 100644 --- a/commands/build.go +++ b/commands/build.go @@ -297,7 +297,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu if res == nil || driverIndex < idx { idx, res = driverIndex, gotRes } - }, allowNoOutput) + }, false, allowNoOutput) err1 := printer.Wait() if err == nil { err = err1 diff --git a/docs/reference/buildx_bake.md b/docs/reference/buildx_bake.md index 73d1799b..c4f67fe4 100644 --- a/docs/reference/buildx_bake.md +++ b/docs/reference/buildx_bake.md @@ -25,6 +25,7 @@ Build from a file | [`--pull`](#pull) | | | Always attempt to pull all referenced images | | `--push` | | | Shorthand for `--set=*.output=type=registry` | | [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) | +| `--sync-output` | | | Ensure all builds complete before beginning output |