diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a9fee690..b0f6f108 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -168,3 +168,36 @@ jobs: DRIVER_OPT: ${{ matrix.driver-opt }} ENDPOINT: ${{ matrix.endpoint }} PLATFORMS: ${{ matrix.platforms }} + + cases: + runs-on: ubuntu-20.04 + needs: + - build + strategy: + fail-fast: false + matrix: + case: + - bake-sync-output + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Install buildx + uses: actions/download-artifact@v3 + with: + name: binary + path: /home/runner/.docker/cli-plugins + - + name: Fix perms and check + run: | + chmod +x /home/runner/.docker/cli-plugins/docker-buildx + docker buildx version + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Test + working-directory: ./test/${{ matrix.case }} + run: | + ./test.sh diff --git a/build/build.go b/build/build.go index 9cf2bc54..b674462f 100644 --- a/build/build.go +++ b/build/build.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -782,11 +783,11 @@ func Invoke(ctx context.Context, cfg ContainerConfig) error { return err } -func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) { - return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, false) +func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, syncOutputs bool) (resp map[string]*client.SolveResponse, err error) { + return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, syncOutputs, false) } -func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { +func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), syncOutputs bool, allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { if len(drivers) == 0 { return nil, errors.Errorf("driver required for build") } @@ -854,9 +855,11 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s } } + msize := 0 for k, opt := range opt { multiDriver := len(m[k]) > 1 hasMobyDriver := false + msize += len(m[k]) for i, dp := range m[k] { di := drivers[dp.driverIndex] if di.Driver.IsMobyDriver() { @@ -928,6 +931,10 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s multiTarget := len(opt) > 1 + buildGrp := &sync.WaitGroup{} + buildGrp.Add(msize) + errCount := int64(0) + for k, opt := range opt { err := func(k string) error { opt := opt @@ -1147,6 +1154,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s res, err := c.Solve(ctx, req) if err != nil { if origErr != nil { + atomic.AddInt64(&errCount, 1) + buildGrp.Done() return nil, err } var reqErr *errdefs.UnsupportedSubrequestError @@ -1158,6 +1167,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s origErr = err continue } + atomic.AddInt64(&errCount, 1) + buildGrp.Done() return nil, err } // buildkit v0.8 vendored in Docker 20.10 does not support typed errors @@ -1167,6 +1178,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s continue } } + atomic.AddInt64(&errCount, 1) + buildGrp.Done() return nil, err } if opt.PrintFunc != nil { @@ -1176,6 +1189,27 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s if resultHandleFunc != nil { resultHandleFunc(dp.driverIndex, &ResultContext{cc, res}) } + + if syncOutputs { + err := res.EachRef(func(ref gateway.Reference) error { + return ref.Evaluate(ctx) + }) + if err != nil { + atomic.AddInt64(&errCount, 1) + buildGrp.Done() + return nil, err + } + } + buildGrp.Done() + if syncOutputs { + buildGrp.Wait() + if atomic.LoadInt64(&errCount) > 0 { + // wait until cancelled + <-ctx.Done() + return nil, ctx.Err() + } + } + return res, nil } }, ch) diff --git a/commands/bake.go b/commands/bake.go index 0629df5b..71cd5408 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -19,9 +19,10 @@ import ( ) type bakeOptions struct { - files []string - overrides []string - printOnly bool + files []string + overrides []string + printOnly bool + syncOutput bool commonOptions } @@ -146,7 +147,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error return nil } - resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer) + resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer, in.syncOutput) if err != nil { return wrapBuildError(err, true) } @@ -190,6 +191,7 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--set=*.output=type=docker"`) flags.BoolVar(&options.printOnly, "print", false, "Print the options without building") flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--set=*.output=type=registry"`) + flags.BoolVar(&options.syncOutput, "sync-output", false, "Ensure all builds complete before beginning output") flags.StringArrayVar(&options.overrides, "set", nil, `Override target value (e.g., "targetpattern.key=value")`) commonBuildFlags(&options.commonOptions, flags) diff --git a/commands/build.go b/commands/build.go index ddcdbb09..21b1f4bf 100644 --- a/commands/build.go +++ b/commands/build.go @@ -297,7 +297,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu if res == nil || driverIndex < idx { idx, res = driverIndex, gotRes } - }, allowNoOutput) + }, false, allowNoOutput) err1 := printer.Wait() if err == nil { err = err1 diff --git a/docs/reference/buildx_bake.md b/docs/reference/buildx_bake.md index 73d1799b..c4f67fe4 100644 --- a/docs/reference/buildx_bake.md +++ b/docs/reference/buildx_bake.md @@ -25,6 +25,7 @@ Build from a file | [`--pull`](#pull) | | | Always attempt to pull all referenced images | | `--push` | | | Shorthand for `--set=*.output=type=registry` | | [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) | +| `--sync-output` | | | Ensure all builds complete before beginning output | diff --git a/test/bake-sync-output/.gitignore b/test/bake-sync-output/.gitignore new file mode 100644 index 00000000..e2e7327c --- /dev/null +++ b/test/bake-sync-output/.gitignore @@ -0,0 +1 @@ +/out diff --git a/test/bake-sync-output/bar.Dockerfile b/test/bake-sync-output/bar.Dockerfile new file mode 100644 index 00000000..ac342679 --- /dev/null +++ b/test/bake-sync-output/bar.Dockerfile @@ -0,0 +1,5 @@ +FROM busybox +RUN echo bar > /bar + +FROM scratch +COPY --from=0 /bar /bar diff --git a/test/bake-sync-output/docker-bake.hcl b/test/bake-sync-output/docker-bake.hcl new file mode 100644 index 00000000..acdabeea --- /dev/null +++ b/test/bake-sync-output/docker-bake.hcl @@ -0,0 +1,13 @@ +group "default" { + targets = ["foo", "bar"] +} + +target "foo" { + dockerfile = "foo.Dockerfile" + output = ["out"] +} + +target "bar" { + dockerfile = "bar.Dockerfile" + output = ["out"] +} diff --git a/test/bake-sync-output/foo.Dockerfile b/test/bake-sync-output/foo.Dockerfile new file mode 100644 index 00000000..b0143c36 --- /dev/null +++ b/test/bake-sync-output/foo.Dockerfile @@ -0,0 +1,5 @@ +FROM busybox +RUN echo foo > /foo + +FROM scratch +COPY --from=0 /foo /foo diff --git a/test/bake-sync-output/test.sh b/test/bake-sync-output/test.sh new file mode 100755 index 00000000..81016518 --- /dev/null +++ b/test/bake-sync-output/test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +rm -rf ./out + +docker buildx bake --print +docker buildx bake --sync-output + +if [[ ! -f ./out/foo || ! -f ./out/bar ]]; then + echo >&2 "error: missing output files" + exit 1 +fi