bake: add new --sync-output flag

This patch introduces a new syncable output option, which ensures that
all builds finish simultaneously in the solver, so that no outputs are
completed independently of each other. This allows bake to easily
express the notion that either all builds should succeed and output or
none of them should.

Signed-off-by: Justin Chadwell <me@jedevc.com>
pull/1197/head
Justin Chadwell 3 years ago
parent 3ed2783f34
commit d52c6230c0

@ -15,6 +15,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"syscall" "syscall"
"time" "time"
@ -782,11 +783,11 @@ func Invoke(ctx context.Context, cfg ContainerConfig) error {
return err return err
} }
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) { func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, syncOutputs bool) (resp map[string]*client.SolveResponse, err error) {
return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, false) return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, syncOutputs, false)
} }
func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) { func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), syncOutputs bool, allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) {
if len(drivers) == 0 { if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build") return nil, errors.Errorf("driver required for build")
} }
@ -854,9 +855,11 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
} }
} }
msize := 0
for k, opt := range opt { for k, opt := range opt {
multiDriver := len(m[k]) > 1 multiDriver := len(m[k]) > 1
hasMobyDriver := false hasMobyDriver := false
msize += len(m[k])
for i, dp := range m[k] { for i, dp := range m[k] {
di := drivers[dp.driverIndex] di := drivers[dp.driverIndex]
if di.Driver.IsMobyDriver() { if di.Driver.IsMobyDriver() {
@ -928,6 +931,10 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
multiTarget := len(opt) > 1 multiTarget := len(opt) > 1
buildGrp := &sync.WaitGroup{}
buildGrp.Add(msize)
errCount := int64(0)
for k, opt := range opt { for k, opt := range opt {
err := func(k string) error { err := func(k string) error {
opt := opt opt := opt
@ -1147,6 +1154,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
res, err := c.Solve(ctx, req) res, err := c.Solve(ctx, req)
if err != nil { if err != nil {
if origErr != nil { if origErr != nil {
atomic.AddInt64(&errCount, 1)
buildGrp.Done()
return nil, err return nil, err
} }
var reqErr *errdefs.UnsupportedSubrequestError var reqErr *errdefs.UnsupportedSubrequestError
@ -1158,6 +1167,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
origErr = err origErr = err
continue continue
} }
atomic.AddInt64(&errCount, 1)
buildGrp.Done()
return nil, err return nil, err
} }
// buildkit v0.8 vendored in Docker 20.10 does not support typed errors // buildkit v0.8 vendored in Docker 20.10 does not support typed errors
@ -1167,6 +1178,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
continue continue
} }
} }
atomic.AddInt64(&errCount, 1)
buildGrp.Done()
return nil, err return nil, err
} }
if opt.PrintFunc != nil { if opt.PrintFunc != nil {
@ -1176,6 +1189,27 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
if resultHandleFunc != nil { if resultHandleFunc != nil {
resultHandleFunc(dp.driverIndex, &ResultContext{cc, res}) resultHandleFunc(dp.driverIndex, &ResultContext{cc, res})
} }
if syncOutputs {
err := res.EachRef(func(ref gateway.Reference) error {
return ref.Evaluate(ctx)
})
if err != nil {
atomic.AddInt64(&errCount, 1)
buildGrp.Done()
return nil, err
}
}
buildGrp.Done()
if syncOutputs {
buildGrp.Wait()
if atomic.LoadInt64(&errCount) > 0 {
// wait until cancelled
<-ctx.Done()
return nil, ctx.Err()
}
}
return res, nil return res, nil
} }
}, ch) }, ch)

@ -19,9 +19,10 @@ import (
) )
type bakeOptions struct { type bakeOptions struct {
files []string files []string
overrides []string overrides []string
printOnly bool printOnly bool
syncOutput bool
commonOptions commonOptions
} }
@ -146,7 +147,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
return nil return nil
} }
resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer) resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer, in.syncOutput)
if err != nil { if err != nil {
return wrapBuildError(err, true) return wrapBuildError(err, true)
} }
@ -190,6 +191,7 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--set=*.output=type=docker"`) flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--set=*.output=type=docker"`)
flags.BoolVar(&options.printOnly, "print", false, "Print the options without building") flags.BoolVar(&options.printOnly, "print", false, "Print the options without building")
flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--set=*.output=type=registry"`) flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--set=*.output=type=registry"`)
flags.BoolVar(&options.syncOutput, "sync-output", false, "Ensure all builds complete before beginning output")
flags.StringArrayVar(&options.overrides, "set", nil, `Override target value (e.g., "targetpattern.key=value")`) flags.StringArrayVar(&options.overrides, "set", nil, `Override target value (e.g., "targetpattern.key=value")`)
commonBuildFlags(&options.commonOptions, flags) commonBuildFlags(&options.commonOptions, flags)

@ -297,7 +297,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
if res == nil || driverIndex < idx { if res == nil || driverIndex < idx {
idx, res = driverIndex, gotRes idx, res = driverIndex, gotRes
} }
}, allowNoOutput) }, false, allowNoOutput)
err1 := printer.Wait() err1 := printer.Wait()
if err == nil { if err == nil {
err = err1 err = err1

@ -25,6 +25,7 @@ Build from a file
| [`--pull`](#pull) | | | Always attempt to pull all referenced images | | [`--pull`](#pull) | | | Always attempt to pull all referenced images |
| `--push` | | | Shorthand for `--set=*.output=type=registry` | | `--push` | | | Shorthand for `--set=*.output=type=registry` |
| [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) | | [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) |
| `--sync-output` | | | Ensure all builds complete before beginning output |
<!---MARKER_GEN_END--> <!---MARKER_GEN_END-->

Loading…
Cancel
Save