diff --git a/bake/bake.go b/bake/bake.go index 295264f6..242b2984 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -8,6 +8,7 @@ import ( "os" "path" "regexp" + "sort" "strconv" "strings" @@ -130,6 +131,12 @@ func ReadTargets(ctx context.Context, files []File, targets, overrides []string, g = []*Group{{Targets: dedupString(gt)}} } + for name, t := range m { + if err := c.loadLinks(name, t, m, o, nil); err != nil { + return nil, nil, err + } + } + return m, g, nil } @@ -303,10 +310,45 @@ func (c Config) expandTargets(pattern string) ([]string, error) { return names, nil } +func (c Config) loadLinks(name string, t *Target, m map[string]*Target, o map[string]map[string]Override, visited []string) error { + visited = append(visited, name) + for _, v := range t.Contexts { + if strings.HasPrefix(v, "target:") { + target := strings.TrimPrefix(v, "target:") + if target == t.Name { + return errors.Errorf("target %s cannot link to itself", target) + } + for _, v := range visited { + if v == target { + return errors.Errorf("infinite loop from %s to %s", name, target) + } + } + t2, ok := m[target] + if !ok { + var err error + t2, err = c.ResolveTarget(target, o) + if err != nil { + return err + } + t2.Outputs = nil + m[target] = t2 + } + if err := c.loadLinks(target, t2, m, o, visited); err != nil { + return err + } + if len(t.Platforms) > 1 && len(t2.Platforms) > 1 { + if !sliceEqual(t.Platforms, t2.Platforms) { + return errors.Errorf("target %s can't be used by %s because it is defined for different platforms %v and %v", target, name, t2.Platforms, t.Platforms) + } + } + } + } + return nil +} + func (c Config) newOverrides(v []string) (map[string]map[string]Override, error) { m := map[string]map[string]Override{} for _, v := range v { - parts := strings.SplitN(v, "=", 2) keys := strings.SplitN(parts[0], ".", 3) if len(keys) < 2 { @@ -351,6 +393,11 @@ func (c Config) newOverrides(v []string) (map[string]map[string]Override, error) o.Value = v } fallthrough + case "contexts": + if len(keys) != 3 { + return nil, errors.Errorf("invalid key %s, contexts requires name", parts[0]) + } + fallthrough default: if len(parts) == 2 { o.Value = parts[1] @@ -461,6 +508,7 @@ type Target struct { Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"` Context *string `json:"context,omitempty" hcl:"context,optional"` + Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional"` Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"` DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"` Args map[string]string `json:"args,omitempty" hcl:"args,optional"` @@ -488,6 +536,15 @@ func (t *Target) normalize() { t.CacheFrom = removeDupes(t.CacheFrom) t.CacheTo = removeDupes(t.CacheTo) t.Outputs = removeDupes(t.Outputs) + + for k, v := range t.Contexts { + if v == "" { + delete(t.Contexts, k) + } + } + if len(t.Contexts) == 0 { + t.Contexts = nil + } } func (t *Target) Merge(t2 *Target) { @@ -506,6 +563,12 @@ func (t *Target) Merge(t2 *Target) { } t.Args[k] = v } + for k, v := range t2.Contexts { + if t.Contexts == nil { + t.Contexts = map[string]string{} + } + t.Contexts[k] = v + } for k, v := range t2.Labels { if t.Labels == nil { t.Labels = map[string]string{} @@ -565,7 +628,14 @@ func (t *Target) AddOverrides(overrides map[string]Override) error { t.Args = map[string]string{} } t.Args[keys[1]] = value - + case "contexts": + if len(keys) != 2 { + return errors.Errorf("contexts require name") + } + if t.Contexts == nil { + t.Contexts = map[string]string{} + } + t.Contexts[keys[1]] = value case "labels": if len(keys) != 2 { return errors.Errorf("labels require name") @@ -693,6 +763,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { bi := build.Inputs{ ContextPath: contextPath, DockerfilePath: dockerfilePath, + NamedContexts: t.Contexts, } if t.DockerfileInline != nil { bi.DockerfileInline = *t.DockerfileInline @@ -811,3 +882,17 @@ func validateTargetName(name string) error { } return nil } + +func sliceEqual(s1, s2 []string) bool { + if len(s1) != len(s2) { + return false + } + sort.Strings(s1) + sort.Strings(s2) + for i := range s1 { + if s1[i] != s2[i] { + return false + } + } + return true +} diff --git a/bake/bake_test.go b/bake/bake_test.go index 955dd0d1..0df5440f 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -353,6 +353,208 @@ func TestOverrideMerge(t *testing.T) { require.Equal(t, "type=registry", m["app"].Outputs[0]) } +func TestReadContexts(t *testing.T) { + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` + target "base" { + contexts = { + foo: "bar" + abc: "def" + } + } + target "app" { + inherits = ["base"] + contexts = { + foo: "baz" + } + } + `), + } + + ctx := context.TODO() + m, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil) + require.NoError(t, err) + + require.Equal(t, 1, len(m)) + _, ok := m["app"] + require.True(t, ok) + + bo, err := TargetsToBuildOpt(m, &Input{}) + require.NoError(t, err) + + ctxs := bo["app"].Inputs.NamedContexts + require.Equal(t, 2, len(ctxs)) + + require.Equal(t, "baz", ctxs["foo"]) + require.Equal(t, "def", ctxs["abc"]) + + m, _, err = ReadTargets(ctx, []File{fp}, []string{"app"}, []string{"app.contexts.foo=bay", "base.contexts.ghi=jkl"}, nil) + require.NoError(t, err) + + require.Equal(t, 1, len(m)) + _, ok = m["app"] + require.True(t, ok) + + bo, err = TargetsToBuildOpt(m, &Input{}) + require.NoError(t, err) + + ctxs = bo["app"].Inputs.NamedContexts + require.Equal(t, 3, len(ctxs)) + + require.Equal(t, "bay", ctxs["foo"]) + require.Equal(t, "def", ctxs["abc"]) + require.Equal(t, "jkl", ctxs["ghi"]) + + // test resetting base values + m, _, err = ReadTargets(ctx, []File{fp}, []string{"app"}, []string{"app.contexts.foo="}, nil) + require.NoError(t, err) + + require.Equal(t, 1, len(m)) + _, ok = m["app"] + require.True(t, ok) + + bo, err = TargetsToBuildOpt(m, &Input{}) + require.NoError(t, err) + + ctxs = bo["app"].Inputs.NamedContexts + require.Equal(t, 1, len(ctxs)) + require.Equal(t, "def", ctxs["abc"]) +} + +func TestReadContextFromTargetUnknown(t *testing.T) { + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` + target "base" { + contexts = { + foo: "bar" + abc: "def" + } + } + target "app" { + contexts = { + foo: "baz" + bar: "target:bar" + } + } + `), + } + + ctx := context.TODO() + _, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to find target bar") +} +func TestReadContextFromTargetChain(t *testing.T) { + ctx := context.TODO() + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` + target "base" { + } + target "mid" { + output = ["foo"] + contexts = { + parent: "target:base" + } + } + target "app" { + contexts = { + foo: "baz" + bar: "target:mid" + } + } + target "unused" {} + `), + } + + m, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil) + require.NoError(t, err) + + require.Equal(t, 3, len(m)) + app, ok := m["app"] + require.True(t, ok) + + require.Equal(t, 2, len(app.Contexts)) + + mid, ok := m["mid"] + require.True(t, ok) + require.Equal(t, 0, len(mid.Outputs)) + require.Equal(t, 1, len(mid.Contexts)) + + base, ok := m["base"] + require.True(t, ok) + require.Equal(t, 0, len(base.Contexts)) +} + +func TestReadContextFromTargetInfiniteLoop(t *testing.T) { + ctx := context.TODO() + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` + target "mid" { + output = ["foo"] + contexts = { + parent: "target:app" + } + } + target "app" { + contexts = { + foo: "baz" + bar: "target:mid" + } + } + `), + } + _, _, err := ReadTargets(ctx, []File{fp}, []string{"app", "mid"}, []string{}, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "infinite loop from") +} + +func TestReadContextFromTargetMultiPlatform(t *testing.T) { + ctx := context.TODO() + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` + target "mid" { + output = ["foo"] + platforms = ["linux/amd64", "linux/arm64"] + } + target "app" { + contexts = { + bar: "target:mid" + } + platforms = ["linux/amd64", "linux/arm64"] + } + `), + } + _, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil) + require.NoError(t, err) +} + +func TestReadContextFromTargetInvalidPlatforms(t *testing.T) { + ctx := context.TODO() + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` + target "mid" { + output = ["foo"] + platforms = ["linux/amd64", "linux/riscv64"] + } + target "app" { + contexts = { + bar: "target:mid" + } + platforms = ["linux/amd64", "linux/arm64"] + } + `), + } + _, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "defined for different platforms") +} + func TestReadTargetsDefault(t *testing.T) { t.Parallel() ctx := context.TODO() diff --git a/build/build.go b/build/build.go index 0b83999b..2c9c07e0 100644 --- a/build/build.go +++ b/build/build.go @@ -23,6 +23,7 @@ import ( "github.com/docker/buildx/util/imagetools" "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/resolver" + "github.com/docker/buildx/util/waitmap" "github.com/docker/cli/opts" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" @@ -34,6 +35,7 @@ import ( gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/upload/uploadprovider" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/progress/progresswriter" @@ -667,8 +669,35 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do } } + // validate that all links between targets use same drivers + for name := range opt { + dps := m[name] + for _, dp := range dps { + for k, v := range dp.so.FrontendAttrs { + if strings.HasPrefix(k, "context:") && strings.HasPrefix(v, "target:") { + k2 := strings.TrimPrefix(v, "target:") + dps2, ok := m[k2] + if !ok { + return nil, errors.Errorf("failed to find target %s for context %s", k2, strings.TrimPrefix(k, "context:")) // should be validated before already + } + var found bool + for _, dp2 := range dps2 { + if dp2.driverIndex == dp.driverIndex { + found = true + break + } + } + if !found { + return nil, errors.Errorf("failed to use %s as context %s for %s because targets build with different drivers", k2, strings.TrimPrefix(k, "context:"), name) + } + } + } + } + } + resp = map[string]*client.SolveResponse{} var respMu sync.Mutex + results := waitmap.New() multiTarget := len(opt) > 1 @@ -793,7 +822,6 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do for i, dp := range dps { so := *dp.so - if multiDriver { for i, e := range so.Exports { switch e.Type { @@ -826,14 +854,42 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do pw := progress.WithPrefix(w, k, multiTarget) c := clients[dp.driverIndex] - - pw = progress.ResetTime(pw) - eg.Go(func() error { + if err := waitContextDeps(ctx, dp.driverIndex, results, &so); err != nil { + return err + } + + pw = progress.ResetTime(pw) defer wg.Done() ch, done := progress.NewChannel(pw) defer func() { <-done }() - rr, err := c.Solve(ctx, nil, so, ch) + + frontendInputs := make(map[string]*pb.Definition) + for key, st := range so.FrontendInputs { + def, err := st.Marshal(ctx) + if err != nil { + return err + } + frontendInputs[key] = def.ToPB() + } + + req := gateway.SolveRequest{ + Frontend: so.Frontend, + FrontendOpt: so.FrontendAttrs, + FrontendInputs: frontendInputs, + } + so.Frontend = "" + so.FrontendAttrs = nil + so.FrontendInputs = nil + + rr, err := c.Build(ctx, so, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := c.Solve(ctx, req) + if err != nil { + return nil, err + } + results.Set(resultKey(dp.driverIndex, k), res) + return res, nil + }, ch) if err != nil { return err } @@ -1084,7 +1140,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr for k, v := range inp.NamedContexts { target.FrontendAttrs["frontend.caps"] = "moby.buildkit.frontend.contexts+forward" - if urlutil.IsGitURL(v) || urlutil.IsURL(v) || strings.HasPrefix(v, "docker-image://") { + if urlutil.IsGitURL(v) || urlutil.IsURL(v) || strings.HasPrefix(v, "docker-image://") || strings.HasPrefix(v, "target:") { target.FrontendAttrs["context:"+k] = v continue } @@ -1111,6 +1167,83 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr return release, nil } +func resultKey(index int, name string) string { + return fmt.Sprintf("%d-%s", index, name) +} + +func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *client.SolveOpt) error { + m := map[string]string{} + for k, v := range so.FrontendAttrs { + if strings.HasPrefix(k, "context:") && strings.HasPrefix(v, "target:") { + target := resultKey(index, strings.TrimPrefix(v, "target:")) + m[target] = k + } + } + if len(m) == 0 { + return nil + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + res, err := results.Get(ctx, keys...) + if err != nil { + return err + } + + for k, v := range m { + r, ok := res[k] + if !ok { + continue + } + rr, ok := r.(*gateway.Result) + if !ok { + return errors.Errorf("invalid result type %T", rr) + } + if so.FrontendAttrs == nil { + so.FrontendAttrs = map[string]string{} + } + if so.FrontendInputs == nil { + so.FrontendInputs = map[string]llb.State{} + } + if len(rr.Refs) > 0 { + for platform, r := range rr.Refs { + st, err := r.ToState() + if err != nil { + return err + } + so.FrontendInputs[k+"::"+platform] = st + so.FrontendAttrs[v+"::"+platform] = "input:" + k + "::" + platform + dt, ok := rr.Metadata["containerimage.config/"+platform] + if !ok { + continue + } + dt, err = json.Marshal(map[string][]byte{"containerimage.config": dt}) + if err != nil { + return err + } + so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt) + } + } + if rr.Ref != nil { + st, err := rr.Ref.ToState() + if err != nil { + return err + } + so.FrontendInputs[k] = st + so.FrontendAttrs[v] = "input:" + k + if dt, ok := rr.Metadata["containerimage.config"]; ok { + dt, err = json.Marshal(map[string][]byte{"containerimage.config": dt}) + if err != nil { + return err + } + so.FrontendAttrs["input-metadata:"+k] = string(dt) + } + } + } + return nil +} + func notSupported(d driver.Driver, f driver.Feature) error { return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx create --use\")", f, d.Factory().Name()) } diff --git a/docs/reference/buildx_bake.md b/docs/reference/buildx_bake.md index 40f66846..01439f45 100644 --- a/docs/reference/buildx_bake.md +++ b/docs/reference/buildx_bake.md @@ -339,7 +339,7 @@ target "db" { Complete list of valid target fields: -`args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `inherits`, `labels`, +`args`, `cache-from`, `cache-to`, `context`, `contexts`, `dockerfile`, `inherits`, `labels`, `no-cache`, `output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target` ### Global scope attributes @@ -799,6 +799,78 @@ $ docker buildx bake --print app } ``` +### Defining additional build contexts and linking targets + +In addition to the main `context` key that defines the build context each target can also define additional named contexts with a map defined with key `contexts`. These values map to the `--build-context` flag in the [build command](buildx_build.md#build-context). + +Inside the Dockerfile these contexts can be used with the `FROM` instruction or `--from` flag. + +The value can be a local source directory, container image (with docker-image:// prefix), Git URL, HTTP URL or a name of another target in the Bake file (with target: prefix). + +#### Pinning alpine image + +```Dockerfile +# Dockerfile +FROM alpine +RUN echo "Hello world" +``` + +```hcl +# docker-bake.hcl +target "app" { + contexts = { + alpine = "docker-image://alpine:3.13" + } +} +``` + +#### Using a secondary source directory + +```Dockerfile +# Dockerfile + +FROM scratch AS src + +FROM golang +COPY --from=src . . +``` + +```hcl +# docker-bake.hcl +target "app" { + contexts = { + src = "../path/to/source" + } +} +``` + +#### Using a result of one target as a base image in another target + +To use a result of one target as a build context of another, specity the target name with `target:` prefix. + +```Dockerfile +# Dockerfile +FROM baseapp +RUN echo "Hello world" +``` + +```hcl +# docker-bake.hcl + +target "base" { + dockerfile = "baseapp.Dockerfile" +} + +target "app" { + contexts = { + baseapp = "target:base" + } +} +``` + +Please note that in most cases you should just use a single multi-stage Dockerfile with multiple targets for similar behavior. This case is recommended when you have multiple Dockerfiles that can't be easily merged into one. + + ### Extension field with Compose [Special extension](https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension) diff --git a/util/waitmap/waitmap.go b/util/waitmap/waitmap.go new file mode 100644 index 00000000..c34b8f0c --- /dev/null +++ b/util/waitmap/waitmap.go @@ -0,0 +1,74 @@ +package waitmap + +import ( + "context" + "sync" +) + +type Map struct { + mu sync.RWMutex + m map[string]interface{} + ch map[string]chan struct{} +} + +func New() *Map { + return &Map{ + m: make(map[string]interface{}), + ch: make(map[string]chan struct{}), + } +} + +func (m *Map) Set(key string, value interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + + m.m[key] = value + + if ch, ok := m.ch[key]; ok { + if ch != nil { + close(ch) + } + } + m.ch[key] = nil +} + +func (m *Map) Get(ctx context.Context, keys ...string) (map[string]interface{}, error) { + if len(keys) == 0 { + return map[string]interface{}{}, nil + } + + if len(keys) > 1 { + out := make(map[string]interface{}) + for _, key := range keys { + mm, err := m.Get(ctx, key) + if err != nil { + return nil, err + } + out[key] = mm[key] + } + return out, nil + } + + key := keys[0] + m.mu.Lock() + ch, ok := m.ch[key] + if !ok { + ch = make(chan struct{}) + m.ch[key] = ch + } + + if ch != nil { + m.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ch: + m.mu.Lock() + } + } + + res := m.m[key] + m.mu.Unlock() + + return map[string]interface{}{key: res}, nil +} diff --git a/util/waitmap/waitmap_test.go b/util/waitmap/waitmap_test.go new file mode 100644 index 00000000..319be611 --- /dev/null +++ b/util/waitmap/waitmap_test.go @@ -0,0 +1,64 @@ +package waitmap + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestGetAfter(t *testing.T) { + m := New() + + m.Set("foo", "bar") + m.Set("bar", "baz") + + ctx := context.TODO() + v, err := m.Get(ctx, "foo", "bar") + require.NoError(t, err) + + require.Equal(t, 2, len(v)) + require.Equal(t, "bar", v["foo"]) + require.Equal(t, "baz", v["bar"]) + + v, err = m.Get(ctx, "foo") + require.NoError(t, err) + require.Equal(t, 1, len(v)) + require.Equal(t, "bar", v["foo"]) +} + +func TestTimeout(t *testing.T) { + m := New() + + m.Set("foo", "bar") + + ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) + defer cancel() + + _, err := m.Get(ctx, "bar") + require.Error(t, err) + require.True(t, errors.Is(err, context.DeadlineExceeded)) +} + +func TestBlocking(t *testing.T) { + m := New() + + m.Set("foo", "bar") + + go func() { + time.Sleep(100 * time.Millisecond) + m.Set("bar", "baz") + time.Sleep(50 * time.Millisecond) + m.Set("baz", "abc") + }() + + ctx := context.TODO() + v, err := m.Get(ctx, "foo", "bar", "baz") + require.NoError(t, err) + require.Equal(t, 3, len(v)) + require.Equal(t, "bar", v["foo"]) + require.Equal(t, "baz", v["bar"]) + require.Equal(t, "abc", v["baz"]) +}