From 1613fde55cdb6892cc318d0afb9d2306207bd93d Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Mon, 20 Mar 2023 12:18:34 +0000 Subject: [PATCH 1/6] bake: allow interception to create derived contexts This patch allows high level clients to define an EvalContext method which can derive a new context given a block and the base parent context. This allows users of the package to intercept evaluation before it begins, and define additional variables and functions that are bound to a single block. Signed-off-by: Justin Chadwell --- bake/hclparser/hclparser.go | 110 +++++++++++++++++++++--------------- bake/hclparser/stdlib.go | 8 +++ 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index ec378cf9..e68c2a8c 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -49,29 +49,29 @@ type parser struct { attrs map[string]*hcl.Attribute funcs map[string]*functionDef - blocks map[string]map[string][]*hcl.Block - blockValues map[*hcl.Block]reflect.Value - blockTypes map[string]reflect.Type + blocks map[string]map[string][]*hcl.Block + blockValues map[*hcl.Block]reflect.Value + blockEvalCtx map[*hcl.Block]*hcl.EvalContext + blockTypes map[string]reflect.Type ectx *hcl.EvalContext progress map[string]struct{} progressF map[string]struct{} progressB map[*hcl.Block]map[string]struct{} - doneF map[string]struct{} doneB map[*hcl.Block]map[string]struct{} } var errUndefined = errors.New("undefined") -func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics { +func (p *parser) loadDeps(ectx *hcl.EvalContext, exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics { fns, hcldiags := funcCalls(exp) if hcldiags.HasErrors() { return hcldiags } for _, fn := range fns { - if err := p.resolveFunction(fn); err != nil { + if err := p.resolveFunction(ectx, fn); err != nil { if allowMissing && errors.Is(err, errUndefined) { continue } @@ -131,7 +131,7 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allow return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr()) } } else { - if err := p.resolveValue(v.RootName()); err != nil { + if err := p.resolveValue(ectx, v.RootName()); err != nil { if allowMissing && errors.Is(err, errUndefined) { continue } @@ -145,16 +145,16 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allow // resolveFunction forces evaluation of a function, storing the result into the // parser. -func (p *parser) resolveFunction(name string) error { - if _, ok := p.doneF[name]; ok { +func (p *parser) resolveFunction(ectx *hcl.EvalContext, name string) error { + if _, ok := p.ectx.Functions[name]; ok { + return nil + } + if _, ok := ectx.Functions[name]; ok { return nil } f, ok := p.funcs[name] if !ok { - if _, ok := p.ectx.Functions[name]; ok { - return nil - } - return errors.Wrapf(errUndefined, "function %q does not exit", name) + return errors.Wrapf(errUndefined, "function %q does not exist", name) } if _, ok := p.progressF[name]; ok { return errors.Errorf("function cycle not allowed for %s", name) @@ -204,7 +204,7 @@ func (p *parser) resolveFunction(name string) error { return diags } - if diags := p.loadDeps(f.Result.Expr, params, false); diags.HasErrors() { + if diags := p.loadDeps(p.ectx, f.Result.Expr, params, false); diags.HasErrors() { return diags } @@ -214,7 +214,6 @@ func (p *parser) resolveFunction(name string) error { if diags.HasErrors() { return diags } - p.doneF[name] = struct{}{} p.ectx.Functions[name] = v return nil @@ -222,10 +221,13 @@ func (p *parser) resolveFunction(name string) error { // resolveValue forces evaluation of a named value, storing the result into the // parser. -func (p *parser) resolveValue(name string) (err error) { +func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { if _, ok := p.ectx.Variables[name]; ok { return nil } + if _, ok := ectx.Variables[name]; ok { + return nil + } if _, ok := p.progress[name]; ok { return errors.Errorf("variable cycle not allowed for %s", name) } @@ -242,9 +244,10 @@ func (p *parser) resolveValue(name string) (err error) { if _, builtin := p.opt.Vars[name]; !ok && !builtin { vr, ok := p.vars[name] if !ok { - return errors.Wrapf(errUndefined, "variable %q does not exit", name) + return errors.Wrapf(errUndefined, "variable %q does not exist", name) } def = vr.Default + ectx = p.ectx } if def == nil { @@ -257,10 +260,10 @@ func (p *parser) resolveValue(name string) (err error) { return } - if diags := p.loadDeps(def.Expr, nil, true); diags.HasErrors() { + if diags := p.loadDeps(ectx, def.Expr, nil, true); diags.HasErrors() { return diags } - vv, diags := def.Expr.Value(p.ectx) + vv, diags := def.Expr.Value(ectx) if diags.HasErrors() { return diags } @@ -364,18 +367,43 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err return FilterExcludeBody(block.Body, filter) } - // load dependencies from all targeted properties + // prepare the output destination and evaluation context t, ok := p.blockTypes[block.Type] if !ok { return nil } + var output reflect.Value + var ectx *hcl.EvalContext + if prev, ok := p.blockValues[block]; ok { + output = prev + ectx = p.blockEvalCtx[block] + } else { + output = reflect.New(t) + setLabel(output, block.Labels[0]) // early attach labels, so we can reference them + + type ectxI interface { + EvalContext(base *hcl.EvalContext, block *hcl.Block) *hcl.EvalContext + } + if v, ok := output.Interface().(ectxI); ok { + ectx = v.EvalContext(p.ectx, block) + if ectx != p.ectx && ectx.Parent() != p.ectx { + return errors.Errorf("EvalContext must return a context with the correct parent") + } + } else { + ectx = p.ectx + } + } + p.blockValues[block] = output + p.blockEvalCtx[block] = ectx + + // load dependencies from all targeted properties schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) content, _, diag := body().PartialContent(schema) if diag.HasErrors() { return diag } for _, a := range content.Attributes { - diag := p.loadDeps(a.Expr, nil, true) + diag := p.loadDeps(ectx, a.Expr, nil, true) if diag.HasErrors() { return diag } @@ -388,18 +416,10 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err } // decode! - var output reflect.Value - if prev, ok := p.blockValues[block]; ok { - output = prev - } else { - output = reflect.New(t) - setLabel(output, block.Labels[0]) // early attach labels, so we can reference them - } - diag = gohcl.DecodeBody(body(), p.ectx, output.Interface()) + diag = gohcl.DecodeBody(body(), ectx, output.Interface()) if diag.HasErrors() { return diag } - p.blockValues[block] = output // mark all targeted properties as done for _, a := range content.Attributes { @@ -417,7 +437,7 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err } } - // store the result into the evaluation context (so if can be referenced) + // store the result into the evaluation context (so it can be referenced) outputType, err := gocty.ImpliedType(output.Interface()) if err != nil { return err @@ -475,20 +495,20 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { attrs: map[string]*hcl.Attribute{}, funcs: map[string]*functionDef{}, - blocks: map[string]map[string][]*hcl.Block{}, - blockValues: map[*hcl.Block]reflect.Value{}, - blockTypes: map[string]reflect.Type{}, - - progress: map[string]struct{}{}, - progressF: map[string]struct{}{}, - progressB: map[*hcl.Block]map[string]struct{}{}, + blocks: map[string]map[string][]*hcl.Block{}, + blockValues: map[*hcl.Block]reflect.Value{}, + blockEvalCtx: map[*hcl.Block]*hcl.EvalContext{}, + blockTypes: map[string]reflect.Type{}, - doneF: map[string]struct{}{}, - doneB: map[*hcl.Block]map[string]struct{}{}, ectx: &hcl.EvalContext{ Variables: map[string]cty.Value{}, - Functions: stdlibFunctions, + Functions: Stdlib(), }, + + progress: map[string]struct{}{}, + progressF: map[string]struct{}{}, + progressB: map[*hcl.Block]map[string]struct{}{}, + doneB: map[*hcl.Block]map[string]struct{}{}, } for _, v := range defs.Variables { @@ -532,7 +552,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { delete(p.attrs, "function") for k := range p.opt.Vars { - _ = p.resolveValue(k) + _ = p.resolveValue(p.ectx, k) } for _, a := range content.Attributes { @@ -548,7 +568,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } for k := range p.vars { - if err := p.resolveValue(k); err != nil { + if err := p.resolveValue(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { return diags } @@ -558,7 +578,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } for k := range p.funcs { - if err := p.resolveFunction(k); err != nil { + if err := p.resolveFunction(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { return diags } @@ -678,7 +698,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } for k := range p.attrs { - if err := p.resolveValue(k); err != nil { + if err := p.resolveValue(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { return diags } diff --git a/bake/hclparser/stdlib.go b/bake/hclparser/stdlib.go index f78b6dce..53d1d416 100644 --- a/bake/hclparser/stdlib.go +++ b/bake/hclparser/stdlib.go @@ -124,3 +124,11 @@ var timestampFunc = function.New(&function.Spec{ return cty.StringVal(time.Now().UTC().Format(time.RFC3339)), nil }, }) + +func Stdlib() map[string]function.Function { + funcs := make(map[string]function.Function, len(stdlibFunctions)) + for k, v := range stdlibFunctions { + funcs[k] = v + } + return funcs +} From 4437802e63df16cc016b2f481dd0d663f4baf033 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Mon, 20 Mar 2023 17:07:34 +0000 Subject: [PATCH 2/6] bake: allow overriding name property Previously, the name property could not be set in the body of a bake target and could only be set for a label. This patch allows the body to override the values of label fields, though the default is still the label. Signed-off-by: Justin Chadwell --- bake/hcl_test.go | 128 +++++++++++++++++++++++++++ bake/hclparser/hclparser.go | 170 ++++++++++++++++++++++++++++++------ 2 files changed, 273 insertions(+), 25 deletions(-) diff --git a/bake/hcl_test.go b/bake/hcl_test.go index ecb0d722..b7483a45 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -634,6 +634,134 @@ func TestHCLMultiFileAttrs(t *testing.T) { require.Equal(t, ptrstr("pre-ghi"), c.Targets[0].Args["v1"]) } +func TestHCLDuplicateTarget(t *testing.T) { + dt := []byte(` + target "app" { + dockerfile = "x" + } + target "app" { + dockerfile = "y" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "app", c.Targets[0].Name) + require.Equal(t, "y", *c.Targets[0].Dockerfile) +} + +func TestHCLRenameTargetAttrs(t *testing.T) { + dt := []byte(` + target "abc" { + name = "xyz" + dockerfile = "foo" + } + + target "def" { + dockerfile = target.xyz.dockerfile + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "xyz", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) + require.Equal(t, "def", c.Targets[1].Name) + require.Equal(t, "foo", *c.Targets[1].Dockerfile) + + dt = []byte(` + target "abc" { + name = "xyz" + dockerfile = "foo" + } + + target "def" { + dockerfile = target.abc.dockerfile + } + `) + + _, err = ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) +} + +func TestHCLRenameTarget(t *testing.T) { + dt := []byte(` + target "abc" { + name = "xyz" + dockerfile = "foo" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "xyz", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) +} + +func TestHCLRenameMerge(t *testing.T) { + dt := []byte(` + target "x" { + name = "y" + dockerfile = "foo" + } + + target "x" { + name = "z" + dockerfile = "bar" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "y", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) + require.Equal(t, "z", c.Targets[1].Name) + require.Equal(t, "bar", *c.Targets[1].Dockerfile) +} + +func TestHCLRenameMultiFile(t *testing.T) { + dt := []byte(` + target "foo" { + name = "bar" + dockerfile = "x" + } + `) + dt2 := []byte(` + target "foo" { + context = "y" + } + `) + dt3 := []byte(` + target "bar" { + target = "z" + } + `) + + c, err := ParseFiles([]File{ + {Data: dt, Name: "c1.hcl"}, + {Data: dt2, Name: "c2.hcl"}, + {Data: dt3, Name: "c3.hcl"}, + }, nil) + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + + require.Equal(t, c.Targets[0].Name, "bar") + require.Equal(t, *c.Targets[0].Dockerfile, "x") + require.Equal(t, *c.Targets[0].Target, "z") + + require.Equal(t, c.Targets[1].Name, "foo") + require.Equal(t, *c.Targets[1].Context, "y") +} + func TestJSONAttributes(t *testing.T) { dt := []byte(`{"FOO": "abc", "variable": {"BAR": {"default": "def"}}, "target": { "app": { "args": {"v1": "pre-${FOO}-${BAR}"}} } }`) diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index e68c2a8c..2c03c584 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -124,11 +124,13 @@ func (p *parser) loadDeps(ectx *hcl.EvalContext, exp hcl.Expression, exclude map } } } - if err := p.resolveBlock(blocks[0], target); err != nil { - if allowMissing && errors.Is(err, errUndefined) { - continue + for _, block := range blocks { + if err := p.resolveBlock(block, target, true); err != nil { + if allowMissing && errors.Is(err, errUndefined) { + continue + } + return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr()) } - return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr()) } } else { if err := p.resolveValue(ectx, v.RootName()); err != nil { @@ -301,12 +303,7 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { // resolveBlock force evaluates a block, storing the result in the parser. If a // target schema is provided, only the attributes and blocks present in the // schema will be evaluated. -func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) { - name := block.Labels[0] - if err := p.opt.ValidateLabel(name); err != nil { - return wrapErrorDiagnostic("Invalid name", err, &block.LabelRanges[0], &block.LabelRanges[0]) - } - +func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema, resolveName bool) (err error) { if _, ok := p.doneB[block]; !ok { p.doneB[block] = map[string]struct{}{} } @@ -314,6 +311,14 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err p.progressB[block] = map[string]struct{}{} } + name, err := p.resolveBlockName(block, resolveName) + if err != nil { + return err + } + if err := p.opt.ValidateLabel(name); err != nil { + return wrapErrorDiagnostic("Invalid name", err, &block.LabelRanges[0], &block.LabelRanges[0]) + } + if target != nil { // filter out attributes and blocks that are already evaluated original := target @@ -379,7 +384,6 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err ectx = p.blockEvalCtx[block] } else { output = reflect.New(t) - setLabel(output, block.Labels[0]) // early attach labels, so we can reference them type ectxI interface { EvalContext(base *hcl.EvalContext, block *hcl.Block) *hcl.EvalContext @@ -398,6 +402,9 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err // load dependencies from all targeted properties schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) + if nameKey, ok := getNameKey(output); ok { + schema.Attributes = append(schema.Attributes, hcl.AttributeSchema{Name: nameKey}) + } content, _, diag := body().PartialContent(schema) if diag.HasErrors() { return diag @@ -409,7 +416,7 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err } } for _, b := range content.Blocks { - err := p.resolveBlock(b, nil) + err := p.resolveBlock(b, nil, true) if err != nil { return err } @@ -420,6 +427,22 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err if diag.HasErrors() { return diag } + if nameKey, ok := getNameKey(output); ok { + for k, v := range content.Attributes { + if k == nameKey { + var name2 string + diag = gohcl.DecodeExpression(v.Expr, ectx, &name) + if diag.HasErrors() { + return diag + } + if name2 != "" { + name = name2 + } + break + } + } + } + setName(output, name) // mark all targeted properties as done for _, a := range content.Attributes { @@ -459,6 +482,66 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err return nil } +// resolveBlockName returns the name of the block, calling resolveBlock to +// evaluate any label fields to correctly resolve the name. +func (p *parser) resolveBlockName(block *hcl.Block, resolveName bool) (string, error) { + defaultName := block.Labels[0] + if !resolveName { + return defaultName, nil + } + + t, ok := p.blockTypes[block.Type] + if !ok { + return "", nil + } + lbl, ok := getNameKey(reflect.New(t)) + if !ok { + return defaultName, nil + } + + if prev, ok := p.blockValues[block]; ok { + // will have previously set name + name, ok := getName(prev) + if ok { + return name, nil + } + return "", errors.New("failed to get label") + } + + target := &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: lbl}}, + Blocks: []hcl.BlockHeaderSchema{{Type: lbl}}, + } + if err := p.resolveBlock(block, target, false); err != nil { + return "", err + } + + name, ok := getName(p.blockValues[block]) + if !ok { + return "", errors.New("failed to get label") + } + + // move block to new name + if name != defaultName { + p.blocks[block.Type][name] = append(p.blocks[block.Type][name], block) + + filtered := make([]*hcl.Block, 0, len(p.blocks[block.Type][defaultName])-1) + for _, b := range p.blocks[block.Type][defaultName] { + if b == block { + continue + } + filtered = append(filtered, b) + } + if len(filtered) != 0 { + p.blocks[block.Type][defaultName] = filtered + } else { + delete(p.blocks[block.Type], defaultName) + } + } + + return name, nil +} + func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { reserved := map[string]struct{}{} schema, _ := gohcl.ImpliedBodySchema(val) @@ -649,7 +732,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { for _, b := range content.Blocks { v := reflect.ValueOf(val) - err := p.resolveBlock(b, nil) + err := p.resolveBlock(b, nil, true) if err != nil { if diag, ok := err.(hcl.Diagnostics); ok { if diag.HasErrors() { @@ -664,13 +747,13 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { vv := p.blockValues[b] t := types[b.Type] - lblIndex := setLabel(vv, b.Labels[0]) - - oldValue, exists := t.values[b.Labels[0]] - if !exists && lblIndex != -1 { + lblIndex, lblExists := getNameIndex(vv) + lblName, _ := getName(vv) + oldValue, exists := t.values[lblName] + if !exists && lblExists { if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice { for i := 0; i < v.Elem().Field(t.idx).Len(); i++ { - if b.Labels[0] == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() { + if lblName == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() { exists = true oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i} break @@ -689,7 +772,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { if slice.IsNil() { slice = reflect.New(t.typ).Elem() } - t.values[b.Labels[0]] = value{Value: vv, idx: slice.Len()} + t.values[lblName] = value{Value: vv, idx: slice.Len()} v.Elem().Field(t.idx).Set(reflect.Append(slice, vv)) } } @@ -730,18 +813,55 @@ func wrapErrorDiagnostic(message string, err error, subject *hcl.Range, context } } -func setLabel(v reflect.Value, lbl string) int { - // cache field index? +func setName(v reflect.Value, name string) { + numFields := v.Elem().Type().NumField() + for i := 0; i < numFields; i++ { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { + if t == "label" { + v.Elem().Field(i).Set(reflect.ValueOf(name)) + } + } + } +} + +func getName(v reflect.Value) (string, bool) { + numFields := v.Elem().Type().NumField() + for i := 0; i < numFields; i++ { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { + if t == "label" { + return v.Elem().Field(i).String(), true + } + } + } + return "", false +} + +func getNameKey(v reflect.Value) (string, bool) { + numFields := v.Elem().Type().NumField() + for i := 0; i < numFields; i++ { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { + if t == "label" { + return parts[0], true + } + } + } + return "", false +} + +func getNameIndex(v reflect.Value) (int, bool) { numFields := v.Elem().Type().NumField() for i := 0; i < numFields; i++ { - for _, t := range strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { if t == "label" { - v.Elem().Field(i).Set(reflect.ValueOf(lbl)) - return i + return i, true } } } - return -1 + return 0, false } func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{}, vars map[string]*variable) hcl.Diagnostics { From 77252f161cb9e955db46a791e80626a1b63949ae Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 21 Mar 2023 10:45:25 +0000 Subject: [PATCH 3/6] bake: add matrix to target block Signed-off-by: Justin Chadwell --- bake/bake.go | 49 ++++ bake/hcl_test.go | 148 +++++++++- bake/hclparser/hclparser.go | 524 ++++++++++++++++++------------------ 3 files changed, 456 insertions(+), 265 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index 1abf5f1b..470de87d 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -23,6 +23,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" + "github.com/zclconf/go-cty/cty" ) var ( @@ -779,6 +780,54 @@ func (t *Target) AddOverrides(overrides map[string]Override) error { return nil } +func (t *Target) EvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { + content, _, err := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, + }) + if err != nil { + return nil, err + } + + attr, ok := content.Attributes["matrix"] + if !ok { + return []*hcl.EvalContext{ectx}, nil + } + if diags := loadDeps(attr.Expr); diags.HasErrors() { + return nil, diags + } + value, err := attr.Expr.Value(ectx) + if err != nil { + return nil, err + } + + if !value.CanIterateElements() { + return nil, errors.Errorf("matrix must be a map") + } + matrix := value.AsValueMap() + + ectxs := []*hcl.EvalContext{ectx} + for k, expr := range matrix { + if !expr.CanIterateElements() { + return nil, errors.Errorf("matrix values must be a list") + } + + ectxs2 := []*hcl.EvalContext{} + for _, v := range expr.AsValueSlice() { + for _, e := range ectxs { + e2 := ectx.NewChild() + e2.Variables = make(map[string]cty.Value) + for k, v := range e.Variables { + e2.Variables[k] = v + } + e2.Variables[k] = v + ectxs2 = append(ectxs2, e2) + } + } + ectxs = ectxs2 + } + return ectxs, nil +} + func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) { m2 := make(map[string]build.Options, len(m)) for k, v := range m { diff --git a/bake/hcl_test.go b/bake/hcl_test.go index b7483a45..95d3b193 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -652,6 +652,22 @@ func TestHCLDuplicateTarget(t *testing.T) { require.Equal(t, "y", *c.Targets[0].Dockerfile) } +func TestHCLRenameTarget(t *testing.T) { + dt := []byte(` + target "abc" { + name = "xyz" + dockerfile = "foo" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "xyz", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) +} + func TestHCLRenameTargetAttrs(t *testing.T) { dt := []byte(` target "abc" { @@ -666,13 +682,31 @@ func TestHCLRenameTargetAttrs(t *testing.T) { c, err := ParseFile(dt, "docker-bake.hcl") require.NoError(t, err) - require.Equal(t, 2, len(c.Targets)) require.Equal(t, "xyz", c.Targets[0].Name) require.Equal(t, "foo", *c.Targets[0].Dockerfile) require.Equal(t, "def", c.Targets[1].Name) require.Equal(t, "foo", *c.Targets[1].Dockerfile) + dt = []byte(` + target "def" { + dockerfile = target.xyz.dockerfile + } + + target "abc" { + name = "xyz" + dockerfile = "foo" + } + `) + + c, err = ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "def", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) + require.Equal(t, "xyz", c.Targets[1].Name) + require.Equal(t, "foo", *c.Targets[1].Dockerfile) + dt = []byte(` target "abc" { name = "xyz" @@ -686,22 +720,20 @@ func TestHCLRenameTargetAttrs(t *testing.T) { _, err = ParseFile(dt, "docker-bake.hcl") require.Error(t, err) -} -func TestHCLRenameTarget(t *testing.T) { - dt := []byte(` + dt = []byte(` + target "def" { + dockerfile = target.abc.dockerfile + } + target "abc" { name = "xyz" dockerfile = "foo" } `) - c, err := ParseFile(dt, "docker-bake.hcl") - require.NoError(t, err) - - require.Equal(t, 1, len(c.Targets)) - require.Equal(t, "xyz", c.Targets[0].Name) - require.Equal(t, "foo", *c.Targets[0].Dockerfile) + _, err = ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) } func TestHCLRenameMerge(t *testing.T) { @@ -762,6 +794,102 @@ func TestHCLRenameMultiFile(t *testing.T) { require.Equal(t, *c.Targets[1].Context, "y") } +func TestHCLMatrixBasic(t *testing.T) { + dt := []byte(` + target "default" { + matrix = { + foo = ["x", "y"] + } + name = foo + dockerfile = "${foo}.Dockerfile" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, c.Targets[0].Name, "x") + require.Equal(t, c.Targets[1].Name, "y") + require.Equal(t, *c.Targets[0].Dockerfile, "x.Dockerfile") + require.Equal(t, *c.Targets[1].Dockerfile, "y.Dockerfile") +} + +func TestHCLMatrixMultiple(t *testing.T) { + dt := []byte(` + target "default" { + matrix = { + foo = ["a"] + bar = ["b", "c"] + baz = ["d", "e", "f"] + } + name = "${foo}-${bar}-${baz}" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 6, len(c.Targets)) + names := make([]string, len(c.Targets)) + for i, t := range c.Targets { + names[i] = t.Name + } + require.ElementsMatch(t, names, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}) +} + +func TestHCLMatrixArgs(t *testing.T) { + dt := []byte(` + a = 1 + variable "b" { + default = 2 + } + target "default" { + matrix = { + foo = [a, b] + } + name = foo + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, c.Targets[0].Name, "1") + require.Equal(t, c.Targets[1].Name, "2") +} + +func TestHCLMatrixErrors(t *testing.T) { + dt := []byte(` + target "default" { + matrix = "test" + } + `) + _, err := ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) + + dt = []byte(` + target "default" { + matrix = { + ["a"] = ["b"] + } + } + `) + _, err = ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) + + dt = []byte(` + target "default" { + matrix = { + a = "b" + } + } + `) + _, err = ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) +} + func TestJSONAttributes(t *testing.T) { dt := []byte(`{"FOO": "abc", "variable": {"BAR": {"default": "def"}}, "target": { "app": { "args": {"v1": "pre-${FOO}-${BAR}"}} } }`) diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index 2c03c584..d36708d0 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -1,7 +1,9 @@ package hclparser import ( + "encoding/binary" "fmt" + "hash/fnv" "math" "math/big" "reflect" @@ -50,16 +52,16 @@ type parser struct { funcs map[string]*functionDef blocks map[string]map[string][]*hcl.Block - blockValues map[*hcl.Block]reflect.Value - blockEvalCtx map[*hcl.Block]*hcl.EvalContext + blockValues map[*hcl.Block][]reflect.Value + blockEvalCtx map[*hcl.Block][]*hcl.EvalContext blockTypes map[string]reflect.Type ectx *hcl.EvalContext - progress map[string]struct{} - progressF map[string]struct{} - progressB map[*hcl.Block]map[string]struct{} - doneB map[*hcl.Block]map[string]struct{} + progressV map[uint64]struct{} + progressF map[uint64]struct{} + progressB map[uint64]map[string]struct{} + doneB map[uint64]map[string]struct{} } var errUndefined = errors.New("undefined") @@ -125,7 +127,7 @@ func (p *parser) loadDeps(ectx *hcl.EvalContext, exp hcl.Expression, exclude map } } for _, block := range blocks { - if err := p.resolveBlock(block, target, true); err != nil { + if err := p.resolveBlock(block, target); err != nil { if allowMissing && errors.Is(err, errUndefined) { continue } @@ -158,10 +160,10 @@ func (p *parser) resolveFunction(ectx *hcl.EvalContext, name string) error { if !ok { return errors.Wrapf(errUndefined, "function %q does not exist", name) } - if _, ok := p.progressF[name]; ok { + if _, ok := p.progressF[key(ectx, name)]; ok { return errors.Errorf("function cycle not allowed for %s", name) } - p.progressF[name] = struct{}{} + p.progressF[key(ectx, name)] = struct{}{} if f.Result == nil { return errors.Errorf("empty result not allowed for %s", name) @@ -230,10 +232,10 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { if _, ok := ectx.Variables[name]; ok { return nil } - if _, ok := p.progress[name]; ok { + if _, ok := p.progressV[key(ectx, name)]; ok { return errors.Errorf("variable cycle not allowed for %s", name) } - p.progress[name] = struct{}{} + p.progressV[key(ectx, name)] = struct{}{} var v *cty.Value defer func() { @@ -303,243 +305,230 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { // resolveBlock force evaluates a block, storing the result in the parser. If a // target schema is provided, only the attributes and blocks present in the // schema will be evaluated. -func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema, resolveName bool) (err error) { - if _, ok := p.doneB[block]; !ok { - p.doneB[block] = map[string]struct{}{} - } - if _, ok := p.progressB[block]; !ok { - p.progressB[block] = map[string]struct{}{} - } - - name, err := p.resolveBlockName(block, resolveName) - if err != nil { - return err - } - if err := p.opt.ValidateLabel(name); err != nil { - return wrapErrorDiagnostic("Invalid name", err, &block.LabelRanges[0], &block.LabelRanges[0]) +func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) { + // prepare the output destination and evaluation context + t, ok := p.blockTypes[block.Type] + if !ok { + return nil } - - if target != nil { - // filter out attributes and blocks that are already evaluated - original := target - target = &hcl.BodySchema{} - for _, a := range original.Attributes { - if _, ok := p.doneB[block][a.Name]; !ok { - target.Attributes = append(target.Attributes, a) - } + var outputs []reflect.Value + var ectxs []*hcl.EvalContext + if prev, ok := p.blockValues[block]; ok { + outputs = prev + ectxs = p.blockEvalCtx[block] + } else { + type ectxI interface { + EvalContexts(base *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) } - for _, b := range original.Blocks { - if _, ok := p.doneB[block][b.Type]; !ok { - target.Blocks = append(target.Blocks, b) + if v, ok := reflect.New(t).Interface().(ectxI); ok { + ectxs, err = v.EvalContexts(p.ectx, block, func(expr hcl.Expression) hcl.Diagnostics { + return p.loadDeps(p.ectx, expr, nil, true) + }) + if err != nil { + return err } + for _, ectx := range ectxs { + if ectx != p.ectx && ectx.Parent() != p.ectx { + return errors.Errorf("EvalContext must return a context with the correct parent") + } + } + } else { + ectxs = append([]*hcl.EvalContext{}, p.ectx) } - if len(target.Attributes) == 0 && len(target.Blocks) == 0 { - return nil + for range ectxs { + outputs = append(outputs, reflect.New(t)) } } + p.blockValues[block] = outputs + p.blockEvalCtx[block] = ectxs - if target != nil { - // detect reference cycles - for _, a := range target.Attributes { - if _, ok := p.progressB[block][a.Name]; ok { - return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name) - } + for i, output := range outputs { + target := target + ectx := ectxs[i] + name, _ := getName(output) + if name == "" { + name = block.Labels[0] } - for _, b := range target.Blocks { - if _, ok := p.progressB[block][b.Type]; ok { - return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type) - } + if err := p.opt.ValidateLabel(name); err != nil { + return err } - for _, a := range target.Attributes { - p.progressB[block][a.Name] = struct{}{} + + if _, ok := p.doneB[key(block, ectx)]; !ok { + p.doneB[key(block, ectx)] = map[string]struct{}{} } - for _, b := range target.Blocks { - p.progressB[block][b.Type] = struct{}{} + if _, ok := p.progressB[key(block, ectx)]; !ok { + p.progressB[key(block, ectx)] = map[string]struct{}{} } - } - // create a filtered body that contains only the target properties - body := func() hcl.Body { if target != nil { - return FilterIncludeBody(block.Body, target) + // filter out attributes and blocks that are already evaluated + original := target + target = &hcl.BodySchema{} + for _, a := range original.Attributes { + if _, ok := p.doneB[key(block, ectx)][a.Name]; !ok { + target.Attributes = append(target.Attributes, a) + } + } + for _, b := range original.Blocks { + if _, ok := p.doneB[key(block, ectx)][b.Type]; !ok { + target.Blocks = append(target.Blocks, b) + } + } + if len(target.Attributes) == 0 && len(target.Blocks) == 0 { + return nil + } } - filter := &hcl.BodySchema{} - for k := range p.doneB[block] { - filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k}) - filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k}) + if target != nil { + // detect reference cycles + for _, a := range target.Attributes { + if _, ok := p.progressB[key(block, ectx)][a.Name]; ok { + return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name) + } + } + for _, b := range target.Blocks { + if _, ok := p.progressB[key(block, ectx)][b.Type]; ok { + return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type) + } + } + for _, a := range target.Attributes { + p.progressB[key(block, ectx)][a.Name] = struct{}{} + } + for _, b := range target.Blocks { + p.progressB[key(block, ectx)][b.Type] = struct{}{} + } } - return FilterExcludeBody(block.Body, filter) - } - // prepare the output destination and evaluation context - t, ok := p.blockTypes[block.Type] - if !ok { - return nil - } - var output reflect.Value - var ectx *hcl.EvalContext - if prev, ok := p.blockValues[block]; ok { - output = prev - ectx = p.blockEvalCtx[block] - } else { - output = reflect.New(t) + // create a filtered body that contains only the target properties + body := func() hcl.Body { + if target != nil { + return FilterIncludeBody(block.Body, target) + } - type ectxI interface { - EvalContext(base *hcl.EvalContext, block *hcl.Block) *hcl.EvalContext - } - if v, ok := output.Interface().(ectxI); ok { - ectx = v.EvalContext(p.ectx, block) - if ectx != p.ectx && ectx.Parent() != p.ectx { - return errors.Errorf("EvalContext must return a context with the correct parent") + filter := &hcl.BodySchema{} + for k := range p.doneB[key(block, ectx)] { + filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k}) + filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k}) } - } else { - ectx = p.ectx + return FilterExcludeBody(block.Body, filter) } - } - p.blockValues[block] = output - p.blockEvalCtx[block] = ectx - // load dependencies from all targeted properties - schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) - if nameKey, ok := getNameKey(output); ok { - schema.Attributes = append(schema.Attributes, hcl.AttributeSchema{Name: nameKey}) - } - content, _, diag := body().PartialContent(schema) - if diag.HasErrors() { - return diag - } - for _, a := range content.Attributes { - diag := p.loadDeps(ectx, a.Expr, nil, true) + // load dependencies from all targeted properties + schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) + if nameKey, ok := getNameKey(output); ok { + schema.Attributes = append(schema.Attributes, hcl.AttributeSchema{Name: nameKey}) + } + content, _, diag := body().PartialContent(schema) if diag.HasErrors() { return diag } - } - for _, b := range content.Blocks { - err := p.resolveBlock(b, nil, true) - if err != nil { - return err + for _, a := range content.Attributes { + diag := p.loadDeps(ectx, a.Expr, nil, true) + if diag.HasErrors() { + return diag + } + } + for _, b := range content.Blocks { + err := p.resolveBlock(b, nil) + if err != nil { + return err + } } - } - // decode! - diag = gohcl.DecodeBody(body(), ectx, output.Interface()) - if diag.HasErrors() { - return diag - } - if nameKey, ok := getNameKey(output); ok { - for k, v := range content.Attributes { - if k == nameKey { - var name2 string - diag = gohcl.DecodeExpression(v.Expr, ectx, &name) - if diag.HasErrors() { - return diag - } - if name2 != "" { - name = name2 + // decode! + diag = gohcl.DecodeBody(body(), ectx, output.Interface()) + if diag.HasErrors() { + return diag + } + if nameKey, ok := getNameKey(output); ok { + for k, v := range content.Attributes { + if k == nameKey { + var name2 string + diag = gohcl.DecodeExpression(v.Expr, ectx, &name) + if diag.HasErrors() { + return diag + } + if name2 != "" { + name = name2 + } + break } - break } } - } - setName(output, name) + setName(output, name) - // mark all targeted properties as done - for _, a := range content.Attributes { - p.doneB[block][a.Name] = struct{}{} - } - for _, b := range content.Blocks { - p.doneB[block][b.Type] = struct{}{} - } - if target != nil { - for _, a := range target.Attributes { - p.doneB[block][a.Name] = struct{}{} + // mark all targeted properties as done + for _, a := range content.Attributes { + p.doneB[key(block, ectx)][a.Name] = struct{}{} } - for _, b := range target.Blocks { - p.doneB[block][b.Type] = struct{}{} + for _, b := range content.Blocks { + p.doneB[key(block, ectx)][b.Type] = struct{}{} + } + if target != nil { + for _, a := range target.Attributes { + p.doneB[key(block, ectx)][a.Name] = struct{}{} + } + for _, b := range target.Blocks { + p.doneB[key(block, ectx)][b.Type] = struct{}{} + } } - } - // store the result into the evaluation context (so it can be referenced) - outputType, err := gocty.ImpliedType(output.Interface()) - if err != nil { - return err - } - outputValue, err := gocty.ToCtyValue(output.Interface(), outputType) - if err != nil { - return err - } - var m map[string]cty.Value - if m2, ok := p.ectx.Variables[block.Type]; ok { - m = m2.AsValueMap() - } - if m == nil { - m = map[string]cty.Value{} + // store the result into the evaluation context (so it can be referenced) + outputType, err := gocty.ImpliedType(output.Interface()) + if err != nil { + return err + } + outputValue, err := gocty.ToCtyValue(output.Interface(), outputType) + if err != nil { + return err + } + var m map[string]cty.Value + if m2, ok := p.ectx.Variables[block.Type]; ok { + m = m2.AsValueMap() + } + if m == nil { + m = map[string]cty.Value{} + } + m[name] = outputValue + p.ectx.Variables[block.Type] = cty.MapVal(m) } - m[name] = outputValue - p.ectx.Variables[block.Type] = cty.MapVal(m) return nil } -// resolveBlockName returns the name of the block, calling resolveBlock to +// resolveBlockNames returns the names of the block, calling resolveBlock to // evaluate any label fields to correctly resolve the name. -func (p *parser) resolveBlockName(block *hcl.Block, resolveName bool) (string, error) { - defaultName := block.Labels[0] - if !resolveName { - return defaultName, nil - } - +func (p *parser) resolveBlockNames(block *hcl.Block) ([]string, error) { t, ok := p.blockTypes[block.Type] if !ok { - return "", nil - } - lbl, ok := getNameKey(reflect.New(t)) - if !ok { - return defaultName, nil + return nil, errors.Errorf("internal error: unknown block type %s", block.Type) } - if prev, ok := p.blockValues[block]; ok { - // will have previously set name - name, ok := getName(prev) - if ok { - return name, nil + nameKey, ok := getNameKey(reflect.New(t)) + if ok { + target := &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: nameKey}}, + Blocks: []hcl.BlockHeaderSchema{{Type: nameKey}}, + } + if err := p.resolveBlock(block, target); err != nil { + return nil, err + } + } else { + if err := p.resolveBlock(block, &hcl.BodySchema{}); err != nil { + return nil, err } - return "", errors.New("failed to get label") - } - - target := &hcl.BodySchema{ - Attributes: []hcl.AttributeSchema{{Name: lbl}}, - Blocks: []hcl.BlockHeaderSchema{{Type: lbl}}, - } - if err := p.resolveBlock(block, target, false); err != nil { - return "", err - } - - name, ok := getName(p.blockValues[block]) - if !ok { - return "", errors.New("failed to get label") } - // move block to new name - if name != defaultName { - p.blocks[block.Type][name] = append(p.blocks[block.Type][name], block) - - filtered := make([]*hcl.Block, 0, len(p.blocks[block.Type][defaultName])-1) - for _, b := range p.blocks[block.Type][defaultName] { - if b == block { - continue - } - filtered = append(filtered, b) - } - if len(filtered) != 0 { - p.blocks[block.Type][defaultName] = filtered - } else { - delete(p.blocks[block.Type], defaultName) + names := make([]string, 0, len(p.blockValues[block])) + for _, prev := range p.blockValues[block] { + name, ok := getName(prev) + if !ok { + return nil, errors.New("internal error: failed to get name") } + names = append(names, name) } - return name, nil + return names, nil } func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { @@ -579,19 +568,18 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { funcs: map[string]*functionDef{}, blocks: map[string]map[string][]*hcl.Block{}, - blockValues: map[*hcl.Block]reflect.Value{}, - blockEvalCtx: map[*hcl.Block]*hcl.EvalContext{}, + blockValues: map[*hcl.Block][]reflect.Value{}, + blockEvalCtx: map[*hcl.Block][]*hcl.EvalContext{}, blockTypes: map[string]reflect.Type{}, - ectx: &hcl.EvalContext{ Variables: map[string]cty.Value{}, Functions: Stdlib(), }, - progress: map[string]struct{}{}, - progressF: map[string]struct{}{}, - progressB: map[*hcl.Block]map[string]struct{}{}, - doneB: map[*hcl.Block]map[string]struct{}{}, + progressV: map[uint64]struct{}{}, + progressF: map[uint64]struct{}{}, + progressB: map[uint64]map[string]struct{}{}, + doneB: map[uint64]map[string]struct{}{}, } for _, v := range defs.Variables { @@ -683,28 +671,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } } - for _, b := range content.Blocks { - if len(b.Labels) == 0 || len(b.Labels) > 1 { - return hcl.Diagnostics{ - &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid block", - Detail: fmt.Sprintf("invalid block label: %v", b.Labels), - Subject: &b.LabelRanges[0], - Context: &b.LabelRanges[0], - }, - } - } - bm, ok := p.blocks[b.Type] - if !ok { - bm = map[string][]*hcl.Block{} - p.blocks[b.Type] = bm - } - - lbl := b.Labels[0] - bm[lbl] = append(bm[lbl], b) - } - type value struct { reflect.Value idx int @@ -715,7 +681,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { values map[string]value } types := map[string]field{} - vt := reflect.ValueOf(val).Elem().Type() for i := 0; i < vt.NumField(); i++ { tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",") @@ -728,11 +693,41 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } } + tmpBlocks := map[string]map[string][]*hcl.Block{} + for _, b := range content.Blocks { + if len(b.Labels) == 0 || len(b.Labels) > 1 { + return hcl.Diagnostics{ + &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid block", + Detail: fmt.Sprintf("invalid block label: %v", b.Labels), + Subject: &b.LabelRanges[0], + Context: &b.LabelRanges[0], + }, + } + } + + bm, ok := tmpBlocks[b.Type] + if !ok { + bm = map[string][]*hcl.Block{} + tmpBlocks[b.Type] = bm + } + + names, err := p.resolveBlockNames(b) + if err != nil { + return wrapErrorDiagnostic("Invalid name", err, &b.LabelRanges[0], &b.LabelRanges[0]) + } + for _, name := range names { + bm[name] = append(bm[name], b) + } + } + p.blocks = tmpBlocks + diags = hcl.Diagnostics{} for _, b := range content.Blocks { v := reflect.ValueOf(val) - err := p.resolveBlock(b, nil, true) + err := p.resolveBlock(b, nil) if err != nil { if diag, ok := err.(hcl.Diagnostics); ok { if diag.HasErrors() { @@ -744,36 +739,37 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } } - vv := p.blockValues[b] - - t := types[b.Type] - lblIndex, lblExists := getNameIndex(vv) - lblName, _ := getName(vv) - oldValue, exists := t.values[lblName] - if !exists && lblExists { - if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice { - for i := 0; i < v.Elem().Field(t.idx).Len(); i++ { - if lblName == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() { - exists = true - oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i} - break + vvs := p.blockValues[b] + for _, vv := range vvs { + t := types[b.Type] + lblIndex, lblExists := getNameIndex(vv) + lblName, _ := getName(vv) + oldValue, exists := t.values[lblName] + if !exists && lblExists { + if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice { + for i := 0; i < v.Elem().Field(t.idx).Len(); i++ { + if lblName == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() { + exists = true + oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i} + break + } } } } - } - if exists { - if m := oldValue.Value.MethodByName("Merge"); m.IsValid() { - m.Call([]reflect.Value{vv}) + if exists { + if m := oldValue.Value.MethodByName("Merge"); m.IsValid() { + m.Call([]reflect.Value{vv}) + } else { + v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv) + } } else { - v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv) - } - } else { - slice := v.Elem().Field(t.idx) - if slice.IsNil() { - slice = reflect.New(t.typ).Elem() + slice := v.Elem().Field(t.idx) + if slice.IsNil() { + slice = reflect.New(t.typ).Elem() + } + t.values[lblName] = value{Value: vv, idx: slice.Len()} + v.Elem().Field(t.idx).Set(reflect.Append(slice, vv)) } - t.values[lblName] = value{Value: vv, idx: slice.Len()} - v.Elem().Field(t.idx).Set(reflect.Append(slice, vv)) } } if diags.HasErrors() { @@ -893,3 +889,21 @@ func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{}, } return fdiags } + +// key returns a unique hash for the given values +func key(ks ...any) uint64 { + hash := fnv.New64a() + for _, k := range ks { + v := reflect.ValueOf(k) + switch v.Kind() { + case reflect.String: + hash.Write([]byte(v.String())) + case reflect.Pointer: + ptr := reflect.ValueOf(k).Pointer() + binary.Write(hash, binary.LittleEndian, uint64(ptr)) + default: + panic(fmt.Sprintf("unknown key kind %s", v.Kind().String())) + } + } + return hash.Sum64() +} From 080687026111e6a03bc5d1030d9d231db03f74d5 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Wed, 22 Mar 2023 17:07:04 +0000 Subject: [PATCH 4/6] bake: generate implicit groups for matrixes Signed-off-by: Justin Chadwell --- bake/bake.go | 19 +++++++- bake/hcl_test.go | 92 +++++++++++++++++++++++++++++++++++-- bake/hclparser/hclparser.go | 37 ++++++++------- 3 files changed, 126 insertions(+), 22 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index 470de87d..3760dae4 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -246,13 +246,28 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) } if len(hclFiles) > 0 { - if err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{ + renamed, err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{ LookupVar: os.LookupEnv, Vars: defaults, ValidateLabel: validateTargetName, - }, &c); err.HasErrors() { + }, &c) + if err.HasErrors() { return nil, err } + + for _, renamed := range renamed { + for oldName, newNames := range renamed { + newNames = dedupSlice(newNames) + if len(newNames) == 1 && oldName == newNames[0] { + continue + } + c.Groups = append(c.Groups, &Group{ + Name: oldName, + Targets: newNames, + }) + } + } + c = dedupeConfig(c) } return &c, nil diff --git a/bake/hcl_test.go b/bake/hcl_test.go index 95d3b193..1580c04f 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -666,6 +666,37 @@ func TestHCLRenameTarget(t *testing.T) { require.Equal(t, 1, len(c.Targets)) require.Equal(t, "xyz", c.Targets[0].Name) require.Equal(t, "foo", *c.Targets[0].Dockerfile) + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "abc", c.Groups[0].Name) + require.Equal(t, []string{"xyz"}, c.Groups[0].Targets) +} + +func TestHCLRenameGroup(t *testing.T) { + dt := []byte(` + group "foo" { + name = "bar" + targets = ["x", "y"] + } + + target "x" { + } + target "y" { + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "x", c.Targets[0].Name) + require.Equal(t, "y", c.Targets[1].Name) + + require.Equal(t, 2, len(c.Groups)) + require.Equal(t, "bar", c.Groups[0].Name) + require.Equal(t, []string{"x", "y"}, c.Groups[0].Targets) + require.Equal(t, "foo", c.Groups[1].Name) + require.Equal(t, []string{"bar"}, c.Groups[1].Targets) } func TestHCLRenameTargetAttrs(t *testing.T) { @@ -736,7 +767,7 @@ func TestHCLRenameTargetAttrs(t *testing.T) { require.Error(t, err) } -func TestHCLRenameMerge(t *testing.T) { +func TestHCLRenameSplit(t *testing.T) { dt := []byte(` target "x" { name = "y" @@ -757,6 +788,10 @@ func TestHCLRenameMerge(t *testing.T) { require.Equal(t, "foo", *c.Targets[0].Dockerfile) require.Equal(t, "z", c.Targets[1].Name) require.Equal(t, "bar", *c.Targets[1].Dockerfile) + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "x", c.Groups[0].Name) + require.Equal(t, []string{"y", "z"}, c.Groups[0].Targets) } func TestHCLRenameMultiFile(t *testing.T) { @@ -813,9 +848,13 @@ func TestHCLMatrixBasic(t *testing.T) { require.Equal(t, c.Targets[1].Name, "y") require.Equal(t, *c.Targets[0].Dockerfile, "x.Dockerfile") require.Equal(t, *c.Targets[1].Dockerfile, "y.Dockerfile") + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "default", c.Groups[0].Name) + require.Equal(t, []string{"x", "y"}, c.Groups[0].Targets) } -func TestHCLMatrixMultiple(t *testing.T) { +func TestHCLMatrixMultipleKeys(t *testing.T) { dt := []byte(` target "default" { matrix = { @@ -835,7 +874,54 @@ func TestHCLMatrixMultiple(t *testing.T) { for i, t := range c.Targets { names[i] = t.Name } - require.ElementsMatch(t, names, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}) + require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, names) + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "default", c.Groups[0].Name) + require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, c.Groups[0].Targets) +} + +func TestHCLMatrixMultipleTargets(t *testing.T) { + dt := []byte(` + target "x" { + matrix = { + foo = ["a", "b"] + } + name = foo + } + target "y" { + matrix = { + bar = ["c", "d"] + } + name = bar + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 4, len(c.Targets)) + names := make([]string, len(c.Targets)) + for i, t := range c.Targets { + names[i] = t.Name + } + require.ElementsMatch(t, []string{"a", "b", "c", "d"}, names) + + require.Equal(t, 2, len(c.Groups)) + names = make([]string, len(c.Groups)) + for i, c := range c.Groups { + names[i] = c.Name + } + require.ElementsMatch(t, []string{"x", "y"}, names) + + for _, g := range c.Groups { + switch g.Name { + case "x": + require.Equal(t, []string{"a", "b"}, g.Targets) + case "y": + require.Equal(t, []string{"c", "d"}, g.Targets) + } + } } func TestHCLMatrixArgs(t *testing.T) { diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index d36708d0..04e11630 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -531,7 +531,7 @@ func (p *parser) resolveBlockNames(block *hcl.Block) ([]string, error) { return names, nil } -func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { +func Parse(b hcl.Body, opt Opt, val interface{}) (map[string]map[string][]string, hcl.Diagnostics) { reserved := map[string]struct{}{} schema, _ := gohcl.ImpliedBodySchema(val) @@ -544,7 +544,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { var defs inputs if err := gohcl.DecodeBody(b, nil, &defs); err != nil { - return err + return nil, err } defsSchema, _ := gohcl.ImpliedBodySchema(defs) @@ -599,18 +599,18 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { content, b, diags := b.PartialContent(schema) if diags.HasErrors() { - return diags + return nil, diags } blocks, b, diags := b.PartialContent(defsSchema) if diags.HasErrors() { - return diags + return nil, diags } attrs, diags := b.JustAttributes() if diags.HasErrors() { if d := removeAttributesDiags(diags, reserved, p.vars); len(d) > 0 { - return d + return nil, d } } @@ -627,7 +627,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } for _, a := range content.Attributes { - return hcl.Diagnostics{ + return nil, hcl.Diagnostics{ &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid attribute", @@ -641,17 +641,17 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { for k := range p.vars { if err := p.resolveValue(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { - return diags + return nil, diags } r := p.vars[k].Body.MissingItemRange() - return wrapErrorDiagnostic("Invalid value", err, &r, &r) + return nil, wrapErrorDiagnostic("Invalid value", err, &r, &r) } } for k := range p.funcs { if err := p.resolveFunction(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { - return diags + return nil, diags } var subject *hcl.Range var context *hcl.Range @@ -667,7 +667,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } } } - return wrapErrorDiagnostic("Invalid function", err, subject, context) + return nil, wrapErrorDiagnostic("Invalid function", err, subject, context) } } @@ -681,6 +681,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { values map[string]value } types := map[string]field{} + renamed := map[string]map[string][]string{} vt := reflect.ValueOf(val).Elem().Type() for i := 0; i < vt.NumField(); i++ { tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",") @@ -691,12 +692,13 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { typ: vt.Field(i).Type, values: make(map[string]value), } + renamed[tags[0]] = map[string][]string{} } tmpBlocks := map[string]map[string][]*hcl.Block{} for _, b := range content.Blocks { if len(b.Labels) == 0 || len(b.Labels) > 1 { - return hcl.Diagnostics{ + return nil, hcl.Diagnostics{ &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid block", @@ -715,10 +717,11 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { names, err := p.resolveBlockNames(b) if err != nil { - return wrapErrorDiagnostic("Invalid name", err, &b.LabelRanges[0], &b.LabelRanges[0]) + return nil, wrapErrorDiagnostic("Invalid name", err, &b.LabelRanges[0], &b.LabelRanges[0]) } for _, name := range names { bm[name] = append(bm[name], b) + renamed[b.Type][b.Labels[0]] = append(renamed[b.Type][b.Labels[0]], name) } } p.blocks = tmpBlocks @@ -735,7 +738,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { continue } } else { - return wrapErrorDiagnostic("Invalid block", err, &b.LabelRanges[0], &b.DefRange) + return nil, wrapErrorDiagnostic("Invalid block", err, &b.LabelRanges[0], &b.DefRange) } } @@ -773,19 +776,19 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } } if diags.HasErrors() { - return diags + return nil, diags } for k := range p.attrs { if err := p.resolveValue(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { - return diags + return nil, diags } - return wrapErrorDiagnostic("Invalid attribute", err, &p.attrs[k].Range, &p.attrs[k].Range) + return nil, wrapErrorDiagnostic("Invalid attribute", err, &p.attrs[k].Range, &p.attrs[k].Range) } } - return nil + return renamed, nil } // wrapErrorDiagnostic wraps an error into a hcl.Diagnostics object. From d699d083993f2e47b3714ba1111d06fba4fd8f1a Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Thu, 30 Mar 2023 10:39:38 +0100 Subject: [PATCH 5/6] bake: add additional tests for matrix behavior Signed-off-by: Justin Chadwell --- bake/hcl_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/bake/hcl_test.go b/bake/hcl_test.go index 1580c04f..f84172c5 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -881,6 +881,61 @@ func TestHCLMatrixMultipleKeys(t *testing.T) { require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, c.Groups[0].Targets) } +func TestHCLMatrixLists(t *testing.T) { + dt := []byte(` + target "foo" { + matrix = { + aa = [["aa", "bb"], ["cc", "dd"]] + } + name = aa[0] + args = { + target = "val${aa[1]}" + } + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "aa", c.Targets[0].Name) + require.Equal(t, ptrstr("valbb"), c.Targets[0].Args["target"]) + require.Equal(t, "cc", c.Targets[1].Name) + require.Equal(t, ptrstr("valdd"), c.Targets[1].Args["target"]) +} + +func TestHCLMatrixMaps(t *testing.T) { + dt := []byte(` + target "foo" { + matrix = { + aa = [ + { + foo = "aa" + bar = "bb" + }, + { + foo = "cc" + bar = "dd" + } + ] + } + name = aa.foo + args = { + target = "val${aa.bar}" + } + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, c.Targets[0].Name, "aa") + require.Equal(t, c.Targets[0].Args["target"], ptrstr("valbb")) + require.Equal(t, c.Targets[1].Name, "cc") + require.Equal(t, c.Targets[1].Args["target"], ptrstr("valdd")) +} + func TestHCLMatrixMultipleTargets(t *testing.T) { dt := []byte(` target "x" { @@ -942,8 +997,40 @@ func TestHCLMatrixArgs(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(c.Targets)) - require.Equal(t, c.Targets[0].Name, "1") - require.Equal(t, c.Targets[1].Name, "2") + require.Equal(t, "1", c.Targets[0].Name) + require.Equal(t, "2", c.Targets[1].Name) +} + +func TestHCLMatrixArgsOverride(t *testing.T) { + dt := []byte(` + variable "ABC" { + default = "def" + } + + target "bar" { + matrix = { + aa = split(",", ABC) + } + name = "bar-${aa}" + args = { + foo = aa + } + } + `) + + c, err := ParseFiles([]File{ + {Data: dt, Name: "docker-bake.hcl"}, + }, map[string]string{"ABC": "11,22,33"}) + require.NoError(t, err) + + require.Equal(t, 3, len(c.Targets)) + require.Equal(t, "bar-11", c.Targets[0].Name) + require.Equal(t, "bar-22", c.Targets[1].Name) + require.Equal(t, "bar-33", c.Targets[2].Name) + + require.Equal(t, ptrstr("11"), c.Targets[0].Args["foo"]) + require.Equal(t, ptrstr("22"), c.Targets[1].Args["foo"]) + require.Equal(t, ptrstr("33"), c.Targets[2].Args["foo"]) } func TestHCLMatrixErrors(t *testing.T) { From a1520ea1b26a097ccabf59713dd6d11f58b38647 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Thu, 30 Mar 2023 11:50:28 +0100 Subject: [PATCH 6/6] bake: additional validation for matrixes This adds the following constraints to the new features: - Explicit renaming with the `name` property is *only* permitted when used with the `matrix` property. - Group does not support either `name` or `matrix` (we may choose to relax this constraint over time). - All generated names must be unique. Signed-off-by: Justin Chadwell --- bake/bake.go | 68 ++++++++++++++++++++- bake/hcl_test.go | 65 +++++++++++--------- bake/hclparser/hclparser.go | 115 +++++++++++++++++------------------- 3 files changed, 157 insertions(+), 91 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index 3760dae4..60e158b7 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -24,6 +24,7 @@ import ( "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" ) var ( @@ -612,6 +613,11 @@ type Target struct { linked bool } +var _ hclparser.WithEvalContexts = &Target{} +var _ hclparser.WithGetName = &Target{} +var _ hclparser.WithEvalContexts = &Group{} +var _ hclparser.WithGetName = &Group{} + func (t *Target) normalize() { t.Attest = removeDupes(t.Attest) t.Tags = removeDupes(t.Tags) @@ -795,7 +801,20 @@ func (t *Target) AddOverrides(overrides map[string]Override) error { return nil } -func (t *Target) EvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { +func (g *Group) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { + content, _, err := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, + }) + if err != nil { + return nil, err + } + if _, ok := content.Attributes["matrix"]; ok { + return nil, errors.Errorf("matrix is not supported for groups") + } + return []*hcl.EvalContext{ectx}, nil +} + +func (t *Target) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { content, _, err := block.Body.PartialContent(&hcl.BodySchema{ Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, }) @@ -843,6 +862,53 @@ func (t *Target) EvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps return ectxs, nil } +func (g *Group) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) { + content, _, diags := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}}, + }) + if diags != nil { + return "", diags + } + + if _, ok := content.Attributes["name"]; ok { + return "", errors.Errorf("name is not supported for groups") + } + if _, ok := content.Attributes["matrix"]; ok { + return "", errors.Errorf("matrix is not supported for groups") + } + return block.Labels[0], nil +} + +func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) { + content, _, diags := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}}, + }) + if diags != nil { + return "", diags + } + + attr, ok := content.Attributes["name"] + if !ok { + return block.Labels[0], nil + } + if _, ok := content.Attributes["matrix"]; !ok { + return "", errors.Errorf("name requires matrix") + } + if diags := loadDeps(attr.Expr); diags.HasErrors() { + return "", diags + } + value, diags := attr.Expr.Value(ectx) + if diags != nil { + return "", diags + } + + value, err := convert.Convert(value, cty.String) + if err != nil { + return "", err + } + return value.AsString(), nil +} + func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) { m2 := make(map[string]build.Options, len(m)) for k, v := range m { diff --git a/bake/hcl_test.go b/bake/hcl_test.go index f84172c5..5010984c 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -660,16 +660,8 @@ func TestHCLRenameTarget(t *testing.T) { } `) - c, err := ParseFile(dt, "docker-bake.hcl") - require.NoError(t, err) - - require.Equal(t, 1, len(c.Targets)) - require.Equal(t, "xyz", c.Targets[0].Name) - require.Equal(t, "foo", *c.Targets[0].Dockerfile) - - require.Equal(t, 1, len(c.Groups)) - require.Equal(t, "abc", c.Groups[0].Name) - require.Equal(t, []string{"xyz"}, c.Groups[0].Targets) + _, err := ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "requires matrix") } func TestHCLRenameGroup(t *testing.T) { @@ -678,31 +670,28 @@ func TestHCLRenameGroup(t *testing.T) { name = "bar" targets = ["x", "y"] } - - target "x" { - } - target "y" { - } `) - c, err := ParseFile(dt, "docker-bake.hcl") - require.NoError(t, err) + _, err := ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "not supported") - require.Equal(t, 2, len(c.Targets)) - require.Equal(t, "x", c.Targets[0].Name) - require.Equal(t, "y", c.Targets[1].Name) + dt = []byte(` + group "foo" { + matrix = { + name = ["x", "y"] + } + } + `) - require.Equal(t, 2, len(c.Groups)) - require.Equal(t, "bar", c.Groups[0].Name) - require.Equal(t, []string{"x", "y"}, c.Groups[0].Targets) - require.Equal(t, "foo", c.Groups[1].Name) - require.Equal(t, []string{"bar"}, c.Groups[1].Targets) + _, err = ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "not supported") } func TestHCLRenameTargetAttrs(t *testing.T) { dt := []byte(` target "abc" { name = "xyz" + matrix = {} dockerfile = "foo" } @@ -726,6 +715,7 @@ func TestHCLRenameTargetAttrs(t *testing.T) { target "abc" { name = "xyz" + matrix = {} dockerfile = "foo" } `) @@ -741,6 +731,7 @@ func TestHCLRenameTargetAttrs(t *testing.T) { dt = []byte(` target "abc" { name = "xyz" + matrix = {} dockerfile = "foo" } @@ -750,7 +741,7 @@ func TestHCLRenameTargetAttrs(t *testing.T) { `) _, err = ParseFile(dt, "docker-bake.hcl") - require.Error(t, err) + require.ErrorContains(t, err, "abc") dt = []byte(` target "def" { @@ -759,23 +750,26 @@ func TestHCLRenameTargetAttrs(t *testing.T) { target "abc" { name = "xyz" + matrix = {} dockerfile = "foo" } `) _, err = ParseFile(dt, "docker-bake.hcl") - require.Error(t, err) + require.ErrorContains(t, err, "abc") } func TestHCLRenameSplit(t *testing.T) { dt := []byte(` target "x" { name = "y" + matrix = {} dockerfile = "foo" } target "x" { name = "z" + matrix = {} dockerfile = "bar" } `) @@ -798,6 +792,7 @@ func TestHCLRenameMultiFile(t *testing.T) { dt := []byte(` target "foo" { name = "bar" + matrix = {} dockerfile = "x" } `) @@ -979,6 +974,20 @@ func TestHCLMatrixMultipleTargets(t *testing.T) { } } +func TestHCLMatrixDuplicateNames(t *testing.T) { + dt := []byte(` + target "default" { + matrix = { + foo = ["a", "b"] + } + name = "c" + } + `) + + _, err := ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) +} + func TestHCLMatrixArgs(t *testing.T) { dt := []byte(` a = 1 @@ -1033,7 +1042,7 @@ func TestHCLMatrixArgsOverride(t *testing.T) { require.Equal(t, ptrstr("33"), c.Targets[2].Args["foo"]) } -func TestHCLMatrixErrors(t *testing.T) { +func TestHCLMatrixBadTypes(t *testing.T) { dt := []byte(` target "default" { matrix = "test" diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index 04e11630..f29c4a8a 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -54,6 +54,7 @@ type parser struct { blocks map[string]map[string][]*hcl.Block blockValues map[*hcl.Block][]reflect.Value blockEvalCtx map[*hcl.Block][]*hcl.EvalContext + blockNames map[*hcl.Block][]string blockTypes map[string]reflect.Type ectx *hcl.EvalContext @@ -64,6 +65,14 @@ type parser struct { doneB map[uint64]map[string]struct{} } +type WithEvalContexts interface { + GetEvalContexts(base *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) +} + +type WithGetName interface { + GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) +} + var errUndefined = errors.New("undefined") func (p *parser) loadDeps(ectx *hcl.EvalContext, exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics { @@ -306,6 +315,11 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { // target schema is provided, only the attributes and blocks present in the // schema will be evaluated. func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) { + // prepare the variable map for this type + if _, ok := p.ectx.Variables[block.Type]; !ok { + p.ectx.Variables[block.Type] = cty.MapValEmpty(cty.Map(cty.String)) + } + // prepare the output destination and evaluation context t, ok := p.blockTypes[block.Type] if !ok { @@ -317,11 +331,8 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err outputs = prev ectxs = p.blockEvalCtx[block] } else { - type ectxI interface { - EvalContexts(base *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) - } - if v, ok := reflect.New(t).Interface().(ectxI); ok { - ectxs, err = v.EvalContexts(p.ectx, block, func(expr hcl.Expression) hcl.Diagnostics { + if v, ok := reflect.New(t).Interface().(WithEvalContexts); ok { + ectxs, err = v.GetEvalContexts(p.ectx, block, func(expr hcl.Expression) hcl.Diagnostics { return p.loadDeps(p.ectx, expr, nil, true) }) if err != nil { @@ -345,12 +356,9 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err for i, output := range outputs { target := target ectx := ectxs[i] - name, _ := getName(output) - if name == "" { - name = block.Labels[0] - } - if err := p.opt.ValidateLabel(name); err != nil { - return err + name := block.Labels[0] + if names, ok := p.blockNames[block]; ok { + name = names[i] } if _, ok := p.doneB[key(block, ectx)]; !ok { @@ -415,9 +423,6 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err // load dependencies from all targeted properties schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) - if nameKey, ok := getNameKey(output); ok { - schema.Attributes = append(schema.Attributes, hcl.AttributeSchema{Name: nameKey}) - } content, _, diag := body().PartialContent(schema) if diag.HasErrors() { return diag @@ -440,22 +445,6 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err if diag.HasErrors() { return diag } - if nameKey, ok := getNameKey(output); ok { - for k, v := range content.Attributes { - if k == nameKey { - var name2 string - diag = gohcl.DecodeExpression(v.Expr, ectx, &name) - if diag.HasErrors() { - return diag - } - if name2 != "" { - name = name2 - } - break - } - } - } - setName(output, name) // mark all targeted properties as done for _, a := range content.Attributes { @@ -499,35 +488,49 @@ func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err err // resolveBlockNames returns the names of the block, calling resolveBlock to // evaluate any label fields to correctly resolve the name. func (p *parser) resolveBlockNames(block *hcl.Block) ([]string, error) { - t, ok := p.blockTypes[block.Type] - if !ok { - return nil, errors.Errorf("internal error: unknown block type %s", block.Type) + if names, ok := p.blockNames[block]; ok { + return names, nil } - nameKey, ok := getNameKey(reflect.New(t)) - if ok { - target := &hcl.BodySchema{ - Attributes: []hcl.AttributeSchema{{Name: nameKey}}, - Blocks: []hcl.BlockHeaderSchema{{Type: nameKey}}, - } - if err := p.resolveBlock(block, target); err != nil { + if err := p.resolveBlock(block, &hcl.BodySchema{}); err != nil { + return nil, err + } + + names := make([]string, 0, len(p.blockValues[block])) + for i, val := range p.blockValues[block] { + ectx := p.blockEvalCtx[block][i] + + name := block.Labels[0] + if err := p.opt.ValidateLabel(name); err != nil { return nil, err } - } else { - if err := p.resolveBlock(block, &hcl.BodySchema{}); err != nil { - return nil, err + + if v, ok := val.Interface().(WithGetName); ok { + var err error + name, err = v.GetName(ectx, block, func(expr hcl.Expression) hcl.Diagnostics { + return p.loadDeps(ectx, expr, nil, true) + }) + if err != nil { + return nil, err + } + if err := p.opt.ValidateLabel(name); err != nil { + return nil, err + } } + + setName(val, name) + names = append(names, name) } - names := make([]string, 0, len(p.blockValues[block])) - for _, prev := range p.blockValues[block] { - name, ok := getName(prev) - if !ok { - return nil, errors.New("internal error: failed to get name") + found := map[string]struct{}{} + for _, name := range names { + if _, ok := found[name]; ok { + return nil, errors.Errorf("duplicate name %q", name) } - names = append(names, name) + found[name] = struct{}{} } + p.blockNames[block] = names return names, nil } @@ -570,6 +573,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) (map[string]map[string][]string blocks: map[string]map[string][]*hcl.Block{}, blockValues: map[*hcl.Block][]reflect.Value{}, blockEvalCtx: map[*hcl.Block][]*hcl.EvalContext{}, + blockNames: map[*hcl.Block][]string{}, blockTypes: map[string]reflect.Type{}, ectx: &hcl.EvalContext{ Variables: map[string]cty.Value{}, @@ -837,19 +841,6 @@ func getName(v reflect.Value) (string, bool) { return "", false } -func getNameKey(v reflect.Value) (string, bool) { - numFields := v.Elem().Type().NumField() - for i := 0; i < numFields; i++ { - parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") - for _, t := range parts[1:] { - if t == "label" { - return parts[0], true - } - } - } - return "", false -} - func getNameIndex(v reflect.Value) (int, bool) { numFields := v.Elem().Type().NumField() for i := 0; i < numFields; i++ {