diff --git a/bake/bake.go b/bake/bake.go index c5ec48af..fc6032f2 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -22,6 +22,8 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" ) var ( @@ -231,13 +233,28 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error) } if len(hclFiles) > 0 { - if err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{ + renamed, err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{ LookupVar: os.LookupEnv, Vars: defaults, ValidateLabel: validateTargetName, - }, &c); err.HasErrors() { + }, &c) + if err.HasErrors() { return nil, err } + + for _, renamed := range renamed { + for oldName, newNames := range renamed { + newNames = dedupSlice(newNames) + if len(newNames) == 1 && oldName == newNames[0] { + continue + } + c.Groups = append(c.Groups, &Group{ + Name: oldName, + Targets: newNames, + }) + } + } + c = dedupeConfig(c) } return &c, nil @@ -582,6 +599,11 @@ type Target struct { linked bool } +var _ hclparser.WithEvalContexts = &Target{} +var _ hclparser.WithGetName = &Target{} +var _ hclparser.WithEvalContexts = &Group{} +var _ hclparser.WithGetName = &Group{} + func (t *Target) normalize() { t.Attest = removeDupes(t.Attest) t.Tags = removeDupes(t.Tags) @@ -765,6 +787,114 @@ func (t *Target) AddOverrides(overrides map[string]Override) error { return nil } +func (g *Group) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { + content, _, err := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, + }) + if err != nil { + return nil, err + } + if _, ok := content.Attributes["matrix"]; ok { + return nil, errors.Errorf("matrix is not supported for groups") + } + return []*hcl.EvalContext{ectx}, nil +} + +func (t *Target) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) { + content, _, err := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "matrix"}}, + }) + if err != nil { + return nil, err + } + + attr, ok := content.Attributes["matrix"] + if !ok { + return []*hcl.EvalContext{ectx}, nil + } + if diags := loadDeps(attr.Expr); diags.HasErrors() { + return nil, diags + } + value, err := attr.Expr.Value(ectx) + if err != nil { + return nil, err + } + + if !value.CanIterateElements() { + return nil, errors.Errorf("matrix must be a map") + } + matrix := value.AsValueMap() + + ectxs := []*hcl.EvalContext{ectx} + for k, expr := range matrix { + if !expr.CanIterateElements() { + return nil, errors.Errorf("matrix values must be a list") + } + + ectxs2 := []*hcl.EvalContext{} + for _, v := range expr.AsValueSlice() { + for _, e := range ectxs { + e2 := ectx.NewChild() + e2.Variables = make(map[string]cty.Value) + for k, v := range e.Variables { + e2.Variables[k] = v + } + e2.Variables[k] = v + ectxs2 = append(ectxs2, e2) + } + } + ectxs = ectxs2 + } + return ectxs, nil +} + +func (g *Group) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) { + content, _, diags := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}}, + }) + if diags != nil { + return "", diags + } + + if _, ok := content.Attributes["name"]; ok { + return "", errors.Errorf("name is not supported for groups") + } + if _, ok := content.Attributes["matrix"]; ok { + return "", errors.Errorf("matrix is not supported for groups") + } + return block.Labels[0], nil +} + +func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) { + content, _, diags := block.Body.PartialContent(&hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}}, + }) + if diags != nil { + return "", diags + } + + attr, ok := content.Attributes["name"] + if !ok { + return block.Labels[0], nil + } + if _, ok := content.Attributes["matrix"]; !ok { + return "", errors.Errorf("name requires matrix") + } + if diags := loadDeps(attr.Expr); diags.HasErrors() { + return "", diags + } + value, diags := attr.Expr.Value(ectx) + if diags != nil { + return "", diags + } + + value, err := convert.Convert(value, cty.String) + if err != nil { + return "", err + } + return value.AsString(), nil +} + func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) { m2 := make(map[string]build.Options, len(m)) for k, v := range m { diff --git a/bake/hcl_test.go b/bake/hcl_test.go index ecb0d722..5010984c 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -634,6 +634,444 @@ func TestHCLMultiFileAttrs(t *testing.T) { require.Equal(t, ptrstr("pre-ghi"), c.Targets[0].Args["v1"]) } +func TestHCLDuplicateTarget(t *testing.T) { + dt := []byte(` + target "app" { + dockerfile = "x" + } + target "app" { + dockerfile = "y" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "app", c.Targets[0].Name) + require.Equal(t, "y", *c.Targets[0].Dockerfile) +} + +func TestHCLRenameTarget(t *testing.T) { + dt := []byte(` + target "abc" { + name = "xyz" + dockerfile = "foo" + } + `) + + _, err := ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "requires matrix") +} + +func TestHCLRenameGroup(t *testing.T) { + dt := []byte(` + group "foo" { + name = "bar" + targets = ["x", "y"] + } + `) + + _, err := ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "not supported") + + dt = []byte(` + group "foo" { + matrix = { + name = ["x", "y"] + } + } + `) + + _, err = ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "not supported") +} + +func TestHCLRenameTargetAttrs(t *testing.T) { + dt := []byte(` + target "abc" { + name = "xyz" + matrix = {} + dockerfile = "foo" + } + + target "def" { + dockerfile = target.xyz.dockerfile + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "xyz", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) + require.Equal(t, "def", c.Targets[1].Name) + require.Equal(t, "foo", *c.Targets[1].Dockerfile) + + dt = []byte(` + target "def" { + dockerfile = target.xyz.dockerfile + } + + target "abc" { + name = "xyz" + matrix = {} + dockerfile = "foo" + } + `) + + c, err = ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "def", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) + require.Equal(t, "xyz", c.Targets[1].Name) + require.Equal(t, "foo", *c.Targets[1].Dockerfile) + + dt = []byte(` + target "abc" { + name = "xyz" + matrix = {} + dockerfile = "foo" + } + + target "def" { + dockerfile = target.abc.dockerfile + } + `) + + _, err = ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "abc") + + dt = []byte(` + target "def" { + dockerfile = target.abc.dockerfile + } + + target "abc" { + name = "xyz" + matrix = {} + dockerfile = "foo" + } + `) + + _, err = ParseFile(dt, "docker-bake.hcl") + require.ErrorContains(t, err, "abc") +} + +func TestHCLRenameSplit(t *testing.T) { + dt := []byte(` + target "x" { + name = "y" + matrix = {} + dockerfile = "foo" + } + + target "x" { + name = "z" + matrix = {} + dockerfile = "bar" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "y", c.Targets[0].Name) + require.Equal(t, "foo", *c.Targets[0].Dockerfile) + require.Equal(t, "z", c.Targets[1].Name) + require.Equal(t, "bar", *c.Targets[1].Dockerfile) + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "x", c.Groups[0].Name) + require.Equal(t, []string{"y", "z"}, c.Groups[0].Targets) +} + +func TestHCLRenameMultiFile(t *testing.T) { + dt := []byte(` + target "foo" { + name = "bar" + matrix = {} + dockerfile = "x" + } + `) + dt2 := []byte(` + target "foo" { + context = "y" + } + `) + dt3 := []byte(` + target "bar" { + target = "z" + } + `) + + c, err := ParseFiles([]File{ + {Data: dt, Name: "c1.hcl"}, + {Data: dt2, Name: "c2.hcl"}, + {Data: dt3, Name: "c3.hcl"}, + }, nil) + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + + require.Equal(t, c.Targets[0].Name, "bar") + require.Equal(t, *c.Targets[0].Dockerfile, "x") + require.Equal(t, *c.Targets[0].Target, "z") + + require.Equal(t, c.Targets[1].Name, "foo") + require.Equal(t, *c.Targets[1].Context, "y") +} + +func TestHCLMatrixBasic(t *testing.T) { + dt := []byte(` + target "default" { + matrix = { + foo = ["x", "y"] + } + name = foo + dockerfile = "${foo}.Dockerfile" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, c.Targets[0].Name, "x") + require.Equal(t, c.Targets[1].Name, "y") + require.Equal(t, *c.Targets[0].Dockerfile, "x.Dockerfile") + require.Equal(t, *c.Targets[1].Dockerfile, "y.Dockerfile") + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "default", c.Groups[0].Name) + require.Equal(t, []string{"x", "y"}, c.Groups[0].Targets) +} + +func TestHCLMatrixMultipleKeys(t *testing.T) { + dt := []byte(` + target "default" { + matrix = { + foo = ["a"] + bar = ["b", "c"] + baz = ["d", "e", "f"] + } + name = "${foo}-${bar}-${baz}" + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 6, len(c.Targets)) + names := make([]string, len(c.Targets)) + for i, t := range c.Targets { + names[i] = t.Name + } + require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, names) + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "default", c.Groups[0].Name) + require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, c.Groups[0].Targets) +} + +func TestHCLMatrixLists(t *testing.T) { + dt := []byte(` + target "foo" { + matrix = { + aa = [["aa", "bb"], ["cc", "dd"]] + } + name = aa[0] + args = { + target = "val${aa[1]}" + } + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "aa", c.Targets[0].Name) + require.Equal(t, ptrstr("valbb"), c.Targets[0].Args["target"]) + require.Equal(t, "cc", c.Targets[1].Name) + require.Equal(t, ptrstr("valdd"), c.Targets[1].Args["target"]) +} + +func TestHCLMatrixMaps(t *testing.T) { + dt := []byte(` + target "foo" { + matrix = { + aa = [ + { + foo = "aa" + bar = "bb" + }, + { + foo = "cc" + bar = "dd" + } + ] + } + name = aa.foo + args = { + target = "val${aa.bar}" + } + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, c.Targets[0].Name, "aa") + require.Equal(t, c.Targets[0].Args["target"], ptrstr("valbb")) + require.Equal(t, c.Targets[1].Name, "cc") + require.Equal(t, c.Targets[1].Args["target"], ptrstr("valdd")) +} + +func TestHCLMatrixMultipleTargets(t *testing.T) { + dt := []byte(` + target "x" { + matrix = { + foo = ["a", "b"] + } + name = foo + } + target "y" { + matrix = { + bar = ["c", "d"] + } + name = bar + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 4, len(c.Targets)) + names := make([]string, len(c.Targets)) + for i, t := range c.Targets { + names[i] = t.Name + } + require.ElementsMatch(t, []string{"a", "b", "c", "d"}, names) + + require.Equal(t, 2, len(c.Groups)) + names = make([]string, len(c.Groups)) + for i, c := range c.Groups { + names[i] = c.Name + } + require.ElementsMatch(t, []string{"x", "y"}, names) + + for _, g := range c.Groups { + switch g.Name { + case "x": + require.Equal(t, []string{"a", "b"}, g.Targets) + case "y": + require.Equal(t, []string{"c", "d"}, g.Targets) + } + } +} + +func TestHCLMatrixDuplicateNames(t *testing.T) { + dt := []byte(` + target "default" { + matrix = { + foo = ["a", "b"] + } + name = "c" + } + `) + + _, err := ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) +} + +func TestHCLMatrixArgs(t *testing.T) { + dt := []byte(` + a = 1 + variable "b" { + default = 2 + } + target "default" { + matrix = { + foo = [a, b] + } + name = foo + } + `) + + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 2, len(c.Targets)) + require.Equal(t, "1", c.Targets[0].Name) + require.Equal(t, "2", c.Targets[1].Name) +} + +func TestHCLMatrixArgsOverride(t *testing.T) { + dt := []byte(` + variable "ABC" { + default = "def" + } + + target "bar" { + matrix = { + aa = split(",", ABC) + } + name = "bar-${aa}" + args = { + foo = aa + } + } + `) + + c, err := ParseFiles([]File{ + {Data: dt, Name: "docker-bake.hcl"}, + }, map[string]string{"ABC": "11,22,33"}) + require.NoError(t, err) + + require.Equal(t, 3, len(c.Targets)) + require.Equal(t, "bar-11", c.Targets[0].Name) + require.Equal(t, "bar-22", c.Targets[1].Name) + require.Equal(t, "bar-33", c.Targets[2].Name) + + require.Equal(t, ptrstr("11"), c.Targets[0].Args["foo"]) + require.Equal(t, ptrstr("22"), c.Targets[1].Args["foo"]) + require.Equal(t, ptrstr("33"), c.Targets[2].Args["foo"]) +} + +func TestHCLMatrixBadTypes(t *testing.T) { + dt := []byte(` + target "default" { + matrix = "test" + } + `) + _, err := ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) + + dt = []byte(` + target "default" { + matrix = { + ["a"] = ["b"] + } + } + `) + _, err = ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) + + dt = []byte(` + target "default" { + matrix = { + a = "b" + } + } + `) + _, err = ParseFile(dt, "docker-bake.hcl") + require.Error(t, err) +} + func TestJSONAttributes(t *testing.T) { dt := []byte(`{"FOO": "abc", "variable": {"BAR": {"default": "def"}}, "target": { "app": { "args": {"v1": "pre-${FOO}-${BAR}"}} } }`) diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index ec378cf9..f29c4a8a 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -1,7 +1,9 @@ package hclparser import ( + "encoding/binary" "fmt" + "hash/fnv" "math" "math/big" "reflect" @@ -49,29 +51,38 @@ type parser struct { attrs map[string]*hcl.Attribute funcs map[string]*functionDef - blocks map[string]map[string][]*hcl.Block - blockValues map[*hcl.Block]reflect.Value - blockTypes map[string]reflect.Type + blocks map[string]map[string][]*hcl.Block + blockValues map[*hcl.Block][]reflect.Value + blockEvalCtx map[*hcl.Block][]*hcl.EvalContext + blockNames map[*hcl.Block][]string + blockTypes map[string]reflect.Type ectx *hcl.EvalContext - progress map[string]struct{} - progressF map[string]struct{} - progressB map[*hcl.Block]map[string]struct{} - doneF map[string]struct{} - doneB map[*hcl.Block]map[string]struct{} + progressV map[uint64]struct{} + progressF map[uint64]struct{} + progressB map[uint64]map[string]struct{} + doneB map[uint64]map[string]struct{} +} + +type WithEvalContexts interface { + GetEvalContexts(base *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) +} + +type WithGetName interface { + GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) } var errUndefined = errors.New("undefined") -func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics { +func (p *parser) loadDeps(ectx *hcl.EvalContext, exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics { fns, hcldiags := funcCalls(exp) if hcldiags.HasErrors() { return hcldiags } for _, fn := range fns { - if err := p.resolveFunction(fn); err != nil { + if err := p.resolveFunction(ectx, fn); err != nil { if allowMissing && errors.Is(err, errUndefined) { continue } @@ -124,14 +135,16 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allow } } } - if err := p.resolveBlock(blocks[0], target); err != nil { - if allowMissing && errors.Is(err, errUndefined) { - continue + for _, block := range blocks { + if err := p.resolveBlock(block, target); err != nil { + if allowMissing && errors.Is(err, errUndefined) { + continue + } + return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr()) } - return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr()) } } else { - if err := p.resolveValue(v.RootName()); err != nil { + if err := p.resolveValue(ectx, v.RootName()); err != nil { if allowMissing && errors.Is(err, errUndefined) { continue } @@ -145,21 +158,21 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allow // resolveFunction forces evaluation of a function, storing the result into the // parser. -func (p *parser) resolveFunction(name string) error { - if _, ok := p.doneF[name]; ok { +func (p *parser) resolveFunction(ectx *hcl.EvalContext, name string) error { + if _, ok := p.ectx.Functions[name]; ok { + return nil + } + if _, ok := ectx.Functions[name]; ok { return nil } f, ok := p.funcs[name] if !ok { - if _, ok := p.ectx.Functions[name]; ok { - return nil - } - return errors.Wrapf(errUndefined, "function %q does not exit", name) + return errors.Wrapf(errUndefined, "function %q does not exist", name) } - if _, ok := p.progressF[name]; ok { + if _, ok := p.progressF[key(ectx, name)]; ok { return errors.Errorf("function cycle not allowed for %s", name) } - p.progressF[name] = struct{}{} + p.progressF[key(ectx, name)] = struct{}{} if f.Result == nil { return errors.Errorf("empty result not allowed for %s", name) @@ -204,7 +217,7 @@ func (p *parser) resolveFunction(name string) error { return diags } - if diags := p.loadDeps(f.Result.Expr, params, false); diags.HasErrors() { + if diags := p.loadDeps(p.ectx, f.Result.Expr, params, false); diags.HasErrors() { return diags } @@ -214,7 +227,6 @@ func (p *parser) resolveFunction(name string) error { if diags.HasErrors() { return diags } - p.doneF[name] = struct{}{} p.ectx.Functions[name] = v return nil @@ -222,14 +234,17 @@ func (p *parser) resolveFunction(name string) error { // resolveValue forces evaluation of a named value, storing the result into the // parser. -func (p *parser) resolveValue(name string) (err error) { +func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { if _, ok := p.ectx.Variables[name]; ok { return nil } - if _, ok := p.progress[name]; ok { + if _, ok := ectx.Variables[name]; ok { + return nil + } + if _, ok := p.progressV[key(ectx, name)]; ok { return errors.Errorf("variable cycle not allowed for %s", name) } - p.progress[name] = struct{}{} + p.progressV[key(ectx, name)] = struct{}{} var v *cty.Value defer func() { @@ -242,9 +257,10 @@ func (p *parser) resolveValue(name string) (err error) { if _, builtin := p.opt.Vars[name]; !ok && !builtin { vr, ok := p.vars[name] if !ok { - return errors.Wrapf(errUndefined, "variable %q does not exit", name) + return errors.Wrapf(errUndefined, "variable %q does not exist", name) } def = vr.Default + ectx = p.ectx } if def == nil { @@ -257,10 +273,10 @@ func (p *parser) resolveValue(name string) (err error) { return } - if diags := p.loadDeps(def.Expr, nil, true); diags.HasErrors() { + if diags := p.loadDeps(ectx, def.Expr, nil, true); diags.HasErrors() { return diags } - vv, diags := def.Expr.Value(p.ectx) + vv, diags := def.Expr.Value(ectx) if diags.HasErrors() { return diags } @@ -299,147 +315,226 @@ func (p *parser) resolveValue(name string) (err error) { // target schema is provided, only the attributes and blocks present in the // schema will be evaluated. func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) { - name := block.Labels[0] - if err := p.opt.ValidateLabel(name); err != nil { - return wrapErrorDiagnostic("Invalid name", err, &block.LabelRanges[0], &block.LabelRanges[0]) + // prepare the variable map for this type + if _, ok := p.ectx.Variables[block.Type]; !ok { + p.ectx.Variables[block.Type] = cty.MapValEmpty(cty.Map(cty.String)) } - if _, ok := p.doneB[block]; !ok { - p.doneB[block] = map[string]struct{}{} - } - if _, ok := p.progressB[block]; !ok { - p.progressB[block] = map[string]struct{}{} + // prepare the output destination and evaluation context + t, ok := p.blockTypes[block.Type] + if !ok { + return nil } - - if target != nil { - // filter out attributes and blocks that are already evaluated - original := target - target = &hcl.BodySchema{} - for _, a := range original.Attributes { - if _, ok := p.doneB[block][a.Name]; !ok { - target.Attributes = append(target.Attributes, a) + var outputs []reflect.Value + var ectxs []*hcl.EvalContext + if prev, ok := p.blockValues[block]; ok { + outputs = prev + ectxs = p.blockEvalCtx[block] + } else { + if v, ok := reflect.New(t).Interface().(WithEvalContexts); ok { + ectxs, err = v.GetEvalContexts(p.ectx, block, func(expr hcl.Expression) hcl.Diagnostics { + return p.loadDeps(p.ectx, expr, nil, true) + }) + if err != nil { + return err } - } - for _, b := range original.Blocks { - if _, ok := p.doneB[block][b.Type]; !ok { - target.Blocks = append(target.Blocks, b) + for _, ectx := range ectxs { + if ectx != p.ectx && ectx.Parent() != p.ectx { + return errors.Errorf("EvalContext must return a context with the correct parent") + } } + } else { + ectxs = append([]*hcl.EvalContext{}, p.ectx) } - if len(target.Attributes) == 0 && len(target.Blocks) == 0 { - return nil + for range ectxs { + outputs = append(outputs, reflect.New(t)) } } + p.blockValues[block] = outputs + p.blockEvalCtx[block] = ectxs - if target != nil { - // detect reference cycles - for _, a := range target.Attributes { - if _, ok := p.progressB[block][a.Name]; ok { - return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name) - } + for i, output := range outputs { + target := target + ectx := ectxs[i] + name := block.Labels[0] + if names, ok := p.blockNames[block]; ok { + name = names[i] } - for _, b := range target.Blocks { - if _, ok := p.progressB[block][b.Type]; ok { - return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type) - } + + if _, ok := p.doneB[key(block, ectx)]; !ok { + p.doneB[key(block, ectx)] = map[string]struct{}{} } - for _, a := range target.Attributes { - p.progressB[block][a.Name] = struct{}{} + if _, ok := p.progressB[key(block, ectx)]; !ok { + p.progressB[key(block, ectx)] = map[string]struct{}{} } - for _, b := range target.Blocks { - p.progressB[block][b.Type] = struct{}{} + + if target != nil { + // filter out attributes and blocks that are already evaluated + original := target + target = &hcl.BodySchema{} + for _, a := range original.Attributes { + if _, ok := p.doneB[key(block, ectx)][a.Name]; !ok { + target.Attributes = append(target.Attributes, a) + } + } + for _, b := range original.Blocks { + if _, ok := p.doneB[key(block, ectx)][b.Type]; !ok { + target.Blocks = append(target.Blocks, b) + } + } + if len(target.Attributes) == 0 && len(target.Blocks) == 0 { + return nil + } } - } - // create a filtered body that contains only the target properties - body := func() hcl.Body { if target != nil { - return FilterIncludeBody(block.Body, target) + // detect reference cycles + for _, a := range target.Attributes { + if _, ok := p.progressB[key(block, ectx)][a.Name]; ok { + return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name) + } + } + for _, b := range target.Blocks { + if _, ok := p.progressB[key(block, ectx)][b.Type]; ok { + return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type) + } + } + for _, a := range target.Attributes { + p.progressB[key(block, ectx)][a.Name] = struct{}{} + } + for _, b := range target.Blocks { + p.progressB[key(block, ectx)][b.Type] = struct{}{} + } + } + + // create a filtered body that contains only the target properties + body := func() hcl.Body { + if target != nil { + return FilterIncludeBody(block.Body, target) + } + + filter := &hcl.BodySchema{} + for k := range p.doneB[key(block, ectx)] { + filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k}) + filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k}) + } + return FilterExcludeBody(block.Body, filter) } - filter := &hcl.BodySchema{} - for k := range p.doneB[block] { - filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k}) - filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k}) + // load dependencies from all targeted properties + schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) + content, _, diag := body().PartialContent(schema) + if diag.HasErrors() { + return diag + } + for _, a := range content.Attributes { + diag := p.loadDeps(ectx, a.Expr, nil, true) + if diag.HasErrors() { + return diag + } + } + for _, b := range content.Blocks { + err := p.resolveBlock(b, nil) + if err != nil { + return err + } } - return FilterExcludeBody(block.Body, filter) - } - // load dependencies from all targeted properties - t, ok := p.blockTypes[block.Type] - if !ok { - return nil - } - schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface()) - content, _, diag := body().PartialContent(schema) - if diag.HasErrors() { - return diag - } - for _, a := range content.Attributes { - diag := p.loadDeps(a.Expr, nil, true) + // decode! + diag = gohcl.DecodeBody(body(), ectx, output.Interface()) if diag.HasErrors() { return diag } - } - for _, b := range content.Blocks { - err := p.resolveBlock(b, nil) + + // mark all targeted properties as done + for _, a := range content.Attributes { + p.doneB[key(block, ectx)][a.Name] = struct{}{} + } + for _, b := range content.Blocks { + p.doneB[key(block, ectx)][b.Type] = struct{}{} + } + if target != nil { + for _, a := range target.Attributes { + p.doneB[key(block, ectx)][a.Name] = struct{}{} + } + for _, b := range target.Blocks { + p.doneB[key(block, ectx)][b.Type] = struct{}{} + } + } + + // store the result into the evaluation context (so it can be referenced) + outputType, err := gocty.ImpliedType(output.Interface()) if err != nil { return err } + outputValue, err := gocty.ToCtyValue(output.Interface(), outputType) + if err != nil { + return err + } + var m map[string]cty.Value + if m2, ok := p.ectx.Variables[block.Type]; ok { + m = m2.AsValueMap() + } + if m == nil { + m = map[string]cty.Value{} + } + m[name] = outputValue + p.ectx.Variables[block.Type] = cty.MapVal(m) } - // decode! - var output reflect.Value - if prev, ok := p.blockValues[block]; ok { - output = prev - } else { - output = reflect.New(t) - setLabel(output, block.Labels[0]) // early attach labels, so we can reference them - } - diag = gohcl.DecodeBody(body(), p.ectx, output.Interface()) - if diag.HasErrors() { - return diag - } - p.blockValues[block] = output + return nil +} - // mark all targeted properties as done - for _, a := range content.Attributes { - p.doneB[block][a.Name] = struct{}{} +// resolveBlockNames returns the names of the block, calling resolveBlock to +// evaluate any label fields to correctly resolve the name. +func (p *parser) resolveBlockNames(block *hcl.Block) ([]string, error) { + if names, ok := p.blockNames[block]; ok { + return names, nil } - for _, b := range content.Blocks { - p.doneB[block][b.Type] = struct{}{} + + if err := p.resolveBlock(block, &hcl.BodySchema{}); err != nil { + return nil, err } - if target != nil { - for _, a := range target.Attributes { - p.doneB[block][a.Name] = struct{}{} + + names := make([]string, 0, len(p.blockValues[block])) + for i, val := range p.blockValues[block] { + ectx := p.blockEvalCtx[block][i] + + name := block.Labels[0] + if err := p.opt.ValidateLabel(name); err != nil { + return nil, err } - for _, b := range target.Blocks { - p.doneB[block][b.Type] = struct{}{} + + if v, ok := val.Interface().(WithGetName); ok { + var err error + name, err = v.GetName(ectx, block, func(expr hcl.Expression) hcl.Diagnostics { + return p.loadDeps(ectx, expr, nil, true) + }) + if err != nil { + return nil, err + } + if err := p.opt.ValidateLabel(name); err != nil { + return nil, err + } } - } - // store the result into the evaluation context (so if can be referenced) - outputType, err := gocty.ImpliedType(output.Interface()) - if err != nil { - return err - } - outputValue, err := gocty.ToCtyValue(output.Interface(), outputType) - if err != nil { - return err + setName(val, name) + names = append(names, name) } - var m map[string]cty.Value - if m2, ok := p.ectx.Variables[block.Type]; ok { - m = m2.AsValueMap() - } - if m == nil { - m = map[string]cty.Value{} + + found := map[string]struct{}{} + for _, name := range names { + if _, ok := found[name]; ok { + return nil, errors.Errorf("duplicate name %q", name) + } + found[name] = struct{}{} } - m[name] = outputValue - p.ectx.Variables[block.Type] = cty.MapVal(m) - return nil + p.blockNames[block] = names + return names, nil } -func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { +func Parse(b hcl.Body, opt Opt, val interface{}) (map[string]map[string][]string, hcl.Diagnostics) { reserved := map[string]struct{}{} schema, _ := gohcl.ImpliedBodySchema(val) @@ -452,7 +547,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { var defs inputs if err := gohcl.DecodeBody(b, nil, &defs); err != nil { - return err + return nil, err } defsSchema, _ := gohcl.ImpliedBodySchema(defs) @@ -475,20 +570,20 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { attrs: map[string]*hcl.Attribute{}, funcs: map[string]*functionDef{}, - blocks: map[string]map[string][]*hcl.Block{}, - blockValues: map[*hcl.Block]reflect.Value{}, - blockTypes: map[string]reflect.Type{}, - - progress: map[string]struct{}{}, - progressF: map[string]struct{}{}, - progressB: map[*hcl.Block]map[string]struct{}{}, - - doneF: map[string]struct{}{}, - doneB: map[*hcl.Block]map[string]struct{}{}, + blocks: map[string]map[string][]*hcl.Block{}, + blockValues: map[*hcl.Block][]reflect.Value{}, + blockEvalCtx: map[*hcl.Block][]*hcl.EvalContext{}, + blockNames: map[*hcl.Block][]string{}, + blockTypes: map[string]reflect.Type{}, ectx: &hcl.EvalContext{ Variables: map[string]cty.Value{}, - Functions: stdlibFunctions, + Functions: Stdlib(), }, + + progressV: map[uint64]struct{}{}, + progressF: map[uint64]struct{}{}, + progressB: map[uint64]map[string]struct{}{}, + doneB: map[uint64]map[string]struct{}{}, } for _, v := range defs.Variables { @@ -508,18 +603,18 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { content, b, diags := b.PartialContent(schema) if diags.HasErrors() { - return diags + return nil, diags } blocks, b, diags := b.PartialContent(defsSchema) if diags.HasErrors() { - return diags + return nil, diags } attrs, diags := b.JustAttributes() if diags.HasErrors() { if d := removeAttributesDiags(diags, reserved, p.vars); len(d) > 0 { - return d + return nil, d } } @@ -532,11 +627,11 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { delete(p.attrs, "function") for k := range p.opt.Vars { - _ = p.resolveValue(k) + _ = p.resolveValue(p.ectx, k) } for _, a := range content.Attributes { - return hcl.Diagnostics{ + return nil, hcl.Diagnostics{ &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Invalid attribute", @@ -548,19 +643,19 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } for k := range p.vars { - if err := p.resolveValue(k); err != nil { + if err := p.resolveValue(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { - return diags + return nil, diags } r := p.vars[k].Body.MissingItemRange() - return wrapErrorDiagnostic("Invalid value", err, &r, &r) + return nil, wrapErrorDiagnostic("Invalid value", err, &r, &r) } } for k := range p.funcs { - if err := p.resolveFunction(k); err != nil { + if err := p.resolveFunction(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { - return diags + return nil, diags } var subject *hcl.Range var context *hcl.Range @@ -576,32 +671,10 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { } } } - return wrapErrorDiagnostic("Invalid function", err, subject, context) + return nil, wrapErrorDiagnostic("Invalid function", err, subject, context) } } - for _, b := range content.Blocks { - if len(b.Labels) == 0 || len(b.Labels) > 1 { - return hcl.Diagnostics{ - &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid block", - Detail: fmt.Sprintf("invalid block label: %v", b.Labels), - Subject: &b.LabelRanges[0], - Context: &b.LabelRanges[0], - }, - } - } - bm, ok := p.blocks[b.Type] - if !ok { - bm = map[string][]*hcl.Block{} - p.blocks[b.Type] = bm - } - - lbl := b.Labels[0] - bm[lbl] = append(bm[lbl], b) - } - type value struct { reflect.Value idx int @@ -612,7 +685,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { values map[string]value } types := map[string]field{} - + renamed := map[string]map[string][]string{} vt := reflect.ValueOf(val).Elem().Type() for i := 0; i < vt.NumField(); i++ { tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",") @@ -623,7 +696,39 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { typ: vt.Field(i).Type, values: make(map[string]value), } + renamed[tags[0]] = map[string][]string{} + } + + tmpBlocks := map[string]map[string][]*hcl.Block{} + for _, b := range content.Blocks { + if len(b.Labels) == 0 || len(b.Labels) > 1 { + return nil, hcl.Diagnostics{ + &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid block", + Detail: fmt.Sprintf("invalid block label: %v", b.Labels), + Subject: &b.LabelRanges[0], + Context: &b.LabelRanges[0], + }, + } + } + + bm, ok := tmpBlocks[b.Type] + if !ok { + bm = map[string][]*hcl.Block{} + tmpBlocks[b.Type] = bm + } + + names, err := p.resolveBlockNames(b) + if err != nil { + return nil, wrapErrorDiagnostic("Invalid name", err, &b.LabelRanges[0], &b.LabelRanges[0]) + } + for _, name := range names { + bm[name] = append(bm[name], b) + renamed[b.Type][b.Labels[0]] = append(renamed[b.Type][b.Labels[0]], name) + } } + p.blocks = tmpBlocks diags = hcl.Diagnostics{} for _, b := range content.Blocks { @@ -637,56 +742,57 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics { continue } } else { - return wrapErrorDiagnostic("Invalid block", err, &b.LabelRanges[0], &b.DefRange) + return nil, wrapErrorDiagnostic("Invalid block", err, &b.LabelRanges[0], &b.DefRange) } } - vv := p.blockValues[b] - - t := types[b.Type] - lblIndex := setLabel(vv, b.Labels[0]) - - oldValue, exists := t.values[b.Labels[0]] - if !exists && lblIndex != -1 { - if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice { - for i := 0; i < v.Elem().Field(t.idx).Len(); i++ { - if b.Labels[0] == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() { - exists = true - oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i} - break + vvs := p.blockValues[b] + for _, vv := range vvs { + t := types[b.Type] + lblIndex, lblExists := getNameIndex(vv) + lblName, _ := getName(vv) + oldValue, exists := t.values[lblName] + if !exists && lblExists { + if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice { + for i := 0; i < v.Elem().Field(t.idx).Len(); i++ { + if lblName == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() { + exists = true + oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i} + break + } } } } - } - if exists { - if m := oldValue.Value.MethodByName("Merge"); m.IsValid() { - m.Call([]reflect.Value{vv}) + if exists { + if m := oldValue.Value.MethodByName("Merge"); m.IsValid() { + m.Call([]reflect.Value{vv}) + } else { + v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv) + } } else { - v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv) - } - } else { - slice := v.Elem().Field(t.idx) - if slice.IsNil() { - slice = reflect.New(t.typ).Elem() + slice := v.Elem().Field(t.idx) + if slice.IsNil() { + slice = reflect.New(t.typ).Elem() + } + t.values[lblName] = value{Value: vv, idx: slice.Len()} + v.Elem().Field(t.idx).Set(reflect.Append(slice, vv)) } - t.values[b.Labels[0]] = value{Value: vv, idx: slice.Len()} - v.Elem().Field(t.idx).Set(reflect.Append(slice, vv)) } } if diags.HasErrors() { - return diags + return nil, diags } for k := range p.attrs { - if err := p.resolveValue(k); err != nil { + if err := p.resolveValue(p.ectx, k); err != nil { if diags, ok := err.(hcl.Diagnostics); ok { - return diags + return nil, diags } - return wrapErrorDiagnostic("Invalid attribute", err, &p.attrs[k].Range, &p.attrs[k].Range) + return nil, wrapErrorDiagnostic("Invalid attribute", err, &p.attrs[k].Range, &p.attrs[k].Range) } } - return nil + return renamed, nil } // wrapErrorDiagnostic wraps an error into a hcl.Diagnostics object. @@ -710,18 +816,42 @@ func wrapErrorDiagnostic(message string, err error, subject *hcl.Range, context } } -func setLabel(v reflect.Value, lbl string) int { - // cache field index? +func setName(v reflect.Value, name string) { + numFields := v.Elem().Type().NumField() + for i := 0; i < numFields; i++ { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { + if t == "label" { + v.Elem().Field(i).Set(reflect.ValueOf(name)) + } + } + } +} + +func getName(v reflect.Value) (string, bool) { + numFields := v.Elem().Type().NumField() + for i := 0; i < numFields; i++ { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { + if t == "label" { + return v.Elem().Field(i).String(), true + } + } + } + return "", false +} + +func getNameIndex(v reflect.Value) (int, bool) { numFields := v.Elem().Type().NumField() for i := 0; i < numFields; i++ { - for _, t := range strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") { + parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") + for _, t := range parts[1:] { if t == "label" { - v.Elem().Field(i).Set(reflect.ValueOf(lbl)) - return i + return i, true } } } - return -1 + return 0, false } func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{}, vars map[string]*variable) hcl.Diagnostics { @@ -753,3 +883,21 @@ func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{}, } return fdiags } + +// key returns a unique hash for the given values +func key(ks ...any) uint64 { + hash := fnv.New64a() + for _, k := range ks { + v := reflect.ValueOf(k) + switch v.Kind() { + case reflect.String: + hash.Write([]byte(v.String())) + case reflect.Pointer: + ptr := reflect.ValueOf(k).Pointer() + binary.Write(hash, binary.LittleEndian, uint64(ptr)) + default: + panic(fmt.Sprintf("unknown key kind %s", v.Kind().String())) + } + } + return hash.Sum64() +} diff --git a/bake/hclparser/stdlib.go b/bake/hclparser/stdlib.go index f78b6dce..53d1d416 100644 --- a/bake/hclparser/stdlib.go +++ b/bake/hclparser/stdlib.go @@ -124,3 +124,11 @@ var timestampFunc = function.New(&function.Spec{ return cty.StringVal(time.Now().UTC().Format(time.RFC3339)), nil }, }) + +func Stdlib() map[string]function.Function { + funcs := make(map[string]function.Function, len(stdlibFunctions)) + for k, v := range stdlibFunctions { + funcs[k] = v + } + return funcs +}