Merge pull request #1690 from jedevc/bake-matrix

Implement matrix for bake targets
pull/1722/head
Justin Chadwell 2 years ago committed by GitHub
commit f7d8bd2055
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -22,6 +22,8 @@ import (
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/pkg/errors"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/convert"
)
var (
@ -231,13 +233,28 @@ func ParseFiles(files []File, defaults map[string]string) (_ *Config, err error)
}
if len(hclFiles) > 0 {
if err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{
renamed, err := hclparser.Parse(hcl.MergeFiles(hclFiles), hclparser.Opt{
LookupVar: os.LookupEnv,
Vars: defaults,
ValidateLabel: validateTargetName,
}, &c); err.HasErrors() {
}, &c)
if err.HasErrors() {
return nil, err
}
for _, renamed := range renamed {
for oldName, newNames := range renamed {
newNames = dedupSlice(newNames)
if len(newNames) == 1 && oldName == newNames[0] {
continue
}
c.Groups = append(c.Groups, &Group{
Name: oldName,
Targets: newNames,
})
}
}
c = dedupeConfig(c)
}
return &c, nil
@ -582,6 +599,11 @@ type Target struct {
linked bool
}
var _ hclparser.WithEvalContexts = &Target{}
var _ hclparser.WithGetName = &Target{}
var _ hclparser.WithEvalContexts = &Group{}
var _ hclparser.WithGetName = &Group{}
func (t *Target) normalize() {
t.Attest = removeDupes(t.Attest)
t.Tags = removeDupes(t.Tags)
@ -765,6 +787,114 @@ func (t *Target) AddOverrides(overrides map[string]Override) error {
return nil
}
func (g *Group) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) {
content, _, err := block.Body.PartialContent(&hcl.BodySchema{
Attributes: []hcl.AttributeSchema{{Name: "matrix"}},
})
if err != nil {
return nil, err
}
if _, ok := content.Attributes["matrix"]; ok {
return nil, errors.Errorf("matrix is not supported for groups")
}
return []*hcl.EvalContext{ectx}, nil
}
func (t *Target) GetEvalContexts(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error) {
content, _, err := block.Body.PartialContent(&hcl.BodySchema{
Attributes: []hcl.AttributeSchema{{Name: "matrix"}},
})
if err != nil {
return nil, err
}
attr, ok := content.Attributes["matrix"]
if !ok {
return []*hcl.EvalContext{ectx}, nil
}
if diags := loadDeps(attr.Expr); diags.HasErrors() {
return nil, diags
}
value, err := attr.Expr.Value(ectx)
if err != nil {
return nil, err
}
if !value.CanIterateElements() {
return nil, errors.Errorf("matrix must be a map")
}
matrix := value.AsValueMap()
ectxs := []*hcl.EvalContext{ectx}
for k, expr := range matrix {
if !expr.CanIterateElements() {
return nil, errors.Errorf("matrix values must be a list")
}
ectxs2 := []*hcl.EvalContext{}
for _, v := range expr.AsValueSlice() {
for _, e := range ectxs {
e2 := ectx.NewChild()
e2.Variables = make(map[string]cty.Value)
for k, v := range e.Variables {
e2.Variables[k] = v
}
e2.Variables[k] = v
ectxs2 = append(ectxs2, e2)
}
}
ectxs = ectxs2
}
return ectxs, nil
}
func (g *Group) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) {
content, _, diags := block.Body.PartialContent(&hcl.BodySchema{
Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}},
})
if diags != nil {
return "", diags
}
if _, ok := content.Attributes["name"]; ok {
return "", errors.Errorf("name is not supported for groups")
}
if _, ok := content.Attributes["matrix"]; ok {
return "", errors.Errorf("matrix is not supported for groups")
}
return block.Labels[0], nil
}
func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error) {
content, _, diags := block.Body.PartialContent(&hcl.BodySchema{
Attributes: []hcl.AttributeSchema{{Name: "name"}, {Name: "matrix"}},
})
if diags != nil {
return "", diags
}
attr, ok := content.Attributes["name"]
if !ok {
return block.Labels[0], nil
}
if _, ok := content.Attributes["matrix"]; !ok {
return "", errors.Errorf("name requires matrix")
}
if diags := loadDeps(attr.Expr); diags.HasErrors() {
return "", diags
}
value, diags := attr.Expr.Value(ectx)
if diags != nil {
return "", diags
}
value, err := convert.Convert(value, cty.String)
if err != nil {
return "", err
}
return value.AsString(), nil
}
func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m))
for k, v := range m {

@ -634,6 +634,444 @@ func TestHCLMultiFileAttrs(t *testing.T) {
require.Equal(t, ptrstr("pre-ghi"), c.Targets[0].Args["v1"])
}
func TestHCLDuplicateTarget(t *testing.T) {
dt := []byte(`
target "app" {
dockerfile = "x"
}
target "app" {
dockerfile = "y"
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, "app", c.Targets[0].Name)
require.Equal(t, "y", *c.Targets[0].Dockerfile)
}
func TestHCLRenameTarget(t *testing.T) {
dt := []byte(`
target "abc" {
name = "xyz"
dockerfile = "foo"
}
`)
_, err := ParseFile(dt, "docker-bake.hcl")
require.ErrorContains(t, err, "requires matrix")
}
func TestHCLRenameGroup(t *testing.T) {
dt := []byte(`
group "foo" {
name = "bar"
targets = ["x", "y"]
}
`)
_, err := ParseFile(dt, "docker-bake.hcl")
require.ErrorContains(t, err, "not supported")
dt = []byte(`
group "foo" {
matrix = {
name = ["x", "y"]
}
}
`)
_, err = ParseFile(dt, "docker-bake.hcl")
require.ErrorContains(t, err, "not supported")
}
func TestHCLRenameTargetAttrs(t *testing.T) {
dt := []byte(`
target "abc" {
name = "xyz"
matrix = {}
dockerfile = "foo"
}
target "def" {
dockerfile = target.xyz.dockerfile
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "xyz", c.Targets[0].Name)
require.Equal(t, "foo", *c.Targets[0].Dockerfile)
require.Equal(t, "def", c.Targets[1].Name)
require.Equal(t, "foo", *c.Targets[1].Dockerfile)
dt = []byte(`
target "def" {
dockerfile = target.xyz.dockerfile
}
target "abc" {
name = "xyz"
matrix = {}
dockerfile = "foo"
}
`)
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "def", c.Targets[0].Name)
require.Equal(t, "foo", *c.Targets[0].Dockerfile)
require.Equal(t, "xyz", c.Targets[1].Name)
require.Equal(t, "foo", *c.Targets[1].Dockerfile)
dt = []byte(`
target "abc" {
name = "xyz"
matrix = {}
dockerfile = "foo"
}
target "def" {
dockerfile = target.abc.dockerfile
}
`)
_, err = ParseFile(dt, "docker-bake.hcl")
require.ErrorContains(t, err, "abc")
dt = []byte(`
target "def" {
dockerfile = target.abc.dockerfile
}
target "abc" {
name = "xyz"
matrix = {}
dockerfile = "foo"
}
`)
_, err = ParseFile(dt, "docker-bake.hcl")
require.ErrorContains(t, err, "abc")
}
func TestHCLRenameSplit(t *testing.T) {
dt := []byte(`
target "x" {
name = "y"
matrix = {}
dockerfile = "foo"
}
target "x" {
name = "z"
matrix = {}
dockerfile = "bar"
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "y", c.Targets[0].Name)
require.Equal(t, "foo", *c.Targets[0].Dockerfile)
require.Equal(t, "z", c.Targets[1].Name)
require.Equal(t, "bar", *c.Targets[1].Dockerfile)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "x", c.Groups[0].Name)
require.Equal(t, []string{"y", "z"}, c.Groups[0].Targets)
}
func TestHCLRenameMultiFile(t *testing.T) {
dt := []byte(`
target "foo" {
name = "bar"
matrix = {}
dockerfile = "x"
}
`)
dt2 := []byte(`
target "foo" {
context = "y"
}
`)
dt3 := []byte(`
target "bar" {
target = "z"
}
`)
c, err := ParseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
{Data: dt3, Name: "c3.hcl"},
}, nil)
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "bar")
require.Equal(t, *c.Targets[0].Dockerfile, "x")
require.Equal(t, *c.Targets[0].Target, "z")
require.Equal(t, c.Targets[1].Name, "foo")
require.Equal(t, *c.Targets[1].Context, "y")
}
func TestHCLMatrixBasic(t *testing.T) {
dt := []byte(`
target "default" {
matrix = {
foo = ["x", "y"]
}
name = foo
dockerfile = "${foo}.Dockerfile"
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "x")
require.Equal(t, c.Targets[1].Name, "y")
require.Equal(t, *c.Targets[0].Dockerfile, "x.Dockerfile")
require.Equal(t, *c.Targets[1].Dockerfile, "y.Dockerfile")
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"x", "y"}, c.Groups[0].Targets)
}
func TestHCLMatrixMultipleKeys(t *testing.T) {
dt := []byte(`
target "default" {
matrix = {
foo = ["a"]
bar = ["b", "c"]
baz = ["d", "e", "f"]
}
name = "${foo}-${bar}-${baz}"
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 6, len(c.Targets))
names := make([]string, len(c.Targets))
for i, t := range c.Targets {
names[i] = t.Name
}
require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, names)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.ElementsMatch(t, []string{"a-b-d", "a-b-e", "a-b-f", "a-c-d", "a-c-e", "a-c-f"}, c.Groups[0].Targets)
}
func TestHCLMatrixLists(t *testing.T) {
dt := []byte(`
target "foo" {
matrix = {
aa = [["aa", "bb"], ["cc", "dd"]]
}
name = aa[0]
args = {
target = "val${aa[1]}"
}
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "aa", c.Targets[0].Name)
require.Equal(t, ptrstr("valbb"), c.Targets[0].Args["target"])
require.Equal(t, "cc", c.Targets[1].Name)
require.Equal(t, ptrstr("valdd"), c.Targets[1].Args["target"])
}
func TestHCLMatrixMaps(t *testing.T) {
dt := []byte(`
target "foo" {
matrix = {
aa = [
{
foo = "aa"
bar = "bb"
},
{
foo = "cc"
bar = "dd"
}
]
}
name = aa.foo
args = {
target = "val${aa.bar}"
}
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "aa")
require.Equal(t, c.Targets[0].Args["target"], ptrstr("valbb"))
require.Equal(t, c.Targets[1].Name, "cc")
require.Equal(t, c.Targets[1].Args["target"], ptrstr("valdd"))
}
func TestHCLMatrixMultipleTargets(t *testing.T) {
dt := []byte(`
target "x" {
matrix = {
foo = ["a", "b"]
}
name = foo
}
target "y" {
matrix = {
bar = ["c", "d"]
}
name = bar
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 4, len(c.Targets))
names := make([]string, len(c.Targets))
for i, t := range c.Targets {
names[i] = t.Name
}
require.ElementsMatch(t, []string{"a", "b", "c", "d"}, names)
require.Equal(t, 2, len(c.Groups))
names = make([]string, len(c.Groups))
for i, c := range c.Groups {
names[i] = c.Name
}
require.ElementsMatch(t, []string{"x", "y"}, names)
for _, g := range c.Groups {
switch g.Name {
case "x":
require.Equal(t, []string{"a", "b"}, g.Targets)
case "y":
require.Equal(t, []string{"c", "d"}, g.Targets)
}
}
}
func TestHCLMatrixDuplicateNames(t *testing.T) {
dt := []byte(`
target "default" {
matrix = {
foo = ["a", "b"]
}
name = "c"
}
`)
_, err := ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
}
func TestHCLMatrixArgs(t *testing.T) {
dt := []byte(`
a = 1
variable "b" {
default = 2
}
target "default" {
matrix = {
foo = [a, b]
}
name = foo
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 2, len(c.Targets))
require.Equal(t, "1", c.Targets[0].Name)
require.Equal(t, "2", c.Targets[1].Name)
}
func TestHCLMatrixArgsOverride(t *testing.T) {
dt := []byte(`
variable "ABC" {
default = "def"
}
target "bar" {
matrix = {
aa = split(",", ABC)
}
name = "bar-${aa}"
args = {
foo = aa
}
}
`)
c, err := ParseFiles([]File{
{Data: dt, Name: "docker-bake.hcl"},
}, map[string]string{"ABC": "11,22,33"})
require.NoError(t, err)
require.Equal(t, 3, len(c.Targets))
require.Equal(t, "bar-11", c.Targets[0].Name)
require.Equal(t, "bar-22", c.Targets[1].Name)
require.Equal(t, "bar-33", c.Targets[2].Name)
require.Equal(t, ptrstr("11"), c.Targets[0].Args["foo"])
require.Equal(t, ptrstr("22"), c.Targets[1].Args["foo"])
require.Equal(t, ptrstr("33"), c.Targets[2].Args["foo"])
}
func TestHCLMatrixBadTypes(t *testing.T) {
dt := []byte(`
target "default" {
matrix = "test"
}
`)
_, err := ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
dt = []byte(`
target "default" {
matrix = {
["a"] = ["b"]
}
}
`)
_, err = ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
dt = []byte(`
target "default" {
matrix = {
a = "b"
}
}
`)
_, err = ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
}
func TestJSONAttributes(t *testing.T) {
dt := []byte(`{"FOO": "abc", "variable": {"BAR": {"default": "def"}}, "target": { "app": { "args": {"v1": "pre-${FOO}-${BAR}"}} } }`)

@ -1,7 +1,9 @@
package hclparser
import (
"encoding/binary"
"fmt"
"hash/fnv"
"math"
"math/big"
"reflect"
@ -49,29 +51,38 @@ type parser struct {
attrs map[string]*hcl.Attribute
funcs map[string]*functionDef
blocks map[string]map[string][]*hcl.Block
blockValues map[*hcl.Block]reflect.Value
blockTypes map[string]reflect.Type
blocks map[string]map[string][]*hcl.Block
blockValues map[*hcl.Block][]reflect.Value
blockEvalCtx map[*hcl.Block][]*hcl.EvalContext
blockNames map[*hcl.Block][]string
blockTypes map[string]reflect.Type
ectx *hcl.EvalContext
progress map[string]struct{}
progressF map[string]struct{}
progressB map[*hcl.Block]map[string]struct{}
doneF map[string]struct{}
doneB map[*hcl.Block]map[string]struct{}
progressV map[uint64]struct{}
progressF map[uint64]struct{}
progressB map[uint64]map[string]struct{}
doneB map[uint64]map[string]struct{}
}
type WithEvalContexts interface {
GetEvalContexts(base *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) ([]*hcl.EvalContext, error)
}
type WithGetName interface {
GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(hcl.Expression) hcl.Diagnostics) (string, error)
}
var errUndefined = errors.New("undefined")
func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics {
func (p *parser) loadDeps(ectx *hcl.EvalContext, exp hcl.Expression, exclude map[string]struct{}, allowMissing bool) hcl.Diagnostics {
fns, hcldiags := funcCalls(exp)
if hcldiags.HasErrors() {
return hcldiags
}
for _, fn := range fns {
if err := p.resolveFunction(fn); err != nil {
if err := p.resolveFunction(ectx, fn); err != nil {
if allowMissing && errors.Is(err, errUndefined) {
continue
}
@ -124,14 +135,16 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allow
}
}
}
if err := p.resolveBlock(blocks[0], target); err != nil {
if allowMissing && errors.Is(err, errUndefined) {
continue
for _, block := range blocks {
if err := p.resolveBlock(block, target); err != nil {
if allowMissing && errors.Is(err, errUndefined) {
continue
}
return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr())
}
return wrapErrorDiagnostic("Invalid expression", err, exp.Range().Ptr(), exp.Range().Ptr())
}
} else {
if err := p.resolveValue(v.RootName()); err != nil {
if err := p.resolveValue(ectx, v.RootName()); err != nil {
if allowMissing && errors.Is(err, errUndefined) {
continue
}
@ -145,21 +158,21 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}, allow
// resolveFunction forces evaluation of a function, storing the result into the
// parser.
func (p *parser) resolveFunction(name string) error {
if _, ok := p.doneF[name]; ok {
func (p *parser) resolveFunction(ectx *hcl.EvalContext, name string) error {
if _, ok := p.ectx.Functions[name]; ok {
return nil
}
if _, ok := ectx.Functions[name]; ok {
return nil
}
f, ok := p.funcs[name]
if !ok {
if _, ok := p.ectx.Functions[name]; ok {
return nil
}
return errors.Wrapf(errUndefined, "function %q does not exit", name)
return errors.Wrapf(errUndefined, "function %q does not exist", name)
}
if _, ok := p.progressF[name]; ok {
if _, ok := p.progressF[key(ectx, name)]; ok {
return errors.Errorf("function cycle not allowed for %s", name)
}
p.progressF[name] = struct{}{}
p.progressF[key(ectx, name)] = struct{}{}
if f.Result == nil {
return errors.Errorf("empty result not allowed for %s", name)
@ -204,7 +217,7 @@ func (p *parser) resolveFunction(name string) error {
return diags
}
if diags := p.loadDeps(f.Result.Expr, params, false); diags.HasErrors() {
if diags := p.loadDeps(p.ectx, f.Result.Expr, params, false); diags.HasErrors() {
return diags
}
@ -214,7 +227,6 @@ func (p *parser) resolveFunction(name string) error {
if diags.HasErrors() {
return diags
}
p.doneF[name] = struct{}{}
p.ectx.Functions[name] = v
return nil
@ -222,14 +234,17 @@ func (p *parser) resolveFunction(name string) error {
// resolveValue forces evaluation of a named value, storing the result into the
// parser.
func (p *parser) resolveValue(name string) (err error) {
func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) {
if _, ok := p.ectx.Variables[name]; ok {
return nil
}
if _, ok := p.progress[name]; ok {
if _, ok := ectx.Variables[name]; ok {
return nil
}
if _, ok := p.progressV[key(ectx, name)]; ok {
return errors.Errorf("variable cycle not allowed for %s", name)
}
p.progress[name] = struct{}{}
p.progressV[key(ectx, name)] = struct{}{}
var v *cty.Value
defer func() {
@ -242,9 +257,10 @@ func (p *parser) resolveValue(name string) (err error) {
if _, builtin := p.opt.Vars[name]; !ok && !builtin {
vr, ok := p.vars[name]
if !ok {
return errors.Wrapf(errUndefined, "variable %q does not exit", name)
return errors.Wrapf(errUndefined, "variable %q does not exist", name)
}
def = vr.Default
ectx = p.ectx
}
if def == nil {
@ -257,10 +273,10 @@ func (p *parser) resolveValue(name string) (err error) {
return
}
if diags := p.loadDeps(def.Expr, nil, true); diags.HasErrors() {
if diags := p.loadDeps(ectx, def.Expr, nil, true); diags.HasErrors() {
return diags
}
vv, diags := def.Expr.Value(p.ectx)
vv, diags := def.Expr.Value(ectx)
if diags.HasErrors() {
return diags
}
@ -299,147 +315,226 @@ func (p *parser) resolveValue(name string) (err error) {
// target schema is provided, only the attributes and blocks present in the
// schema will be evaluated.
func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) {
name := block.Labels[0]
if err := p.opt.ValidateLabel(name); err != nil {
return wrapErrorDiagnostic("Invalid name", err, &block.LabelRanges[0], &block.LabelRanges[0])
// prepare the variable map for this type
if _, ok := p.ectx.Variables[block.Type]; !ok {
p.ectx.Variables[block.Type] = cty.MapValEmpty(cty.Map(cty.String))
}
if _, ok := p.doneB[block]; !ok {
p.doneB[block] = map[string]struct{}{}
}
if _, ok := p.progressB[block]; !ok {
p.progressB[block] = map[string]struct{}{}
// prepare the output destination and evaluation context
t, ok := p.blockTypes[block.Type]
if !ok {
return nil
}
if target != nil {
// filter out attributes and blocks that are already evaluated
original := target
target = &hcl.BodySchema{}
for _, a := range original.Attributes {
if _, ok := p.doneB[block][a.Name]; !ok {
target.Attributes = append(target.Attributes, a)
var outputs []reflect.Value
var ectxs []*hcl.EvalContext
if prev, ok := p.blockValues[block]; ok {
outputs = prev
ectxs = p.blockEvalCtx[block]
} else {
if v, ok := reflect.New(t).Interface().(WithEvalContexts); ok {
ectxs, err = v.GetEvalContexts(p.ectx, block, func(expr hcl.Expression) hcl.Diagnostics {
return p.loadDeps(p.ectx, expr, nil, true)
})
if err != nil {
return err
}
}
for _, b := range original.Blocks {
if _, ok := p.doneB[block][b.Type]; !ok {
target.Blocks = append(target.Blocks, b)
for _, ectx := range ectxs {
if ectx != p.ectx && ectx.Parent() != p.ectx {
return errors.Errorf("EvalContext must return a context with the correct parent")
}
}
} else {
ectxs = append([]*hcl.EvalContext{}, p.ectx)
}
if len(target.Attributes) == 0 && len(target.Blocks) == 0 {
return nil
for range ectxs {
outputs = append(outputs, reflect.New(t))
}
}
p.blockValues[block] = outputs
p.blockEvalCtx[block] = ectxs
if target != nil {
// detect reference cycles
for _, a := range target.Attributes {
if _, ok := p.progressB[block][a.Name]; ok {
return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name)
}
for i, output := range outputs {
target := target
ectx := ectxs[i]
name := block.Labels[0]
if names, ok := p.blockNames[block]; ok {
name = names[i]
}
for _, b := range target.Blocks {
if _, ok := p.progressB[block][b.Type]; ok {
return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type)
}
if _, ok := p.doneB[key(block, ectx)]; !ok {
p.doneB[key(block, ectx)] = map[string]struct{}{}
}
for _, a := range target.Attributes {
p.progressB[block][a.Name] = struct{}{}
if _, ok := p.progressB[key(block, ectx)]; !ok {
p.progressB[key(block, ectx)] = map[string]struct{}{}
}
for _, b := range target.Blocks {
p.progressB[block][b.Type] = struct{}{}
if target != nil {
// filter out attributes and blocks that are already evaluated
original := target
target = &hcl.BodySchema{}
for _, a := range original.Attributes {
if _, ok := p.doneB[key(block, ectx)][a.Name]; !ok {
target.Attributes = append(target.Attributes, a)
}
}
for _, b := range original.Blocks {
if _, ok := p.doneB[key(block, ectx)][b.Type]; !ok {
target.Blocks = append(target.Blocks, b)
}
}
if len(target.Attributes) == 0 && len(target.Blocks) == 0 {
return nil
}
}
}
// create a filtered body that contains only the target properties
body := func() hcl.Body {
if target != nil {
return FilterIncludeBody(block.Body, target)
// detect reference cycles
for _, a := range target.Attributes {
if _, ok := p.progressB[key(block, ectx)][a.Name]; ok {
return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name)
}
}
for _, b := range target.Blocks {
if _, ok := p.progressB[key(block, ectx)][b.Type]; ok {
return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type)
}
}
for _, a := range target.Attributes {
p.progressB[key(block, ectx)][a.Name] = struct{}{}
}
for _, b := range target.Blocks {
p.progressB[key(block, ectx)][b.Type] = struct{}{}
}
}
// create a filtered body that contains only the target properties
body := func() hcl.Body {
if target != nil {
return FilterIncludeBody(block.Body, target)
}
filter := &hcl.BodySchema{}
for k := range p.doneB[key(block, ectx)] {
filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k})
filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k})
}
return FilterExcludeBody(block.Body, filter)
}
filter := &hcl.BodySchema{}
for k := range p.doneB[block] {
filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k})
filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k})
// load dependencies from all targeted properties
schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface())
content, _, diag := body().PartialContent(schema)
if diag.HasErrors() {
return diag
}
for _, a := range content.Attributes {
diag := p.loadDeps(ectx, a.Expr, nil, true)
if diag.HasErrors() {
return diag
}
}
for _, b := range content.Blocks {
err := p.resolveBlock(b, nil)
if err != nil {
return err
}
}
return FilterExcludeBody(block.Body, filter)
}
// load dependencies from all targeted properties
t, ok := p.blockTypes[block.Type]
if !ok {
return nil
}
schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface())
content, _, diag := body().PartialContent(schema)
if diag.HasErrors() {
return diag
}
for _, a := range content.Attributes {
diag := p.loadDeps(a.Expr, nil, true)
// decode!
diag = gohcl.DecodeBody(body(), ectx, output.Interface())
if diag.HasErrors() {
return diag
}
}
for _, b := range content.Blocks {
err := p.resolveBlock(b, nil)
// mark all targeted properties as done
for _, a := range content.Attributes {
p.doneB[key(block, ectx)][a.Name] = struct{}{}
}
for _, b := range content.Blocks {
p.doneB[key(block, ectx)][b.Type] = struct{}{}
}
if target != nil {
for _, a := range target.Attributes {
p.doneB[key(block, ectx)][a.Name] = struct{}{}
}
for _, b := range target.Blocks {
p.doneB[key(block, ectx)][b.Type] = struct{}{}
}
}
// store the result into the evaluation context (so it can be referenced)
outputType, err := gocty.ImpliedType(output.Interface())
if err != nil {
return err
}
outputValue, err := gocty.ToCtyValue(output.Interface(), outputType)
if err != nil {
return err
}
var m map[string]cty.Value
if m2, ok := p.ectx.Variables[block.Type]; ok {
m = m2.AsValueMap()
}
if m == nil {
m = map[string]cty.Value{}
}
m[name] = outputValue
p.ectx.Variables[block.Type] = cty.MapVal(m)
}
// decode!
var output reflect.Value
if prev, ok := p.blockValues[block]; ok {
output = prev
} else {
output = reflect.New(t)
setLabel(output, block.Labels[0]) // early attach labels, so we can reference them
}
diag = gohcl.DecodeBody(body(), p.ectx, output.Interface())
if diag.HasErrors() {
return diag
}
p.blockValues[block] = output
return nil
}
// mark all targeted properties as done
for _, a := range content.Attributes {
p.doneB[block][a.Name] = struct{}{}
// resolveBlockNames returns the names of the block, calling resolveBlock to
// evaluate any label fields to correctly resolve the name.
func (p *parser) resolveBlockNames(block *hcl.Block) ([]string, error) {
if names, ok := p.blockNames[block]; ok {
return names, nil
}
for _, b := range content.Blocks {
p.doneB[block][b.Type] = struct{}{}
if err := p.resolveBlock(block, &hcl.BodySchema{}); err != nil {
return nil, err
}
if target != nil {
for _, a := range target.Attributes {
p.doneB[block][a.Name] = struct{}{}
names := make([]string, 0, len(p.blockValues[block]))
for i, val := range p.blockValues[block] {
ectx := p.blockEvalCtx[block][i]
name := block.Labels[0]
if err := p.opt.ValidateLabel(name); err != nil {
return nil, err
}
for _, b := range target.Blocks {
p.doneB[block][b.Type] = struct{}{}
if v, ok := val.Interface().(WithGetName); ok {
var err error
name, err = v.GetName(ectx, block, func(expr hcl.Expression) hcl.Diagnostics {
return p.loadDeps(ectx, expr, nil, true)
})
if err != nil {
return nil, err
}
if err := p.opt.ValidateLabel(name); err != nil {
return nil, err
}
}
}
// store the result into the evaluation context (so if can be referenced)
outputType, err := gocty.ImpliedType(output.Interface())
if err != nil {
return err
}
outputValue, err := gocty.ToCtyValue(output.Interface(), outputType)
if err != nil {
return err
setName(val, name)
names = append(names, name)
}
var m map[string]cty.Value
if m2, ok := p.ectx.Variables[block.Type]; ok {
m = m2.AsValueMap()
}
if m == nil {
m = map[string]cty.Value{}
found := map[string]struct{}{}
for _, name := range names {
if _, ok := found[name]; ok {
return nil, errors.Errorf("duplicate name %q", name)
}
found[name] = struct{}{}
}
m[name] = outputValue
p.ectx.Variables[block.Type] = cty.MapVal(m)
return nil
p.blockNames[block] = names
return names, nil
}
func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
func Parse(b hcl.Body, opt Opt, val interface{}) (map[string]map[string][]string, hcl.Diagnostics) {
reserved := map[string]struct{}{}
schema, _ := gohcl.ImpliedBodySchema(val)
@ -452,7 +547,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
var defs inputs
if err := gohcl.DecodeBody(b, nil, &defs); err != nil {
return err
return nil, err
}
defsSchema, _ := gohcl.ImpliedBodySchema(defs)
@ -475,20 +570,20 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
attrs: map[string]*hcl.Attribute{},
funcs: map[string]*functionDef{},
blocks: map[string]map[string][]*hcl.Block{},
blockValues: map[*hcl.Block]reflect.Value{},
blockTypes: map[string]reflect.Type{},
progress: map[string]struct{}{},
progressF: map[string]struct{}{},
progressB: map[*hcl.Block]map[string]struct{}{},
doneF: map[string]struct{}{},
doneB: map[*hcl.Block]map[string]struct{}{},
blocks: map[string]map[string][]*hcl.Block{},
blockValues: map[*hcl.Block][]reflect.Value{},
blockEvalCtx: map[*hcl.Block][]*hcl.EvalContext{},
blockNames: map[*hcl.Block][]string{},
blockTypes: map[string]reflect.Type{},
ectx: &hcl.EvalContext{
Variables: map[string]cty.Value{},
Functions: stdlibFunctions,
Functions: Stdlib(),
},
progressV: map[uint64]struct{}{},
progressF: map[uint64]struct{}{},
progressB: map[uint64]map[string]struct{}{},
doneB: map[uint64]map[string]struct{}{},
}
for _, v := range defs.Variables {
@ -508,18 +603,18 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
content, b, diags := b.PartialContent(schema)
if diags.HasErrors() {
return diags
return nil, diags
}
blocks, b, diags := b.PartialContent(defsSchema)
if diags.HasErrors() {
return diags
return nil, diags
}
attrs, diags := b.JustAttributes()
if diags.HasErrors() {
if d := removeAttributesDiags(diags, reserved, p.vars); len(d) > 0 {
return d
return nil, d
}
}
@ -532,11 +627,11 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
delete(p.attrs, "function")
for k := range p.opt.Vars {
_ = p.resolveValue(k)
_ = p.resolveValue(p.ectx, k)
}
for _, a := range content.Attributes {
return hcl.Diagnostics{
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid attribute",
@ -548,19 +643,19 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
}
for k := range p.vars {
if err := p.resolveValue(k); err != nil {
if err := p.resolveValue(p.ectx, k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
return nil, diags
}
r := p.vars[k].Body.MissingItemRange()
return wrapErrorDiagnostic("Invalid value", err, &r, &r)
return nil, wrapErrorDiagnostic("Invalid value", err, &r, &r)
}
}
for k := range p.funcs {
if err := p.resolveFunction(k); err != nil {
if err := p.resolveFunction(p.ectx, k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
return nil, diags
}
var subject *hcl.Range
var context *hcl.Range
@ -576,32 +671,10 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
}
}
}
return wrapErrorDiagnostic("Invalid function", err, subject, context)
return nil, wrapErrorDiagnostic("Invalid function", err, subject, context)
}
}
for _, b := range content.Blocks {
if len(b.Labels) == 0 || len(b.Labels) > 1 {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid block",
Detail: fmt.Sprintf("invalid block label: %v", b.Labels),
Subject: &b.LabelRanges[0],
Context: &b.LabelRanges[0],
},
}
}
bm, ok := p.blocks[b.Type]
if !ok {
bm = map[string][]*hcl.Block{}
p.blocks[b.Type] = bm
}
lbl := b.Labels[0]
bm[lbl] = append(bm[lbl], b)
}
type value struct {
reflect.Value
idx int
@ -612,7 +685,7 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
values map[string]value
}
types := map[string]field{}
renamed := map[string]map[string][]string{}
vt := reflect.ValueOf(val).Elem().Type()
for i := 0; i < vt.NumField(); i++ {
tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",")
@ -623,7 +696,39 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
typ: vt.Field(i).Type,
values: make(map[string]value),
}
renamed[tags[0]] = map[string][]string{}
}
tmpBlocks := map[string]map[string][]*hcl.Block{}
for _, b := range content.Blocks {
if len(b.Labels) == 0 || len(b.Labels) > 1 {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid block",
Detail: fmt.Sprintf("invalid block label: %v", b.Labels),
Subject: &b.LabelRanges[0],
Context: &b.LabelRanges[0],
},
}
}
bm, ok := tmpBlocks[b.Type]
if !ok {
bm = map[string][]*hcl.Block{}
tmpBlocks[b.Type] = bm
}
names, err := p.resolveBlockNames(b)
if err != nil {
return nil, wrapErrorDiagnostic("Invalid name", err, &b.LabelRanges[0], &b.LabelRanges[0])
}
for _, name := range names {
bm[name] = append(bm[name], b)
renamed[b.Type][b.Labels[0]] = append(renamed[b.Type][b.Labels[0]], name)
}
}
p.blocks = tmpBlocks
diags = hcl.Diagnostics{}
for _, b := range content.Blocks {
@ -637,56 +742,57 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
continue
}
} else {
return wrapErrorDiagnostic("Invalid block", err, &b.LabelRanges[0], &b.DefRange)
return nil, wrapErrorDiagnostic("Invalid block", err, &b.LabelRanges[0], &b.DefRange)
}
}
vv := p.blockValues[b]
t := types[b.Type]
lblIndex := setLabel(vv, b.Labels[0])
oldValue, exists := t.values[b.Labels[0]]
if !exists && lblIndex != -1 {
if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice {
for i := 0; i < v.Elem().Field(t.idx).Len(); i++ {
if b.Labels[0] == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() {
exists = true
oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i}
break
vvs := p.blockValues[b]
for _, vv := range vvs {
t := types[b.Type]
lblIndex, lblExists := getNameIndex(vv)
lblName, _ := getName(vv)
oldValue, exists := t.values[lblName]
if !exists && lblExists {
if v.Elem().Field(t.idx).Type().Kind() == reflect.Slice {
for i := 0; i < v.Elem().Field(t.idx).Len(); i++ {
if lblName == v.Elem().Field(t.idx).Index(i).Elem().Field(lblIndex).String() {
exists = true
oldValue = value{Value: v.Elem().Field(t.idx).Index(i), idx: i}
break
}
}
}
}
}
if exists {
if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
m.Call([]reflect.Value{vv})
if exists {
if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
m.Call([]reflect.Value{vv})
} else {
v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv)
}
} else {
v.Elem().Field(t.idx).Index(oldValue.idx).Set(vv)
}
} else {
slice := v.Elem().Field(t.idx)
if slice.IsNil() {
slice = reflect.New(t.typ).Elem()
slice := v.Elem().Field(t.idx)
if slice.IsNil() {
slice = reflect.New(t.typ).Elem()
}
t.values[lblName] = value{Value: vv, idx: slice.Len()}
v.Elem().Field(t.idx).Set(reflect.Append(slice, vv))
}
t.values[b.Labels[0]] = value{Value: vv, idx: slice.Len()}
v.Elem().Field(t.idx).Set(reflect.Append(slice, vv))
}
}
if diags.HasErrors() {
return diags
return nil, diags
}
for k := range p.attrs {
if err := p.resolveValue(k); err != nil {
if err := p.resolveValue(p.ectx, k); err != nil {
if diags, ok := err.(hcl.Diagnostics); ok {
return diags
return nil, diags
}
return wrapErrorDiagnostic("Invalid attribute", err, &p.attrs[k].Range, &p.attrs[k].Range)
return nil, wrapErrorDiagnostic("Invalid attribute", err, &p.attrs[k].Range, &p.attrs[k].Range)
}
}
return nil
return renamed, nil
}
// wrapErrorDiagnostic wraps an error into a hcl.Diagnostics object.
@ -710,18 +816,42 @@ func wrapErrorDiagnostic(message string, err error, subject *hcl.Range, context
}
}
func setLabel(v reflect.Value, lbl string) int {
// cache field index?
func setName(v reflect.Value, name string) {
numFields := v.Elem().Type().NumField()
for i := 0; i < numFields; i++ {
parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",")
for _, t := range parts[1:] {
if t == "label" {
v.Elem().Field(i).Set(reflect.ValueOf(name))
}
}
}
}
func getName(v reflect.Value) (string, bool) {
numFields := v.Elem().Type().NumField()
for i := 0; i < numFields; i++ {
parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",")
for _, t := range parts[1:] {
if t == "label" {
return v.Elem().Field(i).String(), true
}
}
}
return "", false
}
func getNameIndex(v reflect.Value) (int, bool) {
numFields := v.Elem().Type().NumField()
for i := 0; i < numFields; i++ {
for _, t := range strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",") {
parts := strings.Split(v.Elem().Type().Field(i).Tag.Get("hcl"), ",")
for _, t := range parts[1:] {
if t == "label" {
v.Elem().Field(i).Set(reflect.ValueOf(lbl))
return i
return i, true
}
}
}
return -1
return 0, false
}
func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{}, vars map[string]*variable) hcl.Diagnostics {
@ -753,3 +883,21 @@ func removeAttributesDiags(diags hcl.Diagnostics, reserved map[string]struct{},
}
return fdiags
}
// key returns a unique hash for the given values
func key(ks ...any) uint64 {
hash := fnv.New64a()
for _, k := range ks {
v := reflect.ValueOf(k)
switch v.Kind() {
case reflect.String:
hash.Write([]byte(v.String()))
case reflect.Pointer:
ptr := reflect.ValueOf(k).Pointer()
binary.Write(hash, binary.LittleEndian, uint64(ptr))
default:
panic(fmt.Sprintf("unknown key kind %s", v.Kind().String()))
}
}
return hash.Sum64()
}

@ -124,3 +124,11 @@ var timestampFunc = function.New(&function.Spec{
return cty.StringVal(time.Now().UTC().Format(time.RFC3339)), nil
},
})
func Stdlib() map[string]function.Function {
funcs := make(map[string]function.Function, len(stdlibFunctions))
for k, v := range stdlibFunctions {
funcs[k] = v
}
return funcs
}

Loading…
Cancel
Save