bake: support block-based interpolation

This patch adds support for block-based interpolation, so that
properties of blocks can be referenced in the current block and across
other blocks.

Previously, order-of-evaluation did not matter for blocks, and could be
evaluated in any order. However, now that blocks can refer to each
other, we split out this dynamic evaluation order into a separate
resolveBlock function.

Additionally, we need to support partial block evaluations - if block A
refers to property X of block B, when we should only evaluate property
X, and not the entire block. This ensures that we can safely evaluate
blocks that refer to other properties within themselves, and allows
sequences that would otherwise be co-recursive. We take special care in
this logic to ensure that each property is evaluated once *and only*
once - this could otherwise present inconsistencies with stateful
functions, and could risk inconsistent results.

Signed-off-by: Justin Chadwell <me@jedevc.com>

bake/hclparser/body.go

@@ -0,0 +1,103 @@
+package hclparser
+
+import (
+	"github.com/hashicorp/hcl/v2"
+)
+
+type filterBody struct {
+	body    hcl.Body
+	schema  *hcl.BodySchema
+	exclude bool
+}
+
+func FilterIncludeBody(body hcl.Body, schema *hcl.BodySchema) hcl.Body {
+	return &filterBody{
+		body:   body,
+		schema: schema,
+	}
+}
+
+func FilterExcludeBody(body hcl.Body, schema *hcl.BodySchema) hcl.Body {
+	return &filterBody{
+		body:    body,
+		schema:  schema,
+		exclude: true,
+	}
+}
+
+func (b *filterBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) {
+	if b.exclude {
+		schema = subtractSchemas(schema, b.schema)
+	} else {
+		schema = intersectSchemas(schema, b.schema)
+	}
+	content, _, diag := b.body.PartialContent(schema)
+	return content, diag
+}
+
+func (b *filterBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) {
+	if b.exclude {
+		schema = subtractSchemas(schema, b.schema)
+	} else {
+		schema = intersectSchemas(schema, b.schema)
+	}
+	return b.body.PartialContent(schema)
+}
+
+func (b *filterBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) {
+	return b.body.JustAttributes()
+}
+
+func (b *filterBody) MissingItemRange() hcl.Range {
+	return b.body.MissingItemRange()
+}
+
+func intersectSchemas(a, b *hcl.BodySchema) *hcl.BodySchema {
+	result := &hcl.BodySchema{}
+	for _, blockA := range a.Blocks {
+		for _, blockB := range b.Blocks {
+			if blockA.Type == blockB.Type {
+				result.Blocks = append(result.Blocks, blockA)
+				break
+			}
+		}
+	}
+	for _, attrA := range a.Attributes {
+		for _, attrB := range b.Attributes {
+			if attrA.Name == attrB.Name {
+				result.Attributes = append(result.Attributes, attrA)
+				break
+			}
+		}
+	}
+	return result
+}
+
+func subtractSchemas(a, b *hcl.BodySchema) *hcl.BodySchema {
+	result := &hcl.BodySchema{}
+	for _, blockA := range a.Blocks {
+		found := false
+		for _, blockB := range b.Blocks {
+			if blockA.Type == blockB.Type {
+				found = true
+				break
+			}
+		}
+		if !found {
+			result.Blocks = append(result.Blocks, blockA)
+		}
+	}
+	for _, attrA := range a.Attributes {
+		found := false
+		for _, attrB := range b.Attributes {
+			if attrA.Name == attrB.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			result.Attributes = append(result.Attributes, attrA)
+		}
+	}
+	return result
+}

bake/hclparser/hclparser.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hashicorp/hcl/v2/gohcl"
 	"github.com/pkg/errors"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/gocty"
 )
 
 type Opt struct {
@@ -48,11 +49,17 @@ type parser struct {
 	attrs map[string]*hcl.Attribute
 	funcs map[string]*functionDef
 
+	blocks      map[string]map[string][]*hcl.Block
+	blockValues map[*hcl.Block]reflect.Value
+	blockTypes  map[string]reflect.Type
+
 	ectx *hcl.EvalContext
 
 	progress  map[string]struct{}
 	progressF map[string]struct{}
+	progressB map[*hcl.Block]map[string]struct{}
 	doneF     map[string]struct{}
+	doneB     map[*hcl.Block]map[string]struct{}
 }
 
 func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.Diagnostics {
@@ -79,15 +86,69 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.D
 		if _, ok := exclude[v.RootName()]; ok {
 			continue
 		}
-		if err := p.resolveValue(v.RootName()); err != nil {
+		if _, ok := p.blockTypes[v.RootName()]; ok {
-			return hcl.Diagnostics{
+			blockType := v.RootName()
-				&hcl.Diagnostic{
+
-					Severity: hcl.DiagError,
+			split := v.SimpleSplit().Rel
-					Summary:  "Invalid expression",
+			if len(split) == 0 {
-					Detail:   err.Error(),
+				return hcl.Diagnostics{
-					Subject:  v.SourceRange().Ptr(),
+					&hcl.Diagnostic{
-					Context:  v.SourceRange().Ptr(),
+						Severity: hcl.DiagError,
-				},
+						Summary:  "Invalid expression",
+						Detail:   fmt.Sprintf("cannot access %s as a variable", blockType),
+						Subject:  exp.Range().Ptr(),
+						Context:  exp.Range().Ptr(),
+					},
+				}
+			}
+			blockName, ok := split[0].(hcl.TraverseAttr)
+			if !ok {
+				return hcl.Diagnostics{
+					&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Invalid expression",
+						Detail:   fmt.Sprintf("cannot traverse %s without attribute", blockType),
+						Subject:  exp.Range().Ptr(),
+						Context:  exp.Range().Ptr(),
+					},
+				}
+			}
+			blocks := p.blocks[blockType][blockName.Name]
+			if len(blocks) == 0 {
+				continue
+			}
+
+			var target *hcl.BodySchema
+			if len(split) > 1 {
+				if attr, ok := split[1].(hcl.TraverseAttr); ok {
+					target = &hcl.BodySchema{
+						Attributes: []hcl.AttributeSchema{{Name: attr.Name}},
+						Blocks:     []hcl.BlockHeaderSchema{{Type: attr.Name}},
+					}
+				}
+			}
+			if err := p.resolveBlock(blocks[0], target); err != nil {
+				return hcl.Diagnostics{
+					&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Invalid expression",
+						Detail:   err.Error(),
+						Subject:  v.SourceRange().Ptr(),
+						Context:  v.SourceRange().Ptr(),
+					},
+				}
+			}
+		} else {
+			if err := p.resolveValue(v.RootName()); err != nil {
+				return hcl.Diagnostics{
+					&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Invalid expression",
+						Detail:   err.Error(),
+						Subject:  v.SourceRange().Ptr(),
+						Context:  v.SourceRange().Ptr(),
+					},
+				}
 			}
 		}
 	}
@@ -95,6 +156,8 @@ func (p *parser) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.D
 	return nil
 }
 
+// resolveFunction forces evaluation of a function, storing the result into the
+// parser.
 func (p *parser) resolveFunction(name string) error {
 	if _, ok := p.doneF[name]; ok {
 		return nil
@@ -170,6 +233,8 @@ func (p *parser) resolveFunction(name string) error {
 	return nil
 }
 
+// resolveValue forces evaluation of a named value, storing the result into the
+// parser.
 func (p *parser) resolveValue(name string) (err error) {
 	if _, ok := p.ectx.Variables[name]; ok {
 		return nil
@@ -248,6 +313,157 @@ func (p *parser) resolveValue(name string) (err error) {
 	return nil
 }
 
+// resolveBlock force evaluates a block, storing the result in the parser. If a
+// target schema is provided, only the attributes and blocks present in the
+// schema will be evaluated.
+func (p *parser) resolveBlock(block *hcl.Block, target *hcl.BodySchema) (err error) {
+	name := block.Labels[0]
+	if err := p.opt.ValidateLabel(name); err != nil {
+		return hcl.Diagnostics{
+			&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid name",
+				Detail:   err.Error(),
+				Subject:  &block.LabelRanges[0],
+			},
+		}
+	}
+
+	if _, ok := p.doneB[block]; !ok {
+		p.doneB[block] = map[string]struct{}{}
+	}
+	if _, ok := p.progressB[block]; !ok {
+		p.progressB[block] = map[string]struct{}{}
+	}
+
+	if target != nil {
+		// filter out attributes and blocks that are already evaluated
+		original := target
+		target = &hcl.BodySchema{}
+		for _, a := range original.Attributes {
+			if _, ok := p.doneB[block][a.Name]; !ok {
+				target.Attributes = append(target.Attributes, a)
+			}
+		}
+		for _, b := range original.Blocks {
+			if _, ok := p.doneB[block][b.Type]; !ok {
+				target.Blocks = append(target.Blocks, b)
+			}
+		}
+		if len(target.Attributes) == 0 && len(target.Blocks) == 0 {
+			return nil
+		}
+	}
+
+	if target != nil {
+		// detect reference cycles
+		for _, a := range target.Attributes {
+			if _, ok := p.progressB[block][a.Name]; ok {
+				return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, a.Name)
+			}
+		}
+		for _, b := range target.Blocks {
+			if _, ok := p.progressB[block][b.Type]; ok {
+				return errors.Errorf("reference cycle not allowed for %s.%s.%s", block.Type, name, b.Type)
+			}
+		}
+		for _, a := range target.Attributes {
+			p.progressB[block][a.Name] = struct{}{}
+		}
+		for _, b := range target.Blocks {
+			p.progressB[block][b.Type] = struct{}{}
+		}
+	}
+
+	// create a filtered body that contains only the target properties
+	body := func() hcl.Body {
+		if target != nil {
+			return FilterIncludeBody(block.Body, target)
+		}
+
+		filter := &hcl.BodySchema{}
+		for k := range p.doneB[block] {
+			filter.Attributes = append(filter.Attributes, hcl.AttributeSchema{Name: k})
+			filter.Blocks = append(filter.Blocks, hcl.BlockHeaderSchema{Type: k})
+		}
+		return FilterExcludeBody(block.Body, filter)
+	}
+
+	// load dependencies from all targeted properties
+	t, ok := p.blockTypes[block.Type]
+	if !ok {
+		return nil
+	}
+	schema, _ := gohcl.ImpliedBodySchema(reflect.New(t).Interface())
+	content, _, diag := body().PartialContent(schema)
+	if diag.HasErrors() {
+		return diag
+	}
+	for _, a := range content.Attributes {
+		diag := p.loadDeps(a.Expr, nil)
+		if diag.HasErrors() {
+			return diag
+		}
+	}
+	for _, b := range content.Blocks {
+		err := p.resolveBlock(b, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	// decode!
+	var output reflect.Value
+	if prev, ok := p.blockValues[block]; ok {
+		output = prev
+	} else {
+		output = reflect.New(t)
+		setLabel(output, block.Labels[0]) // early attach labels, so we can reference them
+	}
+	diag = gohcl.DecodeBody(body(), p.ectx, output.Interface())
+	if diag.HasErrors() {
+		return diag
+	}
+	p.blockValues[block] = output
+
+	// mark all targeted properties as done
+	for _, a := range content.Attributes {
+		p.doneB[block][a.Name] = struct{}{}
+	}
+	for _, b := range content.Blocks {
+		p.doneB[block][b.Type] = struct{}{}
+	}
+	if target != nil {
+		for _, a := range target.Attributes {
+			p.doneB[block][a.Name] = struct{}{}
+		}
+		for _, b := range target.Blocks {
+			p.doneB[block][b.Type] = struct{}{}
+		}
+	}
+
+	// store the result into the evaluation context (so if can be referenced)
+	outputType, err := gocty.ImpliedType(output.Interface())
+	if err != nil {
+		return err
+	}
+	outputValue, err := gocty.ToCtyValue(output.Interface(), outputType)
+	if err != nil {
+		return err
+	}
+	var m map[string]cty.Value
+	if m2, ok := p.ectx.Variables[block.Type]; ok {
+		m = m2.AsValueMap()
+	}
+	if m == nil {
+		m = map[string]cty.Value{}
+	}
+	m[name] = outputValue
+	p.ectx.Variables[block.Type] = cty.MapVal(m)
+
+	return nil
+}
+
 func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	reserved := map[string]struct{}{}
 	schema, _ := gohcl.ImpliedBodySchema(val)
@@ -284,9 +500,16 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		attrs: map[string]*hcl.Attribute{},
 		funcs: map[string]*functionDef{},
 
+		blocks:      map[string]map[string][]*hcl.Block{},
+		blockValues: map[*hcl.Block]reflect.Value{},
+		blockTypes:  map[string]reflect.Type{},
+
 		progress:  map[string]struct{}{},
 		progressF: map[string]struct{}{},
-		doneF:     map[string]struct{}{},
+		progressB: map[*hcl.Block]map[string]struct{}{},
+
+		doneF: map[string]struct{}{},
+		doneB: map[*hcl.Block]map[string]struct{}{},
 		ectx: &hcl.EvalContext{
 			Variables: map[string]cty.Value{},
 			Functions: stdlibFunctions,
@@ -337,20 +560,15 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		_ = p.resolveValue(k)
 	}
 
-	for k := range p.attrs {
+	for _, a := range content.Attributes {
-		if err := p.resolveValue(k); err != nil {
+		return hcl.Diagnostics{
-			if diags, ok := err.(hcl.Diagnostics); ok {
+			&hcl.Diagnostic{
-				return diags
+				Severity: hcl.DiagError,
-			}
+				Summary:  "Invalid attribute",
-			return hcl.Diagnostics{
+				Detail:   "global attributes currently not supported",
-				&hcl.Diagnostic{
+				Subject:  &a.Range,
-					Severity: hcl.DiagError,
+				Context:  &a.Range,
-					Summary:  "Invalid attribute",
+			},
-					Detail:   err.Error(),
-					Subject:  &p.attrs[k].Range,
-					Context:  &p.attrs[k].Range,
-				},
-			}
 		}
 	}
 
@@ -403,19 +621,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		}
 	}
 
-	for _, a := range content.Attributes {
-		return hcl.Diagnostics{
-			&hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Invalid attribute",
-				Detail:   "global attributes currently not supported",
-				Subject:  &a.Range,
-				Context:  &a.Range,
-			},
-		}
-	}
-
-	m := map[string]map[string][]*hcl.Block{}
 	for _, b := range content.Blocks {
 		if len(b.Labels) == 0 || len(b.Labels) > 1 {
 			return hcl.Diagnostics{
@@ -428,19 +633,16 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 				},
 			}
 		}
-		bm, ok := m[b.Type]
+		bm, ok := p.blocks[b.Type]
 		if !ok {
 			bm = map[string][]*hcl.Block{}
-			m[b.Type] = bm
+			p.blocks[b.Type] = bm
 		}
 
 		lbl := b.Labels[0]
 		bm[lbl] = append(bm[lbl], b)
 	}
 
-	vt := reflect.ValueOf(val).Elem().Type()
-	numFields := vt.NumField()
-
 	type value struct {
 		reflect.Value
 		idx int
@@ -452,9 +654,11 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	}
 	types := map[string]field{}
 
-	for i := 0; i < numFields; i++ {
+	vt := reflect.ValueOf(val).Elem().Type()
+	for i := 0; i < vt.NumField(); i++ {
 		tags := strings.Split(vt.Field(i).Tag.Get("hcl"), ",")
 
+		p.blockTypes[tags[0]] = vt.Field(i).Type.Elem().Elem()
 		types[tags[0]] = field{
 			idx:    i,
 			typ:    vt.Field(i).Type,
@@ -466,29 +670,29 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 	for _, b := range content.Blocks {
 		v := reflect.ValueOf(val)
 
-		t, ok := types[b.Type]
+		err := p.resolveBlock(b, nil)
-		if !ok {
+		if err != nil {
-			continue
+			if diag, ok := err.(hcl.Diagnostics); ok {
-		}
+				if diag.HasErrors() {
-
+					diags = append(diags, diag...)
-		vv := reflect.New(t.typ.Elem().Elem())
+					continue
-		diag := gohcl.DecodeBody(b.Body, p.ectx, vv.Interface())
+				}
-		if diag.HasErrors() {
+			} else {
-			diags = append(diags, diag...)
+				return hcl.Diagnostics{
-			continue
+					&hcl.Diagnostic{
-		}
+						Severity: hcl.DiagError,
-
+						Summary:  "Invalid attribute",
-		if err := opt.ValidateLabel(b.Labels[0]); err != nil {
+						Detail:   err.Error(),
-			return hcl.Diagnostics{
+						Subject:  &b.LabelRanges[0],
-				&hcl.Diagnostic{
+						Context:  &b.DefRange,
-					Severity: hcl.DiagError,
+					},
-					Summary:  "Invalid name",
+				}
-					Detail:   err.Error(),
-					Subject:  &b.LabelRanges[0],
-				},
 			}
 		}
 
+		vv := p.blockValues[b]
+
+		t := types[b.Type]
 		lblIndex := setLabel(vv, b.Labels[0])
 
 		oldValue, exists := t.values[b.Labels[0]]
@@ -502,7 +706,6 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 					}
 				}
 			}
-
 		}
 		if exists {
 			if m := oldValue.Value.MethodByName("Merge"); m.IsValid() {
@@ -523,6 +726,23 @@ func Parse(b hcl.Body, opt Opt, val interface{}) hcl.Diagnostics {
 		return diags
 	}
 
+	for k := range p.attrs {
+		if err := p.resolveValue(k); err != nil {
+			if diags, ok := err.(hcl.Diagnostics); ok {
+				return diags
+			}
+			return hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid attribute",
+					Detail:   err.Error(),
+					Subject:  &p.attrs[k].Range,
+					Context:  &p.attrs[k].Range,
+				},
+			}
+		}
+	}
+
 	return nil
 }