bake: allow variables across files

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
pull/538/head
Tonis Tiigi 4 years ago
parent 927163bf13
commit 0e9d6460db

@ -59,14 +59,11 @@ func ReadLocalFiles(names []string) ([]File, error) {
}
func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
var c Config
for _, f := range files {
cfg, err := ParseFile(f.Data, f.Name)
if err != nil {
return nil, err
}
c = mergeConfig(c, *cfg)
c, err := parseFiles(files)
if err != nil {
return nil, err
}
o, err := c.newOverrides(overrides)
if err != nil {
return nil, err
@ -86,25 +83,73 @@ func ReadTargets(ctx context.Context, files []File, targets, overrides []string)
return m, nil
}
func parseFiles(files []File) (*Config, error) {
var c Config
var fs []*hcl.File
var scs []*StaticConfig
for _, f := range files {
cfg, f, sc, err := parseFile(f.Data, f.Name)
if err != nil {
return nil, err
}
if cfg != nil {
c = mergeConfig(c, *cfg)
} else {
fs = append(fs, f)
scs = append(scs, sc)
}
}
if len(fs) > 0 {
cfg, err := ParseHCL(hcl.MergeFiles(fs), mergeStaticConfig(scs))
if err != nil {
return nil, err
}
c = mergeConfig(c, dedupeConfig(*cfg))
}
return &c, nil
}
func dedupeConfig(c Config) Config {
c2 := c
c2.Targets = make([]*Target, 0, len(c2.Targets))
m := map[string]*Target{}
for _, t := range c.Targets {
if t2, ok := m[t.Name]; ok {
merge(t2, t)
} else {
m[t.Name] = t
c2.Targets = append(c2.Targets, t)
}
}
return c2
}
func ParseFile(dt []byte, fn string) (*Config, error) {
return parseFiles([]File{{Data: dt, Name: fn}})
}
func parseFile(dt []byte, fn string) (*Config, *hcl.File, *StaticConfig, error) {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
return ParseCompose(dt)
c, err := ParseCompose(dt)
return c, nil, nil, err
}
if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
return ParseHCL(dt, fn)
f, sc, err := ParseHCLFile(dt, fn)
return nil, f, sc, err
}
cfg, err := ParseCompose(dt)
if err != nil {
cfg, err2 := ParseHCL(dt, fn)
f, sc, err2 := ParseHCLFile(dt, fn)
if err2 != nil {
return nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
return nil, nil, nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
}
return cfg, nil
return nil, f, sc, nil
}
return cfg, nil
return cfg, nil, nil, nil
}
type Config struct {

@ -12,9 +12,9 @@ import (
"github.com/hashicorp/hcl/v2/ext/tryfunc"
"github.com/hashicorp/hcl/v2/ext/typeexpr"
"github.com/hashicorp/hcl/v2/ext/userfunc"
"github.com/hashicorp/hcl/v2/hclsimple"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/hashicorp/hcl/v2/json"
hcljson "github.com/hashicorp/hcl/v2/json"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
"github.com/zclconf/go-cty/cty"
@ -127,59 +127,74 @@ var (
// Used in the first pass of decoding instead of the Config struct to disallow
// interpolation while parsing variable blocks.
type staticConfig struct {
type StaticConfig struct {
Variables []*Variable `hcl:"variable,block"`
Remain hcl.Body `hcl:",remain"`
}
func ParseHCL(dt []byte, fn string) (_ *Config, err error) {
func mergeStaticConfig(scs []*StaticConfig) *StaticConfig {
if len(scs) == 0 {
return nil
}
sc := scs[0]
for _, s := range scs[1:] {
sc.Variables = append(sc.Variables, s.Variables...)
}
return sc
}
func ParseHCLFile(dt []byte, fn string) (*hcl.File, *StaticConfig, error) {
if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
return parseHCL(dt, fn)
return parseHCLFile(dt, fn)
}
cfg, err := parseHCL(dt, fn+".hcl")
f, sc, err := parseHCLFile(dt, fn+".hcl")
if err != nil {
cfg2, err2 := parseHCL(dt, fn+".json")
f, sc, err2 := parseHCLFile(dt, fn+".json")
if err2 == nil {
return cfg2, nil
return f, sc, nil
}
}
return cfg, err
return f, sc, err
}
func parseHCL(dt []byte, fn string) (_ *Config, err error) {
func parseHCLFile(dt []byte, fn string) (f *hcl.File, _ *StaticConfig, err error) {
defer func() {
err = formatHCLError(dt, err)
}()
// Decode user defined functions, first parsing as hcl and falling back to
// json, returning errors based on the file suffix.
file, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
f, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
if hcldiags.HasErrors() {
var jsondiags hcl.Diagnostics
file, jsondiags = json.Parse(dt, fn)
f, jsondiags = hcljson.Parse(dt, fn)
if jsondiags.HasErrors() {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".json") {
return nil, jsondiags
return nil, nil, jsondiags
}
return nil, hcldiags
return nil, nil, hcldiags
}
}
var sc staticConfig
var sc StaticConfig
// Decode only variable blocks without interpolation.
if err := hclsimple.Decode(fn, dt, nil, &sc); err != nil {
return nil, err
if err := gohcl.DecodeBody(f.Body, nil, &sc); err != nil {
return nil, nil, err
}
return f, &sc, nil
}
func ParseHCL(b hcl.Body, sc *StaticConfig) (_ *Config, err error) {
// Set all variables to their default value if defined.
variables := make(map[string]cty.Value)
for _, variable := range sc.Variables {
variables[variable.Name] = cty.StringVal(variable.Default)
}
userFunctions, _, diags := userfunc.DecodeUserFunctions(file.Body, "function", func() *hcl.EvalContext {
userFunctions, _, diags := userfunc.DecodeUserFunctions(b, "function", func() *hcl.EvalContext {
return &hcl.EvalContext{
Functions: stdlibFunctions,
Variables: variables,
@ -214,7 +229,7 @@ func parseHCL(dt []byte, fn string) (_ *Config, err error) {
var c Config
// Decode with variables and functions.
if err := hclsimple.Decode(fn, dt, ctx, &c); err != nil {
if err := gohcl.DecodeBody(b, ctx, &c); err != nil {
return nil, err
}
return &c, nil

@ -44,7 +44,7 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
@ -104,7 +104,7 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.json")
c, err := ParseFile(dt, "docker-bake.json")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
@ -141,7 +141,7 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
@ -171,7 +171,7 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
@ -200,7 +200,7 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
@ -213,7 +213,7 @@ func TestParseHCL(t *testing.T) {
os.Setenv("BUILD_NUMBER", "456")
c, err = ParseHCL(dt, "docker-bake.hcl")
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
@ -246,7 +246,7 @@ func TestParseHCL(t *testing.T) {
}
`)
_, err := ParseHCL(dt, "docker-bake.hcl")
_, err := ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
require.Contains(t, err.Error(), "docker-bake.hcl:7,17-37: Variables not allowed; Variables may not be used here.")
})
@ -266,7 +266,7 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
@ -275,11 +275,54 @@ func TestParseHCL(t *testing.T) {
os.Setenv("REPO", "docker/buildx")
c, err = ParseHCL(dt, "docker-bake.hcl")
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, []string{"docker/buildx:v1"}, c.Targets[0].Tags)
})
t.Run("MultiFileSharedVariables", func(t *testing.T) {
dt := []byte(`
variable "FOO" {
default = "abc"
}
target "app" {
args = {
v1 = "pre-${FOO}"
}
}
`)
dt2 := []byte(`
target "app" {
args = {
v2 = "${FOO}-post"
}
}
`)
c, err := parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-abc", c.Targets[0].Args["v1"])
require.Equal(t, "abc-post", c.Targets[0].Args["v2"])
os.Setenv("FOO", "def")
c, err = parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-def", c.Targets[0].Args["v1"])
require.Equal(t, "def-post", c.Targets[0].Args["v2"])
})
}

@ -1,108 +0,0 @@
// Package hclsimple is a higher-level entry point for loading HCL
// configuration files directly into Go struct values in a single step.
//
// This package is more opinionated than the rest of the HCL API. See the
// documentation for function Decode for more information.
package hclsimple
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/hashicorp/hcl/v2/json"
)
// Decode parses, decodes, and evaluates expressions in the given HCL source
// code, in a single step.
//
// The main HCL API is built to allow applications that need to decompose
// the processing steps into a pipeline, with different tasks done by
// different parts of the program: parsing the source code into an abstract
// representation, analysing the block structure, evaluating expressions,
// and then extracting the results into a form consumable by the rest of
// the program.
//
// This function does all of those steps in one call, going directly from
// source code to a populated Go struct value.
//
// The "filename" and "src" arguments describe the input configuration. The
// filename is used to add source location context to any returned error
// messages and its suffix will choose one of the two supported syntaxes:
// ".hcl" for native syntax, and ".json" for HCL JSON. The src must therefore
// contain a sequence of bytes that is valid for the selected syntax.
//
// The "ctx" argument provides variables and functions for use during
// expression evaluation. Applications that need no variables nor functions
// can just pass nil.
//
// The "target" argument must be a pointer to a value of a struct type,
// with struct tags as defined by the sibling package "gohcl".
//
// The return type is error but any non-nil error is guaranteed to be
// type-assertable to hcl.Diagnostics for applications that wish to access
// the full error details.
//
// This is a very opinionated function that is intended to serve the needs of
// applications that are just using HCL for simple configuration and don't
// need detailed control over the decoding process. Because this function is
// just wrapping functionality elsewhere, if it doesn't meet your needs then
// please consider copying it into your program and adapting it as needed.
func Decode(filename string, src []byte, ctx *hcl.EvalContext, target interface{}) error {
var file *hcl.File
var diags hcl.Diagnostics
switch suffix := strings.ToLower(filepath.Ext(filename)); suffix {
case ".hcl":
file, diags = hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1})
case ".json":
file, diags = json.Parse(src, filename)
default:
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unsupported file format",
Detail: fmt.Sprintf("Cannot read from %s: unrecognized file format suffix %q.", filename, suffix),
})
return diags
}
if diags.HasErrors() {
return diags
}
diags = gohcl.DecodeBody(file.Body, ctx, target)
if diags.HasErrors() {
return diags
}
return nil
}
// DecodeFile is a wrapper around Decode that first reads the given filename
// from disk. See the Decode documentation for more information.
func DecodeFile(filename string, ctx *hcl.EvalContext, target interface{}) error {
src, err := ioutil.ReadFile(filename)
if err != nil {
if os.IsNotExist(err) {
return hcl.Diagnostics{
{
Severity: hcl.DiagError,
Summary: "Configuration file not found",
Detail: fmt.Sprintf("The configuration file %s does not exist.", filename),
},
}
}
return hcl.Diagnostics{
{
Severity: hcl.DiagError,
Summary: "Failed to read configuration",
Detail: fmt.Sprintf("Can't read %s: %s.", filename, err),
},
}
}
return Decode(filename, src, ctx, target)
}

@ -227,7 +227,6 @@ github.com/hashicorp/hcl/v2/ext/tryfunc
github.com/hashicorp/hcl/v2/ext/typeexpr
github.com/hashicorp/hcl/v2/ext/userfunc
github.com/hashicorp/hcl/v2/gohcl
github.com/hashicorp/hcl/v2/hclsimple
github.com/hashicorp/hcl/v2/hclsyntax
github.com/hashicorp/hcl/v2/hclwrite
github.com/hashicorp/hcl/v2/json

Loading…
Cancel
Save