From 870b38837b21e4212edcfae3f13f7e0e524c2fd4 Mon Sep 17 00:00:00 2001 From: Patrick Van Stee Date: Wed, 15 Apr 2020 21:00:21 -0400 Subject: [PATCH] Allow for user defined functions Signed-off-by: Patrick Van Stee --- README.md | 81 ++++++++- bake/bake.go | 2 + bake/hcl.go | 30 +++- bake/hcl_test.go | 30 ++++ .../hashicorp/hcl/v2/ext/userfunc/README.md | 28 ++++ .../hashicorp/hcl/v2/ext/userfunc/decode.go | 156 ++++++++++++++++++ .../hashicorp/hcl/v2/ext/userfunc/doc.go | 22 +++ .../hashicorp/hcl/v2/ext/userfunc/public.go | 42 +++++ vendor/modules.txt | 1 + 9 files changed, 383 insertions(+), 9 deletions(-) create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/userfunc/README.md create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/userfunc/decode.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/userfunc/doc.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/userfunc/public.go diff --git a/README.md b/README.md index a619c70e..8914295c 100644 --- a/README.md +++ b/README.md @@ -614,13 +614,10 @@ target "db" { Complete list of valid target fields: args, cache-from, cache-to, context, dockerfile, inherits, labels, no-cache, output, platform, pull, secrets, ssh, tags, target -#### HCL variable interpolation +#### HCL variables and functions + +Similar to how Terraform provides a way to [define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable), the HCL file format also supports variable block definitions. These can be used to define variables with values provided by the current environment or a default value when unset. -Similar to how Terraform provides a way to -[define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable), -the HCL file format also supports variable block definitions. These can be used -to define variables with values provided by the current environment or a -default value when unset. Example of using interpolation to tag an image with the git sha: @@ -667,6 +664,78 @@ $ TAG=$(git rev-parse --short HEAD) docker buildx bake --print webapp } ``` + +A [set of generally useful functions](https://github.com/docker/buildx/blob/master/bake/hcl.go#L19-L65) provided by [go-cty](https://github.com/zclconf/go-cty/tree/master/cty/function/stdlib) are avaialble for use in HCL files. In addition, [user defined functions](https://github.com/hashicorp/hcl/tree/hcl2/ext/userfunc) are also supported. + + + +Example of using the `add` function: + +``` +$ cat <<'EOF' > docker-bake.hcl +variable "TAG" { + default = "latest" +} + +group "default" { + targets = ["webapp"] +} + +target "webapp" { + args = { + buildno = "${add(123, 1)}" + } +} +EOF + +$ docker buildx bake --print webapp +{ + "target": { + "webapp": { + "context": ".", + "dockerfile": "Dockerfile", + "args": { + "buildno": "124" + } + } + } +} +``` + +Example of defining an `increment` function: + +``` +$ cat <<'EOF' > docker-bake.hcl +function "increment" { + params = [number] + result = number + 1 +} + +group "default" { + targets = ["webapp"] +} + +target "webapp" { + args = { + buildno = "${increment(123)}" + } +} +EOF + +$ docker buildx bake --print webapp +{ + "target": { + "webapp": { + "context": ".", + "dockerfile": "Dockerfile", + "args": { + "buildno": "124" + } + } + } +} +``` + ### `buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]` Imagetools contains commands for working with manifest lists in the registry. These commands are useful for inspecting multi-platform build results. diff --git a/bake/bake.go b/bake/bake.go index 14028ad0..2f5479be 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -11,6 +11,7 @@ import ( "github.com/docker/buildx/build" "github.com/docker/buildx/util/platformutil" "github.com/docker/docker/pkg/urlutil" + hcl "github.com/hashicorp/hcl/v2" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" ) @@ -73,6 +74,7 @@ type Config struct { Variables []*Variable `json:"-" hcl:"variable,block"` Groups []*Group `json:"groups" hcl:"group,block"` Targets []*Target `json:"targets" hcl:"target,block"` + Remain hcl.Body `json:"-" hcl:",remain"` } func mergeConfig(c1, c2 Config) Config { diff --git a/bake/hcl.go b/bake/hcl.go index 75ae768f..cce83821 100644 --- a/bake/hcl.go +++ b/bake/hcl.go @@ -5,7 +5,9 @@ import ( "strings" hcl "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/ext/userfunc" "github.com/hashicorp/hcl/v2/hclsimple" + "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" "github.com/zclconf/go-cty/cty/function/stdlib" @@ -14,7 +16,7 @@ import ( // Collection of generally useful functions in cty-using applications, which // HCL supports. These functions are available for use in HCL files. var ( - functions = map[string]function.Function{ + stdlibFunctions = map[string]function.Function{ "absolute": stdlib.AbsoluteFunc, "add": stdlib.AddFunc, "and": stdlib.AndFunc, @@ -71,6 +73,21 @@ type staticConfig struct { } func ParseHCL(dt []byte, fn string) (*Config, error) { + // Decode user defined functions. + file, diags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1}) + if diags.HasErrors() { + return nil, diags + } + + userFunctions, _, diags := userfunc.DecodeUserFunctions(file.Body, "function", func() *hcl.EvalContext { + return &hcl.EvalContext{ + Functions: stdlibFunctions, + } + }) + if diags.HasErrors() { + return nil, diags + } + var sc staticConfig // Decode only variable blocks without interpolation. @@ -78,9 +95,8 @@ func ParseHCL(dt []byte, fn string) (*Config, error) { return nil, err } - variables := make(map[string]cty.Value) - // Set all variables to their default value if defined. + variables := make(map[string]cty.Value) for _, variable := range sc.Variables { variables[variable.Name] = cty.StringVal(variable.Default) } @@ -94,6 +110,14 @@ func ParseHCL(dt []byte, fn string) (*Config, error) { } } + functions := make(map[string]function.Function) + for k, v := range stdlibFunctions { + functions[k] = v + } + for k, v := range userFunctions { + functions[k] = v + } + ctx := &hcl.EvalContext{ Variables: variables, Functions: functions, diff --git a/bake/hcl_test.go b/bake/hcl_test.go index 1881543b..7ea047a0 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -93,6 +93,36 @@ func TestParseHCL(t *testing.T) { require.Equal(t, "124", c.Targets[0].Args["buildno"]) }) + t.Run("WithUserDefinedFunctions", func(t *testing.T) { + dt := []byte(` + function "increment" { + params = [number] + result = number + 1 + } + + group "default" { + targets = ["webapp"] + } + + target "webapp" { + args = { + buildno = "${increment(123)}" + } + } + `) + + c, err := ParseHCL(dt, "docker-bake.hcl") + require.NoError(t, err) + + require.Equal(t, 1, len(c.Groups)) + require.Equal(t, "default", c.Groups[0].Name) + require.Equal(t, []string{"webapp"}, c.Groups[0].Targets) + + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, c.Targets[0].Name, "webapp") + require.Equal(t, "124", c.Targets[0].Args["buildno"]) + }) + t.Run("WithVariables", func(t *testing.T) { dt := []byte(` variable "BUILD_NUMBER" { diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/README.md b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/README.md new file mode 100644 index 00000000..5d38046e --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/README.md @@ -0,0 +1,28 @@ +# HCL User Functions Extension + +This HCL extension allows a calling application to support user-defined +functions. + +Functions are defined via a specific block type, like this: + +```hcl +function "add" { + params = [a, b] + result = a + b +} + +function "list" { + params = [] + variadic_param = items + result = items +} +``` + +The extension is implemented as a pre-processor for `cty.Body` objects. Given +a body that may contain functions, the `DecodeUserFunctions` function searches +for blocks that define functions and returns a functions map suitable for +inclusion in a `hcl.EvalContext`. It also returns a new `cty.Body` that +contains the remainder of the content from the given body, allowing for +further processing of remaining content. + +For more information, see [the godoc reference](http://godoc.org/github.com/hashicorp/hcl/v2/ext/userfunc). diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/decode.go b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/decode.go new file mode 100644 index 00000000..6c1e4ca4 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/decode.go @@ -0,0 +1,156 @@ +package userfunc + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" +) + +var funcBodySchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "params", + Required: true, + }, + { + Name: "variadic_param", + Required: false, + }, + { + Name: "result", + Required: true, + }, + }, +} + +func decodeUserFunctions(body hcl.Body, blockType string, contextFunc ContextFunc) (funcs map[string]function.Function, remain hcl.Body, diags hcl.Diagnostics) { + schema := &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: blockType, + LabelNames: []string{"name"}, + }, + }, + } + + content, remain, diags := body.PartialContent(schema) + if diags.HasErrors() { + return nil, remain, diags + } + + // first call to getBaseCtx will populate context, and then the same + // context will be used for all subsequent calls. It's assumed that + // all functions in a given body should see an identical context. + var baseCtx *hcl.EvalContext + getBaseCtx := func() *hcl.EvalContext { + if baseCtx == nil { + if contextFunc != nil { + baseCtx = contextFunc() + } + } + // baseCtx might still be nil here, and that's okay + return baseCtx + } + + funcs = make(map[string]function.Function) +Blocks: + for _, block := range content.Blocks { + name := block.Labels[0] + funcContent, funcDiags := block.Body.Content(funcBodySchema) + diags = append(diags, funcDiags...) + if funcDiags.HasErrors() { + continue + } + + paramsExpr := funcContent.Attributes["params"].Expr + resultExpr := funcContent.Attributes["result"].Expr + var varParamExpr hcl.Expression + if funcContent.Attributes["variadic_param"] != nil { + varParamExpr = funcContent.Attributes["variadic_param"].Expr + } + + var params []string + var varParam string + + paramExprs, paramsDiags := hcl.ExprList(paramsExpr) + diags = append(diags, paramsDiags...) + if paramsDiags.HasErrors() { + continue + } + for _, paramExpr := range paramExprs { + param := hcl.ExprAsKeyword(paramExpr) + if param == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid param element", + Detail: "Each parameter name must be an identifier.", + Subject: paramExpr.Range().Ptr(), + }) + continue Blocks + } + params = append(params, param) + } + + if varParamExpr != nil { + varParam = hcl.ExprAsKeyword(varParamExpr) + if varParam == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variadic_param", + Detail: "The variadic parameter name must be an identifier.", + Subject: varParamExpr.Range().Ptr(), + }) + continue + } + } + + spec := &function.Spec{} + for _, paramName := range params { + spec.Params = append(spec.Params, function.Parameter{ + Name: paramName, + Type: cty.DynamicPseudoType, + }) + } + if varParamExpr != nil { + spec.VarParam = &function.Parameter{ + Name: varParam, + Type: cty.DynamicPseudoType, + } + } + impl := func(args []cty.Value) (cty.Value, error) { + ctx := getBaseCtx() + ctx = ctx.NewChild() + ctx.Variables = make(map[string]cty.Value) + + // The cty function machinery guarantees that we have at least + // enough args to fill all of our params. + for i, paramName := range params { + ctx.Variables[paramName] = args[i] + } + if spec.VarParam != nil { + varArgs := args[len(params):] + ctx.Variables[varParam] = cty.TupleVal(varArgs) + } + + result, diags := resultExpr.Value(ctx) + if diags.HasErrors() { + // Smuggle the diagnostics out via the error channel, since + // a diagnostics sequence implements error. Caller can + // type-assert this to recover the individual diagnostics + // if desired. + return cty.DynamicVal, diags + } + return result, nil + } + spec.Type = func(args []cty.Value) (cty.Type, error) { + val, err := impl(args) + return val.Type(), err + } + spec.Impl = func(args []cty.Value, retType cty.Type) (cty.Value, error) { + return impl(args) + } + funcs[name] = function.New(spec) + } + + return funcs, remain, diags +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/doc.go b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/doc.go new file mode 100644 index 00000000..e4461d45 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/doc.go @@ -0,0 +1,22 @@ +// Package userfunc implements a HCL extension that allows user-defined +// functions in HCL configuration. +// +// Using this extension requires some integration effort on the part of the +// calling application, to pass any declared functions into a HCL evaluation +// context after processing. +// +// The function declaration syntax looks like this: +// +// function "foo" { +// params = ["name"] +// result = "Hello, ${name}!" +// } +// +// When a user-defined function is called, the expression given for the "result" +// attribute is evaluated in an isolated evaluation context that defines variables +// named after the given parameter names. +// +// The block name "function" may be overridden by the calling application, if +// that default name conflicts with an existing block or attribute name in +// the application. +package userfunc diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/public.go b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/public.go new file mode 100644 index 00000000..5415c8c9 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/userfunc/public.go @@ -0,0 +1,42 @@ +package userfunc + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty/function" +) + +// A ContextFunc is a callback used to produce the base EvalContext for +// running a particular set of functions. +// +// This is a function rather than an EvalContext directly to allow functions +// to be decoded before their context is complete. This will be true, for +// example, for applications that wish to allow functions to refer to themselves. +// +// The simplest use of a ContextFunc is to give user functions access to the +// same global variables and functions available elsewhere in an application's +// configuration language, but more complex applications may use different +// contexts to support lexical scoping depending on where in a configuration +// structure a function declaration is found, etc. +type ContextFunc func() *hcl.EvalContext + +// DecodeUserFunctions looks for blocks of the given type in the given body +// and, for each one found, interprets it as a custom function definition. +// +// On success, the result is a mapping of function names to implementations, +// along with a new body that represents the remaining content of the given +// body which can be used for further processing. +// +// The result expression of each function is parsed during decoding but not +// evaluated until the function is called. +// +// If the given ContextFunc is non-nil, it will be called to obtain the +// context in which the function result expressions will be evaluated. If nil, +// or if it returns nil, the result expression will have access only to +// variables named after the declared parameters. A non-nil context turns +// the returned functions into closures, bound to the given context. +// +// If the returned diagnostics set has errors then the function map and +// remain body may be nil or incomplete. +func DecodeUserFunctions(body hcl.Body, blockType string, context ContextFunc) (funcs map[string]function.Function, remain hcl.Body, diags hcl.Diagnostics) { + return decodeUserFunctions(body, blockType, context) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 43a68c1d..e669d397 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -212,6 +212,7 @@ github.com/hashicorp/golang-lru/simplelru # github.com/hashicorp/hcl/v2 v2.4.0 github.com/hashicorp/hcl/v2 github.com/hashicorp/hcl/v2/ext/customdecode +github.com/hashicorp/hcl/v2/ext/userfunc github.com/hashicorp/hcl/v2/gohcl github.com/hashicorp/hcl/v2/hclsimple github.com/hashicorp/hcl/v2/hclsyntax