allow custom annotations and labels into kubernetes manifests

Co-authored-by: Akihiro Suda <suda.kyoto@gmail.com>

Signed-off-by: Jonathan Piché <jpiche@coveo.com>

docs/reference/buildx_create.md

@@ -141,7 +141,7 @@ to achieve that.
 
 Passes additional driver-specific options.
 
-Note: When using quoted values for example for the `nodeselector` or
+Note: When using quoted values for the `nodeselector`, `annotations`, `labels` or
 `tolerations` options, ensure that quotes are escaped correctly for your shell.
 
 #### `docker` driver
@@ -165,6 +165,8 @@ No driver options.
 - `limits.memory` - Sets the limit memory value specified in bytes or with a valid suffix. Example `limits.memory=500Mi`, `limits.memory=4G`
 - `serviceaccount` - Sets the created pod's service account. Example `serviceaccount=example-sa`
 - `"nodeselector=label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
+- `"annotations=domain/thing1=value1,domain/thing2=value2"` - Sets additional annotations on the deployments and pods. No Defaults. Example `annotations=example.com/owner=sarah`
+- `"labels=domain/thing1=value1,domain/thing2=value2"` - Sets additional labels on the deployments and pods. No Defaults. Example `labels=example.com/team=rd`
 - `"tolerations=key=foo,value=bar;key=foo2,operator=exists;key=foo3,effect=NoSchedule"` - Sets the `Pod` tolerations. Accepts the same values as the kube manifest tolera>tions. Key-value pairs are separated by `,`, tolerations are separated by `;`. No Defaults. Example `tolerations=operator=exists`
 - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. Needs Kubernetes 1.19 or later. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
 - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"

driver/kubernetes/factory.go

@@ -148,15 +148,20 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
 		case "serviceaccount":
 			deploymentOpt.ServiceAccountName = v
 		case "nodeselector":
-			kvs := strings.Split(strings.Trim(v, `"`), ",")
+			deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
-			s := map[string]string{}
+			if err != nil {
-			for i := range kvs {
+				return nil, "", "", errors.Wrap(err, "cannot parse node selector")
-				kv := strings.Split(kvs[i], "=")
+			}
-				if len(kv) == 2 {
+		case "annotations":
-					s[kv[0]] = kv[1]
+			deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
-				}
+			if err != nil {
+				return nil, "", "", errors.Wrap(err, "cannot parse annotations")
+			}
+		case "labels":
+			deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
+			if err != nil {
+				return nil, "", "", errors.Wrap(err, "cannot parse labels")
 			}
-			deploymentOpt.NodeSelector = s
 		case "tolerations":
 			ts := strings.Split(v, ";")
 			deploymentOpt.Tolerations = []corev1.Toleration{}
@@ -217,6 +222,19 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
 	return deploymentOpt, loadbalance, namespace, nil
 }
 
+func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {
+	kvs := strings.Split(strings.Trim(in, `"`), itemsep)
+	s := map[string]string{}
+	for i := range kvs {
+		kv := strings.Split(kvs[i], kvsep)
+		if len(kv) != 2 {
+			return nil, errors.Errorf("invalid key-value pair: %s", kvs[i])
+		}
+		s[kv[0]] = kv[1]
+	}
+	return s, nil
+}
+
 func (f *factory) AllowsInstances() bool {
 	return true
 }

driver/kubernetes/factory_test.go

@@ -47,13 +47,13 @@ func TestFactory_processDriverOpts(t *testing.T) {
 				"rootless":        "true",
 				"nodeselector":    "selector1=value1,selector2=value2",
 				"tolerations":     "key=tolerationKey1,value=tolerationValue1,operator=Equal,effect=NoSchedule,tolerationSeconds=60;key=tolerationKey2,operator=Exists",
+				"annotations":     "example.com/expires-after=annotation1,example.com/other=annotation2",
+				"labels":          "example.com/owner=label1,example.com/other=label2",
 				"loadbalance":     "random",
 				"qemu.install":    "true",
 				"qemu.image":      "qemu:latest",
 			}
-			ns := "test"
+			r, loadbalance, ns, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
-			r, loadbalance, ns, err := f.processDriverOpts(cfg.Name, ns, cfg)
 
 			nodeSelectors := map[string]string{
 				"selector1": "value1",
@@ -75,6 +75,16 @@ func TestFactory_processDriverOpts(t *testing.T) {
 				},
 			}
 
+			customAnnotations := map[string]string{
+				"example.com/expires-after": "annotation1",
+				"example.com/other":         "annotation2",
+			}
+
+			customLabels := map[string]string{
+				"example.com/owner": "label1",
+				"example.com/other": "label2",
+			}
+
 			require.NoError(t, err)
 
 			require.Equal(t, "test-ns", ns)
@@ -86,6 +96,8 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			require.Equal(t, "64Mi", r.LimitsMemory)
 			require.True(t, r.Rootless)
 			require.Equal(t, nodeSelectors, r.NodeSelector)
+			require.Equal(t, customAnnotations, r.CustomAnnotations)
+			require.Equal(t, customLabels, r.CustomLabels)
 			require.Equal(t, tolerations, r.Tolerations)
 			require.Equal(t, LoadbalanceRandom, loadbalance)
 			require.True(t, r.Qemu.Install)
@@ -110,6 +122,8 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			require.Equal(t, "", r.LimitsMemory)
 			require.False(t, r.Rootless)
 			require.Empty(t, r.NodeSelector)
+			require.Empty(t, r.CustomAnnotations)
+			require.Empty(t, r.CustomLabels)
 			require.Empty(t, r.Tolerations)
 			require.Equal(t, LoadbalanceSticky, loadbalance)
 			require.False(t, r.Qemu.Install)
@@ -137,6 +151,8 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			require.Equal(t, "", r.LimitsMemory)
 			require.True(t, r.Rootless)
 			require.Empty(t, r.NodeSelector)
+			require.Empty(t, r.CustomAnnotations)
+			require.Empty(t, r.CustomLabels)
 			require.Empty(t, r.Tolerations)
 			require.Equal(t, LoadbalanceSticky, loadbalance)
 			require.False(t, r.Qemu.Install)
@@ -149,9 +165,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"replicas": "invalid",
 			}
-
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
 			require.Error(t, err)
 		},
 	)
@@ -161,9 +175,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"rootless": "invalid",
 			}
-
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
 			require.Error(t, err)
 		},
 	)
@@ -173,9 +185,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"tolerations": "key=foo,value=bar,invalid=foo2",
 			}
-
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
 			require.Error(t, err)
 		},
 	)
@@ -185,9 +195,27 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"tolerations": "key=foo,value=bar,tolerationSeconds=invalid",
 			}
+			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
+			require.Error(t, err)
+		},
+	)
 
+	t.Run(
+		"InvalidCustomAnnotation", func(t *testing.T) {
+			cfg.DriverOpts = map[string]string{
+				"annotations": "key,value",
+			}
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
+			require.Error(t, err)
+		},
+	)
 
+	t.Run(
+		"InvalidCustomLabel", func(t *testing.T) {
+			cfg.DriverOpts = map[string]string{
+				"labels": "key=value=foo",
+			}
+			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
 			require.Error(t, err)
 		},
 	)
@@ -197,9 +225,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"loadbalance": "invalid",
 			}
-
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
 			require.Error(t, err)
 		},
 	)
@@ -209,9 +235,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"qemu.install": "invalid",
 			}
-
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
 			require.Error(t, err)
 		},
 	)
@@ -221,9 +245,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
 			cfg.DriverOpts = map[string]string{
 				"invalid": "foo",
 			}
-
 			_, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
-
 			require.Error(t, err)
 		},
 	)

driver/kubernetes/manifest/manifest.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/docker/buildx/util/platformutil"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -31,24 +32,32 @@ type DeploymentOpt struct {
 	// files mounted at /etc/buildkitd
 	ConfigFiles map[string][]byte
 
-	Rootless       bool
+	Rootless          bool
-	NodeSelector   map[string]string
+	NodeSelector      map[string]string
-	Tolerations    []corev1.Toleration
+	CustomAnnotations map[string]string
-	RequestsCPU    string
+	CustomLabels      map[string]string
-	RequestsMemory string
+	Tolerations       []corev1.Toleration
-	LimitsCPU      string
+	RequestsCPU       string
-	LimitsMemory   string
+	RequestsMemory    string
-	Platforms      []v1.Platform
+	LimitsCPU         string
+	LimitsMemory      string
+	Platforms         []v1.Platform
 }
 
 const (
 	containerName      = "buildkitd"
 	AnnotationPlatform = "buildx.docker.com/platform"
+	LabelApp           = "app"
+)
+
+var (
+	ErrReservedAnnotationPlatform = errors.Errorf("the annotation \"%s\" is reserved and cannot be customized", AnnotationPlatform)
+	ErrReservedLabelApp           = errors.Errorf("the label \"%s\" is reserved and cannot be customized", LabelApp)
 )
 
 func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) {
 	labels := map[string]string{
-		"app": opt.Name,
+		LabelApp: opt.Name,
 	}
 	annotations := map[string]string{}
 	replicas := int32(opt.Replicas)
@@ -59,6 +68,20 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.Config
 		annotations[AnnotationPlatform] = strings.Join(platformutil.Format(opt.Platforms), ",")
 	}
 
+	for k, v := range opt.CustomAnnotations {
+		if k == AnnotationPlatform {
+			return nil, nil, ErrReservedAnnotationPlatform
+		}
+		annotations[k] = v
+	}
+
+	for k, v := range opt.CustomLabels {
+		if k == LabelApp {
+			return nil, nil, ErrReservedLabelApp
+		}
+		labels[k] = v
+	}
+
 	d = &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: appsv1.SchemeGroupVersion.String(),