From 9c06f383bafb07d7f4beb4ed34523fed1e1e145f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Pich=C3=A9?= Date: Wed, 19 Jul 2023 07:48:09 -0400 Subject: [PATCH] allow custom annotations and labels into kubernetes manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Akihiro Suda Signed-off-by: Jonathan Piché --- docs/reference/buildx_create.md | 4 +- driver/kubernetes/factory.go | 34 +++++++++++++---- driver/kubernetes/factory_test.go | 52 ++++++++++++++++++-------- driver/kubernetes/manifest/manifest.go | 41 +++++++++++++++----- 4 files changed, 98 insertions(+), 33 deletions(-) diff --git a/docs/reference/buildx_create.md b/docs/reference/buildx_create.md index 3665cc08..a290c01d 100644 --- a/docs/reference/buildx_create.md +++ b/docs/reference/buildx_create.md @@ -141,7 +141,7 @@ to achieve that. Passes additional driver-specific options. -Note: When using quoted values for example for the `nodeselector` or +Note: When using quoted values for the `nodeselector`, `annotations`, `labels` or `tolerations` options, ensure that quotes are escaped correctly for your shell. #### `docker` driver @@ -165,6 +165,8 @@ No driver options. - `limits.memory` - Sets the limit memory value specified in bytes or with a valid suffix. Example `limits.memory=500Mi`, `limits.memory=4G` - `serviceaccount` - Sets the created pod's service account. Example `serviceaccount=example-sa` - `"nodeselector=label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64` +- `"annotations=domain/thing1=value1,domain/thing2=value2"` - Sets additional annotations on the deployments and pods. No Defaults. Example `annotations=example.com/owner=sarah` +- `"labels=domain/thing1=value1,domain/thing2=value2"` - Sets additional labels on the deployments and pods. No Defaults. Example `labels=example.com/team=rd` - `"tolerations=key=foo,value=bar;key=foo2,operator=exists;key=foo3,effect=NoSchedule"` - Sets the `Pod` tolerations. Accepts the same values as the kube manifest tolera>tions. Key-value pairs are separated by `,`, tolerations are separated by `;`. No Defaults. Example `tolerations=operator=exists` - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. Needs Kubernetes 1.19 or later. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false. - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky" diff --git a/driver/kubernetes/factory.go b/driver/kubernetes/factory.go index 7b71d9b3..044e192d 100644 --- a/driver/kubernetes/factory.go +++ b/driver/kubernetes/factory.go @@ -148,15 +148,20 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg case "serviceaccount": deploymentOpt.ServiceAccountName = v case "nodeselector": - kvs := strings.Split(strings.Trim(v, `"`), ",") - s := map[string]string{} - for i := range kvs { - kv := strings.Split(kvs[i], "=") - if len(kv) == 2 { - s[kv[0]] = kv[1] - } + deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=") + if err != nil { + return nil, "", "", errors.Wrap(err, "cannot parse node selector") + } + case "annotations": + deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=") + if err != nil { + return nil, "", "", errors.Wrap(err, "cannot parse annotations") + } + case "labels": + deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=") + if err != nil { + return nil, "", "", errors.Wrap(err, "cannot parse labels") } - deploymentOpt.NodeSelector = s case "tolerations": ts := strings.Split(v, ";") deploymentOpt.Tolerations = []corev1.Toleration{} @@ -217,6 +222,19 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg return deploymentOpt, loadbalance, namespace, nil } +func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) { + kvs := strings.Split(strings.Trim(in, `"`), itemsep) + s := map[string]string{} + for i := range kvs { + kv := strings.Split(kvs[i], kvsep) + if len(kv) != 2 { + return nil, errors.Errorf("invalid key-value pair: %s", kvs[i]) + } + s[kv[0]] = kv[1] + } + return s, nil +} + func (f *factory) AllowsInstances() bool { return true } diff --git a/driver/kubernetes/factory_test.go b/driver/kubernetes/factory_test.go index 9e33f257..605be96e 100644 --- a/driver/kubernetes/factory_test.go +++ b/driver/kubernetes/factory_test.go @@ -47,13 +47,13 @@ func TestFactory_processDriverOpts(t *testing.T) { "rootless": "true", "nodeselector": "selector1=value1,selector2=value2", "tolerations": "key=tolerationKey1,value=tolerationValue1,operator=Equal,effect=NoSchedule,tolerationSeconds=60;key=tolerationKey2,operator=Exists", + "annotations": "example.com/expires-after=annotation1,example.com/other=annotation2", + "labels": "example.com/owner=label1,example.com/other=label2", "loadbalance": "random", "qemu.install": "true", "qemu.image": "qemu:latest", } - ns := "test" - - r, loadbalance, ns, err := f.processDriverOpts(cfg.Name, ns, cfg) + r, loadbalance, ns, err := f.processDriverOpts(cfg.Name, "test", cfg) nodeSelectors := map[string]string{ "selector1": "value1", @@ -75,6 +75,16 @@ func TestFactory_processDriverOpts(t *testing.T) { }, } + customAnnotations := map[string]string{ + "example.com/expires-after": "annotation1", + "example.com/other": "annotation2", + } + + customLabels := map[string]string{ + "example.com/owner": "label1", + "example.com/other": "label2", + } + require.NoError(t, err) require.Equal(t, "test-ns", ns) @@ -86,6 +96,8 @@ func TestFactory_processDriverOpts(t *testing.T) { require.Equal(t, "64Mi", r.LimitsMemory) require.True(t, r.Rootless) require.Equal(t, nodeSelectors, r.NodeSelector) + require.Equal(t, customAnnotations, r.CustomAnnotations) + require.Equal(t, customLabels, r.CustomLabels) require.Equal(t, tolerations, r.Tolerations) require.Equal(t, LoadbalanceRandom, loadbalance) require.True(t, r.Qemu.Install) @@ -110,6 +122,8 @@ func TestFactory_processDriverOpts(t *testing.T) { require.Equal(t, "", r.LimitsMemory) require.False(t, r.Rootless) require.Empty(t, r.NodeSelector) + require.Empty(t, r.CustomAnnotations) + require.Empty(t, r.CustomLabels) require.Empty(t, r.Tolerations) require.Equal(t, LoadbalanceSticky, loadbalance) require.False(t, r.Qemu.Install) @@ -137,6 +151,8 @@ func TestFactory_processDriverOpts(t *testing.T) { require.Equal(t, "", r.LimitsMemory) require.True(t, r.Rootless) require.Empty(t, r.NodeSelector) + require.Empty(t, r.CustomAnnotations) + require.Empty(t, r.CustomLabels) require.Empty(t, r.Tolerations) require.Equal(t, LoadbalanceSticky, loadbalance) require.False(t, r.Qemu.Install) @@ -149,9 +165,7 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "replicas": "invalid", } - _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) - require.Error(t, err) }, ) @@ -161,9 +175,7 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "rootless": "invalid", } - _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) - require.Error(t, err) }, ) @@ -173,9 +185,7 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "tolerations": "key=foo,value=bar,invalid=foo2", } - _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) - require.Error(t, err) }, ) @@ -185,9 +195,27 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "tolerations": "key=foo,value=bar,tolerationSeconds=invalid", } + _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) + require.Error(t, err) + }, + ) + t.Run( + "InvalidCustomAnnotation", func(t *testing.T) { + cfg.DriverOpts = map[string]string{ + "annotations": "key,value", + } _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) + require.Error(t, err) + }, + ) + t.Run( + "InvalidCustomLabel", func(t *testing.T) { + cfg.DriverOpts = map[string]string{ + "labels": "key=value=foo", + } + _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) require.Error(t, err) }, ) @@ -197,9 +225,7 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "loadbalance": "invalid", } - _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) - require.Error(t, err) }, ) @@ -209,9 +235,7 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "qemu.install": "invalid", } - _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) - require.Error(t, err) }, ) @@ -221,9 +245,7 @@ func TestFactory_processDriverOpts(t *testing.T) { cfg.DriverOpts = map[string]string{ "invalid": "foo", } - _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg) - require.Error(t, err) }, ) diff --git a/driver/kubernetes/manifest/manifest.go b/driver/kubernetes/manifest/manifest.go index 5f5467c8..e745c542 100644 --- a/driver/kubernetes/manifest/manifest.go +++ b/driver/kubernetes/manifest/manifest.go @@ -7,6 +7,7 @@ import ( "github.com/docker/buildx/util/platformutil" v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -31,24 +32,32 @@ type DeploymentOpt struct { // files mounted at /etc/buildkitd ConfigFiles map[string][]byte - Rootless bool - NodeSelector map[string]string - Tolerations []corev1.Toleration - RequestsCPU string - RequestsMemory string - LimitsCPU string - LimitsMemory string - Platforms []v1.Platform + Rootless bool + NodeSelector map[string]string + CustomAnnotations map[string]string + CustomLabels map[string]string + Tolerations []corev1.Toleration + RequestsCPU string + RequestsMemory string + LimitsCPU string + LimitsMemory string + Platforms []v1.Platform } const ( containerName = "buildkitd" AnnotationPlatform = "buildx.docker.com/platform" + LabelApp = "app" +) + +var ( + ErrReservedAnnotationPlatform = errors.Errorf("the annotation \"%s\" is reserved and cannot be customized", AnnotationPlatform) + ErrReservedLabelApp = errors.Errorf("the label \"%s\" is reserved and cannot be customized", LabelApp) ) func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) { labels := map[string]string{ - "app": opt.Name, + LabelApp: opt.Name, } annotations := map[string]string{} replicas := int32(opt.Replicas) @@ -59,6 +68,20 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.Config annotations[AnnotationPlatform] = strings.Join(platformutil.Format(opt.Platforms), ",") } + for k, v := range opt.CustomAnnotations { + if k == AnnotationPlatform { + return nil, nil, ErrReservedAnnotationPlatform + } + annotations[k] = v + } + + for k, v := range opt.CustomLabels { + if k == LabelApp { + return nil, nil, ErrReservedLabelApp + } + labels[k] = v + } + d = &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: appsv1.SchemeGroupVersion.String(),