From 40121c671c80146654dc7018b4d6f7f555cde01f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 3 Nov 2021 18:03:52 -0700 Subject: [PATCH] kubernetes: store config files for k8s Signed-off-by: Tonis Tiigi --- driver/kubernetes/driver.go | 18 ++++---- driver/kubernetes/factory.go | 7 +-- driver/kubernetes/manifest/manifest.go | 59 ++++++++++++++++++++------ 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/driver/kubernetes/driver.go b/driver/kubernetes/driver.go index 58daf373..7e524e4b 100644 --- a/driver/kubernetes/driver.go +++ b/driver/kubernetes/driver.go @@ -41,7 +41,7 @@ type Driver struct { factory driver.Factory minReplicas int deployment *appsv1.Deployment - configMap *corev1.ConfigMap + configMaps []*corev1.ConfigMap clientset *kubernetes.Clientset deploymentClient clientappsv1.DeploymentInterface podClient clientcorev1.PodInterface @@ -65,16 +65,16 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error { return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name) } - if d.configMap != nil { + for _, cfg := range d.configMaps { // create ConfigMap first if exists - _, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{}) + _, err = d.configMapClient.Create(ctx, cfg, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name) + return errors.Wrapf(err, "error while calling configMapClient.Create for %q", cfg.Name) } - _, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{}) + _, err = d.configMapClient.Update(ctx, cfg, metav1.UpdateOptions{}) if err != nil { - return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name) + return errors.Wrapf(err, "error while calling configMapClient.Update for %q", cfg.Name) } } } @@ -171,10 +171,10 @@ func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error { return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name) } } - if d.configMap != nil { - if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil { + for _, cfg := range d.configMaps { + if err := d.configMapClient.Delete(ctx, cfg.Name, metav1.DeleteOptions{}); err != nil { if !apierrors.IsNotFound(err) { - return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name) + return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", cfg.Name) } } } diff --git a/driver/kubernetes/factory.go b/driver/kubernetes/factory.go index 57d1ad59..61985e4b 100644 --- a/driver/kubernetes/factory.go +++ b/driver/kubernetes/factory.go @@ -73,14 +73,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver BuildkitFlags: cfg.BuildkitFlags, Rootless: false, Platforms: cfg.Platforms, + ConfigFiles: cfg.Files, } deploymentOpt.Qemu.Image = bkimage.QemuImage - if cfg, ok := cfg.Files["buildkitd.toml"]; ok { - deploymentOpt.BuildkitConfig = cfg - } - loadbalance := LoadbalanceSticky for k, v := range cfg.DriverOpts { @@ -142,7 +139,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver } } - d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt) + d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt) if err != nil { return nil, err } diff --git a/driver/kubernetes/manifest/manifest.go b/driver/kubernetes/manifest/manifest.go index 3673044a..75268247 100644 --- a/driver/kubernetes/manifest/manifest.go +++ b/driver/kubernetes/manifest/manifest.go @@ -1,6 +1,8 @@ package manifest import ( + "fmt" + "path" "strings" "github.com/docker/buildx/util/platformutil" @@ -25,9 +27,8 @@ type DeploymentOpt struct { } BuildkitFlags []string - // BuildkitConfig - // when not empty, will create configmap with buildkit.toml and mounted - BuildkitConfig []byte + // files mounted at /etc/buildkitd + ConfigFiles map[string][]byte Rootless bool NodeSelector map[string]string @@ -43,7 +44,7 @@ const ( AnnotationPlatform = "buildx.docker.com/platform" ) -func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) { +func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) { labels := map[string]string{ "app": opt.Name, } @@ -103,26 +104,23 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa }, }, } - - if len(opt.BuildkitConfig) > 0 { - c = &corev1.ConfigMap{ + for _, cfg := range splitConfigFiles(opt.ConfigFiles) { + cc := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Namespace: opt.Namespace, - Name: opt.Name + "-config", + Name: opt.Name + "-" + cfg.name, Annotations: annotations, }, - Data: map[string]string{ - "buildkitd.toml": string(opt.BuildkitConfig), - }, + Data: cfg.files, } d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{ - Name: "config", - MountPath: "/etc/buildkit", + Name: cfg.name, + MountPath: path.Join("/etc/buildkit", cfg.path), }} d.Spec.Template.Spec.Volumes = []corev1.Volume{{ @@ -130,11 +128,12 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: c.Name, + Name: cc.Name, }, }, }, }} + c = append(c, cc) } if opt.Qemu.Install { @@ -208,3 +207,35 @@ func toRootless(d *appsv1.Deployment) error { d.Spec.Template.ObjectMeta.Annotations["container.seccomp.security.alpha.kubernetes.io/"+containerName] = "unconfined" return nil } + +type config struct { + name string + path string + files map[string]string +} + +func splitConfigFiles(m map[string][]byte) []config { + var c []config + idx := map[string]int{} + nameIdx := 0 + for k, v := range m { + dir := path.Dir(k) + i, ok := idx[dir] + if !ok { + idx[dir] = len(c) + i = len(c) + name := "config" + if dir != "." { + nameIdx++ + name = fmt.Sprintf("%s-%d", name, nameIdx) + } + c = append(c, config{ + path: dir, + name: name, + files: map[string]string{}, + }) + } + c[i].files[path.Base(k)] = string(v) + } + return c +}