Merge pull request #824 from tonistiigi/config-files-store2

store snapshot of config files on create
pull/818/head
Tõnis Tiigi 3 years ago committed by GitHub
commit d311561a8b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -225,7 +225,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
}
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash)
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
if err != nil {
di.Err = err
return nil
@ -360,7 +360,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b
}
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash)
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash)
if err != nil {
return nil, err
}

@ -7,6 +7,8 @@ import (
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"time"
"github.com/docker/buildx/driver"
@ -134,16 +136,9 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
if err != nil {
return err
}
if f := d.InitConfig.ConfigFile; f != "" {
configFiles, err := confutil.LoadConfigFiles(f)
if err != nil {
return err
}
defer os.RemoveAll(configFiles)
if err := d.copyToContainer(ctx, configFiles, "/"); err != nil {
if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil {
return err
}
}
if err := d.start(ctx, l); err != nil {
return err
}
@ -202,7 +197,14 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
return rc.Close()
}
func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error {
srcPath, err := writeConfigFiles(files)
if err != nil {
return err
}
if srcPath != "" {
defer os.RemoveAll(srcPath)
}
srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{
ChownOpts: &idtools.Identity{UID: 0, GID: 0},
})
@ -210,7 +212,7 @@ func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir str
return err
}
defer srcArchive.Close()
return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{})
return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{})
}
func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) {
l.logger.Log(l.stream, dt)
return len(dt), nil
}
func writeConfigFiles(m map[string][]byte) (_ string, err error) {
// Temp dir that will be copied to the container
tmpDir, err := os.MkdirTemp("", "buildkitd-config")
if err != nil {
return "", err
}
defer func() {
if err != nil {
os.RemoveAll(tmpDir)
}
}()
for f, dt := range m {
f = path.Join(confutil.DefaultBuildKitConfigDir, f)
p := filepath.Join(tmpDir, f)
if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
return "", err
}
if err := os.WriteFile(p, dt, 0600); err != nil {
return "", err
}
}
return tmpDir, nil
}

@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
if cfg.DockerAPI == nil {
return nil, errors.Errorf("docker driver requires docker API access")
}
if cfg.ConfigFile != "" {
if len(cfg.Files) > 0 {
return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file")
}

@ -41,7 +41,7 @@ type Driver struct {
factory driver.Factory
minReplicas int
deployment *appsv1.Deployment
configMap *corev1.ConfigMap
configMaps []*corev1.ConfigMap
clientset *kubernetes.Clientset
deploymentClient clientappsv1.DeploymentInterface
podClient clientcorev1.PodInterface
@ -65,16 +65,16 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name)
}
if d.configMap != nil {
for _, cfg := range d.configMaps {
// create ConfigMap first if exists
_, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{})
_, err = d.configMapClient.Create(ctx, cfg, metav1.CreateOptions{})
if err != nil {
if !apierrors.IsAlreadyExists(err) {
return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name)
return errors.Wrapf(err, "error while calling configMapClient.Create for %q", cfg.Name)
}
_, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{})
_, err = d.configMapClient.Update(ctx, cfg, metav1.UpdateOptions{})
if err != nil {
return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name)
return errors.Wrapf(err, "error while calling configMapClient.Update for %q", cfg.Name)
}
}
}
@ -171,10 +171,10 @@ func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
}
}
if d.configMap != nil {
if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil {
for _, cfg := range d.configMaps {
if err := d.configMapClient.Delete(ctx, cfg.Name, metav1.DeleteOptions{}); err != nil {
if !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name)
return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", cfg.Name)
}
}
}

@ -2,7 +2,6 @@ package kubernetes
import (
"context"
"os"
"strconv"
"strings"
@ -74,18 +73,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
BuildkitFlags: cfg.BuildkitFlags,
Rootless: false,
Platforms: cfg.Platforms,
ConfigFiles: cfg.Files,
}
deploymentOpt.Qemu.Image = bkimage.QemuImage
if cfg.ConfigFile != "" {
buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
if err != nil {
return nil, err
}
deploymentOpt.BuildkitConfig = buildkitConfig
}
loadbalance := LoadbalanceSticky
for k, v := range cfg.DriverOpts {
@ -147,7 +139,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
}
}
d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt)
d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt)
if err != nil {
return nil, err
}

@ -1,6 +1,8 @@
package manifest
import (
"fmt"
"path"
"strings"
"github.com/docker/buildx/util/platformutil"
@ -25,9 +27,8 @@ type DeploymentOpt struct {
}
BuildkitFlags []string
// BuildkitConfig
// when not empty, will create configmap with buildkit.toml and mounted
BuildkitConfig []byte
// files mounted at /etc/buildkitd
ConfigFiles map[string][]byte
Rootless bool
NodeSelector map[string]string
@ -43,7 +44,7 @@ const (
AnnotationPlatform = "buildx.docker.com/platform"
)
func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) {
func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) {
labels := map[string]string{
"app": opt.Name,
}
@ -103,26 +104,23 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
},
},
}
if len(opt.BuildkitConfig) > 0 {
c = &corev1.ConfigMap{
for _, cfg := range splitConfigFiles(opt.ConfigFiles) {
cc := &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: corev1.SchemeGroupVersion.String(),
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: opt.Namespace,
Name: opt.Name + "-config",
Name: opt.Name + "-" + cfg.name,
Annotations: annotations,
},
Data: map[string]string{
"buildkitd.toml": string(opt.BuildkitConfig),
},
Data: cfg.files,
}
d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{
Name: "config",
MountPath: "/etc/buildkit",
Name: cfg.name,
MountPath: path.Join("/etc/buildkit", cfg.path),
}}
d.Spec.Template.Spec.Volumes = []corev1.Volume{{
@ -130,11 +128,12 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: c.Name,
Name: cc.Name,
},
},
},
}}
c = append(c, cc)
}
if opt.Qemu.Install {
@ -208,3 +207,35 @@ func toRootless(d *appsv1.Deployment) error {
d.Spec.Template.ObjectMeta.Annotations["container.seccomp.security.alpha.kubernetes.io/"+containerName] = "unconfined"
return nil
}
type config struct {
name string
path string
files map[string]string
}
func splitConfigFiles(m map[string][]byte) []config {
var c []config
idx := map[string]int{}
nameIdx := 0
for k, v := range m {
dir := path.Dir(k)
i, ok := idx[dir]
if !ok {
idx[dir] = len(c)
i = len(c)
name := "config"
if dir != "." {
nameIdx++
name = fmt.Sprintf("%s-%d", name, nameIdx)
}
c = append(c, config{
path: dir,
name: name,
files: map[string]string{},
})
}
c[i].files[path.Base(k)] = string(v)
}
return c
}

@ -53,7 +53,7 @@ type InitConfig struct {
DockerAPI dockerclient.APIClient
KubeClientConfig KubeClientConfig
BuildkitFlags []string
ConfigFile string
Files map[string][]byte
DriverOpts map[string]string
Auth Auth
Platforms []specs.Platform
@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory {
return nil
}
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
ic := InitConfig{
DockerAPI: api,
KubeClientConfig: kcc,
Name: name,
BuildkitFlags: flags,
ConfigFile: config,
DriverOpts: do,
Auth: auth,
Platforms: platforms,
ContextPathHash: contextPathHash,
Files: files,
}
if f == nil {
var err error

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/containerd/containerd/platforms"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/platformutil"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@ -21,8 +22,9 @@ type Node struct {
Endpoint string
Platforms []specs.Platform
Flags []string
ConfigFile string
DriverOpts map[string]string
Files map[string][]byte
}
func (ng *NodeGroup) Leave(name string) error {
@ -88,10 +90,18 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints
Name: name,
Endpoint: endpoint,
Platforms: pp,
ConfigFile: configFile,
Flags: flags,
DriverOpts: do,
}
if configFile != "" {
files, err := confutil.LoadConfigFiles(configFile)
if err != nil {
return err
}
n.Files = files
}
ng.Nodes = append(ng.Nodes, n)
if err := ng.validateDuplicates(endpoint, len(ng.Nodes)-1); err != nil {

@ -1,6 +1,7 @@
package confutil
import (
"bytes"
"io"
"os"
"path"
@ -20,31 +21,20 @@ const (
// LoadConfigFiles creates a temp directory with BuildKit config and
// registry certificates ready to be copied to a container.
func LoadConfigFiles(bkconfig string) (string, error) {
func LoadConfigFiles(bkconfig string) (map[string][]byte, error) {
if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
return nil, errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
} else if err != nil {
return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
return nil, errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
}
// Load config tree
btoml, err := loadConfigTree(bkconfig)
if err != nil {
return "", err
return nil, err
}
// Temp dir that will be copied to the container
tmpDir, err := os.MkdirTemp("", "buildkitd-config")
if err != nil {
return "", err
}
// Create BuildKit config folders
tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir)
tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil {
return "", err
}
m := make(map[string][]byte)
// Iterate through registry config to copy certs and update
// BuildKit config with the underlying certs' path in the container.
@ -70,19 +60,20 @@ func LoadConfigFiles(bkconfig string) (string, error) {
if regConf == nil {
continue
}
regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
if err := os.Mkdir(regCertsDir, 0755); err != nil {
return "", err
}
pfx := path.Join("certs", regName)
if regConf.Has("ca") {
regCAs := regConf.GetArray("ca").([]string)
if len(regCAs) > 0 {
var cas []string
for _, ca := range regCAs {
cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca)))
if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
return "", err
fp := path.Join(pfx, path.Base(ca))
cas = append(cas, path.Join(DefaultBuildKitConfigDir, fp))
dt, err := readFile(ca)
if err != nil {
return nil, errors.Wrapf(err, "failed to read CA file: %s", ca)
}
m[fp] = dt
}
regConf.Set("ca", cas)
}
@ -98,47 +89,44 @@ func LoadConfigFiles(bkconfig string) (string, error) {
}
key := kp.Get("key").(string)
if len(key) > 0 {
kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key)))
if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
return "", err
fp := path.Join(pfx, path.Base(key))
kp.Set("key", path.Join(DefaultBuildKitConfigDir, fp))
dt, err := readFile(key)
if err != nil {
return nil, errors.Wrapf(err, "failed to read key file: %s", key)
}
m[fp] = dt
}
cert := kp.Get("cert").(string)
if len(cert) > 0 {
kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert)))
if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
return "", err
fp := path.Join(pfx, path.Base(cert))
kp.Set("cert", path.Join(DefaultBuildKitConfigDir, fp))
dt, err := readFile(cert)
if err != nil {
return nil, errors.Wrapf(err, "failed to read cert file: %s", cert)
}
m[fp] = dt
}
}
}
}
}
// Write BuildKit config
bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return "", err
}
_, err = btoml.WriteTo(bkfile)
b := bytes.NewBuffer(nil)
_, err = btoml.WriteTo(b)
if err != nil {
return "", err
return nil, err
}
m["buildkitd.toml"] = b.Bytes()
return tmpDir, nil
return m, nil
}
func copyfile(src string, dst string) error {
sf, err := os.Open(src)
func readFile(fp string) ([]byte, error) {
sf, err := os.Open(fp)
if err != nil {
return err
return nil, err
}
defer sf.Close()
df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer df.Close()
_, err = io.Copy(df, sf)
return err
return io.ReadAll(io.LimitReader(sf, 1024*1024))
}

Loading…
Cancel
Save