store snapshot of config files on create

Files can be reused when container needs to be booted again.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

commands/util.go

@@ -225,7 +225,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
 					}
 				}
 
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash)
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
 				if err != nil {
 					di.Err = err
 					return nil
@@ -360,7 +360,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b
 		}
 	}
 
-	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash)
+	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash)
 	if err != nil {
 		return nil, err
 	}

driver/docker-container/driver.go

@@ -7,6 +7,8 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/buildx/driver"
@@ -134,16 +136,9 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		if err != nil {
 			return err
 		}
-		if f := d.InitConfig.ConfigFile; f != "" {
+		if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil {
-			configFiles, err := confutil.LoadConfigFiles(f)
-			if err != nil {
-				return err
-			}
-			defer os.RemoveAll(configFiles)
-			if err := d.copyToContainer(ctx, configFiles, "/"); err != nil {
 			return err
 		}
-		}
 		if err := d.start(ctx, l); err != nil {
 			return err
 		}
@@ -202,7 +197,14 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
 	return rc.Close()
 }
 
-func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
+func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error {
+	srcPath, err := writeConfigFiles(files)
+	if err != nil {
+		return err
+	}
+	if srcPath != "" {
+		defer os.RemoveAll(srcPath)
+	}
 	srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{
 		ChownOpts: &idtools.Identity{UID: 0, GID: 0},
 	})
@@ -210,7 +212,7 @@ func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir str
 		return err
 	}
 	defer srcArchive.Close()
-	return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{})
+	return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{})
 }
 
 func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
@@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) {
 	l.logger.Log(l.stream, dt)
 	return len(dt), nil
 }
+
+func writeConfigFiles(m map[string][]byte) (_ string, err error) {
+	// Temp dir that will be copied to the container
+	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
+	if err != nil {
+		return "", err
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(tmpDir)
+		}
+	}()
+	for f, dt := range m {
+		f = path.Join(confutil.DefaultBuildKitConfigDir, f)
+		p := filepath.Join(tmpDir, f)
+		if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
+			return "", err
+		}
+		if err := os.WriteFile(p, dt, 0600); err != nil {
+			return "", err
+		}
+	}
+	return tmpDir, nil
+}

driver/docker/factory.go

@@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 	if cfg.DockerAPI == nil {
 		return nil, errors.Errorf("docker driver requires docker API access")
 	}
-	if cfg.ConfigFile != "" {
+	if len(cfg.Files) > 0 {
 		return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file")
 	}
 

driver/kubernetes/factory.go

@@ -2,7 +2,6 @@ package kubernetes
 
 import (
 	"context"
-	"os"
 	"strconv"
 	"strings"
 
@@ -78,12 +77,8 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 
 	deploymentOpt.Qemu.Image = bkimage.QemuImage
 
-	if cfg.ConfigFile != "" {
+	if cfg, ok := cfg.Files["buildkitd.toml"]; ok {
-		buildkitConfig, err := os.ReadFile(cfg.ConfigFile)
+		deploymentOpt.BuildkitConfig = cfg
-		if err != nil {
-			return nil, err
-		}
-		deploymentOpt.BuildkitConfig = buildkitConfig
 	}
 
 	loadbalance := LoadbalanceSticky

driver/manager.go

@@ -53,7 +53,7 @@ type InitConfig struct {
 	DockerAPI        dockerclient.APIClient
 	KubeClientConfig KubeClientConfig
 	BuildkitFlags    []string
-	ConfigFile       string
+	Files            map[string][]byte
 	DriverOpts       map[string]string
 	Auth             Auth
 	Platforms        []specs.Platform
@@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory {
 	return nil
 }
 
-func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
+func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
 	ic := InitConfig{
 		DockerAPI:        api,
 		KubeClientConfig: kcc,
 		Name:             name,
 		BuildkitFlags:    flags,
-		ConfigFile:       config,
 		DriverOpts:       do,
 		Auth:             auth,
 		Platforms:        platforms,
 		ContextPathHash:  contextPathHash,
+		Files:            files,
 	}
 	if f == nil {
 		var err error

store/nodegroup.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/containerd/containerd/platforms"
+	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/platformutil"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -21,8 +22,9 @@ type Node struct {
 	Endpoint   string
 	Platforms  []specs.Platform
 	Flags      []string
-	ConfigFile string
 	DriverOpts map[string]string
+
+	Files map[string][]byte
 }
 
 func (ng *NodeGroup) Leave(name string) error {
@@ -88,10 +90,18 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints
 		Name:       name,
 		Endpoint:   endpoint,
 		Platforms:  pp,
-		ConfigFile: configFile,
 		Flags:      flags,
 		DriverOpts: do,
 	}
+
+	if configFile != "" {
+		files, err := confutil.LoadConfigFiles(configFile)
+		if err != nil {
+			return err
+		}
+		n.Files = files
+	}
+
 	ng.Nodes = append(ng.Nodes, n)
 
 	if err := ng.validateDuplicates(endpoint, len(ng.Nodes)-1); err != nil {

util/confutil/container.go

@@ -1,6 +1,7 @@
 package confutil
 
 import (
+	"bytes"
 	"io"
 	"os"
 	"path"
@@ -20,31 +21,20 @@ const (
 
 // LoadConfigFiles creates a temp directory with BuildKit config and
 // registry certificates ready to be copied to a container.
-func LoadConfigFiles(bkconfig string) (string, error) {
+func LoadConfigFiles(bkconfig string) (map[string][]byte, error) {
 	if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
-		return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
+		return nil, errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
 	} else if err != nil {
-		return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
+		return nil, errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
 	}
 
 	// Load config tree
 	btoml, err := loadConfigTree(bkconfig)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	// Temp dir that will be copied to the container
+	m := make(map[string][]byte)
-	tmpDir, err := os.MkdirTemp("", "buildkitd-config")
-	if err != nil {
-		return "", err
-	}
-
-	// Create BuildKit config folders
-	tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir)
-	tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
-	if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil {
-		return "", err
-	}
 
 	// Iterate through registry config to copy certs and update
 	// BuildKit config with the underlying certs' path in the container.
@@ -70,19 +60,20 @@ func LoadConfigFiles(bkconfig string) (string, error) {
 			if regConf == nil {
 				continue
 			}
-			regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
+			pfx := path.Join("certs", regName)
-			if err := os.Mkdir(regCertsDir, 0755); err != nil {
-				return "", err
-			}
 			if regConf.Has("ca") {
 				regCAs := regConf.GetArray("ca").([]string)
 				if len(regCAs) > 0 {
 					var cas []string
 					for _, ca := range regCAs {
-						cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca)))
+						fp := path.Join(pfx, path.Base(ca))
-						if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
+						cas = append(cas, path.Join(DefaultBuildKitConfigDir, fp))
-							return "", err
+
+						dt, err := readFile(ca)
+						if err != nil {
+							return nil, errors.Wrapf(err, "failed to read CA file: %s", ca)
 						}
+						m[fp] = dt
 					}
 					regConf.Set("ca", cas)
 				}
@@ -98,47 +89,44 @@ func LoadConfigFiles(bkconfig string) (string, error) {
 					}
 					key := kp.Get("key").(string)
 					if len(key) > 0 {
-						kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key)))
+						fp := path.Join(pfx, path.Base(key))
-						if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
+						kp.Set("key", path.Join(DefaultBuildKitConfigDir, fp))
-							return "", err
+						dt, err := readFile(key)
+						if err != nil {
+							return nil, errors.Wrapf(err, "failed to read key file: %s", key)
 						}
+						m[fp] = dt
 					}
 					cert := kp.Get("cert").(string)
 					if len(cert) > 0 {
-						kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert)))
+						fp := path.Join(pfx, path.Base(cert))
-						if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
+						kp.Set("cert", path.Join(DefaultBuildKitConfigDir, fp))
-							return "", err
+						dt, err := readFile(cert)
+						if err != nil {
+							return nil, errors.Wrapf(err, "failed to read cert file: %s", cert)
 						}
+						m[fp] = dt
 					}
 				}
 			}
 		}
 	}
 
-	// Write BuildKit config
+	b := bytes.NewBuffer(nil)
-	bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
+	_, err = btoml.WriteTo(b)
-	if err != nil {
-		return "", err
-	}
-	_, err = btoml.WriteTo(bkfile)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
+	m["buildkitd.toml"] = b.Bytes()
 
-	return tmpDir, nil
+	return m, nil
 }
 
-func copyfile(src string, dst string) error {
+func readFile(fp string) ([]byte, error) {
-	sf, err := os.Open(src)
+	sf, err := os.Open(fp)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer sf.Close()
-	df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600)
+	return io.ReadAll(io.LimitReader(sf, 1024*1024))
-	if err != nil {
-		return err
-	}
-	defer df.Close()
-	_, err = io.Copy(df, sf)
-	return err
 }