From 4c1621cccd064c0e456bacddc99f9a7cd0be8c89 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 2 Nov 2021 22:48:20 -0700 Subject: [PATCH] store snapshot of config files on create Files can be reused when container needs to be booted again. Signed-off-by: Tonis Tiigi --- commands/util.go | 4 +- driver/docker-container/driver.go | 48 +++++++++++++----- driver/docker/factory.go | 2 +- driver/kubernetes/factory.go | 9 +--- driver/manager.go | 6 +-- store/nodegroup.go | 14 +++++- util/confutil/container.go | 82 +++++++++++++------------------ 7 files changed, 92 insertions(+), 73 deletions(-) diff --git a/commands/util.go b/commands/util.go index fbb89f7c..9ec5498a 100644 --- a/commands/util.go +++ b/commands/util.go @@ -225,7 +225,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N } } - d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash) + d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash) if err != nil { di.Err = err return nil @@ -360,7 +360,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b } } - d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash) + d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash) if err != nil { return nil, err } diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go index 80f745ec..9ea1a78a 100644 --- a/driver/docker-container/driver.go +++ b/driver/docker-container/driver.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "net" "os" + "path" + "path/filepath" "time" "github.com/docker/buildx/driver" @@ -134,15 +136,8 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error { if err != nil { return err } - if f := d.InitConfig.ConfigFile; f != "" { - configFiles, err := confutil.LoadConfigFiles(f) - if err != nil { - return err - } - defer os.RemoveAll(configFiles) - if err := d.copyToContainer(ctx, configFiles, "/"); err != nil { - return err - } + if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil { + return err } if err := d.start(ctx, l); err != nil { return err @@ -202,7 +197,14 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error { return rc.Close() } -func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error { +func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error { + srcPath, err := writeConfigFiles(files) + if err != nil { + return err + } + if srcPath != "" { + defer os.RemoveAll(srcPath) + } srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{ ChownOpts: &idtools.Identity{UID: 0, GID: 0}, }) @@ -210,7 +212,7 @@ func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir str return err } defer srcArchive.Close() - return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{}) + return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{}) } func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) { @@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) { l.logger.Log(l.stream, dt) return len(dt), nil } + +func writeConfigFiles(m map[string][]byte) (_ string, err error) { + // Temp dir that will be copied to the container + tmpDir, err := os.MkdirTemp("", "buildkitd-config") + if err != nil { + return "", err + } + defer func() { + if err != nil { + os.RemoveAll(tmpDir) + } + }() + for f, dt := range m { + f = path.Join(confutil.DefaultBuildKitConfigDir, f) + p := filepath.Join(tmpDir, f) + if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil { + return "", err + } + if err := os.WriteFile(p, dt, 0600); err != nil { + return "", err + } + } + return tmpDir, nil +} diff --git a/driver/docker/factory.go b/driver/docker/factory.go index e1d0cae0..c960830e 100644 --- a/driver/docker/factory.go +++ b/driver/docker/factory.go @@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver if cfg.DockerAPI == nil { return nil, errors.Errorf("docker driver requires docker API access") } - if cfg.ConfigFile != "" { + if len(cfg.Files) > 0 { return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file") } diff --git a/driver/kubernetes/factory.go b/driver/kubernetes/factory.go index 9ddc5e56..57d1ad59 100644 --- a/driver/kubernetes/factory.go +++ b/driver/kubernetes/factory.go @@ -2,7 +2,6 @@ package kubernetes import ( "context" - "os" "strconv" "strings" @@ -78,12 +77,8 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver deploymentOpt.Qemu.Image = bkimage.QemuImage - if cfg.ConfigFile != "" { - buildkitConfig, err := os.ReadFile(cfg.ConfigFile) - if err != nil { - return nil, err - } - deploymentOpt.BuildkitConfig = buildkitConfig + if cfg, ok := cfg.Files["buildkitd.toml"]; ok { + deploymentOpt.BuildkitConfig = cfg } loadbalance := LoadbalanceSticky diff --git a/driver/manager.go b/driver/manager.go index 33fd22a8..2fe2ffa1 100644 --- a/driver/manager.go +++ b/driver/manager.go @@ -53,7 +53,7 @@ type InitConfig struct { DockerAPI dockerclient.APIClient KubeClientConfig KubeClientConfig BuildkitFlags []string - ConfigFile string + Files map[string][]byte DriverOpts map[string]string Auth Auth Platforms []specs.Platform @@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory { return nil } -func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) { +func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) { ic := InitConfig{ DockerAPI: api, KubeClientConfig: kcc, Name: name, BuildkitFlags: flags, - ConfigFile: config, DriverOpts: do, Auth: auth, Platforms: platforms, ContextPathHash: contextPathHash, + Files: files, } if f == nil { var err error diff --git a/store/nodegroup.go b/store/nodegroup.go index 5eb7c47c..c9f97a67 100644 --- a/store/nodegroup.go +++ b/store/nodegroup.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/containerd/containerd/platforms" + "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/platformutil" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -21,8 +22,9 @@ type Node struct { Endpoint string Platforms []specs.Platform Flags []string - ConfigFile string DriverOpts map[string]string + + Files map[string][]byte } func (ng *NodeGroup) Leave(name string) error { @@ -88,10 +90,18 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints Name: name, Endpoint: endpoint, Platforms: pp, - ConfigFile: configFile, Flags: flags, DriverOpts: do, } + + if configFile != "" { + files, err := confutil.LoadConfigFiles(configFile) + if err != nil { + return err + } + n.Files = files + } + ng.Nodes = append(ng.Nodes, n) if err := ng.validateDuplicates(endpoint, len(ng.Nodes)-1); err != nil { diff --git a/util/confutil/container.go b/util/confutil/container.go index 9b9447e0..9434b8ff 100644 --- a/util/confutil/container.go +++ b/util/confutil/container.go @@ -1,6 +1,7 @@ package confutil import ( + "bytes" "io" "os" "path" @@ -20,31 +21,20 @@ const ( // LoadConfigFiles creates a temp directory with BuildKit config and // registry certificates ready to be copied to a container. -func LoadConfigFiles(bkconfig string) (string, error) { +func LoadConfigFiles(bkconfig string) (map[string][]byte, error) { if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) { - return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig) + return nil, errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig) } else if err != nil { - return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig) + return nil, errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig) } // Load config tree btoml, err := loadConfigTree(bkconfig) if err != nil { - return "", err + return nil, err } - // Temp dir that will be copied to the container - tmpDir, err := os.MkdirTemp("", "buildkitd-config") - if err != nil { - return "", err - } - - // Create BuildKit config folders - tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir) - tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs") - if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil { - return "", err - } + m := make(map[string][]byte) // Iterate through registry config to copy certs and update // BuildKit config with the underlying certs' path in the container. @@ -70,19 +60,20 @@ func LoadConfigFiles(bkconfig string) (string, error) { if regConf == nil { continue } - regCertsDir := path.Join(tmpBuildKitCertsDir, regName) - if err := os.Mkdir(regCertsDir, 0755); err != nil { - return "", err - } + pfx := path.Join("certs", regName) if regConf.Has("ca") { regCAs := regConf.GetArray("ca").([]string) if len(regCAs) > 0 { var cas []string for _, ca := range regCAs { - cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca))) - if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil { - return "", err + fp := path.Join(pfx, path.Base(ca)) + cas = append(cas, path.Join(DefaultBuildKitConfigDir, fp)) + + dt, err := readFile(ca) + if err != nil { + return nil, errors.Wrapf(err, "failed to read CA file: %s", ca) } + m[fp] = dt } regConf.Set("ca", cas) } @@ -98,47 +89,44 @@ func LoadConfigFiles(bkconfig string) (string, error) { } key := kp.Get("key").(string) if len(key) > 0 { - kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key))) - if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil { - return "", err + fp := path.Join(pfx, path.Base(key)) + kp.Set("key", path.Join(DefaultBuildKitConfigDir, fp)) + dt, err := readFile(key) + if err != nil { + return nil, errors.Wrapf(err, "failed to read key file: %s", key) } + m[fp] = dt } cert := kp.Get("cert").(string) if len(cert) > 0 { - kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert))) - if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil { - return "", err + fp := path.Join(pfx, path.Base(cert)) + kp.Set("cert", path.Join(DefaultBuildKitConfigDir, fp)) + dt, err := readFile(cert) + if err != nil { + return nil, errors.Wrapf(err, "failed to read cert file: %s", cert) } + m[fp] = dt } } } } } - // Write BuildKit config - bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return "", err - } - _, err = btoml.WriteTo(bkfile) + b := bytes.NewBuffer(nil) + _, err = btoml.WriteTo(b) if err != nil { - return "", err + return nil, err } + m["buildkitd.toml"] = b.Bytes() - return tmpDir, nil + return m, nil } -func copyfile(src string, dst string) error { - sf, err := os.Open(src) +func readFile(fp string) ([]byte, error) { + sf, err := os.Open(fp) if err != nil { - return err + return nil, err } defer sf.Close() - df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return err - } - defer df.Close() - _, err = io.Copy(df, sf) - return err + return io.ReadAll(io.LimitReader(sf, 1024*1024)) }