Keep BuildKit state in a volume

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>

commands/rm.go

@@ -11,7 +11,8 @@ import (
 )
 
 type rmOptions struct {
-	builder string
+	builder   string
+	keepState bool
 }
 
 func runRm(dockerCli command.Cli, in rmOptions) error {
@@ -28,7 +29,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		if err != nil {
 			return err
 		}
-		err1 := stop(ctx, dockerCli, ng, true)
+		err1 := rm(ctx, dockerCli, ng, in.keepState)
 		if err := txn.Remove(ng.Name); err != nil {
 			return err
 		}
@@ -40,7 +41,7 @@ func runRm(dockerCli command.Cli, in rmOptions) error {
 		return err
 	}
 	if ng != nil {
-		err1 := stop(ctx, dockerCli, ng, true)
+		err1 := rm(ctx, dockerCli, ng, in.keepState)
 		if err := txn.Remove(ng.Name); err != nil {
 			return err
 		}
@@ -66,10 +67,13 @@ func rmCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 		},
 	}
 
+	flags := cmd.Flags()
+	flags.BoolVar(&options.keepState, "keep-state", false, "Keep BuildKit state")
+
 	return cmd
 }
 
-func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bool) error {
+func rm(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, keepState bool) error {
 	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
 	if err != nil {
 		return err
@@ -79,34 +83,9 @@ func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bo
 			if err := di.Driver.Stop(ctx, true); err != nil {
 				return err
 			}
-			if rm {
+			if err := di.Driver.Rm(ctx, true, !keepState); err != nil {
-				if err := di.Driver.Rm(ctx, true); err != nil {
-					return err
-				}
-			}
-		}
-		if di.Err != nil {
-			err = di.Err
-		}
-	}
-	return err
-}
-
-func stopCurrent(ctx context.Context, dockerCli command.Cli, rm bool) error {
-	dis, err := getDefaultDrivers(ctx, dockerCli, false, "")
-	if err != nil {
-		return err
-	}
-	for _, di := range dis {
-		if di.Driver != nil {
-			if err := di.Driver.Stop(ctx, true); err != nil {
 				return err
 			}
-			if rm {
-				if err := di.Driver.Rm(ctx, true); err != nil {
-					return err
-				}
-			}
 		}
 		if di.Err != nil {
 			err = di.Err

commands/stop.go

@@ -1,6 +1,9 @@
 package commands
 
 import (
+	"context"
+
+	"github.com/docker/buildx/store"
 	"github.com/docker/cli/cli"
 	"github.com/docker/cli/cli/command"
 	"github.com/moby/buildkit/util/appcontext"
@@ -25,7 +28,7 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
 		if err != nil {
 			return err
 		}
-		if err := stop(ctx, dockerCli, ng, false); err != nil {
+		if err := stop(ctx, dockerCli, ng); err != nil {
 			return err
 		}
 		return nil
@@ -36,10 +39,10 @@ func runStop(dockerCli command.Cli, in stopOptions) error {
 		return err
 	}
 	if ng != nil {
-		return stop(ctx, dockerCli, ng, false)
+		return stop(ctx, dockerCli, ng)
 	}
 
-	return stopCurrent(ctx, dockerCli, false)
+	return stopCurrent(ctx, dockerCli)
 }
 
 func stopCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
@@ -66,3 +69,39 @@ func stopCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
 
 	return cmd
 }
+
+func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup) error {
+	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
+	if err != nil {
+		return err
+	}
+	for _, di := range dis {
+		if di.Driver != nil {
+			if err := di.Driver.Stop(ctx, true); err != nil {
+				return err
+			}
+		}
+		if di.Err != nil {
+			err = di.Err
+		}
+	}
+	return err
+}
+
+func stopCurrent(ctx context.Context, dockerCli command.Cli) error {
+	dis, err := getDefaultDrivers(ctx, dockerCli, false, "")
+	if err != nil {
+		return err
+	}
+	for _, di := range dis {
+		if di.Driver != nil {
+			if err := di.Driver.Stop(ctx, true); err != nil {
+				return err
+			}
+		}
+		if di.Err != nil {
+			err = di.Err
+		}
+	}
+	return err
+}

docs/reference/buildx_rm.md

@@ -7,6 +7,13 @@ docker buildx rm [NAME]
 <!---MARKER_GEN_START-->
 Remove a builder instance
 
+### Options
+
+| Name | Description |
+| --- | --- |
+| `--builder string` | Override the configured builder instance |
+| [`--keep-state`](#keep-state) | Keep BuildKit state |
+
 
 <!---MARKER_GEN_END-->
 
@@ -14,3 +21,10 @@ Remove a builder instance
 
 Removes the specified or current builder. It is a no-op attempting to remove the
 default builder.
+
+## Examples
+
+### <a name="keep-state"></a> Keep BuildKit state (--keep-state)
+
+Keep BuildKit state, so it can be reused by a new builder with the same name.
+Currently, only supported by the [`docker-container` driver](buildx_create.md#driver).

driver/docker-container/driver.go

@@ -17,14 +17,18 @@ import (
 	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/util/appdefaults"
 	"github.com/moby/buildkit/util/tracing/detect"
 	"github.com/pkg/errors"
 )
 
+const volumeStateSuffix = "_state"
+
 type Driver struct {
 	driver.InitConfig
 	factory driver.Factory
@@ -103,6 +107,13 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		hc := &container.HostConfig{
 			Privileged: true,
 			UsernsMode: "host",
+			Mounts: []mount.Mount{
+				{
+					Type:   mount.TypeVolume,
+					Source: d.Name + volumeStateSuffix,
+					Target: appdefaults.Root,
+				},
+			},
 		}
 		if d.netMode != "" {
 			hc.NetworkMode = container.NetworkMode(d.netMode)
@@ -226,7 +237,7 @@ func (d *Driver) start(ctx context.Context, l progress.SubLogger) error {
 }
 
 func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
-	container, err := d.DockerAPI.ContainerInspect(ctx, d.Name)
+	ctn, err := d.DockerAPI.ContainerInspect(ctx, d.Name)
 	if err != nil {
 		if dockerclient.IsErrNotFound(err) {
 			return &driver.Info{
@@ -236,7 +247,7 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
 		return nil, err
 	}
 
-	if container.State.Running {
+	if ctn.State.Running {
 		return &driver.Info{
 			Status: driver.Running,
 		}, nil
@@ -258,16 +269,21 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
 
-func (d *Driver) Rm(ctx context.Context, force bool) error {
+func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	info, err := d.Info(ctx)
 	if err != nil {
 		return err
 	}
 	if info.Status != driver.Inactive {
-		return d.DockerAPI.ContainerRemove(ctx, d.Name, dockertypes.ContainerRemoveOptions{
+		if err := d.DockerAPI.ContainerRemove(ctx, d.Name, dockertypes.ContainerRemoveOptions{
 			RemoveVolumes: true,
-			Force:         true,
+			Force:         force,
-		})
+		}); err != nil {
+			return err
+		}
+		if rmVolume {
+			return d.DockerAPI.VolumeRemove(ctx, d.Name+volumeStateSuffix, false)
+		}
 	}
 	return nil
 }

driver/docker/driver.go

@@ -33,7 +33,7 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
 
-func (d *Driver) Rm(ctx context.Context, force bool) error {
+func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	return nil
 }
 

driver/driver.go

@@ -54,7 +54,7 @@ type Driver interface {
 	Bootstrap(context.Context, progress.Logger) error
 	Info(context.Context) (*Info, error)
 	Stop(ctx context.Context, force bool) error
-	Rm(ctx context.Context, force bool) error
+	Rm(ctx context.Context, force bool, rmVolume bool) error
 	Client(ctx context.Context) (*client.Client, error)
 	Features() map[Feature]bool
 	IsMobyDriver() bool

driver/kubernetes/driver.go

@@ -143,7 +143,7 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
 	return nil
 }
 
-func (d *Driver) Rm(ctx context.Context, force bool) error {
+func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error {
 	if err := d.deploymentClient.Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}); err != nil {
 		return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
 	}