diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go
index 54c02abe..e0b477a4 100644
--- a/driver/docker-container/driver.go
+++ b/driver/docker-container/driver.go
@@ -36,7 +36,6 @@ const (
 type Driver struct {
 	driver.InitConfig
 	factory      driver.Factory
-	userNSRemap  bool // true if dockerd is running with userns-remap mode
 	netMode      string
 	image        string
 	cgroupParent string
@@ -120,19 +119,30 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 				},
 			},
 		}
-		if d.userNSRemap {
-			hc.UsernsMode = "host"
-		}
 		if d.netMode != "" {
 			hc.NetworkMode = container.NetworkMode(d.netMode)
 		}
-		if info, err := d.DockerAPI.Info(ctx); err == nil && info.CgroupDriver == "cgroupfs" {
-			// Place all buildkit containers inside this cgroup by default so limits can be attached
-			// to all build activity on the host.
-			hc.CgroupParent = "/docker/buildx"
-			if d.cgroupParent != "" {
-				hc.CgroupParent = d.cgroupParent
+		if info, err := d.DockerAPI.Info(ctx); err == nil {
+			if info.CgroupDriver == "cgroupfs" {
+				// Place all buildkit containers inside this cgroup by default so limits can be attached
+				// to all build activity on the host.
+				hc.CgroupParent = "/docker/buildx"
+				if d.cgroupParent != "" {
+					hc.CgroupParent = d.cgroupParent
+				}
+			}
+
+			secOpts, err := dockertypes.DecodeSecurityOptions(info.SecurityOptions)
+			if err != nil {
+				return err
+			}
+			for _, f := range secOpts {
+				if f.Name == "userns" {
+					hc.UsernsMode = "host"
+					break
+				}
 			}
+
 		}
 		_, err := d.DockerAPI.ContainerCreate(ctx, cfg, hc, &network.NetworkingConfig{}, nil, d.Name)
 		if err != nil {
diff --git a/driver/docker-container/factory.go b/driver/docker-container/factory.go
index 94649b9e..118d9c5a 100644
--- a/driver/docker-container/factory.go
+++ b/driver/docker-container/factory.go
@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/docker/buildx/driver"
-	dockertypes "github.com/docker/docker/api/types"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 )
@@ -41,20 +40,6 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
 		return nil, errors.Errorf("%s driver requires docker API access", f.Name())
 	}
 	d := &Driver{factory: f, InitConfig: cfg}
-	dockerInfo, err := cfg.DockerAPI.Info(ctx)
-	if err != nil {
-		return nil, err
-	}
-	secOpts, err := dockertypes.DecodeSecurityOptions(dockerInfo.SecurityOptions)
-	if err != nil {
-		return nil, err
-	}
-	for _, f := range secOpts {
-		if f.Name == "userns" {
-			d.userNSRemap = true
-			break
-		}
-	}
 	for k, v := range cfg.DriverOpts {
 		switch {
 		case k == "network":