allow multi-node push and imagetools to use custom registry config

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

store/storeutil/storeutil.go

@@ -1,13 +1,17 @@
 package storeutil
 
 import (
+	"bytes"
 	"os"
+	"strings"
 
 	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/imagetools"
+	"github.com/docker/buildx/util/resolver"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/context/docker"
+	buildkitdconfig "github.com/moby/buildkit/cmd/buildkitd/config"
 	"github.com/pkg/errors"
 )
 
@@ -112,5 +116,51 @@ func GetNodeGroup(txn *store.Txn, dockerCli command.Cli, name string) (*store.No
 func GetImageConfig(dockerCli command.Cli, ng *store.NodeGroup) (opt imagetools.Opt, err error) {
 	opt.Auth = dockerCli.ConfigFile()
 
+	if ng == nil || len(ng.Nodes) == 0 {
+		return opt, nil
+	}
+
+	files := ng.Nodes[0].Files
+
+	dt, ok := files["buildkitd.toml"]
+	if !ok {
+		return opt, nil
+	}
+
+	config, err := buildkitdconfig.Load(bytes.NewReader(dt))
+	if err != nil {
+		return opt, err
+	}
+
+	regconfig := make(map[string]resolver.RegistryConfig)
+
+	for k, v := range config.Registries {
+		rc := resolver.RegistryConfig{
+			Mirrors:   v.Mirrors,
+			PlainHTTP: v.PlainHTTP,
+			Insecure:  v.Insecure,
+		}
+		for _, ca := range v.RootCAs {
+			dt, ok := files[strings.TrimPrefix(ca, confutil.DefaultBuildKitConfigDir+"/")]
+			if ok {
+				rc.RootCAs = append(rc.RootCAs, dt)
+			}
+		}
+
+		for _, kp := range v.KeyPairs {
+			key, keyok := files[strings.TrimPrefix(kp.Key, confutil.DefaultBuildKitConfigDir+"/")]
+			cert, certok := files[strings.TrimPrefix(kp.Certificate, confutil.DefaultBuildKitConfigDir+"/")]
+			if keyok && certok {
+				rc.KeyPairs = append(rc.KeyPairs, resolver.TLSKeyPair{
+					Key:         key,
+					Certificate: cert,
+				})
+			}
+		}
+		regconfig[k] = rc
+	}
+
+	opt.RegistryConfig = regconfig
+
 	return opt, nil
 }

util/imagetools/inspect.go

@@ -10,9 +10,10 @@ import (
 
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/docker/buildx/util/resolver"
 	clitypes "github.com/docker/cli/cli/config/types"
 	"github.com/docker/distribution/reference"
-	registryconfig "github.com/moby/buildkit/util/resolver/config"
+	"github.com/moby/buildkit/util/tracing"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -22,23 +23,34 @@ type Auth interface {
 
 type Opt struct {
 	Auth           Auth
-	RegistryConfig map[string]registryconfig.RegistryConfig
+	RegistryConfig map[string]resolver.RegistryConfig
 }
 
 type Resolver struct {
-	auth docker.Authorizer
+	auth  docker.Authorizer
+	hosts docker.RegistryHosts
 }
 
 func New(opt Opt) *Resolver {
 	return &Resolver{
-		auth: docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(opt.Auth)), docker.WithAuthClient(http.DefaultClient)),
+		auth:  docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(opt.Auth)), docker.WithAuthClient(http.DefaultClient)),
+		hosts: resolver.NewRegistryConfig(opt.RegistryConfig),
 	}
 }
 
 func (r *Resolver) resolver() remotes.Resolver {
 	return docker.NewResolver(docker.ResolverOptions{
-		Authorizer: r.auth,
+		Hosts: func(domain string) ([]docker.RegistryHost, error) {
-		Client:     http.DefaultClient,
+			res, err := r.hosts(domain)
+			if err != nil {
+				return nil, err
+			}
+			for i := range res {
+				res[i].Authorizer = r.auth
+			}
+			return res, nil
+		},
+		Client: tracing.DefaultClient,
 	})
 }
 

util/resolver/resolver.go

@@ -0,0 +1,188 @@
+package resolver
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"runtime"
+	"time"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/moby/buildkit/util/tracing"
+	"github.com/pkg/errors"
+)
+
+// TODO: copied from buildkit/util/resolver. Update upstream so we can use the same code.
+
+type RegistryConfig struct {
+	Mirrors   []string
+	PlainHTTP *bool
+	Insecure  *bool
+	RootCAs   [][]byte
+	KeyPairs  []TLSKeyPair
+}
+
+type TLSKeyPair struct {
+	Key         []byte
+	Certificate []byte
+}
+
+func fillInsecureOpts(host string, c RegistryConfig, h docker.RegistryHost) ([]docker.RegistryHost, error) {
+	var hosts []docker.RegistryHost
+
+	tc, err := loadTLSConfig(c)
+	if err != nil {
+		return nil, err
+	}
+	var isHTTP bool
+
+	if c.PlainHTTP != nil && *c.PlainHTTP {
+		isHTTP = true
+	}
+	if c.PlainHTTP == nil {
+		if ok, _ := docker.MatchLocalhost(host); ok {
+			isHTTP = true
+		}
+	}
+
+	if isHTTP {
+		h2 := h
+		h2.Scheme = "http"
+		hosts = append(hosts, h2)
+	}
+	if c.Insecure != nil && *c.Insecure {
+		h2 := h
+		transport := newDefaultTransport()
+		transport.TLSClientConfig = tc
+		h2.Client = &http.Client{
+			Transport: tracing.NewTransport(transport),
+		}
+		tc.InsecureSkipVerify = true
+		hosts = append(hosts, h2)
+	}
+
+	if len(hosts) == 0 {
+		transport := newDefaultTransport()
+		transport.TLSClientConfig = tc
+
+		h.Client = &http.Client{
+			Transport: tracing.NewTransport(transport),
+		}
+		hosts = append(hosts, h)
+	}
+
+	return hosts, nil
+}
+
+func loadTLSConfig(c RegistryConfig) (*tls.Config, error) {
+	tc := &tls.Config{}
+	if len(c.RootCAs) > 0 {
+		systemPool, err := x509.SystemCertPool()
+		if err != nil {
+			if runtime.GOOS == "windows" {
+				systemPool = x509.NewCertPool()
+			} else {
+				return nil, errors.Wrapf(err, "unable to get system cert pool")
+			}
+		}
+		tc.RootCAs = systemPool
+	}
+
+	for _, p := range c.RootCAs {
+		tc.RootCAs.AppendCertsFromPEM(p)
+	}
+
+	for _, kp := range c.KeyPairs {
+		cert, err := tls.X509KeyPair(kp.Certificate, kp.Key)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate)
+		}
+		tc.Certificates = append(tc.Certificates, cert)
+	}
+	return tc, nil
+}
+
+// NewRegistryConfig converts registry config to docker.RegistryHosts callback
+func NewRegistryConfig(m map[string]RegistryConfig) docker.RegistryHosts {
+	return docker.Registries(
+		func(host string) ([]docker.RegistryHost, error) {
+			c, ok := m[host]
+			if !ok {
+				return nil, nil
+			}
+
+			var out []docker.RegistryHost
+
+			for _, mirror := range c.Mirrors {
+				h := docker.RegistryHost{
+					Scheme:       "https",
+					Client:       newDefaultClient(),
+					Host:         mirror,
+					Path:         "/v2",
+					Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve,
+				}
+
+				hosts, err := fillInsecureOpts(mirror, m[mirror], h)
+				if err != nil {
+					return nil, err
+				}
+
+				out = append(out, hosts...)
+			}
+
+			if host == "docker.io" {
+				host = "registry-1.docker.io"
+			}
+
+			h := docker.RegistryHost{
+				Scheme:       "https",
+				Client:       newDefaultClient(),
+				Host:         host,
+				Path:         "/v2",
+				Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve,
+			}
+
+			hosts, err := fillInsecureOpts(host, c, h)
+			if err != nil {
+				return nil, err
+			}
+
+			out = append(out, hosts...)
+			return out, nil
+		},
+		docker.ConfigureDefaultRegistries(
+			docker.WithClient(newDefaultClient()),
+			docker.WithPlainHTTP(docker.MatchLocalhost),
+		),
+	)
+}
+
+func newDefaultClient() *http.Client {
+	return &http.Client{
+		Transport: tracing.NewTransport(newDefaultTransport()),
+	}
+}
+
+// newDefaultTransport is for pull or push client
+//
+// NOTE: For push, there must disable http2 for https because the flow control
+// will limit data transfer. The net/http package doesn't provide http2 tunable
+// settings which limits push performance.
+//
+// REF: https://github.com/golang/go/issues/14077
+func newDefaultTransport() *http.Transport {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 60 * time.Second,
+		}).DialContext,
+		MaxIdleConns:          30,
+		IdleConnTimeout:       120 * time.Second,
+		MaxIdleConnsPerHost:   4,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 5 * time.Second,
+		TLSNextProto:          make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
+	}
+}

vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go

@@ -0,0 +1,115 @@
+package config
+
+import (
+	resolverconfig "github.com/moby/buildkit/util/resolver/config"
+)
+
+// Config provides containerd configuration data for the server
+type Config struct {
+	Debug bool `toml:"debug"`
+
+	// Root is the path to a directory where buildkit will store persistent data
+	Root string `toml:"root"`
+
+	// Entitlements e.g. security.insecure, network.host
+	Entitlements []string `toml:"insecure-entitlements"`
+	// GRPC configuration settings
+	GRPC GRPCConfig `toml:"grpc"`
+
+	Workers struct {
+		OCI        OCIConfig        `toml:"oci"`
+		Containerd ContainerdConfig `toml:"containerd"`
+	} `toml:"worker"`
+
+	Registries map[string]resolverconfig.RegistryConfig `toml:"registry"`
+
+	DNS *DNSConfig `toml:"dns"`
+}
+
+type GRPCConfig struct {
+	Address      []string `toml:"address"`
+	DebugAddress string   `toml:"debugAddress"`
+	UID          *int     `toml:"uid"`
+	GID          *int     `toml:"gid"`
+
+	TLS TLSConfig `toml:"tls"`
+	// MaxRecvMsgSize int    `toml:"max_recv_message_size"`
+	// MaxSendMsgSize int    `toml:"max_send_message_size"`
+}
+
+type TLSConfig struct {
+	Cert string `toml:"cert"`
+	Key  string `toml:"key"`
+	CA   string `toml:"ca"`
+}
+
+type GCConfig struct {
+	GC            *bool      `toml:"gc"`
+	GCKeepStorage int64      `toml:"gckeepstorage"`
+	GCPolicy      []GCPolicy `toml:"gcpolicy"`
+}
+
+type NetworkConfig struct {
+	Mode          string `toml:"networkMode"`
+	CNIConfigPath string `toml:"cniConfigPath"`
+	CNIBinaryPath string `toml:"cniBinaryPath"`
+}
+
+type OCIConfig struct {
+	Enabled          *bool             `toml:"enabled"`
+	Labels           map[string]string `toml:"labels"`
+	Platforms        []string          `toml:"platforms"`
+	Snapshotter      string            `toml:"snapshotter"`
+	Rootless         bool              `toml:"rootless"`
+	NoProcessSandbox bool              `toml:"noProcessSandbox"`
+	GCConfig
+	NetworkConfig
+	// UserRemapUnsupported is unsupported key for testing. The feature is
+	// incomplete and the intention is to make it default without config.
+	UserRemapUnsupported string `toml:"userRemapUnsupported"`
+	// For use in storing the OCI worker binary name that will replace buildkit-runc
+	Binary               string `toml:"binary"`
+	ProxySnapshotterPath string `toml:"proxySnapshotterPath"`
+
+	// StargzSnapshotterConfig is configuration for stargz snapshotter.
+	// We use a generic map[string]interface{} in order to remove the dependency
+	// on stargz snapshotter's config pkg from our config.
+	StargzSnapshotterConfig map[string]interface{} `toml:"stargzSnapshotter"`
+
+	// ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers.
+	// The profile should already be loaded (by a higher level system) before creating a worker.
+	ApparmorProfile string `toml:"apparmor-profile"`
+
+	// MaxParallelism is the maximum number of parallel build steps that can be run at the same time.
+	MaxParallelism int `toml:"max-parallelism"`
+}
+
+type ContainerdConfig struct {
+	Address   string            `toml:"address"`
+	Enabled   *bool             `toml:"enabled"`
+	Labels    map[string]string `toml:"labels"`
+	Platforms []string          `toml:"platforms"`
+	Namespace string            `toml:"namespace"`
+	GCConfig
+	NetworkConfig
+	Snapshotter string `toml:"snapshotter"`
+
+	// ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers.
+	// The profile should already be loaded (by a higher level system) before creating a worker.
+	ApparmorProfile string `toml:"apparmor-profile"`
+
+	MaxParallelism int `toml:"max-parallelism"`
+}
+
+type GCPolicy struct {
+	All          bool     `toml:"all"`
+	KeepBytes    int64    `toml:"keepBytes"`
+	KeepDuration int64    `toml:"keepDuration"`
+	Filters      []string `toml:"filters"`
+}
+
+type DNSConfig struct {
+	Nameservers   []string `toml:"nameservers"`
+	Options       []string `toml:"options"`
+	SearchDomains []string `toml:"searchDomains"`
+}

vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy.go

@@ -0,0 +1,31 @@
+package config
+
+const defaultCap int64 = 2e9 // 2GB
+
+func DefaultGCPolicy(p string, keep int64) []GCPolicy {
+	if keep == 0 {
+		keep = DetectDefaultGCCap(p)
+	}
+	return []GCPolicy{
+		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
+		{
+			Filters:      []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
+			KeepDuration: 48 * 3600, // 48h
+			KeepBytes:    512 * 1e6, // 512MB
+		},
+		// remove any data not used for 60 days
+		{
+			KeepDuration: 60 * 24 * 3600, // 60d
+			KeepBytes:    keep,
+		},
+		// keep the unshared build cache under cap
+		{
+			KeepBytes: keep,
+		},
+		// if previous policies were insufficient start deleting internal data to keep build cache under cap
+		{
+			All:       true,
+			KeepBytes: keep,
+		},
+	}
+}

vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_unix.go

@@ -0,0 +1,17 @@
+// +build !windows
+
+package config
+
+import (
+	"syscall"
+)
+
+func DetectDefaultGCCap(root string) int64 {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return defaultCap
+	}
+	diskSize := int64(st.Bsize) * int64(st.Blocks)
+	avail := diskSize / 10
+	return (avail/(1<<30) + 1) * 1e9 // round up
+}

vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_windows.go

@@ -0,0 +1,7 @@
+// +build windows
+
+package config
+
+func DetectDefaultGCCap(root string) int64 {
+	return defaultCap
+}

vendor/github.com/moby/buildkit/cmd/buildkitd/config/load.go

@@ -0,0 +1,36 @@
+package config
+
+import (
+	"io"
+	"os"
+
+	"github.com/pelletier/go-toml"
+	"github.com/pkg/errors"
+)
+
+// Load loads buildkitd config
+func Load(r io.Reader) (Config, error) {
+	var c Config
+	t, err := toml.LoadReader(r)
+	if err != nil {
+		return c, errors.Wrap(err, "failed to parse config")
+	}
+	err = t.Unmarshal(&c)
+	if err != nil {
+		return c, errors.Wrap(err, "failed to parse config")
+	}
+	return c, nil
+}
+
+// LoadFile loads buildkitd config file
+func LoadFile(fp string) (Config, error) {
+	f, err := os.Open(fp)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return Config{}, nil
+		}
+		return Config{}, errors.Wrapf(err, "failed to load config from %s", fp)
+	}
+	defer f.Close()
+	return Load(f)
+}

vendor/modules.txt

@@ -295,6 +295,7 @@ github.com/moby/buildkit/client/buildid
 github.com/moby/buildkit/client/connhelper
 github.com/moby/buildkit/client/llb
 github.com/moby/buildkit/client/ociindex
+github.com/moby/buildkit/cmd/buildkitd/config
 github.com/moby/buildkit/frontend/gateway/client
 github.com/moby/buildkit/frontend/gateway/grpcclient
 github.com/moby/buildkit/frontend/gateway/pb