diff --git a/store/storeutil/storeutil.go b/store/storeutil/storeutil.go index ea97cb99..b87e3f48 100644 --- a/store/storeutil/storeutil.go +++ b/store/storeutil/storeutil.go @@ -1,13 +1,17 @@ package storeutil import ( + "bytes" "os" + "strings" "github.com/docker/buildx/store" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/imagetools" + "github.com/docker/buildx/util/resolver" "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/context/docker" + buildkitdconfig "github.com/moby/buildkit/cmd/buildkitd/config" "github.com/pkg/errors" ) @@ -112,5 +116,51 @@ func GetNodeGroup(txn *store.Txn, dockerCli command.Cli, name string) (*store.No func GetImageConfig(dockerCli command.Cli, ng *store.NodeGroup) (opt imagetools.Opt, err error) { opt.Auth = dockerCli.ConfigFile() + if ng == nil || len(ng.Nodes) == 0 { + return opt, nil + } + + files := ng.Nodes[0].Files + + dt, ok := files["buildkitd.toml"] + if !ok { + return opt, nil + } + + config, err := buildkitdconfig.Load(bytes.NewReader(dt)) + if err != nil { + return opt, err + } + + regconfig := make(map[string]resolver.RegistryConfig) + + for k, v := range config.Registries { + rc := resolver.RegistryConfig{ + Mirrors: v.Mirrors, + PlainHTTP: v.PlainHTTP, + Insecure: v.Insecure, + } + for _, ca := range v.RootCAs { + dt, ok := files[strings.TrimPrefix(ca, confutil.DefaultBuildKitConfigDir+"/")] + if ok { + rc.RootCAs = append(rc.RootCAs, dt) + } + } + + for _, kp := range v.KeyPairs { + key, keyok := files[strings.TrimPrefix(kp.Key, confutil.DefaultBuildKitConfigDir+"/")] + cert, certok := files[strings.TrimPrefix(kp.Certificate, confutil.DefaultBuildKitConfigDir+"/")] + if keyok && certok { + rc.KeyPairs = append(rc.KeyPairs, resolver.TLSKeyPair{ + Key: key, + Certificate: cert, + }) + } + } + regconfig[k] = rc + } + + opt.RegistryConfig = regconfig + return opt, nil } diff --git a/util/imagetools/inspect.go b/util/imagetools/inspect.go index d8ae52f7..97ae93c2 100644 --- a/util/imagetools/inspect.go +++ b/util/imagetools/inspect.go @@ -10,9 +10,10 @@ import ( "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" + "github.com/docker/buildx/util/resolver" clitypes "github.com/docker/cli/cli/config/types" "github.com/docker/distribution/reference" - registryconfig "github.com/moby/buildkit/util/resolver/config" + "github.com/moby/buildkit/util/tracing" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -22,23 +23,34 @@ type Auth interface { type Opt struct { Auth Auth - RegistryConfig map[string]registryconfig.RegistryConfig + RegistryConfig map[string]resolver.RegistryConfig } type Resolver struct { - auth docker.Authorizer + auth docker.Authorizer + hosts docker.RegistryHosts } func New(opt Opt) *Resolver { return &Resolver{ - auth: docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(opt.Auth)), docker.WithAuthClient(http.DefaultClient)), + auth: docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(opt.Auth)), docker.WithAuthClient(http.DefaultClient)), + hosts: resolver.NewRegistryConfig(opt.RegistryConfig), } } func (r *Resolver) resolver() remotes.Resolver { return docker.NewResolver(docker.ResolverOptions{ - Authorizer: r.auth, - Client: http.DefaultClient, + Hosts: func(domain string) ([]docker.RegistryHost, error) { + res, err := r.hosts(domain) + if err != nil { + return nil, err + } + for i := range res { + res[i].Authorizer = r.auth + } + return res, nil + }, + Client: tracing.DefaultClient, }) } diff --git a/util/resolver/resolver.go b/util/resolver/resolver.go new file mode 100644 index 00000000..1f0c4ba0 --- /dev/null +++ b/util/resolver/resolver.go @@ -0,0 +1,188 @@ +package resolver + +import ( + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "runtime" + "time" + + "github.com/containerd/containerd/remotes/docker" + "github.com/moby/buildkit/util/tracing" + "github.com/pkg/errors" +) + +// TODO: copied from buildkit/util/resolver. Update upstream so we can use the same code. + +type RegistryConfig struct { + Mirrors []string + PlainHTTP *bool + Insecure *bool + RootCAs [][]byte + KeyPairs []TLSKeyPair +} + +type TLSKeyPair struct { + Key []byte + Certificate []byte +} + +func fillInsecureOpts(host string, c RegistryConfig, h docker.RegistryHost) ([]docker.RegistryHost, error) { + var hosts []docker.RegistryHost + + tc, err := loadTLSConfig(c) + if err != nil { + return nil, err + } + var isHTTP bool + + if c.PlainHTTP != nil && *c.PlainHTTP { + isHTTP = true + } + if c.PlainHTTP == nil { + if ok, _ := docker.MatchLocalhost(host); ok { + isHTTP = true + } + } + + if isHTTP { + h2 := h + h2.Scheme = "http" + hosts = append(hosts, h2) + } + if c.Insecure != nil && *c.Insecure { + h2 := h + transport := newDefaultTransport() + transport.TLSClientConfig = tc + h2.Client = &http.Client{ + Transport: tracing.NewTransport(transport), + } + tc.InsecureSkipVerify = true + hosts = append(hosts, h2) + } + + if len(hosts) == 0 { + transport := newDefaultTransport() + transport.TLSClientConfig = tc + + h.Client = &http.Client{ + Transport: tracing.NewTransport(transport), + } + hosts = append(hosts, h) + } + + return hosts, nil +} + +func loadTLSConfig(c RegistryConfig) (*tls.Config, error) { + tc := &tls.Config{} + if len(c.RootCAs) > 0 { + systemPool, err := x509.SystemCertPool() + if err != nil { + if runtime.GOOS == "windows" { + systemPool = x509.NewCertPool() + } else { + return nil, errors.Wrapf(err, "unable to get system cert pool") + } + } + tc.RootCAs = systemPool + } + + for _, p := range c.RootCAs { + tc.RootCAs.AppendCertsFromPEM(p) + } + + for _, kp := range c.KeyPairs { + cert, err := tls.X509KeyPair(kp.Certificate, kp.Key) + if err != nil { + return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate) + } + tc.Certificates = append(tc.Certificates, cert) + } + return tc, nil +} + +// NewRegistryConfig converts registry config to docker.RegistryHosts callback +func NewRegistryConfig(m map[string]RegistryConfig) docker.RegistryHosts { + return docker.Registries( + func(host string) ([]docker.RegistryHost, error) { + c, ok := m[host] + if !ok { + return nil, nil + } + + var out []docker.RegistryHost + + for _, mirror := range c.Mirrors { + h := docker.RegistryHost{ + Scheme: "https", + Client: newDefaultClient(), + Host: mirror, + Path: "/v2", + Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve, + } + + hosts, err := fillInsecureOpts(mirror, m[mirror], h) + if err != nil { + return nil, err + } + + out = append(out, hosts...) + } + + if host == "docker.io" { + host = "registry-1.docker.io" + } + + h := docker.RegistryHost{ + Scheme: "https", + Client: newDefaultClient(), + Host: host, + Path: "/v2", + Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve, + } + + hosts, err := fillInsecureOpts(host, c, h) + if err != nil { + return nil, err + } + + out = append(out, hosts...) + return out, nil + }, + docker.ConfigureDefaultRegistries( + docker.WithClient(newDefaultClient()), + docker.WithPlainHTTP(docker.MatchLocalhost), + ), + ) +} + +func newDefaultClient() *http.Client { + return &http.Client{ + Transport: tracing.NewTransport(newDefaultTransport()), + } +} + +// newDefaultTransport is for pull or push client +// +// NOTE: For push, there must disable http2 for https because the flow control +// will limit data transfer. The net/http package doesn't provide http2 tunable +// settings which limits push performance. +// +// REF: https://github.com/golang/go/issues/14077 +func newDefaultTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 60 * time.Second, + }).DialContext, + MaxIdleConns: 30, + IdleConnTimeout: 120 * time.Second, + MaxIdleConnsPerHost: 4, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 5 * time.Second, + TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), + } +} diff --git a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go new file mode 100644 index 00000000..311e0188 --- /dev/null +++ b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/config.go @@ -0,0 +1,115 @@ +package config + +import ( + resolverconfig "github.com/moby/buildkit/util/resolver/config" +) + +// Config provides containerd configuration data for the server +type Config struct { + Debug bool `toml:"debug"` + + // Root is the path to a directory where buildkit will store persistent data + Root string `toml:"root"` + + // Entitlements e.g. security.insecure, network.host + Entitlements []string `toml:"insecure-entitlements"` + // GRPC configuration settings + GRPC GRPCConfig `toml:"grpc"` + + Workers struct { + OCI OCIConfig `toml:"oci"` + Containerd ContainerdConfig `toml:"containerd"` + } `toml:"worker"` + + Registries map[string]resolverconfig.RegistryConfig `toml:"registry"` + + DNS *DNSConfig `toml:"dns"` +} + +type GRPCConfig struct { + Address []string `toml:"address"` + DebugAddress string `toml:"debugAddress"` + UID *int `toml:"uid"` + GID *int `toml:"gid"` + + TLS TLSConfig `toml:"tls"` + // MaxRecvMsgSize int `toml:"max_recv_message_size"` + // MaxSendMsgSize int `toml:"max_send_message_size"` +} + +type TLSConfig struct { + Cert string `toml:"cert"` + Key string `toml:"key"` + CA string `toml:"ca"` +} + +type GCConfig struct { + GC *bool `toml:"gc"` + GCKeepStorage int64 `toml:"gckeepstorage"` + GCPolicy []GCPolicy `toml:"gcpolicy"` +} + +type NetworkConfig struct { + Mode string `toml:"networkMode"` + CNIConfigPath string `toml:"cniConfigPath"` + CNIBinaryPath string `toml:"cniBinaryPath"` +} + +type OCIConfig struct { + Enabled *bool `toml:"enabled"` + Labels map[string]string `toml:"labels"` + Platforms []string `toml:"platforms"` + Snapshotter string `toml:"snapshotter"` + Rootless bool `toml:"rootless"` + NoProcessSandbox bool `toml:"noProcessSandbox"` + GCConfig + NetworkConfig + // UserRemapUnsupported is unsupported key for testing. The feature is + // incomplete and the intention is to make it default without config. + UserRemapUnsupported string `toml:"userRemapUnsupported"` + // For use in storing the OCI worker binary name that will replace buildkit-runc + Binary string `toml:"binary"` + ProxySnapshotterPath string `toml:"proxySnapshotterPath"` + + // StargzSnapshotterConfig is configuration for stargz snapshotter. + // We use a generic map[string]interface{} in order to remove the dependency + // on stargz snapshotter's config pkg from our config. + StargzSnapshotterConfig map[string]interface{} `toml:"stargzSnapshotter"` + + // ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers. + // The profile should already be loaded (by a higher level system) before creating a worker. + ApparmorProfile string `toml:"apparmor-profile"` + + // MaxParallelism is the maximum number of parallel build steps that can be run at the same time. + MaxParallelism int `toml:"max-parallelism"` +} + +type ContainerdConfig struct { + Address string `toml:"address"` + Enabled *bool `toml:"enabled"` + Labels map[string]string `toml:"labels"` + Platforms []string `toml:"platforms"` + Namespace string `toml:"namespace"` + GCConfig + NetworkConfig + Snapshotter string `toml:"snapshotter"` + + // ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers. + // The profile should already be loaded (by a higher level system) before creating a worker. + ApparmorProfile string `toml:"apparmor-profile"` + + MaxParallelism int `toml:"max-parallelism"` +} + +type GCPolicy struct { + All bool `toml:"all"` + KeepBytes int64 `toml:"keepBytes"` + KeepDuration int64 `toml:"keepDuration"` + Filters []string `toml:"filters"` +} + +type DNSConfig struct { + Nameservers []string `toml:"nameservers"` + Options []string `toml:"options"` + SearchDomains []string `toml:"searchDomains"` +} diff --git a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy.go b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy.go new file mode 100644 index 00000000..6f3f1978 --- /dev/null +++ b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy.go @@ -0,0 +1,31 @@ +package config + +const defaultCap int64 = 2e9 // 2GB + +func DefaultGCPolicy(p string, keep int64) []GCPolicy { + if keep == 0 { + keep = DetectDefaultGCCap(p) + } + return []GCPolicy{ + // if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days + { + Filters: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"}, + KeepDuration: 48 * 3600, // 48h + KeepBytes: 512 * 1e6, // 512MB + }, + // remove any data not used for 60 days + { + KeepDuration: 60 * 24 * 3600, // 60d + KeepBytes: keep, + }, + // keep the unshared build cache under cap + { + KeepBytes: keep, + }, + // if previous policies were insufficient start deleting internal data to keep build cache under cap + { + All: true, + KeepBytes: keep, + }, + } +} diff --git a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_unix.go b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_unix.go new file mode 100644 index 00000000..e022fba8 --- /dev/null +++ b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_unix.go @@ -0,0 +1,17 @@ +// +build !windows + +package config + +import ( + "syscall" +) + +func DetectDefaultGCCap(root string) int64 { + var st syscall.Statfs_t + if err := syscall.Statfs(root, &st); err != nil { + return defaultCap + } + diskSize := int64(st.Bsize) * int64(st.Blocks) + avail := diskSize / 10 + return (avail/(1<<30) + 1) * 1e9 // round up +} diff --git a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_windows.go b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_windows.go new file mode 100644 index 00000000..81d4b82c --- /dev/null +++ b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/gcpolicy_windows.go @@ -0,0 +1,7 @@ +// +build windows + +package config + +func DetectDefaultGCCap(root string) int64 { + return defaultCap +} diff --git a/vendor/github.com/moby/buildkit/cmd/buildkitd/config/load.go b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/load.go new file mode 100644 index 00000000..46e3dafb --- /dev/null +++ b/vendor/github.com/moby/buildkit/cmd/buildkitd/config/load.go @@ -0,0 +1,36 @@ +package config + +import ( + "io" + "os" + + "github.com/pelletier/go-toml" + "github.com/pkg/errors" +) + +// Load loads buildkitd config +func Load(r io.Reader) (Config, error) { + var c Config + t, err := toml.LoadReader(r) + if err != nil { + return c, errors.Wrap(err, "failed to parse config") + } + err = t.Unmarshal(&c) + if err != nil { + return c, errors.Wrap(err, "failed to parse config") + } + return c, nil +} + +// LoadFile loads buildkitd config file +func LoadFile(fp string) (Config, error) { + f, err := os.Open(fp) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Config{}, nil + } + return Config{}, errors.Wrapf(err, "failed to load config from %s", fp) + } + defer f.Close() + return Load(f) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 26161a92..23aa87f9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -295,6 +295,7 @@ github.com/moby/buildkit/client/buildid github.com/moby/buildkit/client/connhelper github.com/moby/buildkit/client/llb github.com/moby/buildkit/client/ociindex +github.com/moby/buildkit/cmd/buildkitd/config github.com/moby/buildkit/frontend/gateway/client github.com/moby/buildkit/frontend/gateway/grpcclient github.com/moby/buildkit/frontend/gateway/pb