allow multi-node push and imagetools to use custom registry config
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>pull/825/head
Tonis Tiigi
3 years ago
parent
88d0775692
commit
c62472121b
@ -0,0 +1,188 @@
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"github.com/containerd/containerd/remotes/docker"
|
||||
"github.com/moby/buildkit/util/tracing"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// TODO: copied from buildkit/util/resolver. Update upstream so we can use the same code.
|
||||
|
||||
type RegistryConfig struct {
|
||||
Mirrors []string
|
||||
PlainHTTP *bool
|
||||
Insecure *bool
|
||||
RootCAs [][]byte
|
||||
KeyPairs []TLSKeyPair
|
||||
}
|
||||
|
||||
type TLSKeyPair struct {
|
||||
Key []byte
|
||||
Certificate []byte
|
||||
}
|
||||
|
||||
func fillInsecureOpts(host string, c RegistryConfig, h docker.RegistryHost) ([]docker.RegistryHost, error) {
|
||||
var hosts []docker.RegistryHost
|
||||
|
||||
tc, err := loadTLSConfig(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var isHTTP bool
|
||||
|
||||
if c.PlainHTTP != nil && *c.PlainHTTP {
|
||||
isHTTP = true
|
||||
}
|
||||
if c.PlainHTTP == nil {
|
||||
if ok, _ := docker.MatchLocalhost(host); ok {
|
||||
isHTTP = true
|
||||
}
|
||||
}
|
||||
|
||||
if isHTTP {
|
||||
h2 := h
|
||||
h2.Scheme = "http"
|
||||
hosts = append(hosts, h2)
|
||||
}
|
||||
if c.Insecure != nil && *c.Insecure {
|
||||
h2 := h
|
||||
transport := newDefaultTransport()
|
||||
transport.TLSClientConfig = tc
|
||||
h2.Client = &http.Client{
|
||||
Transport: tracing.NewTransport(transport),
|
||||
}
|
||||
tc.InsecureSkipVerify = true
|
||||
hosts = append(hosts, h2)
|
||||
}
|
||||
|
||||
if len(hosts) == 0 {
|
||||
transport := newDefaultTransport()
|
||||
transport.TLSClientConfig = tc
|
||||
|
||||
h.Client = &http.Client{
|
||||
Transport: tracing.NewTransport(transport),
|
||||
}
|
||||
hosts = append(hosts, h)
|
||||
}
|
||||
|
||||
return hosts, nil
|
||||
}
|
||||
|
||||
func loadTLSConfig(c RegistryConfig) (*tls.Config, error) {
|
||||
tc := &tls.Config{}
|
||||
if len(c.RootCAs) > 0 {
|
||||
systemPool, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
if runtime.GOOS == "windows" {
|
||||
systemPool = x509.NewCertPool()
|
||||
} else {
|
||||
return nil, errors.Wrapf(err, "unable to get system cert pool")
|
||||
}
|
||||
}
|
||||
tc.RootCAs = systemPool
|
||||
}
|
||||
|
||||
for _, p := range c.RootCAs {
|
||||
tc.RootCAs.AppendCertsFromPEM(p)
|
||||
}
|
||||
|
||||
for _, kp := range c.KeyPairs {
|
||||
cert, err := tls.X509KeyPair(kp.Certificate, kp.Key)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate)
|
||||
}
|
||||
tc.Certificates = append(tc.Certificates, cert)
|
||||
}
|
||||
return tc, nil
|
||||
}
|
||||
|
||||
// NewRegistryConfig converts registry config to docker.RegistryHosts callback
|
||||
func NewRegistryConfig(m map[string]RegistryConfig) docker.RegistryHosts {
|
||||
return docker.Registries(
|
||||
func(host string) ([]docker.RegistryHost, error) {
|
||||
c, ok := m[host]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var out []docker.RegistryHost
|
||||
|
||||
for _, mirror := range c.Mirrors {
|
||||
h := docker.RegistryHost{
|
||||
Scheme: "https",
|
||||
Client: newDefaultClient(),
|
||||
Host: mirror,
|
||||
Path: "/v2",
|
||||
Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve,
|
||||
}
|
||||
|
||||
hosts, err := fillInsecureOpts(mirror, m[mirror], h)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out = append(out, hosts...)
|
||||
}
|
||||
|
||||
if host == "docker.io" {
|
||||
host = "registry-1.docker.io"
|
||||
}
|
||||
|
||||
h := docker.RegistryHost{
|
||||
Scheme: "https",
|
||||
Client: newDefaultClient(),
|
||||
Host: host,
|
||||
Path: "/v2",
|
||||
Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve,
|
||||
}
|
||||
|
||||
hosts, err := fillInsecureOpts(host, c, h)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out = append(out, hosts...)
|
||||
return out, nil
|
||||
},
|
||||
docker.ConfigureDefaultRegistries(
|
||||
docker.WithClient(newDefaultClient()),
|
||||
docker.WithPlainHTTP(docker.MatchLocalhost),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func newDefaultClient() *http.Client {
|
||||
return &http.Client{
|
||||
Transport: tracing.NewTransport(newDefaultTransport()),
|
||||
}
|
||||
}
|
||||
|
||||
// newDefaultTransport is for pull or push client
|
||||
//
|
||||
// NOTE: For push, there must disable http2 for https because the flow control
|
||||
// will limit data transfer. The net/http package doesn't provide http2 tunable
|
||||
// settings which limits push performance.
|
||||
//
|
||||
// REF: https://github.com/golang/go/issues/14077
|
||||
func newDefaultTransport() *http.Transport {
|
||||
return &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 60 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 30,
|
||||
IdleConnTimeout: 120 * time.Second,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 5 * time.Second,
|
||||
TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,115 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
resolverconfig "github.com/moby/buildkit/util/resolver/config"
|
||||
)
|
||||
|
||||
// Config provides containerd configuration data for the server
|
||||
type Config struct {
|
||||
Debug bool `toml:"debug"`
|
||||
|
||||
// Root is the path to a directory where buildkit will store persistent data
|
||||
Root string `toml:"root"`
|
||||
|
||||
// Entitlements e.g. security.insecure, network.host
|
||||
Entitlements []string `toml:"insecure-entitlements"`
|
||||
// GRPC configuration settings
|
||||
GRPC GRPCConfig `toml:"grpc"`
|
||||
|
||||
Workers struct {
|
||||
OCI OCIConfig `toml:"oci"`
|
||||
Containerd ContainerdConfig `toml:"containerd"`
|
||||
} `toml:"worker"`
|
||||
|
||||
Registries map[string]resolverconfig.RegistryConfig `toml:"registry"`
|
||||
|
||||
DNS *DNSConfig `toml:"dns"`
|
||||
}
|
||||
|
||||
type GRPCConfig struct {
|
||||
Address []string `toml:"address"`
|
||||
DebugAddress string `toml:"debugAddress"`
|
||||
UID *int `toml:"uid"`
|
||||
GID *int `toml:"gid"`
|
||||
|
||||
TLS TLSConfig `toml:"tls"`
|
||||
// MaxRecvMsgSize int `toml:"max_recv_message_size"`
|
||||
// MaxSendMsgSize int `toml:"max_send_message_size"`
|
||||
}
|
||||
|
||||
type TLSConfig struct {
|
||||
Cert string `toml:"cert"`
|
||||
Key string `toml:"key"`
|
||||
CA string `toml:"ca"`
|
||||
}
|
||||
|
||||
type GCConfig struct {
|
||||
GC *bool `toml:"gc"`
|
||||
GCKeepStorage int64 `toml:"gckeepstorage"`
|
||||
GCPolicy []GCPolicy `toml:"gcpolicy"`
|
||||
}
|
||||
|
||||
type NetworkConfig struct {
|
||||
Mode string `toml:"networkMode"`
|
||||
CNIConfigPath string `toml:"cniConfigPath"`
|
||||
CNIBinaryPath string `toml:"cniBinaryPath"`
|
||||
}
|
||||
|
||||
type OCIConfig struct {
|
||||
Enabled *bool `toml:"enabled"`
|
||||
Labels map[string]string `toml:"labels"`
|
||||
Platforms []string `toml:"platforms"`
|
||||
Snapshotter string `toml:"snapshotter"`
|
||||
Rootless bool `toml:"rootless"`
|
||||
NoProcessSandbox bool `toml:"noProcessSandbox"`
|
||||
GCConfig
|
||||
NetworkConfig
|
||||
// UserRemapUnsupported is unsupported key for testing. The feature is
|
||||
// incomplete and the intention is to make it default without config.
|
||||
UserRemapUnsupported string `toml:"userRemapUnsupported"`
|
||||
// For use in storing the OCI worker binary name that will replace buildkit-runc
|
||||
Binary string `toml:"binary"`
|
||||
ProxySnapshotterPath string `toml:"proxySnapshotterPath"`
|
||||
|
||||
// StargzSnapshotterConfig is configuration for stargz snapshotter.
|
||||
// We use a generic map[string]interface{} in order to remove the dependency
|
||||
// on stargz snapshotter's config pkg from our config.
|
||||
StargzSnapshotterConfig map[string]interface{} `toml:"stargzSnapshotter"`
|
||||
|
||||
// ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers.
|
||||
// The profile should already be loaded (by a higher level system) before creating a worker.
|
||||
ApparmorProfile string `toml:"apparmor-profile"`
|
||||
|
||||
// MaxParallelism is the maximum number of parallel build steps that can be run at the same time.
|
||||
MaxParallelism int `toml:"max-parallelism"`
|
||||
}
|
||||
|
||||
type ContainerdConfig struct {
|
||||
Address string `toml:"address"`
|
||||
Enabled *bool `toml:"enabled"`
|
||||
Labels map[string]string `toml:"labels"`
|
||||
Platforms []string `toml:"platforms"`
|
||||
Namespace string `toml:"namespace"`
|
||||
GCConfig
|
||||
NetworkConfig
|
||||
Snapshotter string `toml:"snapshotter"`
|
||||
|
||||
// ApparmorProfile is the name of the apparmor profile that should be used to constrain build containers.
|
||||
// The profile should already be loaded (by a higher level system) before creating a worker.
|
||||
ApparmorProfile string `toml:"apparmor-profile"`
|
||||
|
||||
MaxParallelism int `toml:"max-parallelism"`
|
||||
}
|
||||
|
||||
type GCPolicy struct {
|
||||
All bool `toml:"all"`
|
||||
KeepBytes int64 `toml:"keepBytes"`
|
||||
KeepDuration int64 `toml:"keepDuration"`
|
||||
Filters []string `toml:"filters"`
|
||||
}
|
||||
|
||||
type DNSConfig struct {
|
||||
Nameservers []string `toml:"nameservers"`
|
||||
Options []string `toml:"options"`
|
||||
SearchDomains []string `toml:"searchDomains"`
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package config
|
||||
|
||||
const defaultCap int64 = 2e9 // 2GB
|
||||
|
||||
func DefaultGCPolicy(p string, keep int64) []GCPolicy {
|
||||
if keep == 0 {
|
||||
keep = DetectDefaultGCCap(p)
|
||||
}
|
||||
return []GCPolicy{
|
||||
// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
|
||||
{
|
||||
Filters: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
|
||||
KeepDuration: 48 * 3600, // 48h
|
||||
KeepBytes: 512 * 1e6, // 512MB
|
||||
},
|
||||
// remove any data not used for 60 days
|
||||
{
|
||||
KeepDuration: 60 * 24 * 3600, // 60d
|
||||
KeepBytes: keep,
|
||||
},
|
||||
// keep the unshared build cache under cap
|
||||
{
|
||||
KeepBytes: keep,
|
||||
},
|
||||
// if previous policies were insufficient start deleting internal data to keep build cache under cap
|
||||
{
|
||||
All: true,
|
||||
KeepBytes: keep,
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
// +build !windows
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func DetectDefaultGCCap(root string) int64 {
|
||||
var st syscall.Statfs_t
|
||||
if err := syscall.Statfs(root, &st); err != nil {
|
||||
return defaultCap
|
||||
}
|
||||
diskSize := int64(st.Bsize) * int64(st.Blocks)
|
||||
avail := diskSize / 10
|
||||
return (avail/(1<<30) + 1) * 1e9 // round up
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
// +build windows
|
||||
|
||||
package config
|
||||
|
||||
func DetectDefaultGCCap(root string) int64 {
|
||||
return defaultCap
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/pelletier/go-toml"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Load loads buildkitd config
|
||||
func Load(r io.Reader) (Config, error) {
|
||||
var c Config
|
||||
t, err := toml.LoadReader(r)
|
||||
if err != nil {
|
||||
return c, errors.Wrap(err, "failed to parse config")
|
||||
}
|
||||
err = t.Unmarshal(&c)
|
||||
if err != nil {
|
||||
return c, errors.Wrap(err, "failed to parse config")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// LoadFile loads buildkitd config file
|
||||
func LoadFile(fp string) (Config, error) {
|
||||
f, err := os.Open(fp)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return Config{}, nil
|
||||
}
|
||||
return Config{}, errors.Wrapf(err, "failed to load config from %s", fp)
|
||||
}
|
||||
defer f.Close()
|
||||
return Load(f)
|
||||
}
|
||||
Loading…
Reference in New Issue