Merge pull request #1078 from jedevc/remote-driver

Add remote driver
pull/1086/head
Tõnis Tiigi 3 years ago committed by GitHub
commit 38f1138a45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -53,6 +53,7 @@ jobs:
- docker - docker
- docker-container - docker-container
- kubernetes - kubernetes
- remote
buildkit: buildkit:
- moby/buildkit:buildx-stable-1 - moby/buildkit:buildx-stable-1
- moby/buildkit:master - moby/buildkit:master
@ -68,6 +69,8 @@ jobs:
include: include:
- driver: kubernetes - driver: kubernetes
driver-opt: qemu.install=true driver-opt: qemu.install=true
- driver: remote
endpoint: tcp://localhost:1234
exclude: exclude:
- driver: docker - driver: docker
multi-node: mnode-true multi-node: mnode-true
@ -75,6 +78,10 @@ jobs:
buildkit-cfg: bkcfg-true buildkit-cfg: bkcfg-true
- driver: docker-container - driver: docker-container
multi-node: mnode-true multi-node: mnode-true
- driver: remote
multi-node: mnode-true
- driver: remote
buildkit-cfg: bkcfg-true
steps: steps:
- -
name: Checkout name: Checkout
@ -128,6 +135,15 @@ jobs:
if: matrix.driver == 'kubernetes' if: matrix.driver == 'kubernetes'
run: | run: |
kubectl get nodes kubectl get nodes
-
name: Launch remote buildkitd
if: matrix.driver == 'remote'
run: |
docker run -d --privileged \
--name=remote-buildkit \
-p 1234:1234 \
${{ matrix.buildkit }} \
--addr tcp://0.0.0.0:1234
- -
name: Test name: Test
run: | run: |
@ -136,4 +152,5 @@ jobs:
BUILDKIT_IMAGE: ${{ matrix.buildkit }} BUILDKIT_IMAGE: ${{ matrix.buildkit }}
DRIVER: ${{ matrix.driver }} DRIVER: ${{ matrix.driver }}
DRIVER_OPT: ${{ matrix.driver-opt }} DRIVER_OPT: ${{ matrix.driver-opt }}
ENDPOINT: ${{ matrix.endpoint }}
PLATFORMS: ${{ matrix.platforms }} PLATFORMS: ${{ matrix.platforms }}

@ -24,6 +24,7 @@ import (
_ "github.com/docker/buildx/driver/docker" _ "github.com/docker/buildx/driver/docker"
_ "github.com/docker/buildx/driver/docker-container" _ "github.com/docker/buildx/driver/docker-container"
_ "github.com/docker/buildx/driver/kubernetes" _ "github.com/docker/buildx/driver/kubernetes"
_ "github.com/docker/buildx/driver/remote"
) )
func init() { func init() {

@ -60,16 +60,26 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
} }
} }
buildkitHost := os.Getenv("BUILDKIT_HOST")
driverName := in.driver driverName := in.driver
if driverName == "" { if driverName == "" {
f, err := driver.GetDefaultFactory(ctx, dockerCli.Client(), true) if len(args) == 0 && buildkitHost != "" {
if err != nil { driverName = "remote"
return err } else {
} var arg string
if f == nil { if len(args) > 0 {
return errors.Errorf("no valid drivers found") arg = args[0]
}
f, err := driver.GetDefaultFactory(ctx, arg, dockerCli.Client(), true)
if err != nil {
return err
}
if f == nil {
return errors.Errorf("no valid drivers found")
}
driverName = f.Name()
} }
driverName = f.Name()
} }
if driver.GetFactory(driverName, true) == nil { if driver.GetFactory(driverName, true) == nil {
@ -129,44 +139,59 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
} }
var ep string var ep string
var setEp bool
if in.actionLeave { if in.actionLeave {
if err := ng.Leave(in.nodeName); err != nil { if err := ng.Leave(in.nodeName); err != nil {
return err return err
} }
} else { } else {
if len(args) > 0 { switch {
case driverName == "kubernetes":
// naming endpoint to make --append works
ep = (&url.URL{
Scheme: driverName,
Path: "/" + in.name,
RawQuery: (&url.Values{
"deployment": {in.nodeName},
"kubeconfig": {os.Getenv("KUBECONFIG")},
}).Encode(),
}).String()
setEp = false
case driverName == "remote":
if len(args) > 0 {
ep = args[0]
} else if buildkitHost != "" {
ep = buildkitHost
} else {
return errors.Errorf("no remote endpoint provided")
}
ep, err = validateBuildkitEndpoint(ep)
if err != nil {
return err
}
setEp = true
case len(args) > 0:
ep, err = validateEndpoint(dockerCli, args[0]) ep, err = validateEndpoint(dockerCli, args[0])
if err != nil { if err != nil {
return err return err
} }
} else { setEp = true
default:
if dockerCli.CurrentContext() == "default" && dockerCli.DockerEndpoint().TLSData != nil { if dockerCli.CurrentContext() == "default" && dockerCli.DockerEndpoint().TLSData != nil {
return errors.Errorf("could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`") return errors.Errorf("could not create a builder instance with TLS data loaded from environment. Please use `docker context create <context-name>` to create a context for current environment and then create a builder instance with `docker buildx create <context-name>`")
} }
ep, err = storeutil.GetCurrentEndpoint(dockerCli) ep, err = storeutil.GetCurrentEndpoint(dockerCli)
if err != nil { if err != nil {
return err return err
} }
} setEp = false
if in.driver == "kubernetes" {
// naming endpoint to make --append works
ep = (&url.URL{
Scheme: in.driver,
Path: "/" + in.name,
RawQuery: (&url.Values{
"deployment": {in.nodeName},
"kubeconfig": {os.Getenv("KUBECONFIG")},
}).Encode(),
}).String()
} }
m, err := csvToMap(in.driverOpts) m, err := csvToMap(in.driverOpts)
if err != nil { if err != nil {
return err return err
} }
if err := ng.Update(in.nodeName, ep, in.platform, len(args) > 0, in.actionAppend, flags, in.configFile, m); err != nil { if err := ng.Update(in.nodeName, ep, in.platform, setEp, in.actionAppend, flags, in.configFile, m); err != nil {
return err return err
} }
} }

@ -41,6 +41,18 @@ func validateEndpoint(dockerCli command.Cli, ep string) (string, error) {
return h, nil return h, nil
} }
// validateBuildkitEndpoint validates that endpoint is a valid buildkit host
func validateBuildkitEndpoint(ep string) (string, error) {
endpoint, err := url.Parse(ep)
if err != nil {
return "", errors.Wrapf(err, "failed to parse endpoint %s", ep)
}
if endpoint.Scheme != "tcp" && endpoint.Scheme != "unix" {
return "", errors.Errorf("unrecognized url scheme %s", endpoint.Scheme)
}
return ep, nil
}
// driversForNodeGroup returns drivers for a nodegroup instance // driversForNodeGroup returns drivers for a nodegroup instance
func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, contextPathHash string) ([]build.DriverInfo, error) { func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, contextPathHash string) ([]build.DriverInfo, error) {
eg, _ := errgroup.WithContext(ctx) eg, _ := errgroup.WithContext(ctx)
@ -54,11 +66,12 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
return nil, errors.Errorf("failed to find driver %q", f) return nil, errors.Errorf("failed to find driver %q", f)
} }
} else { } else {
dockerapi, err := clientForEndpoint(dockerCli, ng.Nodes[0].Endpoint) ep := ng.Nodes[0].Endpoint
dockerapi, err := clientForEndpoint(dockerCli, ep)
if err != nil { if err != nil {
return nil, err return nil, err
} }
f, err = driver.GetDefaultFactory(ctx, dockerapi, false) f, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -80,6 +93,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
defer func() { defer func() {
dis[i] = di dis[i] = di
}() }()
dockerapi, err := clientForEndpoint(dockerCli, n.Endpoint) dockerapi, err := clientForEndpoint(dockerCli, n.Endpoint)
if err != nil { if err != nil {
di.Err = err di.Err = err
@ -118,7 +132,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
} }
} }
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash) d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, n.Endpoint, dockerapi, imageopt.Auth, kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash)
if err != nil { if err != nil {
di.Err = err di.Err = err
return nil return nil
@ -259,7 +273,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b
return nil, err return nil, err
} }
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), imageopt.Auth, nil, nil, nil, nil, nil, contextPathHash) d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, "", dockerCli.Client(), imageopt.Auth, nil, nil, nil, nil, nil, contextPathHash)
if err != nil { if err != nil {
return nil, err return nil, err
} }

@ -29,7 +29,7 @@ func (*factory) Usage() string {
return "docker-container" return "docker-container"
} }
func (*factory) Priority(ctx context.Context, api dockerclient.APIClient) int { func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
if api == nil { if api == nil {
return priorityUnsupported return priorityUnsupported
} }

@ -26,7 +26,7 @@ func (*factory) Usage() string {
return "docker" return "docker"
} }
func (*factory) Priority(ctx context.Context, api dockerclient.APIClient) int { func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
if api == nil { if api == nil {
return priorityUnsupported return priorityUnsupported
} }

@ -34,7 +34,7 @@ func (*factory) Usage() string {
return DriverName return DriverName
} }
func (*factory) Priority(ctx context.Context, api dockerclient.APIClient) int { func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
if api == nil { if api == nil {
return priorityUnsupported return priorityUnsupported
} }

@ -18,7 +18,7 @@ import (
type Factory interface { type Factory interface {
Name() string Name() string
Usage() string Usage() string
Priority(context.Context, dockerclient.APIClient) int Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int
New(ctx context.Context, cfg InitConfig) (Driver, error) New(ctx context.Context, cfg InitConfig) (Driver, error)
AllowsInstances() bool AllowsInstances() bool
} }
@ -50,6 +50,7 @@ func (k KubeClientConfigInCluster) Namespace() (string, bool, error) {
type InitConfig struct { type InitConfig struct {
// This object needs updates to be generic for different drivers // This object needs updates to be generic for different drivers
Name string Name string
EndpointAddr string
DockerAPI dockerclient.APIClient DockerAPI dockerclient.APIClient
KubeClientConfig KubeClientConfig KubeClientConfig KubeClientConfig
BuildkitFlags []string BuildkitFlags []string
@ -70,7 +71,7 @@ func Register(f Factory) {
drivers[f.Name()] = f drivers[f.Name()] = f
} }
func GetDefaultFactory(ctx context.Context, c dockerclient.APIClient, instanceRequired bool) (Factory, error) { func GetDefaultFactory(ctx context.Context, ep string, c dockerclient.APIClient, instanceRequired bool) (Factory, error) {
if len(drivers) == 0 { if len(drivers) == 0 {
return nil, errors.Errorf("no drivers available") return nil, errors.Errorf("no drivers available")
} }
@ -83,7 +84,7 @@ func GetDefaultFactory(ctx context.Context, c dockerclient.APIClient, instanceRe
if instanceRequired && !f.AllowsInstances() { if instanceRequired && !f.AllowsInstances() {
continue continue
} }
dd = append(dd, p{f: f, priority: f.Priority(ctx, c)}) dd = append(dd, p{f: f, priority: f.Priority(ctx, ep, c)})
} }
sort.Slice(dd, func(i, j int) bool { sort.Slice(dd, func(i, j int) bool {
return dd[i].priority < dd[j].priority return dd[i].priority < dd[j].priority
@ -103,8 +104,9 @@ func GetFactory(name string, instanceRequired bool) Factory {
return nil return nil
} }
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) { func GetDriver(ctx context.Context, name string, f Factory, endpointAddr string, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
ic := InitConfig{ ic := InitConfig{
EndpointAddr: endpointAddr,
DockerAPI: api, DockerAPI: api,
KubeClientConfig: kcc, KubeClientConfig: kcc,
Name: name, Name: name,
@ -117,7 +119,7 @@ func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.API
} }
if f == nil { if f == nil {
var err error var err error
f, err = GetDefaultFactory(ctx, api, false) f, err = GetDefaultFactory(ctx, endpointAddr, api, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }

@ -0,0 +1,80 @@
package remote
import (
"context"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
type Driver struct {
factory driver.Factory
driver.InitConfig
*tlsOpts
}
type tlsOpts struct {
serverName string
caCert string
cert string
key string
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return nil
}
func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
c, err := d.Client(ctx)
if err != nil {
return nil, errors.Wrapf(driver.ErrNotConnecting, err.Error())
}
if _, err := c.ListWorkers(ctx); err != nil {
return nil, errors.Wrapf(driver.ErrNotConnecting, err.Error())
}
return &driver.Info{
Status: driver.Running,
}, nil
}
func (d *Driver) Stop(ctx context.Context, force bool) error {
return nil
}
func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error {
return nil
}
func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
opts := []client.ClientOpt{}
if d.tlsOpts != nil {
opts = append(opts, client.WithCredentials(d.tlsOpts.serverName, d.tlsOpts.caCert, d.tlsOpts.cert, d.tlsOpts.key))
}
return client.New(ctx, d.InitConfig.EndpointAddr, opts...)
}
func (d *Driver) Features() map[driver.Feature]bool {
return map[driver.Feature]bool{
driver.OCIExporter: true,
driver.DockerExporter: false,
driver.CacheExport: true,
driver.MultiPlatform: true,
}
}
func (d *Driver) Factory() driver.Factory {
return d.factory
}
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Config() driver.InitConfig {
return d.InitConfig
}

@ -0,0 +1,115 @@
package remote
import (
"context"
"net/url"
"path/filepath"
"regexp"
"strings"
"github.com/docker/buildx/driver"
dockerclient "github.com/docker/docker/client"
"github.com/pkg/errors"
)
const prioritySupported = 20
const priorityUnsupported = 90
var schemeRegexp = regexp.MustCompile("^(tcp|unix)://")
func init() {
driver.Register(&factory{})
}
type factory struct {
}
func (*factory) Name() string {
return "remote"
}
func (*factory) Usage() string {
return "remote"
}
func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient) int {
if schemeRegexp.MatchString(endpoint) {
return prioritySupported
}
return priorityUnsupported
}
func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver, error) {
if len(cfg.Files) > 0 {
return nil, errors.Errorf("setting config file is not supported for remote driver")
}
if len(cfg.BuildkitFlags) > 0 {
return nil, errors.Errorf("setting buildkit flags is not supported for remote driver")
}
d := &Driver{
factory: f,
InitConfig: cfg,
}
tls := &tlsOpts{}
tlsEnabled := false
for k, v := range cfg.DriverOpts {
switch k {
case "servername":
tls.serverName = v
tlsEnabled = true
case "cacert":
if !filepath.IsAbs(v) {
return nil, errors.Errorf("non-absolute path '%s' provided for %s", v, k)
}
tls.caCert = v
tlsEnabled = true
case "cert":
if !filepath.IsAbs(v) {
return nil, errors.Errorf("non-absolute path '%s' provided for %s", v, k)
}
tls.cert = v
tlsEnabled = true
case "key":
if !filepath.IsAbs(v) {
return nil, errors.Errorf("non-absolute path '%s' provided for %s", v, k)
}
tls.key = v
tlsEnabled = true
default:
return nil, errors.Errorf("invalid driver option %s for remote driver", k)
}
}
if tlsEnabled {
if tls.serverName == "" {
// guess servername as hostname of target address
uri, err := url.Parse(cfg.EndpointAddr)
if err != nil {
return nil, err
}
tls.serverName = uri.Hostname()
}
missing := []string{}
if tls.caCert == "" {
missing = append(missing, "cacert")
}
if tls.cert == "" {
missing = append(missing, "cert")
}
if tls.key == "" {
missing = append(missing, "key")
}
if len(missing) > 0 {
return nil, errors.Errorf("tls enabled, but missing keys %s", strings.Join(missing, ", "))
}
d.tlsOpts = tls
}
return d, nil
}
func (f *factory) AllowsInstances() bool {
return true
}

@ -7,6 +7,7 @@ set -eu -o pipefail
: ${BUILDKIT_CFG=} : ${BUILDKIT_CFG=}
: ${DRIVER=docker-container} : ${DRIVER=docker-container}
: ${DRIVER_OPT=} : ${DRIVER_OPT=}
: ${ENDPOINT=}
: ${MULTI_NODE=0} : ${MULTI_NODE=0}
: ${PLATFORMS=linux/amd64,linux/arm64} : ${PLATFORMS=linux/amd64,linux/arm64}
@ -34,9 +35,11 @@ else
buildPlatformFlag=--platform="${PLATFORMS}" buildPlatformFlag=--platform="${PLATFORMS}"
fi fi
driverOpt=image=${BUILDKIT_IMAGE} if [ "$DRIVER" != "remote" ]; then
driverOpt=${driverOpt:+"${driverOpt},"}image=${BUILDKIT_IMAGE}
fi
if [ -n "$DRIVER_OPT" ]; then if [ -n "$DRIVER_OPT" ]; then
driverOpt=$driverOpt,$DRIVER_OPT driverOpt=${driverOpt:+"${driverOpt},"}$DRIVER_OPT
fi fi
# create builder except for docker driver # create builder except for docker driver
@ -54,9 +57,10 @@ if [ "$DRIVER" != "docker" ]; then
buildxCmd create ${createFlags} \ buildxCmd create ${createFlags} \
--name="${builderName}" \ --name="${builderName}" \
--node="${builderName}-${platform/\//-}" \ --node="${builderName}-${platform/\//-}" \
--platform="${platform}" \
--driver="${DRIVER}" \ --driver="${DRIVER}" \
--driver-opt="${driverOpt}" \ ${driverOpt:+"--driver-opt=${driverOpt}"} \
--platform="${platform}" ${ENDPOINT}
firstNode=0 firstNode=0
done done
else else
@ -66,9 +70,10 @@ if [ "$DRIVER" != "docker" ]; then
fi fi
buildxCmd create ${createFlags} \ buildxCmd create ${createFlags} \
--name="${builderName}" \ --name="${builderName}" \
--platform="${PLATFORMS}" \
--driver="${DRIVER}" \ --driver="${DRIVER}" \
--driver-opt="${driverOpt}" \ ${driverOpt:+"--driver-opt=${driverOpt}"} \
--platform="${PLATFORMS}" ${ENDPOINT}
fi fi
fi fi

Loading…
Cancel
Save