Merge pull request #398 from tonistiigi/remote-bake

bake: remote inputs support
pull/469/head
Tõnis Tiigi 4 years ago committed by GitHub
commit 080e9981c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -5,6 +5,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
"regexp"
"strconv" "strconv"
"strings" "strings"
@ -12,14 +13,55 @@ import (
"github.com/docker/buildx/util/platformutil" "github.com/docker/buildx/util/platformutil"
"github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/pkg/urlutil"
hcl "github.com/hashicorp/hcl/v2" hcl "github.com/hashicorp/hcl/v2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session/auth/authprovider" "github.com/moby/buildkit/session/auth/authprovider"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) { var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
type File struct {
Name string
Data []byte
}
func defaultFilenames() []string {
return []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
}
func ReadLocalFiles(names []string) ([]File, error) {
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
out := make([]File, 0, len(names))
for _, n := range names {
dt, err := ioutil.ReadFile(n)
if err != nil {
if isDefault && errors.Is(err, os.ErrNotExist) {
continue
}
return nil, err
}
out = append(out, File{Name: n, Data: dt})
}
return out, nil
}
func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
var c Config var c Config
for _, f := range files { for _, f := range files {
cfg, err := ParseFile(f) cfg, err := ParseFile(f.Data, f.Name)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -44,12 +86,7 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s
return m, nil return m, nil
} }
func ParseFile(fn string) (*Config, error) { func ParseFile(dt []byte, fn string) (*Config, error) {
dt, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}
fnl := strings.ToLower(fn) fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") { if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
return ParseCompose(dt) return ParseCompose(dt)
@ -336,20 +373,22 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set // Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"` Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"` Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"` Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"` DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"` Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"` Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"` Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"` CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"` CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"` Target *string `json:"target,omitempty" hcl:"target,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"` Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"` SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"` Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"` Outputs []string `json:"output,omitempty" hcl:"output,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"` Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README. // IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
} }
@ -363,10 +402,10 @@ func (t *Target) normalize() {
t.Outputs = removeDupes(t.Outputs) t.Outputs = removeDupes(t.Outputs)
} }
func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) { func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m)) m2 := make(map[string]build.Options, len(m))
for k, v := range m { for k, v := range m {
bo, err := toBuildOpt(v) bo, err := toBuildOpt(v, inp)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -375,7 +414,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
return m2, nil return m2, nil
} }
func toBuildOpt(t *Target) (*build.Options, error) { func updateContext(t *build.Inputs, inp *Input) {
if inp == nil || inp.State == nil {
return
}
if t.ContextPath == "." {
t.ContextPath = inp.URL
return
}
st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
t.ContextState = &st
}
func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
if v := t.Context; v != nil && *v == "-" { if v := t.Context; v != nil && *v == "-" {
return nil, errors.Errorf("context from stdin not allowed in bake") return nil, errors.Errorf("context from stdin not allowed in bake")
} }
@ -387,6 +438,7 @@ func toBuildOpt(t *Target) (*build.Options, error) {
if t.Context != nil { if t.Context != nil {
contextPath = *t.Context contextPath = *t.Context
} }
contextPath = path.Clean(contextPath)
dockerfilePath := "Dockerfile" dockerfilePath := "Dockerfile"
if t.Dockerfile != nil { if t.Dockerfile != nil {
dockerfilePath = *t.Dockerfile dockerfilePath = *t.Dockerfile
@ -405,11 +457,17 @@ func toBuildOpt(t *Target) (*build.Options, error) {
pull = *t.Pull pull = *t.Pull
} }
bi := build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
}
if t.DockerfileInline != nil {
bi.DockerfileInline = *t.DockerfileInline
}
updateContext(&bi, inp)
bo := &build.Options{ bo := &build.Options{
Inputs: build.Inputs{ Inputs: bi,
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
},
Tags: t.Tags, Tags: t.Tags,
BuildArgs: t.Args, BuildArgs: t.Args,
Labels: t.Labels, Labels: t.Labels,
@ -473,6 +531,9 @@ func merge(t1, t2 *Target) *Target {
if t2.Dockerfile != nil { if t2.Dockerfile != nil {
t1.Dockerfile = t2.Dockerfile t1.Dockerfile = t2.Dockerfile
} }
if t2.DockerfileInline != nil {
t1.DockerfileInline = t2.DockerfileInline
}
for k, v := range t2.Args { for k, v := range t2.Args {
if t1.Args == nil { if t1.Args == nil {
t1.Args = map[string]string{} t1.Args = map[string]string{}

@ -2,9 +2,7 @@ package bake
import ( import (
"context" "context"
"io/ioutil"
"os" "os"
"path/filepath"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -12,12 +10,10 @@ import (
func TestReadTargets(t *testing.T) { func TestReadTargets(t *testing.T) {
t.Parallel() t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "config.hcl") fp := File{
err = ioutil.WriteFile(fp, []byte(` Name: "config.hcl",
Data: []byte(`
target "webDEP" { target "webDEP" {
args = { args = {
VAR_INHERITED = "webDEP" VAR_INHERITED = "webDEP"
@ -32,13 +28,13 @@ target "webapp" {
VAR_BOTH = "webapp" VAR_BOTH = "webapp"
} }
inherits = ["webDEP"] inherits = ["webDEP"]
}`), 0600) }`),
require.NoError(t, err) }
ctx := context.TODO() ctx := context.TODO()
t.Run("NoOverrides", func(t *testing.T) { t.Run("NoOverrides", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil) m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 1, len(m)) require.Equal(t, 1, len(m))
@ -50,7 +46,7 @@ target "webapp" {
}) })
t.Run("InvalidTargetOverrides", func(t *testing.T) { t.Run("InvalidTargetOverrides", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"}) _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
require.NotNil(t, err) require.NotNil(t, err)
require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'") require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
}) })
@ -60,7 +56,7 @@ target "webapp" {
os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv") os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv")
defer os.Unsetenv("VAR_FROM_ENV" + t.Name()) defer os.Unsetenv("VAR_FROM_ENV" + t.Name())
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{ m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webapp.args.VAR_UNSET", "webapp.args.VAR_UNSET",
"webapp.args.VAR_EMPTY=", "webapp.args.VAR_EMPTY=",
"webapp.args.VAR_SET=bananas", "webapp.args.VAR_SET=bananas",
@ -89,7 +85,7 @@ target "webapp" {
// building leaf but overriding parent fields // building leaf but overriding parent fields
t.Run("parent", func(t *testing.T) { t.Run("parent", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{ m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webDEP.args.VAR_INHERITED=override", "webDEP.args.VAR_INHERITED=override",
"webDEP.args.VAR_BOTH=override", "webDEP.args.VAR_BOTH=override",
}) })
@ -100,23 +96,23 @@ target "webapp" {
}) })
t.Run("ContextOverride", func(t *testing.T) { t.Run("ContextOverride", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"}) _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
require.NotNil(t, err) require.NotNil(t, err)
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"}) m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, "foo", *m["webapp"].Context) require.Equal(t, "foo", *m["webapp"].Context)
}) })
t.Run("NoCacheOverride", func(t *testing.T) { t.Run("NoCacheOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"}) m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, false, *m["webapp"].NoCache) require.Equal(t, false, *m["webapp"].NoCache)
}) })
t.Run("PullOverride", func(t *testing.T) { t.Run("PullOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"}) m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, false, *m["webapp"].Pull) require.Equal(t, false, *m["webapp"].Pull)
}) })
@ -176,7 +172,7 @@ target "webapp" {
} }
for _, test := range cases { for _, test := range cases {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides) m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
test.check(t, m, err) test.check(t, m, err)
}) })
} }
@ -185,14 +181,11 @@ target "webapp" {
func TestReadTargetsCompose(t *testing.T) { func TestReadTargetsCompose(t *testing.T) {
t.Parallel() t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "docker-compose.yml")
err = ioutil.WriteFile(fp, []byte(`
version: "3"
fp := File{
Name: "docker-compose.yml",
Data: []byte(
`version: "3"
services: services:
db: db:
build: . build: .
@ -203,13 +196,13 @@ services:
dockerfile: Dockerfile.webapp dockerfile: Dockerfile.webapp
args: args:
buildno: 1 buildno: 1
`), 0600) `),
require.NoError(t, err) }
fp2 := filepath.Join(tmpdir, "docker-compose2.yml")
err = ioutil.WriteFile(fp2, []byte(`
version: "3"
fp2 := File{
Name: "docker-compose2.yml",
Data: []byte(
`version: "3"
services: services:
newservice: newservice:
build: . build: .
@ -217,12 +210,12 @@ services:
build: build:
args: args:
buildno2: 12 buildno2: 12
`), 0600) `),
require.NoError(t, err) }
ctx := context.TODO() ctx := context.TODO()
m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil) m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 3, len(m)) require.Equal(t, 3, len(m))

@ -107,6 +107,20 @@ type staticConfig struct {
} }
func ParseHCL(dt []byte, fn string) (_ *Config, err error) { func ParseHCL(dt []byte, fn string) (_ *Config, err error) {
if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
return parseHCL(dt, fn)
}
cfg, err := parseHCL(dt, fn+".hcl")
if err != nil {
cfg2, err2 := parseHCL(dt, fn+".json")
if err2 == nil {
return cfg2, nil
}
}
return cfg, err
}
func parseHCL(dt []byte, fn string) (_ *Config, err error) {
defer func() { defer func() {
err = formatHCLError(dt, err) err = formatHCLError(dt, err)
}() }()
@ -192,15 +206,17 @@ func formatHCLError(dt []byte, err error) error {
if d.Severity != hcl.DiagError { if d.Severity != hcl.DiagError {
continue continue
} }
src := errdefs.Source{ if d.Subject != nil {
Info: &pb.SourceInfo{ src := errdefs.Source{
Filename: d.Subject.Filename, Info: &pb.SourceInfo{
Data: dt, Filename: d.Subject.Filename,
}, Data: dt,
Ranges: []*pb.Range{toErrRange(d.Subject)}, },
Ranges: []*pb.Range{toErrRange(d.Subject)},
}
err = errdefs.WithSource(err, src)
break
} }
err = errdefs.WithSource(err, src)
break
} }
return err return err
} }

@ -0,0 +1,236 @@
package bake
import (
"archive/tar"
"bytes"
"context"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Input struct {
State *llb.State
URL string
}
func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
st, filename, ok := detectHttpContext(url)
if !ok {
st, ok = detectGitContext(url)
if !ok {
return nil, nil, errors.Errorf("not url context")
}
}
inp := &Input{State: st, URL: url}
var files []File
var di *build.DriverInfo
for _, d := range dis {
if d.Err == nil {
di = &d
continue
}
}
if di == nil {
return nil, nil, nil
}
c, err := driver.Boot(ctx, di.Driver, pw)
if err != nil {
return nil, nil, err
}
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
if filename != "" {
files, err = filesFromURLRef(ctx, c, ref, inp, filename, names)
} else {
files, err = filesFromRef(ctx, ref, names)
}
return nil, err
}, ch)
if err != nil {
return nil, nil, err
}
return files, inp, nil
}
func IsRemoteURL(url string) bool {
if _, _, ok := detectHttpContext(url); ok {
return true
}
if _, ok := detectGitContext(url); ok {
return true
}
return false
}
func detectHttpContext(url string) (*llb.State, string, bool) {
if httpPrefix.MatchString(url) {
httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context"))
return &httpContext, "context", true
}
return nil, "", false
}
func detectGitContext(ref string) (*llb.State, bool) {
found := false
if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
found = true
}
for _, prefix := range []string{"git://", "github.com/", "git@"} {
if strings.HasPrefix(ref, prefix) {
found = true
break
}
}
if !found {
return nil, false
}
parts := strings.SplitN(ref, "#", 2)
branch := ""
if len(parts) > 1 {
branch = parts[1]
}
gitOpts := []llb.GitOption{llb.WithCustomName("[internal] load git source " + ref)}
st := llb.Git(parts[0], branch, gitOpts...)
return &st, true
}
func isArchive(header []byte) bool {
for _, m := range [][]byte{
{0x42, 0x5A, 0x68}, // bzip2
{0x1F, 0x8B, 0x08}, // gzip
{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz
} {
if len(header) < len(m) {
continue
}
if bytes.Equal(m, header[:len(m)]) {
return true
}
}
r := tar.NewReader(bytes.NewBuffer(header))
_, err := r.Next()
return err == nil
}
func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) {
stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename})
if err != nil {
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
Range: &gwclient.FileRange{
Length: 1024,
},
})
if err != nil {
return nil, err
}
if isArchive(dt) {
bc := llb.Scratch().File(llb.Copy(inp.State, filename, "/", &llb.CopyInfo{
AttemptUnpack: true,
}))
inp.State = &bc
inp.URL = ""
def, err := bc.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
return filesFromRef(ctx, ref, names)
}
inp.State = nil
name := inp.URL
inp.URL = ""
if len(dt) > stat.Size() {
if stat.Size() > 1024*512 {
return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size")
}
dt, err = ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
})
if err != nil {
return nil, err
}
}
return []File{{Name: name, Data: dt}}, nil
}
func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) {
// TODO: auto-remove parent dir in needed
var files []File
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
for _, name := range names {
_, err := ref.StatFile(ctx, gwclient.StatRequest{Path: name})
if err != nil {
if isDefault {
continue
}
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name})
if err != nil {
return nil, err
}
files = append(files, File{Name: name, Data: dt})
}
return files, nil
}

@ -22,6 +22,7 @@ import (
dockerclient "github.com/docker/docker/client" dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/pkg/urlutil"
"github.com/moby/buildkit/client" "github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session" "github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/entitlements"
@ -61,9 +62,11 @@ type Options struct {
} }
type Inputs struct { type Inputs struct {
ContextPath string ContextPath string
DockerfilePath string DockerfilePath string
InStream io.Reader InStream io.Reader
ContextState *llb.State
DockerfileInline string
} }
type DriverInfo struct { type DriverInfo struct {
@ -173,7 +176,6 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
} }
func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) { func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
availablePlatforms := map[string]int{} availablePlatforms := map[string]int{}
for i, d := range drivers { for i, d := range drivers {
for _, p := range d.Platform { for _, p := range d.Platform {
@ -278,14 +280,7 @@ func toRepoOnly(in string) (string, error) {
return strings.Join(out, ","), nil return strings.Join(out, ","), nil
} }
func isDefaultMobyDriver(d driver.Driver) bool { func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Options, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
_, ok := d.(interface {
IsDefaultMobyDriver()
})
return ok
}
func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
defers := make([]func(), 0, 2) defers := make([]func(), 0, 2)
releaseF := func() { releaseF := func() {
for _, f := range defers { for _, f := range defers {
@ -336,15 +331,11 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
so.FrontendAttrs["multi-platform"] = "true" so.FrontendAttrs["multi-platform"] = "true"
} }
_, isDefaultMobyDriver := d.(interface {
IsDefaultMobyDriver()
})
switch len(opt.Exports) { switch len(opt.Exports) {
case 1: case 1:
// valid // valid
case 0: case 0:
if isDefaultMobyDriver && !noDefaultLoad() { if d.IsMobyDriver() && !noDefaultLoad() {
// backwards compat for docker driver only: // backwards compat for docker driver only:
// this ensures the build results in a docker image. // this ensures the build results in a docker image.
opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}} opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}}
@ -398,7 +389,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
} }
if e.Type == "docker" { if e.Type == "docker" {
if e.Output == nil { if e.Output == nil {
if isDefaultMobyDriver { if d.IsMobyDriver() {
e.Type = "image" e.Type = "image"
} else { } else {
w, cancel, err := dl(e.Attrs["context"]) w, cancel, err := dl(e.Attrs["context"])
@ -412,7 +403,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
return nil, nil, notSupported(d, driver.DockerExporter) return nil, nil, notSupported(d, driver.DockerExporter)
} }
} }
if e.Type == "image" && isDefaultMobyDriver { if e.Type == "image" && d.IsMobyDriver() {
opt.Exports[i].Type = "moby" opt.Exports[i].Type = "moby"
if e.Attrs["push"] != "" { if e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok { if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
@ -425,7 +416,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
so.Exports = opt.Exports so.Exports = opt.Exports
so.Session = opt.Session so.Session = opt.Session
releaseLoad, err := LoadInputs(opt.Inputs, &so) releaseLoad, err := LoadInputs(ctx, d, opt.Inputs, pw, &so)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -479,7 +470,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
return &so, releaseF, nil return &so, releaseF, nil
} }
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, pw progress.Writer) (resp map[string]*client.SolveResponse, err error) { func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
if len(drivers) == 0 { if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build") return nil, errors.Errorf("driver required for build")
} }
@ -491,7 +482,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
var noMobyDriver driver.Driver var noMobyDriver driver.Driver
for _, d := range drivers { for _, d := range drivers {
if !isDefaultMobyDriver(d.Driver) { if !d.Driver.IsMobyDriver() {
noMobyDriver = d.Driver noMobyDriver = d.Driver
break break
} }
@ -506,10 +497,8 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
} }
} }
m, clients, err := resolveDrivers(ctx, drivers, auth, opt, pw) m, clients, err := resolveDrivers(ctx, drivers, auth, opt, w)
if err != nil { if err != nil {
close(pw.Status())
<-pw.Done()
return nil, err return nil, err
} }
@ -522,7 +511,6 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
} }
}() }()
mw := progress.NewMultiWriter(pw)
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
for k, opt := range opt { for k, opt := range opt {
@ -530,8 +518,8 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
for i, dp := range m[k] { for i, dp := range m[k] {
d := drivers[dp.driverIndex].Driver d := drivers[dp.driverIndex].Driver
opt.Platforms = dp.platforms opt.Platforms = dp.platforms
so, release, err := toSolveOpt(d, multiDriver, opt, func(name string) (io.WriteCloser, func(), error) { so, release, err := toSolveOpt(ctx, d, multiDriver, opt, w, func(name string) (io.WriteCloser, func(), error) {
return newDockerLoader(ctx, docker, name, mw) return newDockerLoader(ctx, docker, name, w)
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -559,8 +547,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
var pushNames string var pushNames string
eg.Go(func() error { eg.Go(func() error {
pw := mw.WithPrefix("default", false) pw := progress.WithPrefix(w, "default", false)
defer close(pw.Status())
wg.Wait() wg.Wait()
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -663,23 +650,17 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
} }
func(i int, dp driverPair, so client.SolveOpt) { func(i int, dp driverPair, so client.SolveOpt) {
pw := mw.WithPrefix(k, multiTarget) pw := progress.WithPrefix(w, k, multiTarget)
c := clients[dp.driverIndex] c := clients[dp.driverIndex]
var statusCh chan *client.SolveStatus pw = progress.ResetTime(pw)
if pw != nil {
pw = progress.ResetTime(pw)
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
return pw.Err()
})
}
eg.Go(func() error { eg.Go(func() error {
defer wg.Done() defer wg.Done()
rr, err := c.Solve(ctx, nil, so, statusCh) ch, done := progress.NewChannel(pw)
defer func() { <-done }()
rr, err := c.Solve(ctx, nil, so, ch)
if err != nil { if err != nil {
return err return err
} }
@ -720,7 +701,7 @@ func createTempDockerfile(r io.Reader) (string, error) {
return dir, err return dir, err
} }
func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) { func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) {
if inp.ContextPath == "" { if inp.ContextPath == "" {
return nil, errors.New("please specify build context (e.g. \".\" for the current directory)") return nil, errors.New("please specify build context (e.g. \".\" for the current directory)")
} }
@ -736,6 +717,12 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
) )
switch { switch {
case inp.ContextState != nil:
if target.FrontendInputs == nil {
target.FrontendInputs = make(map[string]llb.State)
}
target.FrontendInputs["context"] = *inp.ContextState
target.FrontendInputs["dockerfile"] = *inp.ContextState
case inp.ContextPath == "-": case inp.ContextPath == "-":
if inp.DockerfilePath == "-" { if inp.DockerfilePath == "-" {
return nil, errStdinConflict return nil, errStdinConflict
@ -746,21 +733,22 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
return nil, errors.Wrap(err, "failed to peek context header from STDIN") return nil, errors.Wrap(err, "failed to peek context header from STDIN")
} }
if !(err == io.EOF && len(magic) == 0) {
if isArchive(magic) { if isArchive(magic) {
// stdin is context // stdin is context
up := uploadprovider.New() up := uploadprovider.New()
target.FrontendAttrs["context"] = up.Add(buf) target.FrontendAttrs["context"] = up.Add(buf)
target.Session = append(target.Session, up) target.Session = append(target.Session, up)
} else { } else {
if inp.DockerfilePath != "" { if inp.DockerfilePath != "" {
return nil, errDockerfileConflict return nil, errDockerfileConflict
}
// stdin is dockerfile
dockerfileReader = buf
inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
target.LocalDirs["context"] = inp.ContextPath
} }
// stdin is dockerfile
dockerfileReader = buf
inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
target.LocalDirs["context"] = inp.ContextPath
} }
case isLocalDir(inp.ContextPath): case isLocalDir(inp.ContextPath):
@ -784,6 +772,10 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
return nil, errors.Errorf("unable to prepare context: path %q not found", inp.ContextPath) return nil, errors.Errorf("unable to prepare context: path %q not found", inp.ContextPath)
} }
if inp.DockerfileInline != "" {
dockerfileReader = strings.NewReader(inp.DockerfileInline)
}
if dockerfileReader != nil { if dockerfileReader != nil {
dockerfileDir, err = createTempDockerfile(dockerfileReader) dockerfileDir, err = createTempDockerfile(dockerfileReader)
if err != nil { if err != nil {
@ -791,6 +783,17 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
} }
toRemove = append(toRemove, dockerfileDir) toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile" dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
}
if urlutil.IsURL(inp.DockerfilePath) {
dockerfileDir, err = createTempDockerfileFromURL(ctx, d, inp.DockerfilePath, pw)
if err != nil {
return nil, err
}
toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
delete(target.FrontendInputs, "dockerfile")
} }
if dockerfileName == "" { if dockerfileName == "" {
@ -818,7 +821,7 @@ func notSupported(d driver.Driver, f driver.Feature) error {
type dockerLoadCallback func(name string) (io.WriteCloser, func(), error) type dockerLoadCallback func(name string) (io.WriteCloser, func(), error)
func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) { func newDockerLoader(ctx context.Context, d DockerAPI, name string, status progress.Writer) (io.WriteCloser, func(), error) {
c, err := d.DockerAPI(name) c, err := d.DockerAPI(name)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
@ -841,7 +844,7 @@ func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress
w.mu.Unlock() w.mu.Unlock()
return return
} }
prog := mw.WithPrefix("", false) prog := progress.WithPrefix(status, "", false)
progress.FromReader(prog, "importing to docker", resp.Body) progress.FromReader(prog, "importing to docker", resp.Body)
}, },
done: done, done: done,

@ -0,0 +1,71 @@
package build
import (
"context"
"io/ioutil"
"path/filepath"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) {
c, err := driver.Boot(ctx, d, pw)
if err != nil {
return "", err
}
var out string
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := llb.HTTP(url, llb.Filename("Dockerfile"), llb.WithCustomNamef("[internal] load %s", url)).Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
stat, err := ref.StatFile(ctx, gwclient.StatRequest{
Path: "Dockerfile",
})
if err != nil {
return nil, err
}
if stat.Size() > 512*1024 {
return nil, errors.Errorf("Dockerfile %s bigger than allowed max size", url)
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: "Dockerfile",
})
if err != nil {
return nil, err
}
dir, err := ioutil.TempDir("", "buildx")
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dt, 0600); err != nil {
return nil, err
}
out = dir
return nil, nil
}, ch)
if err != nil {
return "", err
}
return out, nil
}

@ -1,11 +1,14 @@
package commands package commands
import ( import (
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
"github.com/docker/buildx/bake" "github.com/docker/buildx/bake"
"github.com/docker/buildx/build"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command" "github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/appcontext"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -19,18 +22,16 @@ type bakeOptions struct {
commonOptions commonOptions
} }
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error { func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) {
ctx := appcontext.Context() ctx := appcontext.Context()
if len(in.files) == 0 { var url string
files, err := defaultFiles()
if err != nil { if len(targets) > 0 {
return err if bake.IsRemoteURL(targets[0]) {
} url = targets[0]
if len(files) == 0 { targets = targets[1:]
return errors.Errorf("no docker-compose.yml or docker-bake.hcl found, specify build file with -f/--file")
} }
in.files = files
} }
if len(targets) == 0 { if len(targets) == 0 {
@ -52,8 +53,38 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if in.pull != nil { if in.pull != nil {
overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull)) overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
} }
contextPathHash, _ := os.Getwd()
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
printer := progress.NewPrinter(ctx2, os.Stderr, in.progress)
m, err := bake.ReadTargets(ctx, in.files, targets, overrides) defer func() {
if printer != nil {
err1 := printer.Wait()
if err == nil {
err = err1
}
}
}()
dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash)
if err != nil {
return err
}
var files []bake.File
var inp *bake.Input
if url != "" {
files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
} else {
files, err = bake.ReadLocalFiles(in.files)
}
if err != nil {
return err
}
m, err := bake.ReadTargets(ctx, files, targets, overrides)
if err != nil { if err != nil {
return err return err
} }
@ -63,40 +94,22 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if err != nil { if err != nil {
return err return err
} }
err = printer.Wait()
printer = nil
if err != nil {
return err
}
fmt.Fprintln(dockerCli.Out(), string(dt)) fmt.Fprintln(dockerCli.Out(), string(dt))
return nil return nil
} }
bo, err := bake.TargetsToBuildOpt(m) bo, err := bake.TargetsToBuildOpt(m, inp)
if err != nil { if err != nil {
return err return err
} }
contextPathHash, _ := os.Getwd() _, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
return err
return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash, in.builder)
}
func defaultFiles() ([]string, error) {
fns := []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
out := make([]string, 0, len(fns))
for _, f := range fns {
if _, err := os.Stat(f); err != nil {
if os.IsNotExist(errors.Cause(err)) {
continue
}
return nil, err
}
out = append(out, f)
}
return out, nil
} }
func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {

@ -203,9 +203,14 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
ctx2, cancel := context.WithCancel(context.TODO()) ctx2, cancel := context.WithCancel(context.TODO())
defer cancel() defer cancel()
pw := progress.NewPrinter(ctx2, os.Stderr, progressMode) printer := progress.NewPrinter(ctx2, os.Stderr, progressMode)
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
err1 := printer.Wait()
if err == nil {
err = err1
}
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), pw)
return err return err
} }

@ -171,25 +171,27 @@ func boot(ctx context.Context, ngi *nginfo, dockerCli command.Cli) (bool, error)
return false, nil return false, nil
} }
pw := progress.NewPrinter(context.TODO(), os.Stderr, "auto") printer := progress.NewPrinter(context.TODO(), os.Stderr, "auto")
mw := progress.NewMultiWriter(pw)
eg, _ := errgroup.WithContext(ctx) eg, _ := errgroup.WithContext(ctx)
for _, idx := range toBoot { for _, idx := range toBoot {
func(idx int) { func(idx int) {
eg.Go(func() error { eg.Go(func() error {
pw := mw.WithPrefix(ngi.ng.Nodes[idx].Name, len(toBoot) > 1) pw := progress.WithPrefix(printer, ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
_, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw) _, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
if err != nil { if err != nil {
ngi.drivers[idx].err = err ngi.drivers[idx].err = err
} }
close(pw.Status())
<-pw.Done()
return nil return nil
}) })
}(idx) }(idx)
} }
return true, eg.Wait() err := eg.Wait()
err1 := printer.Wait()
if err == nil {
err = err1
}
return true, err
} }

@ -32,6 +32,10 @@ type Driver struct {
env []string env []string
} }
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error { func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error { return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.DockerAPI.ContainerInspect(ctx, d.Name) _, err := d.DockerAPI.ContainerInspect(ctx, d.Name)

@ -57,4 +57,6 @@ func (d *Driver) Factory() driver.Factory {
return d.factory return d.factory
} }
func (d *Driver) IsDefaultMobyDriver() {} func (d *Driver) IsMobyDriver() bool {
return true
}

@ -57,6 +57,7 @@ type Driver interface {
Rm(ctx context.Context, force bool) error Rm(ctx context.Context, force bool) error
Client(ctx context.Context) (*client.Client, error) Client(ctx context.Context) (*client.Client, error)
Features() map[Feature]bool Features() map[Feature]bool
IsMobyDriver() bool
} }
func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) { func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) {
@ -71,11 +72,7 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, er
if try > 2 { if try > 2 {
return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d) return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
} }
if err := d.Bootstrap(ctx, func(s *client.SolveStatus) { if err := d.Bootstrap(ctx, pw.Write); err != nil {
if pw != nil {
pw.Status() <- s
}
}); err != nil {
return nil, err return nil, err
} }
} }

@ -44,6 +44,10 @@ type Driver struct {
podChooser podchooser.PodChooser podChooser podchooser.PodChooser
} }
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error { func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error { return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{}) _, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})

@ -5,10 +5,12 @@ import (
"io/ioutil" "io/ioutil"
"sort" "sort"
"strings" "strings"
"sync"
"k8s.io/client-go/rest" "k8s.io/client-go/rest"
dockerclient "github.com/docker/docker/client" dockerclient "github.com/docker/docker/client"
"github.com/moby/buildkit/client"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -117,9 +119,27 @@ func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.API
return nil, err return nil, err
} }
} }
return f.New(ctx, ic) d, err := f.New(ctx, ic)
if err != nil {
return nil, err
}
return &cachedDriver{Driver: d}, nil
} }
func GetFactories() map[string]Factory { func GetFactories() map[string]Factory {
return drivers return drivers
} }
type cachedDriver struct {
Driver
client *client.Client
err error
once sync.Once
}
func (d *cachedDriver) Client(ctx context.Context) (*client.Client, error) {
d.once.Do(func() {
d.client, d.err = d.Driver.Client(ctx)
})
return d.client, d.err
}

@ -11,7 +11,6 @@ import (
) )
func FromReader(w Writer, name string, rc io.ReadCloser) { func FromReader(w Writer, name string, rc io.ReadCloser) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID())) dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now() tm := time.Now()
@ -21,9 +20,9 @@ func FromReader(w Writer, name string, rc io.ReadCloser) {
Started: &tm, Started: &tm,
} }
status <- &client.SolveStatus{ w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx}, Vertexes: []*client.Vertex{&vtx},
} })
_, err := io.Copy(ioutil.Discard, rc) _, err := io.Copy(ioutil.Discard, rc)
@ -33,8 +32,7 @@ func FromReader(w Writer, name string, rc io.ReadCloser) {
if err != nil { if err != nil {
vtx2.Error = err.Error() vtx2.Error = err.Error()
} }
status <- &client.SolveStatus{ w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2}, Vertexes: []*client.Vertex{&vtx2},
} })
close(status)
} }

@ -1,101 +1,32 @@
package progress package progress
import ( import (
"context"
"strings" "strings"
"sync"
"github.com/moby/buildkit/client" "github.com/moby/buildkit/client"
"golang.org/x/sync/errgroup"
) )
type MultiWriter struct { func WithPrefix(w Writer, pfx string, force bool) Writer {
w Writer return &prefixed{
eg *errgroup.Group main: w,
once sync.Once pfx: pfx,
ready chan struct{} force: force,
}
func (mw *MultiWriter) WithPrefix(pfx string, force bool) Writer {
in := make(chan *client.SolveStatus)
out := mw.w.Status()
p := &prefixed{
main: mw.w,
in: in,
} }
mw.eg.Go(func() error {
mw.once.Do(func() {
close(mw.ready)
})
for {
select {
case v, ok := <-in:
if ok {
if force {
for _, v := range v.Vertexes {
v.Name = addPrefix(pfx, v.Name)
}
}
out <- v
} else {
return nil
}
case <-mw.Done():
return mw.Err()
}
}
})
return p
}
func (mw *MultiWriter) Done() <-chan struct{} {
return mw.w.Done()
}
func (mw *MultiWriter) Err() error {
return mw.w.Err()
}
func (mw *MultiWriter) Status() chan *client.SolveStatus {
return nil
} }
type prefixed struct { type prefixed struct {
main Writer main Writer
in chan *client.SolveStatus pfx string
} force bool
func (p *prefixed) Done() <-chan struct{} {
return p.main.Done()
}
func (p *prefixed) Err() error {
return p.main.Err()
}
func (p *prefixed) Status() chan *client.SolveStatus {
return p.in
} }
func NewMultiWriter(pw Writer) *MultiWriter { func (p *prefixed) Write(v *client.SolveStatus) {
if pw == nil { if p.force {
return nil for _, v := range v.Vertexes {
} v.Name = addPrefix(p.pfx, v.Name)
eg, _ := errgroup.WithContext(context.TODO()) }
ready := make(chan struct{})
go func() {
<-ready
eg.Wait()
close(pw.Status())
}()
return &MultiWriter{
w: pw,
eg: eg,
ready: ready,
} }
p.main.Write(v)
} }
func addPrefix(pfx, name string) string { func addPrefix(pfx, name string) string {

@ -9,32 +9,27 @@ import (
"github.com/moby/buildkit/util/progress/progressui" "github.com/moby/buildkit/util/progress/progressui"
) )
type printer struct { type Printer struct {
status chan *client.SolveStatus status chan *client.SolveStatus
done <-chan struct{} done <-chan struct{}
err error err error
} }
func (p *printer) Done() <-chan struct{} { func (p *Printer) Wait() error {
return p.done close(p.status)
} <-p.done
func (p *printer) Err() error {
return p.err return p.err
} }
func (p *printer) Status() chan *client.SolveStatus { func (p *Printer) Write(s *client.SolveStatus) {
if p == nil { p.status <- s
return nil
}
return p.status
} }
func NewPrinter(ctx context.Context, out console.File, mode string) Writer { func NewPrinter(ctx context.Context, out console.File, mode string) *Printer {
statusCh := make(chan *client.SolveStatus) statusCh := make(chan *client.SolveStatus)
doneCh := make(chan struct{}) doneCh := make(chan struct{})
pw := &printer{ pw := &Printer{
status: statusCh, status: statusCh,
done: doneCh, done: doneCh,
} }

@ -7,56 +7,45 @@ import (
) )
func ResetTime(in Writer) Writer { func ResetTime(in Writer) Writer {
w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()} return &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
go func() { }
for {
select { func (w *pw) Write(st *client.SolveStatus) {
case <-in.Done(): if w.diff == nil {
return for _, v := range st.Vertexes {
case st, ok := <-w.status: if v.Started != nil {
if !ok { d := v.Started.Sub(w.tm)
close(in.Status()) w.diff = &d
return }
} }
if w.diff == nil { }
for _, v := range st.Vertexes { if w.diff != nil {
if v.Started != nil { for _, v := range st.Vertexes {
d := v.Started.Sub(w.tm) if v.Started != nil {
w.diff = &d d := v.Started.Add(-*w.diff)
} v.Started = &d
} }
} if v.Completed != nil {
if w.diff != nil { d := v.Completed.Add(-*w.diff)
for _, v := range st.Vertexes { v.Completed = &d
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
in.Status() <- st
} }
} }
}() for _, v := range st.Statuses {
return w if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
w.Writer.Write(st)
} }
type pw struct { type pw struct {

@ -9,13 +9,10 @@ import (
) )
type Writer interface { type Writer interface {
Done() <-chan struct{} Write(*client.SolveStatus)
Err() error
Status() chan *client.SolveStatus
} }
func Write(w Writer, name string, f func() error) { func Write(w Writer, name string, f func() error) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID())) dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now() tm := time.Now()
@ -25,9 +22,9 @@ func Write(w Writer, name string, f func() error) {
Started: &tm, Started: &tm,
} }
status <- &client.SolveStatus{ w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx}, Vertexes: []*client.Vertex{&vtx},
} })
err := f() err := f()
@ -37,7 +34,23 @@ func Write(w Writer, name string, f func() error) {
if err != nil { if err != nil {
vtx2.Error = err.Error() vtx2.Error = err.Error()
} }
status <- &client.SolveStatus{ w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2}, Vertexes: []*client.Vertex{&vtx2},
} })
}
func NewChannel(w Writer) (chan *client.SolveStatus, chan struct{}) {
ch := make(chan *client.SolveStatus)
done := make(chan struct{})
go func() {
for {
v, ok := <-ch
if !ok {
close(done)
return
}
w.Write(v)
}
}()
return ch, done
} }

Loading…
Cancel
Save