diff --git a/build/build.go b/build/build.go index 1a838b0b..2ab8484c 100644 --- a/build/build.go +++ b/build/build.go @@ -13,6 +13,7 @@ import ( "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" + dockerclient "github.com/docker/docker/client" "github.com/docker/docker/pkg/urlutil" "github.com/moby/buildkit/client" "github.com/moby/buildkit/session" @@ -61,6 +62,10 @@ type DriverInfo struct { Err error } +type DockerAPI interface { + DockerAPI(name string) (dockerclient.APIClient, error) +} + func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) { err := errors.Errorf("no drivers found") for _, di := range drivers { @@ -74,7 +79,7 @@ func getFirstDriver(drivers []DriverInfo) (driver.Driver, error) { return nil, err } -func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string]*client.SolveResponse, error) { +func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, pw progress.Writer) (map[string]*client.SolveResponse, error) { if len(drivers) == 0 { return nil, errors.Errorf("driver required for build") } @@ -83,7 +88,6 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw return nil, errors.Errorf("multiple drivers currently not supported") } - pwOld := pw d, err := getFirstDriver(drivers) if err != nil { return nil, err @@ -91,10 +95,10 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw _, isDefaultMobyDriver := d.(interface { IsDefaultMobyDriver() }) - c, pw, err := driver.Boot(ctx, d, pw) + c, err := driver.Boot(ctx, d, pw) if err != nil { - close(pwOld.Status()) - <-pwOld.Done() + close(pw.Status()) + <-pw.Done() return nil, err } @@ -173,10 +177,16 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw } if e.Type == "docker" { if e.Output == nil { - if !isDefaultMobyDriver { - return nil, errors.Errorf("loading to docker currently not implemented, specify dest file or -") + if isDefaultMobyDriver { + e.Type = "image" + } else { + w, cancel, err := newDockerLoader(ctx, docker, e.Attrs["context"], mw) + if err != nil { + return nil, err + } + defer cancel() + opt.Exports[i].Output = w } - e.Type = "image" } else if !d.Features()[driver.DockerExporter] { return nil, notSupported(d, driver.DockerExporter) } @@ -245,6 +255,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw var statusCh chan *client.SolveStatus if pw != nil { + pw = progress.ResetTime(pw) statusCh = pw.Status() eg.Go(func() error { <-pw.Done() @@ -380,5 +391,54 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) { } func notSupported(d driver.Driver, f driver.Feature) error { - return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx new\")", f, d.Factory().Name()) + return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx create\")", f, d.Factory().Name()) +} + +func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) { + c, err := d.DockerAPI(name) + if err != nil { + return nil, nil, err + } + + pr, pw := io.Pipe() + started := make(chan struct{}) + w := &waitingWriter{ + PipeWriter: pw, + f: func() { + resp, err := c.ImageLoad(ctx, pr, false) + if err != nil { + pr.CloseWithError(err) + return + } + prog := mw.WithPrefix("", false) + close(started) + progress.FromReader(prog, "importing to docker", resp.Body) + }, + started: started, + } + return w, func() { + pr.Close() + }, nil +} + +type waitingWriter struct { + *io.PipeWriter + f func() + once sync.Once + mu sync.Mutex + err error + started chan struct{} +} + +func (w *waitingWriter) Write(dt []byte) (int, error) { + w.once.Do(func() { + go w.f() + }) + return w.PipeWriter.Write(dt) +} + +func (w *waitingWriter) Close() error { + err := w.PipeWriter.Close() + <-w.started + return err } diff --git a/commands/build.go b/commands/build.go index 7fbf9893..9ce12c18 100644 --- a/commands/build.go +++ b/commands/build.go @@ -127,7 +127,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu defer cancel() pw := progress.NewPrinter(ctx2, os.Stderr, progressMode) - _, err = build.Build(ctx, dis, opts, pw) + _, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), pw) return err } diff --git a/commands/inspect.go b/commands/inspect.go index e21fa01e..b377e76c 100644 --- a/commands/inspect.go +++ b/commands/inspect.go @@ -166,7 +166,7 @@ func boot(ctx context.Context, ngi *nginfo) (bool, error) { func(idx int) { eg.Go(func() error { pw := mw.WithPrefix(ngi.ng.Nodes[idx].Name, len(toBoot) > 1) - _, _, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw) + _, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw) if err != nil { ngi.drivers[idx].err = err } diff --git a/commands/util.go b/commands/util.go index 969471e3..9695e29b 100644 --- a/commands/util.go +++ b/commands/util.go @@ -299,3 +299,18 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo) return eg.Wait() } + +func dockerAPI(dockerCli command.Cli) *api { + return &api{dockerCli: dockerCli} +} + +type api struct { + dockerCli command.Cli +} + +func (a *api) DockerAPI(name string) (dockerclient.APIClient, error) { + if name == "" { + name = a.dockerCli.CurrentContext() + } + return clientForEndpoint(a.dockerCli, name) +} diff --git a/driver/driver.go b/driver/driver.go index aaec253c..2aa114ea 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -2,7 +2,6 @@ package driver import ( "context" - "time" "github.com/moby/buildkit/client" "github.com/pkg/errors" @@ -52,24 +51,24 @@ type Driver interface { Features() map[Feature]bool } -func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, progress.Writer, error) { +func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) { try := 0 for { info, err := d.Info(ctx) if err != nil { - return nil, nil, err + return nil, err } try++ if info.Status != Running { if try > 2 { - return nil, nil, errors.Errorf("failed to bootstrap %T driver in attempts", d) + return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d) } if err := d.Bootstrap(ctx, func(s *client.SolveStatus) { if pw != nil { pw.Status() <- s } }); err != nil { - return nil, nil, err + return nil, err } } @@ -78,72 +77,8 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, pr if errors.Cause(err) == ErrNotRunning && try <= 2 { continue } - return nil, nil, err + return nil, err } - return c, newResetWriter(pw), nil + return c, nil } } - -func newResetWriter(in progress.Writer) progress.Writer { - w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()} - go func() { - for { - select { - case <-in.Done(): - return - case st, ok := <-w.status: - if !ok { - close(in.Status()) - return - } - if w.diff == nil { - for _, v := range st.Vertexes { - if v.Started != nil { - d := v.Started.Sub(w.tm) - w.diff = &d - } - } - } - if w.diff != nil { - for _, v := range st.Vertexes { - if v.Started != nil { - d := v.Started.Add(-*w.diff) - v.Started = &d - } - if v.Completed != nil { - d := v.Completed.Add(-*w.diff) - v.Completed = &d - } - } - for _, v := range st.Statuses { - if v.Started != nil { - d := v.Started.Add(-*w.diff) - v.Started = &d - } - if v.Completed != nil { - d := v.Completed.Add(-*w.diff) - v.Completed = &d - } - v.Timestamp = v.Timestamp.Add(-*w.diff) - } - for _, v := range st.Logs { - v.Timestamp = v.Timestamp.Add(-*w.diff) - } - } - in.Status() <- st - } - } - }() - return w -} - -type pw struct { - progress.Writer - tm time.Time - diff *time.Duration - status chan *client.SolveStatus -} - -func (p *pw) Status() chan *client.SolveStatus { - return p.status -} diff --git a/util/progress/fromreader.go b/util/progress/fromreader.go new file mode 100644 index 00000000..18c8c2da --- /dev/null +++ b/util/progress/fromreader.go @@ -0,0 +1,40 @@ +package progress + +import ( + "io" + "io/ioutil" + "time" + + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/identity" + "github.com/opencontainers/go-digest" +) + +func FromReader(w Writer, name string, rc io.ReadCloser) { + status := w.Status() + dgst := digest.FromBytes([]byte(identity.NewID())) + tm := time.Now() + + vtx := client.Vertex{ + Digest: dgst, + Name: name, + Started: &tm, + } + + status <- &client.SolveStatus{ + Vertexes: []*client.Vertex{&vtx}, + } + + _, err := io.Copy(ioutil.Discard, rc) + + tm2 := time.Now() + vtx2 := vtx + vtx2.Completed = &tm2 + if err != nil { + vtx2.Error = err.Error() + } + status <- &client.SolveStatus{ + Vertexes: []*client.Vertex{&vtx2}, + } + close(status) +} diff --git a/util/progress/reset.go b/util/progress/reset.go new file mode 100644 index 00000000..c2dfcbf9 --- /dev/null +++ b/util/progress/reset.go @@ -0,0 +1,71 @@ +package progress + +import ( + "time" + + "github.com/moby/buildkit/client" +) + +func ResetTime(in Writer) Writer { + w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()} + go func() { + for { + select { + case <-in.Done(): + return + case st, ok := <-w.status: + if !ok { + close(in.Status()) + return + } + if w.diff == nil { + for _, v := range st.Vertexes { + if v.Started != nil { + d := v.Started.Sub(w.tm) + w.diff = &d + } + } + } + if w.diff != nil { + for _, v := range st.Vertexes { + if v.Started != nil { + d := v.Started.Add(-*w.diff) + v.Started = &d + } + if v.Completed != nil { + d := v.Completed.Add(-*w.diff) + v.Completed = &d + } + } + for _, v := range st.Statuses { + if v.Started != nil { + d := v.Started.Add(-*w.diff) + v.Started = &d + } + if v.Completed != nil { + d := v.Completed.Add(-*w.diff) + v.Completed = &d + } + v.Timestamp = v.Timestamp.Add(-*w.diff) + } + for _, v := range st.Logs { + v.Timestamp = v.Timestamp.Add(-*w.diff) + } + } + in.Status() <- st + } + } + }() + return w +} + +type pw struct { + Writer + tm time.Time + diff *time.Duration + status chan *client.SolveStatus +} + +func (p *pw) Status() chan *client.SolveStatus { + return p.status +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 575edd2a..f98d1bc1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -139,10 +139,10 @@ github.com/docker/compose-on-kubernetes/api/compose/impersonation # github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible github.com/docker/distribution/reference github.com/docker/distribution/digestset +github.com/docker/distribution/registry/api/errcode github.com/docker/distribution/manifest/manifestlist github.com/docker/distribution github.com/docker/distribution/manifest/schema2 -github.com/docker/distribution/registry/api/errcode github.com/docker/distribution/registry/api/v2 github.com/docker/distribution/registry/client github.com/docker/distribution/registry/client/auth @@ -154,8 +154,8 @@ github.com/docker/distribution/registry/storage/cache/memory github.com/docker/distribution/uuid github.com/docker/distribution/metrics # github.com/docker/docker v1.14.0-0.20190410063227-d9d9eccdc862 -github.com/docker/docker/pkg/urlutil github.com/docker/docker/client +github.com/docker/docker/pkg/urlutil github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/api/types/network @@ -163,21 +163,22 @@ github.com/docker/docker/pkg/stdcopy github.com/docker/docker/pkg/namesgenerator github.com/docker/docker/api/types/mount github.com/docker/docker/api/types/versions +github.com/docker/docker/api github.com/docker/docker/api/types/events github.com/docker/docker/api/types/filters +github.com/docker/docker/api/types/image github.com/docker/docker/api/types/registry +github.com/docker/docker/api/types/swarm +github.com/docker/docker/api/types/time +github.com/docker/docker/api/types/volume +github.com/docker/docker/errdefs github.com/docker/docker/pkg/homedir github.com/docker/docker/pkg/system github.com/docker/docker/pkg/term github.com/docker/docker/registry github.com/docker/docker/api/types/blkiodev -github.com/docker/docker/api/types/swarm -github.com/docker/docker/api -github.com/docker/docker/api/types/image -github.com/docker/docker/api/types/time -github.com/docker/docker/api/types/volume -github.com/docker/docker/errdefs github.com/docker/docker/api/types/strslice +github.com/docker/docker/api/types/swarm/runtime github.com/docker/docker/pkg/jsonmessage github.com/docker/docker/pkg/idtools github.com/docker/docker/pkg/mount @@ -186,7 +187,6 @@ github.com/docker/docker/pkg/ioutils github.com/docker/docker/pkg/stringid github.com/docker/docker/pkg/tarsum github.com/docker/docker/registry/resumable -github.com/docker/docker/api/types/swarm/runtime github.com/docker/docker/pkg/fileutils github.com/docker/docker/pkg/longpath # github.com/docker/docker-credential-helpers v0.6.1 @@ -196,8 +196,8 @@ github.com/docker/docker-credential-helpers/credentials github.com/docker/go/canonical/json # github.com/docker/go-connections v0.4.0 github.com/docker/go-connections/nat -github.com/docker/go-connections/tlsconfig github.com/docker/go-connections/sockets +github.com/docker/go-connections/tlsconfig # github.com/docker/go-events v0.0.0-20170721190031-9461782956ad github.com/docker/go-events # github.com/docker/go-metrics v0.0.0-20170502235133-d466d4f6fd96 @@ -396,10 +396,10 @@ golang.org/x/net/http2 golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/trace +golang.org/x/net/proxy golang.org/x/net/http/httpguts golang.org/x/net/http2/hpack golang.org/x/net/idna -golang.org/x/net/proxy golang.org/x/net/internal/timeseries golang.org/x/net/internal/socks # golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f