From b3d7b5a2ec1e8b2a40d2e2d5031eb27e49dd4f17 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 26 Mar 2019 11:39:37 -0700 Subject: [PATCH] vendor: update buildkit Signed-off-by: Tonis Tiigi --- go.mod | 3 +- go.sum | 5 +- .../docker/docker/pkg/stdcopy/stdcopy.go | 190 ++++++++++++++++++ .../github.com/moby/buildkit/client/client.go | 45 ++++- .../buildkit/client/connhelper/connhelper.go | 37 ++++ vendor/modules.txt | 10 +- 6 files changed, 279 insertions(+), 11 deletions(-) create mode 100644 vendor/github.com/docker/docker/pkg/stdcopy/stdcopy.go create mode 100644 vendor/github.com/moby/buildkit/client/connhelper/connhelper.go diff --git a/go.mod b/go.mod index 6fc06909..a1120fbb 100644 --- a/go.mod +++ b/go.mod @@ -53,9 +53,10 @@ require ( github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/miekg/pkcs11 v0.0.0-20190322140431-074fd7a1ed19 // indirect - github.com/moby/buildkit v0.4.1-0.20190322070013-03feb5e28f65 + github.com/moby/buildkit v0.4.1-0.20190326070013-325bd96b6b62 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/runtime-spec v1.0.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect diff --git a/go.sum b/go.sum index 915169bd..7077fffc 100644 --- a/go.sum +++ b/go.sum @@ -63,7 +63,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20190315220205-a8ed825ac853 h1:tTngnoO/B6HQnJ+pK8tN7kEAhmhIfaJOutqq/A4/JTM= github.com/denisenkom/go-mssqldb v0.0.0-20190315220205-a8ed825ac853/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= -github.com/docker/cli v0.0.0-20190131223713-234462756460/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v0.0.0-20190321234815-f40f9c240ab0 h1:E7NTtHfZYV+iu35yZ49AbrxqhMHpiOl3FstDYm38vQ0= github.com/docker/cli v0.0.0-20190321234815-f40f9c240ab0/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/compose-on-kubernetes v0.4.19-0.20190128150448-356b2919c496 h1:90ytrX1dbzL7Uf/hHiuWwvywC+gikHv4hkAy4CwRTbs= @@ -184,8 +183,8 @@ github.com/miekg/pkcs11 v0.0.0-20190322140431-074fd7a1ed19/go.mod h1:WCBAbTOdfhH github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/buildkit v0.4.1-0.20190322070013-03feb5e28f65 h1:t9GGPuQ85Reqzlnu7Vy7QBb+SrztgASQ5ek8CSEqdyI= -github.com/moby/buildkit v0.4.1-0.20190322070013-03feb5e28f65/go.mod h1:hxXlABlNvROrpptYGUMgF2V7VrHpjphi23E63UjlVP4= +github.com/moby/buildkit v0.4.1-0.20190326070013-325bd96b6b62 h1:GulTASjmiroM1LFiX4HX1zo7nQ7vRm3BbzdUou7UMaw= +github.com/moby/buildkit v0.4.1-0.20190326070013-325bd96b6b62/go.mod h1:CGtUCEvQ3mcP73SmixGWRxEOECm+hCUJsdfNVKPNzVA= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= diff --git a/vendor/github.com/docker/docker/pkg/stdcopy/stdcopy.go b/vendor/github.com/docker/docker/pkg/stdcopy/stdcopy.go new file mode 100644 index 00000000..8f6e0a73 --- /dev/null +++ b/vendor/github.com/docker/docker/pkg/stdcopy/stdcopy.go @@ -0,0 +1,190 @@ +package stdcopy // import "github.com/docker/docker/pkg/stdcopy" + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "sync" +) + +// StdType is the type of standard stream +// a writer can multiplex to. +type StdType byte + +const ( + // Stdin represents standard input stream type. + Stdin StdType = iota + // Stdout represents standard output stream type. + Stdout + // Stderr represents standard error steam type. + Stderr + // Systemerr represents errors originating from the system that make it + // into the multiplexed stream. + Systemerr + + stdWriterPrefixLen = 8 + stdWriterFdIndex = 0 + stdWriterSizeIndex = 4 + + startingBufLen = 32*1024 + stdWriterPrefixLen + 1 +) + +var bufPool = &sync.Pool{New: func() interface{} { return bytes.NewBuffer(nil) }} + +// stdWriter is wrapper of io.Writer with extra customized info. +type stdWriter struct { + io.Writer + prefix byte +} + +// Write sends the buffer to the underneath writer. +// It inserts the prefix header before the buffer, +// so stdcopy.StdCopy knows where to multiplex the output. +// It makes stdWriter to implement io.Writer. +func (w *stdWriter) Write(p []byte) (n int, err error) { + if w == nil || w.Writer == nil { + return 0, errors.New("Writer not instantiated") + } + if p == nil { + return 0, nil + } + + header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix} + binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(p))) + buf := bufPool.Get().(*bytes.Buffer) + buf.Write(header[:]) + buf.Write(p) + + n, err = w.Writer.Write(buf.Bytes()) + n -= stdWriterPrefixLen + if n < 0 { + n = 0 + } + + buf.Reset() + bufPool.Put(buf) + return +} + +// NewStdWriter instantiates a new Writer. +// Everything written to it will be encapsulated using a custom format, +// and written to the underlying `w` stream. +// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection. +// `t` indicates the id of the stream to encapsulate. +// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr. +func NewStdWriter(w io.Writer, t StdType) io.Writer { + return &stdWriter{ + Writer: w, + prefix: byte(t), + } +} + +// StdCopy is a modified version of io.Copy. +// +// StdCopy will demultiplex `src`, assuming that it contains two streams, +// previously multiplexed together using a StdWriter instance. +// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`. +// +// StdCopy will read until it hits EOF on `src`. It will then return a nil error. +// In other words: if `err` is non nil, it indicates a real underlying error. +// +// `written` will hold the total number of bytes written to `dstout` and `dsterr`. +func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { + var ( + buf = make([]byte, startingBufLen) + bufLen = len(buf) + nr, nw int + er, ew error + out io.Writer + frameSize int + ) + + for { + // Make sure we have at least a full header + for nr < stdWriterPrefixLen { + var nr2 int + nr2, er = src.Read(buf[nr:]) + nr += nr2 + if er == io.EOF { + if nr < stdWriterPrefixLen { + return written, nil + } + break + } + if er != nil { + return 0, er + } + } + + stream := StdType(buf[stdWriterFdIndex]) + // Check the first byte to know where to write + switch stream { + case Stdin: + fallthrough + case Stdout: + // Write on stdout + out = dstout + case Stderr: + // Write on stderr + out = dsterr + case Systemerr: + // If we're on Systemerr, we won't write anywhere. + // NB: if this code changes later, make sure you don't try to write + // to outstream if Systemerr is the stream + out = nil + default: + return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex]) + } + + // Retrieve the size of the frame + frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4])) + + // Check if the buffer is big enough to read the frame. + // Extend it if necessary. + if frameSize+stdWriterPrefixLen > bufLen { + buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...) + bufLen = len(buf) + } + + // While the amount of bytes read is less than the size of the frame + header, we keep reading + for nr < frameSize+stdWriterPrefixLen { + var nr2 int + nr2, er = src.Read(buf[nr:]) + nr += nr2 + if er == io.EOF { + if nr < frameSize+stdWriterPrefixLen { + return written, nil + } + break + } + if er != nil { + return 0, er + } + } + + // we might have an error from the source mixed up in our multiplexed + // stream. if we do, return it. + if stream == Systemerr { + return written, fmt.Errorf("error from daemon in stream: %s", string(buf[stdWriterPrefixLen:frameSize+stdWriterPrefixLen])) + } + + // Write the retrieved frame (without header) + nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen]) + if ew != nil { + return 0, ew + } + + // If the frame has not been fully written: error + if nw != frameSize { + return 0, io.ErrShortWrite + } + written += int64(nw) + + // Move the rest of the buffer to the beginning + copy(buf, buf[frameSize+stdWriterPrefixLen:]) + // Move the index + nr -= frameSize + stdWriterPrefixLen + } +} diff --git a/vendor/github.com/moby/buildkit/client/client.go b/vendor/github.com/moby/buildkit/client/client.go index 6126acdc..6b824094 100644 --- a/vendor/github.com/moby/buildkit/client/client.go +++ b/vendor/github.com/moby/buildkit/client/client.go @@ -5,9 +5,12 @@ import ( "crypto/tls" "crypto/x509" "io/ioutil" + "net" + "time" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" controlapi "github.com/moby/buildkit/api/services/control" + "github.com/moby/buildkit/client/connhelper" "github.com/moby/buildkit/util/appdefaults" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -23,9 +26,8 @@ type ClientOpt interface{} // New returns a new buildkit client. Address can be empty for the system-default address. func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error) { - gopts := []grpc.DialOption{ - grpc.WithDialer(dialer), - } + gopts := []grpc.DialOption{} + needDialer := true needWithInsecure := true for _, o := range opts { if _, ok := o.(*withFailFast); ok { @@ -44,6 +46,19 @@ func New(ctx context.Context, address string, opts ...ClientOpt) (*Client, error grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(wt.tracer, otgrpc.LogPayloads())), grpc.WithStreamInterceptor(otgrpc.OpenTracingStreamClientInterceptor(wt.tracer))) } + if wd, ok := o.(*withDialer); ok { + gopts = append(gopts, grpc.WithDialer(wd.dialer)) + needDialer = false + } + } + if needDialer { + dialFn, err := resolveDialer(address) + if err != nil { + return nil, err + } + // TODO(AkihiroSuda): use WithContextDialer (requires grpc 1.19) + // https://github.com/grpc/grpc-go/commit/40cb5618f475e7b9d61aa7920ae4b04ef9bbaf89 + gopts = append(gopts, grpc.WithDialer(dialFn)) } if needWithInsecure { gopts = append(gopts, grpc.WithInsecure()) @@ -75,6 +90,14 @@ func WithFailFast() ClientOpt { return &withFailFast{} } +type withDialer struct { + dialer func(string, time.Duration) (net.Conn, error) +} + +func WithDialer(df func(string, time.Duration) (net.Conn, error)) ClientOpt { + return &withDialer{dialer: df} +} + type withCredentials struct { ServerName string CACert string @@ -128,3 +151,19 @@ func WithTracer(t opentracing.Tracer) ClientOpt { type withTracer struct { tracer opentracing.Tracer } + +func resolveDialer(address string) (func(string, time.Duration) (net.Conn, error), error) { + ch, err := connhelper.GetConnectionHelper(address) + if err != nil { + return nil, err + } + if ch != nil { + f := func(a string, _ time.Duration) (net.Conn, error) { + ctx := context.Background() + return ch.ContextDialer(ctx, a) + } + return f, nil + } + // basic dialer + return dialer, nil +} diff --git a/vendor/github.com/moby/buildkit/client/connhelper/connhelper.go b/vendor/github.com/moby/buildkit/client/connhelper/connhelper.go new file mode 100644 index 00000000..9fa5edcd --- /dev/null +++ b/vendor/github.com/moby/buildkit/client/connhelper/connhelper.go @@ -0,0 +1,37 @@ +// Package connhelper provides helpers for connecting to a remote daemon host with custom logic. +package connhelper + +import ( + "context" + "net" + "net/url" + + "github.com/docker/cli/cli/connhelper/commandconn" +) + +// ConnectionHelper allows to connect to a remote host with custom stream provider binary. +type ConnectionHelper struct { + // ContextDialer can be passed to grpc.WithContextDialer + ContextDialer func(ctx context.Context, addr string) (net.Conn, error) +} + +// GetConnectionHelper returns BuildKit-specific connection helper for the given URL. +// GetConnectionHelper returns nil without error when no helper is registered for the scheme. +// +// docker:// URL requires BuildKit v0.5.0 or later in the container. +func GetConnectionHelper(daemonURL string) (*ConnectionHelper, error) { + u, err := url.Parse(daemonURL) + if err != nil { + return nil, err + } + switch scheme := u.Scheme; scheme { + case "docker": + container := u.Host + return &ConnectionHelper{ + ContextDialer: func(ctx context.Context, addr string) (net.Conn, error) { + return commandconn.New(ctx, "docker", "exec", "-i", container, "buildctl", "dial-stdio") + }, + }, nil + } + return nil, err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 26e3a157..c6118478 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -116,8 +116,8 @@ github.com/docker/cli/cli/version github.com/docker/cli/internal/containerizedengine github.com/docker/cli/opts github.com/docker/cli/types -github.com/docker/cli/cli/config/credentials github.com/docker/cli/cli/connhelper/commandconn +github.com/docker/cli/cli/config/credentials github.com/docker/cli/cli/connhelper/ssh github.com/docker/cli/kubernetes github.com/docker/cli/cli/manifest/types @@ -151,6 +151,7 @@ github.com/docker/docker/client github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/api/types/network +github.com/docker/docker/pkg/stdcopy github.com/docker/docker/api/types/events github.com/docker/docker/api/types/filters github.com/docker/docker/api/types/registry @@ -245,23 +246,24 @@ github.com/konsorten/go-windows-terminal-sequences github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/miekg/pkcs11 v0.0.0-20190322140431-074fd7a1ed19 github.com/miekg/pkcs11 -# github.com/moby/buildkit v0.4.1-0.20190322070013-03feb5e28f65 +# github.com/moby/buildkit v0.4.1-0.20190326070013-325bd96b6b62 github.com/moby/buildkit/client github.com/moby/buildkit/session github.com/moby/buildkit/session/secrets/secretsprovider github.com/moby/buildkit/session/sshforward/sshprovider -github.com/moby/buildkit/util/progress/progressui github.com/moby/buildkit/session/auth/authprovider github.com/moby/buildkit/util/appcontext +github.com/moby/buildkit/identity +github.com/moby/buildkit/util/progress/progressui github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types github.com/moby/buildkit/client/buildid +github.com/moby/buildkit/client/connhelper github.com/moby/buildkit/client/llb github.com/moby/buildkit/client/ociindex github.com/moby/buildkit/frontend/gateway/client github.com/moby/buildkit/frontend/gateway/grpcclient github.com/moby/buildkit/frontend/gateway/pb -github.com/moby/buildkit/identity github.com/moby/buildkit/session/content github.com/moby/buildkit/session/filesync github.com/moby/buildkit/session/grpchijack