From 290e25917c71f2a29c34961c0166d14abb4a145f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 20 Sep 2020 17:44:16 -0700 Subject: [PATCH] build: allow dockerfile from URL Signed-off-by: Tonis Tiigi --- build/build.go | 47 +++++++++++++++++++------------- build/url.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++ driver/manager.go | 22 ++++++++++++++- 3 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 build/url.go diff --git a/build/build.go b/build/build.go index 13fd0e33..e468eb61 100644 --- a/build/build.go +++ b/build/build.go @@ -285,7 +285,7 @@ func isDefaultMobyDriver(d driver.Driver) bool { return ok } -func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) { +func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Options, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) { defers := make([]func(), 0, 2) releaseF := func() { for _, f := range defers { @@ -425,7 +425,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal so.Exports = opt.Exports so.Session = opt.Session - releaseLoad, err := LoadInputs(opt.Inputs, &so) + releaseLoad, err := LoadInputs(ctx, d, opt.Inputs, pw, &so) if err != nil { return nil, nil, err } @@ -530,7 +530,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do for i, dp := range m[k] { d := drivers[dp.driverIndex].Driver opt.Platforms = dp.platforms - so, release, err := toSolveOpt(d, multiDriver, opt, func(name string) (io.WriteCloser, func(), error) { + so, release, err := toSolveOpt(ctx, d, multiDriver, opt, pw, func(name string) (io.WriteCloser, func(), error) { return newDockerLoader(ctx, docker, name, mw) }) if err != nil { @@ -720,7 +720,7 @@ func createTempDockerfile(r io.Reader) (string, error) { return dir, err } -func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) { +func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) { if inp.ContextPath == "" { return nil, errors.New("please specify build context (e.g. \".\" for the current directory)") } @@ -746,21 +746,22 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) { if err != nil && err != io.EOF { return nil, errors.Wrap(err, "failed to peek context header from STDIN") } - - if isArchive(magic) { - // stdin is context - up := uploadprovider.New() - target.FrontendAttrs["context"] = up.Add(buf) - target.Session = append(target.Session, up) - } else { - if inp.DockerfilePath != "" { - return nil, errDockerfileConflict + if !(err == io.EOF && len(magic) == 0) { + if isArchive(magic) { + // stdin is context + up := uploadprovider.New() + target.FrontendAttrs["context"] = up.Add(buf) + target.Session = append(target.Session, up) + } else { + if inp.DockerfilePath != "" { + return nil, errDockerfileConflict + } + // stdin is dockerfile + dockerfileReader = buf + inp.ContextPath, _ = ioutil.TempDir("", "empty-dir") + toRemove = append(toRemove, inp.ContextPath) + target.LocalDirs["context"] = inp.ContextPath } - // stdin is dockerfile - dockerfileReader = buf - inp.ContextPath, _ = ioutil.TempDir("", "empty-dir") - toRemove = append(toRemove, inp.ContextPath) - target.LocalDirs["context"] = inp.ContextPath } case isLocalDir(inp.ContextPath): @@ -791,6 +792,16 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) { } toRemove = append(toRemove, dockerfileDir) dockerfileName = "Dockerfile" + target.FrontendAttrs["dockerfilekey"] = "dockerfile" + } + if urlutil.IsURL(inp.DockerfilePath) { + dockerfileDir, err = createTempDockerfileFromURL(ctx, d, inp.DockerfilePath, pw) + if err != nil { + return nil, err + } + toRemove = append(toRemove, dockerfileDir) + dockerfileName = "Dockerfile" + target.FrontendAttrs["dockerfilekey"] = "dockerfile" } if dockerfileName == "" { diff --git a/build/url.go b/build/url.go new file mode 100644 index 00000000..e369e9ba --- /dev/null +++ b/build/url.go @@ -0,0 +1,69 @@ +package build + +import ( + "context" + "io/ioutil" + "path/filepath" + + "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/progress" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/pkg/errors" +) + +func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) { + c, err := d.Client(ctx) + if err != nil { + return "", err + } + var out string + _, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { + def, err := llb.HTTP(url, llb.Filename("Dockerfile"), llb.WithCustomNamef("[internal] load %s", url)).Marshal(ctx) + if err != nil { + return nil, err + } + + res, err := c.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + stat, err := ref.StatFile(ctx, gwclient.StatRequest{ + Path: "Dockerfile", + }) + if err != nil { + return nil, err + } + if stat.Size() > 512*1024 { + return nil, errors.Errorf("Dockerfile %s bigger than allowed max size", url) + } + + dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + Filename: "Dockerfile", + }) + if err != nil { + return nil, err + } + dir, err := ioutil.TempDir("", "buildx") + if err != nil { + return nil, err + } + if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dt, 0600); err != nil { + return nil, err + } + out = dir + return nil, nil + }, pw.Status()) + + if err != nil { + return "", err + } + return out, nil +} diff --git a/driver/manager.go b/driver/manager.go index a54b4237..f573431f 100644 --- a/driver/manager.go +++ b/driver/manager.go @@ -5,10 +5,12 @@ import ( "io/ioutil" "sort" "strings" + "sync" "k8s.io/client-go/rest" dockerclient "github.com/docker/docker/client" + "github.com/moby/buildkit/client" "github.com/pkg/errors" ) @@ -117,9 +119,27 @@ func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.API return nil, err } } - return f.New(ctx, ic) + d, err := f.New(ctx, ic) + if err != nil { + return nil, err + } + return &cachedDriver{Driver: d}, nil } func GetFactories() map[string]Factory { return drivers } + +type cachedDriver struct { + Driver + client *client.Client + err error + once sync.Once +} + +func (d *cachedDriver) Client(ctx context.Context) (*client.Client, error) { + d.once.Do(func() { + d.client, d.err = d.Driver.Client(ctx) + }) + return d.client, d.err +}