diff --git a/bake/bake.go b/bake/bake.go index a8c7c37f..beefc37f 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path" + "regexp" "strconv" "strings" @@ -12,14 +13,55 @@ import ( "github.com/docker/buildx/util/platformutil" "github.com/docker/docker/pkg/urlutil" hcl "github.com/hashicorp/hcl/v2" + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" ) -func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) { +var httpPrefix = regexp.MustCompile(`^https?://`) +var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`) + +type File struct { + Name string + Data []byte +} + +func defaultFilenames() []string { + return []string{ + "docker-compose.yml", // support app + "docker-compose.yaml", // support app + "docker-bake.json", + "docker-bake.override.json", + "docker-bake.hcl", + "docker-bake.override.hcl", + } +} + +func ReadLocalFiles(names []string) ([]File, error) { + isDefault := false + if len(names) == 0 { + isDefault = true + names = defaultFilenames() + } + out := make([]File, 0, len(names)) + + for _, n := range names { + dt, err := ioutil.ReadFile(n) + if err != nil { + if isDefault && errors.Is(err, os.ErrNotExist) { + continue + } + return nil, err + } + out = append(out, File{Name: n, Data: dt}) + } + return out, nil +} + +func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) { var c Config for _, f := range files { - cfg, err := ParseFile(f) + cfg, err := ParseFile(f.Data, f.Name) if err != nil { return nil, err } @@ -44,12 +86,7 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s return m, nil } -func ParseFile(fn string) (*Config, error) { - dt, err := ioutil.ReadFile(fn) - if err != nil { - return nil, err - } - +func ParseFile(dt []byte, fn string) (*Config, error) { fnl := strings.ToLower(fn) if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") { return ParseCompose(dt) @@ -350,6 +387,7 @@ type Target struct { Outputs []string `json:"output,omitempty" hcl:"output,optional"` Pull *bool `json:"pull,omitempty" hcl:"pull,optional"` NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"` + // IMPORTANT: if you add more fields here, do not forget to update newOverrides and README. } @@ -363,10 +401,10 @@ func (t *Target) normalize() { t.Outputs = removeDupes(t.Outputs) } -func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) { +func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) { m2 := make(map[string]build.Options, len(m)) for k, v := range m { - bo, err := toBuildOpt(v) + bo, err := toBuildOpt(v, inp) if err != nil { return nil, err } @@ -375,7 +413,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) { return m2, nil } -func toBuildOpt(t *Target) (*build.Options, error) { +func updateContext(t *build.Inputs, inp *Input) { + if inp == nil || inp.State == nil { + return + } + if t.ContextPath == "." { + t.ContextPath = inp.URL + return + } + st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath)) + t.ContextState = &st +} + +func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { if v := t.Context; v != nil && *v == "-" { return nil, errors.Errorf("context from stdin not allowed in bake") } @@ -387,6 +437,7 @@ func toBuildOpt(t *Target) (*build.Options, error) { if t.Context != nil { contextPath = *t.Context } + contextPath = path.Clean(contextPath) dockerfilePath := "Dockerfile" if t.Dockerfile != nil { dockerfilePath = *t.Dockerfile @@ -405,11 +456,14 @@ func toBuildOpt(t *Target) (*build.Options, error) { pull = *t.Pull } + bi := build.Inputs{ + ContextPath: contextPath, + DockerfilePath: dockerfilePath, + } + updateContext(&bi, inp) + bo := &build.Options{ - Inputs: build.Inputs{ - ContextPath: contextPath, - DockerfilePath: dockerfilePath, - }, + Inputs: bi, Tags: t.Tags, BuildArgs: t.Args, Labels: t.Labels, diff --git a/bake/bake_test.go b/bake/bake_test.go index 49d64981..cf8b081c 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -2,9 +2,7 @@ package bake import ( "context" - "io/ioutil" "os" - "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -12,12 +10,10 @@ import ( func TestReadTargets(t *testing.T) { t.Parallel() - tmpdir, err := ioutil.TempDir("", "bake") - require.NoError(t, err) - defer os.RemoveAll(tmpdir) - fp := filepath.Join(tmpdir, "config.hcl") - err = ioutil.WriteFile(fp, []byte(` + fp := File{ + Name: "config.hcl", + Data: []byte(` target "webDEP" { args = { VAR_INHERITED = "webDEP" @@ -32,13 +28,13 @@ target "webapp" { VAR_BOTH = "webapp" } inherits = ["webDEP"] -}`), 0600) - require.NoError(t, err) +}`), + } ctx := context.TODO() t.Run("NoOverrides", func(t *testing.T) { - m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil) + m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil) require.NoError(t, err) require.Equal(t, 1, len(m)) @@ -50,7 +46,7 @@ target "webapp" { }) t.Run("InvalidTargetOverrides", func(t *testing.T) { - _, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"}) + _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"}) require.NotNil(t, err) require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'") }) @@ -60,7 +56,7 @@ target "webapp" { os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv") defer os.Unsetenv("VAR_FROM_ENV" + t.Name()) - m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{ + m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{ "webapp.args.VAR_UNSET", "webapp.args.VAR_EMPTY=", "webapp.args.VAR_SET=bananas", @@ -89,7 +85,7 @@ target "webapp" { // building leaf but overriding parent fields t.Run("parent", func(t *testing.T) { - m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{ + m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{ "webDEP.args.VAR_INHERITED=override", "webDEP.args.VAR_BOTH=override", }) @@ -100,23 +96,23 @@ target "webapp" { }) t.Run("ContextOverride", func(t *testing.T) { - _, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"}) + _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"}) require.NotNil(t, err) - m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"}) + m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"}) require.NoError(t, err) require.Equal(t, "foo", *m["webapp"].Context) }) t.Run("NoCacheOverride", func(t *testing.T) { - m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"}) + m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"}) require.NoError(t, err) require.Equal(t, false, *m["webapp"].NoCache) }) t.Run("PullOverride", func(t *testing.T) { - m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"}) + m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"}) require.NoError(t, err) require.Equal(t, false, *m["webapp"].Pull) }) @@ -176,7 +172,7 @@ target "webapp" { } for _, test := range cases { t.Run(test.name, func(t *testing.T) { - m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides) + m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides) test.check(t, m, err) }) } @@ -185,14 +181,11 @@ target "webapp" { func TestReadTargetsCompose(t *testing.T) { t.Parallel() - tmpdir, err := ioutil.TempDir("", "bake") - require.NoError(t, err) - defer os.RemoveAll(tmpdir) - - fp := filepath.Join(tmpdir, "docker-compose.yml") - err = ioutil.WriteFile(fp, []byte(` -version: "3" + fp := File{ + Name: "docker-compose.yml", + Data: []byte( + `version: "3" services: db: build: . @@ -203,13 +196,13 @@ services: dockerfile: Dockerfile.webapp args: buildno: 1 -`), 0600) - require.NoError(t, err) - - fp2 := filepath.Join(tmpdir, "docker-compose2.yml") - err = ioutil.WriteFile(fp2, []byte(` -version: "3" +`), + } + fp2 := File{ + Name: "docker-compose2.yml", + Data: []byte( + `version: "3" services: newservice: build: . @@ -217,12 +210,12 @@ services: build: args: buildno2: 12 -`), 0600) - require.NoError(t, err) +`), + } ctx := context.TODO() - m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil) + m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil) require.NoError(t, err) require.Equal(t, 3, len(m)) diff --git a/bake/remote.go b/bake/remote.go new file mode 100644 index 00000000..4e28946b --- /dev/null +++ b/bake/remote.go @@ -0,0 +1,234 @@ +package bake + +import ( + "archive/tar" + "bytes" + "context" + "strings" + + "github.com/docker/buildx/build" + "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/progress" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/pkg/errors" +) + +type Input struct { + State *llb.State + URL string +} + +func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) { + st, filename, ok := detectHttpContext(url) + if !ok { + st, ok = detectGitContext(url) + if !ok { + return nil, nil, errors.Errorf("not url context") + } + } + + inp := &Input{State: st, URL: url} + var files []File + + var di *build.DriverInfo + for _, d := range dis { + if d.Err == nil { + di = &d + continue + } + } + if di == nil { + return nil, nil, nil + } + + c, err := driver.Boot(ctx, di.Driver, pw) + if err != nil { + return nil, nil, err + } + + _, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + res, err := c.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + if filename != "" { + files, err = filesFromURLRef(ctx, c, ref, inp, filename, names) + } else { + files, err = filesFromRef(ctx, ref, names) + } + return nil, err + }, progress.NewChannel(pw)) + + if err != nil { + return nil, nil, err + } + + return files, inp, nil +} + +func IsRemoteURL(url string) bool { + if _, _, ok := detectHttpContext(url); ok { + return true + } + if _, ok := detectGitContext(url); ok { + return true + } + return false +} + +func detectHttpContext(url string) (*llb.State, string, bool) { + if httpPrefix.MatchString(url) { + httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context")) + return &httpContext, "context", true + } + return nil, "", false +} + +func detectGitContext(ref string) (*llb.State, bool) { + found := false + if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) { + found = true + } + + for _, prefix := range []string{"git://", "github.com/", "git@"} { + if strings.HasPrefix(ref, prefix) { + found = true + break + } + } + if !found { + return nil, false + } + + parts := strings.SplitN(ref, "#", 2) + branch := "" + if len(parts) > 1 { + branch = parts[1] + } + gitOpts := []llb.GitOption{llb.WithCustomName("[internal] load git source " + ref)} + + st := llb.Git(parts[0], branch, gitOpts...) + return &st, true +} + +func isArchive(header []byte) bool { + for _, m := range [][]byte{ + {0x42, 0x5A, 0x68}, // bzip2 + {0x1F, 0x8B, 0x08}, // gzip + {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz + } { + if len(header) < len(m) { + continue + } + if bytes.Equal(m, header[:len(m)]) { + return true + } + } + + r := tar.NewReader(bytes.NewBuffer(header)) + _, err := r.Next() + return err == nil +} + +func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) { + stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename}) + if err != nil { + return nil, err + } + + dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + Filename: filename, + Range: &gwclient.FileRange{ + Length: 1024, + }, + }) + if err != nil { + return nil, err + } + + if isArchive(dt) { + bc := llb.Scratch().File(llb.Copy(inp.State, filename, "/", &llb.CopyInfo{ + AttemptUnpack: true, + })) + inp.State = &bc + inp.URL = "" + def, err := bc.Marshal(ctx) + if err != nil { + return nil, err + } + res, err := c.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + return filesFromRef(ctx, ref, names) + } + + inp.State = nil + name := inp.URL + inp.URL = "" + + if len(dt) > stat.Size() { + if stat.Size() > 1024*512 { + return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size") + } + + dt, err = ref.ReadFile(ctx, gwclient.ReadRequest{ + Filename: filename, + }) + if err != nil { + return nil, err + } + } + + return []File{{Name: name, Data: dt}}, nil +} + +func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) { + // TODO: auto-remove parent dir in needed + var files []File + + isDefault := false + if len(names) == 0 { + isDefault = true + names = defaultFilenames() + } + + for _, name := range names { + _, err := ref.StatFile(ctx, gwclient.StatRequest{Path: name}) + if err != nil { + if isDefault { + continue + } + return nil, err + } + dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name}) + if err != nil { + return nil, err + } + files = append(files, File{Name: name, Data: dt}) + } + + return files, nil +} diff --git a/build/build.go b/build/build.go index 1b6e09cf..7f7b229d 100644 --- a/build/build.go +++ b/build/build.go @@ -22,6 +22,7 @@ import ( dockerclient "github.com/docker/docker/client" "github.com/docker/docker/pkg/urlutil" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/util/entitlements" @@ -64,6 +65,7 @@ type Inputs struct { ContextPath string DockerfilePath string InStream io.Reader + ContextState *llb.State } type DriverInfo struct { @@ -723,6 +725,12 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr ) switch { + case inp.ContextState != nil: + if target.FrontendInputs == nil { + target.FrontendInputs = make(map[string]llb.State) + } + target.FrontendInputs["context"] = *inp.ContextState + target.FrontendInputs["dockerfile"] = *inp.ContextState case inp.ContextPath == "-": if inp.DockerfilePath == "-" { return nil, errStdinConflict @@ -789,6 +797,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr toRemove = append(toRemove, dockerfileDir) dockerfileName = "Dockerfile" target.FrontendAttrs["dockerfilekey"] = "dockerfile" + delete(target.FrontendInputs, "dockerfile") } if dockerfileName == "" { diff --git a/build/url.go b/build/url.go index 451b0146..bbde6b47 100644 --- a/build/url.go +++ b/build/url.go @@ -14,7 +14,7 @@ import ( ) func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) { - c, err := d.Client(ctx) + c, err := driver.Boot(ctx, d, pw) if err != nil { return "", err } diff --git a/commands/bake.go b/commands/bake.go index c5ab0bc7..0d37b100 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -1,11 +1,14 @@ package commands import ( + "context" "encoding/json" "fmt" "os" "github.com/docker/buildx/bake" + "github.com/docker/buildx/build" + "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" "github.com/moby/buildkit/util/appcontext" "github.com/pkg/errors" @@ -19,18 +22,16 @@ type bakeOptions struct { commonOptions } -func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error { +func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) { ctx := appcontext.Context() - if len(in.files) == 0 { - files, err := defaultFiles() - if err != nil { - return err - } - if len(files) == 0 { - return errors.Errorf("no docker-compose.yml or docker-bake.hcl found, specify build file with -f/--file") + var url string + + if len(targets) > 0 { + if bake.IsRemoteURL(targets[0]) { + url = targets[0] + targets = targets[1:] } - in.files = files } if len(targets) == 0 { @@ -52,8 +53,38 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error { if in.pull != nil { overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull)) } + contextPathHash, _ := os.Getwd() + + ctx2, cancel := context.WithCancel(context.TODO()) + defer cancel() + printer := progress.NewPrinter(ctx2, os.Stderr, in.progress) - m, err := bake.ReadTargets(ctx, in.files, targets, overrides) + defer func() { + if printer != nil { + err1 := printer.Wait() + if err == nil { + err = err1 + } + } + }() + + dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash) + if err != nil { + return err + } + + var files []bake.File + var inp *bake.Input + if url != "" { + files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer) + } else { + files, err = bake.ReadLocalFiles(in.files) + } + if err != nil { + return err + } + + m, err := bake.ReadTargets(ctx, files, targets, overrides) if err != nil { return err } @@ -63,40 +94,22 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error { if err != nil { return err } + err = printer.Wait() + printer = nil + if err != nil { + return err + } fmt.Fprintln(dockerCli.Out(), string(dt)) return nil } - bo, err := bake.TargetsToBuildOpt(m) + bo, err := bake.TargetsToBuildOpt(m, inp) if err != nil { return err } - contextPathHash, _ := os.Getwd() - - return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash, in.builder) -} - -func defaultFiles() ([]string, error) { - fns := []string{ - "docker-compose.yml", // support app - "docker-compose.yaml", // support app - "docker-bake.json", - "docker-bake.override.json", - "docker-bake.hcl", - "docker-bake.override.hcl", - } - out := make([]string, 0, len(fns)) - for _, f := range fns { - if _, err := os.Stat(f); err != nil { - if os.IsNotExist(errors.Cause(err)) { - continue - } - return nil, err - } - out = append(out, f) - } - return out, nil + _, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer) + return err } func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {