Merge branch 'master' into patch-1

pull/384/head
Andy MacKinlay 4 years ago committed by GitHub
commit 576b9caaa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,185 @@
name: build
on:
workflow_dispatch:
push:
branches:
- 'master'
tags:
- 'v*'
pull_request:
branches:
- 'master'
env:
REPO_SLUG_ORIGIN: "moby/buildkit:master"
CACHEKEY_BINARIES: "binaries"
PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
jobs:
base:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Cache ${{ env.CACHEKEY_BINARIES }}
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Build ${{ env.CACHEKEY_BINARIES }}
run: |
./hack/build_ci_first_pass binaries
env:
CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
CACHEDIR_TO: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}-new
-
# FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850
name: Move cache
run: |
rm -rf /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
mv /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}-new /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
test:
runs-on: ubuntu-latest
needs: [base]
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Cache ${{ env.CACHEKEY_BINARIES }}
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Test
run: |
make test
env:
TEST_COVERAGE: 1
TESTFLAGS: -v --parallel=6 --timeout=20m
CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
-
name: Send to Codecov
uses: codecov/codecov-action@v1
with:
file: ./coverage/coverage.txt
cross:
runs-on: ubuntu-latest
needs: [base]
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Cache ${{ env.CACHEKEY_BINARIES }}
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Cross
run: |
make cross
env:
TARGETPLATFORM: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64
CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
binaries:
runs-on: ubuntu-latest
needs: [test, cross]
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Prepare
id: prep
run: |
TAG=pr
if [[ $GITHUB_REF == refs/tags/v* ]]; then
TAG=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
fi
echo ::set-output name=tag::${TAG}
-
name: Cache ${{ env.CACHEKEY_BINARIES }}
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Build ${{ steps.prep.outputs.tag }}
run: |
./hack/release "${{ steps.prep.outputs.tag }}" release-out
env:
PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64
CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}
-
name: Move artifacts
run: |
mv ./release-out/**/* ./release-out/
-
name: Upload artifacts
uses: actions/upload-artifact@v2
with:
name: buildx
path: ./release-out/*
if-no-files-found: error
-
name: GitHub Release
if: startsWith(github.ref, 'refs/tags/v')
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
draft: true
files: ./release-out/*
name: ${{ steps.prep.outputs.tag }}

@ -0,0 +1,25 @@
# Workflow used to make a request to proxy.golang.org to refresh cache on https://pkg.go.dev/github.com/docker/buildx
# when a released of buildx is produced
name: godev
on:
push:
tags:
- 'v*'
jobs:
update:
runs-on: ubuntu-latest
steps:
-
name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.13
-
name: Call pkg.go.dev
run: |
go get github.com/${GITHUB_REPOSITORY}@${GITHUB_REF#refs/tags/}
env:
GO111MODULE: on
GOPROXY: https://proxy.golang.org

@ -0,0 +1,39 @@
name: validate
on:
workflow_dispatch:
push:
branches:
- 'master'
tags:
- 'v*'
pull_request:
branches:
- 'master'
env:
REPO_SLUG_ORIGIN: "moby/buildkit:master"
jobs:
validate:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
target:
- lint
- validate-vendor
- validate-docs
steps:
-
name: Checkout
uses: actions/checkout@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-
name: Run
run: |
make ${{ matrix.target }}

1
.gitignore vendored

@ -1,2 +1,3 @@
bin
coverage
cross-out

@ -0,0 +1,30 @@
run:
timeout: 10m
skip-files:
- ".*\\.pb\\.go$"
modules-download-mode: vendor
build-tags:
linters:
enable:
- gofmt
- govet
- deadcode
- goimports
- ineffassign
- misspell
- unused
- varcheck
- golint
- staticcheck
- typecheck
- structcheck
disable-all: true
issues:
exclude-rules:
- linters:
- golint
text: "stutters"

@ -1,35 +0,0 @@
dist: bionic
sudo: required
install:
- docker run --name buildkit --rm -d --privileged -p 1234:1234 $REPO_SLUG_ORIGIN --addr tcp://0.0.0.0:1234
- sudo docker cp buildkit:/usr/bin/buildctl /usr/bin/
- export BUILDKIT_HOST=tcp://0.0.0.0:1234
env:
global:
- PLATFORMS="linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
- CROSS_PLATFORMS="${PLATFORMS},darwin/amd64,windows/amd64"
- PREFER_BUILDCTL="1"
script:
- make binaries validate-all && TARGETPLATFORM="${CROSS_PLATFORMS}" ./hack/cross
deploy:
- provider: script
script: PLATFORMS="${CROSS_PLATFORMS}" ./hack/release $TRAVIS_TAG release-out
on:
repo: docker/buildx
tags: true
condition: $TRAVIS_TAG =~ ^v[0-9]
- provider: releases
api_key:
secure: "VKVL+tyS3BfqjM4VMGHoHJbcKY4mqq4AGrclVEvBnt0gm1LkGeKxSheCZgF1EC4oSV8rCy6dkoRWL0PLkl895MIl20Z4v53o1NOQ4Fn0A+eptnrld8jYUkL5PcD+kdEqv2GkBn7vO6E/fwYY/wH9FYlE+fXUa0c/YQGqNGS+XVDtgkftqBV+F2EzaIwk+D+QClFBRmKvIbXrUQASi1K6K2eT3gvzR4zh679TSdI2nbnTKtE06xG1PBFVmb1Ux3/Jz4yHFvf2d3M1mOyqIBsozKoyxisiFQxnm3FjhPrdlZJ9oy/nsQM3ahQKJ3DF8hiLI1LxcxRa6wo//t3uu2eJSYl/c5nu0T7gVw4sChQNy52fUhEGoDTDwYoAxsLSDXcpj1jevRsKvxt/dh2e2De1a9HYj5oM+z2O+pcyiY98cKDbhe2miUqUdiYMBy24xUunB46zVcJF3pIqCYtw5ts8ES6Ixn3u+4OGV/hMDrVdiG2bOZtNVkdbKMEkOEBGa3parPJ69jh6og639kdAD3DFxyZn3YKYuJlcNShn3tj6iPokBYhlLwwf8vuEV7gK7G0rDS9yxuF03jgkwpBBF2wy+u1AbJv241T7v2ZB8H8VlYyHA0E5pnoWbw+lIOTy4IAc8gIesMvDuFFi4r1okhiAt/24U0p4aAohjh1nPuU3spY="
file: release-out/**/*
skip_cleanup: true
file_glob: true
on:
repo: docker/buildx
tags: true
condition: $TRAVIS_TAG =~ ^v[0-9]

@ -0,0 +1,13 @@
ignore: |
/vendor
extends: default
yaml-files:
- '*.yaml'
- '*.yml'
rules:
truthy: disable
line-length: disable
document-start: disable

@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1.1-experimental
# syntax=docker/dockerfile:1.2
ARG DOCKERD_VERSION=19.03-rc
ARG DOCKERD_VERSION=19.03
ARG CLI_VERSION=19.03
FROM docker:$DOCKERD_VERSION AS dockerd-release
@ -8,7 +8,7 @@ FROM docker:$DOCKERD_VERSION AS dockerd-release
# xgo is a helper for golang cross-compilation
FROM --platform=$BUILDPLATFORM tonistiigi/xx:golang@sha256:6f7d999551dd471b58f70716754290495690efa8421e0a1fcf18eb11d0c0a537 AS xgo
FROM --platform=$BUILDPLATFORM golang:1.13-alpine AS gobase
FROM --platform=$BUILDPLATFORM golang:1.16-alpine AS gobase
COPY --from=xgo / /
RUN apk add --no-cache file git
ENV GOFLAGS=-mod=vendor

@ -150,6 +150,8 @@ made through a pull request.
[Org.Maintainers]
people = [
"akihirosuda",
"crazy-max",
"tiborvass",
"tonistiigi",
]
@ -176,6 +178,16 @@ made through a pull request.
# All other sections should refer to people by their canonical key
# in the people section.
[people.akihirosuda]
Name = "Akihiro Suda"
Email = "akihiro.suda.cz@hco.ntt.co.jp"
GitHub = "AkihiroSuda"
[people.crazy-max]
Name = "Kevin Alvarez"
Email = "contact@crazymax.dev"
GitHub = "crazy-max"
[people.thajeztah]
Name = "Sebastiaan van Stijn"
Email = "github@gone.nl"

@ -7,9 +7,12 @@ binaries:
binaries-cross:
EXPORT_LOCAL=cross-out ./hack/cross
cross:
./hack/cross
install: binaries
mkdir -p ~/.docker/cli-plugins
cp bin/buildx ~/.docker/cli-plugins/docker-buildx
install bin/buildx ~/.docker/cli-plugins/docker-buildx
lint:
./hack/lint
@ -20,12 +23,18 @@ test:
validate-vendor:
./hack/validate-vendor
validate-all: lint test validate-vendor
validate-docs:
./hack/validate-docs
validate-all: lint test validate-vendor validate-docs
vendor:
./hack/update-vendor
docs:
./hack/update-docs
generate-authors:
./hack/generate-authors
.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors
.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs

@ -1,16 +1,20 @@
# buildx
### Docker CLI plugin for extended build capabilities with BuildKit
_buildx is Tech Preview_
[![PkgGoDev](https://img.shields.io/badge/go.dev-docs-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/docker/buildx)
[![Build Status](https://github.com/docker/buildx/workflows/build/badge.svg)](https://github.com/docker/buildx/actions?query=workflow%3Abuild)
[![Go Report Card](https://goreportcard.com/badge/github.com/docker/buildx)](https://goreportcard.com/report/github.com/docker/buildx)
[![codecov](https://codecov.io/gh/docker/buildx/branch/master/graph/badge.svg)](https://codecov.io/gh/docker/buildx)
### TL;DR
`buildx` is a Docker CLI plugin for extended build capabilities with [BuildKit](https://github.com/moby/buildkit).
Key features:
- Familiar UI from `docker build`
- Full BuildKit capabilities with container driver
- Multiple builder instance support
- Multi-node builds for cross-platform images
- Compose build support
- WIP: High-level build constructs (`bake`)
- High-level build constructs (`bake`)
- In-container driver support (both Docker and Kubernetes)
# Table of Contents
@ -25,28 +29,29 @@ _buildx is Tech Preview_
* [Building multi-platform images](#building-multi-platform-images)
* [High-level build options](#high-level-build-options)
- [Documentation](#documentation)
+ [`buildx build [OPTIONS] PATH | URL | -`](#buildx-build-options-path--url---)
+ [`buildx create [OPTIONS] [CONTEXT|ENDPOINT]`](#buildx-create-options-contextendpoint)
+ [`buildx use NAME`](#buildx-use-name)
+ [`buildx inspect [NAME]`](#buildx-inspect-name)
+ [`buildx ls`](#buildx-ls)
+ [`buildx stop [NAME]`](#buildx-stop-name)
+ [`buildx rm [NAME]`](#buildx-rm-name)
+ [`buildx bake [OPTIONS] [TARGET...]`](#buildx-bake-options-target)
+ [`buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]`](#buildx-imagetools-create-options-source-source)
+ [`buildx imagetools inspect NAME`](#buildx-imagetools-inspect-name)
+ [`buildx build [OPTIONS] PATH | URL | -`](docs/reference/buildx_build.md)
+ [`buildx create [OPTIONS] [CONTEXT|ENDPOINT]`](docs/reference/buildx_create.md)
+ [`buildx use NAME`](docs/reference/buildx_use.md)
+ [`buildx inspect [NAME]`](docs/reference/buildx_inspect.md)
+ [`buildx ls`](docs/reference/buildx_ls.md)
+ [`buildx stop [NAME]`](docs/reference/buildx_stop.md)
+ [`buildx rm [NAME]`](docs/reference/buildx_rm.md)
+ [`buildx bake [OPTIONS] [TARGET...]`](docs/reference/buildx_bake.md)
+ [`buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]`](docs/reference/buildx_imagetools_create.md)
+ [`buildx imagetools inspect NAME`](docs/reference/buildx_imagetools_inspect.md)
- [Setting buildx as default builder in Docker 19.03+](#setting-buildx-as-default-builder-in-docker-1903)
- [Contributing](#contributing)
# Installing
Using `buildx` as a docker CLI plugin requires using Docker 19.03. A limited set of functionality works with older versions of Docker when invoking the binary directly.
Using `buildx` as a docker CLI plugin requires using Docker 19.03 or newer. A limited set of functionality works with older versions of Docker when invoking the binary directly.
### Docker CE
### Docker
`buildx` comes bundled by default with Docker Desktop and Docker CE starting with 19.03, but may not be present in all repackaged versions for Linux distributions (in which case follow the 'Binary release' instructions below). When it is included, it requires experimental mode to be enabled on the Docker CLI.
To enable it, `"experimental": "enabled"` can be added to the CLI configuration file `~/.docker/config.json`. An alternative is to set the `DOCKER_CLI_EXPERIMENTAL=enabled` environment variable.
`buildx` comes bundled by default with Docker Desktop, and with Docker CE packages starting with 19.03, but may not be present in all repackaged versions for Linux distributions (in which case follow the 'Binary release' instructions below).
When it is included, it requires experimental mode to be enabled on the Docker CLI.
To enable it, add `"experimental": "enabled"` to the CLI configuration file `~/.docker/config.json`, or set the `DOCKER_CLI_EXPERIMENTAL=enabled` environment variable.
### Binary release
@ -59,13 +64,8 @@ chmod a+x ~/.docker/cli-plugins/docker-buildx
# Building
### with Docker 18.09+
```
$ git clone git://github.com/docker/buildx && cd buildx
$ make install
```
### with buildx or Docker 19.03
### with buildx or Docker 19.03+
```
$ export DOCKER_BUILDKIT=1
$ docker build --platform=local -o . git://github.com/docker/buildx
@ -73,6 +73,12 @@ $ mkdir -p ~/.docker/cli-plugins
$ mv buildx ~/.docker/cli-plugins/docker-buildx
```
### with Docker 18.09+
```
$ git clone git://github.com/docker/buildx && cd buildx
$ make install
```
# Getting started
## Building with buildx
@ -92,7 +98,7 @@ Buildx will always build using the BuildKit engine and does not require `DOCKER_
Buildx build command supports the features available for `docker build` including the new features in Docker 19.03 such as outputs configuration, inline build caching or specifying target platform. In addition, buildx supports new features not yet available for regular `docker build` like building manifest lists, distributed caching, exporting build results to OCI image tarballs etc.
Buildx is supposed to be flexible and can be run in different configurations that are exposed through a driver concept. Currently, we support a "docker" driver that uses the BuildKit library bundled into the docker daemon binary, and a "docker-container" driver that automatically launches BuildKit inside a Docker container. We plan to add more drivers in the future, for example, one that would allow running buildx inside an (unprivileged) container.
Buildx is supposed to be flexible and can be run in different configurations that are exposed through a driver concept. Currently, we support a "docker" driver that uses the BuildKit library bundled into the Docker daemon binary, and a "docker-container" driver that automatically launches BuildKit inside a Docker container. We plan to add more drivers in the future, for example, one that would allow running buildx inside an (unprivileged) container.
The user experience of using buildx is very similar across drivers, but there are some features that are not currently supported by the "docker" driver, because the BuildKit library bundled into docker daemon currently uses a different storage component. In contrast, all images built with "docker" driver are automatically added to the "docker images" view by default, whereas when using other drivers the method for outputting an image needs to be selected with `--output`.
@ -105,7 +111,7 @@ Buildx allows you to create new instances of isolated builders. This can be used
New instances can be created with `docker buildx create` command. This will create a new builder instance with a single node based on your current configuration. To use a remote node you can specify the `DOCKER_HOST` or remote context name while creating the new builder. After creating a new instance you can manage its lifecycle with the `inspect`, `stop` and `rm` commands and list all available builders with `ls`. After creating a new builder you can also append new nodes to it.
To switch between different builders use `docker buildx use <name>`. After running this command the build commands would automatically keep using this builder.
To switch between different builders, use `docker buildx use <name>`. After running this command the build commands would automatically keep using this builder.
Docker 19.03 also features a new `docker context` command that can be used for giving names for remote Docker API endpoints. Buildx integrates with `docker context` so that all of your contexts automatically get a default builder instance. While creating a new builder instance or when adding a node to it you can also set the context name as the target.
@ -117,7 +123,7 @@ When invoking a build, the `--platform` flag can be used to specify the target p
Multi-platform images can be built by mainly three different strategies that are all supported by buildx and Dockerfiles. You can use the QEMU emulation support in the kernel, build on multiple native nodes using the same builder instance or use a stage in Dockerfile to cross-compile to different architectures.
QEMU is the easiest way to get started if your node already supports it (e.g. if you are using Docker Desktop). It requires no changes to your Dockerfile and BuildKit will automatically detect the secondary architectures that are available. When BuildKit needs to run a binary for a different architecture it will automatically load it through a binary registered in the binfmt_misc handler. For QEMU binaries registered with binfmt_misc on the host OS to work transparently inside containers they must be registed with the fix_binary flag. This requires a kernel >= 4.8 and binfmt-support >= 2.1.7. You can check for proper registration by checking if `F` is among the flags in `/proc/sys/fs/binfmt_misc/qemu-*`. While Docker Desktop comes preconfigured with binfmt_misc support for additional platforms, for other installations it likely needs to be installed using [`tonistiigi/binfmt`](https://github.com/tonistiigi/binfmt) image.
QEMU is the easiest way to get started if your node already supports it (e.g. if you are using Docker Desktop). It requires no changes to your Dockerfile and BuildKit will automatically detect the secondary architectures that are available. When BuildKit needs to run a binary for a different architecture it will automatically load it through a binary registered in the binfmt_misc handler. For QEMU binaries registered with binfmt_misc on the host OS to work transparently inside containers they must be registered with the fix_binary flag. This requires a kernel >= 4.8 and binfmt-support >= 2.1.7. You can check for proper registration by checking if `F` is among the flags in `/proc/sys/fs/binfmt_misc/qemu-*`. While Docker Desktop comes preconfigured with binfmt_misc support for additional platforms, for other installations it likely needs to be installed using [`tonistiigi/binfmt`](https://github.com/tonistiigi/binfmt) image.
```
$ docker run --privileged --rm tonistiigi/binfmt --install all
@ -155,659 +161,7 @@ Currently, the bake command supports building images from compose files, similar
There is also support for custom build rules from HCL/JSON files allowing better code reuse and different target groups. The design of bake is in very early stages and we are looking for feedback from users.
# Documentation
### `buildx build [OPTIONS] PATH | URL | -`
The `buildx build` command starts a build using BuildKit. This command is similar to the UI of `docker build` command and takes the same flags and arguments.
Options:
| Flag | Description |
| --- | --- |
| --add-host [] | Add a custom host-to-IP mapping (host:ip)
| --allow [] | Allow extra privileged entitlement, e.g. network.host, security.insecure
| --build-arg [] | Set build-time variables
| --cache-from [] | External cache sources (eg. user/app:cache, type=local,src=path/to/dir)
| --cache-to [] | Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)
| --file string | Name of the Dockerfile (Default is 'PATH/Dockerfile')
| --iidfile string | Write the image ID to the file
| --label [] | Set metadata for an image
| --load | Shorthand for --output=type=docker
| --network string | Set the networking mode for the RUN instructions during build (default "default")
| --no-cache | Do not use cache when building the image
| --output [] | Output destination (format: type=local,dest=path)
| --platform [] | Set target platform for build
| --progress string | Set type of progress output (auto, plain, tty). Use plain to show container output (default "auto")
| --pull | Always attempt to pull a newer version of the image
| --push | Shorthand for --output=type=registry
| --secret [] | Secret file to expose to the build: id=mysecret,src=/local/secret
| --ssh [] | SSH agent socket or keys to expose to the build (format: default\|&#60;id&#62;[=&#60;socket&#62;\|&#60;key&#62;[,&#60;key&#62;]])
| --tag [] | Name and optionally a tag in the 'name:tag' format
| --target string | Set the target build stage to build.
For documentation on most of these flags refer to `docker build` documentation in https://docs.docker.com/engine/reference/commandline/build/ . In here well document a subset of the new flags.
#### ` --platform=value[,value]`
Set the target platform for the build. All `FROM` commands inside the Dockerfile without their own `--platform` flag will pull base images for this platform and this value will also be the platform of the resulting image. The default value will be the current platform of the buildkit daemon.
When using `docker-container` driver with `buildx`, this flag can accept multiple values as an input separated by a comma. With multiple values the result will be built for all of the specified platforms and joined together into a single manifest list.
If the`Dockerfile` needs to invoke the `RUN` command, the builder needs runtime support for the specified platform. In a clean setup, you can only execute `RUN` commands for your system architecture. If your kernel supports binfmt_misc https://en.wikipedia.org/wiki/Binfmt_misc launchers for secondary architectures buildx will pick them up automatically. Docker desktop releases come with binfmt_misc automatically configured for `arm64` and `arm` architectures. You can see what runtime platforms your current builder instance supports by running `docker buildx inspect --bootstrap`.
Inside a `Dockerfile`, you can access the current platform value through `TARGETPLATFORM` build argument. Please refer to `docker build` documentation for the full description of automatic platform argument variants https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope .
The formatting for the platform specifier is defined in https://github.com/containerd/containerd/blob/v1.2.6/platforms/platforms.go#L63 .
Examples:
```
docker buildx build --platform=linux/arm64 .
docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 .
docker buildx build --platform=darwin .
```
#### `-o, --output=[PATH,-,type=TYPE[,KEY=VALUE]`
Sets the export action for the build result. In `docker build` all builds finish by creating a container image and exporting it to `docker images`. `buildx` makes this step configurable allowing results to be exported directly to the client, oci image tarballs, registry etc.
Buildx with `docker` driver currently only supports local, tarball exporter and image exporter. `docker-container` driver supports all the exporters.
If just the path is specified as a value, `buildx` will use the local exporter with this path as the destination. If the value is “-”, `buildx` will use `tar` exporter and write to `stdout`.
Examples:
```
docker buildx build -o . .
docker buildx build -o outdir .
docker buildx build -o - - > out.tar
docker buildx build -o type=docker .
docker buildx build -o type=docker,dest=- . > myimage.tar
docker buildx build -t tonistiigi/foo -o type=registry
````
Supported exported types are:
##### `local`
The `local` export type writes all result files to a directory on the client. The new files will be owned by the current user. On multi-platform builds, all results will be put in subdirectories by their platform.
Attribute key:
- `dest` - destination directory where files will be written
##### `tar`
The `tar` export type writes all result files as a single tarball on the client. On multi-platform builds all results will be put in subdirectories by their platform.
Attribute key:
- `dest` - destination path where tarball will be written. “-” writes to stdout.
##### `oci`
The `oci` export type writes the result image or manifest list as an OCI image layout tarball https://github.com/opencontainers/image-spec/blob/master/image-layout.md on the client.
Attribute key:
- `dest` - destination path where tarball will be written. “-” writes to stdout.
##### `docker`
The `docker` export type writes the single-platform result image as a Docker image specification tarball https://github.com/moby/moby/blob/master/image/spec/v1.2.md on the client. Tarballs created by this exporter are also OCI compatible.
Currently, multi-platform images cannot be exported with the `docker` export type. The most common usecase for multi-platform images is to directly push to a registry (see [`registry`](#registry)).
Attribute keys:
- `dest` - destination path where tarball will be written. If not specified the tar will be loaded automatically to the current docker instance.
- `context` - name for the docker context where to import the result
##### `image`
The `image` exporter writes the build result as an image or a manifest list. When using `docker` driver the image will appear in `docker images`. Optionally image can be automatically pushed to a registry by specifying attributes.
Attribute keys:
- `name` - name (references) for the new image.
- `push` - boolean to automatically push the image.
##### `registry`
The `registry` exporter is a shortcut for `type=image,push=true`.
#### `--push`
Shorthand for [`--output=type=registry`](#registry). Will automatically push the build result to registry.
#### `--load`
Shorthand for [`--output=type=docker`](#docker). Will automatically load the single-platform build result to `docker images`.
#### `--cache-from=[NAME|type=TYPE[,KEY=VALUE]]`
Use an external cache source for a build. Supported types are `registry` and `local`. The `registry` source can import cache from a cache manifest or (special) image configuration on the registry. The `local` source can import cache from local files previously exported with `--cache-to`.
If no type is specified, `registry` exporter is used with a specified reference.
`docker` driver currently only supports importing build cache from the registry.
Examples:
```
docker buildx build --cache-from=user/app:cache .
docker buildx build --cache-from=user/app .
docker buildx build --cache-from=type=registry,ref=user/app .
docker buildx build --cache-from=type=local,src=path/to/cache .
```
#### `--cache-to=[NAME|type=TYPE[,KEY=VALUE]]`
Export build cache to an external cache destination. Supported types are `registry`, `local` and `inline`. Registry exports build cache to a cache manifest in the registry, local exports cache to a local directory on the client and inline writes the cache metadata into the image configuration.
`docker` driver currently only supports exporting inline cache metadata to image configuration. Alternatively, `--build-arg BUILDKIT_INLINE_CACHE=1` can be used to trigger inline cache exporter.
Attribute key:
- `mode` - Specifies how many layers are exported with the cache. “min” on only exports layers already in the final build build stage, “max” exports layers for all stages. Metadata is always exported for the whole build.
Examples:
```
docker buildx build --cache-to=user/app:cache .
docker buildx build --cache-to=type=inline .
docker buildx build --cache-to=type=registry,ref=user/app .
docker buildx build --cache-to=type=local,dest=path/to/cache .
```
#### `--allow=ENTITLEMENT`
Allow extra privileged entitlement. List of entitlements:
- `network.host` - Allows executions with host networking.
- `security.insecure` - Allows executions without sandbox. See [related Dockerfile extensions](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---securityinsecuresandbox).
For entitlements to be enabled, the `buildkitd` daemon also needs to allow them with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](#--buildkitd-flags-flags))
Example:
```
$ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
$ docker buildx build --allow security.insecure .
```
### `buildx create [OPTIONS] [CONTEXT|ENDPOINT]`
Create makes a new builder instance pointing to a docker context or endpoint, where context is the name of a context from `docker context ls` and endpoint is the address for docker socket (eg. `DOCKER_HOST` value).
By default, the current docker configuration is used for determining the context/endpoint value.
Builder instances are isolated environments where builds can be invoked. All docker contexts also get the default builder instance.
Options:
| Flag | Description |
| --- | --- |
| --append | Append a node to builder instead of changing it
| --buildkitd-flags string | Flags for buildkitd daemon
| --config string | BuildKit config file
| --driver string | Driver to use (eg. docker-container)
| --driver-opt stringArray | Options for the driver
| --leave | Remove a node from builder instead of changing it
| --name string | Builder instance name
| --node string | Create/modify node with given name
| --platform stringArray | Fixed platforms for current node
| --use | Set the current builder instance
#### `--append`
Changes the action of the command to appends a new node to an existing builder specified by `--name`. Buildx will choose an appropriate node for a build based on the platforms it supports.
Example:
```
$ docker buildx create mycontext1
eager_beaver
$ docker buildx create --name eager_beaver --append mycontext2
eager_beaver
```
#### `--buildkitd-flags FLAGS`
Adds flags when starting the buildkitd daemon. They take precedence over the configuration file specified by [`--config`](#--config-file). See `buildkitd --help` for the available flags.
Example:
```
--buildkitd-flags '--debug --debugaddr 0.0.0.0:6666'
```
#### `--config FILE`
Specifies the configuration file for the buildkitd daemon to use. The configuration can be overridden by [`--buildkitd-flags`](#--buildkitd-flags-flags). See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
#### `--driver DRIVER`
Sets the builder driver to be used. There are two available drivers, each have their own specificities.
- `docker` - Uses the builder that is built into the docker daemon. With this driver, the [`--load`](#--load) flag is implied by default on `buildx build`. However, building multi-platform images or exporting cache is not currently supported.
- `docker-container` - Uses a buildkit container that will be spawned via docker. With this driver, both building multi-platform images and exporting cache are supported. However, images built will not automatically appear in `docker images` (see [`build --load`](#--load)).
- `kubernetes` - Uses a kubernetes pods. With this driver , you can spin up pods with defined buildkit container image to build your images.
#### `--driver-opt OPTIONS`
Passes additional driver-specific options. Details for each driver:
- `docker` - No driver options
- `docker-container`
- `image=IMAGE` - Sets the container image to be used for running buildkit.
- `network=NETMODE` - Sets the network mode for running the buildkit container.
- Example:
```
--driver docker-container --driver-opt image=moby/buildkit:master,network=host
```
- `kubernetes`
- `image=IMAGE` - Sets the container image to be used for running buildkit.
- `namespace=NS` - Sets the Kubernetes namespace. Defaults to the current namespace.
- `replicas=N` - Sets the number of `Pod` replicas. Defaults to 1.
- `nodeselector="label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
- `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
- `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
#### `--leave`
Changes the action of the command to removes a node from a builder. The builder needs to be specified with `--name` and node that is removed is set with `--node`.
Example:
```
docker buildx create --name mybuilder --node mybuilder0 --leave
```
#### `--name NAME`
Specifies the name of the builder to be created or modified. If none is specified, one will be automatically generated.
#### `--node NODE`
Specifies the name of the node to be created or modified. If none is specified, it is the name of the builder it belongs to, with an index number suffix.
#### `--platform PLATFORMS`
Sets the platforms supported by the node. It expects a comma-separated list of platforms of the form OS/architecture/variant. The node will also automatically detect the platforms it supports, but manual values take priority over the detected ones and can be used when multiple nodes support building for the same platform.
Example:
```
docker buildx create --platform linux/amd64
docker buildx create --platform linux/arm64,linux/arm/v8
```
#### `--use`
Automatically switches the current builder to the newly created one. Equivalent to running `docker buildx use $(docker buildx create ...)`.
### `buildx use NAME`
Switches the current builder instance. Build commands invoked after this command will run on a specified builder. Alternatively, a context name can be used to switch to the default builder of that context.
### `buildx inspect [NAME]`
Shows information about the current or specified builder.
Example:
```
Name: elated_tesla
Driver: docker-container
Nodes:
Name: elated_tesla0
Endpoint: unix:///var/run/docker.sock
Status: running
Platforms: linux/amd64
Name: elated_tesla1
Endpoint: ssh://ubuntu@1.2.3.4
Status: running
Platforms: linux/arm64, linux/arm/v7, linux/arm/v6
```
#### `--bootstrap`
Ensures that the builder is running before inspecting it. If the driver is `docker-container`, then `--bootstrap` starts the buildkit container and waits until it is operational. Bootstrapping is automatically done during build, it is thus not necessary. The same BuildKit container is used during the lifetime of the associated builder node (as displayed in `buildx ls`).
### `buildx ls`
Lists all builder instances and the nodes for each instance
Example:
```
docker buildx ls
NAME/NODE DRIVER/ENDPOINT STATUS PLATFORMS
elated_tesla * docker-container
elated_tesla0 unix:///var/run/docker.sock running linux/amd64
elated_tesla1 ssh://ubuntu@1.2.3.4 running linux/arm64, linux/arm/v7, linux/arm/v6
default docker
default default running linux/amd64
```
Each builder has one or more nodes associated with it. The current builders name is marked with a `*`.
### `buildx stop [NAME]`
Stops the specified or current builder. This will not prevent buildx build to restart the builder. The implementation of stop depends on the driver.
### `buildx rm [NAME]`
Removes the specified or current builder. It is a no-op attempting to remove the default builder.
### `buildx bake [OPTIONS] [TARGET...]`
Bake is a high-level build command.
Each specified target will run in parallel as part of the build.
Options:
| Flag | Description |
| --- | --- |
| -f, --file stringArray | Build definition file
| --load | Shorthand for --set=*.output=type=docker
| --no-cache | Do not use cache when building the image
| --print | Print the options without building
| --progress string | Set type of progress output (auto, plain, tty). Use plain to show container output (default "auto")
| --pull | Always attempt to pull a newer version of the image
| --push | Shorthand for --set=*.output=type=registry
| --set stringArray | Override target value (eg: targetpattern.key=value)
#### `-f, --file FILE`
Specifies the bake definition file. The file can be a Docker Compose, JSON or HCL file. If multiple files are specified they are all read and configurations are combined. By default, if no files are specified, the following are parsed:
docker-compose.yml
docker-compose.yaml
docker-bake.json
docker-bake.override.json
docker-bake.hcl
docker-bake.override.hcl
#### `--no-cache`
Same as `build --no-cache`. Do not use cache when building the image.
#### `--print`
Prints the resulting options of the targets desired to be built, in a JSON format, without starting a build.
```
$ docker buildx bake -f docker-bake.hcl --print db
{
"target": {
"db": {
"context": "./",
"dockerfile": "Dockerfile",
"tags": [
"docker.io/tiborvass/db"
]
}
}
}
```
#### `--progress`
Same as `build --progress`. Set type of progress output (auto, plain, tty). Use plain to show container output (default "auto").
#### `--pull`
Same as `build --pull`.
#### `--set targetpattern.key[.subkey]=value`
Override target configurations from command line. The pattern matching syntax is defined in https://golang.org/pkg/path/#Match.
Example:
```
docker buildx bake --set target.args.mybuildarg=value
docker buildx bake --set target.platform=linux/arm64
docker buildx bake --set foo*.args.mybuildarg=value # overrides build arg for all targets starting with 'foo'
docker buildx bake --set *.platform=linux/arm64 # overrides platform for all targets
docker buildx bake --set foo*.no-cache # bypass caching only for targets starting with 'foo'
```
Complete list of overridable fields:
args, cache-from, cache-to, context, dockerfile, labels, no-cache, output, platform, pull, secrets, ssh, tags, target
#### File definition
In addition to compose files, bake supports a JSON and an equivalent HCL file format for defining build groups and targets.
A target reflects a single docker build invocation with the same options that you would specify for `docker build`. A group is a grouping of targets.
Multiple files can include the same target and final build options will be determined by merging them together.
In the case of compose files, each service corresponds to a target.
A group can specify its list of targets with the `targets` option. A target can inherit build options by setting the `inherits` option to the list of targets or groups to inherit from.
Note: Design of bake command is work in progress, the user experience may change based on feedback.
Example HCL defintion:
```
group "default" {
targets = ["db", "webapp-dev"]
}
target "webapp-dev" {
dockerfile = "Dockerfile.webapp"
tags = ["docker.io/username/webapp"]
}
target "webapp-release" {
inherits = ["webapp-dev"]
platforms = ["linux/amd64", "linux/arm64"]
}
target "db" {
dockerfile = "Dockerfile.db"
tags = ["docker.io/username/db"]
}
```
Complete list of valid target fields:
args, cache-from, cache-to, context, dockerfile, inherits, labels, no-cache, output, platform, pull, secrets, ssh, tags, target
#### HCL variables and functions
Similar to how Terraform provides a way to [define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable), the HCL file format also supports variable block definitions. These can be used to define variables with values provided by the current environment or a default value when unset.
Example of using interpolation to tag an image with the git sha:
```
$ cat <<'EOF' > docker-bake.hcl
variable "TAG" {
default = "latest"
}
group "default" {
targets = ["webapp"]
}
target "webapp" {
tags = ["docker.io/username/webapp:${TAG}"]
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"tags": [
"docker.io/username/webapp:latest"
]
}
}
}
$ TAG=$(git rev-parse --short HEAD) docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"tags": [
"docker.io/username/webapp:985e9e9"
]
}
}
}
```
A [set of generally useful functions](https://github.com/docker/buildx/blob/master/bake/hcl.go#L19-L65) provided by [go-cty](https://github.com/zclconf/go-cty/tree/master/cty/function/stdlib) are avaialble for use in HCL files. In addition, [user defined functions](https://github.com/hashicorp/hcl/tree/hcl2/ext/userfunc) are also supported.
Example of using the `add` function:
```
$ cat <<'EOF' > docker-bake.hcl
variable "TAG" {
default = "latest"
}
group "default" {
targets = ["webapp"]
}
target "webapp" {
args = {
buildno = "${add(123, 1)}"
}
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"args": {
"buildno": "124"
}
}
}
}
```
Example of defining an `increment` function:
```
$ cat <<'EOF' > docker-bake.hcl
function "increment" {
params = [number]
result = number + 1
}
group "default" {
targets = ["webapp"]
}
target "webapp" {
args = {
buildno = "${increment(123)}"
}
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"args": {
"buildno": "124"
}
}
}
}
```
### `buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]`
Imagetools contains commands for working with manifest lists in the registry. These commands are useful for inspecting multi-platform build results.
Create creates a new manifest list based on source manifests. The source manifests can be manifest lists or single platform distribution manifests and must already exist in the registry where the new manifest is created. If only one source is specified create performs a carbon copy.
Options:
| Flag | Description |
| --- | --- |
| --append | Append to existing manifest
| --dry-run | Show final image instead of pushing
| -f, --file stringArray | Read source descriptor from file
| -t, --tag stringArray | Set reference for new image
#### `--append`
Append appends the new sources to an existing manifest list in the destination.
#### `--dry-run`
Do not push the image, just show it.
#### `-f, --file FILE`
Reads source from files. A source can be a manifest digest, manifest reference or a JSON of OCI descriptor object.
#### `-t, --tag IMAGE`
Name of the image to be created.
Examples:
```
docker buildx imagetools create --dry-run alpine@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907 sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
docker buildx imagetools create -t tonistiigi/myapp -f image1 -f image2
```
### `buildx imagetools inspect NAME`
Show details of image in the registry.
Example:
```
$ docker buildx imagetools inspect alpine
Name: docker.io/library/alpine:latest
MediaType: application/vnd.docker.distribution.manifest.list.v2+json
Digest: sha256:28ef97b8686a0b5399129e9b763d5b7e5ff03576aa5580d6f4182a49c5fe1913
Manifests:
Name: docker.io/library/alpine:latest@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907
MediaType: application/vnd.docker.distribution.manifest.v2+json
Platform: linux/amd64
Name: docker.io/library/alpine:latest@sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
MediaType: application/vnd.docker.distribution.manifest.v2+json
Platform: linux/arm/v6
...
```
#### `--raw`
Raw prints the original JSON bytes instead of the formatted output.
[`buildx bake` Reference Docs](docs/reference/buildx_bake.md)
# Setting buildx as default builder in Docker 19.03+

@ -5,26 +5,66 @@ import (
"io/ioutil"
"os"
"path"
"regexp"
"strconv"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/docker/pkg/urlutil"
hcl "github.com/hashicorp/hcl/v2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/pkg/errors"
)
func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) {
var c Config
for _, f := range files {
cfg, err := ParseFile(f)
var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
type File struct {
Name string
Data []byte
}
func defaultFilenames() []string {
return []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
}
func ReadLocalFiles(names []string) ([]File, error) {
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
out := make([]File, 0, len(names))
for _, n := range names {
dt, err := ioutil.ReadFile(n)
if err != nil {
if isDefault && errors.Is(err, os.ErrNotExist) {
continue
}
return nil, err
}
c = mergeConfig(c, *cfg)
out = append(out, File{Name: n, Data: dt})
}
return out, nil
}
func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
c, err := parseFiles(files)
if err != nil {
return nil, err
}
o, err := c.newOverrides(overrides)
if err != nil {
return nil, err
@ -44,30 +84,73 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s
return m, nil
}
func ParseFile(fn string) (*Config, error) {
dt, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
func parseFiles(files []File) (*Config, error) {
var c Config
var fs []*hcl.File
var scs []*StaticConfig
for _, f := range files {
cfg, f, sc, err := parseFile(f.Data, f.Name)
if err != nil {
return nil, err
}
if cfg != nil {
c = mergeConfig(c, *cfg)
} else {
fs = append(fs, f)
scs = append(scs, sc)
}
}
if len(fs) > 0 {
cfg, err := ParseHCL(hcl.MergeFiles(fs), mergeStaticConfig(scs))
if err != nil {
return nil, err
}
c = mergeConfig(c, dedupeConfig(*cfg))
}
return &c, nil
}
func dedupeConfig(c Config) Config {
c2 := c
c2.Targets = make([]*Target, 0, len(c2.Targets))
m := map[string]*Target{}
for _, t := range c.Targets {
if t2, ok := m[t.Name]; ok {
merge(t2, t)
} else {
m[t.Name] = t
c2.Targets = append(c2.Targets, t)
}
}
return c2
}
func ParseFile(dt []byte, fn string) (*Config, error) {
return parseFiles([]File{{Data: dt, Name: fn}})
}
func parseFile(dt []byte, fn string) (*Config, *hcl.File, *StaticConfig, error) {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
return ParseCompose(dt)
c, err := ParseCompose(dt)
return c, nil, nil, err
}
if strings.HasSuffix(fnl, ".json") || strings.HasSuffix(fnl, ".hcl") {
return ParseHCL(dt, fn)
f, sc, err := ParseHCLFile(dt, fn)
return nil, f, sc, err
}
cfg, err := ParseCompose(dt)
if err != nil {
cfg, err2 := ParseHCL(dt, fn)
f, sc, err2 := ParseHCLFile(dt, fn)
if err2 != nil {
return nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
return nil, nil, nil, errors.Errorf("failed to parse %s: parsing yaml: %s, parsing hcl: %s", fn, err.Error(), err2.Error())
}
return cfg, nil
return nil, f, sc, nil
}
return cfg, nil
return cfg, nil, nil, nil
}
type Config struct {
@ -320,8 +403,15 @@ func (c Config) target(name string, visited map[string]struct{}, overrides map[s
}
type Variable struct {
Name string `json:"-" hcl:"name,label"`
Default string `json:"default,omitempty" hcl:"default,optional"`
Name string `json:"-" hcl:"name,label"`
Default *hcl.Attribute `json:"default,omitempty" hcl:"default,optional"`
}
type Function struct {
Name string `json:"-" hcl:"name,label"`
Params *hcl.Attribute `json:"params,omitempty" hcl:"params"`
Variadic *hcl.Attribute `json:"variadic_param,omitempty" hcl:"variadic_params"`
Result *hcl.Attribute `json:"result,omitempty" hcl:"result"`
}
type Group struct {
@ -336,20 +426,22 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
}
@ -363,10 +455,10 @@ func (t *Target) normalize() {
t.Outputs = removeDupes(t.Outputs)
}
func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m))
for k, v := range m {
bo, err := toBuildOpt(v)
bo, err := toBuildOpt(v, inp)
if err != nil {
return nil, err
}
@ -375,7 +467,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
return m2, nil
}
func toBuildOpt(t *Target) (*build.Options, error) {
func updateContext(t *build.Inputs, inp *Input) {
if inp == nil || inp.State == nil {
return
}
if t.ContextPath == "." {
t.ContextPath = inp.URL
return
}
st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
t.ContextState = &st
}
func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
if v := t.Context; v != nil && *v == "-" {
return nil, errors.Errorf("context from stdin not allowed in bake")
}
@ -387,6 +491,7 @@ func toBuildOpt(t *Target) (*build.Options, error) {
if t.Context != nil {
contextPath = *t.Context
}
contextPath = path.Clean(contextPath)
dockerfilePath := "Dockerfile"
if t.Dockerfile != nil {
dockerfilePath = *t.Dockerfile
@ -405,11 +510,17 @@ func toBuildOpt(t *Target) (*build.Options, error) {
pull = *t.Pull
}
bi := build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
}
if t.DockerfileInline != nil {
bi.DockerfileInline = *t.DockerfileInline
}
updateContext(&bi, inp)
bo := &build.Options{
Inputs: build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
},
Inputs: bi,
Tags: t.Tags,
BuildArgs: t.Args,
Labels: t.Labels,
@ -425,13 +536,17 @@ func toBuildOpt(t *Target) (*build.Options, error) {
bo.Session = append(bo.Session, authprovider.NewDockerAuthProvider(os.Stderr))
secrets, err := build.ParseSecretSpecs(t.Secrets)
secrets, err := buildflags.ParseSecretSpecs(t.Secrets)
if err != nil {
return nil, err
}
bo.Session = append(bo.Session, secrets)
ssh, err := build.ParseSSHSpecs(t.SSH)
sshSpecs := t.SSH
if len(sshSpecs) == 0 && buildflags.IsGitSSH(contextPath) {
sshSpecs = []string{"default"}
}
ssh, err := buildflags.ParseSSHSpecs(sshSpecs)
if err != nil {
return nil, err
}
@ -441,19 +556,19 @@ func toBuildOpt(t *Target) (*build.Options, error) {
bo.Target = *t.Target
}
cacheImports, err := build.ParseCacheEntry(t.CacheFrom)
cacheImports, err := buildflags.ParseCacheEntry(t.CacheFrom)
if err != nil {
return nil, err
}
bo.CacheFrom = cacheImports
cacheExports, err := build.ParseCacheEntry(t.CacheTo)
cacheExports, err := buildflags.ParseCacheEntry(t.CacheTo)
if err != nil {
return nil, err
}
bo.CacheTo = cacheExports
outputs, err := build.ParseOutputs(t.Outputs)
outputs, err := buildflags.ParseOutputs(t.Outputs)
if err != nil {
return nil, err
}
@ -473,6 +588,9 @@ func merge(t1, t2 *Target) *Target {
if t2.Dockerfile != nil {
t1.Dockerfile = t2.Dockerfile
}
if t2.DockerfileInline != nil {
t1.DockerfileInline = t2.DockerfileInline
}
for k, v := range t2.Args {
if t1.Args == nil {
t1.Args = map[string]string{}
@ -526,6 +644,9 @@ func removeDupes(s []string) []string {
if _, ok := seen[v]; ok {
continue
}
if v == "" {
continue
}
seen[v] = struct{}{}
s[i] = v
i++

@ -2,9 +2,7 @@ package bake
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
@ -12,12 +10,10 @@ import (
func TestReadTargets(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "config.hcl")
err = ioutil.WriteFile(fp, []byte(`
fp := File{
Name: "config.hcl",
Data: []byte(`
target "webDEP" {
args = {
VAR_INHERITED = "webDEP"
@ -32,13 +28,13 @@ target "webapp" {
VAR_BOTH = "webapp"
}
inherits = ["webDEP"]
}`), 0600)
require.NoError(t, err)
}`),
}
ctx := context.TODO()
t.Run("NoOverrides", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil)
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))
@ -50,7 +46,7 @@ target "webapp" {
})
t.Run("InvalidTargetOverrides", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
require.NotNil(t, err)
require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
})
@ -60,7 +56,7 @@ target "webapp" {
os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv")
defer os.Unsetenv("VAR_FROM_ENV" + t.Name())
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webapp.args.VAR_UNSET",
"webapp.args.VAR_EMPTY=",
"webapp.args.VAR_SET=bananas",
@ -89,7 +85,7 @@ target "webapp" {
// building leaf but overriding parent fields
t.Run("parent", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webDEP.args.VAR_INHERITED=override",
"webDEP.args.VAR_BOTH=override",
})
@ -100,23 +96,23 @@ target "webapp" {
})
t.Run("ContextOverride", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
require.NotNil(t, err)
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
require.NoError(t, err)
require.Equal(t, "foo", *m["webapp"].Context)
})
t.Run("NoCacheOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].NoCache)
})
t.Run("PullOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].Pull)
})
@ -176,7 +172,7 @@ target "webapp" {
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides)
m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
test.check(t, m, err)
})
}
@ -185,14 +181,11 @@ target "webapp" {
func TestReadTargetsCompose(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "docker-compose.yml")
err = ioutil.WriteFile(fp, []byte(`
version: "3"
fp := File{
Name: "docker-compose.yml",
Data: []byte(
`version: "3"
services:
db:
build: .
@ -203,13 +196,13 @@ services:
dockerfile: Dockerfile.webapp
args:
buildno: 1
`), 0600)
require.NoError(t, err)
fp2 := filepath.Join(tmpdir, "docker-compose2.yml")
err = ioutil.WriteFile(fp2, []byte(`
version: "3"
`),
}
fp2 := File{
Name: "docker-compose2.yml",
Data: []byte(
`version: "3"
services:
newservice:
build: .
@ -217,12 +210,12 @@ services:
build:
args:
buildno2: 12
`), 0600)
require.NoError(t, err)
`),
}
ctx := context.TODO()
m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil)
m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
require.NoError(t, err)
require.Equal(t, 3, len(m))

@ -56,7 +56,7 @@ func ParseCompose(dt []byte) (*Config, error) {
if reflect.DeepEqual(s.Build, zeroBuildConfig) {
// if not make sure they're setting an image or it's invalid d-c.yml
if s.Image == "" {
return nil, fmt.Errorf("compose file invalid: service %s has neither an image nor a build context specified. At least one must be provided.", s.Name)
return nil, fmt.Errorf("compose file invalid: service %s has neither an image nor a build context specified. At least one must be provided", s.Name)
}
continue
}

@ -1,14 +1,28 @@
package bake
import (
"math"
"math/big"
"os"
"reflect"
"strconv"
"strings"
"unsafe"
"github.com/docker/buildx/util/userfunc"
"github.com/hashicorp/go-cty-funcs/cidr"
"github.com/hashicorp/go-cty-funcs/crypto"
"github.com/hashicorp/go-cty-funcs/encoding"
"github.com/hashicorp/go-cty-funcs/uuid"
hcl "github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/ext/userfunc"
"github.com/hashicorp/hcl/v2/hclsimple"
"github.com/hashicorp/hcl/v2/ext/tryfunc"
"github.com/hashicorp/hcl/v2/ext/typeexpr"
"github.com/hashicorp/hcl/v2/gohcl"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/hashicorp/hcl/v2/json"
hcljson "github.com/hashicorp/hcl/v2/json"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/solver/pb"
"github.com/pkg/errors"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
"github.com/zclconf/go-cty/cty/function/stdlib"
@ -21,16 +35,26 @@ var (
"absolute": stdlib.AbsoluteFunc,
"add": stdlib.AddFunc,
"and": stdlib.AndFunc,
"base64decode": encoding.Base64DecodeFunc,
"base64encode": encoding.Base64EncodeFunc,
"bcrypt": crypto.BcryptFunc,
"byteslen": stdlib.BytesLenFunc,
"bytesslice": stdlib.BytesSliceFunc,
"can": tryfunc.CanFunc,
"ceil": stdlib.CeilFunc,
"chomp": stdlib.ChompFunc,
"chunklist": stdlib.ChunklistFunc,
"ceil": stdlib.CeilFunc,
"cidrhost": cidr.HostFunc,
"cidrnetmask": cidr.NetmaskFunc,
"cidrsubnet": cidr.SubnetFunc,
"cidrsubnets": cidr.SubnetsFunc,
"csvdecode": stdlib.CSVDecodeFunc,
"coalesce": stdlib.CoalesceFunc,
"coalescelist": stdlib.CoalesceListFunc,
"compact": stdlib.CompactFunc,
"concat": stdlib.ConcatFunc,
"contains": stdlib.ContainsFunc,
"convert": typeexpr.ConvertFunc,
"distinct": stdlib.DistinctFunc,
"divide": stdlib.DivideFunc,
"element": stdlib.ElementFunc,
@ -57,6 +81,7 @@ var (
"lookup": stdlib.LookupFunc,
"lower": stdlib.LowerFunc,
"max": stdlib.MaxFunc,
"md5": crypto.Md5Func,
"merge": stdlib.MergeFunc,
"min": stdlib.MinFunc,
"modulo": stdlib.ModuloFunc,
@ -70,13 +95,19 @@ var (
"range": stdlib.RangeFunc,
"regexall": stdlib.RegexAllFunc,
"regex": stdlib.RegexFunc,
"regex_replace": stdlib.RegexReplaceFunc,
"reverse": stdlib.ReverseFunc,
"reverselist": stdlib.ReverseListFunc,
"rsadecrypt": crypto.RsaDecryptFunc,
"sethaselement": stdlib.SetHasElementFunc,
"setintersection": stdlib.SetIntersectionFunc,
"setproduct": stdlib.SetProductFunc,
"setsubtract": stdlib.SetSubtractFunc,
"setsymmetricdifference": stdlib.SetSymmetricDifferenceFunc,
"setunion": stdlib.SetUnionFunc,
"sha1": crypto.Sha1Func,
"sha256": crypto.Sha256Func,
"sha512": crypto.Sha512Func,
"signum": stdlib.SignumFunc,
"slice": stdlib.SliceFunc,
"sort": stdlib.SortFunc,
@ -90,85 +121,527 @@ var (
"trimprefix": stdlib.TrimPrefixFunc,
"trimspace": stdlib.TrimSpaceFunc,
"trimsuffix": stdlib.TrimSuffixFunc,
"try": tryfunc.TryFunc,
"upper": stdlib.UpperFunc,
"urlencode": encoding.URLEncodeFunc,
"uuidv4": uuid.V4Func,
"uuidv5": uuid.V5Func,
"values": stdlib.ValuesFunc,
"zipmap": stdlib.ZipmapFunc,
}
)
// Used in the first pass of decoding instead of the Config struct to disallow
// interpolation while parsing variable blocks.
type staticConfig struct {
type StaticConfig struct {
Variables []*Variable `hcl:"variable,block"`
Functions []*Function `hcl:"function,block"`
Remain hcl.Body `hcl:",remain"`
attrs hcl.Attributes
defaults map[string]*hcl.Attribute
funcDefs map[string]*Function
funcs map[string]function.Function
env map[string]string
ectx hcl.EvalContext
progress map[string]struct{}
progressF map[string]struct{}
}
func ParseHCL(dt []byte, fn string) (*Config, error) {
// Decode user defined functions, first parsing as hcl and falling back to
// json, returning errors based on the file suffix.
file, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
func mergeStaticConfig(scs []*StaticConfig) *StaticConfig {
if len(scs) == 0 {
return nil
}
sc := scs[0]
for _, s := range scs[1:] {
sc.Variables = append(sc.Variables, s.Variables...)
sc.Functions = append(sc.Functions, s.Functions...)
for k, v := range s.attrs {
sc.attrs[k] = v
}
}
return sc
}
func (sc *StaticConfig) EvalContext(withEnv bool) (*hcl.EvalContext, error) {
// json parser also parses blocks as attributes
delete(sc.attrs, "target")
delete(sc.attrs, "function")
sc.defaults = map[string]*hcl.Attribute{}
for _, v := range sc.Variables {
if v.Name == "target" {
continue
}
sc.defaults[v.Name] = v.Default
}
sc.env = map[string]string{}
if withEnv {
// Override default with values from environment.
for _, v := range os.Environ() {
parts := strings.SplitN(v, "=", 2)
name, value := parts[0], parts[1]
sc.env[name] = value
}
}
sc.funcDefs = map[string]*Function{}
for _, v := range sc.Functions {
sc.funcDefs[v.Name] = v
}
sc.ectx = hcl.EvalContext{
Variables: map[string]cty.Value{},
Functions: stdlibFunctions,
}
sc.funcs = map[string]function.Function{}
sc.progress = map[string]struct{}{}
sc.progressF = map[string]struct{}{}
for k := range sc.attrs {
if err := sc.resolveValue(k); err != nil {
return nil, err
}
}
for k := range sc.defaults {
if err := sc.resolveValue(k); err != nil {
return nil, err
}
}
for k := range sc.funcDefs {
if err := sc.resolveFunction(k); err != nil {
return nil, err
}
}
return &sc.ectx, nil
}
type jsonExp interface {
ExprList() []hcl.Expression
ExprMap() []hcl.KeyValuePair
}
func elementExpressions(je jsonExp, exp hcl.Expression) []hcl.Expression {
list := je.ExprList()
if len(list) != 0 {
exp := make([]hcl.Expression, 0, len(list))
for _, e := range list {
if je, ok := e.(jsonExp); ok {
exp = append(exp, elementExpressions(je, e)...)
}
}
return exp
}
kvlist := je.ExprMap()
if len(kvlist) != 0 {
exp := make([]hcl.Expression, 0, len(kvlist)*2)
for _, p := range kvlist {
exp = append(exp, p.Key)
if je, ok := p.Value.(jsonExp); ok {
exp = append(exp, elementExpressions(je, p.Value)...)
}
}
return exp
}
return []hcl.Expression{exp}
}
func jsonFuncCallsRecursive(exp hcl.Expression) ([]string, error) {
je, ok := exp.(jsonExp)
if !ok {
return nil, errors.Errorf("invalid expression type %T", exp)
}
m := map[string]struct{}{}
for _, e := range elementExpressions(je, exp) {
if err := appendJSONFuncCalls(e, m); err != nil {
return nil, err
}
}
arr := make([]string, 0, len(m))
for n := range m {
arr = append(arr, n)
}
return arr, nil
}
func appendJSONFuncCalls(exp hcl.Expression, m map[string]struct{}) error {
v := reflect.ValueOf(exp)
if v.Kind() != reflect.Ptr || v.IsNil() {
return errors.Errorf("invalid json expression kind %T %v", exp, v.Kind())
}
if v.Elem().Kind() != reflect.Struct {
return errors.Errorf("invalid json expression pointer to %T %v", exp, v.Elem().Kind())
}
src := v.Elem().FieldByName("src")
if src.IsZero() {
return errors.Errorf("%v has no property src", v.Elem().Type())
}
if src.Kind() != reflect.Interface {
return errors.Errorf("%v src is not interface: %v", src.Type(), src.Kind())
}
src = src.Elem()
if src.IsNil() {
return nil
}
if src.Kind() == reflect.Ptr {
src = src.Elem()
}
if src.Kind() != reflect.Struct {
return errors.Errorf("%v is not struct: %v", src.Type(), src.Kind())
}
// hcl/v2/json/ast#stringVal
val := src.FieldByName("Value")
if val.IsZero() {
return nil
}
rng := src.FieldByName("SrcRange")
if val.IsZero() {
return nil
}
var stringVal struct {
Value string
SrcRange hcl.Range
}
if !val.Type().AssignableTo(reflect.ValueOf(stringVal.Value).Type()) {
return nil
}
if !rng.Type().AssignableTo(reflect.ValueOf(stringVal.SrcRange).Type()) {
return nil
}
// reflect.Set does not work for unexported fields
stringVal.Value = *(*string)(unsafe.Pointer(val.UnsafeAddr()))
stringVal.SrcRange = *(*hcl.Range)(unsafe.Pointer(rng.UnsafeAddr()))
expr, diags := hclsyntax.ParseExpression([]byte(stringVal.Value), stringVal.SrcRange.Filename, stringVal.SrcRange.Start)
if diags.HasErrors() {
return nil
}
fns, err := funcCalls(expr)
if err != nil {
return err
}
for _, fn := range fns {
m[fn] = struct{}{}
}
return nil
}
func funcCalls(exp hcl.Expression) ([]string, hcl.Diagnostics) {
node, ok := exp.(hclsyntax.Node)
if !ok {
fns, err := jsonFuncCallsRecursive(exp)
if err != nil {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: exp.Range().Ptr(),
Context: exp.Range().Ptr(),
},
}
}
return fns, nil
}
var funcnames []string
hcldiags := hclsyntax.VisitAll(node, func(n hclsyntax.Node) hcl.Diagnostics {
if fe, ok := n.(*hclsyntax.FunctionCallExpr); ok {
funcnames = append(funcnames, fe.Name)
}
return nil
})
if hcldiags.HasErrors() {
var jsondiags hcl.Diagnostics
file, jsondiags = json.Parse(dt, fn)
if jsondiags.HasErrors() {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".json") {
return nil, jsondiags
} else {
return nil, hcldiags
return nil, hcldiags
}
return funcnames, nil
}
func (sc *StaticConfig) loadDeps(exp hcl.Expression, exclude map[string]struct{}) hcl.Diagnostics {
fns, hcldiags := funcCalls(exp)
if hcldiags.HasErrors() {
return hcldiags
}
for _, fn := range fns {
if err := sc.resolveFunction(fn); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: exp.Range().Ptr(),
Context: exp.Range().Ptr(),
},
}
}
}
for _, v := range exp.Variables() {
if _, ok := exclude[v.RootName()]; ok {
continue
}
if err := sc.resolveValue(v.RootName()); err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid expression",
Detail: err.Error(),
Subject: v.SourceRange().Ptr(),
Context: v.SourceRange().Ptr(),
},
}
}
}
userFunctions, _, diags := userfunc.DecodeUserFunctions(file.Body, "function", func() *hcl.EvalContext {
return &hcl.EvalContext{
Functions: stdlibFunctions,
return nil
}
func (sc *StaticConfig) resolveFunction(name string) error {
if _, ok := sc.funcs[name]; ok {
return nil
}
f, ok := sc.funcDefs[name]
if !ok {
if _, ok := sc.ectx.Functions[name]; ok {
return nil
}
return errors.Errorf("undefined function %s", name)
}
if _, ok := sc.progressF[name]; ok {
return errors.Errorf("function cycle not allowed for %s", name)
}
sc.progressF[name] = struct{}{}
paramExprs, paramsDiags := hcl.ExprList(f.Params.Expr)
if paramsDiags.HasErrors() {
return paramsDiags
}
var diags hcl.Diagnostics
params := map[string]struct{}{}
for _, paramExpr := range paramExprs {
param := hcl.ExprAsKeyword(paramExpr)
if param == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid param element",
Detail: "Each parameter name must be an identifier.",
Subject: paramExpr.Range().Ptr(),
})
}
params[param] = struct{}{}
}
var variadic hcl.Expression
if f.Variadic != nil {
variadic = f.Variadic.Expr
param := hcl.ExprAsKeyword(variadic)
if param == "" {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid param element",
Detail: "Each parameter name must be an identifier.",
Subject: f.Variadic.Range.Ptr(),
})
}
params[param] = struct{}{}
}
if diags.HasErrors() {
return diags
}
if diags := sc.loadDeps(f.Result.Expr, params); diags.HasErrors() {
return diags
}
v, diags := userfunc.NewFunction(f.Params.Expr, variadic, f.Result.Expr, func() *hcl.EvalContext {
return &sc.ectx
})
if diags.HasErrors() {
return nil, diags
return diags
}
sc.funcs[name] = v
sc.ectx.Functions[name] = v
var sc staticConfig
return nil
}
// Decode only variable blocks without interpolation.
if err := hclsimple.Decode(fn, dt, nil, &sc); err != nil {
return nil, err
func (sc *StaticConfig) resolveValue(name string) (err error) {
if _, ok := sc.ectx.Variables[name]; ok {
return nil
}
if _, ok := sc.progress[name]; ok {
return errors.Errorf("variable cycle not allowed for %s", name)
}
sc.progress[name] = struct{}{}
// Set all variables to their default value if defined.
variables := make(map[string]cty.Value)
for _, variable := range sc.Variables {
variables[variable.Name] = cty.StringVal(variable.Default)
var v *cty.Value
defer func() {
if v != nil {
sc.ectx.Variables[name] = *v
}
}()
def, ok := sc.attrs[name]
if !ok {
def, ok = sc.defaults[name]
if !ok {
return errors.Errorf("undefined variable %q", name)
}
}
if def == nil {
vv := cty.StringVal(sc.env[name])
v = &vv
return
}
// Override default with values from environment.
for _, env := range os.Environ() {
parts := strings.SplitN(env, "=", 2)
name, value := parts[0], parts[1]
if _, ok := variables[name]; ok {
variables[name] = cty.StringVal(value)
if diags := sc.loadDeps(def.Expr, nil); diags.HasErrors() {
return diags
}
vv, diags := def.Expr.Value(&sc.ectx)
if diags.HasErrors() {
return diags
}
_, isVar := sc.defaults[name]
if envv, ok := sc.env[name]; ok && isVar {
if vv.Type().Equals(cty.Bool) {
b, err := strconv.ParseBool(envv)
if err != nil {
return errors.Wrapf(err, "failed to parse %s as bool", name)
}
vv := cty.BoolVal(b)
v = &vv
return nil
} else if vv.Type().Equals(cty.String) {
vv := cty.StringVal(envv)
v = &vv
return nil
} else if vv.Type().Equals(cty.Number) {
n, err := strconv.ParseFloat(envv, 64)
if err == nil && (math.IsNaN(n) || math.IsInf(n, 0)) {
err = errors.Errorf("invalid number value")
}
if err != nil {
return errors.Wrapf(err, "failed to parse %s as number", name)
}
vv := cty.NumberVal(big.NewFloat(n))
v = &vv
return nil
} else {
// TODO: support lists with csv values
return errors.Errorf("unsupported type %s for variable %s", v.Type(), name)
}
}
v = &vv
return nil
}
func ParseHCLFile(dt []byte, fn string) (*hcl.File, *StaticConfig, error) {
if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
return parseHCLFile(dt, fn)
}
f, sc, err := parseHCLFile(dt, fn+".hcl")
if err != nil {
f, sc, err2 := parseHCLFile(dt, fn+".json")
if err2 == nil {
return f, sc, nil
}
}
return f, sc, err
}
func parseHCLFile(dt []byte, fn string) (f *hcl.File, _ *StaticConfig, err error) {
defer func() {
err = formatHCLError(dt, err)
}()
// Decode user defined functions, first parsing as hcl and falling back to
// json, returning errors based on the file suffix.
f, hcldiags := hclsyntax.ParseConfig(dt, fn, hcl.Pos{Line: 1, Column: 1})
if hcldiags.HasErrors() {
var jsondiags hcl.Diagnostics
f, jsondiags = hcljson.Parse(dt, fn)
if jsondiags.HasErrors() {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".json") {
return nil, nil, jsondiags
}
return nil, nil, hcldiags
}
}
functions := make(map[string]function.Function)
for k, v := range stdlibFunctions {
functions[k] = v
var sc StaticConfig
// Decode only variable blocks without interpolation.
if err := gohcl.DecodeBody(f.Body, nil, &sc); err != nil {
return nil, nil, err
}
for k, v := range userFunctions {
functions[k] = v
attrs, diags := f.Body.JustAttributes()
if diags.HasErrors() {
for _, d := range diags {
if d.Detail != "Blocks are not allowed here." {
return nil, nil, diags
}
}
}
sc.attrs = attrs
return f, &sc, nil
}
ctx := &hcl.EvalContext{
Variables: variables,
Functions: functions,
func ParseHCL(b hcl.Body, sc *StaticConfig) (_ *Config, err error) {
ctx, err := sc.EvalContext(true)
if err != nil {
return nil, err
}
var c Config
// Decode with variables and functions.
if err := hclsimple.Decode(fn, dt, ctx, &c); err != nil {
if err := gohcl.DecodeBody(b, ctx, &c); err != nil {
return nil, err
}
return &c, nil
}
func formatHCLError(dt []byte, err error) error {
if err == nil {
return nil
}
diags, ok := err.(hcl.Diagnostics)
if !ok {
return err
}
for _, d := range diags {
if d.Severity != hcl.DiagError {
continue
}
if d.Subject != nil {
src := errdefs.Source{
Info: &pb.SourceInfo{
Filename: d.Subject.Filename,
Data: dt,
},
Ranges: []*pb.Range{toErrRange(d.Subject)},
}
err = errdefs.WithSource(err, src)
break
}
}
return err
}
func toErrRange(in *hcl.Range) *pb.Range {
return &pb.Range{
Start: pb.Position{Line: int32(in.Start.Line), Character: int32(in.Start.Column)},
End: pb.Position{Line: int32(in.End.Line), Character: int32(in.End.Column)},
}
}

@ -7,11 +7,9 @@ import (
"github.com/stretchr/testify/require"
)
func TestParseHCL(t *testing.T) {
func TestHCLBasic(t *testing.T) {
t.Parallel()
t.Run("Basic", func(t *testing.T) {
dt := []byte(`
dt := []byte(`
group "default" {
targets = ["db", "webapp"]
}
@ -44,32 +42,32 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
require.NoError(t, err)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
require.Equal(t, 4, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "db")
require.Equal(t, "./db", *c.Targets[0].Context)
require.Equal(t, 4, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "db")
require.Equal(t, "./db", *c.Targets[0].Context)
require.Equal(t, c.Targets[1].Name, "webapp")
require.Equal(t, 1, len(c.Targets[1].Args))
require.Equal(t, "123", c.Targets[1].Args["buildno"])
require.Equal(t, c.Targets[1].Name, "webapp")
require.Equal(t, 1, len(c.Targets[1].Args))
require.Equal(t, "123", c.Targets[1].Args["buildno"])
require.Equal(t, c.Targets[2].Name, "cross")
require.Equal(t, 2, len(c.Targets[2].Platforms))
require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
require.Equal(t, c.Targets[2].Name, "cross")
require.Equal(t, 2, len(c.Targets[2].Platforms))
require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
require.Equal(t, c.Targets[3].Name, "webapp-plus")
require.Equal(t, 1, len(c.Targets[3].Args))
require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
})
require.Equal(t, c.Targets[3].Name, "webapp-plus")
require.Equal(t, 1, len(c.Targets[3].Args))
require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
}
t.Run("BasicInJSON", func(t *testing.T) {
dt := []byte(`
func TestHCLBasicInJSON(t *testing.T) {
dt := []byte(`
{
"group": {
"default": {
@ -104,32 +102,32 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.json")
require.NoError(t, err)
c, err := ParseFile(dt, "docker-bake.json")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"db", "webapp"}, c.Groups[0].Targets)
require.Equal(t, 4, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "db")
require.Equal(t, "./db", *c.Targets[0].Context)
require.Equal(t, 4, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "db")
require.Equal(t, "./db", *c.Targets[0].Context)
require.Equal(t, c.Targets[1].Name, "webapp")
require.Equal(t, 1, len(c.Targets[1].Args))
require.Equal(t, "123", c.Targets[1].Args["buildno"])
require.Equal(t, c.Targets[1].Name, "webapp")
require.Equal(t, 1, len(c.Targets[1].Args))
require.Equal(t, "123", c.Targets[1].Args["buildno"])
require.Equal(t, c.Targets[2].Name, "cross")
require.Equal(t, 2, len(c.Targets[2].Platforms))
require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
require.Equal(t, c.Targets[2].Name, "cross")
require.Equal(t, 2, len(c.Targets[2].Platforms))
require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[2].Platforms)
require.Equal(t, c.Targets[3].Name, "webapp-plus")
require.Equal(t, 1, len(c.Targets[3].Args))
require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
})
require.Equal(t, c.Targets[3].Name, "webapp-plus")
require.Equal(t, 1, len(c.Targets[3].Args))
require.Equal(t, map[string]string{"IAMCROSS": "true"}, c.Targets[3].Args)
}
t.Run("WithFunctions", func(t *testing.T) {
dt := []byte(`
func TestHCLWithFunctions(t *testing.T) {
dt := []byte(`
group "default" {
targets = ["webapp"]
}
@ -141,20 +139,20 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
require.NoError(t, err)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "124", c.Targets[0].Args["buildno"])
})
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "124", c.Targets[0].Args["buildno"])
}
t.Run("WithUserDefinedFunctions", func(t *testing.T) {
dt := []byte(`
func TestHCLWithUserDefinedFunctions(t *testing.T) {
dt := []byte(`
function "increment" {
params = [number]
result = number + 1
@ -171,20 +169,20 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
require.NoError(t, err)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "124", c.Targets[0].Args["buildno"])
})
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "124", c.Targets[0].Args["buildno"])
}
t.Run("WithVariables", func(t *testing.T) {
dt := []byte(`
func TestHCLWithVariables(t *testing.T) {
dt := []byte(`
variable "BUILD_NUMBER" {
default = "123"
}
@ -200,54 +198,371 @@ func TestParseHCL(t *testing.T) {
}
`)
c, err := ParseHCL(dt, "docker-bake.hcl")
require.NoError(t, err)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "123", c.Targets[0].Args["buildno"])
os.Setenv("BUILD_NUMBER", "456")
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "123", c.Targets[0].Args["buildno"])
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
os.Setenv("BUILD_NUMBER", "456")
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "456", c.Targets[0].Args["buildno"])
}
func TestHCLWithVariablesInFunctions(t *testing.T) {
dt := []byte(`
variable "REPO" {
default = "user/repo"
}
function "tag" {
params = [tag]
result = ["${REPO}:${tag}"]
}
target "webapp" {
tags = tag("v1")
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, []string{"user/repo:v1"}, c.Targets[0].Tags)
os.Setenv("REPO", "docker/buildx")
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, []string{"docker/buildx:v1"}, c.Targets[0].Tags)
}
func TestHCLMultiFileSharedVariables(t *testing.T) {
dt := []byte(`
variable "FOO" {
default = "abc"
}
target "app" {
args = {
v1 = "pre-${FOO}"
}
}
`)
dt2 := []byte(`
target "app" {
args = {
v2 = "${FOO}-post"
}
}
`)
c, err = ParseHCL(dt, "docker-bake.hcl")
require.NoError(t, err)
c, err := parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-abc", c.Targets[0].Args["v1"])
require.Equal(t, "abc-post", c.Targets[0].Args["v2"])
require.Equal(t, 1, len(c.Groups))
require.Equal(t, "default", c.Groups[0].Name)
require.Equal(t, []string{"webapp"}, c.Groups[0].Targets)
os.Setenv("FOO", "def")
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "webapp")
require.Equal(t, "456", c.Targets[0].Args["buildno"])
c, err = parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-def", c.Targets[0].Args["v1"])
require.Equal(t, "def-post", c.Targets[0].Args["v2"])
}
t.Run("WithIncorrectVariables", func(t *testing.T) {
dt := []byte(`
variable "DEFAULT_BUILD_NUMBER" {
default = "1"
func TestHCLVarsWithVars(t *testing.T) {
os.Unsetenv("FOO")
dt := []byte(`
variable "FOO" {
default = upper("${BASE}def")
}
variable "BAR" {
default = "-${FOO}-"
}
target "app" {
args = {
v1 = "pre-${BAR}"
}
}
`)
dt2 := []byte(`
variable "BASE" {
default = "abc"
}
target "app" {
args = {
v2 = "${FOO}-post"
}
}
`)
variable "BUILD_NUMBER" {
default = "${DEFAULT_BUILD_NUMBER}"
c, err := parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre--ABCDEF-", c.Targets[0].Args["v1"])
require.Equal(t, "ABCDEF-post", c.Targets[0].Args["v2"])
os.Setenv("BASE", "new")
c, err = parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre--NEWDEF-", c.Targets[0].Args["v1"])
require.Equal(t, "NEWDEF-post", c.Targets[0].Args["v2"])
}
func TestHCLTypedVariables(t *testing.T) {
os.Unsetenv("FOO")
dt := []byte(`
variable "FOO" {
default = 3
}
variable "IS_FOO" {
default = true
}
target "app" {
args = {
v1 = FOO > 5 ? "higher" : "lower"
v2 = IS_FOO ? "yes" : "no"
}
}
`)
group "default" {
targets = ["webapp"]
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "lower", c.Targets[0].Args["v1"])
require.Equal(t, "yes", c.Targets[0].Args["v2"])
os.Setenv("FOO", "5.1")
os.Setenv("IS_FOO", "0")
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "higher", c.Targets[0].Args["v1"])
require.Equal(t, "no", c.Targets[0].Args["v2"])
os.Setenv("FOO", "NaN")
_, err = ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
require.Contains(t, err.Error(), "failed to parse FOO as number")
os.Setenv("FOO", "0")
os.Setenv("IS_FOO", "maybe")
_, err = ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
require.Contains(t, err.Error(), "failed to parse IS_FOO as bool")
}
func TestHCLVariableCycle(t *testing.T) {
dt := []byte(`
variable "FOO" {
default = BAR
}
variable "FOO2" {
default = FOO
}
variable "BAR" {
default = FOO
}
target "app" {}
`)
target "webapp" {
_, err := ParseFile(dt, "docker-bake.hcl")
require.Error(t, err)
require.Contains(t, err.Error(), "variable cycle not allowed")
}
func TestHCLAttrs(t *testing.T) {
dt := []byte(`
FOO="abc"
BAR="attr-${FOO}def"
target "app" {
args = {
buildno = "${BUILD_NUMBER}"
"v1": BAR
}
}
`)
_, err := ParseHCL(dt, "docker-bake.hcl")
require.Error(t, err)
require.Contains(t, err.Error(), "docker-bake.hcl:7,17-37: Variables not allowed; Variables may not be used here.")
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "attr-abcdef", c.Targets[0].Args["v1"])
// env does not apply if no variable
os.Setenv("FOO", "bar")
c, err = ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "attr-abcdef", c.Targets[0].Args["v1"])
// attr-multifile
}
func TestHCLAttrsCustomType(t *testing.T) {
dt := []byte(`
platforms=["linux/arm64", "linux/amd64"]
target "app" {
platforms = platforms
args = {
"v1": platforms[0]
}
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, []string{"linux/arm64", "linux/amd64"}, c.Targets[0].Platforms)
require.Equal(t, "linux/arm64", c.Targets[0].Args["v1"])
}
func TestHCLMultiFileAttrs(t *testing.T) {
os.Unsetenv("FOO")
dt := []byte(`
variable "FOO" {
default = "abc"
}
target "app" {
args = {
v1 = "pre-${FOO}"
}
}
`)
dt2 := []byte(`
FOO="def"
`)
c, err := parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-def", c.Targets[0].Args["v1"])
os.Setenv("FOO", "ghi")
c, err = parseFiles([]File{
{Data: dt, Name: "c1.hcl"},
{Data: dt2, Name: "c2.hcl"},
})
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-ghi", c.Targets[0].Args["v1"])
}
func TestJSONAttributes(t *testing.T) {
dt := []byte(`{"FOO": "abc", "variable": {"BAR": {"default": "def"}}, "target": { "app": { "args": {"v1": "pre-${FOO}-${BAR}"}} } }`)
c, err := ParseFile(dt, "docker-bake.json")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-abc-def", c.Targets[0].Args["v1"])
}
func TestJSONFunctions(t *testing.T) {
dt := []byte(`{
"FOO": "abc",
"function": {
"myfunc": {
"params": ["inp"],
"result": "<${upper(inp)}-${FOO}>"
}
},
"target": {
"app": {
"args": {
"v1": "pre-${myfunc(\"foo\")}"
}
}
}}`)
c, err := ParseFile(dt, "docker-bake.json")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "pre-<FOO-abc>", c.Targets[0].Args["v1"])
}
func TestHCLFunctionInAttr(t *testing.T) {
dt := []byte(`
function "brace" {
params = [inp]
result = "[${inp}]"
}
function "myupper" {
params = [val]
result = "${upper(val)} <> ${brace(v2)}"
}
v1=myupper("foo")
v2=lower("BAZ")
target "app" {
args = {
"v1": v1
}
}
`)
c, err := ParseFile(dt, "docker-bake.hcl")
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, c.Targets[0].Name, "app")
require.Equal(t, "FOO <> [baz]", c.Targets[0].Args["v1"])
}

@ -0,0 +1,236 @@
package bake
import (
"archive/tar"
"bytes"
"context"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Input struct {
State *llb.State
URL string
}
func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
st, filename, ok := detectHTTPContext(url)
if !ok {
st, ok = detectGitContext(url)
if !ok {
return nil, nil, errors.Errorf("not url context")
}
}
inp := &Input{State: st, URL: url}
var files []File
var di *build.DriverInfo
for _, d := range dis {
if d.Err == nil {
di = &d
continue
}
}
if di == nil {
return nil, nil, nil
}
c, err := driver.Boot(ctx, di.Driver, pw)
if err != nil {
return nil, nil, err
}
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
if filename != "" {
files, err = filesFromURLRef(ctx, c, ref, inp, filename, names)
} else {
files, err = filesFromRef(ctx, ref, names)
}
return nil, err
}, ch)
if err != nil {
return nil, nil, err
}
return files, inp, nil
}
func IsRemoteURL(url string) bool {
if _, _, ok := detectHTTPContext(url); ok {
return true
}
if _, ok := detectGitContext(url); ok {
return true
}
return false
}
func detectHTTPContext(url string) (*llb.State, string, bool) {
if httpPrefix.MatchString(url) {
httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context"))
return &httpContext, "context", true
}
return nil, "", false
}
func detectGitContext(ref string) (*llb.State, bool) {
found := false
if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
found = true
}
for _, prefix := range []string{"git://", "github.com/", "git@"} {
if strings.HasPrefix(ref, prefix) {
found = true
break
}
}
if !found {
return nil, false
}
parts := strings.SplitN(ref, "#", 2)
branch := ""
if len(parts) > 1 {
branch = parts[1]
}
gitOpts := []llb.GitOption{llb.WithCustomName("[internal] load git source " + ref)}
st := llb.Git(parts[0], branch, gitOpts...)
return &st, true
}
func isArchive(header []byte) bool {
for _, m := range [][]byte{
{0x42, 0x5A, 0x68}, // bzip2
{0x1F, 0x8B, 0x08}, // gzip
{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz
} {
if len(header) < len(m) {
continue
}
if bytes.Equal(m, header[:len(m)]) {
return true
}
}
r := tar.NewReader(bytes.NewBuffer(header))
_, err := r.Next()
return err == nil
}
func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) {
stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename})
if err != nil {
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
Range: &gwclient.FileRange{
Length: 1024,
},
})
if err != nil {
return nil, err
}
if isArchive(dt) {
bc := llb.Scratch().File(llb.Copy(inp.State, filename, "/", &llb.CopyInfo{
AttemptUnpack: true,
}))
inp.State = &bc
inp.URL = ""
def, err := bc.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
return filesFromRef(ctx, ref, names)
}
inp.State = nil
name := inp.URL
inp.URL = ""
if len(dt) > stat.Size() {
if stat.Size() > 1024*512 {
return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size")
}
dt, err = ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
})
if err != nil {
return nil, err
}
}
return []File{{Name: name, Data: dt}}, nil
}
func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) {
// TODO: auto-remove parent dir in needed
var files []File
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
for _, name := range names {
_, err := ref.StatFile(ctx, gwclient.StatRequest{Path: name})
if err != nil {
if isDefault {
continue
}
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name})
if err != nil {
return nil, err
}
files = append(files, File{Name: name, Data: dt})
}
return files, nil
}

@ -3,6 +3,7 @@ package build
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
@ -11,6 +12,7 @@ import (
"strconv"
"strings"
"sync"
"time"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
@ -19,12 +21,18 @@ import (
"github.com/docker/buildx/util/progress"
clitypes "github.com/docker/cli/cli/config/types"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/docker/pkg/urlutil"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/util/apicaps"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/progress/progresswriter"
"github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@ -61,9 +69,11 @@ type Options struct {
}
type Inputs struct {
ContextPath string
DockerfilePath string
InStream io.Reader
ContextPath string
DockerfilePath string
InStream io.Reader
ContextState *llb.State
DockerfileInline string
}
type DriverInfo struct {
@ -102,6 +112,7 @@ type driverPair struct {
driverIndex int
platforms []specs.Platform
so *client.SolveOpt
bopts gateway.BuildOpts
}
func driverIndexes(m map[string][]driverPair) []int {
@ -172,8 +183,40 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
return m
}
func resolveDrivers(ctx context.Context, drivers []DriverInfo, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
dps, clients, err := resolveDriversBase(ctx, drivers, auth, opt, pw)
if err != nil {
return nil, nil, err
}
bopts := make([]gateway.BuildOpts, len(clients))
eg, ctx := errgroup.WithContext(ctx)
for i, c := range clients {
func(i int, c *client.Client) {
eg.Go(func() error {
clients[i].Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
bopts[i] = c.BuildOpts()
return nil, nil
}, nil)
return nil
})
}(i, c)
}
if err := eg.Wait(); err != nil {
return nil, nil, err
}
for key := range dps {
for i, dp := range dps[key] {
dps[key][i].bopts = bopts[dp.driverIndex]
}
}
return dps, clients, nil
}
func resolveDriversBase(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
availablePlatforms := map[string]int{}
for i, d := range drivers {
for _, p := range d.Platform {
@ -238,6 +281,7 @@ func resolveDrivers(ctx context.Context, drivers []DriverInfo, opt map[string]Op
workers[i] = ww
return nil
})
}(i)
}
@ -278,14 +322,7 @@ func toRepoOnly(in string) (string, error) {
return strings.Join(out, ","), nil
}
func isDefaultMobyDriver(d driver.Driver) bool {
_, ok := d.(interface {
IsDefaultMobyDriver()
})
return ok
}
func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Options, bopts gateway.BuildOpts, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
defers := make([]func(), 0, 2)
releaseF := func() {
for _, f := range defers {
@ -322,29 +359,51 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
}
}
cacheTo := make([]client.CacheOptionsEntry, 0, len(opt.CacheTo))
for _, e := range opt.CacheTo {
if e.Type == "gha" {
if !bopts.LLBCaps.Contains(apicaps.CapID("cache.gha")) {
continue
}
}
cacheTo = append(cacheTo, e)
}
cacheFrom := make([]client.CacheOptionsEntry, 0, len(opt.CacheFrom))
for _, e := range opt.CacheFrom {
if e.Type == "gha" {
if !bopts.LLBCaps.Contains(apicaps.CapID("cache.gha")) {
continue
}
}
cacheFrom = append(cacheFrom, e)
}
so := client.SolveOpt{
Frontend: "dockerfile.v0",
FrontendAttrs: map[string]string{},
LocalDirs: map[string]string{},
CacheExports: opt.CacheTo,
CacheImports: opt.CacheFrom,
CacheExports: cacheTo,
CacheImports: cacheFrom,
AllowedEntitlements: opt.Allow,
}
if v, ok := opt.BuildArgs["BUILDKIT_MULTI_PLATFORM"]; ok {
if v, _ := strconv.ParseBool(v); v {
so.FrontendAttrs["multi-platform"] = "true"
}
}
if multiDriver {
// force creation of manifest list
so.FrontendAttrs["multi-platform"] = "true"
}
_, isDefaultMobyDriver := d.(interface {
IsDefaultMobyDriver()
})
switch len(opt.Exports) {
case 1:
// valid
case 0:
if isDefaultMobyDriver && !noDefaultLoad() {
if d.IsMobyDriver() && !noDefaultLoad() {
// backwards compat for docker driver only:
// this ensures the build results in a docker image.
opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}}
@ -398,7 +457,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
}
if e.Type == "docker" {
if e.Output == nil {
if isDefaultMobyDriver {
if d.IsMobyDriver() {
e.Type = "image"
} else {
w, cancel, err := dl(e.Attrs["context"])
@ -412,11 +471,13 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
return nil, nil, notSupported(d, driver.DockerExporter)
}
}
if e.Type == "image" && isDefaultMobyDriver {
if e.Type == "image" && d.IsMobyDriver() {
opt.Exports[i].Type = "moby"
if e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, nil, errors.Errorf("auto-push is currently not implemented for docker driver, please create a new builder instance")
if ok, _ := strconv.ParseBool(e.Attrs["push-by-digest"]); ok {
return nil, nil, errors.Errorf("push-by-digest is currently not implemented for docker driver, please create a new builder instance")
}
}
}
}
@ -425,7 +486,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
so.Exports = opt.Exports
so.Session = opt.Session
releaseLoad, err := LoadInputs(opt.Inputs, &so)
releaseLoad, err := LoadInputs(ctx, d, opt.Inputs, pw, &so)
if err != nil {
return nil, nil, err
}
@ -461,9 +522,11 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
// setup networkmode
switch opt.NetworkMode {
case "host", "none":
case "host":
so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
so.AllowedEntitlements = append(so.AllowedEntitlements, entitlements.EntitlementNetworkHost)
case "none":
so.FrontendAttrs["force-network-mode"] = opt.NetworkMode
case "", "default":
default:
return nil, nil, errors.Errorf("network mode %q not supported by buildkit", opt.NetworkMode)
@ -479,7 +542,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
return &so, releaseF, nil
}
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, pw progress.Writer) (resp map[string]*client.SolveResponse, err error) {
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build")
}
@ -491,7 +554,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
var noMobyDriver driver.Driver
for _, d := range drivers {
if !isDefaultMobyDriver(d.Driver) {
if !d.Driver.IsMobyDriver() {
noMobyDriver = d.Driver
break
}
@ -506,10 +569,8 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}
m, clients, err := resolveDrivers(ctx, drivers, opt, pw)
m, clients, err := resolveDrivers(ctx, drivers, auth, opt, w)
if err != nil {
close(pw.Status())
<-pw.Done()
return nil, err
}
@ -522,16 +583,19 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}()
mw := progress.NewMultiWriter(pw)
eg, ctx := errgroup.WithContext(ctx)
for k, opt := range opt {
multiDriver := len(m[k]) > 1
hasMobyDriver := false
for i, dp := range m[k] {
d := drivers[dp.driverIndex].Driver
if d.IsMobyDriver() {
hasMobyDriver = true
}
opt.Platforms = dp.platforms
so, release, err := toSolveOpt(d, multiDriver, opt, func(name string) (io.WriteCloser, func(), error) {
return newDockerLoader(ctx, docker, name, mw)
so, release, err := toSolveOpt(ctx, d, multiDriver, opt, dp.bopts, w, func(name string) (io.WriteCloser, func(), error) {
return newDockerLoader(ctx, docker, name, w)
})
if err != nil {
return nil, err
@ -539,6 +603,28 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
defers = append(defers, release)
m[k][i].so = so
}
for _, at := range opt.Session {
if s, ok := at.(interface {
SetLogger(progresswriter.Logger)
}); ok {
s.SetLogger(func(s *client.SolveStatus) {
w.Write(s)
})
}
}
// validate for multi-node push
if hasMobyDriver && multiDriver {
for _, dp := range m[k] {
for _, e := range dp.so.Exports {
if e.Type == "moby" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
return nil, errors.Errorf("multi-node push can't currently be performed with the docker driver, please switch to a different driver")
}
}
}
}
}
}
resp = map[string]*client.SolveResponse{}
@ -559,8 +645,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
var pushNames string
eg.Go(func() error {
pw := mw.WithPrefix("default", false)
defer close(pw.Status())
pw := progress.WithPrefix(w, "default", false)
wg.Wait()
select {
case <-ctx.Done():
@ -663,27 +748,43 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
func(i int, dp driverPair, so client.SolveOpt) {
pw := mw.WithPrefix(k, multiTarget)
pw := progress.WithPrefix(w, k, multiTarget)
c := clients[dp.driverIndex]
var statusCh chan *client.SolveStatus
if pw != nil {
pw = progress.ResetTime(pw)
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
return pw.Err()
})
}
pw = progress.ResetTime(pw)
eg.Go(func() error {
defer wg.Done()
rr, err := c.Solve(ctx, nil, so, statusCh)
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
rr, err := c.Solve(ctx, nil, so, ch)
if err != nil {
return err
}
res[i] = rr
d := drivers[dp.driverIndex].Driver
if d.IsMobyDriver() {
for _, e := range so.Exports {
if e.Type == "moby" && e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
pushNames = e.Attrs["name"]
if pushNames == "" {
return errors.Errorf("tag is needed when pushing to registry")
}
pw := progress.ResetTime(pw)
for _, name := range strings.Split(pushNames, ",") {
if err := progress.Wrap(fmt.Sprintf("pushing %s with docker", name), pw.Write, func(l progress.SubLogger) error {
return pushWithMoby(ctx, d, name, l)
}); err != nil {
return err
}
}
}
}
}
}
return nil
})
@ -704,6 +805,86 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
return resp, nil
}
func pushWithMoby(ctx context.Context, d driver.Driver, name string, l progress.SubLogger) error {
api := d.Config().DockerAPI
if api == nil {
return errors.Errorf("invalid empty Docker API reference") // should never happen
}
creds, err := imagetools.RegistryAuthForRef(name, d.Config().Auth)
if err != nil {
return err
}
rc, err := api.ImagePush(ctx, name, types.ImagePushOptions{
RegistryAuth: creds,
})
if err != nil {
return err
}
started := map[string]*client.VertexStatus{}
defer func() {
for _, st := range started {
if st.Completed == nil {
now := time.Now()
st.Completed = &now
l.SetStatus(st)
}
}
}()
dec := json.NewDecoder(rc)
var parsedError error
for {
var jm jsonmessage.JSONMessage
if err := dec.Decode(&jm); err != nil {
if parsedError != nil {
return parsedError
}
if err == io.EOF {
break
}
return err
}
if jm.ID != "" {
id := "pushing layer " + jm.ID
st, ok := started[id]
if !ok {
if jm.Progress != nil || jm.Status == "Pushed" {
now := time.Now()
st = &client.VertexStatus{
ID: id,
Started: &now,
}
started[id] = st
} else {
continue
}
}
st.Timestamp = time.Now()
if jm.Progress != nil {
st.Current = jm.Progress.Current
st.Total = jm.Progress.Total
}
if jm.Error != nil {
now := time.Now()
st.Completed = &now
}
if jm.Status == "Pushed" {
now := time.Now()
st.Completed = &now
st.Current = st.Total
}
l.SetStatus(st)
}
if jm.Error != nil {
parsedError = jm.Error
}
}
return nil
}
func createTempDockerfile(r io.Reader) (string, error) {
dir, err := ioutil.TempDir("", "dockerfile")
if err != nil {
@ -720,7 +901,7 @@ func createTempDockerfile(r io.Reader) (string, error) {
return dir, err
}
func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) {
if inp.ContextPath == "" {
return nil, errors.New("please specify build context (e.g. \".\" for the current directory)")
}
@ -736,6 +917,12 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
)
switch {
case inp.ContextState != nil:
if target.FrontendInputs == nil {
target.FrontendInputs = make(map[string]llb.State)
}
target.FrontendInputs["context"] = *inp.ContextState
target.FrontendInputs["dockerfile"] = *inp.ContextState
case inp.ContextPath == "-":
if inp.DockerfilePath == "-" {
return nil, errStdinConflict
@ -746,21 +933,22 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
if err != nil && err != io.EOF {
return nil, errors.Wrap(err, "failed to peek context header from STDIN")
}
if isArchive(magic) {
// stdin is context
up := uploadprovider.New()
target.FrontendAttrs["context"] = up.Add(buf)
target.Session = append(target.Session, up)
} else {
if inp.DockerfilePath != "" {
return nil, errDockerfileConflict
if !(err == io.EOF && len(magic) == 0) {
if isArchive(magic) {
// stdin is context
up := uploadprovider.New()
target.FrontendAttrs["context"] = up.Add(buf)
target.Session = append(target.Session, up)
} else {
if inp.DockerfilePath != "" {
return nil, errDockerfileConflict
}
// stdin is dockerfile
dockerfileReader = buf
inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
target.LocalDirs["context"] = inp.ContextPath
}
// stdin is dockerfile
dockerfileReader = buf
inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
target.LocalDirs["context"] = inp.ContextPath
}
case isLocalDir(inp.ContextPath):
@ -784,6 +972,10 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
return nil, errors.Errorf("unable to prepare context: path %q not found", inp.ContextPath)
}
if inp.DockerfileInline != "" {
dockerfileReader = strings.NewReader(inp.DockerfileInline)
}
if dockerfileReader != nil {
dockerfileDir, err = createTempDockerfile(dockerfileReader)
if err != nil {
@ -791,17 +983,30 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
}
toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
}
if urlutil.IsURL(inp.DockerfilePath) {
dockerfileDir, err = createTempDockerfileFromURL(ctx, d, inp.DockerfilePath, pw)
if err != nil {
return nil, err
}
toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
delete(target.FrontendInputs, "dockerfile")
}
if dockerfileName == "" {
dockerfileName = "Dockerfile"
}
target.FrontendAttrs["filename"] = dockerfileName
if dockerfileDir != "" {
target.LocalDirs["dockerfile"] = dockerfileDir
dockerfileName = handleLowercaseDockerfile(dockerfileDir, dockerfileName)
}
target.FrontendAttrs["filename"] = dockerfileName
release := func() {
for _, dir := range toRemove {
os.RemoveAll(dir)
@ -816,7 +1021,7 @@ func notSupported(d driver.Driver, f driver.Feature) error {
type dockerLoadCallback func(name string) (io.WriteCloser, func(), error)
func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) {
func newDockerLoader(ctx context.Context, d DockerAPI, name string, status progress.Writer) (io.WriteCloser, func(), error) {
c, err := d.DockerAPI(name)
if err != nil {
return nil, nil, err
@ -839,7 +1044,7 @@ func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress
w.mu.Unlock()
return
}
prog := mw.WithPrefix("", false)
prog := progress.WithPrefix(status, "", false)
progress.FromReader(prog, "importing to docker", resp.Body)
},
done: done,
@ -851,7 +1056,10 @@ func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress
}
func noDefaultLoad() bool {
v := os.Getenv("BUILDX_NO_DEFAULT_LOAD")
v, ok := os.LookupEnv("BUILDX_NO_DEFAULT_LOAD")
if !ok {
return false
}
b, err := strconv.ParseBool(v)
if err != nil {
logrus.Warnf("invalid non-bool value for BUILDX_NO_DEFAULT_LOAD: %s", v)
@ -886,3 +1094,40 @@ func (w *waitingWriter) Close() error {
}
return err
}
// handle https://github.com/moby/moby/pull/10858
func handleLowercaseDockerfile(dir, p string) string {
if filepath.Base(p) != "Dockerfile" {
return p
}
f, err := os.Open(filepath.Dir(filepath.Join(dir, p)))
if err != nil {
return p
}
names, err := f.Readdirnames(-1)
if err != nil {
return p
}
foundLowerCase := false
for _, n := range names {
if n == "Dockerfile" {
return p
}
if n == "dockerfile" {
foundLowerCase = true
}
}
if foundLowerCase {
return filepath.Join(filepath.Dir(p), "dockerfile")
}
return p
}
func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}

@ -0,0 +1,71 @@
package build
import (
"context"
"io/ioutil"
"path/filepath"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) {
c, err := driver.Boot(ctx, d, pw)
if err != nil {
return "", err
}
var out string
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := llb.HTTP(url, llb.Filename("Dockerfile"), llb.WithCustomNamef("[internal] load %s", url)).Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
stat, err := ref.StatFile(ctx, gwclient.StatRequest{
Path: "Dockerfile",
})
if err != nil {
return nil, err
}
if stat.Size() > 512*1024 {
return nil, errors.Errorf("Dockerfile %s bigger than allowed max size", url)
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: "Dockerfile",
})
if err != nil {
return nil, err
}
dir, err := ioutil.TempDir("", "buildx")
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dt, 0600); err != nil {
return nil, err
}
out = dir
return nil, nil
}, ch)
if err != nil {
return "", err
}
return out, nil
}

@ -7,11 +7,14 @@ import (
"github.com/containerd/containerd/pkg/seed"
"github.com/docker/buildx/commands"
"github.com/docker/buildx/version"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli-plugins/manager"
"github.com/docker/cli/cli-plugins/plugin"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/debug"
cliflags "github.com/docker/cli/cli/flags"
"github.com/spf13/cobra"
"github.com/moby/buildkit/solver/errdefs"
"github.com/moby/buildkit/util/stack"
// FIXME: "k8s.io/client-go/plugin/pkg/client/auth/azure" is excluded because of compilation error
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
@ -27,6 +30,7 @@ var experimental string
func init() {
seed.WithTimeAndRand()
stack.SetVersionInfo(version.Version, version.Revision)
}
func main() {
@ -47,13 +51,42 @@ func main() {
}
}
plugin.Run(func(dockerCli command.Cli) *cobra.Command {
return commands.NewRootCmd("buildx", true, dockerCli)
},
manager.Metadata{
SchemaVersion: "0.1.0",
Vendor: "Docker Inc.",
Version: version.Version,
Experimental: experimental != "",
})
dockerCli, err := command.NewDockerCli()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
p := commands.NewRootCmd("buildx", true, dockerCli)
meta := manager.Metadata{
SchemaVersion: "0.1.0",
Vendor: "Docker Inc.",
Version: version.Version,
Experimental: experimental != "",
}
if err := plugin.RunPlugin(dockerCli, p, meta); err != nil {
if sterr, ok := err.(cli.StatusError); ok {
if sterr.Status != "" {
fmt.Fprintln(dockerCli.Err(), sterr.Status)
}
// StatusError should only be used for errors, and all errors should
// have a non-zero exit status, so never exit with 0
if sterr.StatusCode == 0 {
os.Exit(1)
}
os.Exit(sterr.StatusCode)
}
for _, s := range errdefs.Sources(err) {
s.Print(dockerCli.Err())
}
if debug.IsEnabled() {
fmt.Fprintf(dockerCli.Err(), "error: %+v", stack.Formatter(err))
} else {
fmt.Fprintf(dockerCli.Err(), "error: %v\n", err)
}
os.Exit(1)
}
}

@ -0,0 +1 @@
comment: false

@ -1,11 +1,14 @@
package commands
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/docker/buildx/bake"
"github.com/docker/buildx/build"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/appcontext"
"github.com/pkg/errors"
@ -19,18 +22,16 @@ type bakeOptions struct {
commonOptions
}
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) {
ctx := appcontext.Context()
if len(in.files) == 0 {
files, err := defaultFiles()
if err != nil {
return err
}
if len(files) == 0 {
return errors.Errorf("no docker-compose.yml or docker-bake.hcl found, specify build file with -f/--file")
var url string
if len(targets) > 0 {
if bake.IsRemoteURL(targets[0]) {
url = targets[0]
targets = targets[1:]
}
in.files = files
}
if len(targets) == 0 {
@ -52,8 +53,38 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if in.pull != nil {
overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
}
contextPathHash, _ := os.Getwd()
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
printer := progress.NewPrinter(ctx2, os.Stderr, in.progress)
m, err := bake.ReadTargets(ctx, in.files, targets, overrides)
defer func() {
if printer != nil {
err1 := printer.Wait()
if err == nil {
err = err1
}
}
}()
dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash)
if err != nil {
return err
}
var files []bake.File
var inp *bake.Input
if url != "" {
files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
} else {
files, err = bake.ReadLocalFiles(in.files)
}
if err != nil {
return err
}
m, err := bake.ReadTargets(ctx, files, targets, overrides)
if err != nil {
return err
}
@ -63,40 +94,22 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if err != nil {
return err
}
err = printer.Wait()
printer = nil
if err != nil {
return err
}
fmt.Fprintln(dockerCli.Out(), string(dt))
return nil
}
bo, err := bake.TargetsToBuildOpt(m)
bo, err := bake.TargetsToBuildOpt(m, inp)
if err != nil {
return err
}
contextPathHash, _ := os.Getwd()
return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash, in.builder)
}
func defaultFiles() ([]string, error) {
fns := []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
out := make([]string, 0, len(fns))
for _, f := range fns {
if _, err := os.Stat(f); err != nil {
if os.IsNotExist(errors.Cause(err)) {
continue
}
return nil, err
}
out = append(out, f)
}
return out, nil
_, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
return err
}
func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {

@ -7,6 +7,7 @@ import (
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli"
@ -15,6 +16,7 @@ import (
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/util/appcontext"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
@ -62,11 +64,14 @@ type buildOptions struct {
}
type commonOptions struct {
builder string
noCache *bool
progress string
pull *bool
builder string
noCache *bool
progress string
pull *bool
// golangci-lint#826
// nolint:structcheck
exportPush bool
// nolint:structcheck
exportLoad bool
}
@ -75,7 +80,7 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
return errors.Errorf("squash currently not implemented")
}
if in.quiet {
return errors.Errorf("quiet currently not implemented")
logrus.Warnf("quiet currently not implemented")
}
ctx := appcontext.Context()
@ -114,19 +119,23 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(os.Stderr))
secrets, err := build.ParseSecretSpecs(in.secrets)
secrets, err := buildflags.ParseSecretSpecs(in.secrets)
if err != nil {
return err
}
opts.Session = append(opts.Session, secrets)
ssh, err := build.ParseSSHSpecs(in.ssh)
sshSpecs := in.ssh
if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.contextPath) {
sshSpecs = []string{"default"}
}
ssh, err := buildflags.ParseSSHSpecs(sshSpecs)
if err != nil {
return err
}
opts.Session = append(opts.Session, ssh)
outputs, err := build.ParseOutputs(in.outputs)
outputs, err := buildflags.ParseOutputs(in.outputs)
if err != nil {
return err
}
@ -167,19 +176,19 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
opts.Exports = outputs
cacheImports, err := build.ParseCacheEntry(in.cacheFrom)
cacheImports, err := buildflags.ParseCacheEntry(in.cacheFrom)
if err != nil {
return err
}
opts.CacheFrom = cacheImports
cacheExports, err := build.ParseCacheEntry(in.cacheTo)
cacheExports, err := buildflags.ParseCacheEntry(in.cacheTo)
if err != nil {
return err
}
opts.CacheTo = cacheExports
allow, err := build.ParseEntitlements(in.allow)
allow, err := buildflags.ParseEntitlements(in.allow)
if err != nil {
return err
}
@ -202,9 +211,14 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
pw := progress.NewPrinter(ctx2, os.Stderr, progressMode)
printer := progress.NewPrinter(ctx2, os.Stderr, progressMode)
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
err1 := printer.Wait()
if err == nil {
err = err1
}
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), pw)
return err
}
@ -229,8 +243,12 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
flags.BoolVar(&options.exportLoad, "load", false, "Shorthand for --output=type=docker")
flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Name and optionally a tag in the 'name:tag' format")
flags.SetAnnotation("tag", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t"})
flags.StringArrayVar(&options.buildArgs, "build-arg", []string{}, "Set build-time variables")
flags.SetAnnotation("build-arg", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg"})
flags.StringVarP(&options.dockerfileName, "file", "f", "", "Name of the Dockerfile (Default is 'PATH/Dockerfile')")
flags.SetAnnotation("file", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f"})
flags.StringArrayVar(&options.labels, "label", []string{}, "Set metadata for an image")
@ -238,6 +256,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
flags.StringArrayVar(&options.cacheTo, "cache-to", []string{}, "Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)")
flags.StringVar(&options.target, "target", "", "Set the target build stage to build.")
flags.SetAnnotation("target", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target"})
flags.StringSliceVar(&options.allow, "allow", []string{}, "Allow extra privileged entitlement, e.g. network.host, security.insecure")
@ -245,6 +264,7 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success")
flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build")
flags.StringSliceVar(&options.extraHosts, "add-host", []string{}, "Add a custom host-to-IP mapping (host:ip)")
flags.SetAnnotation("add-host", "docs.external.url", []string{"https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host"})
flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file")
flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer")
flags.MarkHidden("quiet")
@ -305,7 +325,13 @@ func buildCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
func commonBuildFlags(options *commonOptions, flags *pflag.FlagSet) {
options.noCache = flags.Bool("no-cache", false, "Do not use cache when building the image")
flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
defaultProgress, ok := os.LookupEnv("BUILDX_PROGRESS_DEFAULT")
if !ok {
defaultProgress = "auto"
}
flags.StringVar(&options.progress, "progress", defaultProgress, "Set type of progress output (auto, plain, tty). Use plain to show container output")
options.pull = flags.Bool("pull", false, "Always attempt to pull a newer version of the image")
}

@ -3,6 +3,7 @@ package commands
import (
"encoding/csv"
"fmt"
"net/url"
"os"
"strings"
@ -145,7 +146,14 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
if in.driver == "kubernetes" {
// naming endpoint to make --append works
ep = fmt.Sprintf("%s://%s?deployment=%s", in.driver, in.name, in.nodeName)
ep = (&url.URL{
Scheme: in.driver,
Path: "/" + in.name,
RawQuery: (&url.Values{
"deployment": {in.nodeName},
"kubeconfig": {os.Getenv("KUBECONFIG")},
}).Encode(),
}).String()
}
m, err := csvToMap(in.driverOpts)

@ -109,7 +109,6 @@ func duCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
options.builder = rootOpts.builder
return runDiskUsage(dockerCli, options)
},
Annotations: map[string]string{"version": "1.00"},
}
flags := cmd.Flags()
@ -186,8 +185,6 @@ func printSummary(tw *tabwriter.Writer, dus [][]*client.UsageInfo) {
}
}
tw = tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
if shared > 0 {
fmt.Fprintf(tw, "Shared:\t%.2f\n", units.Bytes(shared))
fmt.Fprintf(tw, "Private:\t%.2f\n", units.Bytes(total-shared))

@ -118,7 +118,15 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
return err
}
srcs[i].Ref = nil
srcs[i].Desc = desc
if srcs[i].Desc.Digest == "" {
srcs[i].Desc = desc
} else {
var err error
srcs[i].Desc, err = mergeDesc(desc, srcs[i].Desc)
if err != nil {
return err
}
}
return nil
})
}(i)
@ -168,7 +176,7 @@ func parseSources(in []string) ([]*src, error) {
for i, in := range in {
s, err := parseSource(in)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, refereces and descriptors", in)
return nil, errors.Wrapf(err, "failed to parse source %q, valid sources are digests, references and descriptors", in)
}
out[i] = s
}
@ -238,3 +246,19 @@ func createCmd(dockerCli command.Cli) *cobra.Command {
return cmd
}
func mergeDesc(d1, d2 ocispec.Descriptor) (ocispec.Descriptor, error) {
if d2.Size != 0 && d1.Size != d2.Size {
return ocispec.Descriptor{}, errors.Errorf("invalid size mismatch for %s, %d != %d", d1.Digest, d2.Size, d1.Size)
}
if d2.MediaType != "" {
d1.MediaType = d2.MediaType
}
if len(d2.Annotations) != 0 {
d1.Annotations = d2.Annotations // no merge so support removes
}
if d2.Platform != nil {
d1.Platform = d2.Platform // missing items filled in later from image config
}
return d1, nil
}

@ -38,7 +38,7 @@ func runInspect(dockerCli command.Cli, in inspectOptions, name string) error {
// case images.MediaTypeDockerSchema2Manifest, specs.MediaTypeImageManifest:
// TODO: handle distribution manifest and schema1
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
imagetools.PrintManifestList(dt, desc, name, os.Stdout)
return imagetools.PrintManifestList(dt, desc, name, os.Stdout)
default:
fmt.Printf("%s\n", dt)
}

@ -79,12 +79,14 @@ func runInspect(dockerCli command.Cli, in inspectOptions) error {
err = loadNodeGroupData(timeoutCtx, dockerCli, ngi)
var bootNgi *nginfo
if in.bootstrap {
var ok bool
ok, err = boot(ctx, ngi)
ok, err = boot(ctx, ngi, dockerCli)
if err != nil {
return err
}
bootNgi = ngi
if ok {
ngi = &nginfo{ng: ng}
err = loadNodeGroupData(ctx, dockerCli, ngi)
@ -113,6 +115,8 @@ func runInspect(dockerCli command.Cli, in inspectOptions) error {
fmt.Fprintf(w, "Error:\t%s\n", err.Error())
} else if err := ngi.drivers[i].err; err != nil {
fmt.Fprintf(w, "Error:\t%s\n", err.Error())
} else if bootNgi != nil && len(bootNgi.drivers) > i && bootNgi.drivers[i].err != nil {
fmt.Fprintf(w, "Error:\t%s\n", bootNgi.drivers[i].err.Error())
} else {
fmt.Fprintf(w, "Status:\t%s\n", ngi.drivers[i].info.Status)
if len(n.Flags) > 0 {
@ -153,7 +157,7 @@ func inspectCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
return cmd
}
func boot(ctx context.Context, ngi *nginfo) (bool, error) {
func boot(ctx context.Context, ngi *nginfo, dockerCli command.Cli) (bool, error) {
toBoot := make([]int, 0, len(ngi.drivers))
for i, d := range ngi.drivers {
if d.err != nil || d.di.Err != nil || d.di.Driver == nil || d.info == nil {
@ -167,25 +171,27 @@ func boot(ctx context.Context, ngi *nginfo) (bool, error) {
return false, nil
}
pw := progress.NewPrinter(context.TODO(), os.Stderr, "auto")
mw := progress.NewMultiWriter(pw)
printer := progress.NewPrinter(context.TODO(), os.Stderr, "auto")
eg, _ := errgroup.WithContext(ctx)
for _, idx := range toBoot {
func(idx int) {
eg.Go(func() error {
pw := mw.WithPrefix(ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
pw := progress.WithPrefix(printer, ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
_, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
if err != nil {
ngi.drivers[idx].err = err
}
close(pw.Status())
<-pw.Done()
return nil
})
}(idx)
}
return true, eg.Wait()
err := eg.Wait()
err1 := printer.Wait()
if err == nil {
err = err1
}
return true, err
}

@ -129,13 +129,12 @@ func pruneCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
cmd := &cobra.Command{
Use: "prune",
Short: "Remove build cache ",
Short: "Remove build cache",
Args: cli.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
options.builder = rootOpts.builder
return runPrune(dockerCli, options)
},
Annotations: map[string]string{"version": "1.00"},
}
flags := cmd.Flags()

@ -93,7 +93,7 @@ func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bo
}
func stopCurrent(ctx context.Context, dockerCli command.Cli, rm bool) error {
dis, err := getDefaultDrivers(ctx, dockerCli, "")
dis, err := getDefaultDrivers(ctx, dockerCli, false, "")
if err != nil {
return err
}

@ -2,6 +2,7 @@ package commands
import (
"context"
"net/url"
"os"
"path/filepath"
"strings"
@ -13,23 +14,38 @@ import (
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/context/docker"
"github.com/docker/cli/cli/context/kubernetes"
ctxstore "github.com/docker/cli/cli/context/store"
dopts "github.com/docker/cli/opts"
dockerclient "github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"k8s.io/client-go/tools/clientcmd"
)
// getStore returns current builder instance store
func getStore(dockerCli command.Cli) (*store.Txn, func(), error) {
dir := filepath.Dir(dockerCli.ConfigFile().Filename)
s, err := store.New(dir)
s, err := store.New(getConfigStorePath(dockerCli))
if err != nil {
return nil, nil, err
}
return s.Txn()
}
// getConfigStorePath will look for correct configuration store path;
// if `$BUILDX_CONFIG` is set - use it, otherwise use parent directory
// of Docker config file (i.e. `${DOCKER_CONFIG}/buildx`)
func getConfigStorePath(dockerCli command.Cli) string {
if buildxConfig := os.Getenv("BUILDX_CONFIG"); buildxConfig != "" {
logrus.Debugf("using config store %q based in \"$BUILDX_CONFIG\" environment variable", buildxConfig)
return buildxConfig
}
buildxConfig := filepath.Join(filepath.Dir(dockerCli.ConfigFile().Filename), "buildx")
logrus.Debugf("using default config store %q", buildxConfig)
return buildxConfig
}
// getCurrentEndpoint returns the current default endpoint value
func getCurrentEndpoint(dockerCli command.Cli) (string, error) {
name := dockerCli.CurrentContext()
@ -180,20 +196,34 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
contextStore := dockerCli.ContextStore()
var kcc driver.KubeClientConfig
kcc, err = kubernetes.ConfigFromContext(n.Endpoint, contextStore)
kcc, err = configFromContext(n.Endpoint, contextStore)
if err != nil {
// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
// try again with name="default".
// FIXME: n should retain real context name.
kcc, err = kubernetes.ConfigFromContext("default", contextStore)
kcc, err = configFromContext("default", contextStore)
if err != nil {
logrus.Error(err)
}
}
tryToUseKubeConfigInCluster := false
if kcc == nil {
kcc = driver.KubeClientConfigInCluster{}
tryToUseKubeConfigInCluster = true
} else {
if _, err := kcc.ClientConfig(); err != nil {
tryToUseKubeConfigInCluster = true
}
}
if tryToUseKubeConfigInCluster {
kccInCluster := driver.KubeClientConfigInCluster{}
if _, err := kccInCluster.ClientConfig(); err == nil {
logrus.Debug("using kube config in cluster")
kcc = kccInCluster
}
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, kcc, n.Flags, n.ConfigFile, assignDriverOptsByDriverInfo(n.DriverOpts, di), contextPathHash)
d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash)
if err != nil {
di.Err = err
return nil
@ -211,18 +241,19 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
return dis, nil
}
// pass platform as driver opts to provide for some drive, like kubernetes
func assignDriverOptsByDriverInfo(opts map[string]string, driveInfo build.DriverInfo) map[string]string {
m := map[string]string{}
if len(driveInfo.Platform) > 0 {
m["platform"] = strings.Join(platformutil.Format(driveInfo.Platform), ",")
}
func configFromContext(endpointName string, s ctxstore.Reader) (clientcmd.ClientConfig, error) {
if strings.HasPrefix(endpointName, "kubernetes://") {
u, _ := url.Parse(endpointName)
for key := range opts {
m[key] = opts[key]
if kubeconfig := u.Query().Get("kubeconfig"); kubeconfig != "" {
clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig},
&clientcmd.ConfigOverrides{},
)
return clientConfig, nil
}
}
return m
return kubernetes.ConfigFromContext(endpointName, s)
}
// clientForEndpoint returns a docker client for an endpoint
@ -268,10 +299,29 @@ func clientForEndpoint(dockerCli command.Cli, name string) (dockerclient.APIClie
}
func getInstanceOrDefault(ctx context.Context, dockerCli command.Cli, instance, contextPathHash string) ([]build.DriverInfo, error) {
var defaultOnly bool
if instance == "default" && instance != dockerCli.CurrentContext() {
return nil, errors.Errorf("use `docker --context=default buildx` to switch to default context")
}
if instance == "default" || instance == dockerCli.CurrentContext() {
instance = ""
defaultOnly = true
}
list, err := dockerCli.ContextStore().List()
if err != nil {
return nil, err
}
for _, l := range list {
if l.Name == instance {
return nil, errors.Errorf("use `docker --context=%s buildx` to switch to context %s", instance, instance)
}
}
if instance != "" {
return getInstanceByName(ctx, dockerCli, instance, contextPathHash)
}
return getDefaultDrivers(ctx, dockerCli, contextPathHash)
return getDefaultDrivers(ctx, dockerCli, defaultOnly, contextPathHash)
}
func getInstanceByName(ctx context.Context, dockerCli command.Cli, instance, contextPathHash string) ([]build.DriverInfo, error) {
@ -289,23 +339,25 @@ func getInstanceByName(ctx context.Context, dockerCli command.Cli, instance, con
}
// getDefaultDrivers returns drivers based on current cli config
func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, contextPathHash string) ([]build.DriverInfo, error) {
func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly bool, contextPathHash string) ([]build.DriverInfo, error) {
txn, release, err := getStore(dockerCli)
if err != nil {
return nil, err
}
defer release()
ng, err := getCurrentInstance(txn, dockerCli)
if err != nil {
return nil, err
}
if !defaultOnly {
ng, err := getCurrentInstance(txn, dockerCli)
if err != nil {
return nil, err
}
if ng != nil {
return driversForNodeGroup(ctx, dockerCli, ng, contextPathHash)
if ng != nil {
return driversForNodeGroup(ctx, dockerCli, ng, contextPathHash)
}
}
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), nil, nil, "", nil, contextPathHash)
d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash)
if err != nil {
return nil, err
}
@ -370,13 +422,24 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)
return err
}
// skip when multi drivers
if len(ngi.drivers) == 1 {
kubernetesDriverCount := 0
for _, di := range ngi.drivers {
if di.info != nil && len(di.info.DynamicNodes) > 0 {
kubernetesDriverCount++
}
}
isAllKubernetesDrivers := len(ngi.drivers) == kubernetesDriverCount
if isAllKubernetesDrivers {
var drivers []dinfo
var dynamicNodes []store.Node
for _, di := range ngi.drivers {
// dynamic nodes are used in Kubernetes driver.
// Kubernetes pods are dynamically mapped to BuildKit Nodes.
if di.info != nil && len(di.info.DynamicNodes) > 0 {
var drivers []dinfo
for i := 0; i < len(di.info.DynamicNodes); i++ {
// all []dinfo share *build.DriverInfo and *driver.Info
diClone := di
@ -385,14 +448,16 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)
}
drivers = append(drivers, di)
}
// not append (remove the static nodes in the store)
ngi.ng.Nodes = di.info.DynamicNodes
ngi.ng.Dynamic = true
ngi.drivers = drivers
return nil
dynamicNodes = append(dynamicNodes, di.info.DynamicNodes...)
}
}
// not append (remove the static nodes in the store)
ngi.ng.Nodes = dynamicNodes
ngi.drivers = drivers
ngi.ng.Dynamic = true
}
return nil
}

@ -17,7 +17,7 @@ func runVersion(dockerCli command.Cli) error {
func versionCmd(dockerCli command.Cli) *cobra.Command {
cmd := &cobra.Command{
Use: "version",
Short: "Show buildx version information ",
Short: "Show buildx version information",
Args: cli.ExactArgs(0),
RunE: func(cmd *cobra.Command, args []string) error {
return runVersion(dockerCli)

@ -0,0 +1,198 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"github.com/docker/buildx/commands"
"github.com/docker/cli/cli/command"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
const descriptionSourcePath = "docs/reference/"
func generateDocs(opts *options) error {
dockerCLI, err := command.NewDockerCli()
if err != nil {
return err
}
cmd := &cobra.Command{
Use: "docker [OPTIONS] COMMAND [ARG...]",
Short: "The base command for the Docker CLI.",
}
cmd.AddCommand(commands.NewRootCmd("buildx", true, dockerCLI))
return genCmd(cmd, opts.target)
}
func getMDFilename(cmd *cobra.Command) string {
name := cmd.CommandPath()
if i := strings.Index(name, " "); i >= 0 {
name = name[i+1:]
}
return strings.ReplaceAll(name, " ", "_") + ".md"
}
func genCmd(cmd *cobra.Command, dir string) error {
for _, c := range cmd.Commands() {
if err := genCmd(c, dir); err != nil {
return err
}
}
if !cmd.HasParent() {
return nil
}
mdFile := getMDFilename(cmd)
fullPath := filepath.Join(dir, mdFile)
content, err := ioutil.ReadFile(fullPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return errors.Wrapf(err, "%s does not exist", mdFile)
}
}
cs := string(content)
markerStart := "<!---MARKER_GEN_START-->"
markerEnd := "<!---MARKER_GEN_END-->"
start := strings.Index(cs, markerStart)
end := strings.Index(cs, markerEnd)
if start == -1 {
return errors.Errorf("no start marker in %s", mdFile)
}
if end == -1 {
return errors.Errorf("no end marker in %s", mdFile)
}
out, err := cmdOutput(cmd, cs)
if err != nil {
return err
}
cont := cs[:start] + markerStart + "\n" + out + "\n" + cs[end:]
fi, err := os.Stat(fullPath)
if err != nil {
return err
}
if err := ioutil.WriteFile(fullPath, []byte(cont), fi.Mode()); err != nil {
return errors.Wrapf(err, "failed to write %s", fullPath)
}
log.Printf("updated %s", fullPath)
return nil
}
func makeLink(txt, link string, f *pflag.Flag, isAnchor bool) string {
link = "#" + link
annotations, ok := f.Annotations["docs.external.url"]
if ok && len(annotations) > 0 {
link = annotations[0]
} else {
if !isAnchor {
return txt
}
}
return "[" + txt + "](" + link + ")"
}
func cmdOutput(cmd *cobra.Command, old string) (string, error) {
b := &strings.Builder{}
desc := cmd.Short
if cmd.Long != "" {
desc = cmd.Long
}
if desc != "" {
fmt.Fprintf(b, "%s\n\n", desc)
}
if len(cmd.Aliases) != 0 {
fmt.Fprintf(b, "### Aliases\n\n`%s`", cmd.Name())
for _, a := range cmd.Aliases {
fmt.Fprintf(b, ", `%s`", a)
}
fmt.Fprint(b, "\n\n")
}
if len(cmd.Commands()) != 0 {
fmt.Fprint(b, "### Subcommands\n\n")
fmt.Fprint(b, "| Name | Description |\n")
fmt.Fprint(b, "| --- | --- |\n")
for _, c := range cmd.Commands() {
fmt.Fprintf(b, "| [`%s`](%s) | %s |\n", c.Name(), getMDFilename(c), c.Short)
}
fmt.Fprint(b, "\n\n")
}
hasFlags := cmd.Flags().HasAvailableFlags()
cmd.Flags().AddFlagSet(cmd.InheritedFlags())
if hasFlags {
fmt.Fprint(b, "### Options\n\n")
fmt.Fprint(b, "| Name | Description |\n")
fmt.Fprint(b, "| --- | --- |\n")
cmd.Flags().VisitAll(func(f *pflag.Flag) {
if f.Hidden {
return
}
isLink := strings.Contains(old, "<a name=\""+f.Name+"\"></a>")
fmt.Fprint(b, "| ")
if f.Shorthand != "" {
name := "`-" + f.Shorthand + "`"
name = makeLink(name, f.Name, f, isLink)
fmt.Fprintf(b, "%s, ", name)
}
name := "`--" + f.Name
if f.Value.Type() != "bool" {
name += " " + f.Value.Type()
}
name += "`"
name = makeLink(name, f.Name, f, isLink)
fmt.Fprintf(b, "%s | %s |\n", name, f.Usage)
})
fmt.Fprintln(b, "")
}
return b.String(), nil
}
type options struct {
target string
}
func parseArgs() (*options, error) {
opts := &options{}
flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
flags.StringVar(&opts.target, "target", descriptionSourcePath, "Docs directory")
err := flags.Parse(os.Args[1:])
return opts, err
}
func main() {
if err := run(); err != nil {
log.Printf("error: %+v", err)
os.Exit(1)
}
}
func run() error {
opts, err := parseArgs()
if err != nil {
return err
}
if err := generateDocs(opts); err != nil {
return err
}
return nil
}

@ -0,0 +1,31 @@
# buildx
```
docker buildx [OPTIONS] COMMAND
```
<!---MARKER_GEN_START-->
Build with BuildKit
### Subcommands
| Name | Description |
| --- | --- |
| [`bake`](buildx_bake.md) | Build from a file |
| [`build`](buildx_build.md) | Start a build |
| [`create`](buildx_create.md) | Create a new builder instance |
| [`du`](buildx_du.md) | Disk usage |
| [`imagetools`](buildx_imagetools.md) | Commands to work on images in registry |
| [`inspect`](buildx_inspect.md) | Inspect current builder instance |
| [`install`](buildx_install.md) | Install buildx as a 'docker builder' alias |
| [`ls`](buildx_ls.md) | List builder instances |
| [`prune`](buildx_prune.md) | Remove build cache |
| [`rm`](buildx_rm.md) | Remove a builder instance |
| [`stop`](buildx_stop.md) | Stop builder instance |
| [`uninstall`](buildx_uninstall.md) | Uninstall the 'docker builder' alias |
| [`use`](buildx_use.md) | Set the current builder instance |
| [`version`](buildx_version.md) | Show buildx version information |
<!---MARKER_GEN_END-->

@ -0,0 +1,373 @@
# buildx bake
```
docker buildx bake [OPTIONS] [TARGET...]
```
<!---MARKER_GEN_START-->
Build from a file
### Aliases
`bake`, `f`
### Options
| Name | Description |
| --- | --- |
| `--builder string` | Override the configured builder instance |
| [`-f`](#file), [`--file stringArray`](#file) | Build definition file |
| `--load` | Shorthand for --set=*.output=type=docker |
| [`--no-cache`](#no-cache) | Do not use cache when building the image |
| [`--print`](#print) | Print the options without building |
| [`--progress string`](#progress) | Set type of progress output (auto, plain, tty). Use plain to show container output |
| [`--pull`](#pull) | Always attempt to pull a newer version of the image |
| `--push` | Shorthand for --set=*.output=type=registry |
| [`--set stringArray`](#set) | Override target value (eg: targetpattern.key=value) |
<!---MARKER_GEN_END-->
## Description
Bake is a high-level build command. Each specified target will run in parallel
as part of the build.
Read [High-level build options](https://github.com/docker/buildx#high-level-build-options) for introduction.
Please note that `buildx bake` command may receive backwards incompatible features in the future if needed. We are looking for feedback on improving the command and extending the functionality further.
## Examples
### <a name="file"></a> Specify a build definition file (-f, --file)
By default, `buildx bake` looks for build definition files in the current directory,
the following are parsed:
- `docker-compose.yml`
- `docker-compose.yaml`
- `docker-bake.json`
- `docker-bake.override.json`
- `docker-bake.hcl`
- `docker-bake.override.hcl`
Use the `-f` / `--file` option to specify the build definition file to use. The
file can be a Docker Compose, JSON or HCL file. If multiple files are specified
they are all read and configurations are combined.
The following example uses a Docker Compose file named `docker-compose.dev.yaml`
as build definition file, and builds all targets in the file:
```console
$ docker buildx bake -f docker-compose.dev.yaml
[+] Building 66.3s (30/30) FINISHED
=> [frontend internal] load build definition from Dockerfile 0.1s
=> => transferring dockerfile: 36B 0.0s
=> [backend internal] load build definition from Dockerfile 0.2s
=> => transferring dockerfile: 3.73kB 0.0s
=> [database internal] load build definition from Dockerfile 0.1s
=> => transferring dockerfile: 5.77kB 0.0s
...
```
Pass the names of the targets to build, to build only specific target(s). The
following example builds the `backend` and `database` targets that are defined
in the `docker-compose.dev.yaml` file, skipping the build for the `frontend`
target:
```console
$ docker buildx bake -f docker-compose.dev.yaml backend database
[+] Building 2.4s (13/13) FINISHED
=> [backend internal] load build definition from Dockerfile 0.1s
=> => transferring dockerfile: 81B 0.0s
=> [database internal] load build definition from Dockerfile 0.2s
=> => transferring dockerfile: 36B 0.0s
=> [backend internal] load .dockerignore 0.3s
...
```
### <a name="no-cache"></a> Do not use cache when building the image (--no-cache)
Same as `build --no-cache`. Do not use cache when building the image.
### <a name="print"></a> Print the options without building (--print)
Prints the resulting options of the targets desired to be built, in a JSON format,
without starting a build.
```console
$ docker buildx bake -f docker-bake.hcl --print db
{
"target": {
"db": {
"context": "./",
"dockerfile": "Dockerfile",
"tags": [
"docker.io/tiborvass/db"
]
}
}
}
```
### <a name="progress"></a> Set type of progress output (--progress)
Same as `build --progress`. Set type of progress output (auto, plain, tty). Use
plain to show container output (default "auto").
The following example uses `plain` output during the build:
```console
$ docker buildx bake --progress=plain
#2 [backend internal] load build definition from Dockerfile.test
#2 sha256:de70cb0bb6ed8044f7b9b1b53b67f624e2ccfb93d96bb48b70c1fba562489618
#2 ...
#1 [database internal] load build definition from Dockerfile.test
#1 sha256:453cb50abd941762900a1212657a35fc4aad107f5d180b0ee9d93d6b74481bce
#1 transferring dockerfile: 36B done
#1 DONE 0.1s
...
```
### <a name="pull"></a> Always attempt to pull a newer version of the image (--pull)
Same as `build --pull`.
### <a name="set"></a> Override target configurations from command line (--set)
```
--set targetpattern.key[.subkey]=value
```
Override target configurations from command line. The pattern matching syntax is
defined in https://golang.org/pkg/path/#Match.
**Examples**
```console
$ docker buildx bake --set target.args.mybuildarg=value
$ docker buildx bake --set target.platform=linux/arm64
$ docker buildx bake --set foo*.args.mybuildarg=value # overrides build arg for all targets starting with 'foo'
$ docker buildx bake --set *.platform=linux/arm64 # overrides platform for all targets
$ docker buildx bake --set foo*.no-cache # bypass caching only for targets starting with 'foo'
```
Complete list of overridable fields:
args, cache-from, cache-to, context, dockerfile, labels, no-cache, output, platform,
pull, secrets, ssh, tags, target
### File definition
In addition to compose files, bake supports a JSON and an equivalent HCL file
format for defining build groups and targets.
A target reflects a single docker build invocation with the same options that
you would specify for `docker build`. A group is a grouping of targets.
Multiple files can include the same target and final build options will be
determined by merging them together.
In the case of compose files, each service corresponds to a target.
A group can specify its list of targets with the `targets` option. A target can
inherit build options by setting the `inherits` option to the list of targets or
groups to inherit from.
Note: Design of bake command is work in progress, the user experience may change
based on feedback.
**Example HCL definition**
```hcl
group "default" {
targets = ["db", "webapp-dev"]
}
target "webapp-dev" {
dockerfile = "Dockerfile.webapp"
tags = ["docker.io/username/webapp"]
}
target "webapp-release" {
inherits = ["webapp-dev"]
platforms = ["linux/amd64", "linux/arm64"]
}
target "db" {
dockerfile = "Dockerfile.db"
tags = ["docker.io/username/db"]
}
```
Complete list of valid target fields:
`args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `inherits`, `labels`,
`no-cache`, `output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target`
### HCL variables and functions
Similar to how Terraform provides a way to [define variables](https://www.terraform.io/docs/configuration/variables.html#declaring-an-input-variable),
the HCL file format also supports variable block definitions. These can be used
to define variables with values provided by the current environment, or a default
value when unset.
Example of using interpolation to tag an image with the git sha:
```console
$ cat <<'EOF' > docker-bake.hcl
variable "TAG" {
default = "latest"
}
group "default" {
targets = ["webapp"]
}
target "webapp" {
tags = ["docker.io/username/webapp:${TAG}"]
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"tags": [
"docker.io/username/webapp:latest"
]
}
}
}
$ TAG=$(git rev-parse --short HEAD) docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"tags": [
"docker.io/username/webapp:985e9e9"
]
}
}
}
```
A [set of generally useful functions](https://github.com/docker/buildx/blob/master/bake/hcl.go#L19-L65)
provided by [go-cty](https://github.com/zclconf/go-cty/tree/main/cty/function/stdlib)
are available for use in HCL files. In addition, [user defined functions](https://github.com/hashicorp/hcl/tree/main/ext/userfunc)
are also supported.
Example of using the `add` function:
```console
$ cat <<'EOF' > docker-bake.hcl
variable "TAG" {
default = "latest"
}
group "default" {
targets = ["webapp"]
}
target "webapp" {
args = {
buildno = "${add(123, 1)}"
}
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"args": {
"buildno": "124"
}
}
}
}
```
Example of defining an `increment` function:
```console
$ cat <<'EOF' > docker-bake.hcl
function "increment" {
params = [number]
result = number + 1
}
group "default" {
targets = ["webapp"]
}
target "webapp" {
args = {
buildno = "${increment(123)}"
}
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"args": {
"buildno": "124"
}
}
}
}
```
Example of only adding tags if a variable is not empty using an `notequal`
function:
```console
$ cat <<'EOF' > docker-bake.hcl
variable "TAG" {default="" }
group "default" {
targets = [
"webapp",
]
}
target "webapp" {
context="."
dockerfile="Dockerfile"
tags = [
"my-image:latest",
notequal("",TAG) ? "my-image:${TAG}": "",
]
}
EOF
$ docker buildx bake --print webapp
{
"target": {
"webapp": {
"context": ".",
"dockerfile": "Dockerfile",
"tags": [
"my-image:latest"
]
}
}
}
```

@ -0,0 +1,271 @@
# buildx build
```
docker buildx build [OPTIONS] PATH | URL | -
```
<!---MARKER_GEN_START-->
Start a build
### Aliases
`build`, `b`
### Options
| Name | Description |
| --- | --- |
| [`--add-host stringSlice`](https://docs.docker.com/engine/reference/commandline/build/#add-entries-to-container-hosts-file---add-host) | Add a custom host-to-IP mapping (host:ip) |
| [`--allow stringSlice`](#allow) | Allow extra privileged entitlement, e.g. network.host, security.insecure |
| [`--build-arg stringArray`](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg) | Set build-time variables |
| `--builder string` | Override the configured builder instance |
| [`--cache-from stringArray`](#cache-from) | External cache sources (eg. user/app:cache, type=local,src=path/to/dir) |
| [`--cache-to stringArray`](#cache-to) | Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir) |
| [`-f`](https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f), [`--file string`](https://docs.docker.com/engine/reference/commandline/build/#specify-a-dockerfile--f) | Name of the Dockerfile (Default is 'PATH/Dockerfile') |
| `--iidfile string` | Write the image ID to the file |
| `--label stringArray` | Set metadata for an image |
| [`--load`](#load) | Shorthand for --output=type=docker |
| `--network string` | Set the networking mode for the RUN instructions during build |
| `--no-cache` | Do not use cache when building the image |
| [`-o`](#output), [`--output stringArray`](#output) | Output destination (format: type=local,dest=path) |
| [`--platform stringArray`](#platform) | Set target platform for build |
| `--progress string` | Set type of progress output (auto, plain, tty). Use plain to show container output |
| `--pull` | Always attempt to pull a newer version of the image |
| [`--push`](#push) | Shorthand for --output=type=registry |
| `--secret stringArray` | Secret file to expose to the build: id=mysecret,src=/local/secret |
| `--ssh stringArray` | SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]]) |
| [`-t`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t), [`--tag stringArray`](https://docs.docker.com/engine/reference/commandline/build/#tag-an-image--t) | Name and optionally a tag in the 'name:tag' format |
| [`--target string`](https://docs.docker.com/engine/reference/commandline/build/#specifying-target-build-stage---target) | Set the target build stage to build. |
<!---MARKER_GEN_END-->
## Description
The `buildx build` command starts a build using BuildKit. This command is similar
to the UI of `docker build` command and takes the same flags and arguments.
For documentation on most of these flags, refer to the [`docker build`
documentation](https://docs.docker.com/engine/reference/commandline/build/). In
here well document a subset of the new flags.
## Examples
### <a name="platform"></a> Set the target platforms for the build (--platform)
```
--platform=value[,value]
```
Set the target platform for the build. All `FROM` commands inside the Dockerfile
without their own `--platform` flag will pull base images for this platform and
this value will also be the platform of the resulting image. The default value
will be the current platform of the buildkit daemon.
When using `docker-container` driver with `buildx`, this flag can accept multiple
values as an input separated by a comma. With multiple values the result will be
built for all of the specified platforms and joined together into a single manifest
list.
If the `Dockerfile` needs to invoke the `RUN` command, the builder needs runtime
support for the specified platform. In a clean setup, you can only execute `RUN`
commands for your system architecture.
If your kernel supports [`binfmt_misc`](https://en.wikipedia.org/wiki/Binfmt_misc)
launchers for secondary architectures, buildx will pick them up automatically.
Docker desktop releases come with `binfmt_misc` automatically configured for `arm64`
and `arm` architectures. You can see what runtime platforms your current builder
instance supports by running `docker buildx inspect --bootstrap`.
Inside a `Dockerfile`, you can access the current platform value through
`TARGETPLATFORM` build argument. Please refer to the [`docker build`
documentation](https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope)
for the full description of automatic platform argument variants .
The formatting for the platform specifier is defined in the [containerd source
code](https://github.com/containerd/containerd/blob/v1.4.3/platforms/platforms.go#L63).
**Examples**
```console
$ docker buildx build --platform=linux/arm64 .
$ docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 .
$ docker buildx build --platform=darwin .
```
### <a name="output"></a> Set the export action for the build result (-o, --output)
```
-o, --output=[PATH,-,type=TYPE[,KEY=VALUE]
```
Sets the export action for the build result. In `docker build` all builds finish
by creating a container image and exporting it to `docker images`. `buildx` makes
this step configurable allowing results to be exported directly to the client,
oci image tarballs, registry etc.
Buildx with `docker` driver currently only supports local, tarball exporter and
image exporter. `docker-container` driver supports all the exporters.
If just the path is specified as a value, `buildx` will use the local exporter
with this path as the destination. If the value is "-", `buildx` will use `tar`
exporter and write to `stdout`.
**Examples**
```console
$ docker buildx build -o . .
$ docker buildx build -o outdir .
$ docker buildx build -o - - > out.tar
$ docker buildx build -o type=docker .
$ docker buildx build -o type=docker,dest=- . > myimage.tar
$ docker buildx build -t tonistiigi/foo -o type=registry
```
Supported exported types are:
#### `local`
The `local` export type writes all result files to a directory on the client. The
new files will be owned by the current user. On multi-platform builds, all results
will be put in subdirectories by their platform.
Attribute key:
- `dest` - destination directory where files will be written
#### `tar`
The `tar` export type writes all result files as a single tarball on the client.
On multi-platform builds all results will be put in subdirectories by their platform.
Attribute key:
- `dest` - destination path where tarball will be written. “-” writes to stdout.
#### `oci`
The `oci` export type writes the result image or manifest list as an [OCI image
layout](https://github.com/opencontainers/image-spec/blob/v1.0.1/image-layout.md)
tarball on the client.
Attribute key:
- `dest` - destination path where tarball will be written. “-” writes to stdout.
#### `docker`
The `docker` export type writes the single-platform result image as a [Docker image
specification](https://github.com/docker/docker/blob/v20.10.2/image/spec/v1.2.md)
tarball on the client. Tarballs created by this exporter are also OCI compatible.
Currently, multi-platform images cannot be exported with the `docker` export type.
The most common usecase for multi-platform images is to directly push to a registry
(see [`registry`](#registry)).
Attribute keys:
- `dest` - destination path where tarball will be written. If not specified the
tar will be loaded automatically to the current docker instance.
- `context` - name for the docker context where to import the result
#### `image`
The `image` exporter writes the build result as an image or a manifest list. When
using `docker` driver the image will appear in `docker images`. Optionally, image
can be automatically pushed to a registry by specifying attributes.
Attribute keys:
- `name` - name (references) for the new image.
- `push` - boolean to automatically push the image.
#### `registry`
The `registry` exporter is a shortcut for `type=image,push=true`.
### <a name="push"></a> Push the build result to a registry (--push)
Shorthand for [`--output=type=registry`](#registry). Will automatically push the
build result to registry.
### <a name="load"></a> Load the single-platform build result to `docker images` (--load)
Shorthand for [`--output=type=docker`](#docker). Will automatically load the
single-platform build result to `docker images`.
### <a name="cache-from"></a> Use an external cache source for a build (--cache-from)
```
--cache-from=[NAME|type=TYPE[,KEY=VALUE]]
```
Use an external cache source for a build. Supported types are `registry` and `local`.
The `registry` source can import cache from a cache manifest or (special) image
configuration on the registry. The `local` source can import cache from local
files previously exported with `--cache-to`.
If no type is specified, `registry` exporter is used with a specified reference.
`docker` driver currently only supports importing build cache from the registry.
**Examples**
```console
$ docker buildx build --cache-from=user/app:cache .
$ docker buildx build --cache-from=user/app .
$ docker buildx build --cache-from=type=registry,ref=user/app .
$ docker buildx build --cache-from=type=local,src=path/to/cache .
```
### <a name="cache-to"></a> Export build cache to an external cache destination (--cache-to)
```
--cache-to=[NAME|type=TYPE[,KEY=VALUE]]
```
Export build cache to an external cache destination. Supported types are `registry`,
`local` and `inline`. Registry exports build cache to a cache manifest in the
registry, local exports cache to a local directory on the client and inline writes
the cache metadata into the image configuration.
`docker` driver currently only supports exporting inline cache metadata to image
configuration. Alternatively, `--build-arg BUILDKIT_INLINE_CACHE=1` can be used
to trigger inline cache exporter.
Attribute key:
- `mode` - Specifies how many layers are exported with the cache. “min” on only
exports layers already in the final build stage, “max” exports layers for
all stages. Metadata is always exported for the whole build.
**Examples**
```console
$ docker buildx build --cache-to=user/app:cache .
$ docker buildx build --cache-to=type=inline .
$ docker buildx build --cache-to=type=registry,ref=user/app .
$ docker buildx build --cache-to=type=local,dest=path/to/cache .
```
### <a name="allow"></a> Allow extra privileged entitlement (--allow)
```
--allow=ENTITLEMENT
```
Allow extra privileged entitlement. List of entitlements:
- `network.host` - Allows executions with host networking.
- `security.insecure` - Allows executions without sandbox. See
[related Dockerfile extensions](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---securityinsecuresandbox).
For entitlements to be enabled, the `buildkitd` daemon also needs to allow them
with `--allow-insecure-entitlement` (see [`create --buildkitd-flags`](buildx_create.md#--buildkitd-flags-flags))
**Examples**
```console
$ docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
$ docker buildx build --allow security.insecure .
```

@ -0,0 +1,189 @@
# buildx create
```
docker buildx create [OPTIONS] [CONTEXT|ENDPOINT]
```
<!---MARKER_GEN_START-->
Create a new builder instance
### Options
| Name | Description |
| --- | --- |
| [`--append`](#append) | Append a node to builder instead of changing it |
| `--builder string` | Override the configured builder instance |
| [`--buildkitd-flags string`](#buildkitd-flags) | Flags for buildkitd daemon |
| [`--config string`](#config) | BuildKit config file |
| [`--driver string`](#driver) | Driver to use (available: []) |
| [`--driver-opt stringArray`](#driver-opt) | Options for the driver |
| [`--leave`](#leave) | Remove a node from builder instead of changing it |
| [`--name string`](#name) | Builder instance name |
| [`--node string`](#node) | Create/modify node with given name |
| [`--platform stringArray`](#platform) | Fixed platforms for current node |
| [`--use`](#use) | Set the current builder instance |
<!---MARKER_GEN_END-->
## Description
Create makes a new builder instance pointing to a docker context or endpoint,
where context is the name of a context from `docker context ls` and endpoint is
the address for docker socket (eg. `DOCKER_HOST` value).
By default, the current Docker configuration is used for determining the
context/endpoint value.
Builder instances are isolated environments where builds can be invoked. All
Docker contexts also get the default builder instance.
## Examples
### <a name="append"></a> Append a new node to an existing builder (--append)
The `--append` flag changes the action of the command to append a new node to an
existing builder specified by `--name`. Buildx will choose an appropriate node
for a build based on the platforms it supports.
**Examples**
```console
$ docker buildx create mycontext1
eager_beaver
$ docker buildx create --name eager_beaver --append mycontext2
eager_beaver
```
### <a name="buildkitd-flags"></a> Specify options for the buildkitd daemon (--buildkitd-flags)
```
--buildkitd-flags FLAGS
```
Adds flags when starting the buildkitd daemon. They take precedence over the
configuration file specified by [`--config`](#--config-file). See `buildkitd --help`
for the available flags.
**Example**
```
--buildkitd-flags '--debug --debugaddr 0.0.0.0:6666'
```
### <a name="config"></a> Specify a configuration file for the buildkitd daemon (--config)
```
--config FILE
```
Specifies the configuration file for the buildkitd daemon to use. The configuration
can be overridden by [`--buildkitd-flags`](#--buildkitd-flags-flags).
See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).
### <a name="driver"></a> Set the builder driver to use (--driver)
```
--driver DRIVER
```
Sets the builder driver to be used. There are two available drivers, each have
their own specificities.
- `docker` - Uses the builder that is built into the docker daemon. With this
driver, the [`--load`](buildx_build.md#--load) flag is implied by default on
`buildx build`. However, building multi-platform images or exporting cache is
not currently supported.
- `docker-container` - Uses a buildkit container that will be spawned via docker.
With this driver, both building multi-platform images and exporting cache are
supported. However, images built will not automatically appear in `docker images`
(see [`build --load`](buildx_build.md#--load)).
- `kubernetes` - Uses a kubernetes pods. With this driver, you can spin up pods
with defined buildkit container image to build your images.
### <a name="driver-opt"></a> Set additional driver-specific options (--driver-opt)
```
--driver-opt OPTIONS
```
Passes additional driver-specific options. Details for each driver:
- `docker` - No driver options
- `docker-container`
- `image=IMAGE` - Sets the container image to be used for running buildkit.
- `network=NETMODE` - Sets the network mode for running the buildkit container.
- Example:
```console
--driver docker-container --driver-opt image=moby/buildkit:master,network=host
```
- `kubernetes`
- `image=IMAGE` - Sets the container image to be used for running buildkit.
- `namespace=NS` - Sets the Kubernetes namespace. Defaults to the current namespace.
- `replicas=N` - Sets the number of `Pod` replicas. Defaults to 1.
- `requests.cpu` - Sets the request CPU value specified in units of Kubernetes CPU. Example `requests.cpu=100m`, `requests.cpu=2`
- `requests.memory` - Sets the request memory value specified in bytes or with a valid suffix. Example `requests.memory=500Mi`, `requests.memory=4G`
- `limits.cpu` - Sets the limit CPU value specified in units of Kubernetes CPU. Example `limits.cpu=100m`, `limits.cpu=2`
- `limits.memory` - Sets the limit memory value specified in bytes or with a valid suffix. Example `limits.memory=500Mi`, `limits.memory=4G`
- `nodeselector="label1=value1,label2=value2"` - Sets the kv of `Pod` nodeSelector. No Defaults. Example `nodeselector=kubernetes.io/arch=arm64`
- `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
- `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
### <a name="leave"></a> Remove a node from a builder (--leave)
The `--leave` flag changes the action of the command to remove a node from a
builder. The builder needs to be specified with `--name` and node that is removed
is set with `--node`.
**Examples**
```console
$ docker buildx create --name mybuilder --node mybuilder0 --leave
```
### <a name="name"></a> Specify the name of the builder (--name)
```
--name NAME
```
The `--name` flag specifies the name of the builder to be created or modified.
If none is specified, one will be automatically generated.
### <a name="node"></a> Specify the name of the node (--node)
```
--node NODE
```
The `--node` flag specifies the name of the node to be created or modified. If
none is specified, it is the name of the builder it belongs to, with an index
number suffix.
### <a name="platform"></a> Set the platforms supported by the node
```
--platform PLATFORMS
```
The `--platform` flag sets the platforms supported by the node. It expects a
comma-separated list of platforms of the form OS/architecture/variant. The node
will also automatically detect the platforms it supports, but manual values take
priority over the detected ones and can be used when multiple nodes support
building for the same platform.
**Examples**
```console
$ docker buildx create --platform linux/amd64
$ docker buildx create --platform linux/arm64,linux/arm/v8
```
### <a name="use"></a> Automatically switch to the newly created builder
The `--use` flag automatically switches the current builder to the newly created
one. Equivalent to running `docker buildx use $(docker buildx create ...)`.

@ -0,0 +1,19 @@
# buildx du
```
docker buildx du
```
<!---MARKER_GEN_START-->
Disk usage
### Options
| Name | Description |
| --- | --- |
| `--builder string` | Override the configured builder instance |
| `--filter filter` | Provide filter values |
| `--verbose` | Provide a more verbose output |
<!---MARKER_GEN_END-->

@ -0,0 +1,24 @@
# buildx imagetools
```
docker buildx imagetools [OPTIONS] COMMAND
```
<!---MARKER_GEN_START-->
Commands to work on images in registry
### Subcommands
| Name | Description |
| --- | --- |
| [`create`](buildx_imagetools_create.md) | Create a new image based on source images |
| [`inspect`](buildx_imagetools_inspect.md) | Show details of image in the registry |
<!---MARKER_GEN_END-->
## Description
Imagetools contains commands for working with manifest lists in the registry.
These commands are useful for inspecting multi-platform build results.

@ -0,0 +1,80 @@
# buildx imagetools create
```
docker buildx imagetools create [OPTIONS] [SOURCE] [SOURCE...]
```
<!---MARKER_GEN_START-->
Create a new image based on source images
### Options
| Name | Description |
| --- | --- |
| [`--append`](#append) | Append to existing manifest |
| `--builder string` | Override the configured builder instance |
| [`--dry-run`](#dry-run) | Show final image instead of pushing |
| [`-f`](#file), [`--file stringArray`](#file) | Read source descriptor from file |
| [`-t`](#tag), [`--tag stringArray`](#tag) | Set reference for new image |
<!---MARKER_GEN_END-->
## Description
Imagetools contains commands for working with manifest lists in the registry.
These commands are useful for inspecting multi-platform build results.
Create a new manifest list based on source manifests. The source manifests can
be manifest lists or single platform distribution manifests and must already
exist in the registry where the new manifest is created. If only one source is
specified, create performs a carbon copy.
## Examples
### <a name="append"></a> Append new sources to an existing manifest list (--append)
Use the `--append` flag to append the new sources to an existing manifest list
in the destination.
### <a name="dry-run"></a> Show final image instead of pushing (--dry-run)
Use the `--dry-run` flag to not push the image, just show it.
### <a name="file"></a> Read source descriptor from a file (-f, --file)
```
-f FILE or --file FILE
```
Reads source from files. A source can be a manifest digest, manifest reference,
or a JSON of OCI descriptor object.
In order to define annotations or additional platform properties like `os.version` and
`os.features` you need to add them in the OCI descriptor object encoded in JSON.
```
docker buildx imagetools inspect --raw alpine | jq '.manifests[0] | .platform."os.version"="10.1"' > descr.json
docker buildx imagetools create -f descr.json myuser/image
```
The descriptor in the file is merged with existing descriptor in the registry if it exists.
The supported fields for the descriptor are defined in [OCI spec](https://github.com/opencontainers/image-spec/blob/master/descriptor.md#properties) .
### <a name="tag"></a> Set reference for new image (-t, --tag)
```
-t IMAGE or --tag IMAGE
```
Use the `-t` or `--tag` flag to set the name of the image to be created.
**Examples**
```console
$ docker buildx imagetools create --dry-run alpine@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907 sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
$ docker buildx imagetools create -t tonistiigi/myapp -f image1 -f image2
```

@ -0,0 +1,47 @@
# buildx imagetools inspect
```
docker buildx imagetools inspect [OPTIONS] NAME
```
<!---MARKER_GEN_START-->
Show details of image in the registry
### Options
| Name | Description |
| --- | --- |
| `--builder string` | Override the configured builder instance |
| [`--raw`](#raw) | Show original JSON manifest |
<!---MARKER_GEN_END-->
## Description
Show details of image in the registry.
Example:
```console
$ docker buildx imagetools inspect alpine
Name: docker.io/library/alpine:latest
MediaType: application/vnd.docker.distribution.manifest.list.v2+json
Digest: sha256:28ef97b8686a0b5399129e9b763d5b7e5ff03576aa5580d6f4182a49c5fe1913
Manifests:
Name: docker.io/library/alpine:latest@sha256:5c40b3c27b9f13c873fefb2139765c56ce97fd50230f1f2d5c91e55dec171907
MediaType: application/vnd.docker.distribution.manifest.v2+json
Platform: linux/amd64
Name: docker.io/library/alpine:latest@sha256:c4ba6347b0e4258ce6a6de2401619316f982b7bcc529f73d2a410d0097730204
MediaType: application/vnd.docker.distribution.manifest.v2+json
Platform: linux/arm/v6
...
```
### <a name="raw"></a> Show original, unformatted JSON manifest (--raw)
Use the `--raw` option to print the original JSON bytes instead of the formatted
output.

@ -0,0 +1,58 @@
# buildx inspect
```
docker buildx inspect [NAME]
```
<!---MARKER_GEN_START-->
Inspect current builder instance
### Options
| Name | Description |
| --- | --- |
| [`--bootstrap`](#bootstrap) | Ensure builder has booted before inspecting |
| `--builder string` | Override the configured builder instance |
<!---MARKER_GEN_END-->
## Description
Shows information about the current or specified builder.
## Examples
### Get information about a builder instance
By default, `inspect` shows information about the current builder. Specify the
name of the builder to inspect to get information about that builder.
The following example shows information about a builder instance named
`elated_tesla`:
```console
$ docker buildx inspect elated_tesla
Name: elated_tesla
Driver: docker-container
Nodes:
Name: elated_tesla0
Endpoint: unix:///var/run/docker.sock
Status: running
Platforms: linux/amd64
Name: elated_tesla1
Endpoint: ssh://ubuntu@1.2.3.4
Status: running
Platforms: linux/arm64, linux/arm/v7, linux/arm/v6
```
### <a name="bootstrap"></a> Ensure that the builder is running before inspecting (--bootstrap)
Use the `--bootstrap` option to ensure that the builder is running before
inspecting it. If the driver is `docker-container`, then `--bootstrap` starts
the buildkit container and waits until it is operational. Bootstrapping is
automatically done during build, and therefore not necessary. The same BuildKit
container is used during the lifetime of the associated builder node (as
displayed in `buildx ls`).

@ -0,0 +1,11 @@
# buildx install
```
docker buildx install
```
<!---MARKER_GEN_START-->
Install buildx as a 'docker builder' alias
<!---MARKER_GEN_END-->

@ -0,0 +1,31 @@
# buildx ls
```
docker buildx ls
```
<!---MARKER_GEN_START-->
List builder instances
<!---MARKER_GEN_END-->
## Description
Lists all builder instances and the nodes for each instance
**Example**
```console
$ docker buildx ls
NAME/NODE DRIVER/ENDPOINT STATUS PLATFORMS
elated_tesla * docker-container
elated_tesla0 unix:///var/run/docker.sock running linux/amd64
elated_tesla1 ssh://ubuntu@1.2.3.4 running linux/arm64, linux/arm/v7, linux/arm/v6
default docker
default default running linux/amd64
```
Each builder has one or more nodes associated with it. The current builder's
name is marked with a `*`.

@ -0,0 +1,23 @@
# buildx prune
```
docker buildx prune
```
<!---MARKER_GEN_START-->
Remove build cache
### Options
| Name | Description |
| --- | --- |
| `-a`, `--all` | Remove all unused images, not just dangling ones |
| `--builder string` | Override the configured builder instance |
| `--filter filter` | Provide filter values (e.g. 'until=24h') |
| `-f`, `--force` | Do not prompt for confirmation |
| `--keep-storage bytes` | Amount of disk space to keep for cache |
| `--verbose` | Provide a more verbose output |
<!---MARKER_GEN_END-->

@ -0,0 +1,16 @@
# buildx rm
```
docker buildx rm [NAME]
```
<!---MARKER_GEN_START-->
Remove a builder instance
<!---MARKER_GEN_END-->
## Description
Removes the specified or current builder. It is a no-op attempting to remove the
default builder.

@ -0,0 +1,16 @@
# buildx stop
```
docker buildx stop [NAME]
```
<!---MARKER_GEN_START-->
Stop builder instance
<!---MARKER_GEN_END-->
## Description
Stops the specified or current builder. This will not prevent buildx build to
restart the builder. The implementation of stop depends on the driver.

@ -0,0 +1,11 @@
# buildx uninstall
```
docker buildx uninstall
```
<!---MARKER_GEN_START-->
Uninstall the 'docker builder' alias
<!---MARKER_GEN_END-->

@ -0,0 +1,25 @@
# buildx use
```
docker buildx use [OPTIONS] NAME
```
<!---MARKER_GEN_START-->
Set the current builder instance
### Options
| Name | Description |
| --- | --- |
| `--builder string` | Override the configured builder instance |
| `--default` | Set builder as default for current context |
| `--global` | Builder persists context changes |
<!---MARKER_GEN_END-->
## Description
Switches the current builder instance. Build commands invoked after this command
will run on a specified builder. Alternatively, a context name can be used to
switch to the default builder of that context.

@ -0,0 +1,21 @@
# buildx version
```
docker buildx version
```
<!---MARKER_GEN_START-->
Show buildx version information
<!---MARKER_GEN_END-->
## Examples
### View version information
```console
$ docker buildx version
github.com/docker/buildx v0.5.1-docker 11057da37336192bfc57d81e02359ba7ba848e4a
```

@ -2,5 +2,5 @@ package bkimage
const (
DefaultImage = "moby/buildkit:buildx-stable-1" // TODO: make this verified
DefaultRootlessImage = "moby/buildkit:v0.6.2-rootless"
DefaultRootlessImage = DefaultImage + "-rootless"
)

@ -12,6 +12,7 @@ import (
"github.com/docker/buildx/driver"
"github.com/docker/buildx/driver/bkimage"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
"github.com/docker/docker/api/types"
dockertypes "github.com/docker/docker/api/types"
@ -31,6 +32,14 @@ type Driver struct {
env []string
}
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Config() driver.InitConfig {
return d.InitConfig
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.DockerAPI.ContainerInspect(ctx, d.Name)
@ -59,7 +68,13 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
}
if err := l.Wrap("pulling image "+imageName, func() error {
rc, err := d.DockerAPI.ImageCreate(ctx, imageName, types.ImageCreateOptions{})
ra, err := imagetools.RegistryAuthForRef(imageName, d.Auth)
if err != nil {
return err
}
rc, err := d.DockerAPI.ImageCreate(ctx, imageName, types.ImageCreateOptions{
RegistryAuth: ra,
})
if err != nil {
return err
}
@ -86,11 +101,12 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
if err := l.Wrap("creating container "+d.Name, func() error {
hc := &container.HostConfig{
Privileged: true,
UsernsMode: "host",
}
if d.netMode != "" {
hc.NetworkMode = container.NetworkMode(d.netMode)
}
_, err := d.DockerAPI.ContainerCreate(ctx, cfg, hc, &network.NetworkingConfig{}, d.Name)
_, err := d.DockerAPI.ContainerCreate(ctx, cfg, hc, &network.NetworkingConfig{}, nil, d.Name)
if err != nil {
return err
}
@ -263,7 +279,7 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
conn = demuxConn(conn)
return client.New(ctx, "", client.WithDialer(func(string, time.Duration) (net.Conn, error) {
return client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
return conn, nil
}))
}

@ -3,7 +3,6 @@ package docker
import (
"context"
"net"
"time"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
@ -39,7 +38,7 @@ func (d *Driver) Rm(ctx context.Context, force bool) error {
}
func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
return client.New(ctx, "", client.WithDialer(func(string, time.Duration) (net.Conn, error) {
return client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
return d.DockerAPI.DialHijack(ctx, "/grpc", "h2c", nil)
}))
}
@ -48,9 +47,8 @@ func (d *Driver) Features() map[driver.Feature]bool {
return map[driver.Feature]bool{
driver.OCIExporter: false,
driver.DockerExporter: false,
driver.CacheExport: false,
driver.MultiPlatform: false,
driver.CacheExport: false,
driver.MultiPlatform: false,
}
}
@ -58,4 +56,10 @@ func (d *Driver) Factory() driver.Factory {
return d.factory
}
func (d *Driver) IsDefaultMobyDriver() {}
func (d *Driver) IsMobyDriver() bool {
return true
}
func (d *Driver) Config() driver.InitConfig {
return d.InitConfig
}

@ -5,6 +5,7 @@ import (
"github.com/docker/buildx/store"
"github.com/docker/buildx/util/progress"
clitypes "github.com/docker/cli/cli/config/types"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
@ -44,6 +45,10 @@ type Info struct {
DynamicNodes []store.Node
}
type Auth interface {
GetAuthConfig(registryHostname string) (clitypes.AuthConfig, error)
}
type Driver interface {
Factory() Factory
Bootstrap(context.Context, progress.Logger) error
@ -52,6 +57,8 @@ type Driver interface {
Rm(ctx context.Context, force bool) error
Client(ctx context.Context) (*client.Client, error)
Features() map[Feature]bool
IsMobyDriver() bool
Config() InitConfig
}
func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) {
@ -66,11 +73,7 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, er
if try > 2 {
return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
}
if err := d.Bootstrap(ctx, func(s *client.SolveStatus) {
if pw != nil {
pw.Status() <- s
}
}); err != nil {
if err := d.Bootstrap(ctx, pw.Write); err != nil {
return nil, err
}
}

@ -44,12 +44,19 @@ type Driver struct {
podChooser podchooser.PodChooser
}
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Config() driver.InitConfig {
return d.InitConfig
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.deploymentClient.Get(d.deployment.Name, metav1.GetOptions{})
_, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
if err != nil {
// TODO: return err if err != ErrNotFound
_, err = d.deploymentClient.Create(d.deployment)
_, err = d.deploymentClient.Create(ctx, d.deployment, metav1.CreateOptions{})
if err != nil {
return errors.Wrapf(err, "error while calling deploymentClient.Create for %q", d.deployment.Name)
}
@ -72,7 +79,7 @@ func (d *Driver) wait(ctx context.Context) error {
depl *appsv1.Deployment
)
for try := 0; try < 100; try++ {
depl, err = d.deploymentClient.Get(d.deployment.Name, metav1.GetOptions{})
depl, err = d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
if err == nil {
if depl.Status.ReadyReplicas >= int32(d.minReplicas) {
return nil
@ -90,7 +97,7 @@ func (d *Driver) wait(ctx context.Context) error {
}
func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
depl, err := d.deploymentClient.Get(d.deployment.Name, metav1.GetOptions{})
depl, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})
if err != nil {
// TODO: return err if err != ErrNotFound
return &driver.Info{
@ -102,7 +109,7 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) {
Status: driver.Stopped,
}, nil
}
pods, err := podchooser.ListRunningPods(d.podClient, depl)
pods, err := podchooser.ListRunningPods(ctx, d.podClient, depl)
if err != nil {
return nil, err
}
@ -136,7 +143,7 @@ func (d *Driver) Stop(ctx context.Context, force bool) error {
}
func (d *Driver) Rm(ctx context.Context, force bool) error {
if err := d.deploymentClient.Delete(d.deployment.Name, nil); err != nil {
if err := d.deploymentClient.Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}); err != nil {
return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name)
}
return nil
@ -162,7 +169,7 @@ func (d *Driver) Client(ctx context.Context) (*client.Client, error) {
if err != nil {
return nil, err
}
return client.New(ctx, "", client.WithDialer(func(string, time.Duration) (net.Conn, error) {
return client.New(ctx, "", client.WithContextDialer(func(context.Context, string) (net.Conn, error) {
return conn, nil
}))
}

@ -9,7 +9,6 @@ import (
"github.com/docker/buildx/driver/bkimage"
"github.com/docker/buildx/driver/kubernetes/manifest"
"github.com/docker/buildx/driver/kubernetes/podchooser"
"github.com/docker/buildx/util/platformutil"
dockerclient "github.com/docker/docker/client"
"github.com/pkg/errors"
"k8s.io/client-go/kubernetes"
@ -71,6 +70,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
Replicas: 1,
BuildkitFlags: cfg.BuildkitFlags,
Rootless: false,
Platforms: cfg.Platforms,
}
loadbalance := LoadbalanceSticky
imageOverride := ""
@ -85,20 +85,20 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
if err != nil {
return nil, err
}
case "requests.cpu":
deploymentOpt.RequestsCPU = v
case "requests.memory":
deploymentOpt.RequestsMemory = v
case "limits.cpu":
deploymentOpt.LimitsCPU = v
case "limits.memory":
deploymentOpt.LimitsMemory = v
case "rootless":
deploymentOpt.Rootless, err = strconv.ParseBool(v)
if err != nil {
return nil, err
}
deploymentOpt.Image = bkimage.DefaultRootlessImage
case "platform":
if v != "" {
platforms, err := platformutil.Parse(strings.Split(v, ","))
if err != nil {
return nil, err
}
deploymentOpt.Platforms = platforms
}
case "nodeselector":
kvs := strings.Split(strings.Trim(v, `"`), ",")
s := map[string]string{}

@ -7,18 +7,23 @@ import (
v1 "github.com/opencontainers/image-spec/specs-go/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type DeploymentOpt struct {
Namespace string
Name string
Image string
Replicas int
BuildkitFlags []string
Rootless bool
NodeSelector map[string]string
Platforms []v1.Platform
Namespace string
Name string
Image string
Replicas int
BuildkitFlags []string
Rootless bool
NodeSelector map[string]string
RequestsCPU string
RequestsMemory string
LimitsCPU string
LimitsMemory string
Platforms []v1.Platform
}
const (
@ -76,6 +81,10 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
},
},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{},
Limits: corev1.ResourceList{},
},
},
},
},
@ -92,6 +101,38 @@ func NewDeployment(opt *DeploymentOpt) (*appsv1.Deployment, error) {
d.Spec.Template.Spec.NodeSelector = opt.NodeSelector
}
if opt.RequestsCPU != "" {
reqCPU, err := resource.ParseQuantity(opt.RequestsCPU)
if err != nil {
return nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = reqCPU
}
if opt.RequestsMemory != "" {
reqMemory, err := resource.ParseQuantity(opt.RequestsMemory)
if err != nil {
return nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = reqMemory
}
if opt.LimitsCPU != "" {
limCPU, err := resource.ParseQuantity(opt.LimitsCPU)
if err != nil {
return nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU] = limCPU
}
if opt.LimitsMemory != "" {
limMemory, err := resource.ParseQuantity(opt.LimitsMemory)
if err != nil {
return nil, err
}
d.Spec.Template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = limMemory
}
return d, nil
}

@ -25,7 +25,7 @@ type RandomPodChooser struct {
}
func (pc *RandomPodChooser) ChoosePod(ctx context.Context) (*corev1.Pod, error) {
pods, err := ListRunningPods(pc.PodClient, pc.Deployment)
pods, err := ListRunningPods(ctx, pc.PodClient, pc.Deployment)
if err != nil {
return nil, err
}
@ -46,7 +46,7 @@ type StickyPodChooser struct {
}
func (pc *StickyPodChooser) ChoosePod(ctx context.Context) (*corev1.Pod, error) {
pods, err := ListRunningPods(pc.PodClient, pc.Deployment)
pods, err := ListRunningPods(ctx, pc.PodClient, pc.Deployment)
if err != nil {
return nil, err
}
@ -70,7 +70,7 @@ func (pc *StickyPodChooser) ChoosePod(ctx context.Context) (*corev1.Pod, error)
return podMap[chosen], nil
}
func ListRunningPods(client clientcorev1.PodInterface, depl *appsv1.Deployment) ([]*corev1.Pod, error) {
func ListRunningPods(ctx context.Context, client clientcorev1.PodInterface, depl *appsv1.Deployment) ([]*corev1.Pod, error) {
selector, err := metav1.LabelSelectorAsSelector(depl.Spec.Selector)
if err != nil {
return nil, err
@ -78,7 +78,7 @@ func ListRunningPods(client clientcorev1.PodInterface, depl *appsv1.Deployment)
listOpts := metav1.ListOptions{
LabelSelector: selector.String(),
}
podList, err := client.List(listOpts)
podList, err := client.List(ctx, listOpts)
if err != nil {
return nil, err
}

@ -5,10 +5,13 @@ import (
"io/ioutil"
"sort"
"strings"
"sync"
"k8s.io/client-go/rest"
dockerclient "github.com/docker/docker/client"
"github.com/moby/buildkit/client"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -52,6 +55,8 @@ type InitConfig struct {
BuildkitFlags []string
ConfigFile string
DriverOpts map[string]string
Auth Auth
Platforms []specs.Platform
// ContextPathHash can be used for determining pods in the driver instance
ContextPathHash string
}
@ -98,7 +103,7 @@ func GetFactory(name string, instanceRequired bool) Factory {
return nil
}
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, kcc KubeClientConfig, flags []string, config string, do map[string]string, contextPathHash string) (Driver, error) {
func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) {
ic := InitConfig{
DockerAPI: api,
KubeClientConfig: kcc,
@ -106,6 +111,8 @@ func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.API
BuildkitFlags: flags,
ConfigFile: config,
DriverOpts: do,
Auth: auth,
Platforms: platforms,
ContextPathHash: contextPathHash,
}
if f == nil {
@ -115,9 +122,27 @@ func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.API
return nil, err
}
}
return f.New(ctx, ic)
d, err := f.New(ctx, ic)
if err != nil {
return nil, err
}
return &cachedDriver{Driver: d}, nil
}
func GetFactories() map[string]Factory {
return drivers
}
type cachedDriver struct {
Driver
client *client.Client
err error
once sync.Once
}
func (d *cachedDriver) Client(ctx context.Context) (*client.Client, error) {
d.once.Do(func() {
d.client, d.err = d.Driver.Client(ctx)
})
return d.client, d.err
}

@ -1,72 +1,71 @@
module github.com/docker/buildx
go 1.13
require (
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect
github.com/bugsnag/bugsnag-go v1.4.1 // indirect
github.com/bugsnag/panicwrap v1.2.0 // indirect
github.com/cenkalti/backoff v2.1.1+incompatible // indirect
github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e // indirect
github.com/containerd/console v0.0.0-20191219165238-8375c3424e4d
github.com/containerd/containerd v1.4.0-0
github.com/containerd/console v1.0.1
github.com/containerd/containerd v1.5.0-beta.4
github.com/denisenkom/go-mssqldb v0.0.0-20190315220205-a8ed825ac853 // indirect
github.com/docker/cli v0.0.0-20200227165822-2298e6a3fe24
github.com/docker/cli v20.10.5+incompatible
github.com/docker/compose-on-kubernetes v0.4.19-0.20190128150448-356b2919c496 // indirect
github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c
github.com/docker/docker-credential-helpers v0.6.1 // indirect
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v20.10.5+incompatible
github.com/docker/docker-credential-helpers v0.6.4-0.20210125172408-38bea2ce277a // indirect
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/libtrust v0.0.0-20150526203908-9cbd2a1374f4 // indirect
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/gofrs/flock v0.7.2
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/fvbommel/sortorder v1.0.1 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/gofrs/flock v0.7.3
github.com/gofrs/uuid v3.3.0+incompatible // indirect
github.com/google/certificate-transparency-go v1.0.21 // indirect
github.com/google/shlex v0.0.0-20150127133951-6f45313302b9
github.com/googleapis/gnostic v0.3.1 // indirect
github.com/gophercloud/gophercloud v0.6.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hashicorp/hcl/v2 v2.6.0
github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840
github.com/hashicorp/hcl/v2 v2.8.2
github.com/jinzhu/gorm v1.9.2 // indirect
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
github.com/jinzhu/now v1.0.0 // indirect
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/lib/pq v1.0.0 // indirect
github.com/mattn/go-shellwords v1.0.5 // indirect
github.com/lib/pq v1.10.0 // indirect
github.com/mattn/go-sqlite3 v1.10.0 // indirect
github.com/miekg/pkcs11 v0.0.0-20190322140431-074fd7a1ed19 // indirect
github.com/moby/buildkit v0.7.2
github.com/opencontainers/go-digest v1.0.0-rc1
github.com/moby/buildkit v0.8.2-0.20210401015549-df49b648c8bf
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/opencontainers/selinux v1.3.3 // indirect
github.com/pkg/errors v0.9.1
github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/sirupsen/logrus v1.7.0
github.com/spf13/cobra v1.1.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.7.0
github.com/theupdateframework/notary v0.6.1 // indirect
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect
github.com/zclconf/go-cty v1.4.0
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/zclconf/go-cty v1.7.1
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
gopkg.in/fatih/pool.v2 v2.0.0 // indirect
gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
k8s.io/api v0.16.7
k8s.io/apimachinery v0.16.7
k8s.io/client-go v0.16.7
vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect
k8s.io/api v0.20.1
k8s.io/apimachinery v0.20.1
k8s.io/client-go v0.20.1
)
replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
replace github.com/docker/docker => github.com/docker/docker v1.4.2-0.20200227233006-38f52c9fec82
replace (
// protobuf: corresponds to containerd (through buildkit)
github.com/golang/protobuf => github.com/golang/protobuf v1.3.5
github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
replace github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
go 1.13
// genproto: corresponds to containerd (through buildkit)
google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
// grpc: corresponds to protobuf
google.golang.org/grpc => google.golang.org/grpc v1.30.0
)

933
go.sum

File diff suppressed because it is too large Load Diff

@ -1,13 +0,0 @@
{
"Vendor": true,
"Deadline": "8m",
"Exclude": [".*.pb.go"],
"DisableAll": true,
"Enable": [
"gofmt",
"goimports",
"ineffassign",
"vet",
"deadcode"
]
}

@ -1,56 +1,16 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -eu
: ${TARGETPLATFORM=$CLI_PLATFORM}
: ${CONTINUOUS_INTEGRATION=}
set -ex
platformFlag=""
if [ -n "$TARGETPLATFORM" ]; then
platformFlag="--platform $TARGETPLATFORM"
fi
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
binariesDocker() {
mkdir -p bin/tmp
export DOCKER_BUILDKIT=1
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
platformFlag=""
if [ -n "$TARGETPLATFORM" ]; then
platformFlag="--build-arg=TARGETPLATFORM=$TARGETPLATFORM"
fi
docker build $platformFlag --target=binaries --iidfile $iidfile --force-rm .
iid=$(cat $iidfile)
containerID=$(docker create $iid copy)
docker cp $containerID:/ bin/tmp
mv bin/tmp/build* bin/
rm -rf bin/tmp
docker rm $containerID
docker rmi -f $iid
rm -f $iidfile
}
binaries() {
platformFlag=""
if [ ! -z "$TARGETPLATFORM" ]; then
platformFlag="--frontend-opt=platform=$TARGETPLATFORM"
fi
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--frontend-opt target=binaries $platformFlag \
--output type=local,dest=./bin/
}
case $buildmode in
"buildkit")
binaries
;;
"docker-buildkit")
binariesDocker
;;
*)
echo "buildctl or docker with buildkit support is required"
exit 1
;;
esac
buildxCmd build $platformFlag \
--target "binaries" \
--output "type=local,dest=./bin/" \
.

@ -0,0 +1,38 @@
#!/usr/bin/env bash
TYP=$1
. $(dirname $0)/util
set -e
usage() {
echo "usage: ./hack/build_ci_first_pass <binaries>"
exit 1
}
if [ -z "$TYP" ]; then
usage
fi
importCacheFlags=""
exportCacheFlags=""
if [ "$GITHUB_ACTIONS" = "true" ]; then
if [ -n "$cacheRefFrom" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheRefFrom"
fi
if [ -n "$cacheRefTo" ]; then
exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo"
fi
fi
case $TYP in
"binaries")
buildxCmd build $importCacheFlags $exportCacheFlags \
--target "binaries" \
$currentcontext
;;
*)
echo >&2 "Unknown type $TYP"
exit 1
;;
esac

@ -1,20 +1,24 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -e
: ${TARGETPLATFORM=linux/amd64,linux/arm/v7,linux/arm64,darwin/amd64,windows/amd64,linux/ppc64le,linux/s390x}
: ${CONTINUOUS_INTEGRATION=}
: ${EXPORT_LOCAL=}
set -ex
importCacheFlags=""
if [ "$GITHUB_ACTIONS" = "true" ]; then
if [ -n "$cacheRefFrom" ]; then
importCacheFlags="--cache-from=type=local,src=$cacheRefFrom"
fi
fi
exportFlag=""
if [ -n "$EXPORT_LOCAL" ]; then
exportFlag="--output=type=local,dest=$EXPORT_LOCAL"
fi
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain";
exportFlag="--output=type=local,dest=$EXPORT_LOCAL"
fi
buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. --opt platform=$TARGETPLATFORM $exportFlag --opt target=binaries
buildxCmd build $importCacheFlags $exportFlag \
--target "binaries" \
--platform "$TARGETPLATFORM" \
$currentcontext

@ -0,0 +1,29 @@
# syntax = docker/dockerfile:1.2
FROM golang:1.16-alpine AS docsgen
WORKDIR /src
RUN --mount=target=. \
--mount=target=/root/.cache,type=cache \
go build -mod=vendor -o /out/docsgen ./docs/docsgen
FROM alpine AS gen
RUN apk add --no-cache rsync git
WORKDIR /src
COPY --from=docsgen /out/docsgen /usr/bin
RUN --mount=target=/context \
--mount=target=.,type=tmpfs,readwrite \
rsync -a /context/. . && \
docsgen && \
mkdir /out && cp -r docs/reference /out
FROM scratch AS update
COPY --from=gen /out /out
FROM gen AS validate
RUN --mount=target=/context \
--mount=target=.,type=tmpfs,readwrite \
rsync -a /context/. . && \
git add -A && \
rm -rf docs/reference/* && \
cp -rf /out/* ./docs/ && \
./hack/validate-docs check

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.0-experimental
# syntax=docker/dockerfile:1.2
FROM golang:1.13-alpine
RUN apk add --no-cache git
RUN go get -u gopkg.in/alecthomas/gometalinter.v1 \
&& mv /go/bin/gometalinter.v1 /go/bin/gometalinter \
&& gometalinter --install
FROM golang:1.16-alpine
RUN apk add --no-cache gcc musl-dev yamllint
RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.36.0
WORKDIR /go/src/github.com/docker/buildx
RUN --mount=target=/go/src/github.com/docker/buildx \
gometalinter --config=gometalinter.json ./...
RUN --mount=target=/go/src/github.com/docker/buildx --mount=target=/root/.cache,type=cache \
golangci-lint run
RUN --mount=target=/go/src/github.com/docker/buildx --mount=target=/root/.cache,type=cache \
yamllint -c .yamllint.yml --strict .

@ -1,5 +1,6 @@
# syntax = docker/dockerfile:1.0-experimental
FROM golang:1.13-alpine AS vendored
# syntax = docker/dockerfile:1.2
FROM golang:1.16-alpine AS vendored
RUN apk add --no-cache git rsync
WORKDIR /src
RUN --mount=target=/context \

@ -1,37 +1,6 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -eu -o pipefail -x
set -eu
: ${CONTINUOUS_INTEGRATION=}
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
lintDocker() {
export DOCKER_BUILDKIT=1
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
docker build --iidfile $iidfile -f ./hack/dockerfiles/lint.Dockerfile --force-rm .
iid=$(cat $iidfile)
docker rmi $iid
rm -f $iidfile
}
lint() {
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--frontend-opt filename=./hack/dockerfiles/lint.Dockerfile
}
case $buildmode in
"buildkit")
lint
;;
"docker-buildkit")
lintDocker
;;
*)
echo "buildctl or docker with buildkit support is required"
exit 1
;;
esac
buildxCmd build --file ./hack/dockerfiles/lint.Dockerfile .

@ -3,14 +3,10 @@
TAG=$1
OUT=$2
. $(dirname $0)/util
set -eu -o pipefail
: ${PLATFORMS=linux/amd64}
: ${CONTINUOUS_INTEGRATION=}
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
usage() {
echo "usage: ./hack/release <tag> <out>"
@ -21,12 +17,15 @@ if [ -z "$TAG" ] || [ -z "$OUT" ]; then
usage
fi
importCacheFlags=""
if [[ -n "$cacheRefFrom" ]] && [[ "$cacheType" = "local" ]]; then
for ref in $cacheRefFrom; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
fi
set -x
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--opt target=release \
--opt platform=$PLATFORMS \
--exporter local \
--exporter-opt output=$OUT
buildxCmd build $importCacheFlags \
--target "release" \
--platform "$PLATFORMS" \
--output "type=local,dest=$OUT" \
$currentcontext

@ -3,51 +3,45 @@
. $(dirname $0)/util
set -eu -o pipefail
: ${CONTINUOUS_INTEGRATION=}
: ${BUILDX_NOCACHE=}
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
: ${TEST_COVERAGE=}
importCacheFlags=""
if [ -n "$cacheRefFrom" ]; then
if [ "$cacheType" = "local" ]; then
for ref in $cacheRefFrom; do
importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref "
done
fi
fi
iid="buildx-tests"
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
set -x
case $buildmode in
"buildkit")
tmpfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
buildctl build $progressFlag --frontend=dockerfile.v0 \
--local context=. --local dockerfile=. \
--frontend-opt target=integration-tests \
--output type=docker,name=$iid,dest=$tmpfile
docker load -i $tmpfile
rm $tmpfile
;;
"docker-buildkit")
export DOCKER_BUILDKIT=1
docker build --iidfile $iidfile --target integration-tests --force-rm .
iid=$(cat $iidfile)
;;
*)
echo "docker with buildkit support is required"
exit 1
;;
esac
coverageVol=""
coverageFlags=""
if [ "$TEST_COVERAGE" = "1" ]; then
covdir="$(pwd)/coverage"
mkdir -p "$covdir"
coverageVol="-v $covdir:/coverage"
coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic"
fi
buildxCmd build $importCacheFlags \
--target "integration-tests" \
--output "type=docker,name=$iid" \
$currentcontext
cacheVolume="buildx-cache"
if ! docker inspect "$cacheVolume" > /dev/null 2>&1 ; then
cacheVolume=$(docker create --name=buildx-cache -v /root/.cache -v /go/pkg/mod alpine)
if ! docker inspect "$cacheVolume" > /dev/null 2>&1; then
cacheVolume=$(docker create --name=buildx-cache -v /root/.cache -v /go/pkg/mod alpine)
fi
docker run --rm -v /tmp --volumes-from=$cacheVolume --privileged $iid go test ${TESTFLAGS:--v} ${TESTPKGS:-./...}
docker run --rm -v /tmp $coverageVol --volumes-from=$cacheVolume --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...}
if [ -n "$BUILDX_NOCACHE" ]; then
docker rm -v $cacheVolume
fi
case $buildmode in
"docker-buildkit")
rm "$iidfile"
docker rmi $iid
;;
esac
rm "$iidfile"
docker rmi $iid

@ -0,0 +1,16 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -eu
output=$(mktemp -d -t buildx-output.XXXXXXXXXX)
buildxCmd build \
--target "update" \
--output "type=local,dest=$output" \
--file "./hack/dockerfiles/docs.Dockerfile" \
.
rm -rf ./docs/reference/*
cp -R "$output"/out/* ./docs/
rm -rf $output

@ -1,45 +1,16 @@
#!/usr/bin/env bash
. $(dirname $0)/util
set -eu -o pipefail -x
set -eu
: ${CONTINUOUS_INTEGRATION=}
output=$(mktemp -d -t buildx-output.XXXXXXXXXX)
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
buildxCmd build \
--target "update" \
--output "type=local,dest=$output" \
--file "./hack/dockerfiles/vendor.Dockerfile" \
.
case $buildmode in
"buildkit")
output=$(mktemp -d -t buildctl-output.XXXXXXXXXX)
buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
--frontend-opt target=update \
--frontend-opt filename=./hack/dockerfiles/vendor.Dockerfile \
--output type=local,dest=$output
rm -rf ./vendor
cp -R "$output/out/" .
rm -rf $output
;;
*)
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
case $buildmode in
"docker-buildkit")
export DOCKER_BUILDKIT=1
docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --target update --force-rm .
;;
*)
echo "buildctl or docker with buildkit support is required"
exit 1
;;
esac
iid=$(cat $iidfile)
cid=$(docker create $iid noop)
rm -rf ./vendor
docker cp $cid:/out/go.mod .
docker cp $cid:/out/go.sum .
docker cp $cid:/out/vendor .
docker rm $cid
rm -f $iidfile
;;
esac
rm -rf ./vendor
cp -R "$output"/out/* .
rm -rf $output

@ -1,26 +1,42 @@
#!/usr/bin/env sh
: ${CI=}
: ${PREFER_BUILDCTL=}
: ${PREFER_LEGACY=}
: ${CLI_PLATFORM=}
: ${GITHUB_ACTIONS=}
: ${CACHEDIR_FROM=}
: ${CACHEDIR_TO=}
newerEqualThan() { # $1=minimum wanted version $2=actual-version
[ "$1" = "$(printf "$1\n$2" | sort -V | head -n 1)" ]
}
buildmode="legacy"
if [ "$PREFER_BUILDCTL" = "1" ]; then
buildmode="buildkit";
else
serverVersion=$(docker info --format '{{.ServerVersion}}')
experimental=$(docker info --format '{{.ExperimentalBuild}}')
if [ "$PREFER_LEGACY" != "1" ] && ( newerEqualThan "18.09" $serverVersion || \
( newerEqualThan "18.06" $serverVersion && [ "true" = "$experimental" ] ) || \
[ "$DOCKER_BUILDKIT" = "1" ]); then
buildmode="docker-buildkit";
fi
echo >&2 "WARNING: PREFER_BUILDCTL is no longer supported. Ignoring."
fi
if [ "$PREFER_LEGACY" = "1" ]; then
echo >&2 "WARNING: PREFER_LEGACY is no longer supported. Ignoring."
fi
progressFlag=""
if [ "$CI" = "true" ]; then
progressFlag="--progress=plain"
fi
buildxCmd() {
if docker buildx version >/dev/null 2>&1; then
set -x
docker buildx "$@" $progressFlag
elif buildx version >/dev/null 2>&1; then
set -x
buildx "$@" $progressFlag
elif docker version >/dev/null 2>&1; then
set -x
DOCKER_BUILDKIT=1 docker "$@" $progressFlag
else
echo >&2 "ERROR: Please enable DOCKER_BUILDKIT or install standalone buildx"
exit 1
fi
}
if [ -z "$CLI_PLATFORM" ]; then
if [ "$(uname -s)" = "Darwin" ]; then
arch="$(uname -m)"
@ -33,3 +49,18 @@ if [ -z "$CLI_PLATFORM" ]; then
fi
fi
cacheType=""
cacheRefFrom=""
cacheRefTo=""
currentref=""
if [ "$GITHUB_ACTIONS" = "true" ]; then
currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF"
cacheType="local"
cacheRefFrom="$CACHEDIR_FROM"
cacheRefTo="$CACHEDIR_TO"
fi
currentcontext="."
if [ -n "$currentref" ]; then
currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref"
fi

@ -0,0 +1,29 @@
#!/usr/bin/env sh
set -eu
case ${1:-} in
'')
. $(dirname $0)/util
buildxCmd build \
--target validate \
--file ./hack/dockerfiles/docs.Dockerfile \
.
;;
check)
status="$(git status --porcelain -- docs/reference 2>/dev/null)"
diffs=$(echo "$status" | grep -v '^[RAD] ' || true)
if [ "$diffs" ]; then
{
set +x
echo 'The result of ./hack/update-docs differs'
echo
echo "$diffs"
echo
echo 'Please vendor your package with ./hack/update-docs'
echo
} >&2
exit 1
fi
echo 'Congratulations! All docs changes are done the right way.'
;;
esac

@ -1,40 +1,20 @@
#!/usr/bin/env sh
set -eu
: ${CONTINUOUS_INTEGRATION=}
: ${DOCKER_BUILDKIT=}
progressFlag=""
if [ "$CONTINUOUS_INTEGRATION" = "true" ]; then progressFlag="--progress=plain"; fi
case ${1:-} in
'')
. $(dirname $0)/util
case $buildmode in
"buildkit")
buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. --frontend-opt filename=./hack/dockerfiles/vendor.Dockerfile --frontend-opt target=validate
'')
. $(dirname $0)/util
buildxCmd build \
--target validate \
--file ./hack/dockerfiles/vendor.Dockerfile \
.
;;
"docker-buildkit")
export DOCKER_BUILDKIT=1
iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --target validate --force-rm . || exit 1
iid=$(cat $iidfile)
docker rmi $iid
rm -f $iidfile
;;
*)
echo "buildkit support is required"
exit 1
;;
esac
;;
check)
status="$(git status --porcelain -- go.mod go.sum vendor 2>/dev/null)"
diffs=$(echo "$status" | grep -v '^[RAD] ' || true)
if [ "$diffs" ]; then
check)
status="$(git status --porcelain -- go.mod go.sum vendor 2>/dev/null)"
diffs=$(echo "$status" | grep -v '^[RAD] ' || true)
if [ "$diffs" ]; then
{
set +x
set +x
echo 'The result of "make vendor" differs'
echo
echo "$diffs"
@ -43,7 +23,7 @@ check)
echo
} >&2
exit 1
fi
echo 'Congratulations! All vendoring changes are done the right way.'
;;
fi
echo 'Congratulations! All vendoring changes are done the right way.'
;;
esac

@ -15,7 +15,6 @@ import (
)
func New(root string) (*Store, error) {
root = filepath.Join(root, "buildx")
if err := os.MkdirAll(filepath.Join(root, "instances"), 0700); err != nil {
return nil, err
}

@ -1,7 +1,8 @@
package build
package buildflags
import (
"encoding/csv"
"os"
"strings"
"github.com/moby/buildkit/client"
@ -45,6 +46,9 @@ func ParseCacheEntry(in []string) ([]client.CacheOptionsEntry, error) {
if im.Type == "" {
return nil, errors.Errorf("type required form> %q", in)
}
if !addGithubToken(&im) {
continue
}
imports = append(imports, im)
}
return imports, nil
@ -58,3 +62,20 @@ func isRefOnlyFormat(in []string) bool {
}
return true
}
func addGithubToken(ci *client.CacheOptionsEntry) bool {
if ci.Type != "gha" {
return true
}
if _, ok := ci.Attrs["token"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
ci.Attrs["token"] = v
}
}
if _, ok := ci.Attrs["url"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok {
ci.Attrs["url"] = v
}
}
return ci.Attrs["token"] != "" && ci.Attrs["url"] != ""
}

@ -1,4 +1,4 @@
package build
package buildflags
import (
"github.com/moby/buildkit/util/entitlements"

@ -1,4 +1,4 @@
package build
package buildflags
import (
"encoding/csv"

@ -1,4 +1,4 @@
package build
package buildflags
import (
"encoding/csv"
@ -10,7 +10,7 @@ import (
)
func ParseSecretSpecs(sl []string) (session.Attachable, error) {
fs := make([]secretsprovider.FileSource, 0, len(sl))
fs := make([]secretsprovider.Source, 0, len(sl))
for _, v := range sl {
s, err := parseSecret(v)
if err != nil {
@ -18,22 +18,23 @@ func ParseSecretSpecs(sl []string) (session.Attachable, error) {
}
fs = append(fs, *s)
}
store, err := secretsprovider.NewFileStore(fs)
store, err := secretsprovider.NewStore(fs)
if err != nil {
return nil, err
}
return secretsprovider.NewSecretProvider(store), nil
}
func parseSecret(value string) (*secretsprovider.FileSource, error) {
func parseSecret(value string) (*secretsprovider.Source, error) {
csvReader := csv.NewReader(strings.NewReader(value))
fields, err := csvReader.Read()
if err != nil {
return nil, errors.Wrap(err, "failed to parse csv secret")
}
fs := secretsprovider.FileSource{}
fs := secretsprovider.Source{}
var typ string
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
key := strings.ToLower(parts[0])
@ -45,16 +46,23 @@ func parseSecret(value string) (*secretsprovider.FileSource, error) {
value := parts[1]
switch key {
case "type":
if value != "file" {
if value != "file" && value != "env" {
return nil, errors.Errorf("unsupported secret type %q", value)
}
typ = value
case "id":
fs.ID = value
case "source", "src":
fs.FilePath = value
case "env":
fs.Env = value
default:
return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
}
}
if typ == "env" && fs.Env == "" {
fs.Env = fs.FilePath
fs.FilePath = ""
}
return &fs, nil
}

@ -1,10 +1,11 @@
package build
package buildflags
import (
"strings"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/sshforward/sshprovider"
"github.com/moby/buildkit/util/gitutil"
)
func ParseSSHSpecs(sl []string) (session.Attachable, error) {
@ -29,3 +30,9 @@ func parseSSH(value string) (*sshprovider.AgentConfig, error) {
}
return &cfg, nil
}
// IsGitSSH returns true if the given repo URL is accessed over ssh
func IsGitSSH(url string) bool {
_, gitProtocol := gitutil.ParseProtocol(url)
return gitProtocol == gitutil.SSHProtocol
}

@ -47,13 +47,16 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
switch mt {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p := descs[i].Platform
if descs[i].Platform == nil {
p, err := r.loadPlatform(ctx, in, dt)
if err != nil {
p = &ocispec.Platform{}
}
if p.OS == "" || p.Architecture == "" {
if err := r.loadPlatform(ctx, p, in, dt); err != nil {
return err
}
descs[i].Platform = p
}
descs[i].Platform = p
case images.MediaTypeDockerSchema1Manifest:
return errors.Errorf("schema1 manifests are not allowed in manifest lists")
}
@ -166,24 +169,35 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D
return err
}
func (r *Resolver) loadPlatform(ctx context.Context, in string, dt []byte) (*ocispec.Platform, error) {
func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error {
var manifest ocispec.Manifest
if err := json.Unmarshal(dt, &manifest); err != nil {
return nil, errors.WithStack(err)
return errors.WithStack(err)
}
dt, err := r.GetDescriptor(ctx, in, manifest.Config)
if err != nil {
return nil, err
return err
}
var p ocispec.Platform
if err := json.Unmarshal(dt, &p); err != nil {
return nil, errors.WithStack(err)
return errors.WithStack(err)
}
p = platforms.Normalize(p)
return &p, nil
if p2.Architecture == "" {
p2.Architecture = p.Architecture
if p2.Variant == "" {
p2.Variant = p.Variant
}
}
if p2.OS == "" {
p2.OS = p.OS
}
return nil
}
func detectMediaType(dt []byte) (string, error) {

@ -3,6 +3,8 @@ package imagetools
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"io"
"net/http"
@ -107,3 +109,26 @@ func toCredentialsFunc(a Auth) func(string) (string, string, error) {
return ac.Username, ac.Password, nil
}
}
func RegistryAuthForRef(ref string, a Auth) (string, error) {
if a == nil {
return "", nil
}
r, err := parseRef(ref)
if err != nil {
return "", err
}
host := reference.Domain(r)
if host == "docker.io" {
host = "https://index.docker.io/v1/"
}
ac, err := a.GetAuthConfig(host)
if err != nil {
return "", err
}
buf, err := json.Marshal(ac)
if err != nil {
return "", err
}
return base64.URLEncoding.EncodeToString(buf), nil
}

@ -11,7 +11,6 @@ import (
)
func FromReader(w Writer, name string, rc io.ReadCloser) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now()
@ -21,9 +20,9 @@ func FromReader(w Writer, name string, rc io.ReadCloser) {
Started: &tm,
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx},
}
})
_, err := io.Copy(ioutil.Discard, rc)
@ -33,8 +32,7 @@ func FromReader(w Writer, name string, rc io.ReadCloser) {
if err != nil {
vtx2.Error = err.Error()
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2},
}
close(status)
})
}

@ -1,101 +1,32 @@
package progress
import (
"context"
"strings"
"sync"
"github.com/moby/buildkit/client"
"golang.org/x/sync/errgroup"
)
type MultiWriter struct {
w Writer
eg *errgroup.Group
once sync.Once
ready chan struct{}
}
func (mw *MultiWriter) WithPrefix(pfx string, force bool) Writer {
in := make(chan *client.SolveStatus)
out := mw.w.Status()
p := &prefixed{
main: mw.w,
in: in,
func WithPrefix(w Writer, pfx string, force bool) Writer {
return &prefixed{
main: w,
pfx: pfx,
force: force,
}
mw.eg.Go(func() error {
mw.once.Do(func() {
close(mw.ready)
})
for {
select {
case v, ok := <-in:
if ok {
if force {
for _, v := range v.Vertexes {
v.Name = addPrefix(pfx, v.Name)
}
}
out <- v
} else {
return nil
}
case <-mw.Done():
return mw.Err()
}
}
})
return p
}
func (mw *MultiWriter) Done() <-chan struct{} {
return mw.w.Done()
}
func (mw *MultiWriter) Err() error {
return mw.w.Err()
}
func (mw *MultiWriter) Status() chan *client.SolveStatus {
return nil
}
type prefixed struct {
main Writer
in chan *client.SolveStatus
}
func (p *prefixed) Done() <-chan struct{} {
return p.main.Done()
}
func (p *prefixed) Err() error {
return p.main.Err()
}
func (p *prefixed) Status() chan *client.SolveStatus {
return p.in
main Writer
pfx string
force bool
}
func NewMultiWriter(pw Writer) *MultiWriter {
if pw == nil {
return nil
}
eg, _ := errgroup.WithContext(context.TODO())
ready := make(chan struct{})
go func() {
<-ready
eg.Wait()
close(pw.Status())
}()
return &MultiWriter{
w: pw,
eg: eg,
ready: ready,
func (p *prefixed) Write(v *client.SolveStatus) {
if p.force {
for _, v := range v.Vertexes {
v.Name = addPrefix(p.pfx, v.Name)
}
}
p.main.Write(v)
}
func addPrefix(pfx, name string) string {

@ -2,6 +2,8 @@ package progress
import (
"context"
"io"
"io/ioutil"
"os"
"github.com/containerd/console"
@ -9,47 +11,55 @@ import (
"github.com/moby/buildkit/util/progress/progressui"
)
type printer struct {
const (
PrinterModeAuto = "auto"
PrinterModeTty = "tty"
PrinterModePlain = "plain"
PrinterModeQuiet = "quiet"
)
type Printer struct {
status chan *client.SolveStatus
done <-chan struct{}
err error
}
func (p *printer) Done() <-chan struct{} {
return p.done
}
func (p *printer) Err() error {
func (p *Printer) Wait() error {
close(p.status)
<-p.done
return p.err
}
func (p *printer) Status() chan *client.SolveStatus {
if p == nil {
return nil
}
return p.status
func (p *Printer) Write(s *client.SolveStatus) {
p.status <- s
}
func NewPrinter(ctx context.Context, out console.File, mode string) Writer {
func NewPrinter(ctx context.Context, out console.File, mode string) *Printer {
statusCh := make(chan *client.SolveStatus)
doneCh := make(chan struct{})
pw := &printer{
pw := &Printer{
status: statusCh,
done: doneCh,
}
if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && mode == "auto" {
if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && mode == PrinterModeAuto {
mode = v
}
go func() {
var c console.Console
if cons, err := console.ConsoleFromFile(out); err == nil && (mode == "auto" || mode == "tty") {
c = cons
var w io.Writer = out
switch mode {
case PrinterModeQuiet:
w = ioutil.Discard
case PrinterModeAuto, PrinterModeTty:
if cons, err := console.ConsoleFromFile(out); err == nil {
c = cons
}
}
// not using shared context to not disrupt display but let is finish reporting errors
pw.err = progressui.DisplaySolveStatus(ctx, "", c, out, statusCh)
pw.err = progressui.DisplaySolveStatus(ctx, "", c, w, statusCh)
close(doneCh)
}()
return pw

@ -13,6 +13,7 @@ type Logger func(*client.SolveStatus)
type SubLogger interface {
Wrap(name string, fn func() error) error
Log(stream int, dt []byte)
SetStatus(*client.VertexStatus)
}
func Wrap(name string, l Logger, fn func(SubLogger) error) (err error) {
@ -88,3 +89,10 @@ func (sl *subLogger) Log(stream int, dt []byte) {
}},
})
}
func (sl *subLogger) SetStatus(st *client.VertexStatus) {
st.Vertex = sl.dgst
sl.logger(&client.SolveStatus{
Statuses: []*client.VertexStatus{st},
})
}

@ -7,56 +7,45 @@ import (
)
func ResetTime(in Writer) Writer {
w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
go func() {
for {
select {
case <-in.Done():
return
case st, ok := <-w.status:
if !ok {
close(in.Status())
return
}
if w.diff == nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Sub(w.tm)
w.diff = &d
}
}
}
if w.diff != nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
in.Status() <- st
return &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
}
func (w *pw) Write(st *client.SolveStatus) {
if w.diff == nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Sub(w.tm)
w.diff = &d
}
}
}
if w.diff != nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
}()
return w
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
w.Writer.Write(st)
}
type pw struct {
@ -66,6 +55,6 @@ type pw struct {
status chan *client.SolveStatus
}
func (p *pw) Status() chan *client.SolveStatus {
return p.status
func (w *pw) Status() chan *client.SolveStatus {
return w.status
}

@ -9,13 +9,10 @@ import (
)
type Writer interface {
Done() <-chan struct{}
Err() error
Status() chan *client.SolveStatus
Write(*client.SolveStatus)
}
func Write(w Writer, name string, f func() error) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now()
@ -25,9 +22,9 @@ func Write(w Writer, name string, f func() error) {
Started: &tm,
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx},
}
})
err := f()
@ -37,7 +34,23 @@ func Write(w Writer, name string, f func() error) {
if err != nil {
vtx2.Error = err.Error()
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2},
}
})
}
func NewChannel(w Writer) (chan *client.SolveStatus, chan struct{}) {
ch := make(chan *client.SolveStatus)
done := make(chan struct{})
go func() {
for {
v, ok := <-ch
if !ok {
close(done)
return
}
w.Write(v)
}
}()
return ch, done
}

@ -25,4 +25,4 @@ inclusion in a `hcl.EvalContext`. It also returns a new `cty.Body` that
contains the remainder of the content from the given body, allowing for
further processing of remaining content.
For more information, see [the godoc reference](http://godoc.org/github.com/hashicorp/hcl/v2/ext/userfunc).
For more information, see [the godoc reference](https://pkg.go.dev/github.com/hashicorp/hcl/v2/ext/userfunc?tab=doc).

@ -0,0 +1,79 @@
package userfunc
import (
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty/function"
)
var funcBodySchema = &hcl.BodySchema{
Attributes: []hcl.AttributeSchema{
{
Name: "params",
Required: true,
},
{
Name: "variadic_param",
Required: false,
},
{
Name: "result",
Required: true,
},
},
}
func decodeUserFunctions(body hcl.Body, blockType string, contextFunc ContextFunc) (funcs map[string]function.Function, remain hcl.Body, diags hcl.Diagnostics) {
schema := &hcl.BodySchema{
Blocks: []hcl.BlockHeaderSchema{
{
Type: blockType,
LabelNames: []string{"name"},
},
},
}
content, remain, diags := body.PartialContent(schema)
if diags.HasErrors() {
return nil, remain, diags
}
// first call to getBaseCtx will populate context, and then the same
// context will be used for all subsequent calls. It's assumed that
// all functions in a given body should see an identical context.
var baseCtx *hcl.EvalContext
getBaseCtx := func() *hcl.EvalContext {
if baseCtx == nil {
if contextFunc != nil {
baseCtx = contextFunc()
}
}
// baseCtx might still be nil here, and that's okay
return baseCtx
}
funcs = make(map[string]function.Function)
for _, block := range content.Blocks {
name := block.Labels[0]
funcContent, funcDiags := block.Body.Content(funcBodySchema)
diags = append(diags, funcDiags...)
if funcDiags.HasErrors() {
continue
}
paramsExpr := funcContent.Attributes["params"].Expr
resultExpr := funcContent.Attributes["result"].Expr
var varParamExpr hcl.Expression
if funcContent.Attributes["variadic_param"] != nil {
varParamExpr = funcContent.Attributes["variadic_param"].Expr
}
f, funcDiags := NewFunction(paramsExpr, varParamExpr, resultExpr, getBaseCtx)
if funcDiags.HasErrors() {
diags = append(diags, funcDiags...)
continue
}
funcs[name] = f
}
return funcs, remain, diags
}

@ -0,0 +1,125 @@
package userfunc
import (
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)
// A ContextFunc is a callback used to produce the base EvalContext for
// running a particular set of functions.
//
// This is a function rather than an EvalContext directly to allow functions
// to be decoded before their context is complete. This will be true, for
// example, for applications that wish to allow functions to refer to themselves.
//
// The simplest use of a ContextFunc is to give user functions access to the
// same global variables and functions available elsewhere in an application's
// configuration language, but more complex applications may use different
// contexts to support lexical scoping depending on where in a configuration
// structure a function declaration is found, etc.
type ContextFunc func() *hcl.EvalContext
// DecodeUserFunctions looks for blocks of the given type in the given body
// and, for each one found, interprets it as a custom function definition.
//
// On success, the result is a mapping of function names to implementations,
// along with a new body that represents the remaining content of the given
// body which can be used for further processing.
//
// The result expression of each function is parsed during decoding but not
// evaluated until the function is called.
//
// If the given ContextFunc is non-nil, it will be called to obtain the
// context in which the function result expressions will be evaluated. If nil,
// or if it returns nil, the result expression will have access only to
// variables named after the declared parameters. A non-nil context turns
// the returned functions into closures, bound to the given context.
//
// If the returned diagnostics set has errors then the function map and
// remain body may be nil or incomplete.
func DecodeUserFunctions(body hcl.Body, blockType string, context ContextFunc) (funcs map[string]function.Function, remain hcl.Body, diags hcl.Diagnostics) {
return decodeUserFunctions(body, blockType, context)
}
// NewFunction creates a new function instance from preparsed HCL expressions.
func NewFunction(paramsExpr, varParamExpr, resultExpr hcl.Expression, getBaseCtx func() *hcl.EvalContext) (function.Function, hcl.Diagnostics) {
var params []string
var varParam string
paramExprs, paramsDiags := hcl.ExprList(paramsExpr)
if paramsDiags.HasErrors() {
return function.Function{}, paramsDiags
}
for _, paramExpr := range paramExprs {
param := hcl.ExprAsKeyword(paramExpr)
if param == "" {
return function.Function{}, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: "Invalid param element",
Detail: "Each parameter name must be an identifier.",
Subject: paramExpr.Range().Ptr(),
}}
}
params = append(params, param)
}
if varParamExpr != nil {
varParam = hcl.ExprAsKeyword(varParamExpr)
if varParam == "" {
return function.Function{}, hcl.Diagnostics{{
Severity: hcl.DiagError,
Summary: "Invalid variadic_param",
Detail: "The variadic parameter name must be an identifier.",
Subject: varParamExpr.Range().Ptr(),
}}
}
}
spec := &function.Spec{}
for _, paramName := range params {
spec.Params = append(spec.Params, function.Parameter{
Name: paramName,
Type: cty.DynamicPseudoType,
})
}
if varParamExpr != nil {
spec.VarParam = &function.Parameter{
Name: varParam,
Type: cty.DynamicPseudoType,
}
}
impl := func(args []cty.Value) (cty.Value, error) {
ctx := getBaseCtx()
ctx = ctx.NewChild()
ctx.Variables = make(map[string]cty.Value)
// The cty function machinery guarantees that we have at least
// enough args to fill all of our params.
for i, paramName := range params {
ctx.Variables[paramName] = args[i]
}
if spec.VarParam != nil {
varArgs := args[len(params):]
ctx.Variables[varParam] = cty.TupleVal(varArgs)
}
result, diags := resultExpr.Value(ctx)
if diags.HasErrors() {
// Smuggle the diagnostics out via the error channel, since
// a diagnostics sequence implements error. Caller can
// type-assert this to recover the individual diagnostics
// if desired.
return cty.DynamicVal, diags
}
return result, nil
}
spec.Type = func(args []cty.Value) (cty.Type, error) {
val, err := impl(args)
return val.Type(), err
}
spec.Impl = func(args []cty.Value, retType cty.Type) (cty.Value, error) {
return impl(args)
}
return function.New(spec), nil
}

@ -61,25 +61,14 @@ var (
instID = &cachedValue{k: "instance/id", trim: true}
)
var (
defaultClient = &Client{hc: &http.Client{
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
ResponseHeaderTimeout: 2 * time.Second,
},
}}
subscribeClient = &Client{hc: &http.Client{
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
},
}}
)
var defaultClient = &Client{hc: &http.Client{
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
},
}}
// NotDefinedError is returned when requested metadata is not defined.
//
@ -206,10 +195,9 @@ func systemInfoSuggestsGCE() bool {
return name == "Google" || name == "Google Compute Engine"
}
// Subscribe calls Client.Subscribe on a client designed for subscribing (one with no
// ResponseHeaderTimeout).
// Subscribe calls Client.Subscribe on the default client.
func Subscribe(suffix string, fn func(v string, ok bool) error) error {
return subscribeClient.Subscribe(suffix, fn)
return defaultClient.Subscribe(suffix, fn)
}
// Get calls Client.Get on the default client.
@ -227,6 +215,9 @@ func InternalIP() (string, error) { return defaultClient.InternalIP() }
// ExternalIP returns the instance's primary external (public) IP address.
func ExternalIP() (string, error) { return defaultClient.ExternalIP() }
// Email calls Client.Email on the default client.
func Email(serviceAccount string) (string, error) { return defaultClient.Email(serviceAccount) }
// Hostname returns the instance's hostname. This will be of the form
// "<instanceID>.c.<projID>.internal".
func Hostname() (string, error) { return defaultClient.Hostname() }
@ -277,9 +268,14 @@ type Client struct {
hc *http.Client
}
// NewClient returns a Client that can be used to fetch metadata. All HTTP requests
// will use the given http.Client instead of the default client.
// NewClient returns a Client that can be used to fetch metadata.
// Returns the client that uses the specified http.Client for HTTP requests.
// If nil is specified, returns the default client.
func NewClient(c *http.Client) *Client {
if c == nil {
return defaultClient
}
return &Client{hc: c}
}
@ -301,7 +297,10 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
host = metadataIP
}
u := "http://" + host + "/computeMetadata/v1/" + suffix
req, _ := http.NewRequest("GET", u, nil)
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return "", "", err
}
req.Header.Set("Metadata-Flavor", "Google")
req.Header.Set("User-Agent", userAgent)
res, err := c.hc.Do(req)
@ -367,6 +366,16 @@ func (c *Client) InternalIP() (string, error) {
return c.getTrimmed("instance/network-interfaces/0/ip")
}
// Email returns the email address associated with the service account.
// The account may be empty or the string "default" to use the instance's
// main account.
func (c *Client) Email(serviceAccount string) (string, error) {
if serviceAccount == "" {
serviceAccount = "default"
}
return c.getTrimmed("instance/service-accounts/" + serviceAccount + "/email")
}
// ExternalIP returns the instance's primary external (public) IP address.
func (c *Client) ExternalIP() (string, error) {
return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip")
@ -394,11 +403,7 @@ func (c *Client) InstanceTags() ([]string, error) {
// InstanceName returns the current VM's instance ID string.
func (c *Client) InstanceName() (string, error) {
host, err := c.Hostname()
if err != nil {
return "", err
}
return strings.Split(host, ".")[0], nil
return c.getTrimmed("instance/name")
}
// Zone returns the current VM's zone, such as "us-central1-b".

@ -0,0 +1 @@
* @microsoft/containerplat

@ -5,21 +5,14 @@ package winio
import (
"os"
"runtime"
"syscall"
"unsafe"
)
//sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
//sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
const (
fileBasicInfo = 0
fileIDInfo = 0x12
"golang.org/x/sys/windows"
)
// FileBasicInfo contains file access time and file attributes information.
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
pad uint32 // padding
}
@ -27,7 +20,7 @@ type FileBasicInfo struct {
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &FileBasicInfo{}
if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -36,13 +29,32 @@ func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return nil
}
// FileStandardInfo contains extended information for the file.
// FILE_STANDARD_INFO in WinBase.h
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_standard_info
type FileStandardInfo struct {
AllocationSize, EndOfFile int64
NumberOfLinks uint32
DeletePending, Directory bool
}
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return si, nil
}
// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
// unique on a system.
type FileIDInfo struct {
@ -53,7 +65,7 @@ type FileIDInfo struct {
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)

@ -3,7 +3,7 @@ module github.com/Microsoft/go-winio
go 1.12
require (
github.com/pkg/errors v0.8.1
github.com/sirupsen/logrus v1.4.1
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.7.0
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
)

@ -1,18 +1,14 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

@ -1,3 +1,5 @@
// +build windows
package winio
import (

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save