From 6cfef7fa36d6b3d2c820a2b38af8295c5b6cfe65 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 3 Sep 2021 22:49:59 +0200 Subject: [PATCH] Bake workflow Signed-off-by: CrazyMax --- .dockerignore | 2 +- .github/workflows/build.yml | 144 ++-------------------------- .github/workflows/validate.yml | 8 -- .mailmap | 9 +- AUTHORS | 40 +++++++- Dockerfile | 40 ++++---- Makefile | 42 +++++--- README.md | 22 ++--- docker-bake.hcl | 128 +++++++++++++++++++++++++ hack/binaries | 16 ---- hack/build_ci_first_pass | 38 -------- hack/cross | 24 ----- hack/demo-env/entrypoint.sh | 4 - hack/dockerfiles/authors.Dockerfile | 33 +++++++ hack/dockerfiles/docs.Dockerfile | 31 +++--- hack/dockerfiles/vendor.Dockerfile | 34 ++++--- hack/generate-authors | 21 ---- hack/lint | 6 -- hack/release | 39 +++----- hack/shell | 11 ++- hack/test | 47 --------- hack/update-docs | 13 +-- hack/update-vendor | 13 +-- hack/util | 66 ------------- hack/validate-docs | 29 ------ hack/validate-vendor | 29 ------ 26 files changed, 344 insertions(+), 545 deletions(-) create mode 100644 docker-bake.hcl delete mode 100755 hack/binaries delete mode 100755 hack/build_ci_first_pass delete mode 100755 hack/cross create mode 100644 hack/dockerfiles/authors.Dockerfile delete mode 100755 hack/generate-authors delete mode 100755 hack/lint delete mode 100755 hack/test delete mode 100755 hack/util delete mode 100755 hack/validate-docs delete mode 100755 hack/validate-vendor diff --git a/.dockerignore b/.dockerignore index 4abb92c2..e1d2025e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ bin/ cross-out/ -release-out/ \ No newline at end of file +release-out/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d2e55cb..f2a99a3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,61 +16,15 @@ on: env: REPO_SLUG: "docker/buildx-bin" REPO_SLUG_ORIGIN: "moby/buildkit:master" - CACHEKEY_BINARIES: "binaries" - PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" + RELEASE_OUT: "./release-out" jobs: - base: + build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - - - name: Cache ${{ env.CACHEKEY_BINARIES }} - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}- - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - - - name: Build ${{ env.CACHEKEY_BINARIES }} - run: | - ./hack/build_ci_first_pass binaries - env: - CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - CACHEDIR_TO: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}-new - - - # FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850 - name: Move cache - run: | - rm -rf /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - mv /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}-new /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - - test: - runs-on: ubuntu-latest - needs: [base] - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Cache ${{ env.CACHEKEY_BINARIES }} - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}- - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -83,91 +37,15 @@ jobs: name: Test run: | make test - env: - TEST_COVERAGE: 1 - TESTFLAGS: -v --parallel=6 --timeout=20m - CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - name: Send to Codecov uses: codecov/codecov-action@v2 with: file: ./coverage/coverage.txt - - cross: - runs-on: ubuntu-latest - needs: [base] - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Cache ${{ env.CACHEKEY_BINARIES }} - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}- - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - - name: Cross + name: Build binaries run: | - make cross - env: - TARGETPLATFORM: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - - binaries: - runs-on: ubuntu-latest - needs: [test, cross] - env: - RELEASE_OUT: ./release-out - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Prepare - id: prep - run: | - TAG=pr - if [[ $GITHUB_REF == refs/tags/v* ]]; then - TAG=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - fi - echo ::set-output name=tag::${TAG} - - - name: Cache ${{ env.CACHEKEY_BINARIES }} - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}- - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - - - name: Build ${{ steps.prep.outputs.tag }} - run: | - ./hack/release ${{ env.RELEASE_OUT }} - env: - PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - CHECKSUMS: 1 - CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} + make release - name: Upload artifacts uses: actions/upload-artifact@v2 @@ -186,6 +64,7 @@ jobs: type=ref,event=branch type=ref,event=pr type=semver,pattern={{version}} + bake-target: meta-helper - name: Login to DockerHub if: github.event_name != 'pull_request' @@ -195,15 +74,13 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push image - uses: docker/build-push-action@v2 + uses: docker/bake-action@v1 with: - context: . - target: binaries + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: image-cross push: ${{ github.event_name != 'pull_request' }} - cache-from: type=local,src=/tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} - platforms: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - name: GitHub Release if: startsWith(github.ref, 'refs/tags/v') @@ -213,4 +90,3 @@ jobs: with: draft: true files: ${{ env.RELEASE_OUT }}/* - name: ${{ steps.prep.outputs.tag }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 53e0774f..ed506742 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -13,9 +13,6 @@ on: - 'master' - 'v[0-9]*' -env: - REPO_SLUG_ORIGIN: "moby/buildkit:master" - jobs: validate: runs-on: ubuntu-latest @@ -30,11 +27,6 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - name: Run run: | diff --git a/.mailmap b/.mailmap index e09bf4d7..4e45f1aa 100644 --- a/.mailmap +++ b/.mailmap @@ -1,6 +1,13 @@ # This file lists all individuals having contributed content to the repository. -# For how it is generated, see `hack/generate-authors`. +# For how it is generated, see hack/dockerfiles/authors.Dockerfile. +CrazyMax +CrazyMax <1951866+crazy-max@users.noreply.github.com> +CrazyMax +Sebastiaan van Stijn +Sebastiaan van Stijn Tibor Vass Tibor Vass Tõnis Tiigi +Ulysses Souza +Wang Jinglei diff --git a/AUTHORS b/AUTHORS index 224634d3..abf95bf7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,7 +1,45 @@ # This file lists all individuals having contributed content to the repository. -# For how it is generated, see `scripts/generate-authors.sh`. +# For how it is generated, see hack/dockerfiles/authors.Dockerfile. +Akihiro Suda +Alex Couture-Beil +Andrew Haines +Andy MacKinlay +Anthony Poschen +Artur Klauser +Batuhan Apaydın Bin Du +Brandon Philips Brian Goff +CrazyMax +dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> +Devin Bayer +Djordje Lukic +Dmytro Makovey +Donghui Wang <977675308@qq.com> +faust +Felipe Santos +Fernando Miguel +gfrancesco +gracenoah +Hollow Man +Ilya Dmitrichenko +Jack Laxson +Jean-Yves Gastaud +khs1994 +Kotaro Adachi +l00397676 +Michal Augustyn +Patrick Van Stee +Saul Shanabrook +Sebastiaan van Stijn +SHIMA Tatsuya +Silvin Lubecki +Solomon Hykes +Sune Keller Tibor Vass Tõnis Tiigi +Ulysses Souza +Wang Jinglei +Xiang Dai <764524258@qq.com> +zelahi diff --git a/Dockerfile b/Dockerfile index a66f8e2d..5f011509 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ # syntax=docker/dockerfile:1.3 ARG GO_VERSION=1.17.0 -ARG DOCKERD_VERSION=19.03 -ARG CLI_VERSION=19.03 +ARG DOCKERD_VERSION=20.10.8 FROM docker:$DOCKERD_VERSION AS dockerd-release @@ -26,24 +25,22 @@ RUN --mount=target=. \ FROM gobase AS buildx-build ENV CGO_ENABLED=0 ARG TARGETPLATFORM -RUN --mount=target=. --mount=target=/root/.cache,type=cache \ - --mount=target=/go/pkg/mod,type=cache \ - --mount=source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildx-version \ +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=bind,source=/tmp/.ldflags,target=/tmp/.ldflags,from=buildx-version \ set -x; xx-go build -ldflags "$(cat /tmp/.ldflags)" -o /usr/bin/buildx ./cmd/buildx && \ xx-verify --static /usr/bin/buildx -FROM buildx-build AS integration-tests -COPY . . +FROM buildx-build AS test +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/go/pkg/mod \ + go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic ./... && \ + go tool cover -func=/tmp/coverage.txt -# FROM golang:1.12-alpine AS docker-cli-build -# RUN apk add -U git bash coreutils gcc musl-dev -# ENV CGO_ENABLED=0 -# ARG REPO=github.com/tiborvass/cli -# ARG BRANCH=cli-plugin-aliases -# ARG CLI_VERSION -# WORKDIR /go/src/github.com/docker/cli -# RUN git clone git://$REPO . && git checkout $BRANCH -# RUN ./scripts/build/binary +FROM scratch AS test-coverage +COPY --from=test /tmp/coverage.txt /coverage.txt FROM scratch AS binaries-unix COPY --from=buildx-build /usr/bin/buildx / @@ -56,28 +53,29 @@ COPY --from=buildx-build /usr/bin/buildx /buildx.exe FROM binaries-$TARGETOS AS binaries +# Release FROM --platform=$BUILDPLATFORM alpine AS releaser WORKDIR /work ARG TARGETPLATFORM RUN --mount=from=binaries \ - --mount=source=/tmp/.version,target=/tmp/.version,from=buildx-version \ + --mount=type=bind,source=/tmp/.version,target=/tmp/.version,from=buildx-version \ mkdir -p /out && cp buildx* "/out/buildx-$(cat /tmp/.version).$(echo $TARGETPLATFORM | sed 's/\//-/g')$(ls buildx* | sed -e 's/^buildx//')" FROM scratch AS release COPY --from=releaser /out/ / -FROM alpine AS demo-env +# Shell +FROM docker:$DOCKERD_VERSION AS dockerd-release +FROM alpine AS shell RUN apk add --no-cache iptables tmux git vim less openssh RUN mkdir -p /usr/local/lib/docker/cli-plugins && ln -s /usr/local/bin/buildx /usr/local/lib/docker/cli-plugins/docker-buildx COPY ./hack/demo-env/entrypoint.sh /usr/local/bin COPY ./hack/demo-env/tmux.conf /root/.tmux.conf COPY --from=dockerd-release /usr/local/bin /usr/local/bin -#COPY --from=docker-cli-build /go/src/github.com/docker/cli/build/docker /usr/local/bin - WORKDIR /work COPY ./hack/demo-env/examples . COPY --from=binaries / /usr/local/bin/ VOLUME /var/lib/docker ENTRYPOINT ["entrypoint.sh"] -FROM binaries \ No newline at end of file +FROM binaries diff --git a/Makefile b/Makefile index 4d03a8de..c8e0de01 100644 --- a/Makefile +++ b/Makefile @@ -1,32 +1,48 @@ +ifneq (, $(BUILDX_BIN)) + export BUILDX_CMD = $(BUILDX_BIN) +else ifneq (, $(shell docker buildx version)) + export BUILDX_CMD = docker buildx +else ifneq (, $(shell which buildx)) + export BUILDX_CMD = $(which buildx) +else + $(error "Buildx is required: https://github.com/docker/buildx#installing") +endif + +export BIN_OUT = ./bin +export RELEASE_OUT = ./release-out + shell: ./hack/shell binaries: - ./hack/binaries + $(BUILDX_CMD) bake binaries binaries-cross: - EXPORT_LOCAL=cross-out ./hack/cross - -cross: - ./hack/cross + $(BUILDX_CMD) bake binaries-cross install: binaries mkdir -p ~/.docker/cli-plugins install bin/buildx ~/.docker/cli-plugins/docker-buildx +release: + ./hack/release + +validate-all: lint test validate-vendor validate-docs + lint: - ./hack/lint + $(BUILDX_CMD) bake lint test: - ./hack/test + $(BUILDX_CMD) bake test validate-vendor: - ./hack/validate-vendor + $(BUILDX_CMD) bake validate-vendor validate-docs: - ./hack/validate-docs + $(BUILDX_CMD) bake validate-docs -validate-all: lint test validate-vendor validate-docs +validate-authors: + $(BUILDX_CMD) bake validate-authors vendor: ./hack/update-vendor @@ -34,7 +50,7 @@ vendor: docs: ./hack/update-docs -generate-authors: - ./hack/generate-authors +authors: + $(BUILDX_CMD) bake update-authors -.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs +.PHONY: shell binaries binaries-cross install release validate-all lint validate-vendor validate-docs validate-authors vendor docs authors diff --git a/README.md b/README.md index 296c0d98..71dde960 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,6 @@ Key features: - [From `Dockerfile`](#from-dockerfile) - [Set buildx as the default builder](#set-buildx-as-the-default-builder) - [Building](#building) - - [with Docker 18.09+](#with-docker-1809) - - [with buildx or Docker 19.03](#with-buildx-or-docker-1903) - [Getting started](#getting-started) - [Building with buildx](#building-with-buildx) - [Working with builder instances](#working-with-builder-instances) @@ -90,25 +88,25 @@ RUN docker buildx version # Set buildx as the default builder Running the command [`docker buildx install`](docs/reference/buildx_install.md) -sets up docker builder command as an alias to `docker buildx`. This results in -the ability to have `docker build` use the current buildx builder. +sets up docker builder command as an alias to `docker buildx build`. This +results in the ability to have `docker build` use the current buildx builder. To remove this alias, run [`docker buildx uninstall`](docs/reference/buildx_uninstall.md). # Building -## With buildx or Docker 19.03+ - ```console -$ export DOCKER_BUILDKIT=1 -$ docker build --platform=local -o . git://github.com/docker/buildx +# Buildx 0.6+ +$ docker buildx bake "git://github.com/docker/buildx" $ mkdir -p ~/.docker/cli-plugins -$ mv buildx ~/.docker/cli-plugins/docker-buildx -``` +$ mv ./bin/buildx ~/.docker/cli-plugins/docker-buildx -## With Docker 18.09+ +# Docker 19.03+ +$ DOCKER_BUILDKIT=1 docker build --platform=local -o . "git://github.com/docker/buildx" +$ mkdir -p ~/.docker/cli-plugins +$ mv buildx ~/.docker/cli-plugins/docker-buildx -```console +# Local $ git clone git://github.com/docker/buildx && cd buildx $ make install ``` diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 00000000..a8502678 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,128 @@ +variable "GO_VERSION" { + default = "1.17.0" +} +variable "BIN_OUT" { + default = "./bin" +} +variable "RELEASE_OUT" { + default = "./release-out" +} + +// Special target: https://github.com/docker/metadata-action#bake-definition +target "meta-helper" { + tags = ["docker/buildx-bin:local"] +} + +target "_common" { + args = { + GO_VERSION = GO_VERSION + BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1 + } +} + +group "default" { + targets = ["binaries"] +} + +group "validate" { + targets = ["lint", "validate-vendor", "validate-docs"] +} + +target "lint" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/lint.Dockerfile" + output = ["type=cacheonly"] +} + +target "validate-vendor" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/vendor.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "validate-docs" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/docs.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "validate-authors" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/authors.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "update-vendor" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/vendor.Dockerfile" + target = "update" + output = ["."] +} + +target "update-docs" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/docs.Dockerfile" + target = "update" + output = ["./docs/reference"] +} + +target "update-authors" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/authors.Dockerfile" + target = "update" + output = ["."] +} + +target "test" { + inherits = ["_common"] + target = "test-coverage" + output = ["./coverage"] +} + +target "binaries" { + inherits = ["_common"] + target = "binaries" + output = [BIN_OUT] + platforms = ["local"] +} + +target "binaries-cross" { + inherits = ["binaries"] + platforms = [ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm/v6", + "linux/arm/v7", + "linux/arm64", + "linux/ppc64le", + "linux/riscv64", + "linux/s390x", + "windows/amd64", + "windows/arm64" + ] +} + +target "release" { + inherits = ["binaries-cross"] + target = "release" + output = [RELEASE_OUT] +} + +target "image" { + inherits = ["meta-helper", "binaries"] + output = ["type=image"] +} + +target "image-cross" { + inherits = ["meta-helper", "binaries-cross"] + output = ["type=image"] +} + +target "image-local" { + inherits = ["image"] + output = ["type=docker"] +} diff --git a/hack/binaries b/hack/binaries deleted file mode 100755 index c6b673f5..00000000 --- a/hack/binaries +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -: ${TARGETPLATFORM=$CLI_PLATFORM} - -platformFlag="" -if [ -n "$TARGETPLATFORM" ]; then - platformFlag="--platform $TARGETPLATFORM" -fi - -buildxCmd build $platformFlag \ - --target "binaries" \ - --output "type=local,dest=./bin/" \ - . diff --git a/hack/build_ci_first_pass b/hack/build_ci_first_pass deleted file mode 100755 index 8a6f23b9..00000000 --- a/hack/build_ci_first_pass +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -TYP=$1 - -. $(dirname $0)/util -set -e - -usage() { - echo "usage: ./hack/build_ci_first_pass " - exit 1 -} - -if [ -z "$TYP" ]; then - usage -fi - -importCacheFlags="" -exportCacheFlags="" -if [ "$GITHUB_ACTIONS" = "true" ]; then - if [ -n "$cacheRefFrom" ]; then - importCacheFlags="--cache-from=type=local,src=$cacheRefFrom" - fi - if [ -n "$cacheRefTo" ]; then - exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo" - fi -fi - -case $TYP in - "binaries") - buildxCmd build $importCacheFlags $exportCacheFlags \ - --target "binaries" \ - $currentcontext - ;; - *) - echo >&2 "Unknown type $TYP" - exit 1 - ;; -esac diff --git a/hack/cross b/hack/cross deleted file mode 100755 index 7726b4be..00000000 --- a/hack/cross +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -: ${TARGETPLATFORM=linux/amd64,linux/arm/v7,linux/arm64,darwin/amd64,windows/amd64,linux/ppc64le,linux/s390x,linux/riscv64} -: ${EXPORT_LOCAL=} - -importCacheFlags="" -if [ "$GITHUB_ACTIONS" = "true" ]; then - if [ -n "$cacheRefFrom" ]; then - importCacheFlags="--cache-from=type=local,src=$cacheRefFrom" - fi -fi - -exportFlag="" -if [ -n "$EXPORT_LOCAL" ]; then - exportFlag="--output=type=local,dest=$EXPORT_LOCAL" -fi - -buildxCmd build $importCacheFlags $exportFlag \ - --target "binaries" \ - --platform "$TARGETPLATFORM" \ - $currentcontext diff --git a/hack/demo-env/entrypoint.sh b/hack/demo-env/entrypoint.sh index ca4346f0..da31fe7a 100755 --- a/hack/demo-env/entrypoint.sh +++ b/hack/demo-env/entrypoint.sh @@ -13,7 +13,3 @@ else ( $dockerdCmd &>/var/log/dockerd.log & ) exec ash fi - - - - diff --git a/hack/dockerfiles/authors.Dockerfile b/hack/dockerfiles/authors.Dockerfile new file mode 100644 index 00000000..68a1a90b --- /dev/null +++ b/hack/dockerfiles/authors.Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1.3-labs + +FROM alpine:3.14 AS gen +RUN apk add --no-cache git +WORKDIR /src +RUN --mount=type=bind,target=. < /out/AUTHORS +cat /out/AUTHORS +EOT + +FROM scratch AS update +COPY --from=gen /out / + +FROM gen AS validate +RUN --mount=type=bind,target=.,rw <&2 'ERROR: Authors result differs. Please update with "make authors"' + git status --porcelain -- AUTHORS + exit 1 +fi +EOT diff --git a/hack/dockerfiles/docs.Dockerfile b/hack/dockerfiles/docs.Dockerfile index 339d2c72..47a06e6f 100644 --- a/hack/dockerfiles/docs.Dockerfile +++ b/hack/dockerfiles/docs.Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.3 +# syntax=docker/dockerfile:1.3-labs ARG GO_VERSION=1.17.0 @@ -13,19 +13,28 @@ RUN apk add --no-cache rsync git WORKDIR /src COPY --from=docsgen /out/docsgen /usr/bin RUN --mount=target=/context \ - --mount=target=.,type=tmpfs,readwrite \ - rsync -a /context/. . && \ - docsgen && \ - mkdir /out && cp -r docs/reference /out + --mount=target=.,type=tmpfs <&2 'ERROR: Docs result differs. Please update with "make docs"' + git status --porcelain -- docs/reference + exit 1 +fi +EOT diff --git a/hack/dockerfiles/vendor.Dockerfile b/hack/dockerfiles/vendor.Dockerfile index 3d4c9b93..967f9d58 100644 --- a/hack/dockerfiles/vendor.Dockerfile +++ b/hack/dockerfiles/vendor.Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.3 +# syntax=docker/dockerfile:1.3-labs ARG GO_VERSION=1.17.0 @@ -6,20 +6,30 @@ FROM golang:${GO_VERSION}-alpine AS vendored RUN apk add --no-cache git rsync WORKDIR /src RUN --mount=target=/context \ - --mount=target=.,type=tmpfs,readwrite \ - --mount=target=/go/pkg/mod,type=cache \ - rsync -a /context/. . && \ - go mod tidy && go mod vendor && \ - mkdir /out && cp -r go.mod go.sum vendor /out + --mount=target=.,type=tmpfs \ + --mount=target=/go/pkg/mod,type=cache <&2 'ERROR: Vendor result differs. Please vendor your package with "make vendor"' + git status --porcelain -- go.mod go.sum vendor + exit 1 +fi +EOT diff --git a/hack/generate-authors b/hack/generate-authors deleted file mode 100755 index c154d263..00000000 --- a/hack/generate-authors +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -if [ -x "$(command -v greadlink)" ]; then - # on macOS, GNU readlink is ava (greadlink) can be installed through brew install coreutils - cd "$(dirname "$(greadlink -f "$BASH_SOURCE")")/.." -else - cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.." -fi - -# see also ".mailmap" for how email addresses and names are deduplicated - -{ - cat <<-'EOH' - # This file lists all individuals having contributed content to the repository. - # For how it is generated, see `scripts/generate-authors.sh`. - EOH - echo - git log --format='%aN <%aE>' | LC_ALL=C.UTF-8 sort -uf -} > AUTHORS diff --git a/hack/lint b/hack/lint deleted file mode 100755 index 56804aa4..00000000 --- a/hack/lint +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -buildxCmd build --file ./hack/dockerfiles/lint.Dockerfile . diff --git a/hack/release b/hack/release index 2d96aad3..4e9471c3 100755 --- a/hack/release +++ b/hack/release @@ -1,37 +1,20 @@ #!/usr/bin/env bash -OUT=${1:-release-out} - -. $(dirname $0)/util set -eu -o pipefail -: ${PLATFORMS=linux/amd64} -: ${CHECKSUMS=} - -importCacheFlags="" -if [[ -n "$cacheRefFrom" ]] && [[ "$cacheType" = "local" ]]; then - for ref in $cacheRefFrom; do - importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref " - done -fi +: ${BUILDX_CMD=docker buildx} +: ${RELEASE_OUT=./release-out} -buildxCmd build $importCacheFlags \ - --target "release" \ - --platform "$PLATFORMS" \ - --output "type=local,dest=$OUT" \ - $currentcontext +# release +(set -x ; ${BUILDX_CMD} bake --set "*.output=$RELEASE_OUT" release) # wrap binaries -{ set +x; } 2>/dev/null -if [[ $PLATFORMS =~ "," ]]; then - mv -f ./$OUT/**/* ./$OUT/ - find ./$OUT -type d -empty -delete -fi +mv -f ./${RELEASE_OUT}/**/* ./${RELEASE_OUT}/ +find ./${RELEASE_OUT} -type d -empty -delete -if [ -n "$CHECKSUMS" ]; then - if ! type shasum > /dev/null 2>&1; then - echo >&2 "ERROR: shasum is required" - exit 1 - fi - find ./$OUT/ -type f \( -iname "buildx-*" ! -iname "*darwin*" \) -print0 | sort -z | xargs -r0 shasum -a 256 -b | sed 's# .*/# #' > ./$OUT/checksums.txt +# checksums +if ! type shasum > /dev/null 2>&1; then + echo >&2 "ERROR: shasum is required" + exit 1 fi +find ./${RELEASE_OUT}/ -type f \( -iname "buildx-*" ! -iname "*darwin*" \) -print0 | sort -z | xargs -r0 shasum -a 256 -b | sed 's# .*/# #' > ./${RELEASE_OUT}/checksums.txt diff --git a/hack/shell b/hack/shell index 4f31165e..cd2901a5 100755 --- a/hack/shell +++ b/hack/shell @@ -2,17 +2,18 @@ set -e +: ${BUILDX_CMD=docker buildx} : ${TMUX=} function clean { - docker rmi $(cat $iidfile) + docker rmi $iid } -iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) -DOCKER_BUILDKIT=1 docker build --iidfile $iidfile --target demo-env . +iid=buildx-shell +(set -x ; ${BUILDX_CMD} build --output "type=docker,name=$iid" --target shell .) trap clean EXIT SSH= if [ -n "$MOUNT_SSH_AUTH_SOCK" ]; then - SSH="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" + SSH="-v $SSH_AUTH_SOCK:$SSH_AUTH_SOCK -e SSH_AUTH_SOCK" fi -docker run $SSH -it --privileged --rm -e TMUX_ENTRYPOINT=$TMUX $(cat $iidfile) +docker run $SSH -it --privileged --rm -e TMUX_ENTRYPOINT=$TMUX $iid diff --git a/hack/test b/hack/test deleted file mode 100755 index e1d6fc5c..00000000 --- a/hack/test +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu -o pipefail - -: ${BUILDX_NOCACHE=} -: ${TEST_COVERAGE=} - -importCacheFlags="" -if [ -n "$cacheRefFrom" ]; then - if [ "$cacheType" = "local" ]; then - for ref in $cacheRefFrom; do - importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref " - done - fi -fi - -iid="buildx-tests" -iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) - -coverageVol="" -coverageFlags="" -if [ "$TEST_COVERAGE" = "1" ]; then - covdir="$(pwd)/coverage" - mkdir -p "$covdir" - coverageVol="-v $covdir:/coverage" - coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic" -fi - -buildxCmd build $importCacheFlags \ - --target "integration-tests" \ - --output "type=docker,name=$iid" \ - $currentcontext - -cacheVolume="buildx-cache" -if ! docker inspect "$cacheVolume" > /dev/null 2>&1; then - cacheVolume=$(docker create --name=buildx-cache -v /root/.cache -v /go/pkg/mod alpine) -fi - -docker run --rm -v /tmp $coverageVol --volumes-from=$cacheVolume --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...} - -if [ -n "$BUILDX_NOCACHE" ]; then - docker rm -v $cacheVolume -fi - -rm "$iidfile" -docker rmi $iid diff --git a/hack/update-docs b/hack/update-docs index 14d6ef7e..5b9c1bef 100755 --- a/hack/update-docs +++ b/hack/update-docs @@ -1,16 +1,11 @@ #!/usr/bin/env bash -. $(dirname $0)/util -set -eu +set -eu -o pipefail -output=$(mktemp -d -t buildx-output.XXXXXXXXXX) - -buildxCmd build \ - --target "update" \ - --output "type=local,dest=$output" \ - --file "./hack/dockerfiles/docs.Dockerfile" \ - . +: ${BUILDX_CMD=docker buildx} +output=$(mktemp -d -t buildx-output.XXXXXXXXXX) +(set -x ; ${BUILDX_CMD} bake --set "*.output=$output" update-docs) rm -rf ./docs/reference/* cp -R "$output"/out/* ./docs/ rm -rf $output diff --git a/hack/update-vendor b/hack/update-vendor index 18dd3237..983a7f0c 100755 --- a/hack/update-vendor +++ b/hack/update-vendor @@ -1,16 +1,11 @@ #!/usr/bin/env bash -. $(dirname $0)/util -set -eu +set -eu -o pipefail -output=$(mktemp -d -t buildx-output.XXXXXXXXXX) - -buildxCmd build \ - --target "update" \ - --output "type=local,dest=$output" \ - --file "./hack/dockerfiles/vendor.Dockerfile" \ - . +: ${BUILDX_CMD=docker buildx} +output=$(mktemp -d -t buildx-output.XXXXXXXXXX) +(set -x ; ${BUILDX_CMD} bake --set "*.output=$output" update-vendor) rm -rf ./vendor cp -R "$output"/out/* . rm -rf $output diff --git a/hack/util b/hack/util deleted file mode 100755 index cc634197..00000000 --- a/hack/util +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env sh - -: ${CI=} -: ${PREFER_BUILDCTL=} -: ${PREFER_LEGACY=} -: ${CLI_PLATFORM=} -: ${GITHUB_ACTIONS=} -: ${CACHEDIR_FROM=} -: ${CACHEDIR_TO=} - -if [ "$PREFER_BUILDCTL" = "1" ]; then - echo >&2 "WARNING: PREFER_BUILDCTL is no longer supported. Ignoring." -fi - -if [ "$PREFER_LEGACY" = "1" ]; then - echo >&2 "WARNING: PREFER_LEGACY is no longer supported. Ignoring." -fi - -progressFlag="" -if [ "$CI" = "true" ]; then - progressFlag="--progress=plain" -fi - -buildxCmd() { - if docker buildx version >/dev/null 2>&1; then - set -x - docker buildx "$@" $progressFlag - elif buildx version >/dev/null 2>&1; then - set -x - buildx "$@" $progressFlag - elif docker version >/dev/null 2>&1; then - set -x - DOCKER_BUILDKIT=1 docker "$@" $progressFlag - else - echo >&2 "ERROR: Please enable DOCKER_BUILDKIT or install standalone buildx" - exit 1 - fi -} - -if [ -z "$CLI_PLATFORM" ]; then - if [ "$(uname -s)" = "Darwin" ]; then - arch="$(uname -m)" - if [ "$arch" = "x86_64" ]; then - arch="amd64" - fi - CLI_PLATFORM="darwin/$arch" - elif uname -s | grep MINGW > /dev/null 2>&1 ; then - CLI_PLATFORM="windows/amd64" - fi -fi - -cacheType="" -cacheRefFrom="" -cacheRefTo="" -currentref="" -if [ "$GITHUB_ACTIONS" = "true" ]; then - currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF" - cacheType="local" - cacheRefFrom="$CACHEDIR_FROM" - cacheRefTo="$CACHEDIR_TO" -fi - -currentcontext="." -if [ -n "$currentref" ]; then - currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref" -fi diff --git a/hack/validate-docs b/hack/validate-docs deleted file mode 100755 index 4aa0cf26..00000000 --- a/hack/validate-docs +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case ${1:-} in - '') - . $(dirname $0)/util - buildxCmd build \ - --target validate \ - --file ./hack/dockerfiles/docs.Dockerfile \ - . - ;; - check) - status="$(git status --porcelain -- docs/reference 2>/dev/null)" - diffs=$(echo "$status" | grep -v '^[RAD] ' || true) - if [ "$diffs" ]; then - { - set +x - echo 'The result of ./hack/update-docs differs' - echo - echo "$diffs" - echo - echo 'Please vendor your package with ./hack/update-docs' - echo - } >&2 - exit 1 - fi - echo 'Congratulations! All docs changes are done the right way.' - ;; -esac diff --git a/hack/validate-vendor b/hack/validate-vendor deleted file mode 100755 index d60a6fe9..00000000 --- a/hack/validate-vendor +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case ${1:-} in - '') - . $(dirname $0)/util - buildxCmd build \ - --target validate \ - --file ./hack/dockerfiles/vendor.Dockerfile \ - . - ;; - check) - status="$(git status --porcelain -- go.mod go.sum vendor 2>/dev/null)" - diffs=$(echo "$status" | grep -v '^[RAD] ' || true) - if [ "$diffs" ]; then - { - set +x - echo 'The result of "make vendor" differs' - echo - echo "$diffs" - echo - echo 'Please vendor your package with "make vendor"' - echo - } >&2 - exit 1 - fi - echo 'Congratulations! All vendoring changes are done the right way.' - ;; -esac