From e762e46b4bd930d8ad437d19c2367f8a8ebb1394 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:11:32 +0200 Subject: [PATCH 1/9] ci: merge test workflow to build one Signed-off-by: CrazyMax --- .github/workflows/build.yml | 56 ++++++++++++++++++++----- .github/workflows/test.yml | 81 ------------------------------------- 2 files changed, 45 insertions(+), 92 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4edce420..aa1769ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,10 +27,34 @@ env: jobs: test: runs-on: ubuntu-22.04 + env: + TESTFLAGS: "-v --parallel=6 --timeout=30m" + TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m" + GOTESTSUM_FORMAT: "standard-verbose" + strategy: + fail-fast: false + matrix: + worker: + - docker + - docker-container + - remote + pkg: + - ./tests + typ: + - integration + include: + - pkg: ./... + skip-integration-tests: 1 steps: - name: Checkout uses: actions/checkout@v3 + - + name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -40,19 +64,29 @@ jobs: buildkitd-flags: --debug - name: Test - uses: docker/bake-action@v3 + run: | + export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') + ./hack/test ${{ matrix.typ }} + env: + TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}" + TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" + TESTPKGS: "${{ matrix.pkg }}" + SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" + - + name: Generate annotations + if: always() + uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9 with: - targets: test - set: | - *.cache-from=type=gha,scope=test - *.cache-to=type=gha,scope=test + directory: ./bin/testreports - - name: Upload coverage - uses: codecov/codecov-action@v3 + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 with: - directory: ${{ env.DESTDIR }}/coverage + name: test-reports + path: ./bin/testreports - prepare: + prepare-binaries: runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.platforms.outputs.matrix }} @@ -73,11 +107,11 @@ jobs: binaries: runs-on: ubuntu-22.04 needs: - - prepare + - prepare-binaries strategy: fail-fast: false matrix: - platform: ${{ fromJson(needs.prepare.outputs.matrix) }} + platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }} steps: - name: Prepare diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index dad6f7d1..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: test - -on: - workflow_dispatch: - push: - branches: - - 'master' - - 'v[0-9]*' - tags: - - 'v*' - pull_request: - paths-ignore: - - '.github/releases.json' - - 'README.md' - - 'docs/**' - -env: - SETUP_BUILDX_VERSION: "latest" - SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - TESTFLAGS: "-v --parallel=6 --timeout=30m" - TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m" - GOTESTSUM_FORMAT: "standard-verbose" - -jobs: - test: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - worker: - - docker - - docker-container - - remote - pkg: - - ./tests - typ: - - integration - # already run as part of build.yml - # include: - # - pkg: ./... - # skip-integration-tests: 1 - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Test - run: | - export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') - ./hack/test ${{ matrix.typ }} - env: - TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}" - TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" - TESTPKGS: "${{ matrix.pkg }}" - SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - - - name: Generate annotations - if: always() - uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9 - with: - directory: ./bin/testreports - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-reports - path: ./bin/testreports From 73c17ef4d25bf2e4cd072ccb965d928fdc60ae9e Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:12:22 +0200 Subject: [PATCH 2/9] ci: send test reports to codecov Signed-off-by: CrazyMax --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aa1769ac..200b6a87 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,6 +72,12 @@ jobs: TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" TESTPKGS: "${{ matrix.pkg }}" SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" + - + name: Send to Codecov + if: always() + uses: codecov/codecov-action@v3 + with: + directory: ./bin/testreports - name: Generate annotations if: always() From ecf8dd0a26459855f563238c718e3d39ef9f50ee Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:24:01 +0200 Subject: [PATCH 3/9] test: type integration not needed Signed-off-by: CrazyMax --- .github/workflows/build.yml | 4 +--- hack/test | 29 ++++++----------------------- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 200b6a87..11be1800 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,8 +40,6 @@ jobs: - remote pkg: - ./tests - typ: - - integration include: - pkg: ./... skip-integration-tests: 1 @@ -66,7 +64,7 @@ jobs: name: Test run: | export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') - ./hack/test ${{ matrix.typ }} + ./hack/test env: TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}" TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" diff --git a/hack/test b/hack/test index 9dbb8104..82fabf1a 100755 --- a/hack/test +++ b/hack/test @@ -6,11 +6,10 @@ set -eu -o pipefail : "${CACHE_FROM=}" : "${CACHE_TO=}" -: ${TEST_INTEGRATION=} -: ${TEST_REPORT_SUFFIX=} -: ${TEST_KEEP_CACHE=} -: ${TEST_DOCKERD=} -: ${TEST_BUILDKIT_IMAGE=} +: "${TEST_REPORT_SUFFIX=}" +: "${TEST_KEEP_CACHE=}" +: "${TEST_DOCKERD=}" +: "${TEST_BUILDKIT_IMAGE=}" if [ -n "$CACHE_FROM" ]; then for cfrom in $CACHE_FROM; do @@ -23,20 +22,6 @@ if [ -n "$CACHE_TO" ]; then done fi -if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi - -while test $# -gt 0; do - case "$1" in - integration) - TEST_INTEGRATION=1 - ;; - *) - echo "unknown arg $1" - ;; - esac - shift -done - iid="buildx-tests" testReportsDir="$(pwd)/bin/testreports" @@ -52,10 +37,8 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine fi -if [ "$TEST_INTEGRATION" == 1 ]; then - cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) - docker start -a -i $cid -fi +cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) +docker start -a -i $cid if [ "$TEST_KEEP_CACHE" != "1" ]; then docker rm -v $cacheVolume From b9cf46785b1120205be84ddc30b7901c47c8ce6c Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:30:25 +0200 Subject: [PATCH 4/9] test: use trap to remove cache volume Signed-off-by: CrazyMax --- hack/test | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/hack/test b/hack/test index 82fabf1a..5f8820aa 100755 --- a/hack/test +++ b/hack/test @@ -36,10 +36,9 @@ cacheVolume="buildx-test-cache" if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine fi +if [ "$TEST_KEEP_CACHE" != "1" ]; then + trap 'docker rm -v $cacheVolume' EXIT +fi cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) docker start -a -i $cid - -if [ "$TEST_KEEP_CACHE" != "1" ]; then - docker rm -v $cacheVolume -fi From 74b121be66aa82d2cb7caad7277a4a46667559c1 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:44:21 +0200 Subject: [PATCH 5/9] test: opt to avoid building the integration tests image Signed-off-by: CrazyMax --- hack/test | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/hack/test b/hack/test index 5f8820aa..b1cea083 100755 --- a/hack/test +++ b/hack/test @@ -3,35 +3,25 @@ set -eu -o pipefail : "${BUILDX_CMD=docker buildx}" -: "${CACHE_FROM=}" -: "${CACHE_TO=}" + +: "${TEST_IMAGE_BUILD=1}" +: "${TEST_IMAGE_ID=buildx-tests}" : "${TEST_REPORT_SUFFIX=}" : "${TEST_KEEP_CACHE=}" : "${TEST_DOCKERD=}" : "${TEST_BUILDKIT_IMAGE=}" -if [ -n "$CACHE_FROM" ]; then - for cfrom in $CACHE_FROM; do - setFlags+=(--set "*.cache-from=$cfrom") - done -fi -if [ -n "$CACHE_TO" ]; then - for cto in $CACHE_TO; do - setFlags+=(--set "*.cache-to=$cto") - done +if [ "$TEST_IMAGE_BUILD" = "1" ]; then + ${BUILDX_CMD} bake integration-test --set "*.output=type=docker,name=$TEST_IMAGE_ID" fi -iid="buildx-tests" - testReportsDir="$(pwd)/bin/testreports" mkdir -p "$testReportsDir" testReportsVol="-v $testReportsDir:/testreports" gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml" gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic" -${BUILDX_CMD} bake integration-test "${setFlags[@]}" --set "*.output=type=docker,name=$iid" - cacheVolume="buildx-test-cache" if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine @@ -40,5 +30,5 @@ if [ "$TEST_KEEP_CACHE" != "1" ]; then trap 'docker rm -v $cacheVolume' EXIT fi -cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) +cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) docker start -a -i $cid From 98dedd32259b455b9fcfa57b2da699940e6c5439 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:45:42 +0200 Subject: [PATCH 6/9] ci: cache integration tests base and build in dedicated step Signed-off-by: CrazyMax --- .github/workflows/build.yml | 41 ++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11be1800..6cbe48b5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,14 +23,44 @@ env: BUILDKIT_IMAGE: "moby/buildkit:latest" REPO_SLUG: "docker/buildx-bin" DESTDIR: "./bin" + TEST_CACHE_SCOPE: "test" jobs: + prepare-test: + runs-on: ubuntu-22.04 + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.BUILDKIT_IMAGE }} + buildkitd-flags: --debug + - + name: Build + uses: docker/bake-action@v3 + with: + targets: integration-test-base + set: | + *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} + *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} + test: runs-on: ubuntu-22.04 + needs: + - prepare-test env: TESTFLAGS: "-v --parallel=6 --timeout=30m" TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m" GOTESTSUM_FORMAT: "standard-verbose" + TEST_IMAGE_BUILD: "0" + TEST_IMAGE_ID: "buildx-tests" strategy: fail-fast: false matrix: @@ -47,9 +77,6 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -60,6 +87,14 @@ jobs: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.BUILDKIT_IMAGE }} buildkitd-flags: --debug + - + name: Build test image + uses: docker/bake-action@v3 + with: + targets: integration-test + set: | + *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} + *.output=type=docker,name=${{ env.TEST_IMAGE_ID }} - name: Test run: | From 55377b2b0f9f36878002e42ff6365b0f7a1a284e Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 00:57:16 +0200 Subject: [PATCH 7/9] make: test-unit and test-integration targets Also switch test target to run the test script. Signed-off-by: CrazyMax --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 29fca1a3..916421e8 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,15 @@ lint: .PHONY: test test: - $(BUILDX_CMD) bake test + ./hack/test + +.PHONY: test-unit +test-unit: + TESTPKGS=./... SKIP_INTEGRATION_TESTS=1 ./hack/test + +.PHONY: test +test-integration: + TESTPKGS=./tests ./hack/test .PHONY: validate-vendor validate-vendor: From a4db138c5ef4bb399652a9340c7cd7d3a2e5421c Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 01:06:06 +0200 Subject: [PATCH 8/9] docs: update CONTRIBUTING.md to run tests Signed-off-by: CrazyMax --- .github/CONTRIBUTING.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 24461870..9a3fdfab 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -116,6 +116,43 @@ commit automatically with `git commit -s`. ### Run the unit- and integration-tests +Running tests: + +```bash +make test +``` + +This runs all unit and integration tests, in a containerized environment. +Locally, every package can be tested separately with standard Go tools, but +integration tests are skipped if local user doesn't have enough permissions or +worker binaries are not installed. + +```bash +# run unit tests only +make test-unit + +# run integration tests only +make test-integration + +# test a specific package +TESTPKGS=./bake make test + +# run all integration tests with a specific worker +TESTFLAGS="--run=//worker=docker-container -v" make test +``` + +> **Note** +> +> Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant +> images in a docker volume if you are repeatedly calling `make test`. This +> helps to avoid rate limiting on the remote registry side. + +> **Note** +> +> If you are working behind a proxy, you can set some of or all +> `HTTP_PROXY=http://ip:port`, `HTTPS_PROXY=http://ip:port`, `NO_PROXY=http://ip:port` +> for the test framework to specify the proxy build args. + To enter a demo container environment and experiment, you may run: ``` From 38b4eef5c6b54999b053aa44456aa27404c5a217 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 30 May 2023 01:54:17 +0200 Subject: [PATCH 9/9] ci: make release and bin-image depend on test Signed-off-by: CrazyMax --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cbe48b5..d23af019 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -188,6 +188,8 @@ jobs: bin-image: runs-on: ubuntu-22.04 + needs: + - test if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }} steps: - @@ -239,6 +241,7 @@ jobs: release: runs-on: ubuntu-22.04 needs: + - test - binaries steps: -