diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 24461870..9a3fdfab 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -116,6 +116,43 @@ commit automatically with `git commit -s`. ### Run the unit- and integration-tests +Running tests: + +```bash +make test +``` + +This runs all unit and integration tests, in a containerized environment. +Locally, every package can be tested separately with standard Go tools, but +integration tests are skipped if local user doesn't have enough permissions or +worker binaries are not installed. + +```bash +# run unit tests only +make test-unit + +# run integration tests only +make test-integration + +# test a specific package +TESTPKGS=./bake make test + +# run all integration tests with a specific worker +TESTFLAGS="--run=//worker=docker-container -v" make test +``` + +> **Note** +> +> Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant +> images in a docker volume if you are repeatedly calling `make test`. This +> helps to avoid rate limiting on the remote registry side. + +> **Note** +> +> If you are working behind a proxy, you can set some of or all +> `HTTP_PROXY=http://ip:port`, `HTTPS_PROXY=http://ip:port`, `NO_PROXY=http://ip:port` +> for the test framework to specify the proxy build args. + To enter a demo container environment and experiment, you may run: ``` diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4edce420..d23af019 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,14 +23,63 @@ env: BUILDKIT_IMAGE: "moby/buildkit:latest" REPO_SLUG: "docker/buildx-bin" DESTDIR: "./bin" + TEST_CACHE_SCOPE: "test" jobs: + prepare-test: + runs-on: ubuntu-22.04 + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.BUILDKIT_IMAGE }} + buildkitd-flags: --debug + - + name: Build + uses: docker/bake-action@v3 + with: + targets: integration-test-base + set: | + *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} + *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} + test: runs-on: ubuntu-22.04 + needs: + - prepare-test + env: + TESTFLAGS: "-v --parallel=6 --timeout=30m" + TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m" + GOTESTSUM_FORMAT: "standard-verbose" + TEST_IMAGE_BUILD: "0" + TEST_IMAGE_ID: "buildx-tests" + strategy: + fail-fast: false + matrix: + worker: + - docker + - docker-container + - remote + pkg: + - ./tests + include: + - pkg: ./... + skip-integration-tests: 1 steps: - name: Checkout uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -39,20 +88,44 @@ jobs: driver-opts: image=${{ env.BUILDKIT_IMAGE }} buildkitd-flags: --debug - - name: Test + name: Build test image uses: docker/bake-action@v3 with: - targets: test + targets: integration-test set: | - *.cache-from=type=gha,scope=test - *.cache-to=type=gha,scope=test + *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} + *.output=type=docker,name=${{ env.TEST_IMAGE_ID }} - - name: Upload coverage + name: Test + run: | + export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') + ./hack/test + env: + TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}" + TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" + TESTPKGS: "${{ matrix.pkg }}" + SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" + - + name: Send to Codecov + if: always() uses: codecov/codecov-action@v3 with: - directory: ${{ env.DESTDIR }}/coverage + directory: ./bin/testreports + - + name: Generate annotations + if: always() + uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9 + with: + directory: ./bin/testreports + - + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: ./bin/testreports - prepare: + prepare-binaries: runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.platforms.outputs.matrix }} @@ -73,11 +146,11 @@ jobs: binaries: runs-on: ubuntu-22.04 needs: - - prepare + - prepare-binaries strategy: fail-fast: false matrix: - platform: ${{ fromJson(needs.prepare.outputs.matrix) }} + platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }} steps: - name: Prepare @@ -115,6 +188,8 @@ jobs: bin-image: runs-on: ubuntu-22.04 + needs: + - test if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }} steps: - @@ -166,6 +241,7 @@ jobs: release: runs-on: ubuntu-22.04 needs: + - test - binaries steps: - diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index dad6f7d1..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: test - -on: - workflow_dispatch: - push: - branches: - - 'master' - - 'v[0-9]*' - tags: - - 'v*' - pull_request: - paths-ignore: - - '.github/releases.json' - - 'README.md' - - 'docs/**' - -env: - SETUP_BUILDX_VERSION: "latest" - SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - TESTFLAGS: "-v --parallel=6 --timeout=30m" - TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m" - GOTESTSUM_FORMAT: "standard-verbose" - -jobs: - test: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - worker: - - docker - - docker-container - - remote - pkg: - - ./tests - typ: - - integration - # already run as part of build.yml - # include: - # - pkg: ./... - # skip-integration-tests: 1 - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Test - run: | - export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') - ./hack/test ${{ matrix.typ }} - env: - TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}" - TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" - TESTPKGS: "${{ matrix.pkg }}" - SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - - - name: Generate annotations - if: always() - uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9 - with: - directory: ./bin/testreports - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v3 - with: - name: test-reports - path: ./bin/testreports diff --git a/Makefile b/Makefile index 29fca1a3..916421e8 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,15 @@ lint: .PHONY: test test: - $(BUILDX_CMD) bake test + ./hack/test + +.PHONY: test-unit +test-unit: + TESTPKGS=./... SKIP_INTEGRATION_TESTS=1 ./hack/test + +.PHONY: test +test-integration: + TESTPKGS=./tests ./hack/test .PHONY: validate-vendor validate-vendor: diff --git a/hack/test b/hack/test index 9dbb8104..b1cea083 100755 --- a/hack/test +++ b/hack/test @@ -3,41 +3,18 @@ set -eu -o pipefail : "${BUILDX_CMD=docker buildx}" -: "${CACHE_FROM=}" -: "${CACHE_TO=}" - -: ${TEST_INTEGRATION=} -: ${TEST_REPORT_SUFFIX=} -: ${TEST_KEEP_CACHE=} -: ${TEST_DOCKERD=} -: ${TEST_BUILDKIT_IMAGE=} - -if [ -n "$CACHE_FROM" ]; then - for cfrom in $CACHE_FROM; do - setFlags+=(--set "*.cache-from=$cfrom") - done -fi -if [ -n "$CACHE_TO" ]; then - for cto in $CACHE_TO; do - setFlags+=(--set "*.cache-to=$cto") - done -fi -if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi +: "${TEST_IMAGE_BUILD=1}" +: "${TEST_IMAGE_ID=buildx-tests}" -while test $# -gt 0; do - case "$1" in - integration) - TEST_INTEGRATION=1 - ;; - *) - echo "unknown arg $1" - ;; - esac - shift -done +: "${TEST_REPORT_SUFFIX=}" +: "${TEST_KEEP_CACHE=}" +: "${TEST_DOCKERD=}" +: "${TEST_BUILDKIT_IMAGE=}" -iid="buildx-tests" +if [ "$TEST_IMAGE_BUILD" = "1" ]; then + ${BUILDX_CMD} bake integration-test --set "*.output=type=docker,name=$TEST_IMAGE_ID" +fi testReportsDir="$(pwd)/bin/testreports" mkdir -p "$testReportsDir" @@ -45,18 +22,13 @@ testReportsVol="-v $testReportsDir:/testreports" gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml" gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic" -${BUILDX_CMD} bake integration-test "${setFlags[@]}" --set "*.output=type=docker,name=$iid" - cacheVolume="buildx-test-cache" if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine fi - -if [ "$TEST_INTEGRATION" == 1 ]; then - cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) - docker start -a -i $cid -fi - if [ "$TEST_KEEP_CACHE" != "1" ]; then - docker rm -v $cacheVolume + trap 'docker rm -v $cacheVolume' EXIT fi + +cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) +docker start -a -i $cid