Merge pull request #1851 from crazy-max/test-flow

test: update workflow
pull/1856/head
CrazyMax 2 years ago committed by GitHub
commit f1788002e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -116,6 +116,43 @@ commit automatically with `git commit -s`.
### Run the unit- and integration-tests
Running tests:
```bash
make test
```
This runs all unit and integration tests, in a containerized environment.
Locally, every package can be tested separately with standard Go tools, but
integration tests are skipped if local user doesn't have enough permissions or
worker binaries are not installed.
```bash
# run unit tests only
make test-unit
# run integration tests only
make test-integration
# test a specific package
TESTPKGS=./bake make test
# run all integration tests with a specific worker
TESTFLAGS="--run=//worker=docker-container -v" make test
```
> **Note**
>
> Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant
> images in a docker volume if you are repeatedly calling `make test`. This
> helps to avoid rate limiting on the remote registry side.
> **Note**
>
> If you are working behind a proxy, you can set some of or all
> `HTTP_PROXY=http://ip:port`, `HTTPS_PROXY=http://ip:port`, `NO_PROXY=http://ip:port`
> for the test framework to specify the proxy build args.
To enter a demo container environment and experiment, you may run:
```

@ -23,14 +23,63 @@ env:
BUILDKIT_IMAGE: "moby/buildkit:latest"
REPO_SLUG: "docker/buildx-bin"
DESTDIR: "./bin"
TEST_CACHE_SCOPE: "test"
jobs:
prepare-test:
runs-on: ubuntu-22.04
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
version: ${{ env.BUILDX_VERSION }}
driver-opts: image=${{ env.BUILDKIT_IMAGE }}
buildkitd-flags: --debug
-
name: Build
uses: docker/bake-action@v3
with:
targets: integration-test-base
set: |
*.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
*.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
test:
runs-on: ubuntu-22.04
needs:
- prepare-test
env:
TESTFLAGS: "-v --parallel=6 --timeout=30m"
TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
GOTESTSUM_FORMAT: "standard-verbose"
TEST_IMAGE_BUILD: "0"
TEST_IMAGE_ID: "buildx-tests"
strategy:
fail-fast: false
matrix:
worker:
- docker
- docker-container
- remote
pkg:
- ./tests
include:
- pkg: ./...
skip-integration-tests: 1
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
@ -39,20 +88,44 @@ jobs:
driver-opts: image=${{ env.BUILDKIT_IMAGE }}
buildkitd-flags: --debug
-
name: Test
name: Build test image
uses: docker/bake-action@v3
with:
targets: test
targets: integration-test
set: |
*.cache-from=type=gha,scope=test
*.cache-to=type=gha,scope=test
*.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }}
*.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
-
name: Upload coverage
name: Test
run: |
export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
./hack/test
env:
TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}"
TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
TESTPKGS: "${{ matrix.pkg }}"
SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
-
name: Send to Codecov
if: always()
uses: codecov/codecov-action@v3
with:
directory: ${{ env.DESTDIR }}/coverage
directory: ./bin/testreports
-
name: Generate annotations
if: always()
uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9
with:
directory: ./bin/testreports
-
name: Upload test reports
if: always()
uses: actions/upload-artifact@v3
with:
name: test-reports
path: ./bin/testreports
prepare:
prepare-binaries:
runs-on: ubuntu-22.04
outputs:
matrix: ${{ steps.platforms.outputs.matrix }}
@ -73,11 +146,11 @@ jobs:
binaries:
runs-on: ubuntu-22.04
needs:
- prepare
- prepare-binaries
strategy:
fail-fast: false
matrix:
platform: ${{ fromJson(needs.prepare.outputs.matrix) }}
platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }}
steps:
-
name: Prepare
@ -115,6 +188,8 @@ jobs:
bin-image:
runs-on: ubuntu-22.04
needs:
- test
if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }}
steps:
-
@ -166,6 +241,7 @@ jobs:
release:
runs-on: ubuntu-22.04
needs:
- test
- binaries
steps:
-

@ -1,81 +0,0 @@
name: test
on:
workflow_dispatch:
push:
branches:
- 'master'
- 'v[0-9]*'
tags:
- 'v*'
pull_request:
paths-ignore:
- '.github/releases.json'
- 'README.md'
- 'docs/**'
env:
SETUP_BUILDX_VERSION: "latest"
SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest"
TESTFLAGS: "-v --parallel=6 --timeout=30m"
TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m"
GOTESTSUM_FORMAT: "standard-verbose"
jobs:
test:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
worker:
- docker
- docker-container
- remote
pkg:
- ./tests
typ:
- integration
# already run as part of build.yml
# include:
# - pkg: ./...
# skip-integration-tests: 1
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Expose GitHub Runtime
uses: crazy-max/ghaction-github-runtime@v2
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
version: ${{ env.SETUP_BUILDX_VERSION }}
driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
buildkitd-flags: --debug
-
name: Test
run: |
export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')
./hack/test ${{ matrix.typ }}
env:
TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}"
TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
TESTPKGS: "${{ matrix.pkg }}"
SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
-
name: Generate annotations
if: always()
uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9
with:
directory: ./bin/testreports
-
name: Upload test reports
if: always()
uses: actions/upload-artifact@v3
with:
name: test-reports
path: ./bin/testreports

@ -45,7 +45,15 @@ lint:
.PHONY: test
test:
$(BUILDX_CMD) bake test
./hack/test
.PHONY: test-unit
test-unit:
TESTPKGS=./... SKIP_INTEGRATION_TESTS=1 ./hack/test
.PHONY: test
test-integration:
TESTPKGS=./tests ./hack/test
.PHONY: validate-vendor
validate-vendor:

@ -3,41 +3,18 @@
set -eu -o pipefail
: "${BUILDX_CMD=docker buildx}"
: "${CACHE_FROM=}"
: "${CACHE_TO=}"
: ${TEST_INTEGRATION=}
: ${TEST_REPORT_SUFFIX=}
: ${TEST_KEEP_CACHE=}
: ${TEST_DOCKERD=}
: ${TEST_BUILDKIT_IMAGE=}
if [ -n "$CACHE_FROM" ]; then
for cfrom in $CACHE_FROM; do
setFlags+=(--set "*.cache-from=$cfrom")
done
fi
if [ -n "$CACHE_TO" ]; then
for cto in $CACHE_TO; do
setFlags+=(--set "*.cache-to=$cto")
done
fi
if [ "$#" == 0 ]; then TEST_INTEGRATION=1; fi
: "${TEST_IMAGE_BUILD=1}"
: "${TEST_IMAGE_ID=buildx-tests}"
while test $# -gt 0; do
case "$1" in
integration)
TEST_INTEGRATION=1
;;
*)
echo "unknown arg $1"
;;
esac
shift
done
: "${TEST_REPORT_SUFFIX=}"
: "${TEST_KEEP_CACHE=}"
: "${TEST_DOCKERD=}"
: "${TEST_BUILDKIT_IMAGE=}"
iid="buildx-tests"
if [ "$TEST_IMAGE_BUILD" = "1" ]; then
${BUILDX_CMD} bake integration-test --set "*.output=type=docker,name=$TEST_IMAGE_ID"
fi
testReportsDir="$(pwd)/bin/testreports"
mkdir -p "$testReportsDir"
@ -45,18 +22,13 @@ testReportsVol="-v $testReportsDir:/testreports"
gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml"
gotestArgs="-mod=vendor -coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic"
${BUILDX_CMD} bake integration-test "${setFlags[@]}" --set "*.output=type=docker,name=$iid"
cacheVolume="buildx-test-cache"
if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine
fi
if [ "$TEST_INTEGRATION" == 1 ]; then
cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
docker start -a -i $cid
fi
if [ "$TEST_KEEP_CACHE" != "1" ]; then
docker rm -v $cacheVolume
trap 'docker rm -v $cacheVolume' EXIT
fi
cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e TEST_DOCKERD -e TEST_BUILDKIT_IMAGE -e SKIP_INTEGRATION_TESTS -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $TEST_IMAGE_ID gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
docker start -a -i $cid

Loading…
Cancel
Save