Merge pull request #774 from crazy-max/driver-e2e

Driver e2e tests

.github/workflows/e2e.yml

@@ -0,0 +1,100 @@
+name: e2e
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+      - 'v[0-9]*'
+  pull_request:
+    branches:
+      - 'master'
+      - 'v[0-9]*'
+
+jobs:
+  driver:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        driver:
+          - docker
+          - docker-container
+          - kubernetes
+        buildkit:
+          - moby/buildkit:buildx-stable-1
+          - moby/buildkit:master
+        buildkit-cfg:
+          - bkcfg-false
+          - bkcfg-true
+        multi-node:
+          - mnode-false
+          - mnode-true
+        platforms:
+          - linux/amd64,linux/arm64
+        include:
+          - driver: kubernetes
+            driver-opt: qemu.install=true
+        exclude:
+          - driver: docker
+            multi-node: mnode-true
+          - driver: docker
+            buildkit-cfg: bkcfg-true
+          - driver: docker-container
+            multi-node: mnode-true
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        if: matrix.driver == 'docker' || matrix.driver == 'docker-container'
+      -
+        name: Install buildx
+        run: |
+          make install
+          docker buildx version
+      -
+        name: Init env vars
+        run: |
+          # BuildKit cfg
+          if [ "${{ matrix.buildkit-cfg }}" = "bkcfg-true" ]; then
+            cat > "/tmp/buildkitd.toml" <<EOL
+          [worker.oci]
+            max-parallelism = 2
+          EOL
+            echo "BUILDKIT_CFG=/tmp/buildkitd.toml" >> $GITHUB_ENV
+          fi
+          # Multi node
+          if [ "${{ matrix.multi-node }}" = "mnode-true" ]; then
+            echo "MULTI_NODE=1" >> $GITHUB_ENV
+          else
+            echo "MULTI_NODE=0" >> $GITHUB_ENV
+          fi
+      -
+        name: Install k3s
+        if: matrix.driver == 'kubernetes'
+        uses: debianmaster/actions-k3s@v1.0.3
+        id: k3s
+        with:
+          version: v1.21.2-k3s1
+      -
+        name: Config k3s
+        if: matrix.driver == 'kubernetes'
+        run: |
+          (set -x ; cat ${{ steps.k3s.outputs.kubeconfig }})
+      -
+        name: Check k3s nodes
+        if: matrix.driver == 'kubernetes'
+        run: |
+          kubectl get nodes
+      -
+        name: Test
+        run: |
+          make test-driver
+        env:
+          BUILDKIT_IMAGE: ${{ matrix.buildkit }}
+          DRIVER: ${{ matrix.driver }}
+          DRIVER_OPT: ${{ matrix.driver-opt }}
+          PLATFORMS: ${{ matrix.platforms }}

Makefile

@@ -44,6 +44,9 @@ validate-docs:
 validate-authors:
 	$(BUILDX_CMD) bake validate-authors
 
+test-driver:
+	./hack/test-driver
+
 vendor:
 	./hack/update-vendor
 

hack/test-driver

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+: ${BUILDX_CMD=docker buildx}
+: ${BUILDKIT_IMAGE=moby/buildkit:buildx-stable-1}
+: ${BUILDKIT_CFG=}
+: ${DRIVER=docker-container}
+: ${DRIVER_OPT=}
+: ${MULTI_NODE=0}
+: ${PLATFORMS=linux/amd64,linux/arm64}
+
+function clean {
+  rm -rf "$context"
+  ${BUILDX_CMD} rm "$builderName"
+}
+
+context=$(mktemp -d -t buildx-output.XXXXXXXXXX)
+dockerfile=${context}/Dockerfile
+trap clean EXIT
+
+builderName=buildx-test-$(openssl rand -hex 16)
+buildPlatformFlag=
+if [ "$DRIVER" = "docker" ]; then
+  builderName=default
+else
+  buildPlatformFlag=--platform="${PLATFORMS}"
+fi
+
+driverOpt=image=${BUILDKIT_IMAGE}
+if [ -n "$DRIVER_OPT" ]; then
+  driverOpt=$driverOpt,$DRIVER_OPT
+fi
+
+# create builder except for docker driver
+if [ "$DRIVER" != "docker" ]; then
+  if [ "${MULTI_NODE}" = "1" ]; then
+    firstNode=1
+    for platform in ${PLATFORMS//,/ }; do
+      createFlags=""
+      if [ -f "$BUILDKIT_CFG" ]; then
+        createFlags="$createFlags --config=${BUILDKIT_CFG}"
+      fi
+      if [ "$firstNode" = "0" ]; then
+        createFlags="$createFlags --append"
+      fi
+      (
+        set -x
+        ${BUILDX_CMD} create ${createFlags} \
+          --name="${builderName}" \
+          --node="${builderName}-${platform/\//-}" \
+          --driver="${DRIVER}" \
+          --platform="${platform}"
+      )
+      firstNode=0
+    done
+  else
+    createFlags=""
+    if [ -f "$BUILDKIT_CFG" ]; then
+      createFlags="$createFlags --config=${BUILDKIT_CFG}"
+    fi
+    (
+      set -x
+      ${BUILDX_CMD} create ${createFlags} \
+        --name="${builderName}" \
+        --driver="${DRIVER}" \
+        --platform="${PLATFORMS}"
+    )
+  fi
+fi
+
+# multi-platform not supported by docker driver
+buildPlatformFlag=
+if [ "$DRIVER" != "docker" ]; then
+  buildPlatformFlag=--platform="${PLATFORMS}"
+fi
+
+set -x
+
+# inspect and bootstrap
+${BUILDX_CMD} inspect --bootstrap --builder="${builderName}"
+
+# create dockerfile
+cat > "${dockerfile}" <<EOL
+FROM busybox as build
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+RUN echo "I am running on \$BUILDPLATFORM, building for \$TARGETPLATFORM" > /log
+FROM busybox
+COPY --from=build /log /log
+RUN cat /log
+RUN uname -a
+EOL
+
+# build
+${BUILDX_CMD} build ${buildPlatformFlag} \
+  --output="type=cacheonly" \
+  --builder="${builderName}" \
+  "${context}"