From 299fd19c494363393439ce8253a2cc262ee8eeae Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 11 Feb 2022 08:15:47 +0100 Subject: [PATCH 1/2] build: enhance metadata json output Signed-off-by: CrazyMax --- commands/bake.go | 25 ++++++++++++++----------- commands/build.go | 33 ++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/commands/bake.go b/commands/bake.go index db7b9498..94a125e9 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -13,7 +13,6 @@ import ( "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/tracing" "github.com/docker/cli/cli/command" - "github.com/docker/docker/pkg/ioutils" "github.com/moby/buildkit/util/appcontext" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -152,16 +151,20 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error } if len(in.metadataFile) > 0 && resp != nil { - mdata := map[string]map[string]string{} - for k, r := range resp { - mdata[k] = r.ExporterResponse - } - mdatab, err := json.MarshalIndent(mdata, "", " ") - if err != nil { - return err - } - if err := ioutils.AtomicWriteFile(in.metadataFile, mdatab, 0644); err != nil { - return err + if len(resp) == 1 { + for _, r := range resp { + if err := writeMetadataFile(in.metadataFile, decodeExporterResponse(r.ExporterResponse)); err != nil { + return err + } + } + } else { + dt := make(map[string]interface{}) + for t, r := range resp { + dt[t] = decodeExporterResponse(r.ExporterResponse) + } + if err := writeMetadataFile(in.metadataFile, dt); err != nil { + return err + } } } diff --git a/commands/build.go b/commands/build.go index 404f9564..6eb3de54 100644 --- a/commands/build.go +++ b/commands/build.go @@ -3,6 +3,7 @@ package commands import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -251,11 +252,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu } if len(metadataFile) > 0 && resp != nil { - mdatab, err := json.MarshalIndent(resp[defaultTargetName].ExporterResponse, "", " ") - if err != nil { - return "", err - } - if err := ioutils.AtomicWriteFile(metadataFile, mdatab, 0644); err != nil { + if err := writeMetadataFile(metadataFile, decodeExporterResponse(resp[defaultTargetName].ExporterResponse)); err != nil { return "", err } } @@ -496,6 +493,32 @@ func parseContextNames(values []string) (map[string]string, error) { return result, nil } +func writeMetadataFile(filename string, dt interface{}) error { + b, err := json.MarshalIndent(dt, "", " ") + if err != nil { + return err + } + return ioutils.AtomicWriteFile(filename, b, 0644) +} + +func decodeExporterResponse(exporterResponse map[string]string) map[string]interface{} { + out := make(map[string]interface{}) + for k, v := range exporterResponse { + dt, err := base64.StdEncoding.DecodeString(v) + if err != nil { + out[k] = v + continue + } + var raw map[string]interface{} + if err = json.Unmarshal(dt, &raw); err != nil || len(raw) == 0 { + out[k] = v + continue + } + out[k] = json.RawMessage(dt) + } + return out +} + func wrapBuildError(err error) error { if err == nil { return nil From ecf215b927be3f5179c261572e008204c16f43fc Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 11 Feb 2022 08:16:35 +0100 Subject: [PATCH 2/2] e2e: add bake build and display metadata json Signed-off-by: CrazyMax --- .github/workflows/e2e.yml | 1 + hack/test-driver | 108 +++++++++++++++++++++++++++++--------- 2 files changed, 85 insertions(+), 24 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 00343e07..41c8dfa4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,6 +31,7 @@ jobs: - mnode-false - mnode-true platforms: + - linux/amd64 - linux/amd64,linux/arm64 include: - driver: kubernetes diff --git a/hack/test-driver b/hack/test-driver index ddcc7072..12818dc0 100755 --- a/hack/test-driver +++ b/hack/test-driver @@ -10,13 +10,20 @@ set -eu -o pipefail : ${MULTI_NODE=0} : ${PLATFORMS=linux/amd64,linux/arm64} +function buildxCmd { + (set -x ; $BUILDX_CMD "$@") +} + function clean { rm -rf "$context" - ${BUILDX_CMD} rm "$builderName" + if [ "$builderName" != "default" ]; then + buildxCmd rm "$builderName" + fi } context=$(mktemp -d -t buildx-output.XXXXXXXXXX) dockerfile=${context}/Dockerfile +bakedef=${context}/docker-bake.hcl trap clean EXIT builderName=buildx-test-$(openssl rand -hex 16) @@ -44,15 +51,12 @@ if [ "$DRIVER" != "docker" ]; then if [ "$firstNode" = "0" ]; then createFlags="$createFlags --append" fi - ( - set -x - ${BUILDX_CMD} create ${createFlags} \ - --name="${builderName}" \ - --node="${builderName}-${platform/\//-}" \ - --driver="${DRIVER}" \ - --driver-opt="${driverOpt}" \ - --platform="${platform}" - ) + buildxCmd create ${createFlags} \ + --name="${builderName}" \ + --node="${builderName}-${platform/\//-}" \ + --driver="${DRIVER}" \ + --driver-opt="${driverOpt}" \ + --platform="${platform}" firstNode=0 done else @@ -60,27 +64,37 @@ if [ "$DRIVER" != "docker" ]; then if [ -f "$BUILDKIT_CFG" ]; then createFlags="$createFlags --config=${BUILDKIT_CFG}" fi - ( - set -x - ${BUILDX_CMD} create ${createFlags} \ - --name="${builderName}" \ - --driver="${DRIVER}" \ - --driver-opt="${driverOpt}" \ - --platform="${PLATFORMS}" - ) + buildxCmd create ${createFlags} \ + --name="${builderName}" \ + --driver="${DRIVER}" \ + --driver-opt="${driverOpt}" \ + --platform="${PLATFORMS}" fi fi +function buildOutput { + local name=$1 + if [ "$DRIVER" != "docker" ]; then + if [ "${MULTI_NODE}" = "1" ]; then + echo "type=cacheonly" + else + echo "type=oci,dest=${context}/${name}.tar" + fi + else + echo "type=docker,name=${name}" + fi +} + # multi-platform not supported by docker driver buildPlatformFlag= +bakePlatformFlag= if [ "$DRIVER" != "docker" ]; then buildPlatformFlag=--platform="${PLATFORMS}" + bakePlatformFlag=--set="*.platform=${PLATFORMS}" fi -set -x - # inspect and bootstrap -${BUILDX_CMD} inspect --bootstrap --builder="${builderName}" +buildxCmd inspect --bootstrap --builder="${builderName}" # create dockerfile cat > "${dockerfile}" < /log -FROM busybox + +FROM busybox AS log COPY --from=build /log /log RUN cat /log RUN uname -a + +FROM busybox AS hello +RUN echo hello > /hello + +FROM scratch +COPY --from=log /log /log +COPY --from=hello /hello /hello EOL # build -${BUILDX_CMD} build ${buildPlatformFlag} \ - --output="type=cacheonly" \ +buildxCmd build ${buildPlatformFlag} \ + --output="$(buildOutput buildx-test-build)" \ --builder="${builderName}" \ + --metadata-file="${context}/metadata-build.json" \ "${context}" +cat "${context}/metadata-build.json" + +# create bake def +cat > "${bakedef}" <