Merge pull request #946 from crazy-max/metadata-output

build: enhance metadata json output

.github/workflows/e2e.yml

@@ -31,6 +31,7 @@ jobs:
           - mnode-false
           - mnode-true
         platforms:
+          - linux/amd64
           - linux/amd64,linux/arm64
         include:
           - driver: kubernetes

commands/bake.go

@@ -13,7 +13,6 @@ import (
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli/command"
-	"github.com/docker/docker/pkg/ioutils"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -152,16 +151,20 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
 	}
 
 	if len(in.metadataFile) > 0 && resp != nil {
-		mdata := map[string]map[string]string{}
+		if len(resp) == 1 {
-		for k, r := range resp {
+			for _, r := range resp {
-			mdata[k] = r.ExporterResponse
+				if err := writeMetadataFile(in.metadataFile, decodeExporterResponse(r.ExporterResponse)); err != nil {
-		}
+					return err
-		mdatab, err := json.MarshalIndent(mdata, "", "  ")
+				}
-		if err != nil {
+			}
-			return err
+		} else {
-		}
+			dt := make(map[string]interface{})
-		if err := ioutils.AtomicWriteFile(in.metadataFile, mdatab, 0644); err != nil {
+			for t, r := range resp {
-			return err
+				dt[t] = decodeExporterResponse(r.ExporterResponse)
+			}
+			if err := writeMetadataFile(in.metadataFile, dt); err != nil {
+				return err
+			}
 		}
 	}
 

commands/build.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -251,11 +252,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
 	}
 
 	if len(metadataFile) > 0 && resp != nil {
-		mdatab, err := json.MarshalIndent(resp[defaultTargetName].ExporterResponse, "", "  ")
+		if err := writeMetadataFile(metadataFile, decodeExporterResponse(resp[defaultTargetName].ExporterResponse)); err != nil {
-		if err != nil {
-			return "", err
-		}
-		if err := ioutils.AtomicWriteFile(metadataFile, mdatab, 0644); err != nil {
 			return "", err
 		}
 	}
@@ -496,6 +493,32 @@ func parseContextNames(values []string) (map[string]string, error) {
 	return result, nil
 }
 
+func writeMetadataFile(filename string, dt interface{}) error {
+	b, err := json.MarshalIndent(dt, "", "  ")
+	if err != nil {
+		return err
+	}
+	return ioutils.AtomicWriteFile(filename, b, 0644)
+}
+
+func decodeExporterResponse(exporterResponse map[string]string) map[string]interface{} {
+	out := make(map[string]interface{})
+	for k, v := range exporterResponse {
+		dt, err := base64.StdEncoding.DecodeString(v)
+		if err != nil {
+			out[k] = v
+			continue
+		}
+		var raw map[string]interface{}
+		if err = json.Unmarshal(dt, &raw); err != nil || len(raw) == 0 {
+			out[k] = v
+			continue
+		}
+		out[k] = json.RawMessage(dt)
+	}
+	return out
+}
+
 func wrapBuildError(err error) error {
 	if err == nil {
 		return nil

hack/test-driver

@@ -10,13 +10,20 @@ set -eu -o pipefail
 : ${MULTI_NODE=0}
 : ${PLATFORMS=linux/amd64,linux/arm64}
 
+function buildxCmd {
+  (set -x ; $BUILDX_CMD "$@")
+}
+
 function clean {
   rm -rf "$context"
-  ${BUILDX_CMD} rm "$builderName"
+  if [ "$builderName" != "default" ]; then
+    buildxCmd rm "$builderName"
+  fi
 }
 
 context=$(mktemp -d -t buildx-output.XXXXXXXXXX)
 dockerfile=${context}/Dockerfile
+bakedef=${context}/docker-bake.hcl
 trap clean EXIT
 
 builderName=buildx-test-$(openssl rand -hex 16)
@@ -44,15 +51,12 @@ if [ "$DRIVER" != "docker" ]; then
       if [ "$firstNode" = "0" ]; then
         createFlags="$createFlags --append"
       fi
-      (
+      buildxCmd create ${createFlags} \
-        set -x
+        --name="${builderName}" \
-        ${BUILDX_CMD} create ${createFlags} \
+        --node="${builderName}-${platform/\//-}" \
-          --name="${builderName}" \
+        --driver="${DRIVER}" \
-          --node="${builderName}-${platform/\//-}" \
+        --driver-opt="${driverOpt}" \
-          --driver="${DRIVER}" \
+        --platform="${platform}"
-          --driver-opt="${driverOpt}" \
-          --platform="${platform}"
-      )
       firstNode=0
     done
   else
@@ -60,27 +64,37 @@ if [ "$DRIVER" != "docker" ]; then
     if [ -f "$BUILDKIT_CFG" ]; then
       createFlags="$createFlags --config=${BUILDKIT_CFG}"
     fi
-    (
+    buildxCmd create ${createFlags} \
-      set -x
+      --name="${builderName}" \
-      ${BUILDX_CMD} create ${createFlags} \
+      --driver="${DRIVER}" \
-        --name="${builderName}" \
+      --driver-opt="${driverOpt}" \
-        --driver="${DRIVER}" \
+      --platform="${PLATFORMS}"
-        --driver-opt="${driverOpt}" \
-        --platform="${PLATFORMS}"
-    )
   fi
 fi
 
+function buildOutput {
+  local name=$1
+  if [ "$DRIVER" != "docker" ]; then
+    if [ "${MULTI_NODE}" = "1" ]; then
+      echo "type=cacheonly"
+    else
+      echo "type=oci,dest=${context}/${name}.tar"
+    fi
+  else
+    echo "type=docker,name=${name}"
+  fi
+}
+
 # multi-platform not supported by docker driver
 buildPlatformFlag=
+bakePlatformFlag=
 if [ "$DRIVER" != "docker" ]; then
   buildPlatformFlag=--platform="${PLATFORMS}"
+  bakePlatformFlag=--set="*.platform=${PLATFORMS}"
 fi
 
-set -x
-
 # inspect and bootstrap
-${BUILDX_CMD} inspect --bootstrap --builder="${builderName}"
+buildxCmd inspect --bootstrap --builder="${builderName}"
 
 # create dockerfile
 cat > "${dockerfile}" <<EOL
@@ -88,14 +102,60 @@ FROM busybox as build
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
 RUN echo "I am running on \$BUILDPLATFORM, building for \$TARGETPLATFORM" > /log
-FROM busybox
+
+FROM busybox AS log
 COPY --from=build /log /log
 RUN cat /log
 RUN uname -a
+
+FROM busybox AS hello
+RUN echo hello > /hello
+
+FROM scratch
+COPY --from=log /log /log
+COPY --from=hello /hello /hello
 EOL
 
 # build
-${BUILDX_CMD} build ${buildPlatformFlag} \
+buildxCmd build ${buildPlatformFlag} \
-  --output="type=cacheonly" \
+  --output="$(buildOutput buildx-test-build)" \
   --builder="${builderName}" \
+  --metadata-file="${context}/metadata-build.json" \
   "${context}"
+cat "${context}/metadata-build.json"
+
+# create bake def
+cat > "${bakedef}" <<EOL
+group "default" {
+  targets = ["release"]
+}
+group "all" {
+  targets = ["log", "hello"]
+}
+target "release" {
+  output = ["$(buildOutput buildx-test-bake-release)"]
+}
+target "log" {
+  output = ["$(buildOutput buildx-test-bake-log)"]
+}
+target "hello" {
+  output = ["$(buildOutput buildx-test-bake-hello)"]
+}
+EOL
+
+# bake default target
+buildxCmd bake ${bakePlatformFlag} \
+  --file="${bakedef}" \
+  --builder="${builderName}" \
+  --set "*.context=${context}" \
+  --metadata-file="${context}/metadata-bake-def.json"
+cat "${context}/metadata-bake-def.json"
+
+# bake all target
+buildxCmd bake ${bakePlatformFlag} \
+  --file="${bakedef}" \
+  --builder="${builderName}" \
+  --set "*.context=${context}" \
+  --metadata-file="${context}/metadata-bake-all.json" \
+  all
+cat "${context}/metadata-bake-all.json"