diff --git a/build/build.go b/build/build.go
index 3728711d..8e8f7d63 100644
--- a/build/build.go
+++ b/build/build.go
@@ -604,13 +604,6 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
 		}
 	}
 
-	// Propagate SOURCE_DATE_EPOCH from the client env
-	if v := os.Getenv("SOURCE_DATE_EPOCH"); v != "" {
-		if _, ok := so.FrontendAttrs["build-arg:SOURCE_DATE_EPOCH"]; !ok {
-			so.FrontendAttrs["build-arg:SOURCE_DATE_EPOCH"] = v
-		}
-	}
-
 	// set platforms
 	if len(opt.Platforms) != 0 {
 		pp := make([]string, len(opt.Platforms))
diff --git a/commands/build.go b/commands/build.go
index 3d33ab65..fef0b48c 100644
--- a/commands/build.go
+++ b/commands/build.go
@@ -100,6 +100,13 @@ func (o *buildOptions) toControllerOptions() (controllerapi.BuildOptions, error)
 		Opts:           &o.CommonOptions,
 	}
 
+	// TODO: extract env var parsing to a method easily usable by library consumers
+	if v := os.Getenv("SOURCE_DATE_EPOCH"); v != "" {
+		if _, ok := opts.BuildArgs["SOURCE_DATE_EPOCH"]; !ok {
+			opts.BuildArgs["SOURCE_DATE_EPOCH"] = v
+		}
+	}
+
 	inAttests := append([]string{}, o.attests...)
 	if o.provenance != "" {
 		inAttests = append(inAttests, buildflags.CanonicalizeAttest("provenance", o.provenance))