diff --git a/Makefile b/Makefile
index 9c67a1a9..2be17502 100644
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,7 @@ validate-vendor:
validate-docs:
./hack/validate-docs
-
+
validate-all: lint test validate-vendor validate-docs
vendor:
@@ -34,7 +34,10 @@ vendor:
docs:
./hack/update-docs
+yamldocs: ## generate documentation YAML files consumed by docs repo
+ ./hack/generate-yamldocs
+
generate-authors:
./hack/generate-authors
-.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs
+.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs yamldocs
diff --git a/docs/yamlgen/generate.go b/docs/yamlgen/generate.go
new file mode 100644
index 00000000..a6bb6b39
--- /dev/null
+++ b/docs/yamlgen/generate.go
@@ -0,0 +1,117 @@
+package main
+
+import (
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/docker/buildx/commands"
+ "github.com/docker/cli/cli/command"
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+)
+
+const descriptionSourcePath = "docs/reference/"
+
+func generateCliYaml(opts *options) error {
+ dockerCLI, err := command.NewDockerCli()
+ if err != nil {
+ return err
+ }
+ cmd := &cobra.Command{
+ Use: "docker [OPTIONS] COMMAND [ARG...]",
+ Short: "The base command for the Docker CLI.",
+ }
+ cmd.AddCommand(commands.NewRootCmd("buildx", true, dockerCLI))
+ disableFlagsInUseLine(cmd)
+ source := filepath.Join(opts.source, descriptionSourcePath)
+ fmt.Println("Markdown source:", source)
+ if err := loadLongDescription(cmd, source); err != nil {
+ return err
+ }
+
+ if err := os.MkdirAll(opts.target, 0755); err != nil {
+ return err
+ }
+
+ cmd.DisableAutoGenTag = true
+ return GenYamlTree(cmd, opts.target)
+}
+
+func disableFlagsInUseLine(cmd *cobra.Command) {
+ visitAll(cmd, func(ccmd *cobra.Command) {
+ // do not add a `[flags]` to the end of the usage line.
+ ccmd.DisableFlagsInUseLine = true
+ })
+}
+
+// visitAll will traverse all commands from the root.
+// This is different from the VisitAll of cobra.Command where only parents
+// are checked.
+func visitAll(root *cobra.Command, fn func(*cobra.Command)) {
+ for _, cmd := range root.Commands() {
+ visitAll(cmd, fn)
+ }
+ fn(root)
+}
+
+func loadLongDescription(parentCmd *cobra.Command, path string) error {
+ for _, cmd := range parentCmd.Commands() {
+ if cmd.HasSubCommands() {
+ if err := loadLongDescription(cmd, path); err != nil {
+ return err
+ }
+ }
+ name := cmd.CommandPath()
+ log.Println("INFO: Generating docs for", name)
+ if i := strings.Index(name, " "); i >= 0 {
+ // remove root command / binary name
+ name = name[i+1:]
+ }
+ if name == "" {
+ continue
+ }
+ mdFile := strings.ReplaceAll(name, " ", "_") + ".md"
+ fullPath := filepath.Join(path, mdFile)
+ content, err := ioutil.ReadFile(fullPath)
+ if os.IsNotExist(err) {
+ log.Printf("WARN: %s does not exist, skipping\n", mdFile)
+ continue
+ }
+ if err != nil {
+ return err
+ }
+ applyDescriptionAndExamples(cmd, string(content))
+ }
+ return nil
+}
+
+type options struct {
+ source string
+ target string
+}
+
+func parseArgs() (*options, error) {
+ opts := &options{}
+ cwd, _ := os.Getwd()
+ flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
+ flags.StringVar(&opts.source, "root", cwd, "Path to project root")
+ flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files")
+ err := flags.Parse(os.Args[1:])
+ return opts, err
+}
+
+func main() {
+ opts, err := parseArgs()
+ if err != nil {
+ log.Println(err)
+ }
+ fmt.Println("Project root: ", opts.source)
+ fmt.Println("YAML output dir:", opts.target)
+ if err := generateCliYaml(opts); err != nil {
+ log.Println("Failed to generate yaml files:", err)
+ }
+}
diff --git a/docs/yamlgen/markdown.go b/docs/yamlgen/markdown.go
new file mode 100644
index 00000000..0ad78613
--- /dev/null
+++ b/docs/yamlgen/markdown.go
@@ -0,0 +1,73 @@
+package main
+
+import (
+ "regexp"
+ "strings"
+ "unicode"
+)
+
+var (
+ // mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce
+ // false positives for (e.g.) comments in code-blocks (# this is a comment),
+ // so should not be used as a generic regex for other purposes.
+ mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`)
+ // htmlAnchor matches inline HTML anchors. This is intended to only match anchors
+ // for our use-case; DO NOT consider using this as a generic regex, or at least
+ // not before reading https://stackoverflow.com/a/1732454/1811501.
+ htmlAnchor = regexp.MustCompile(`\s*`)
+)
+
+// getSections returns all H2 sections by title (lowercase)
+func getSections(mdString string) map[string]string {
+ parsedContent := strings.Split("\n"+mdString, "\n## ")
+ sections := make(map[string]string, len(parsedContent))
+ for _, s := range parsedContent {
+ if strings.HasPrefix(s, "#") {
+ // not a H2 Section
+ continue
+ }
+ parts := strings.SplitN(s, "\n", 2)
+ if len(parts) == 2 {
+ sections[strings.ToLower(parts[0])] = parts[1]
+ }
+ }
+ return sections
+}
+
+// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in
+// YAML. It removes trailing whitespace and substitutes tabs for four spaces
+// to prevent YAML switching to use "compact" form; ("line1 \nline\t2\n")
+// which, although equivalent, is hard to read.
+func cleanupMarkDown(mdString string) (md string, anchors []string) {
+ // remove leading/trailing whitespace, and replace tabs in the whole content
+ mdString = strings.TrimSpace(mdString)
+ mdString = strings.ReplaceAll(mdString, "\t", " ")
+ mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "")
+
+ var id string
+ // replace trailing whitespace per line, and handle custom anchors
+ lines := strings.Split(mdString, "\n")
+ for i := 0; i < len(lines); i++ {
+ lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace)
+ lines[i], id = convertHTMLAnchor(lines[i])
+ if id != "" {
+ anchors = append(anchors, id)
+ }
+ }
+ return strings.Join(lines, "\n"), anchors
+}
+
+// convertHTMLAnchor converts inline anchor-tags in headings ()
+// to an extended-markdown property ({#myanchor}). Extended Markdown properties
+// are not supported in GitHub Flavored Markdown, but are supported by Jekyll,
+// and lead to cleaner HTML in our docs, and prevents duplicate anchors.
+// It returns the converted MarkDown heading and the custom ID (if present)
+func convertHTMLAnchor(mdLine string) (md string, customID string) {
+ if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 {
+ if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 {
+ customID = a[1]
+ mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}"
+ }
+ }
+ return mdLine, customID
+}
diff --git a/docs/yamlgen/markdown_test.go b/docs/yamlgen/markdown_test.go
new file mode 100644
index 00000000..1d244c96
--- /dev/null
+++ b/docs/yamlgen/markdown_test.go
@@ -0,0 +1,132 @@
+package main
+
+import "testing"
+
+func TestCleanupMarkDown(t *testing.T) {
+ tests := []struct {
+ doc, in, expected string
+ }{
+ {
+ doc: "whitespace around sections",
+ in: `
+
+ ## Section start
+
+Some lines.
+And more lines.
+
+`,
+ expected: `## Section start
+
+Some lines.
+And more lines.`,
+ },
+ {
+ doc: "lines with inline tabs",
+ in: `## Some Heading
+
+A line with tabs in it.
+Tabs should be replaced by spaces`,
+ expected: `## Some Heading
+
+A line with tabs in it.
+Tabs should be replaced by spaces`,
+ },
+ {
+ doc: "lines with trailing spaces",
+ in: `## Some Heading with spaces
+
+This is a line.
+ This is an indented line
+
+### Some other heading
+
+Last line.`,
+ expected: `## Some Heading with spaces
+
+This is a line.
+ This is an indented line
+
+### Some other heading
+
+Last line.`,
+ },
+ {
+ doc: "lines with trailing tabs",
+ in: `## Some Heading with tabs
+
+This is a line.
+ This is an indented line
+
+### Some other heading
+
+Last line.`,
+ expected: `## Some Heading with tabs
+
+This is a line.
+ This is an indented line
+
+### Some other heading
+
+Last line.`,
+ },
+ }
+ for _, tc := range tests {
+ tc := tc
+ t.Run(tc.doc, func(t *testing.T) {
+ out, _ := cleanupMarkDown(tc.in)
+ if out != tc.expected {
+ t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out)
+ }
+ })
+ }
+}
+
+func TestConvertHTMLAnchor(t *testing.T) {
+ tests := []struct {
+ in, id, expected string
+ }{
+ {
+ in: `# Heading 1`,
+ id: "heading1",
+ expected: `# Heading 1 {#heading1}`,
+ },
+ {
+ in: `## Heading 2 `,
+ id: "heading2",
+ expected: `## Heading 2 {#heading2}`,
+ },
+ {
+ in: `### Heading 3`,
+ id: "heading3",
+ expected: `### Heading 3 {#heading3}`,
+ },
+ {
+ in: `#### Heading 4`,
+ id: "heading4",
+ expected: `#### Heading 4 {#heading4}`,
+ },
+ {
+ in: `##### Heading 5`,
+ id: "heading5",
+ expected: `##### Heading 5 {#heading5}`,
+ },
+ {
+ in: `###### hello!Heading 6`,
+ id: "",
+ expected: `###### hello!Heading 6`,
+ },
+ }
+ for _, tc := range tests {
+ tc := tc
+ t.Run(tc.in, func(t *testing.T) {
+ out, id := convertHTMLAnchor(tc.in)
+ if id != tc.id {
+ t.Fatalf("expected: %s, actual: %s\n", tc.id, id)
+ }
+ if out != tc.expected {
+ t.Fatalf("\nexpected: %s\nactual: %s\n", tc.expected, out)
+ }
+ })
+ }
+}
diff --git a/docs/yamlgen/yaml.go b/docs/yamlgen/yaml.go
new file mode 100644
index 00000000..72128693
--- /dev/null
+++ b/docs/yamlgen/yaml.go
@@ -0,0 +1,304 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+ yaml "gopkg.in/yaml.v2"
+)
+
+type cmdOption struct {
+ Option string
+ Shorthand string `yaml:",omitempty"`
+ ValueType string `yaml:"value_type,omitempty"`
+ DefaultValue string `yaml:"default_value,omitempty"`
+ Description string `yaml:",omitempty"`
+ // DetailsURL contains an anchor-id or link for more information on this flag
+ DetailsURL string `yaml:"details_url,omitempty"`
+ Deprecated bool
+ MinAPIVersion string `yaml:"min_api_version,omitempty"`
+ Experimental bool
+ ExperimentalCLI bool
+ Kubernetes bool
+ Swarm bool
+ OSType string `yaml:"os_type,omitempty"`
+}
+
+type cmdDoc struct {
+ Name string `yaml:"command"`
+ SeeAlso []string `yaml:"parent,omitempty"`
+ Version string `yaml:"engine_version,omitempty"`
+ Aliases string `yaml:",omitempty"`
+ Short string `yaml:",omitempty"`
+ Long string `yaml:",omitempty"`
+ Usage string `yaml:",omitempty"`
+ Pname string `yaml:",omitempty"`
+ Plink string `yaml:",omitempty"`
+ Cname []string `yaml:",omitempty"`
+ Clink []string `yaml:",omitempty"`
+ Options []cmdOption `yaml:",omitempty"`
+ InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"`
+ Example string `yaml:"examples,omitempty"`
+ Deprecated bool
+ MinAPIVersion string `yaml:"min_api_version,omitempty"`
+ Experimental bool
+ ExperimentalCLI bool
+ Kubernetes bool
+ Swarm bool
+ OSType string `yaml:"os_type,omitempty"`
+}
+
+// GenYamlTree creates yaml structured ref files
+func GenYamlTree(cmd *cobra.Command, dir string) error {
+ emptyStr := func(s string) string { return "" }
+ return GenYamlTreeCustom(cmd, dir, emptyStr)
+}
+
+// GenYamlTreeCustom creates yaml structured ref files
+func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error {
+ for _, c := range cmd.Commands() {
+ if !c.Runnable() && !c.HasAvailableSubCommands() {
+ // skip non-runnable commands without subcommands
+ // but *do* generate YAML for hidden and deprecated commands
+ // the YAML will have those included as metadata, so that the
+ // documentation repository can decide whether or not to present them
+ continue
+ }
+ if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil {
+ return err
+ }
+ }
+ if !cmd.HasParent() {
+ return nil
+ }
+ basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
+ filename := filepath.Join(dir, basename)
+ f, err := os.Create(filename)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ if _, err := io.WriteString(f, filePrepender(filename)); err != nil {
+ return err
+ }
+ return GenYamlCustom(cmd, f)
+}
+
+// GenYamlCustom creates custom yaml output
+// nolint: gocyclo
+func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
+ cliDoc := cmdDoc{
+ Name: cmd.CommandPath(),
+ Aliases: strings.Join(cmd.Aliases, ", "),
+ Short: forceMultiLine(cmd.Short),
+ Long: forceMultiLine(cmd.Long),
+ Deprecated: len(cmd.Deprecated) > 0,
+ }
+
+ if len(cliDoc.Long) == 0 {
+ cliDoc.Long = cliDoc.Short
+ }
+
+ if cmd.Runnable() {
+ cliDoc.Usage = cmd.UseLine()
+ }
+
+ if len(cmd.Example) > 0 {
+ cliDoc.Example = cmd.Example
+ }
+
+ // Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack`
+ for curr := cmd; curr != nil; curr = curr.Parent() {
+ if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" {
+ cliDoc.MinAPIVersion = v
+ }
+ if _, ok := curr.Annotations["experimental"]; ok && !cliDoc.Experimental {
+ cliDoc.Experimental = true
+ }
+ if _, ok := curr.Annotations["experimentalCLI"]; ok && !cliDoc.ExperimentalCLI {
+ cliDoc.ExperimentalCLI = true
+ }
+ if _, ok := curr.Annotations["kubernetes"]; ok && !cliDoc.Kubernetes {
+ cliDoc.Kubernetes = true
+ }
+ if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm {
+ cliDoc.Swarm = true
+ }
+ if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" {
+ cliDoc.OSType = o
+ }
+ }
+
+ var anchors = make(map[string]struct{})
+ if a, ok := cmd.Annotations["anchors"]; ok && a != "" {
+ for _, anchor := range strings.Split(a, ",") {
+ anchors[anchor] = struct{}{}
+ }
+ }
+
+ flags := cmd.NonInheritedFlags()
+ if flags.HasFlags() {
+ cliDoc.Options = genFlagResult(flags, anchors)
+ }
+ flags = cmd.InheritedFlags()
+ if flags.HasFlags() {
+ cliDoc.InheritedOptions = genFlagResult(flags, anchors)
+ }
+
+ if hasSeeAlso(cmd) {
+ if cmd.HasParent() {
+ parent := cmd.Parent()
+ cliDoc.Pname = parent.CommandPath()
+ cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml"
+ cmd.VisitParents(func(c *cobra.Command) {
+ if c.DisableAutoGenTag {
+ cmd.DisableAutoGenTag = c.DisableAutoGenTag
+ }
+ })
+ }
+
+ children := cmd.Commands()
+ sort.Sort(byName(children))
+
+ for _, child := range children {
+ if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() {
+ continue
+ }
+ currentChild := cliDoc.Name + " " + child.Name()
+ cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
+ link := currentChild + ".yaml"
+ cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1))
+ }
+ }
+
+ final, err := yaml.Marshal(&cliDoc)
+ if err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+ if _, err := fmt.Fprintln(w, string(final)); err != nil {
+ return err
+ }
+ return nil
+}
+
+func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption {
+ var (
+ result []cmdOption
+ opt cmdOption
+ )
+
+ flags.VisitAll(func(flag *pflag.Flag) {
+ opt = cmdOption{
+ Option: flag.Name,
+ ValueType: flag.Value.Type(),
+ DefaultValue: forceMultiLine(flag.DefValue),
+ Description: forceMultiLine(flag.Usage),
+ Deprecated: len(flag.Deprecated) > 0,
+ }
+
+ if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 {
+ opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com")
+ } else if _, ok = anchors[flag.Name]; ok {
+ opt.DetailsURL = "#" + flag.Name
+ }
+
+ // Todo, when we mark a shorthand is deprecated, but specify an empty message.
+ // The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
+ // Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
+ if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
+ opt.Shorthand = flag.Shorthand
+ }
+ if _, ok := flag.Annotations["experimental"]; ok {
+ opt.Experimental = true
+ }
+ if v, ok := flag.Annotations["version"]; ok {
+ opt.MinAPIVersion = v[0]
+ }
+ if _, ok := flag.Annotations["experimentalCLI"]; ok {
+ opt.ExperimentalCLI = true
+ }
+ if _, ok := flag.Annotations["kubernetes"]; ok {
+ opt.Kubernetes = true
+ }
+ if _, ok := flag.Annotations["swarm"]; ok {
+ opt.Swarm = true
+ }
+
+ // Note that the annotation can have multiple ostypes set, however, multiple
+ // values are currently not used (and unlikely will).
+ //
+ // To simplify usage of the os_type property in the YAML, and for consistency
+ // with the same property for commands, we're only using the first ostype that's set.
+ if ostypes, ok := flag.Annotations["ostype"]; ok && len(opt.OSType) == 0 && len(ostypes) > 0 {
+ opt.OSType = ostypes[0]
+ }
+
+ result = append(result, opt)
+ })
+
+ return result
+}
+
+// Temporary workaround for yaml lib generating incorrect yaml with long strings
+// that do not contain \n.
+func forceMultiLine(s string) string {
+ s = strings.TrimSpace(s)
+ if len(s) > 60 && !strings.Contains(s, "\n") {
+ s = s + "\n"
+ }
+ return s
+}
+
+// Small duplication for cobra utils
+func hasSeeAlso(cmd *cobra.Command) bool {
+ if cmd.HasParent() {
+ return true
+ }
+ for _, c := range cmd.Commands() {
+ if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
+ continue
+ }
+ return true
+ }
+ return false
+}
+
+// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the
+// "Description" and "Examples" H2 sections in mdString (if present).
+func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) {
+ sections := getSections(mdString)
+ var (
+ anchors []string
+ md string
+ )
+ if sections["description"] != "" {
+ md, anchors = cleanupMarkDown(sections["description"])
+ cmd.Long = md
+ anchors = append(anchors, md)
+ }
+ if sections["examples"] != "" {
+ md, anchors = cleanupMarkDown(sections["examples"])
+ cmd.Example = md
+ anchors = append(anchors, md)
+ }
+ if len(anchors) > 0 {
+ if cmd.Annotations == nil {
+ cmd.Annotations = make(map[string]string)
+ }
+ cmd.Annotations["anchors"] = strings.Join(anchors, ",")
+ }
+}
+
+type byName []*cobra.Command
+
+func (s byName) Len() int { return len(s) }
+func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() }
diff --git a/go.mod b/go.mod
index 55422bea..3b7459b4 100644
--- a/go.mod
+++ b/go.mod
@@ -54,6 +54,7 @@ require (
gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
gopkg.in/fatih/pool.v2 v2.0.0 // indirect
gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
+ gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.20.1
k8s.io/apimachinery v0.20.1
k8s.io/client-go v0.20.1
diff --git a/hack/dockerfiles/yamldocs.Dockerfile b/hack/dockerfiles/yamldocs.Dockerfile
new file mode 100644
index 00000000..6c1a4db1
--- /dev/null
+++ b/hack/dockerfiles/yamldocs.Dockerfile
@@ -0,0 +1,19 @@
+# syntax = docker/dockerfile:1.2
+
+FROM golang:1.16-alpine AS yamlgen
+WORKDIR /src
+RUN --mount=target=. \
+ --mount=target=/root/.cache,type=cache \
+ go build -mod=vendor -o /out/yamlgen ./docs/yamlgen
+
+FROM alpine AS gen
+RUN apk add --no-cache rsync git
+WORKDIR /src
+COPY --from=yamlgen /out/yamlgen /usr/bin
+RUN --mount=target=/context \
+ --mount=target=.,type=tmpfs,readwrite \
+ rsync -a /context/. . \
+ && yamlgen --target /out/yaml
+
+FROM scratch AS update
+COPY --from=gen /out /
diff --git a/hack/generate-yamldocs b/hack/generate-yamldocs
new file mode 100755
index 00000000..2946371d
--- /dev/null
+++ b/hack/generate-yamldocs
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+. "$(dirname "$0")"/util
+set -eu
+
+buildxCmd build \
+ --output "type=local,dest=./bin/docs/" \
+ --file "./hack/dockerfiles/yamldocs.Dockerfile" \
+ --progress=plain \
+ .