diff --git a/Makefile b/Makefile index 9c67a1a9..2be17502 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ validate-vendor: validate-docs: ./hack/validate-docs - + validate-all: lint test validate-vendor validate-docs vendor: @@ -34,7 +34,10 @@ vendor: docs: ./hack/update-docs +yamldocs: ## generate documentation YAML files consumed by docs repo + ./hack/generate-yamldocs + generate-authors: ./hack/generate-authors -.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs +.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs yamldocs diff --git a/docs/yamlgen/generate.go b/docs/yamlgen/generate.go new file mode 100644 index 00000000..a6bb6b39 --- /dev/null +++ b/docs/yamlgen/generate.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "github.com/docker/buildx/commands" + "github.com/docker/cli/cli/command" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const descriptionSourcePath = "docs/reference/" + +func generateCliYaml(opts *options) error { + dockerCLI, err := command.NewDockerCli() + if err != nil { + return err + } + cmd := &cobra.Command{ + Use: "docker [OPTIONS] COMMAND [ARG...]", + Short: "The base command for the Docker CLI.", + } + cmd.AddCommand(commands.NewRootCmd("buildx", true, dockerCLI)) + disableFlagsInUseLine(cmd) + source := filepath.Join(opts.source, descriptionSourcePath) + fmt.Println("Markdown source:", source) + if err := loadLongDescription(cmd, source); err != nil { + return err + } + + if err := os.MkdirAll(opts.target, 0755); err != nil { + return err + } + + cmd.DisableAutoGenTag = true + return GenYamlTree(cmd, opts.target) +} + +func disableFlagsInUseLine(cmd *cobra.Command) { + visitAll(cmd, func(ccmd *cobra.Command) { + // do not add a `[flags]` to the end of the usage line. + ccmd.DisableFlagsInUseLine = true + }) +} + +// visitAll will traverse all commands from the root. +// This is different from the VisitAll of cobra.Command where only parents +// are checked. +func visitAll(root *cobra.Command, fn func(*cobra.Command)) { + for _, cmd := range root.Commands() { + visitAll(cmd, fn) + } + fn(root) +} + +func loadLongDescription(parentCmd *cobra.Command, path string) error { + for _, cmd := range parentCmd.Commands() { + if cmd.HasSubCommands() { + if err := loadLongDescription(cmd, path); err != nil { + return err + } + } + name := cmd.CommandPath() + log.Println("INFO: Generating docs for", name) + if i := strings.Index(name, " "); i >= 0 { + // remove root command / binary name + name = name[i+1:] + } + if name == "" { + continue + } + mdFile := strings.ReplaceAll(name, " ", "_") + ".md" + fullPath := filepath.Join(path, mdFile) + content, err := ioutil.ReadFile(fullPath) + if os.IsNotExist(err) { + log.Printf("WARN: %s does not exist, skipping\n", mdFile) + continue + } + if err != nil { + return err + } + applyDescriptionAndExamples(cmd, string(content)) + } + return nil +} + +type options struct { + source string + target string +} + +func parseArgs() (*options, error) { + opts := &options{} + cwd, _ := os.Getwd() + flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) + flags.StringVar(&opts.source, "root", cwd, "Path to project root") + flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files") + err := flags.Parse(os.Args[1:]) + return opts, err +} + +func main() { + opts, err := parseArgs() + if err != nil { + log.Println(err) + } + fmt.Println("Project root: ", opts.source) + fmt.Println("YAML output dir:", opts.target) + if err := generateCliYaml(opts); err != nil { + log.Println("Failed to generate yaml files:", err) + } +} diff --git a/docs/yamlgen/markdown.go b/docs/yamlgen/markdown.go new file mode 100644 index 00000000..0ad78613 --- /dev/null +++ b/docs/yamlgen/markdown.go @@ -0,0 +1,73 @@ +package main + +import ( + "regexp" + "strings" + "unicode" +) + +var ( + // mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce + // false positives for (e.g.) comments in code-blocks (# this is a comment), + // so should not be used as a generic regex for other purposes. + mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`) + // htmlAnchor matches inline HTML anchors. This is intended to only match anchors + // for our use-case; DO NOT consider using this as a generic regex, or at least + // not before reading https://stackoverflow.com/a/1732454/1811501. + htmlAnchor = regexp.MustCompile(`\s*`) +) + +// getSections returns all H2 sections by title (lowercase) +func getSections(mdString string) map[string]string { + parsedContent := strings.Split("\n"+mdString, "\n## ") + sections := make(map[string]string, len(parsedContent)) + for _, s := range parsedContent { + if strings.HasPrefix(s, "#") { + // not a H2 Section + continue + } + parts := strings.SplitN(s, "\n", 2) + if len(parts) == 2 { + sections[strings.ToLower(parts[0])] = parts[1] + } + } + return sections +} + +// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in +// YAML. It removes trailing whitespace and substitutes tabs for four spaces +// to prevent YAML switching to use "compact" form; ("line1 \nline\t2\n") +// which, although equivalent, is hard to read. +func cleanupMarkDown(mdString string) (md string, anchors []string) { + // remove leading/trailing whitespace, and replace tabs in the whole content + mdString = strings.TrimSpace(mdString) + mdString = strings.ReplaceAll(mdString, "\t", " ") + mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "") + + var id string + // replace trailing whitespace per line, and handle custom anchors + lines := strings.Split(mdString, "\n") + for i := 0; i < len(lines); i++ { + lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace) + lines[i], id = convertHTMLAnchor(lines[i]) + if id != "" { + anchors = append(anchors, id) + } + } + return strings.Join(lines, "\n"), anchors +} + +// convertHTMLAnchor converts inline anchor-tags in headings () +// to an extended-markdown property ({#myanchor}). Extended Markdown properties +// are not supported in GitHub Flavored Markdown, but are supported by Jekyll, +// and lead to cleaner HTML in our docs, and prevents duplicate anchors. +// It returns the converted MarkDown heading and the custom ID (if present) +func convertHTMLAnchor(mdLine string) (md string, customID string) { + if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 { + if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 { + customID = a[1] + mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}" + } + } + return mdLine, customID +} diff --git a/docs/yamlgen/markdown_test.go b/docs/yamlgen/markdown_test.go new file mode 100644 index 00000000..1d244c96 --- /dev/null +++ b/docs/yamlgen/markdown_test.go @@ -0,0 +1,132 @@ +package main + +import "testing" + +func TestCleanupMarkDown(t *testing.T) { + tests := []struct { + doc, in, expected string + }{ + { + doc: "whitespace around sections", + in: ` + + ## Section start + +Some lines. +And more lines. + +`, + expected: `## Section start + +Some lines. +And more lines.`, + }, + { + doc: "lines with inline tabs", + in: `## Some Heading + +A line with tabs in it. +Tabs should be replaced by spaces`, + expected: `## Some Heading + +A line with tabs in it. +Tabs should be replaced by spaces`, + }, + { + doc: "lines with trailing spaces", + in: `## Some Heading with spaces + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + expected: `## Some Heading with spaces + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + }, + { + doc: "lines with trailing tabs", + in: `## Some Heading with tabs + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + expected: `## Some Heading with tabs + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + }, + } + for _, tc := range tests { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + out, _ := cleanupMarkDown(tc.in) + if out != tc.expected { + t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out) + } + }) + } +} + +func TestConvertHTMLAnchor(t *testing.T) { + tests := []struct { + in, id, expected string + }{ + { + in: `# Heading 1`, + id: "heading1", + expected: `# Heading 1 {#heading1}`, + }, + { + in: `## Heading 2 `, + id: "heading2", + expected: `## Heading 2 {#heading2}`, + }, + { + in: `### Heading 3`, + id: "heading3", + expected: `### Heading 3 {#heading3}`, + }, + { + in: `#### Heading 4`, + id: "heading4", + expected: `#### Heading 4 {#heading4}`, + }, + { + in: `##### Heading 5`, + id: "heading5", + expected: `##### Heading 5 {#heading5}`, + }, + { + in: `###### hello!Heading 6`, + id: "", + expected: `###### hello!Heading 6`, + }, + } + for _, tc := range tests { + tc := tc + t.Run(tc.in, func(t *testing.T) { + out, id := convertHTMLAnchor(tc.in) + if id != tc.id { + t.Fatalf("expected: %s, actual: %s\n", tc.id, id) + } + if out != tc.expected { + t.Fatalf("\nexpected: %s\nactual: %s\n", tc.expected, out) + } + }) + } +} diff --git a/docs/yamlgen/yaml.go b/docs/yamlgen/yaml.go new file mode 100644 index 00000000..72128693 --- /dev/null +++ b/docs/yamlgen/yaml.go @@ -0,0 +1,304 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + yaml "gopkg.in/yaml.v2" +) + +type cmdOption struct { + Option string + Shorthand string `yaml:",omitempty"` + ValueType string `yaml:"value_type,omitempty"` + DefaultValue string `yaml:"default_value,omitempty"` + Description string `yaml:",omitempty"` + // DetailsURL contains an anchor-id or link for more information on this flag + DetailsURL string `yaml:"details_url,omitempty"` + Deprecated bool + MinAPIVersion string `yaml:"min_api_version,omitempty"` + Experimental bool + ExperimentalCLI bool + Kubernetes bool + Swarm bool + OSType string `yaml:"os_type,omitempty"` +} + +type cmdDoc struct { + Name string `yaml:"command"` + SeeAlso []string `yaml:"parent,omitempty"` + Version string `yaml:"engine_version,omitempty"` + Aliases string `yaml:",omitempty"` + Short string `yaml:",omitempty"` + Long string `yaml:",omitempty"` + Usage string `yaml:",omitempty"` + Pname string `yaml:",omitempty"` + Plink string `yaml:",omitempty"` + Cname []string `yaml:",omitempty"` + Clink []string `yaml:",omitempty"` + Options []cmdOption `yaml:",omitempty"` + InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"` + Example string `yaml:"examples,omitempty"` + Deprecated bool + MinAPIVersion string `yaml:"min_api_version,omitempty"` + Experimental bool + ExperimentalCLI bool + Kubernetes bool + Swarm bool + OSType string `yaml:"os_type,omitempty"` +} + +// GenYamlTree creates yaml structured ref files +func GenYamlTree(cmd *cobra.Command, dir string) error { + emptyStr := func(s string) string { return "" } + return GenYamlTreeCustom(cmd, dir, emptyStr) +} + +// GenYamlTreeCustom creates yaml structured ref files +func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error { + for _, c := range cmd.Commands() { + if !c.Runnable() && !c.HasAvailableSubCommands() { + // skip non-runnable commands without subcommands + // but *do* generate YAML for hidden and deprecated commands + // the YAML will have those included as metadata, so that the + // documentation repository can decide whether or not to present them + continue + } + if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil { + return err + } + } + if !cmd.HasParent() { + return nil + } + basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml" + filename := filepath.Join(dir, basename) + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.WriteString(f, filePrepender(filename)); err != nil { + return err + } + return GenYamlCustom(cmd, f) +} + +// GenYamlCustom creates custom yaml output +// nolint: gocyclo +func GenYamlCustom(cmd *cobra.Command, w io.Writer) error { + cliDoc := cmdDoc{ + Name: cmd.CommandPath(), + Aliases: strings.Join(cmd.Aliases, ", "), + Short: forceMultiLine(cmd.Short), + Long: forceMultiLine(cmd.Long), + Deprecated: len(cmd.Deprecated) > 0, + } + + if len(cliDoc.Long) == 0 { + cliDoc.Long = cliDoc.Short + } + + if cmd.Runnable() { + cliDoc.Usage = cmd.UseLine() + } + + if len(cmd.Example) > 0 { + cliDoc.Example = cmd.Example + } + + // Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack` + for curr := cmd; curr != nil; curr = curr.Parent() { + if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" { + cliDoc.MinAPIVersion = v + } + if _, ok := curr.Annotations["experimental"]; ok && !cliDoc.Experimental { + cliDoc.Experimental = true + } + if _, ok := curr.Annotations["experimentalCLI"]; ok && !cliDoc.ExperimentalCLI { + cliDoc.ExperimentalCLI = true + } + if _, ok := curr.Annotations["kubernetes"]; ok && !cliDoc.Kubernetes { + cliDoc.Kubernetes = true + } + if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm { + cliDoc.Swarm = true + } + if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" { + cliDoc.OSType = o + } + } + + var anchors = make(map[string]struct{}) + if a, ok := cmd.Annotations["anchors"]; ok && a != "" { + for _, anchor := range strings.Split(a, ",") { + anchors[anchor] = struct{}{} + } + } + + flags := cmd.NonInheritedFlags() + if flags.HasFlags() { + cliDoc.Options = genFlagResult(flags, anchors) + } + flags = cmd.InheritedFlags() + if flags.HasFlags() { + cliDoc.InheritedOptions = genFlagResult(flags, anchors) + } + + if hasSeeAlso(cmd) { + if cmd.HasParent() { + parent := cmd.Parent() + cliDoc.Pname = parent.CommandPath() + cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml" + cmd.VisitParents(func(c *cobra.Command) { + if c.DisableAutoGenTag { + cmd.DisableAutoGenTag = c.DisableAutoGenTag + } + }) + } + + children := cmd.Commands() + sort.Sort(byName(children)) + + for _, child := range children { + if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() { + continue + } + currentChild := cliDoc.Name + " " + child.Name() + cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name()) + link := currentChild + ".yaml" + cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1)) + } + } + + final, err := yaml.Marshal(&cliDoc) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + if _, err := fmt.Fprintln(w, string(final)); err != nil { + return err + } + return nil +} + +func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption { + var ( + result []cmdOption + opt cmdOption + ) + + flags.VisitAll(func(flag *pflag.Flag) { + opt = cmdOption{ + Option: flag.Name, + ValueType: flag.Value.Type(), + DefaultValue: forceMultiLine(flag.DefValue), + Description: forceMultiLine(flag.Usage), + Deprecated: len(flag.Deprecated) > 0, + } + + if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 { + opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com") + } else if _, ok = anchors[flag.Name]; ok { + opt.DetailsURL = "#" + flag.Name + } + + // Todo, when we mark a shorthand is deprecated, but specify an empty message. + // The flag.ShorthandDeprecated is empty as the shorthand is deprecated. + // Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok. + if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 { + opt.Shorthand = flag.Shorthand + } + if _, ok := flag.Annotations["experimental"]; ok { + opt.Experimental = true + } + if v, ok := flag.Annotations["version"]; ok { + opt.MinAPIVersion = v[0] + } + if _, ok := flag.Annotations["experimentalCLI"]; ok { + opt.ExperimentalCLI = true + } + if _, ok := flag.Annotations["kubernetes"]; ok { + opt.Kubernetes = true + } + if _, ok := flag.Annotations["swarm"]; ok { + opt.Swarm = true + } + + // Note that the annotation can have multiple ostypes set, however, multiple + // values are currently not used (and unlikely will). + // + // To simplify usage of the os_type property in the YAML, and for consistency + // with the same property for commands, we're only using the first ostype that's set. + if ostypes, ok := flag.Annotations["ostype"]; ok && len(opt.OSType) == 0 && len(ostypes) > 0 { + opt.OSType = ostypes[0] + } + + result = append(result, opt) + }) + + return result +} + +// Temporary workaround for yaml lib generating incorrect yaml with long strings +// that do not contain \n. +func forceMultiLine(s string) string { + s = strings.TrimSpace(s) + if len(s) > 60 && !strings.Contains(s, "\n") { + s = s + "\n" + } + return s +} + +// Small duplication for cobra utils +func hasSeeAlso(cmd *cobra.Command) bool { + if cmd.HasParent() { + return true + } + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { + continue + } + return true + } + return false +} + +// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the +// "Description" and "Examples" H2 sections in mdString (if present). +func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) { + sections := getSections(mdString) + var ( + anchors []string + md string + ) + if sections["description"] != "" { + md, anchors = cleanupMarkDown(sections["description"]) + cmd.Long = md + anchors = append(anchors, md) + } + if sections["examples"] != "" { + md, anchors = cleanupMarkDown(sections["examples"]) + cmd.Example = md + anchors = append(anchors, md) + } + if len(anchors) > 0 { + if cmd.Annotations == nil { + cmd.Annotations = make(map[string]string) + } + cmd.Annotations["anchors"] = strings.Join(anchors, ",") + } +} + +type byName []*cobra.Command + +func (s byName) Len() int { return len(s) } +func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() } diff --git a/go.mod b/go.mod index 55422bea..3b7459b4 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.20.1 k8s.io/apimachinery v0.20.1 k8s.io/client-go v0.20.1 diff --git a/hack/dockerfiles/yamldocs.Dockerfile b/hack/dockerfiles/yamldocs.Dockerfile new file mode 100644 index 00000000..6c1a4db1 --- /dev/null +++ b/hack/dockerfiles/yamldocs.Dockerfile @@ -0,0 +1,19 @@ +# syntax = docker/dockerfile:1.2 + +FROM golang:1.16-alpine AS yamlgen +WORKDIR /src +RUN --mount=target=. \ + --mount=target=/root/.cache,type=cache \ + go build -mod=vendor -o /out/yamlgen ./docs/yamlgen + +FROM alpine AS gen +RUN apk add --no-cache rsync git +WORKDIR /src +COPY --from=yamlgen /out/yamlgen /usr/bin +RUN --mount=target=/context \ + --mount=target=.,type=tmpfs,readwrite \ + rsync -a /context/. . \ + && yamlgen --target /out/yaml + +FROM scratch AS update +COPY --from=gen /out / diff --git a/hack/generate-yamldocs b/hack/generate-yamldocs new file mode 100755 index 00000000..2946371d --- /dev/null +++ b/hack/generate-yamldocs @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +. "$(dirname "$0")"/util +set -eu + +buildxCmd build \ + --output "type=local,dest=./bin/docs/" \ + --file "./hack/dockerfiles/yamldocs.Dockerfile" \ + --progress=plain \ + .