Add YAML-docs generator

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
pull/91/head
Sebastiaan van Stijn 6 years ago
parent 98d337af21
commit 9d9c027e2a
No known key found for this signature in database
GPG Key ID: 76698F39D527CE8C

@ -25,7 +25,7 @@ validate-vendor:
validate-docs:
./hack/validate-docs
validate-all: lint test validate-vendor validate-docs
vendor:
@ -34,7 +34,10 @@ vendor:
docs:
./hack/update-docs
yamldocs: ## generate documentation YAML files consumed by docs repo
./hack/generate-yamldocs
generate-authors:
./hack/generate-authors
.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs
.PHONY: vendor lint shell binaries install binaries-cross validate-all generate-authors validate-docs docs yamldocs

@ -0,0 +1,117 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"github.com/docker/buildx/commands"
"github.com/docker/cli/cli/command"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
const descriptionSourcePath = "docs/reference/"
func generateCliYaml(opts *options) error {
dockerCLI, err := command.NewDockerCli()
if err != nil {
return err
}
cmd := &cobra.Command{
Use: "docker [OPTIONS] COMMAND [ARG...]",
Short: "The base command for the Docker CLI.",
}
cmd.AddCommand(commands.NewRootCmd("buildx", true, dockerCLI))
disableFlagsInUseLine(cmd)
source := filepath.Join(opts.source, descriptionSourcePath)
fmt.Println("Markdown source:", source)
if err := loadLongDescription(cmd, source); err != nil {
return err
}
if err := os.MkdirAll(opts.target, 0755); err != nil {
return err
}
cmd.DisableAutoGenTag = true
return GenYamlTree(cmd, opts.target)
}
func disableFlagsInUseLine(cmd *cobra.Command) {
visitAll(cmd, func(ccmd *cobra.Command) {
// do not add a `[flags]` to the end of the usage line.
ccmd.DisableFlagsInUseLine = true
})
}
// visitAll will traverse all commands from the root.
// This is different from the VisitAll of cobra.Command where only parents
// are checked.
func visitAll(root *cobra.Command, fn func(*cobra.Command)) {
for _, cmd := range root.Commands() {
visitAll(cmd, fn)
}
fn(root)
}
func loadLongDescription(parentCmd *cobra.Command, path string) error {
for _, cmd := range parentCmd.Commands() {
if cmd.HasSubCommands() {
if err := loadLongDescription(cmd, path); err != nil {
return err
}
}
name := cmd.CommandPath()
log.Println("INFO: Generating docs for", name)
if i := strings.Index(name, " "); i >= 0 {
// remove root command / binary name
name = name[i+1:]
}
if name == "" {
continue
}
mdFile := strings.ReplaceAll(name, " ", "_") + ".md"
fullPath := filepath.Join(path, mdFile)
content, err := ioutil.ReadFile(fullPath)
if os.IsNotExist(err) {
log.Printf("WARN: %s does not exist, skipping\n", mdFile)
continue
}
if err != nil {
return err
}
applyDescriptionAndExamples(cmd, string(content))
}
return nil
}
type options struct {
source string
target string
}
func parseArgs() (*options, error) {
opts := &options{}
cwd, _ := os.Getwd()
flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
flags.StringVar(&opts.source, "root", cwd, "Path to project root")
flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files")
err := flags.Parse(os.Args[1:])
return opts, err
}
func main() {
opts, err := parseArgs()
if err != nil {
log.Println(err)
}
fmt.Println("Project root: ", opts.source)
fmt.Println("YAML output dir:", opts.target)
if err := generateCliYaml(opts); err != nil {
log.Println("Failed to generate yaml files:", err)
}
}

@ -0,0 +1,73 @@
package main
import (
"regexp"
"strings"
"unicode"
)
var (
// mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce
// false positives for (e.g.) comments in code-blocks (# this is a comment),
// so should not be used as a generic regex for other purposes.
mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`)
// htmlAnchor matches inline HTML anchors. This is intended to only match anchors
// for our use-case; DO NOT consider using this as a generic regex, or at least
// not before reading https://stackoverflow.com/a/1732454/1811501.
htmlAnchor = regexp.MustCompile(`<a\s+(?:name|id)="?([^"]+)"?\s*></a>\s*`)
)
// getSections returns all H2 sections by title (lowercase)
func getSections(mdString string) map[string]string {
parsedContent := strings.Split("\n"+mdString, "\n## ")
sections := make(map[string]string, len(parsedContent))
for _, s := range parsedContent {
if strings.HasPrefix(s, "#") {
// not a H2 Section
continue
}
parts := strings.SplitN(s, "\n", 2)
if len(parts) == 2 {
sections[strings.ToLower(parts[0])] = parts[1]
}
}
return sections
}
// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in
// YAML. It removes trailing whitespace and substitutes tabs for four spaces
// to prevent YAML switching to use "compact" form; ("line1 \nline\t2\n")
// which, although equivalent, is hard to read.
func cleanupMarkDown(mdString string) (md string, anchors []string) {
// remove leading/trailing whitespace, and replace tabs in the whole content
mdString = strings.TrimSpace(mdString)
mdString = strings.ReplaceAll(mdString, "\t", " ")
mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "")
var id string
// replace trailing whitespace per line, and handle custom anchors
lines := strings.Split(mdString, "\n")
for i := 0; i < len(lines); i++ {
lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace)
lines[i], id = convertHTMLAnchor(lines[i])
if id != "" {
anchors = append(anchors, id)
}
}
return strings.Join(lines, "\n"), anchors
}
// convertHTMLAnchor converts inline anchor-tags in headings (<a name=myanchor></a>)
// to an extended-markdown property ({#myanchor}). Extended Markdown properties
// are not supported in GitHub Flavored Markdown, but are supported by Jekyll,
// and lead to cleaner HTML in our docs, and prevents duplicate anchors.
// It returns the converted MarkDown heading and the custom ID (if present)
func convertHTMLAnchor(mdLine string) (md string, customID string) {
if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 {
if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 {
customID = a[1]
mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}"
}
}
return mdLine, customID
}

@ -0,0 +1,132 @@
package main
import "testing"
func TestCleanupMarkDown(t *testing.T) {
tests := []struct {
doc, in, expected string
}{
{
doc: "whitespace around sections",
in: `
## Section start
Some lines.
And more lines.
`,
expected: `## Section start
Some lines.
And more lines.`,
},
{
doc: "lines with inline tabs",
in: `## Some Heading
A line with tabs in it.
Tabs should be replaced by spaces`,
expected: `## Some Heading
A line with tabs in it.
Tabs should be replaced by spaces`,
},
{
doc: "lines with trailing spaces",
in: `## Some Heading with spaces
This is a line.
This is an indented line
### Some other heading
Last line.`,
expected: `## Some Heading with spaces
This is a line.
This is an indented line
### Some other heading
Last line.`,
},
{
doc: "lines with trailing tabs",
in: `## Some Heading with tabs
This is a line.
This is an indented line
### Some other heading
Last line.`,
expected: `## Some Heading with tabs
This is a line.
This is an indented line
### Some other heading
Last line.`,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.doc, func(t *testing.T) {
out, _ := cleanupMarkDown(tc.in)
if out != tc.expected {
t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out)
}
})
}
}
func TestConvertHTMLAnchor(t *testing.T) {
tests := []struct {
in, id, expected string
}{
{
in: `# <a name=heading1></a> Heading 1`,
id: "heading1",
expected: `# Heading 1 {#heading1}`,
},
{
in: `## Heading 2<a name=heading2></a> `,
id: "heading2",
expected: `## Heading 2 {#heading2}`,
},
{
in: `### <a id=heading3></a>Heading 3`,
id: "heading3",
expected: `### Heading 3 {#heading3}`,
},
{
in: `#### <a id="heading4"></a> Heading 4`,
id: "heading4",
expected: `#### Heading 4 {#heading4}`,
},
{
in: `##### <a id="heading5" ></a> Heading 5`,
id: "heading5",
expected: `##### Heading 5 {#heading5}`,
},
{
in: `###### <a id=hello href=foo>hello!</a>Heading 6`,
id: "",
expected: `###### <a id=hello href=foo>hello!</a>Heading 6`,
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.in, func(t *testing.T) {
out, id := convertHTMLAnchor(tc.in)
if id != tc.id {
t.Fatalf("expected: %s, actual: %s\n", tc.id, id)
}
if out != tc.expected {
t.Fatalf("\nexpected: %s\nactual: %s\n", tc.expected, out)
}
})
}
}

@ -0,0 +1,304 @@
package main
import (
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
yaml "gopkg.in/yaml.v2"
)
type cmdOption struct {
Option string
Shorthand string `yaml:",omitempty"`
ValueType string `yaml:"value_type,omitempty"`
DefaultValue string `yaml:"default_value,omitempty"`
Description string `yaml:",omitempty"`
// DetailsURL contains an anchor-id or link for more information on this flag
DetailsURL string `yaml:"details_url,omitempty"`
Deprecated bool
MinAPIVersion string `yaml:"min_api_version,omitempty"`
Experimental bool
ExperimentalCLI bool
Kubernetes bool
Swarm bool
OSType string `yaml:"os_type,omitempty"`
}
type cmdDoc struct {
Name string `yaml:"command"`
SeeAlso []string `yaml:"parent,omitempty"`
Version string `yaml:"engine_version,omitempty"`
Aliases string `yaml:",omitempty"`
Short string `yaml:",omitempty"`
Long string `yaml:",omitempty"`
Usage string `yaml:",omitempty"`
Pname string `yaml:",omitempty"`
Plink string `yaml:",omitempty"`
Cname []string `yaml:",omitempty"`
Clink []string `yaml:",omitempty"`
Options []cmdOption `yaml:",omitempty"`
InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"`
Example string `yaml:"examples,omitempty"`
Deprecated bool
MinAPIVersion string `yaml:"min_api_version,omitempty"`
Experimental bool
ExperimentalCLI bool
Kubernetes bool
Swarm bool
OSType string `yaml:"os_type,omitempty"`
}
// GenYamlTree creates yaml structured ref files
func GenYamlTree(cmd *cobra.Command, dir string) error {
emptyStr := func(s string) string { return "" }
return GenYamlTreeCustom(cmd, dir, emptyStr)
}
// GenYamlTreeCustom creates yaml structured ref files
func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error {
for _, c := range cmd.Commands() {
if !c.Runnable() && !c.HasAvailableSubCommands() {
// skip non-runnable commands without subcommands
// but *do* generate YAML for hidden and deprecated commands
// the YAML will have those included as metadata, so that the
// documentation repository can decide whether or not to present them
continue
}
if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil {
return err
}
}
if !cmd.HasParent() {
return nil
}
basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
filename := filepath.Join(dir, basename)
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
if _, err := io.WriteString(f, filePrepender(filename)); err != nil {
return err
}
return GenYamlCustom(cmd, f)
}
// GenYamlCustom creates custom yaml output
// nolint: gocyclo
func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
cliDoc := cmdDoc{
Name: cmd.CommandPath(),
Aliases: strings.Join(cmd.Aliases, ", "),
Short: forceMultiLine(cmd.Short),
Long: forceMultiLine(cmd.Long),
Deprecated: len(cmd.Deprecated) > 0,
}
if len(cliDoc.Long) == 0 {
cliDoc.Long = cliDoc.Short
}
if cmd.Runnable() {
cliDoc.Usage = cmd.UseLine()
}
if len(cmd.Example) > 0 {
cliDoc.Example = cmd.Example
}
// Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack`
for curr := cmd; curr != nil; curr = curr.Parent() {
if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" {
cliDoc.MinAPIVersion = v
}
if _, ok := curr.Annotations["experimental"]; ok && !cliDoc.Experimental {
cliDoc.Experimental = true
}
if _, ok := curr.Annotations["experimentalCLI"]; ok && !cliDoc.ExperimentalCLI {
cliDoc.ExperimentalCLI = true
}
if _, ok := curr.Annotations["kubernetes"]; ok && !cliDoc.Kubernetes {
cliDoc.Kubernetes = true
}
if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm {
cliDoc.Swarm = true
}
if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" {
cliDoc.OSType = o
}
}
var anchors = make(map[string]struct{})
if a, ok := cmd.Annotations["anchors"]; ok && a != "" {
for _, anchor := range strings.Split(a, ",") {
anchors[anchor] = struct{}{}
}
}
flags := cmd.NonInheritedFlags()
if flags.HasFlags() {
cliDoc.Options = genFlagResult(flags, anchors)
}
flags = cmd.InheritedFlags()
if flags.HasFlags() {
cliDoc.InheritedOptions = genFlagResult(flags, anchors)
}
if hasSeeAlso(cmd) {
if cmd.HasParent() {
parent := cmd.Parent()
cliDoc.Pname = parent.CommandPath()
cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml"
cmd.VisitParents(func(c *cobra.Command) {
if c.DisableAutoGenTag {
cmd.DisableAutoGenTag = c.DisableAutoGenTag
}
})
}
children := cmd.Commands()
sort.Sort(byName(children))
for _, child := range children {
if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() {
continue
}
currentChild := cliDoc.Name + " " + child.Name()
cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
link := currentChild + ".yaml"
cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1))
}
}
final, err := yaml.Marshal(&cliDoc)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if _, err := fmt.Fprintln(w, string(final)); err != nil {
return err
}
return nil
}
func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption {
var (
result []cmdOption
opt cmdOption
)
flags.VisitAll(func(flag *pflag.Flag) {
opt = cmdOption{
Option: flag.Name,
ValueType: flag.Value.Type(),
DefaultValue: forceMultiLine(flag.DefValue),
Description: forceMultiLine(flag.Usage),
Deprecated: len(flag.Deprecated) > 0,
}
if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 {
opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com")
} else if _, ok = anchors[flag.Name]; ok {
opt.DetailsURL = "#" + flag.Name
}
// Todo, when we mark a shorthand is deprecated, but specify an empty message.
// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
opt.Shorthand = flag.Shorthand
}
if _, ok := flag.Annotations["experimental"]; ok {
opt.Experimental = true
}
if v, ok := flag.Annotations["version"]; ok {
opt.MinAPIVersion = v[0]
}
if _, ok := flag.Annotations["experimentalCLI"]; ok {
opt.ExperimentalCLI = true
}
if _, ok := flag.Annotations["kubernetes"]; ok {
opt.Kubernetes = true
}
if _, ok := flag.Annotations["swarm"]; ok {
opt.Swarm = true
}
// Note that the annotation can have multiple ostypes set, however, multiple
// values are currently not used (and unlikely will).
//
// To simplify usage of the os_type property in the YAML, and for consistency
// with the same property for commands, we're only using the first ostype that's set.
if ostypes, ok := flag.Annotations["ostype"]; ok && len(opt.OSType) == 0 && len(ostypes) > 0 {
opt.OSType = ostypes[0]
}
result = append(result, opt)
})
return result
}
// Temporary workaround for yaml lib generating incorrect yaml with long strings
// that do not contain \n.
func forceMultiLine(s string) string {
s = strings.TrimSpace(s)
if len(s) > 60 && !strings.Contains(s, "\n") {
s = s + "\n"
}
return s
}
// Small duplication for cobra utils
func hasSeeAlso(cmd *cobra.Command) bool {
if cmd.HasParent() {
return true
}
for _, c := range cmd.Commands() {
if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
continue
}
return true
}
return false
}
// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the
// "Description" and "Examples" H2 sections in mdString (if present).
func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) {
sections := getSections(mdString)
var (
anchors []string
md string
)
if sections["description"] != "" {
md, anchors = cleanupMarkDown(sections["description"])
cmd.Long = md
anchors = append(anchors, md)
}
if sections["examples"] != "" {
md, anchors = cleanupMarkDown(sections["examples"])
cmd.Example = md
anchors = append(anchors, md)
}
if len(anchors) > 0 {
if cmd.Annotations == nil {
cmd.Annotations = make(map[string]string)
}
cmd.Annotations["anchors"] = strings.Join(anchors, ",")
}
}
type byName []*cobra.Command
func (s byName) Len() int { return len(s) }
func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() }

@ -54,6 +54,7 @@ require (
gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
gopkg.in/fatih/pool.v2 v2.0.0 // indirect
gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.20.1
k8s.io/apimachinery v0.20.1
k8s.io/client-go v0.20.1

@ -0,0 +1,19 @@
# syntax = docker/dockerfile:1.2
FROM golang:1.16-alpine AS yamlgen
WORKDIR /src
RUN --mount=target=. \
--mount=target=/root/.cache,type=cache \
go build -mod=vendor -o /out/yamlgen ./docs/yamlgen
FROM alpine AS gen
RUN apk add --no-cache rsync git
WORKDIR /src
COPY --from=yamlgen /out/yamlgen /usr/bin
RUN --mount=target=/context \
--mount=target=.,type=tmpfs,readwrite \
rsync -a /context/. . \
&& yamlgen --target /out/yaml
FROM scratch AS update
COPY --from=gen /out /

@ -0,0 +1,10 @@
#!/usr/bin/env bash
. "$(dirname "$0")"/util
set -eu
buildxCmd build \
--output "type=local,dest=./bin/docs/" \
--file "./hack/dockerfiles/yamldocs.Dockerfile" \
--progress=plain \
.
Loading…
Cancel
Save