diff --git a/cmd/wfctl/docs.go b/cmd/wfctl/docs.go index 22c3c8c3..119e993f 100644 --- a/cmd/wfctl/docs.go +++ b/cmd/wfctl/docs.go @@ -46,16 +46,30 @@ Examples: func runDocsGenerate(args []string) error { fs := flag.NewFlagSet("docs generate", flag.ContinueOnError) output := fs.String("output", "./docs/generated/", "Output directory for generated documentation") + out := fs.String("out", "", "Output directory for generated Go API documentation (alias for --output in API mode)") + source := fs.String("source", "", "Repository source directory for Go API documentation") + modulePath := fs.String("module", "", "Go module path for Go API documentation") + version := fs.String("version", "", "Released version or tag for generated Go API documentation") + packages := fs.String("packages", "", "Comma-separated package paths to include in Go API documentation") + registry := fs.String("registry", "", "Plugin registry JSON path or URL for plugin Go API documentation") + cacheDir := fs.String("cache-dir", "", "Directory used to cache cloned plugin repositories") + subjects := fs.String("subjects", "workflow", "Comma-separated API doc subjects to generate: workflow,plugins") + maxVersionLines := fs.Int("max-version-lines", 3, "Maximum released major/minor version lines to include in metadata") pluginDir := fs.String("plugin-dir", "", "Directory containing external plugin manifests (plugin.json)") title := fs.String("title", "", "Application title (default: derived from config)") fs.Usage = func() { - fmt.Fprintf(fs.Output(), `Usage: wfctl docs generate [options] + fmt.Fprintf(fs.Output(), `Usage: + wfctl docs generate [options] + wfctl docs generate --source --out --module --version --packages Generate Markdown documentation with Mermaid diagrams from a workflow configuration file. If -plugin-dir is specified, external plugin manifests (plugin.json) are loaded and described in the output. +When --source is provided, generate Go API reference Markdown from released +Workflow or plugin packages and write version metadata to versions.json. + Options: `) fs.PrintDefaults() @@ -65,6 +79,24 @@ Options: return err } + if *source != "" || *modulePath != "" || *packages != "" || *registry != "" { + outDir := *out + if outDir == "" { + outDir = *output + } + return runDocsGenerateAPI(docsGenerateAPIOptions{ + Source: *source, + Out: outDir, + Module: *modulePath, + Version: *version, + Packages: *packages, + Registry: *registry, + CacheDir: *cacheDir, + Subjects: *subjects, + MaxVersionLines: *maxVersionLines, + }) + } + if fs.NArg() < 1 { fs.Usage() return fmt.Errorf("config file path is required") diff --git a/cmd/wfctl/docs_generate.go b/cmd/wfctl/docs_generate.go new file mode 100644 index 00000000..7df8e90f --- /dev/null +++ b/cmd/wfctl/docs_generate.go @@ -0,0 +1,735 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "go/ast" + "go/doc" + "go/parser" + "go/printer" + "go/token" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "golang.org/x/mod/semver" +) + +type docsGenerateAPIOptions struct { + Source string + Out string + Module string + Version string + Packages string + Registry string + CacheDir string + Subjects string + MaxVersionLines int +} + +type docsAPIMetadata struct { + SchemaVersion int `json:"schemaVersion"` + GeneratedAt string `json:"generatedAt"` + Subject string `json:"subject"` + Subjects []string `json:"subjects"` + Versions map[string][]string `json:"versions"` + Packages []docsAPIPackageMeta `json:"packages"` + Warnings []string `json:"warnings"` +} + +type docsAPIPackageMeta struct { + Subject string `json:"subject"` + Name string `json:"name"` + ImportPath string `json:"importPath"` + Version string `json:"version"` + Path string `json:"path"` + Synopsis string `json:"synopsis,omitempty"` +} + +type goListPackage struct { + Dir string `json:"Dir"` + ImportPath string `json:"ImportPath"` + Name string `json:"Name"` + Doc string `json:"Doc"` + GoFiles []string `json:"GoFiles"` +} + +type renderedDocsPackage struct { + Meta docsAPIPackageMeta + Doc string + Warnings []string +} + +func runDocsGenerateAPI(opts docsGenerateAPIOptions) error { + if strings.TrimSpace(opts.Source) == "" { + return fmt.Errorf("docs generate: --source is required for Go API docs") + } + if strings.TrimSpace(opts.Out) == "" { + return fmt.Errorf("docs generate: --out is required for Go API docs") + } + if strings.TrimSpace(opts.Module) == "" { + return fmt.Errorf("docs generate: --module is required for Go API docs") + } + if strings.TrimSpace(opts.Version) == "" { + return fmt.Errorf("docs generate: --version is required for Go API docs") + } + subjects := splitDocsCSV(opts.Subjects) + if len(subjects) == 0 { + subjects = []string{"workflow"} + } + packages := splitDocsCSV(opts.Packages) + if docsContainsString(subjects, "workflow") && len(packages) == 0 { + return fmt.Errorf("docs generate: --packages must include at least one package") + } + if docsContainsString(subjects, "plugins") && strings.TrimSpace(opts.Registry) == "" { + return fmt.Errorf("docs generate: --registry is required when --subjects includes plugins") + } + + source, err := filepath.Abs(opts.Source) + if err != nil { + return fmt.Errorf("docs generate: resolve source: %w", err) + } + out, err := filepath.Abs(opts.Out) + if err != nil { + return fmt.Errorf("docs generate: resolve out: %w", err) + } + if err := os.MkdirAll(out, 0o750); err != nil { + return fmt.Errorf("docs generate: create output dir: %w", err) + } + + ctx := context.Background() + rendered := make([]renderedDocsPackage, 0, len(packages)) + var warnings []string + if docsContainsString(subjects, "workflow") { + for _, pkg := range packages { + docPkg, err := renderWorkflowAPIPackage(ctx, source, opts.Module, opts.Version, pkg) + if err != nil { + warnings = append(warnings, err.Error()) + continue + } + rendered = append(rendered, docPkg) + warnings = append(warnings, docPkg.Warnings...) + } + } + if docsContainsString(subjects, "plugins") { + pluginDocs, pluginWarnings, err := renderRegistryPluginAPIPackages(ctx, opts) + if err != nil { + return err + } + rendered = append(rendered, pluginDocs...) + warnings = append(warnings, pluginWarnings...) + } + if len(rendered) == 0 { + return fmt.Errorf("docs generate: no packages generated") + } + + for i := range rendered { + pkg := &rendered[i] + dest := filepath.Join(out, filepath.FromSlash(pkg.Meta.Path)) + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return fmt.Errorf("docs generate: create %s: %w", filepath.Dir(dest), err) + } + // #nosec G306 -- generated documentation artifacts are intended to be readable by tooling. + if err := os.WriteFile(dest, []byte(pkg.Doc), 0o644); err != nil { + return fmt.Errorf("docs generate: write %s: %w", dest, err) + } + fmt.Printf(" create %s\n", dest) + } + + meta := docsAPIMetadata{ + SchemaVersion: 1, + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Subject: subjects[0], + Subjects: subjects, + Versions: map[string][]string{}, + Packages: make([]docsAPIPackageMeta, 0, len(rendered)), + Warnings: warnings, + } + for i := range rendered { + pkg := &rendered[i] + meta.Packages = append(meta.Packages, pkg.Meta) + key := pkg.Meta.Subject + if pkg.Meta.Subject == "plugin" { + key = "plugins/" + pluginSlug(pkg.Meta.Name) + } + if pkg.Meta.Version != "" && !docsContainsString(meta.Versions[key], pkg.Meta.Version) { + meta.Versions[key] = append(meta.Versions[key], pkg.Meta.Version) + } + } + sort.Slice(meta.Packages, func(i, j int) bool { + return meta.Packages[i].ImportPath < meta.Packages[j].ImportPath + }) + limitDocsVersionLines(meta.Versions, opts.MaxVersionLines) + metaBytes, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return fmt.Errorf("docs generate: marshal metadata: %w", err) + } + metaPath := filepath.Join(out, "versions.json") + // #nosec G306 -- generated documentation metadata is intended to be readable by tooling. + if err := os.WriteFile(metaPath, append(metaBytes, '\n'), 0o644); err != nil { + return fmt.Errorf("docs generate: write versions.json: %w", err) + } + fmt.Printf(" create %s\n", metaPath) + fmt.Printf("\nGenerated %d Go API package doc(s) in %s\n", len(rendered), out) + return nil +} + +func renderWorkflowAPIPackage(ctx context.Context, source, modulePath, version, pkgRel string) (renderedDocsPackage, error) { + pkgRel = strings.Trim(strings.TrimSpace(pkgRel), "/") + if pkgRel == "" { + return renderedDocsPackage{}, fmt.Errorf("docs generate: empty package path") + } + listPkg, err := goListAPIPackage(ctx, source, modulePath, pkgRel) + if err != nil { + return renderedDocsPackage{}, fmt.Errorf("docs generate: go list %s: %w", pkgRel, err) + } + docPkg, fset, err := parseDocPackage(listPkg) + if err != nil { + return renderedDocsPackage{}, fmt.Errorf("docs generate: parse %s: %w", pkgRel, err) + } + route := "workflow/latest/" + strings.Trim(pkgRel, "/") + "/index.md" + synopsis := docPkg.Synopsis(docPkg.Doc) + if synopsis == "" { + synopsis = listPkg.Doc + } + meta := docsAPIPackageMeta{ + Subject: "workflow", + Name: docPkg.Name, + ImportPath: listPkg.ImportPath, + Version: version, + Path: route, + Synopsis: synopsis, + } + return renderedDocsPackage{ + Meta: meta, + Doc: renderPackageMarkdown(fset, docPkg, meta, workflowSourceLink(version, pkgRel), nil), + }, nil +} + +func goListAPIPackage(ctx context.Context, source, modulePath, pkgRel string) (goListPackage, error) { + importPath := strings.TrimRight(modulePath, "/") + if strings.Trim(pkgRel, "/") != "" && pkgRel != "." { + importPath += "/" + strings.Trim(pkgRel, "/") + } + cmd := exec.CommandContext(ctx, "go", "list", "-json", importPath) // #nosec G204 -- fixed go command with package arg from CLI input. + cmd.Dir = source + cmd.Env = append(os.Environ(), "GOWORK=off") + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return goListPackage{}, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + var pkg goListPackage + if err := json.Unmarshal(out, &pkg); err != nil { + return goListPackage{}, err + } + return pkg, nil +} + +func parseDocPackage(listPkg goListPackage) (*doc.Package, *token.FileSet, error) { + fset := token.NewFileSet() + files := make([]*ast.File, 0, len(listPkg.GoFiles)) + for _, name := range listPkg.GoFiles { + if strings.HasSuffix(name, "_test.go") { + continue + } + path := filepath.Join(listPkg.Dir, name) + file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return nil, nil, err + } + files = append(files, file) + } + if len(files) == 0 { + return nil, nil, fmt.Errorf("no Go package found in %s", listPkg.Dir) + } + docPkg, err := doc.NewFromFiles(fset, files, listPkg.ImportPath) + if err != nil { + return nil, nil, err + } + return docPkg, fset, nil +} + +func renderPackageMarkdown(fset *token.FileSet, pkg *doc.Package, meta docsAPIPackageMeta, sourceLink string, warnings []string) string { + var b strings.Builder + fmt.Fprintf(&b, "# package %s\n\n", pkg.Name) + fmt.Fprintf(&b, "Import path: `%s`\n\n", meta.ImportPath) + fmt.Fprintf(&b, "Version: `%s`\n\n", meta.Version) + fmt.Fprintf(&b, "Source: %s\n\n", sourceLink) + b.WriteString("## Warnings\n\n") + if len(warnings) == 0 { + b.WriteString("None\n\n") + } else { + for _, warning := range warnings { + fmt.Fprintf(&b, "- %s\n", warning) + } + b.WriteString("\n") + } + if strings.TrimSpace(pkg.Doc) != "" { + b.WriteString("## Synopsis\n\n") + b.WriteString(strings.TrimSpace(pkg.Doc)) + b.WriteString("\n\n") + } + renderValues(&b, fset, "Constants", pkg.Consts) + renderValues(&b, fset, "Variables", pkg.Vars) + renderFuncs(&b, fset, "Functions", pkg.Funcs) + renderTypes(&b, fset, pkg.Types) + return b.String() +} + +func renderValues(b *strings.Builder, fset *token.FileSet, heading string, values []*doc.Value) { + if len(values) == 0 { + return + } + fmt.Fprintf(b, "## %s\n\n", heading) + for _, value := range values { + if strings.TrimSpace(value.Doc) != "" { + b.WriteString(strings.TrimSpace(value.Doc)) + b.WriteString("\n\n") + } + writeDecl(b, fset, value.Decl) + } +} + +func renderFuncs(b *strings.Builder, fset *token.FileSet, heading string, funcs []*doc.Func) { + if len(funcs) == 0 { + return + } + fmt.Fprintf(b, "## %s\n\n", heading) + for _, fn := range funcs { + fmt.Fprintf(b, "### func %s\n\n", fn.Name) + if strings.TrimSpace(fn.Doc) != "" { + b.WriteString(strings.TrimSpace(fn.Doc)) + b.WriteString("\n\n") + } + writeDecl(b, fset, fn.Decl) + } +} + +func renderTypes(b *strings.Builder, fset *token.FileSet, types []*doc.Type) { + if len(types) == 0 { + return + } + b.WriteString("## Types\n\n") + for _, typ := range types { + fmt.Fprintf(b, "### type %s\n\n", typ.Name) + if strings.TrimSpace(typ.Doc) != "" { + b.WriteString(strings.TrimSpace(typ.Doc)) + b.WriteString("\n\n") + } + writeDecl(b, fset, typ.Decl) + renderFuncs(b, fset, "Functions", typ.Funcs) + renderFuncs(b, fset, "Methods", typ.Methods) + } +} + +func writeDecl(b *strings.Builder, fset *token.FileSet, decl ast.Node) { + var buf bytes.Buffer + if err := printer.Fprint(&buf, fset, decl); err != nil { + return + } + b.WriteString("```go\n") + b.WriteString(strings.TrimSpace(buf.String())) + b.WriteString("\n```\n\n") +} + +func workflowSourceLink(version, pkgRel string) string { + ref := strings.TrimSpace(version) + if ref == "" || ref == "latest" { + ref = "main" + } + return "https://github.com/GoCodeAlone/workflow/tree/" + ref + "/" + strings.Trim(pkgRel, "/") +} + +func renderRegistryPluginAPIPackages(ctx context.Context, opts docsGenerateAPIOptions) ([]renderedDocsPackage, []string, error) { + manifests, err := loadDocsRegistry(ctx, opts.Registry) + if err != nil { + return nil, nil, fmt.Errorf("docs generate: load registry: %w", err) + } + cacheDir := opts.CacheDir + if cacheDir == "" { + cacheDir = filepath.Join(os.TempDir(), "wfctl-docs-plugin-cache") + } + if err := os.MkdirAll(cacheDir, 0o750); err != nil { + return nil, nil, fmt.Errorf("docs generate: create cache dir: %w", err) + } + var rendered []renderedDocsPackage + var warnings []string + allowLocalPluginSources := docsRegistryLocalSource(opts.Registry) + for i := range manifests { + manifest := &manifests[i] + if strings.TrimSpace(manifest.Name) == "" { + warnings = append(warnings, "plugin registry entry missing name") + continue + } + if !trustedGoCodeAloneRepo(manifest.Repository) { + warnings = append(warnings, fmt.Sprintf("%s skipped: repository %q is outside the GoCodeAlone GitHub trust boundary", manifest.Name, manifest.Repository)) + continue + } + if strings.TrimSpace(manifest.Version) == "" { + warnings = append(warnings, fmt.Sprintf("%s skipped: missing version", manifest.Name)) + continue + } + if _, err := safeDocsPluginSlug(manifest.Name); err != nil { + warnings = append(warnings, fmt.Sprintf("%s %s skipped: %v", manifest.Name, manifest.Version, err)) + continue + } + checkout, err := checkoutDocsPluginRepo(ctx, manifest, cacheDir, allowLocalPluginSources) + if err != nil { + warnings = append(warnings, fmt.Sprintf("%s %s skipped: %v", manifest.Name, manifest.Version, err)) + continue + } + modulePath, err := readGoModulePath(filepath.Join(checkout, "go.mod")) + if err != nil { + warnings = append(warnings, fmt.Sprintf("%s %s skipped: %v", manifest.Name, manifest.Version, err)) + continue + } + docPkg, err := renderPluginAPIPackage(ctx, checkout, modulePath, manifest) + if err != nil { + warnings = append(warnings, fmt.Sprintf("%s %s skipped: %v", manifest.Name, manifest.Version, err)) + continue + } + rendered = append(rendered, docPkg) + } + return rendered, warnings, nil +} + +func loadDocsRegistry(ctx context.Context, ref string) ([]RegistryManifest, error) { + var data []byte + var err error + if strings.HasPrefix(ref, "https://") || strings.HasPrefix(ref, "http://") { + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ref, nil) + if reqErr != nil { + return nil, reqErr + } + client := &http.Client{Timeout: 30 * time.Second} + resp, httpErr := client.Do(req) + if httpErr != nil { + return nil, httpErr + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned HTTP %d", resp.StatusCode) + } + data, err = io.ReadAll(resp.Body) + } else { + data, err = os.ReadFile(ref) + } + if err != nil { + return nil, err + } + var envelope struct { + Plugins []RegistryManifest `json:"plugins"` + } + if err := json.Unmarshal(data, &envelope); err == nil && envelope.Plugins != nil { + return envelope.Plugins, nil + } + var manifests []RegistryManifest + if err := json.Unmarshal(data, &manifests); err != nil { + return nil, err + } + return manifests, nil +} + +func checkoutDocsPluginRepo(ctx context.Context, manifest *RegistryManifest, cacheDir string, allowLocalSource bool) (string, error) { + slug, err := safeDocsPluginSlug(manifest.Name) + if err != nil { + return "", err + } + dest, err := docsPluginCacheDestination(cacheDir, slug) + if err != nil { + return "", err + } + cloneSource, err := docsPluginCloneSource(manifest, allowLocalSource) + if err != nil { + return "", err + } + if docsPluginRepoExists(dest) { + if err := refreshDocsPluginRepo(ctx, dest, cloneSource, manifest.Version); err == nil { + return dest, nil + } + } + if err := os.RemoveAll(dest); err != nil { + return "", err + } + args := []string{"clone", "--depth", "1", "--branch", manifest.Version, cloneSource, dest} + cmd := exec.CommandContext(ctx, "git", args...) // #nosec G204 -- fixed git command; args are not shell-expanded. + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git clone tag %s: %w: %s", manifest.Version, err, strings.TrimSpace(stderr.String())) + } + return dest, nil +} + +func renderPluginAPIPackage(ctx context.Context, checkout, modulePath string, manifest *RegistryManifest) (renderedDocsPackage, error) { + listPkg, err := goListAPIPackage(ctx, checkout, modulePath, ".") + if err != nil { + return renderedDocsPackage{}, err + } + docPkg, fset, err := parseDocPackage(listPkg) + if err != nil { + return renderedDocsPackage{}, err + } + slug, err := safeDocsPluginSlug(manifest.Name) + if err != nil { + return renderedDocsPackage{}, err + } + route := "plugins/" + slug + "/latest/index.md" + synopsis := docPkg.Synopsis(docPkg.Doc) + if synopsis == "" { + synopsis = listPkg.Doc + } + meta := docsAPIPackageMeta{ + Subject: "plugin", + Name: manifest.Name, + ImportPath: listPkg.ImportPath, + Version: manifest.Version, + Path: route, + Synopsis: synopsis, + } + return renderedDocsPackage{ + Meta: meta, + Doc: renderPackageMarkdown(fset, docPkg, meta, pluginSourceLink(manifest.Repository, manifest.Version), nil), + }, nil +} + +func readGoModulePath(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "module" { + return fields[1], nil + } + } + return "", fmt.Errorf("module path not found in %s", path) +} + +func trustedGoCodeAloneRepo(repo string) bool { + parsed, err := url.Parse(strings.TrimSpace(repo)) + if err != nil { + return false + } + if parsed.Scheme != "https" || parsed.Host != "github.com" { + return false + } + segments := strings.Split(strings.Trim(parsed.Path, "/"), "/") + if len(segments) < 2 || segments[0] != "GoCodeAlone" { + return false + } + for _, segment := range segments { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} + +func pluginSlug(name string) string { + name = strings.TrimSpace(name) + name = strings.TrimSuffix(name, ".git") + name = strings.TrimPrefix(name, "workflow-plugin-") + if idx := strings.LastIndex(name, "/"); idx >= 0 { + name = name[idx+1:] + name = strings.TrimPrefix(name, "workflow-plugin-") + } + return name +} + +func safeDocsPluginSlug(name string) (string, error) { + slug := pluginSlug(name) + if slug == "" || slug == "." || slug == ".." { + return "", fmt.Errorf("invalid plugin slug %q", slug) + } + if strings.ContainsAny(slug, `/\`) { + return "", fmt.Errorf("invalid plugin slug %q", slug) + } + for i, r := range slug { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + continue + } + if i > 0 && (r == '-' || r == '_' || r == '.') { + continue + } + return "", fmt.Errorf("invalid plugin slug %q", slug) + } + return slug, nil +} + +func docsPluginCacheDestination(cacheDir, slug string) (string, error) { + absCacheDir, err := filepath.Abs(cacheDir) + if err != nil { + return "", err + } + dest := filepath.Join(absCacheDir, slug) + rel, err := filepath.Rel(absCacheDir, dest) + if err != nil { + return "", err + } + if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("invalid plugin cache destination %q", dest) + } + return dest, nil +} + +func docsPluginRepoExists(dest string) bool { + info, err := os.Stat(filepath.Join(dest, ".git")) + return err == nil && info.IsDir() +} + +func refreshDocsPluginRepo(ctx context.Context, dest, cloneSource, version string) error { + remote, err := runDocsGit(ctx, dest, "remote", "get-url", "origin") + if err != nil { + return err + } + if strings.TrimSpace(remote) != cloneSource { + return fmt.Errorf("cached repository remote %q does not match %q", strings.TrimSpace(remote), cloneSource) + } + tagRef := "refs/tags/" + version + ":refs/tags/" + version + if _, err := runDocsGit(ctx, dest, "fetch", "--depth", "1", "--force", "origin", tagRef); err != nil { + return err + } + if _, err := runDocsGit(ctx, dest, "checkout", "--detach", version); err != nil { + return err + } + if _, err := runDocsGit(ctx, dest, "reset", "--hard"); err != nil { + return err + } + if _, err := runDocsGit(ctx, dest, "clean", "-fdx"); err != nil { + return err + } + return nil +} + +func runDocsGit(ctx context.Context, dir string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "git", args...) // #nosec G204 -- fixed git command; args are not shell-expanded. + cmd.Dir = dir + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return string(out), nil +} + +func docsPluginCloneSource(manifest *RegistryManifest, allowLocalSource bool) (string, error) { + source := strings.TrimSpace(manifest.Source) + if source == "" { + source = strings.TrimSpace(manifest.Repository) + } + if docsPluginLocalCloneSource(source) { + if allowLocalSource { + return source, nil + } + return "", fmt.Errorf("source %q is outside the GoCodeAlone GitHub trust boundary", source) + } + if trustedGoCodeAloneRepo(source) { + return source, nil + } + return "", fmt.Errorf("source %q is outside the GoCodeAlone GitHub trust boundary", source) +} + +func docsRegistryLocalSource(registry string) bool { + registry = strings.TrimSpace(registry) + return !strings.HasPrefix(registry, "https://") && !strings.HasPrefix(registry, "http://") +} + +func docsPluginLocalCloneSource(source string) bool { + source = strings.TrimSpace(source) + if source == "" { + return false + } + if filepath.IsAbs(source) || strings.HasPrefix(source, ".") { + return true + } + if strings.Contains(source, "://") || strings.HasPrefix(source, "git@") || strings.Contains(source, ":") { + return false + } + return true +} + +func limitDocsVersionLines(versions map[string][]string, maxLines int) { + for key, values := range versions { + values = uniqueDocsVersions(values) + sort.SliceStable(values, func(i, j int) bool { + return compareDocsVersions(values[i], values[j]) > 0 + }) + if maxLines > 0 && len(values) > maxLines { + values = values[:maxLines] + } + versions[key] = values + } +} + +func uniqueDocsVersions(values []string) []string { + seen := make(map[string]struct{}, len(values)) + unique := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + unique = append(unique, value) + } + return unique +} + +func compareDocsVersions(left, right string) int { + leftValid := semver.IsValid(left) + rightValid := semver.IsValid(right) + if leftValid && rightValid { + return semver.Compare(left, right) + } + if leftValid { + return 1 + } + if rightValid { + return -1 + } + return strings.Compare(left, right) +} + +func pluginSourceLink(repository, version string) string { + repository = strings.TrimSuffix(strings.TrimSpace(repository), ".git") + if repository == "" { + return "" + } + return repository + "/tree/" + version +} + +func docsContainsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func splitDocsCSV(raw string) []string { + var out []string + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} diff --git a/cmd/wfctl/docs_generate_test.go b/cmd/wfctl/docs_generate_test.go new file mode 100644 index 00000000..4b0a4e02 --- /dev/null +++ b/cmd/wfctl/docs_generate_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "go/doc" + "go/token" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestDocsGenerateHelpIncludesAPIDocFlags(t *testing.T) { + out, err := captureStderr(t, func() error { + return runDocsGenerate([]string{"--help"}) + }) + if !errors.Is(err, flag.ErrHelp) { + t.Fatalf("expected flag.ErrHelp, got %v", err) + } + for _, want := range []string{"--source", "--out", "--module", "--version", "--packages", "--registry", "--subjects"} { + if !strings.Contains(out, strings.TrimPrefix(want, "--")) { + t.Fatalf("help output missing %s:\n%s", want, out) + } + } +} + +func TestDocsGenerateWorkflowPackages(t *testing.T) { + repoRoot, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + outDir := t.TempDir() + _, err = captureStdout(t, func() error { + return runDocsGenerate([]string{ + "--source", repoRoot, + "--out", outDir, + "--module", "github.com/GoCodeAlone/workflow", + "--version", "v0.75.0", + "--packages", "plugin,plugin/sdk,plugin/external/sdk", + }) + }) + if err != nil { + t.Fatalf("docs generate: %v", err) + } + + metaPath := filepath.Join(outDir, "versions.json") + rawMeta, err := os.ReadFile(metaPath) + if err != nil { + t.Fatalf("read versions.json: %v", err) + } + var meta struct { + SchemaVersion int `json:"schemaVersion"` + Subject string + Versions map[string][]string `json:"versions"` + Packages []struct { + Subject string `json:"subject"` + ImportPath string `json:"importPath"` + Version string `json:"version"` + Path string `json:"path"` + Synopsis string `json:"synopsis"` + } `json:"packages"` + Warnings []string `json:"warnings"` + } + if err := json.Unmarshal(rawMeta, &meta); err != nil { + t.Fatalf("parse versions.json: %v", err) + } + if meta.SchemaVersion != 1 { + t.Fatalf("schemaVersion = %d, want 1", meta.SchemaVersion) + } + if got := meta.Versions["workflow"]; len(got) != 1 || got[0] != "v0.75.0" { + t.Fatalf("workflow versions = %v, want [v0.75.0]", got) + } + if len(meta.Packages) != 3 { + t.Fatalf("packages = %d, want 3 (%+v)", len(meta.Packages), meta.Packages) + } + if len(meta.Warnings) != 0 { + t.Fatalf("warnings = %v, want none", meta.Warnings) + } + + docPath := filepath.Join(outDir, "workflow", "latest", "plugin", "index.md") + if mode := statFileMode(t, docPath); mode != 0o644 { + t.Fatalf("generated doc mode = %04o, want 0644", mode) + } + if mode := statFileMode(t, metaPath); mode != 0o644 { + t.Fatalf("versions.json mode = %04o, want 0644", mode) + } + rawDoc, err := os.ReadFile(docPath) + if err != nil { + t.Fatalf("read generated plugin doc: %v", err) + } + doc := string(rawDoc) + for _, want := range []string{ + "# package plugin", + "Import path: `github.com/GoCodeAlone/workflow/plugin`", + "Version: `v0.75.0`", + "https://github.com/GoCodeAlone/workflow/tree/v0.75.0/plugin", + "## Types", + } { + if !strings.Contains(doc, want) { + t.Fatalf("generated doc missing %q:\n%s", want, doc) + } + } +} + +func TestDocsGenerateRegistryPlugins(t *testing.T) { + pluginRepo := createDocsPluginRepo(t, "github.com/GoCodeAlone/workflow-plugin-alpha", "alpha", "v0.1.0") + registryPath := filepath.Join(t.TempDir(), "registry.json") + registry := `{ + "plugins": [ + { + "name": "workflow-plugin-alpha", + "version": "v0.1.0", + "repository": "https://github.com/GoCodeAlone/workflow-plugin-alpha", + "source": ` + strconvQuote(pluginRepo) + ` + }, + { + "name": "workflow-plugin-missing", + "version": "v0.2.0", + "repository": "https://github.com/GoCodeAlone/workflow-plugin-missing", + "source": ` + strconvQuote(pluginRepo) + ` + }, + { + "name": "workflow-plugin-outside", + "version": "v0.1.0", + "repository": "https://github.com/Other/workflow-plugin-outside", + "source": ` + strconvQuote(pluginRepo) + ` + }, + { + "name": "workflow-plugin-..", + "version": "v0.1.0", + "repository": "https://github.com/GoCodeAlone/workflow-plugin-dotdot", + "source": ` + strconvQuote(pluginRepo) + ` + }, + { + "name": "workflow-plugin-untrusted-source", + "version": "v0.1.0", + "repository": "https://github.com/GoCodeAlone/workflow-plugin-untrusted-source", + "source": "https://github.com/Other/workflow-plugin-untrusted-source" + } + ] +}` + if err := os.WriteFile(registryPath, []byte(registry), 0o600); err != nil { + t.Fatal(err) + } + + outDir := t.TempDir() + cacheDir := filepath.Join(t.TempDir(), "cache") + _, err := captureStdout(t, func() error { + return runDocsGenerate([]string{ + "--source", ".", + "--out", outDir, + "--module", "github.com/GoCodeAlone/workflow", + "--version", "v0.75.0", + "--registry", registryPath, + "--cache-dir", cacheDir, + "--subjects", "plugins", + }) + }) + if err != nil { + t.Fatalf("docs generate registry plugins: %v", err) + } + + docPath := filepath.Join(outDir, "plugins", "alpha", "latest", "index.md") + rawDoc, err := os.ReadFile(docPath) + if err != nil { + t.Fatalf("read generated plugin doc: %v", err) + } + doc := string(rawDoc) + for _, want := range []string{ + "# package alpha", + "Import path: `github.com/GoCodeAlone/workflow-plugin-alpha`", + "Version: `v0.1.0`", + "https://github.com/GoCodeAlone/workflow-plugin-alpha/tree/v0.1.0", + "## Functions", + } { + if !strings.Contains(doc, want) { + t.Fatalf("generated plugin doc missing %q:\n%s", want, doc) + } + } + + rawMeta, err := os.ReadFile(filepath.Join(outDir, "versions.json")) + if err != nil { + t.Fatal(err) + } + var meta docsAPIMetadata + if err := json.Unmarshal(rawMeta, &meta); err != nil { + t.Fatal(err) + } + if got := meta.Versions["plugins/alpha"]; len(got) != 1 || got[0] != "v0.1.0" { + t.Fatalf("plugins/alpha versions = %v, want [v0.1.0]", got) + } + if len(meta.Packages) != 1 || meta.Packages[0].Path != "plugins/alpha/latest/index.md" { + t.Fatalf("packages = %+v, want generated alpha plugin route", meta.Packages) + } + joinedWarnings := strings.Join(meta.Warnings, "\n") + for _, want := range []string{"workflow-plugin-missing", "v0.2.0", "workflow-plugin-outside", "trust boundary", "invalid plugin slug", "workflow-plugin-untrusted-source"} { + if !strings.Contains(joinedWarnings, want) { + t.Fatalf("warnings missing %q:\n%s", want, joinedWarnings) + } + } +} + +func TestLimitDocsVersionLines(t *testing.T) { + versions := map[string][]string{ + "workflow": { + "v0.74.3", + "v0.74.2", + "v0.75.0", + "v1.0.0", + "v0.73.9", + "v0.75.0", + }, + } + + limitDocsVersionLines(versions, 2) + + want := []string{"v1.0.0", "v0.75.0"} + got := versions["workflow"] + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("workflow versions = %v, want %v", got, want) + } +} + +func TestDocsPluginCloneSourceRejectsLocalSourceFromRemoteRegistry(t *testing.T) { + manifest := &RegistryManifest{ + Name: "workflow-plugin-alpha", + Repository: "https://github.com/GoCodeAlone/workflow-plugin-alpha", + Source: "../workflow-plugin-alpha", + } + + if _, err := docsPluginCloneSource(manifest, false); err == nil { + t.Fatal("expected remote registry local source to be rejected") + } + + source, err := docsPluginCloneSource(manifest, true) + if err != nil { + t.Fatalf("local registry source rejected: %v", err) + } + if source != manifest.Source { + t.Fatalf("source = %q, want %q", source, manifest.Source) + } +} + +func TestTrustedGoCodeAloneRepoRejectsDotSegments(t *testing.T) { + for _, repo := range []string{ + "https://github.com/GoCodeAlone/../Other/workflow-plugin-alpha", + "https://github.com/GoCodeAlone/%2e%2e/Other/workflow-plugin-alpha", + "https://github.com/GoCodeAlone/./workflow-plugin-alpha", + } { + if trustedGoCodeAloneRepo(repo) { + t.Fatalf("trustedGoCodeAloneRepo(%q) = true, want false", repo) + } + } + if !trustedGoCodeAloneRepo("https://github.com/GoCodeAlone/workflow-plugin-alpha") { + t.Fatal("expected normal GoCodeAlone HTTPS repo to be trusted") + } +} + +func TestCheckoutDocsPluginRepoReusesCache(t *testing.T) { + pluginRepo := createDocsPluginRepo(t, "github.com/GoCodeAlone/workflow-plugin-alpha", "alpha", "v0.1.0") + cacheDir := t.TempDir() + manifest := &RegistryManifest{ + Name: "workflow-plugin-alpha", + Version: "v0.1.0", + Repository: "https://github.com/GoCodeAlone/workflow-plugin-alpha", + Source: pluginRepo, + } + + checkout, err := checkoutDocsPluginRepo(context.Background(), manifest, cacheDir, true) + if err != nil { + t.Fatalf("first checkout: %v", err) + } + sentinel := filepath.Join(checkout, ".git", "wfctl-cache-sentinel") + if err := os.WriteFile(sentinel, []byte("cached"), 0o600); err != nil { + t.Fatalf("write sentinel: %v", err) + } + + secondCheckout, err := checkoutDocsPluginRepo(context.Background(), manifest, cacheDir, true) + if err != nil { + t.Fatalf("second checkout: %v", err) + } + if secondCheckout != checkout { + t.Fatalf("checkout path = %q, want %q", secondCheckout, checkout) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("cache sentinel was removed: %v", err) + } +} + +func TestRenderPackageMarkdownWarnings(t *testing.T) { + rendered := renderPackageMarkdown(token.NewFileSet(), &doc.Package{Name: "alpha"}, docsAPIPackageMeta{ + ImportPath: "github.com/GoCodeAlone/workflow-plugin-alpha", + Version: "v0.1.0", + }, "https://github.com/GoCodeAlone/workflow-plugin-alpha/tree/v0.1.0", []string{"missing package docs"}) + + if !strings.Contains(rendered, "- missing package docs") { + t.Fatalf("rendered doc missing warning:\n%s", rendered) + } + if strings.Contains(rendered, "## Warnings\n\nNone") { + t.Fatalf("rendered doc still claims warnings are none:\n%s", rendered) + } +} + +func createDocsPluginRepo(t *testing.T, modulePath, packageName, tag string) string { + t.Helper() + dir := t.TempDir() + write := func(rel, content string) { + t.Helper() + path := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + write("go.mod", "module "+modulePath+"\n\ngo 1.26\n") + write(packageName+".go", "package "+packageName+"\n\n// PluginName returns the fixture plugin name.\nfunc PluginName() string { return "+strconvQuote(packageName)+" }\n") + runDocsTestGit(t, dir, "init") + runDocsTestGit(t, dir, "config", "user.email", "docs@example.test") + runDocsTestGit(t, dir, "config", "user.name", "Docs Test") + runDocsTestGit(t, dir, "add", ".") + runDocsTestGit(t, dir, "commit", "-m", "initial") + runDocsTestGit(t, dir, "tag", tag) + return dir +} + +func runDocsTestGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +func strconvQuote(s string) string { + b, _ := json.Marshal(s) + return string(b) +} + +func statFileMode(t *testing.T, path string) os.FileMode { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + return info.Mode().Perm() +} diff --git a/cmd/wfctl/testdata/docs-registry.json b/cmd/wfctl/testdata/docs-registry.json new file mode 100644 index 00000000..a5b818f5 --- /dev/null +++ b/cmd/wfctl/testdata/docs-registry.json @@ -0,0 +1,3 @@ +{ + "plugins": [] +} diff --git a/docs/WFCTL.md b/docs/WFCTL.md index 212c002b..0182e634 100644 --- a/docs/WFCTL.md +++ b/docs/WFCTL.md @@ -1944,19 +1944,29 @@ WFCTL_CONFIRM_PRUNE=1 wfctl infra rotate-and-prune \ ### `docs generate` -Generate Markdown documentation with Mermaid diagrams from a workflow configuration file. Produces a set of `.md` files describing modules, pipelines, workflows, external plugins, and system architecture. +Generate Markdown documentation. With a config file, this produces Mermaid-backed application docs for modules, pipelines, workflows, external plugins, and system architecture. With `--source`, this produces Go API reference Markdown and `versions.json` metadata for released Workflow and plugin packages. ``` wfctl docs generate [options] +wfctl docs generate --source --out --module --version --packages ``` | Flag | Default | Description | |------|---------|-------------| | `-output` | `./docs/generated/` | Output directory for generated documentation | +| `-out` | _(same as `-output`)_ | Output directory for Go API docs | +| `-source` | _(none)_ | Repository source directory for Go API docs | +| `-module` | _(none)_ | Go module path for Go API docs | +| `-version` | _(none)_ | Released version/tag represented by the generated docs | +| `-packages` | _(none)_ | Comma-separated package paths to include | +| `-registry` | _(none)_ | Plugin registry JSON path or URL for plugin API docs | +| `-cache-dir` | _(none)_ | Directory for cached plugin repository checkouts | +| `-subjects` | `workflow` | API doc subjects to generate: `workflow`, `plugins` | +| `-max-version-lines` | `3` | Maximum released version lines to include in metadata | | `-plugin-dir` | _(none)_ | Directory containing external plugin manifests (`plugin.json`) | | `-title` | _(derived from config filename)_ | Application title used in the README | -**Generated files:** +**Generated config documentation files:** | File | Description | |------|-------------| @@ -1974,8 +1984,15 @@ wfctl docs generate workflow.yaml wfctl docs generate -output ./docs/ workflow.yaml wfctl docs generate -output ./docs/ -plugin-dir ./plugins/ workflow.yaml wfctl docs generate -output ./docs/ -title "Order Service" workflow.yaml +wfctl docs generate --source . --out ./docs/api --module github.com/GoCodeAlone/workflow --version v0.75.0 --packages plugin,plugin/sdk,plugin/external/sdk +wfctl docs generate --source . --out ./docs/api --module github.com/GoCodeAlone/workflow --version v0.75.0 --subjects plugins --registry ./registry.json --cache-dir ./.cache/wfctl-docs ``` +Plugin API generation checks out each trusted `https://github.com/GoCodeAlone/*` +repository at the exact registry version tag. Missing tags and repositories +outside that trust boundary are recorded as warnings in `versions.json` without +failing the whole docs run. + --- ### `api extract` diff --git a/plugins/http/modules.go b/plugins/http/modules.go index 79ab78ff..cee770ea 100644 --- a/plugins/http/modules.go +++ b/plugins/http/modules.go @@ -77,16 +77,28 @@ func normalizeListenPort(v any) (int, bool) { case int16: return validListenPort(int(p)) case int32: + if p > 65535 { + return 0, false + } return validListenPort(int(p)) case int64: + if p > 65535 { + return 0, false + } return validListenPort(int(p)) case uint: + if p > 65535 { + return 0, false + } return validListenPort(int(p)) case uint8: return validListenPort(int(p)) case uint16: return validListenPort(int(p)) case uint32: + if p > 65535 { + return 0, false + } return validListenPort(int(p)) case uint64: if p > 65535 {