From af223418efb1a749a4e3d525a292e890e3103b3f Mon Sep 17 00:00:00 2001 From: samzong Date: Wed, 1 Jul 2026 14:01:34 -0400 Subject: [PATCH] feat(lathe): add __lathe verify for generated CLI contract validation Add a hidden __lathe verify --json command that checks root help, catalog schema, commands show parity, required flags, and auth status behavior without network calls. Support silent exit codes for verify failures. ## Considered and deferred - pkg/lathe/verify.go:54 [BOT-NIT]: --json flag is documented for agents but verify always emits JSON; flag exists for contract discoverability only. - pkg/lathe/verify.go:171 [BOT-TASTE]: isJSONBody treats empty media type as JSON, matching OpenAPI default body semantics. Signed-off-by: samzong --- README.md | 1 + docs/cli-usage.md | 3 + pkg/lathe/lathe.go | 7 +- pkg/lathe/verify.go | 247 +++++++++++++++++++++++++++++++++++++ pkg/lathe/verify_test.go | 142 +++++++++++++++++++++ pkg/runtime/errors.go | 8 ++ pkg/runtime/errors_test.go | 24 ++++ 7 files changed, 429 insertions(+), 3 deletions(-) create mode 100644 pkg/lathe/verify.go create mode 100644 pkg/lathe/verify_test.go diff --git a/README.md b/README.md index 63a56db..44e24d4 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,7 @@ Lathe runtime commands that should not occupy generated API command names live u | Command | Effect | |---|---| +| ` __lathe verify --json` | Verify the generated CLI contract without making network calls. | | ` __lathe version` | Print version information. | | ` __lathe completion ` | Generate shell completion scripts. | diff --git a/docs/cli-usage.md b/docs/cli-usage.md index 67db937..b3a068e 100644 --- a/docs/cli-usage.md +++ b/docs/cli-usage.md @@ -325,6 +325,7 @@ go build -o bin/acmectl ./cmd/acmectl Generated CLIs expose machine-readable contracts. Agents should use this loop: ```sh +bin/acmectl __lathe verify --json bin/acmectl search "create user" --json bin/acmectl commands show users users create --json bin/acmectl commands schema --json @@ -336,6 +337,8 @@ bin/acmectl users users create --set email=alice@example.com -o json Rules: - Treat `search` output as candidates only. +- Run `__lathe verify --json` after building a generated CLI to check the local + command contract before live calls. - Inspect exact command details with `commands show` before execution. - Use `examples` from command detail when overlays provide runnable command metadata. - Run `auth status --hostname ` before authenticated commands. diff --git a/pkg/lathe/lathe.go b/pkg/lathe/lathe.go index b2efb64..7baaebe 100644 --- a/pkg/lathe/lathe.go +++ b/pkg/lathe/lathe.go @@ -54,19 +54,20 @@ func NewApp(m *config.Manifest) *cobra.Command { if m.Update.GitHub != nil { cmd.AddCommand(updateCmd(m)) } - cmd.AddCommand(metaCmd(m.CLI.Name)) + cmd.AddCommand(metaCmd(m)) return cmd } -func metaCmd(cliName string) *cobra.Command { +func metaCmd(m *config.Manifest) *cobra.Command { cmd := &cobra.Command{ Use: metaCommandName, Short: "Lathe control commands", Hidden: true, } cmd.AddCommand(versionCmd()) + cmd.AddCommand(verifyCmd(m)) cmd.InitDefaultCompletionCmd() - rewriteCompletionHelp(cmd, cliName) + rewriteCompletionHelp(cmd, m.CLI.Name) return cmd } diff --git a/pkg/lathe/verify.go b/pkg/lathe/verify.go new file mode 100644 index 0000000..07d100d --- /dev/null +++ b/pkg/lathe/verify.go @@ -0,0 +1,247 @@ +package lathe + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "reflect" + "strings" + + "github.com/spf13/cobra" + + "github.com/lathe-cli/lathe/pkg/config" + "github.com/lathe-cli/lathe/pkg/runtime" +) + +type verifyReport struct { + OK bool `json:"ok"` + Checks []verifyCheck `json:"checks"` +} + +type verifyCheck struct { + Name string `json:"name"` + OK bool `json:"ok"` + Error string `json:"error,omitempty"` +} + +type verifyFailedError struct{} + +func (verifyFailedError) Error() string { + return "generated CLI verify failed" +} + +func (verifyFailedError) SilentExitCode() int { + return runtime.ExitGeneral +} + +func verifyCmd(m *config.Manifest) *cobra.Command { + cmd := &cobra.Command{ + Use: "verify", + Short: "Verify generated CLI contract", + RunE: func(cmd *cobra.Command, _ []string) error { + report := verifyGenerated(cmd.Root(), m) + if err := writeJSON(cmd, report); err != nil { + return err + } + if !report.OK { + return verifyFailedError{} + } + return nil + }, + } + cmd.Flags().Bool("json", false, "Emit verify JSON") + return cmd +} + +func verifyGenerated(root *cobra.Command, m *config.Manifest) verifyReport { + report := verifyReport{OK: true} + catalog := runtime.BuildCatalog(root, catalogOptions(m, false)) + + report.add("root_help", verifyRootHelp(root, m.CLI.Name)) + report.add("commands_schema", verifyCommandsSchema(root, catalog)) + report.add("commands_json", verifyCommandsJSON(catalog)) + report.add("catalog_nonempty", verifyCatalogNonempty(catalog)) + for _, entry := range catalog.Commands { + report.add("commands_show:"+strings.Join(entry.Path, " "), verifyCatalogEntry(root, m, entry)) + } + report.add("auth_status_unauthenticated", verifyAuthStatusUnauthenticated(m)) + + return report +} + +func (r *verifyReport) add(name string, err error) { + check := verifyCheck{Name: name, OK: err == nil} + if err != nil { + check.Error = err.Error() + r.OK = false + } + r.Checks = append(r.Checks, check) +} + +func verifyRootHelp(root *cobra.Command, cliName string) error { + if root == nil { + return errors.New("root command is nil") + } + if root.Use != cliName { + return fmt.Errorf("root use = %q, want %q", root.Use, cliName) + } + if findCommand(root, []string{"commands"}) == nil { + return errors.New("missing commands command") + } + if findCommand(root, []string{"search"}) == nil { + return errors.New("missing search command") + } + for _, want := range []string{"commands --json", "commands show", "search"} { + if !strings.Contains(root.Long, want) { + return fmt.Errorf("root help missing %q", want) + } + } + if root.UsageString() == "" { + return errors.New("root usage is empty") + } + return nil +} + +func verifyCommandsSchema(root *cobra.Command, catalog runtime.Catalog) error { + if findCommand(root, []string{"commands", "schema"}) == nil { + return errors.New("missing commands schema command") + } + if catalog.CatalogSchemaVersion != runtime.CatalogSchemaVersion { + return fmt.Errorf("catalog schema = %d, want %d", catalog.CatalogSchemaVersion, runtime.CatalogSchemaVersion) + } + return nil +} + +func verifyCommandsJSON(catalog runtime.Catalog) error { + _, err := json.Marshal(catalog) + return err +} + +func verifyCatalogNonempty(catalog runtime.Catalog) error { + if len(catalog.Commands) == 0 { + return errors.New("visible generated command catalog is empty") + } + return nil +} + +func verifyCatalogEntry(root *cobra.Command, m *config.Manifest, entry runtime.CatalogCommand) error { + path := strings.Join(entry.Path, " ") + if len(entry.Path) == 0 { + return errors.New("catalog entry path is empty") + } + cmd := findCommand(root, entry.Path) + if cmd == nil { + return fmt.Errorf("cobra command not found for path %q", path) + } + found, ok := runtime.FindCatalogCommand(root, entry.Path, catalogOptions(m, false)) + if !ok { + return fmt.Errorf("commands show cannot find %q", path) + } + if !reflect.DeepEqual(found, entry) { + return fmt.Errorf("commands show mismatch for %q", path) + } + for _, flag := range entry.Flags { + if flag.Flag == "" { + return fmt.Errorf("%q has catalog flag with empty name", path) + } + if cmd.Flags().Lookup(flag.Flag) == nil { + return fmt.Errorf("%q catalog requires missing --%s flag", path, flag.Flag) + } + } + if entry.Body != nil && entry.Body.Required { + switch { + case isJSONBody(entry.Body.MediaType): + for _, name := range []string{"file", "set", "set-str"} { + if cmd.Flags().Lookup(name) == nil { + return fmt.Errorf("%q required body missing --%s flag", path, name) + } + } + case isMultipartBody(entry.Body.MediaType): + default: + if cmd.Flags().Lookup("file") == nil { + return fmt.Errorf("%q required body missing --file flag", path) + } + } + } + return nil +} + +func isJSONBody(mediaType string) bool { + mt := normalizedMediaType(mediaType) + return mt == "" || mt == "application/json" || strings.HasSuffix(mt, "+json") +} + +func isMultipartBody(mediaType string) bool { + return normalizedMediaType(mediaType) == "multipart/form-data" +} + +func normalizedMediaType(mediaType string) string { + mt, _, _ := strings.Cut(strings.ToLower(strings.TrimSpace(mediaType)), ";") + return strings.TrimSpace(mt) +} + +func verifyAuthStatusUnauthenticated(m *config.Manifest) error { + if m == nil { + return errors.New("manifest is nil") + } + manifest := *m + if manifest.CLI.ConfigDir == "" { + manifest.CLI.ConfigDir = manifest.CLI.Name + } + if manifest.CLI.ConfigDirEnv == "" { + manifest.CLI.ConfigDirEnv = strings.ToUpper(manifest.CLI.Name) + "_CONFIG_DIR" + } + tempDir, err := os.MkdirTemp("", manifest.CLI.Name+"-verify-*") + if err != nil { + return err + } + defer func() { + _ = os.RemoveAll(tempDir) + }() + + old, hadOld := os.LookupEnv(manifest.CLI.ConfigDirEnv) + if err := os.Setenv(manifest.CLI.ConfigDirEnv, tempDir); err != nil { + return err + } + defer func() { + if hadOld { + _ = os.Setenv(manifest.CLI.ConfigDirEnv, old) + } else { + _ = os.Unsetenv(manifest.CLI.ConfigDirEnv) + } + config.Bind(m) + }() + + root := NewApp(&manifest) + root.SetOut(io.Discard) + root.SetErr(io.Discard) + root.SetArgs([]string{"auth", "status"}) + err = root.Execute() + if err == nil { + return errors.New("auth status unexpectedly passed with empty isolated config") + } + if runtime.ClassifyError(err).Code != runtime.CodeNotAuthenticated { + return fmt.Errorf("auth status error = %v, want not authenticated", err) + } + return nil +} + +func findCommand(root *cobra.Command, path []string) *cobra.Command { + cur := root + for _, segment := range path { + var next *cobra.Command + for _, child := range cur.Commands() { + if child.Name() == segment { + next = child + break + } + } + if next == nil { + return nil + } + cur = next + } + return cur +} diff --git a/pkg/lathe/verify_test.go b/pkg/lathe/verify_test.go new file mode 100644 index 0000000..96c9731 --- /dev/null +++ b/pkg/lathe/verify_test.go @@ -0,0 +1,142 @@ +package lathe + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/lathe-cli/lathe/pkg/runtime" +) + +func TestRunVerifyGeneratedJSON(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run(RunOptions{ + Manifest: []byte("cli:\n name: myctl\n short: test cli\n"), + Mount: func(root *cobra.Command) error { + runtime.Build(root, "demo", []runtime.CommandSpec{ + { + Group: "Users", + Use: "get-user", + Short: "Get a user", + Method: "GET", + PathTpl: "/users/{id}", + Params: []runtime.ParamSpec{{ + Name: "id", + Flag: "id", + In: runtime.InPath, + GoType: "string", + Required: true, + }}, + }, + { + Group: "Users", + Use: "create-user", + Short: "Create a user", + Method: "POST", + PathTpl: "/users", + RequestBody: &runtime.RequestBody{Required: true, MediaType: "application/json"}, + }, + }) + skills := &cobra.Command{Use: "skills"} + pkg := &cobra.Command{Use: "package"} + pkg.Flags().String("file", "", "") + pkg.Flags().String("github-url", "", "") + pkg.Flags().String("app-id", "", "") + pkg.Flags().String("skill-id", "", "") + runtime.AttachCatalogCommand(pkg, "console-rest", runtime.CommandSpec{ + Group: "Skills", + Use: "package", + Short: "Package skill", + Method: "POST", + PathTpl: "/skills/package", + RequestBody: &runtime.RequestBody{Required: true, MediaType: "multipart/form-data"}, + Params: []runtime.ParamSpec{ + {Name: "file", Flag: "file", In: runtime.InFormData, GoType: "string", Required: true}, + {Name: "githubUrl", Flag: "github-url", In: runtime.InFormData, GoType: "string", Required: true}, + {Name: "appId", Flag: "app-id", In: runtime.InFormData, GoType: "string", Required: true}, + {Name: "skillId", Flag: "skill-id", In: runtime.InFormData, GoType: "string", Required: true}, + }, + }) + skills.AddCommand(pkg) + root.AddCommand(skills) + return nil + }, + }, []string{"__lathe", "verify", "--json"}, &stdout, &stderr) + if code != runtime.ExitOK { + t.Fatalf("exit = %d, stderr = %q", code, stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q", stderr.String()) + } + var report verifyReport + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("invalid JSON: %v\n%s", err, stdout.String()) + } + if !report.OK { + t.Fatalf("report = %+v", report) + } + for _, want := range []string{ + "root_help", + "commands_schema", + "commands_json", + "catalog_nonempty", + "commands_show:demo users get-user", + "commands_show:demo users create-user", + "commands_show:skills package", + "auth_status_unauthenticated", + } { + if !verifyReportHasCheck(report, want) { + t.Fatalf("report missing %q: %+v", want, report.Checks) + } + } +} + +func TestRunVerifyGeneratedFailureReturnsJSONOnly(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run(RunOptions{ + Manifest: []byte("cli:\n name: myctl\n"), + Mount: func(root *cobra.Command) error { + bad := &cobra.Command{Use: "bad"} + runtime.AttachCatalogCommand(bad, "demo", runtime.CommandSpec{ + Use: "bad", + Params: []runtime.ParamSpec{{ + Name: "id", + Flag: "id", + In: runtime.InPath, + GoType: "string", + Required: true, + }}, + }) + root.AddCommand(bad) + return nil + }, + }, []string{"__lathe", "verify", "--json"}, &stdout, &stderr) + if code != runtime.ExitGeneral { + t.Fatalf("exit = %d, want %d", code, runtime.ExitGeneral) + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q", stderr.String()) + } + var report verifyReport + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatalf("invalid JSON: %v\n%s", err, stdout.String()) + } + if report.OK { + t.Fatalf("report unexpectedly passed: %+v", report) + } + if !strings.Contains(stdout.String(), "missing --id") { + t.Fatalf("report missing flag failure:\n%s", stdout.String()) + } +} + +func verifyReportHasCheck(report verifyReport, name string) bool { + for _, check := range report.Checks { + if check.Name == name && check.OK { + return true + } + } + return false +} diff --git a/pkg/runtime/errors.go b/pkg/runtime/errors.go index df8a02f..dab1b43 100644 --- a/pkg/runtime/errors.go +++ b/pkg/runtime/errors.go @@ -71,6 +71,10 @@ type jsonErrorEnvelope struct { Error LatheError `json:"error"` } +type silentExitError interface { + SilentExitCode() int +} + func FormatError(err error, format string, w io.Writer) int { le := ClassifyError(err) if le == nil { @@ -92,6 +96,10 @@ func Execute(cmd *cobra.Command) int { if err == nil { return ExitOK } + var silent silentExitError + if errors.As(err, &silent) { + return silent.SilentExitCode() + } format, _ := cmd.PersistentFlags().GetString("output") return FormatError(err, format, os.Stderr) } diff --git a/pkg/runtime/errors_test.go b/pkg/runtime/errors_test.go index 1a478e5..22d0a4c 100644 --- a/pkg/runtime/errors_test.go +++ b/pkg/runtime/errors_test.go @@ -7,6 +7,8 @@ import ( "fmt" "strings" "testing" + + "github.com/spf13/cobra" ) func TestClassifyError_Nil(t *testing.T) { @@ -102,3 +104,25 @@ func TestLatheError_Unwrap(t *testing.T) { t.Error("expected Unwrap to expose cause") } } + +type testSilentExitError struct{} + +func (testSilentExitError) Error() string { + return "hidden" +} + +func (testSilentExitError) SilentExitCode() int { + return ExitUsage +} + +func TestExecuteSilentExitError(t *testing.T) { + cmd := &cobra.Command{ + Use: "demo", + RunE: func(*cobra.Command, []string) error { + return testSilentExitError{} + }, + } + if code := Execute(cmd); code != ExitUsage { + t.Fatalf("exit = %d, want %d", code, ExitUsage) + } +}