diff --git a/apple/Sources/NLCIApple/App.swift b/apple/Sources/NLCIApple/App.swift index ba10f51..0539473 100644 --- a/apple/Sources/NLCIApple/App.swift +++ b/apple/Sources/NLCIApple/App.swift @@ -38,13 +38,15 @@ func buildPrompt(_ input: BridgeInput) -> String { struct NLCIApple { static func main() { Task { - // --ping mode: check availability and exit + // --ping mode: check availability and advertise capabilities so + // Go callers can gate metadata-mode requests on what this binary + // actually supports. if CommandLine.arguments.contains("--ping") { if let errStr = checkAvailability() { emit(BridgeOutput(error: errStr)) exit(1) } - emit(BridgeOutput(command: "ok", explanation: "Apple Intelligence is available")) + emit(BridgeOutput(capabilities: ["command", "metadata"])) exit(0) } @@ -70,19 +72,28 @@ struct NLCIApple { exit(1) } - // Run inference + // Run inference. The mode field selects which @Generable schema + // the model is constrained to produce. do { let session = LanguageModelSession(instructions: input.system) - let response = try await session.respond( - to: buildPrompt(input), - generating: CommandResult.self - ) - - emit(BridgeOutput( - command: response.content.command, - explanation: response.content.explanation - )) + switch input.mode { + case "metadata": + let response = try await session.respond( + to: buildPrompt(input), + generating: ToolMetadata.self + ) + emit(BridgeOutput(metadata: response.content)) + default: // "command" + let response = try await session.respond( + to: buildPrompt(input), + generating: CommandResult.self + ) + emit(BridgeOutput( + command: response.content.command, + explanation: response.content.explanation + )) + } exit(0) } catch { emit(BridgeOutput(error: "inference:\(error.localizedDescription)")) diff --git a/apple/Sources/NLCIApple/BridgeInput.swift b/apple/Sources/NLCIApple/BridgeInput.swift index fbeba9f..c4cae68 100644 --- a/apple/Sources/NLCIApple/BridgeInput.swift +++ b/apple/Sources/NLCIApple/BridgeInput.swift @@ -1,7 +1,11 @@ import Foundation /// BridgeInput is the JSON payload sent from Go to the Swift subprocess via stdin. +/// `mode` selects which @Generable schema the model should produce. Defaults +/// to "command" so older Go binaries that don't set the field continue to get +/// today's behavior. struct BridgeInput: Codable { + let mode: String let system: String let schema: String let examples: [[String]] @@ -10,6 +14,7 @@ struct BridgeInput: Codable { // Custom decoder so missing keys or null never crash. init(from decoder: Decoder) throws { let c = try decoder.container(keyedBy: CodingKeys.self) + mode = try c.decodeIfPresent(String.self, forKey: .mode) ?? "command" system = try c.decode(String.self, forKey: .system) schema = try c.decodeIfPresent(String.self, forKey: .schema) ?? "" examples = try c.decodeIfPresent([[String]].self, forKey: .examples) ?? [] diff --git a/apple/Sources/NLCIApple/BridgeOutput.swift b/apple/Sources/NLCIApple/BridgeOutput.swift index adffb77..0eb57db 100644 --- a/apple/Sources/NLCIApple/BridgeOutput.swift +++ b/apple/Sources/NLCIApple/BridgeOutput.swift @@ -1,20 +1,52 @@ import Foundation /// BridgeOutput is the JSON payload written by the Swift subprocess to stdout. +/// +/// In command mode `command`/`explanation` carry the result; in metadata mode +/// `metadata` carries it instead. `--ping` responses populate `capabilities` +/// so callers can detect whether this binary supports the metadata schema +/// without trying it speculatively. struct BridgeOutput: Codable { + let mode: String? let command: String? let explanation: String? + let metadata: ToolMetadata? + let capabilities: [String]? let error: String? init(command: String, explanation: String) { + self.mode = "command" self.command = command self.explanation = explanation + self.metadata = nil + self.capabilities = nil + self.error = nil + } + + init(metadata: ToolMetadata) { + self.mode = "metadata" + self.command = nil + self.explanation = nil + self.metadata = metadata + self.capabilities = nil + self.error = nil + } + + init(capabilities: [String]) { + self.mode = nil + self.command = "ok" + self.explanation = "Apple Intelligence is available" + self.metadata = nil + self.capabilities = capabilities self.error = nil } init(error: String) { + self.mode = nil self.command = nil self.explanation = nil + self.metadata = nil + self.capabilities = nil self.error = error } } diff --git a/apple/Sources/NLCIApple/MetadataResult.swift b/apple/Sources/NLCIApple/MetadataResult.swift new file mode 100644 index 0000000..2e0fb67 --- /dev/null +++ b/apple/Sources/NLCIApple/MetadataResult.swift @@ -0,0 +1,46 @@ +import Foundation +import FoundationModels + +/// ToolMetadata is the structured output of `nlci init` enrichment mode. +/// It mirrors the Go EnrichmentMetadata struct in cmd/nlci; the bridge maps +/// 1:1 onto BridgeOutput.metadata. Each field carries a @Guide describing the +/// content the model should produce, anchored in the verified command surface +/// the user prompt provides. +@Generable +struct ToolMetadata: Codable { + @Guide(description: "A single-line summary of what the tool is for, capped at roughly 80 characters. Capture the purpose, not the surface. Bad: 'docker CLI'. Good: 'Manage Docker containers, images, networks, and volumes'.") + var description: String + + @Guide(description: "A multi-line system prompt (4-7 lines) suitable to be used directly as the system prompt for an LLM that translates intent into commands for this tool. Mention the binary name, any tool-specific behavior derived from the help text (defaults, required flags), and safety reminders. Avoid generic boilerplate that would apply to any CLI.") + var systemPrompt: String + + @Guide(description: "Safety patterns: which command substrings need confirmation before execution, and which are absolutely forbidden. Ground each pick in real verbs from the help text or verified command list.") + var safety: SafetyRules + + @Guide(description: "Map of natural-language synonym words to the command paths or capability names they refer to. Aim for 5-15 entries that cover the common ways a user might phrase intent.") + var synonyms: [SynonymEntry] +} + +/// SafetyRules carry destructive-pattern lists. Both arrays may be empty for +/// non-destructive tools (e.g. read-only query CLIs). +@Generable +struct SafetyRules: Codable { + @Guide(description: "Command-substring patterns that should prompt the user before running. Use full subcommand paths (e.g. 'docker rm', not just 'rm'). Look for delete/remove/destroy/drop/purge/reset/force/prune verbs in the verified commands. Empty array if the tool has no destructive operations.") + var requireConfirmation: [String] + + @Guide(description: "Command-substring patterns the user should never run. Reserved for irreversible plus sweeping operations (e.g. 'docker system prune --all --volumes --force'). Usually empty.") + var forbidden: [String] +} + +/// SynonymEntry pairs a natural-language word with one or more command paths +/// or capability names. We use an array of pairs rather than a Swift +/// Dictionary because @Generable's macro support for [String: [String]] is +/// not documented; arrays of structs are the well-trodden path. +@Generable +struct SynonymEntry: Codable { + @Guide(description: "A short natural-language word a user might say (e.g. 'list', 'delete', 'show', 'clean', 'fetch').") + var keyword: String + + @Guide(description: "Command paths or capability names from this tool that the keyword refers to. At least one entry. Prefer paths that exist in the verified command list.") + var targets: [String] +} diff --git a/cmd/nlci/main.go b/cmd/nlci/main.go index d80af4b..c796767 100644 --- a/cmd/nlci/main.go +++ b/cmd/nlci/main.go @@ -441,14 +441,12 @@ func enrichInitDiscovery(ctx context.Context, br *backend.Router, toolName strin } // runMetadataEnrichment is the §B call site shared by both command_tree and -// flag_driven scaffolds. Best-effort: any failure (no router available, LLM -// error, malformed YAML, timeout) returns nil so callers fall through to -// hardcoded defaults rather than blocking the scaffold write. -// -// Apple Intelligence is skipped here: nlci-apple's Swift bridge constrains -// output to a Generable CommandResult struct, so it can't return free-form -// YAML. Run init under Ollama / llama.cpp / LM Studio to populate the -// metadata fields. Hardcoded fallback applies otherwise. +// flag_driven scaffolds. It prefers a backend's native MetadataGenerator +// capability (Apple Intelligence's @Generable schemas) when available and +// falls back to the textual YAML prompt path on backends that don't +// implement it (Ollama / llama.cpp / LM Studio). Best-effort throughout: +// any failure returns nil so the caller falls through to hardcoded defaults +// rather than blocking the scaffold write. func runMetadataEnrichment(ctx context.Context, opts scaffoldOptions, toolName, mode, helpText string, commands []definition.Command, quiet bool) *EnrichmentMetadata { br := opts.BackendRouter if br == nil { @@ -460,24 +458,37 @@ func runMetadataEnrichment(ctx context.Context, opts scaffoldOptions, toolName, return nil } - // Resolve so we know which backend will run, then skip Apple cleanly. - if _, err := br.Resolve(ctx); err != nil { + active, err := br.Resolve(ctx) + if err != nil { return nil } - if br.ActiveName() == "apple" { + + enrichCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + // Native structured generation: Apple advertises this via @Generable. + if mg, ok := active.(backend.MetadataGenerator); ok { if !quiet { - fmt.Println("Metadata enrichment skipped: Apple backend constrains output to single commands. Use --backend ollama|llamacpp|lmstudio for richer metadata.") + fmt.Println("Enriching scaffold metadata via native structured generation…") } - return nil + result, err := mg.GenerateMetadata(enrichCtx, backend.MetadataRequest{ + System: initEnrichmentSystemPrompt(), + Prompt: buildInitEnrichmentPrompt(toolName, toolName, mode, helpText, commands), + }) + if err == nil && result != nil { + return convertNativeMetadata(result) + } + if !quiet { + fmt.Printf("Native metadata enrichment failed, falling back to text prompt: %v\n", err) + } + // Fall through to text-prompt path below. } - enrichCtx, cancel := context.WithTimeout(ctx, 20*time.Second) - defer cancel() - + // Text-prompt path: ask for a YAML block in the model's command field + // and parse it. Used by backends that don't expose structured output. if !quiet { fmt.Println("Enriching scaffold metadata via inference…") } - meta, err := enrichInitMetadata(enrichCtx, br, toolName, toolName, mode, helpText, commands) if err != nil { if !quiet { @@ -490,6 +501,29 @@ func runMetadataEnrichment(ctx context.Context, opts scaffoldOptions, toolName, return meta } +// convertNativeMetadata maps a backend.MetadataResult (returned by the +// Apple bridge's @Generable structs) into the cmd-local EnrichmentMetadata +// shape used by the scaffold writers. Applies the same placeholder-leak +// guard the YAML parser uses, so a hallucinated "" in the +// description trips the same fallback. +func convertNativeMetadata(r *backend.MetadataResult) *EnrichmentMetadata { + if r == nil { + return nil + } + if containsBracketedToken(r.Description) || containsBracketedToken(r.SystemPrompt) { + return nil + } + return &EnrichmentMetadata{ + Description: r.Description, + SystemPrompt: r.SystemPrompt, + Safety: definition.Safety{ + RequireConfirmation: r.Safety.RequireConfirmation, + Forbidden: r.Safety.Forbidden, + }, + Synonyms: r.Synonyms, + } +} + // capabilitiesAsCommands adapts a flag-driven capability list into the // definition.Command shape used by the §B prompt. Only Name and Description // are needed — the prompt cares about the surface, not flag bindings. diff --git a/cmd/nlci/main_test.go b/cmd/nlci/main_test.go index d655b00..6e60db3 100644 --- a/cmd/nlci/main_test.go +++ b/cmd/nlci/main_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/jakejimenez/nlci/internal/backend" "github.com/jakejimenez/nlci/internal/definition" ) @@ -155,3 +156,44 @@ func TestMergeSynonyms_NilLLM(t *testing.T) { t.Fatalf("nil llm should pass heuristic through unchanged: %+v", merged) } } + +func TestConvertNativeMetadata_HappyPath(t *testing.T) { + r := &backend.MetadataResult{ + Description: "Manage Docker things", + SystemPrompt: "You are a docker expert.", + Safety: backend.Safety{ + RequireConfirmation: []string{"docker rm"}, + Forbidden: []string{}, + }, + Synonyms: map[string][]string{"list": {"ps"}}, + } + got := convertNativeMetadata(r) + if got == nil { + t.Fatalf("expected conversion, got nil") + } + if got.Description != "Manage Docker things" { + t.Fatalf("description: got %q", got.Description) + } + if len(got.Safety.RequireConfirmation) != 1 || got.Safety.RequireConfirmation[0] != "docker rm" { + t.Fatalf("safety: got %+v", got.Safety) + } + if got.Synonyms["list"][0] != "ps" { + t.Fatalf("synonyms: got %+v", got.Synonyms) + } +} + +func TestConvertNativeMetadata_RejectsPlaceholderInDescription(t *testing.T) { + r := &backend.MetadataResult{ + Description: " CLI", + SystemPrompt: "Generic.", + } + if got := convertNativeMetadata(r); got != nil { + t.Fatalf("expected nil for placeholder in description, got %+v", got) + } +} + +func TestConvertNativeMetadata_NilInput(t *testing.T) { + if got := convertNativeMetadata(nil); got != nil { + t.Fatalf("expected nil for nil input, got %+v", got) + } +} diff --git a/internal/backend/apple.go b/internal/backend/apple.go index bae0209..700a144 100644 --- a/internal/backend/apple.go +++ b/internal/backend/apple.go @@ -14,11 +14,13 @@ import ( // The Swift binary reads a JSON BridgeInput from stdin and writes a // JSON BridgeOutput to stdout. type AppleBackend struct { - binaryPath string + binaryPath string + capabilities []string // populated at Ping time; empty slice means "old binary, command-only" } // bridgeInput mirrors BridgeInput.swift. type bridgeInput struct { + Mode string `json:"mode,omitempty"` System string `json:"system"` Schema string `json:"schema"` Examples [][2]string `json:"examples"` @@ -27,9 +29,32 @@ type bridgeInput struct { // bridgeOutput mirrors BridgeOutput.swift. type bridgeOutput struct { - Command string `json:"command"` - Explanation string `json:"explanation"` - Error string `json:"error"` + Mode string `json:"mode,omitempty"` + Command string `json:"command,omitempty"` + Explanation string `json:"explanation,omitempty"` + Metadata *appleMetadata `json:"metadata,omitempty"` + Capabilities []string `json:"capabilities,omitempty"` + Error string `json:"error,omitempty"` +} + +// appleMetadata mirrors ToolMetadata.swift, including the array-of-pairs +// shape for synonyms (FoundationModels' @Generable doesn't support +// Dictionary directly). +type appleMetadata struct { + Description string `json:"description"` + SystemPrompt string `json:"systemPrompt"` + Safety appleSafetyRules `json:"safety"` + Synonyms []appleSynonymEntry `json:"synonyms"` +} + +type appleSafetyRules struct { + RequireConfirmation []string `json:"requireConfirmation"` + Forbidden []string `json:"forbidden"` +} + +type appleSynonymEntry struct { + Keyword string `json:"keyword"` + Targets []string `json:"targets"` } // NewApple creates an AppleBackend. binaryPath is the path to the nlci-apple binary. @@ -40,21 +65,42 @@ func NewApple(binaryPath string) *AppleBackend { func (a *AppleBackend) Name() string { return "apple" } // Ping checks that the nlci-apple binary exists and that Apple Intelligence -// is available on this device. +// is available on this device. The ping response also advertises which +// schemas the binary supports (e.g. "command", "metadata") so callers can +// gate optional capabilities without speculatively trying them. func (a *AppleBackend) Ping(ctx context.Context) error { if _, err := os.Stat(a.binaryPath); err != nil { return fmt.Errorf("%w: nlci-apple binary not found at %s", ErrBackendUnavailable, a.binaryPath) } - // Run with --ping flag to check Apple Intelligence availability cmd := exec.CommandContext(ctx, a.binaryPath, "--ping") + var stdout bytes.Buffer + cmd.Stdout = &stdout if err := cmd.Run(); err != nil { return fmt.Errorf("%w: nlci-apple ping failed: %s", ErrBackendUnavailable, err) } + // Parse capabilities; old binaries don't emit them, leaving the slice nil. + var out bridgeOutput + if err := json.Unmarshal(stdout.Bytes(), &out); err == nil { + a.capabilities = out.Capabilities + } + return nil } +// HasCapability reports whether the underlying Swift binary advertised the +// named mode at ping time. Returns false for binaries that predate the +// capability advertisement (i.e. only support the implicit "command" mode). +func (a *AppleBackend) HasCapability(name string) bool { + for _, c := range a.capabilities { + if c == name { + return true + } + } + return false +} + // Generate runs the Swift subprocess, sends the request via stdin, // and reads the response from stdout. func (a *AppleBackend) Generate(ctx context.Context, r Request) (Response, error) { @@ -106,3 +152,87 @@ func (a *AppleBackend) Generate(ctx context.Context, r Request) (Response, error Explanation: out.Explanation, }, nil } + +// GenerateMetadata implements MetadataGenerator. It invokes the Swift bridge +// in metadata mode, which constrains the model output to a Generable +// ToolMetadata schema instead of the default CommandResult. +// +// Returns an error advising a rebuild when the binary doesn't advertise the +// "metadata" capability — old binaries silently produce a CommandResult and +// the metadata field would be absent. +func (a *AppleBackend) GenerateMetadata(ctx context.Context, r MetadataRequest) (*MetadataResult, error) { + if !a.HasCapability("metadata") { + return nil, fmt.Errorf("apple: nlci-apple binary does not advertise metadata mode — run `make build-apple && make install-apple` to upgrade") + } + + input := bridgeInput{ + Mode: "metadata", + System: r.System, + Examples: make([][2]string, 0), + Intent: r.Prompt, + } + + inputJSON, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("apple: marshal metadata input: %w", err) + } + + cmd := exec.CommandContext(ctx, a.binaryPath) + cmd.Stdin = bytes.NewReader(inputJSON) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if stderrStr != "" { + return nil, fmt.Errorf("apple: metadata subprocess failed: %w\n%s", err, stderrStr) + } + return nil, fmt.Errorf("apple: metadata subprocess failed: %w", err) + } + + var out bridgeOutput + if err := json.Unmarshal(stdout.Bytes(), &out); err != nil { + return nil, fmt.Errorf("apple: invalid metadata output: %w (raw: %s)", err, stdout.String()) + } + + if out.Error != "" { + if strings.HasPrefix(out.Error, "unavailable:") { + return nil, fmt.Errorf("%w: %s", ErrBackendUnavailable, out.Error) + } + return nil, fmt.Errorf("apple: %s", out.Error) + } + + if out.Metadata == nil { + return nil, fmt.Errorf("apple: metadata field absent in response (raw: %s)", stdout.String()) + } + + syn := make(map[string][]string, len(out.Metadata.Synonyms)) + for _, e := range out.Metadata.Synonyms { + if e.Keyword == "" || len(e.Targets) == 0 { + continue + } + // Append + dedupe in case the model emits the same keyword twice. + seen := make(map[string]bool, len(syn[e.Keyword])) + for _, t := range syn[e.Keyword] { + seen[t] = true + } + for _, t := range e.Targets { + if !seen[t] { + syn[e.Keyword] = append(syn[e.Keyword], t) + seen[t] = true + } + } + } + + return &MetadataResult{ + Description: out.Metadata.Description, + SystemPrompt: out.Metadata.SystemPrompt, + Safety: Safety{ + RequireConfirmation: out.Metadata.Safety.RequireConfirmation, + Forbidden: out.Metadata.Safety.Forbidden, + }, + Synonyms: syn, + }, nil +} diff --git a/internal/backend/metadata.go b/internal/backend/metadata.go new file mode 100644 index 0000000..de42563 --- /dev/null +++ b/internal/backend/metadata.go @@ -0,0 +1,37 @@ +package backend + +import "context" + +// MetadataRequest is the input to a structured-output metadata-generation +// call. Mirrors Request but uses Prompt instead of Intent for clarity — this +// is not a single-command intent, it's a request for structured tool +// metadata derived from help text. +type MetadataRequest struct { + System string + Prompt string +} + +// MetadataResult mirrors cmd/nlci's EnrichmentMetadata so backends with +// native structured output (e.g. Apple Intelligence's @Generable schemas) +// can produce it directly without a textual YAML round-trip. +type MetadataResult struct { + Description string + SystemPrompt string + Safety Safety + Synonyms map[string][]string +} + +// Safety duplicates definition.Safety in this package to avoid an import +// cycle (definition imports backend transitively in some test paths). +type Safety struct { + RequireConfirmation []string + Forbidden []string +} + +// MetadataGenerator is implemented by backends that support native +// structured-output metadata generation. Backends that don't implement it +// fall through to the textual YAML prompt path in cmd/nlci. This is +// idiomatic Go capability detection (cf. io.WriterTo, fs.ReadDirFS). +type MetadataGenerator interface { + GenerateMetadata(ctx context.Context, r MetadataRequest) (*MetadataResult, error) +}