diff --git a/README.md b/README.md index 110d2e6..d15e971 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,59 @@ keyprobe --huggingface keyprobe --replicate ``` +### Requirement Flags + +Validate specific capabilities or models with **fuzzy matching** and **short aliases**: + +```sh +# Capability validation +keyprobe --require-streaming # Real-time streaming +keyprobe --require-embeddings # Vector embeddings +keyprobe --require-chat # Chat completion +keyprobe --require-assistants # AI assistants +keyprobe --require-dall-e # Image generation + +# Model validation +keyprobe --require-model "gpt-4o" # Specific model access +keyprobe --require-model "claude-3-5-sonnet" + +# Combined validation +keyprobe --require-streaming --require-model "gpt-4o" + +# Provider-specific +keyprobe --anthropic --require-files # Anthropic files API +keyprobe --openai --require-whisper # OpenAI speech-to-text + +# Fuzzy matching works too! +keyprobe --require "streamig" # Matches "Streaming" +keyprobe --require "embedings" # Matches "Embeddings" +``` + +### Common Use Cases + +```sh +# Real-time applications +keyprobe --require-streaming sk-ant-api03-... + +# Vector search & RAG +keyprobe --require-embeddings sk-proj-... + +# Advanced reasoning models +keyprobe --require-model "gpt-4o" sk-proj-... + +# Production deployment +keyprobe --require-chat --require-streaming sk-proj-... + +# Multi-modal AI +keyprobe --require-dall-e --require-whisper sk-proj-... + +# AI assistants & agents +keyprobe --require-assistants sk-proj-... + +# Spelling mistakes work too! +keyprobe --require "streamig" sk-ant-api03-... +``` + ## Output **CLI (default)** @@ -84,12 +137,36 @@ keyprobe --output json } ``` + +## ⚡ Short Aliases +No quotes needed for common capabilities: + +```bash +keyprobe --require-streaming # --require "Streaming" +keyprobe --require-embeddings # --require "Embeddings" +keyprobe --require-chat # --require "Messages API (chat)" +keyprobe --require-batch # --require "Batch API" +keyprobe --require-files # --require "Files API (beta)" +keyprobe --require-assistants # --require "Assistants API" +keyprobe --require-dall-e # --require "DALL-E 3 (image gen)" +keyprobe --require-whisper # --require "Whisper (speech-to-text)" +keyprobe --require-fine-tuning # --require "Fine-tuning" +``` + + ## Exit codes | Code | Meaning | |------|---------| -| `0` | Key is valid | -| `1` | Key is invalid or an error occurred | +| `0` | Key is valid and all requirements met | +| `1` | Key is invalid, error occurred, or requirements failed | + +**Exit code 1 scenarios:** +- API key is invalid or expired +- Required capability is not available (`--require`) +- Required model is not accessible (`--require-model`) +- Network errors or API failures +- Invalid command-line arguments ## Supported providers @@ -108,6 +185,202 @@ keyprobe --output json | DeepSeek | 64-char key | | Cohere | `sk-cohere-` or 40-char key | +## GitHub Actions Integration + +KeyProbe integrates seamlessly with GitHub Actions for automated API key validation in CI/CD pipelines with support for capability and model requirements. + +### Quick Setup + +1. **Add your API key to GitHub Secrets:** + - Go to your repository → Settings → Secrets and variables → Actions + - Add a new secret named `LLM_API_KEY` with your API key value + +2. **Copy this workflow to `.github/workflows/keyprobe.yml`:** + ```yaml + name: KeyProbe API Key Validation + + on: + workflow_dispatch: + inputs: + required_capability: + description: 'Required capability (e.g., "Streaming", "Embeddings")' + required: false + type: string + default: '' + required_model: + description: 'Required model (e.g., "gpt-4o", "claude-3-sonnet")' + required: false + type: string + default: '' + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] + + jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + + - name: Build KeyProbe CLI + run: | + git clone https://github.com/datumbrain/keyprobe.git keyprobe-temp + cd keyprobe-temp + go build -o ../keyprobe . + cd .. + rm -rf keyprobe-temp + + - name: Safety Check - Verify API Key Secret + run: | + # Fail early if LLM_API_KEY secret is missing or empty + if [ -z "$LLM_API_KEY" ]; then + echo "❌ LLM_API_KEY secret is missing or empty" + echo "Please add LLM_API_KEY to your repository secrets" + exit 1 + fi + echo "✅ API key secret is present" + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + + - name: Validate API Key and Requirements + run: | + # Build KeyProbe command with optional requirement flags + CMD="./keyprobe" + + # Add --require flag if capability is specified + if [ -n "${{ github.event.inputs.required_capability }}" ]; then + CMD="$CMD --require '${{ github.event.inputs.required_capability }}'" + echo "🔍 Checking for required capability: ${{ github.event.inputs.required_capability }}" + fi + + # Add --require-model flag if model is specified + if [ -n "${{ github.event.inputs.required_model }}" ]; then + CMD="$CMD --require-model '${{ github.event.inputs.required_model }}'" + echo "🔍 Checking for required model: ${{ github.event.inputs.required_model }}" + fi + + # Add API key + CMD="$CMD '$LLM_API_KEY'" + + echo "🚀 Running command: $CMD" + echo "" + + # Execute validation - KeyProbe will exit with code 1 if: + # - API key is invalid + # - Required capability is not available + # - Required model is not accessible + if eval "$CMD"; then + echo "" + echo "✅ All validations passed successfully" + else + echo "" + echo "❌ Validation failed - workflow will exit with error" + exit 1 + fi + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + ``` + +### Usage Examples + +**Basic validation (runs automatically on pushes/PRs):** +- Workflow validates the API key stored in `LLM_API_KEY` secret +- If invalid, the workflow fails and blocks merges + +**Manual validation with capability requirement:** +1. Go to Actions → KeyProbe API Key Validation → Run workflow +2. Enter "Streaming" in the "Required capability" field +3. Leave "Required model" empty +4. Click "Run workflow" +5. Workflow validates the key and streaming capability + +**Manual validation with model requirement:** +1. Go to Actions → KeyProbe API Key Validation → Run workflow +2. Leave "Required capability" empty +3. Enter "gpt-4o" in the "Required model" field +4. Click "Run workflow" +5. Workflow validates the key and GPT-4o access + +**Manual validation with both requirements:** +1. Go to Actions → KeyProbe API Key Validation → Run workflow +2. Enter "Streaming" in "Required capability" +3. Enter "gpt-4o" in "Required model" +4. Click "Run workflow" +5. Workflow validates key, streaming capability, and GPT-4o access + +### Common CI/CD Scenarios + +```yaml +# Example: Require streaming for real-time features (using short alias) +- name: Validate Streaming Capability + run: | + ./keyprobe --require-streaming "$LLM_API_KEY" + +# Example: Require specific model for production +- name: Validate Production Model + run: | + ./keyprobe --require-model "gpt-4o" "$LLM_API_KEY" + +# Example: Comprehensive validation with short aliases +- name: Validate All Requirements + run: | + ./keyprobe --require-streaming --require-model "gpt-4o" "$LLM_API_KEY" + +# Example: Multiple capabilities validation +- name: Validate Multiple Capabilities + run: | + ./keyprobe --require-chat --require-embeddings --require-assistants "$LLM_API_KEY" + +# Example: Provider-specific validation +- name: Validate Anthropic Features + run: | + ./keyprobe --anthropic --require-streaming --require-files "$LLM_API_KEY" +``` + +### Requirements + +- No external dependencies required - workflow uses KeyProbe's built-in requirement flags +- KeyProbe handles all validation logic internally +- Optional JSON validation step for detailed reporting + +### How Failure Works + +KeyProbe uses **exit codes** to automatically control workflow behavior: + +- **Exit code 0**: Key is valid and all requirements met → Workflow continues ✅ +- **Exit code 1**: Key is invalid, requirements failed, or error occurred → Workflow fails ❌ + +**Exit code 1 scenarios:** +- API key is invalid or expired +- Required capability is not available (`--require`) +- Required model is not accessible (`--require-model`) +- Network errors or API failures +- Invalid command-line arguments + +**CI/CD Integration:** +- Failed validation blocks deployments and pull request merges +- No additional logic needed - GitHub Actions respects exit codes automatically +- Clear error messages show exactly what failed (key validity, missing capability, or model access) +- JSON validation step provides detailed diagnostics for troubleshooting + +### Production Features + +- **Robust validation**: Uses KeyProbe's built-in requirement flags with proper error handling +- **Flexible requirements**: Support for both capability and model validation +- **Security**: API keys only accessed via GitHub Secrets +- **Detailed reporting**: Optional JSON validation for troubleshooting +- **Manual control**: Optional requirement checking for specific deployment scenarios +- **Clean integration**: Works seamlessly with existing CI/CD pipelines + ## License MIT diff --git a/main.go b/main.go index f77d0bb..f79e14a 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,155 @@ func cyan(s string) string { return colorCyan + s + colorReset } func bold(s string) string { return colorBold + s + colorReset } func dim(s string) string { return colorDim + s + colorReset } +// ─── Capability Requirements System ───────────────────────────────────────────── + +// CapabilityAlias maps aliases and fuzzy matches to canonical capability names +var CapabilityAlias = map[string][]string{ + "Streaming": {"streaming", "stream", "real-time", "realtime", "sse", "server-sent-events"}, + "Messages API (chat)": {"messages", "chat", "chat-api", "messages-api", "completion", "completions"}, + "Embeddings": {"embeddings", "embedding", "vector", "vectors", "text-embedding"}, + "Batch API": {"batch", "batches", "batch-api", "async", "asynchronous"}, + "Files API (beta)": {"files", "file-upload", "documents", "files-api", "file-api"}, + "Claude Code (CLI)": {"claude-code", "cli", "command-line", "tool-use"}, + "DALL-E 3 (image gen)": {"dall-e", "dalle", "image-gen", "image-generation", "images", "vision"}, + "Whisper (speech-to-text)": {"whisper", "speech-to-text", "stt", "audio", "transcription"}, + "TTS (text-to-speech)": {"tts", "text-to-speech", "speech", "audio-gen", "voice"}, + "Assistants API": {"assistants", "assistant", "agents", "tools", "function-calling"}, + "Fine-tuning": {"fine-tuning", "finetune", "training", "custom-models"}, + "Image generation (Imagen)": {"imagen", "image-gen", "images", "vision"}, +} + +// normalizeString for fuzzy matching +func normalizeString(s string) string { + s = strings.ToLower(s) + s = strings.ReplaceAll(s, " ", "") + s = strings.ReplaceAll(s, "-", "") + s = strings.ReplaceAll(s, "_", "") + s = strings.ReplaceAll(s, "(", "") + s = strings.ReplaceAll(s, ")", "") + s = strings.ReplaceAll(s, "+", "") + return s +} + +// levenshteinDistance calculates the edit distance between two strings +func levenshteinDistance(a, b string) int { + a, b = strings.ToLower(a), strings.ToLower(b) + if len(a) == 0 { + return len(b) + } + if len(b) == 0 { + return len(a) + } + if a == b { + return 0 + } + + // Ensure a is the shorter string + if len(a) > len(b) { + a, b = b, a + } + + prevRow := make([]int, len(b)+1) + for i := range prevRow { + prevRow[i] = i + } + + for i, ca := range a { + currRow := []int{i + 1} + for j, cb := range b { + cost := 0 + if ca != cb { + cost = 1 + } + min := prevRow[j+1] + 1 // deletion + if currRow[j]+1 < min { + min = currRow[j] + 1 // insertion + } + if prevRow[j]+cost < min { + min = prevRow[j] + cost // substitution + } + currRow = append(currRow, min) + } + prevRow = currRow + } + + return prevRow[len(b)] +} + +// findBestCapabilityMatch finds the best matching capability using fuzzy matching +func findBestCapabilityMatch(input string, availableCapabilities []string) (string, bool) { + input = normalizeString(input) + + // First try exact matches with aliases + for canonical, aliases := range CapabilityAlias { + for _, alias := range aliases { + if normalizeString(alias) == input { + return canonical, true + } + } + } + + // Then try exact matches with capability names + for _, cap := range availableCapabilities { + if normalizeString(cap) == input { + return cap, true + } + } + + // Fuzzy matching - find closest match + bestMatch := "" + bestScore := 0.8 // threshold for fuzzy matching + + for _, cap := range availableCapabilities { + normalizedCap := normalizeString(cap) + + // Check if input is contained in capability or vice versa + if strings.Contains(normalizedCap, input) || strings.Contains(input, normalizedCap) { + return cap, true + } + + // Calculate similarity score based on levenshtein distance + distance := levenshteinDistance(input, normalizedCap) + maxLen := max(len(input), len(normalizedCap)) + similarity := 1.0 - float64(distance)/float64(maxLen) + + if similarity > bestScore { + bestScore = similarity + bestMatch = cap + } + } + + if bestMatch != "" { + return bestMatch, true + } + + // Also check aliases for fuzzy matches + for canonical, aliases := range CapabilityAlias { + for _, alias := range aliases { + normalizedAlias := normalizeString(alias) + + if strings.Contains(normalizedAlias, input) || strings.Contains(input, normalizedAlias) { + return canonical, true + } + + distance := levenshteinDistance(input, normalizedAlias) + maxLen := max(len(input), len(normalizedAlias)) + similarity := 1.0 - float64(distance)/float64(maxLen) + + if similarity > bestScore { + bestScore = similarity + bestMatch = canonical + } + } + } + + if bestMatch != "" { + return bestMatch, true + } + + return "", false +} + // ─── Provider detection ─────────────────────────────────────────────────────── type Provider string @@ -109,13 +258,14 @@ type Capability struct { } type ValidationResult struct { - Provider Provider - Valid bool - Error string - Model string - OrgID string - Capabilities []Capability - RawDetails map[string]string + Provider Provider + Valid bool + Error string + Model string + OrgID string + Capabilities []Capability + RawDetails map[string]string + RequirementFailed string } type jsonCapability struct { @@ -125,12 +275,13 @@ type jsonCapability struct { } type jsonResult struct { - Provider string `json:"provider"` - Valid bool `json:"valid"` - Error string `json:"error,omitempty"` - Model string `json:"model,omitempty"` - Capabilities []jsonCapability `json:"capabilities,omitempty"` - Details map[string]string `json:"details,omitempty"` + Provider string `json:"provider"` + Valid bool `json:"valid"` + Error string `json:"error,omitempty"` + Model string `json:"model,omitempty"` + Capabilities []jsonCapability `json:"capabilities,omitempty"` + Details map[string]string `json:"details,omitempty"` + RequirementFailed string `json:"requirement_failed,omitempty"` } // ─── HTTP helpers ───────────────────────────────────────────────────────────── @@ -875,11 +1026,12 @@ func printResult(r ValidationResult) { func printResultJSON(r ValidationResult) { label := providerLabels[r.Provider] out := jsonResult{ - Provider: label, - Valid: r.Valid, - Error: r.Error, - Model: r.Model, - Details: r.RawDetails, + Provider: label, + Valid: r.Valid, + Error: r.Error, + Model: r.Model, + Details: r.RawDetails, + RequirementFailed: r.RequirementFailed, } for _, c := range r.Capabilities { out.Capabilities = append(out.Capabilities, jsonCapability{ @@ -919,23 +1071,96 @@ func usage() { keyprobe --replicate keyprobe --output json # output as JSON +%s + # Requirement validation (with fuzzy matching) + keyprobe --require "Streaming" # require specific capability + keyprobe --require-streaming # short alias (same as above) + keyprobe --require "embedings" # spelling mistakes tolerated + keyprobe --require-model "gpt-4o" # require specific model + %s Validates the key against the provider's live API and reports which capabilities (models, APIs, features) are accessible. Detection is automatic when no --provider flag is given. Use --output json for machine-readable output. + + Requirement flags: + --require Exit with error if capability not available + --require- Short alias for --require (e.g., --require-streaming) + --require-model Exit with error if model not accessible + + Features: + • Fuzzy matching: Spelling mistakes are tolerated + • Short aliases: Use --require-streaming instead of --require "Streaming" + • Auto-suggestion: Shows closest match when capability not found + + Exit codes: + 0 Key is valid and all requirements met + 1 Key is invalid, error occurred, or requirements failed `, bold("keyprobe"), dim("v"+version), bold("Usage:"), bold("Examples:"), + bold("Requirements:"), bold("Notes:"), ) } // ─── Main ───────────────────────────────────────────────────────────────────── +// checkCapability checks if a specific capability is available in the validation result +func checkCapability(result ValidationResult, requiredCapability string) bool { + if requiredCapability == "" { + return true + } + + // Extract available capability names from the result + var availableCaps []string + for _, cap := range result.Capabilities { + availableCaps = append(availableCaps, cap.Name) + } + + // Use fuzzy matching to find the best match + matchedCapability, found := findBestCapabilityMatch(requiredCapability, availableCaps) + if !found { + return false + } + + // Check if the matched capability is available + for _, cap := range result.Capabilities { + if cap.Name == matchedCapability { + return cap.Available + } + } + return false +} + +// checkModel checks if a specific model is available for the provider +func checkModel(result ValidationResult, requiredModel string) bool { + if requiredModel == "" { + return true + } + // For OpenAI, check the models list + if result.Provider == ProviderOpenAI { + // We would need to store the models list, but for now check capabilities + for _, cap := range result.Capabilities { + if strings.Contains(strings.ToLower(cap.Name), strings.ToLower(requiredModel)) { + return cap.Available + } + } + } + // For other providers, check capability names that might contain model info + for _, cap := range result.Capabilities { + if strings.Contains(strings.ToLower(cap.Name), strings.ToLower(requiredModel)) || + strings.Contains(strings.ToLower(cap.Description), strings.ToLower(requiredModel)) { + return cap.Available + } + } + return false +} + func main() { fs := flag.NewFlagSet("keyprobe", flag.ExitOnError) fs.Usage = usage @@ -954,13 +1179,16 @@ func main() { fReplicate := fs.Bool("replicate", false, "") fOutput := fs.String("output", "cli", "") fVersion := fs.Bool("version", false, "") + fRequire := fs.String("require", "", "") + fRequireModel := fs.String("require-model", "", "") // Extract the API key (first non-flag argument) so flags can appear anywhere. - // String flags like --output consume the next token as their value, so we + // String flags like --output, --require, --require-model consume the next token as their value, so we // must skip those values when hunting for the key. - stringFlags := map[string]bool{"output": true} + stringFlags := map[string]bool{"output": true, "require": true, "require-model": true} var key string var flagArgs []string + var requireAlias string // Store --require-streaming style aliases rawArgs := os.Args[1:] for i := 0; i < len(rawArgs); i++ { arg := rawArgs[i] @@ -973,6 +1201,10 @@ func main() { // --flag value form: consume next token as value flagArgs = append(flagArgs, arg, rawArgs[i+1]) i++ + } else if strings.HasPrefix(name, "require-") { + // Handle --require-streaming style aliases + requireAlias = strings.TrimPrefix(name, "require-") + // Don't add to flagArgs since it's not a defined flag } else { flagArgs = append(flagArgs, arg) } @@ -994,6 +1226,15 @@ func main() { os.Exit(0) } + // Handle --require-streaming style aliases + if requireAlias != "" { + if *fRequire != "" { + fmt.Fprintln(os.Stderr, red("error: cannot use both --require and --require- flags")) + os.Exit(1) + } + *fRequire = requireAlias + } + if key == "" { fmt.Fprintln(os.Stderr, red("error: API key is required")) usage() @@ -1056,13 +1297,50 @@ func main() { fmt.Print("\r \r") // clear "Validating..." } + // Check requirements after validation (only if key is valid) + requiredCapability := *fRequire + requiredModel := *fRequireModel + + // Check if requirements are met before printing (only if key is valid) + requirementFailed := "" + if result.Valid { + if requiredCapability != "" && !checkCapability(result, requiredCapability) { + // Try to find the closest match for suggestion + var availableCaps []string + for _, cap := range result.Capabilities { + availableCaps = append(availableCaps, cap.Name) + } + + suggestion := "" + if closestMatch, found := findBestCapabilityMatch(requiredCapability, availableCaps); found { + suggestion = fmt.Sprintf(" Did you mean '%s'?", closestMatch) + } else { + // Show general suggestion + suggestion = " Use --help to see available options" + } + + requirementFailed = fmt.Sprintf("Capability '%s' is not available.%s", requiredCapability, suggestion) + } else if requiredModel != "" && !checkModel(result, requiredModel) { + requirementFailed = fmt.Sprintf("Model '%s' is not available", requiredModel) + } + } + + // Update result with requirement failure info for JSON output + if requirementFailed != "" { + result.RequirementFailed = requirementFailed + } + if outputFormat == "json" { printResultJSON(result) } else { printResult(result) + if requirementFailed != "" { + fmt.Fprintf(os.Stderr, "\n%s %s\n", red("✗ Requirement failed:"), requirementFailed) + } } - if !result.Valid { + // Exit with error code if requirements failed or key is invalid + if requirementFailed != "" || !result.Valid { os.Exit(1) } }