From cc01af1c7c366f56ab31dc5a78edc02e18dd102e Mon Sep 17 00:00:00 2001 From: Simba Zhang Date: Fri, 6 Mar 2026 21:08:02 -0800 Subject: [PATCH] feat: add config.yaml and platform parameter docs for Aegis skill system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Create docs/skill-params.md documenting platform env vars (AEGIS_GATEWAY_URL, AEGIS_VLM_URL, AEGIS_SKILL_ID, AEGIS_SKILL_PARAMS, AEGIS_PORTS) and the config.yaml format for user-configurable params - Create config.yaml for home-security-benchmark with mode (llm/vlm/full) and noOpen user params — no platform params (Aegis auto-injects those) - Update run-benchmark.cjs to read AEGIS_SKILL_PARAMS: merge skillParams.noOpen into NO_OPEN, add TEST_MODE for suite filtering (llm skips VLM Scene Analysis, vlm keeps only VLM suites, full runs all) - Update SKILL.md with User Configuration section referencing config.yaml and linking to docs/skill-params.md --- docs/skill-params.md | 104 ++++++++++++++++++ .../analysis/home-security-benchmark/SKILL.md | 11 ++ .../home-security-benchmark/config.yaml | 13 +++ .../scripts/run-benchmark.cjs | 31 +++++- 4 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 docs/skill-params.md create mode 100644 skills/analysis/home-security-benchmark/config.yaml diff --git a/docs/skill-params.md b/docs/skill-params.md new file mode 100644 index 00000000..dd8306fc --- /dev/null +++ b/docs/skill-params.md @@ -0,0 +1,104 @@ +# Aegis Skill Platform Parameters + +Aegis automatically injects these environment variables to every skill process. Skills should **not** ask users to configure these — they are provided by the platform. + +## Platform Parameters (auto-injected) + +| Env Var | Type | Description | +|---------|------|-------------| +| `AEGIS_GATEWAY_URL` | `string` | LLM gateway endpoint (e.g. `http://localhost:5407`). Proxies to whatever LLM provider the user has configured (OpenAI, Anthropic, local). Skills should use this for all LLM calls — it handles auth, routing, and model selection. | +| `AEGIS_VLM_URL` | `string` | Local VLM (Vision Language Model) server endpoint (e.g. `http://localhost:5405`). Available when the user has a local VLM running. | +| `AEGIS_SKILL_ID` | `string` | The skill's unique identifier (e.g. `home-security-benchmark`). | +| `AEGIS_SKILL_PARAMS` | `JSON string` | User-configured parameters from `config.yaml` (see below). | +| `AEGIS_PORTS` | `JSON string` | All Aegis service ports as JSON. Use the URL vars above instead of parsing this directly. | + +## User Parameters (from config.yaml) + +Skills can define user-configurable parameters in a `config.yaml` file alongside `SKILL.md`. Aegis parses this at install time and renders a config panel in the UI. User values are passed as JSON via `AEGIS_SKILL_PARAMS`. + +### config.yaml Format + +```yaml +params: + - key: mode + label: Test Mode + type: select + options: [option1, option2, option3] + default: option1 + description: "Human-readable description shown in the config panel" + + - key: verbose + label: Verbose Output + type: boolean + default: false + description: "Enable detailed logging" + + - key: threshold + label: Confidence Threshold + type: number + default: 0.7 + description: "Minimum confidence score (0.0–1.0)" + + - key: apiEndpoint + label: Custom API Endpoint + type: string + default: "" + description: "Optional override for external API" +``` + +Supported types: `string`, `boolean`, `select`, `number` + +### Reading config.yaml in Your Skill + +```javascript +// Node.js — parse AEGIS_SKILL_PARAMS +let skillParams = {}; +try { skillParams = JSON.parse(process.env.AEGIS_SKILL_PARAMS || '{}'); } catch {} + +const mode = skillParams.mode || 'default'; +const verbose = skillParams.verbose || false; +``` + +```python +# Python — parse AEGIS_SKILL_PARAMS +import os, json + +skill_params = json.loads(os.environ.get('AEGIS_SKILL_PARAMS', '{}')) +mode = skill_params.get('mode', 'default') +verbose = skill_params.get('verbose', False) +``` + +### Precedence + +``` +CLI flags > AEGIS_SKILL_PARAMS > Platform env vars > Defaults +``` + +When a skill supports both CLI arguments and `AEGIS_SKILL_PARAMS`, CLI flags should take priority. Platform-injected env vars (like `AEGIS_GATEWAY_URL`) are always available regardless of `config.yaml`. + +## Gateway as Proxy + +The gateway (`AEGIS_GATEWAY_URL`) is an OpenAI-compatible proxy. Skills call it like any OpenAI endpoint — the gateway handles: + +- **API key management** — user configures keys in Aegis settings +- **Provider routing** — OpenAI, Anthropic, local models +- **Model selection** — user picks model in Aegis UI + +Skills should **not** need raw API keys. If a skill needs direct provider access in the future, Aegis will expose additional env vars (`AEGIS_LLM_API_KEY`, `AEGIS_LLM_PROVIDER`, etc.) — but this is not yet implemented. + +### Example: Calling the Gateway + +```javascript +const gatewayUrl = process.env.AEGIS_GATEWAY_URL || 'http://localhost:5407'; + +const response = await fetch(`${gatewayUrl}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + messages: [{ role: 'user', content: 'Hello' }], + stream: false, + }), +}); +``` + +No API key header needed — the gateway injects it. diff --git a/skills/analysis/home-security-benchmark/SKILL.md b/skills/analysis/home-security-benchmark/SKILL.md index 42cc01a9..c5fba21c 100644 --- a/skills/analysis/home-security-benchmark/SKILL.md +++ b/skills/analysis/home-security-benchmark/SKILL.md @@ -59,6 +59,17 @@ node scripts/run-benchmark.cjs --no-open > **Note**: URLs should be base URLs (e.g. `http://localhost:5405`). The benchmark appends `/v1/chat/completions` automatically. Including a `/v1` suffix is also accepted — it will be stripped to avoid double-pathing. +### User Configuration (config.yaml) + +This skill includes a [`config.yaml`](config.yaml) that defines user-configurable parameters. Aegis parses this at install time and renders a config panel in the UI. Values are delivered via `AEGIS_SKILL_PARAMS`. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `mode` | select | `llm` | Which suites to run: `llm` (96 tests), `vlm` (35 tests), or `full` (131 tests) | +| `noOpen` | boolean | `false` | Skip auto-opening the HTML report in browser | + +Platform parameters like `AEGIS_GATEWAY_URL` and `AEGIS_VLM_URL` are auto-injected by Aegis — they are **not** in `config.yaml`. See [Aegis Skill Platform Parameters](../../../docs/skill-params.md) for the full platform contract. + ### CLI Arguments (standalone fallback) | Argument | Default | Description | diff --git a/skills/analysis/home-security-benchmark/config.yaml b/skills/analysis/home-security-benchmark/config.yaml new file mode 100644 index 00000000..c643fb7c --- /dev/null +++ b/skills/analysis/home-security-benchmark/config.yaml @@ -0,0 +1,13 @@ +params: + - key: mode + label: Test Mode + type: select + options: [llm, vlm, full] + default: llm + description: "Which test suites to run: llm-only, vlm-only, or full" + + - key: noOpen + label: Don't auto-open report + type: boolean + default: false + description: Skip opening the HTML report in browser after completion diff --git a/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs b/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs index 2a72417b..bd9483ea 100644 --- a/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs +++ b/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs @@ -73,18 +73,19 @@ Tests: 131 total (96 LLM + 35 VLM) across 16 suites process.exit(0); } +// Parse skill parameters if running as Aegis skill +let skillParams = {}; +try { skillParams = JSON.parse(process.env.AEGIS_SKILL_PARAMS || '{}'); } catch { } + // Aegis provides config via env vars; CLI args are fallback for standalone const GATEWAY_URL = process.env.AEGIS_GATEWAY_URL || getArg('gateway', 'http://localhost:5407'); const VLM_URL = process.env.AEGIS_VLM_URL || getArg('vlm', ''); const RESULTS_DIR = getArg('out', path.join(os.homedir(), '.aegis-ai', 'benchmarks')); -const NO_OPEN = args.includes('--no-open'); +const IS_SKILL_MODE = !!process.env.AEGIS_SKILL_ID; +const NO_OPEN = args.includes('--no-open') || skillParams.noOpen || false; +const TEST_MODE = skillParams.mode || 'full'; const TIMEOUT_MS = 30000; const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures'); -const IS_SKILL_MODE = !!process.env.AEGIS_SKILL_ID; - -// Parse skill parameters if running as Aegis skill -let skillParams = {}; -try { skillParams = JSON.parse(process.env.AEGIS_SKILL_PARAMS || '{}'); } catch { } // ─── Skill Protocol: JSON lines on stdout, human text on stderr ────────────── @@ -1706,6 +1707,24 @@ async function main() { // Emit ready event (Aegis listens for this) emit({ event: 'ready', model: results.model.name, system: results.system.cpu }); + // Filter suites by test mode (from AEGIS_SKILL_PARAMS or default 'full') + if (TEST_MODE !== 'full') { + const isVlmSuite = (name) => name.includes('VLM Scene') || name.includes('📸'); + const originalCount = suites.length; + if (TEST_MODE === 'llm') { + // Remove VLM image-analysis suites (VLM-to-Alert Triage stays — it's LLM-based text triage) + for (let i = suites.length - 1; i >= 0; i--) { + if (isVlmSuite(suites[i].name)) suites.splice(i, 1); + } + } else if (TEST_MODE === 'vlm') { + // Keep only VLM image-analysis suites (requires VLM URL) + for (let i = suites.length - 1; i >= 0; i--) { + if (!isVlmSuite(suites[i].name)) suites.splice(i, 1); + } + } + log(` Filter: ${TEST_MODE} mode → ${suites.length}/${originalCount} suites selected`); + } + const suiteStart = Date.now(); await runSuites(); results.totals.timeMs = Date.now() - suiteStart;