Skip to content

Commit 9007d79

Browse files
jonathanpeterwujonathanpeterwu
committed
feat(retrieval): Add real LLM provider and retrieval audit system
STA-317: LLM-Driven Context Retrieval - Add AnthropicLLMProvider using official SDK for real LLM analysis - Add RetrievalAuditStore to record retrieval decisions - Auto-create LLM provider when ANTHROPIC_API_KEY is available - Add CLI commands: stackmemory retrieval audit/stats/reasoning/status - Record audit entries with reasoning, confidence, and token usage - Support cleanup of old audit entries Closes STA-317
1 parent b363302 commit 9007d79

5 files changed

Lines changed: 867 additions & 2 deletions

File tree

‎src/cli/commands/retrieval.ts‎

Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
/**
2+
* Retrieval CLI Commands
3+
* View and manage LLM-driven context retrieval settings and audit logs
4+
*/
5+
6+
import { Command } from 'commander';
7+
import Database from 'better-sqlite3';
8+
import { join } from 'path';
9+
import { existsSync } from 'fs';
10+
import chalk from 'chalk';
11+
import { RetrievalAuditStore } from '../../core/retrieval/retrieval-audit.js';
12+
13+
export function createRetrievalCommands(): Command {
14+
const retrieval = new Command('retrieval')
15+
.alias('ret')
16+
.description('Manage LLM-driven context retrieval');
17+
18+
// Audit subcommand
19+
retrieval
20+
.command('audit')
21+
.description('View retrieval audit log')
22+
.option('-l, --limit <n>', 'Number of entries to show', '10')
23+
.option('-q, --query <text>', 'Filter by query text')
24+
.option('-v, --verbose', 'Show full details')
25+
.action(async (options) => {
26+
const projectRoot = process.cwd();
27+
const dbPath = join(projectRoot, '.stackmemory', 'context.db');
28+
29+
if (!existsSync(dbPath)) {
30+
console.log(
31+
chalk.red('StackMemory not initialized in this directory.')
32+
);
33+
console.log(chalk.gray('Run "stackmemory init" first.'));
34+
return;
35+
}
36+
37+
const db = new Database(dbPath);
38+
39+
try {
40+
// Get project ID
41+
let projectId = 'default';
42+
try {
43+
const row = db
44+
.prepare(`SELECT value FROM metadata WHERE key = 'project_id'`)
45+
.get() as any;
46+
if (row?.value) projectId = row.value;
47+
} catch {
48+
// Use default
49+
}
50+
51+
const auditStore = new RetrievalAuditStore(db, projectId);
52+
53+
let entries;
54+
if (options.query) {
55+
entries = auditStore.searchByQuery(
56+
options.query,
57+
parseInt(options.limit)
58+
);
59+
console.log(
60+
chalk.blue(
61+
`\nRetrieval audit entries matching "${options.query}":\n`
62+
)
63+
);
64+
} else {
65+
entries = auditStore.getRecent(parseInt(options.limit));
66+
console.log(chalk.blue('\nRecent retrieval audit entries:\n'));
67+
}
68+
69+
if (entries.length === 0) {
70+
console.log(chalk.gray('No audit entries found.'));
71+
console.log(
72+
chalk.gray(
73+
'\nRetrieval audit records decisions made by the LLM-driven'
74+
)
75+
);
76+
console.log(
77+
chalk.gray(
78+
'context retrieval system. Run some queries to generate entries.'
79+
)
80+
);
81+
return;
82+
}
83+
84+
for (const entry of entries) {
85+
const date = new Date(entry.timestamp).toLocaleString();
86+
const providerIcon =
87+
entry.provider === 'anthropic'
88+
? chalk.green('LLM')
89+
: entry.provider === 'cached'
90+
? chalk.yellow('CACHE')
91+
: chalk.gray('HEUR');
92+
93+
console.log(
94+
`${chalk.cyan(entry.id.slice(0, 8))} ${chalk.gray(date)} [${providerIcon}]`
95+
);
96+
console.log(
97+
` Query: ${chalk.white(entry.query.slice(0, 60))}${entry.query.length > 60 ? '...' : ''}`
98+
);
99+
console.log(
100+
` Confidence: ${formatConfidence(entry.confidenceScore)} | ` +
101+
`Tokens: ${entry.tokensUsed}/${entry.tokenBudget} | ` +
102+
`Time: ${entry.analysisTimeMs}ms | ` +
103+
`Complexity: ${entry.queryComplexity}`
104+
);
105+
106+
if (options.verbose) {
107+
console.log(
108+
` Frames: ${entry.framesRetrieved.join(', ') || 'none'}`
109+
);
110+
console.log(
111+
` Reasoning: ${chalk.gray(entry.reasoning.slice(0, 200))}${entry.reasoning.length > 200 ? '...' : ''}`
112+
);
113+
}
114+
115+
console.log('');
116+
}
117+
} finally {
118+
db.close();
119+
}
120+
});
121+
122+
// Stats subcommand
123+
retrieval
124+
.command('stats')
125+
.description('Show retrieval statistics')
126+
.action(async () => {
127+
const projectRoot = process.cwd();
128+
const dbPath = join(projectRoot, '.stackmemory', 'context.db');
129+
130+
if (!existsSync(dbPath)) {
131+
console.log(
132+
chalk.red('StackMemory not initialized in this directory.')
133+
);
134+
return;
135+
}
136+
137+
const db = new Database(dbPath);
138+
139+
try {
140+
let projectId = 'default';
141+
try {
142+
const row = db
143+
.prepare(`SELECT value FROM metadata WHERE key = 'project_id'`)
144+
.get() as any;
145+
if (row?.value) projectId = row.value;
146+
} catch {
147+
// Use default
148+
}
149+
150+
const auditStore = new RetrievalAuditStore(db, projectId);
151+
const stats = auditStore.getStats();
152+
153+
console.log(chalk.blue('\nRetrieval Statistics\n'));
154+
155+
console.log(`Total retrievals: ${chalk.white(stats.totalRetrievals)}`);
156+
console.log(
157+
`Average confidence: ${formatConfidence(stats.avgConfidence)}`
158+
);
159+
console.log(
160+
`Average tokens used: ${chalk.white(Math.round(stats.avgTokensUsed))}`
161+
);
162+
console.log(
163+
`Average analysis time: ${chalk.white(Math.round(stats.avgAnalysisTime))}ms`
164+
);
165+
166+
console.log('\nProvider breakdown:');
167+
for (const [provider, count] of Object.entries(
168+
stats.providerBreakdown
169+
)) {
170+
const pct =
171+
stats.totalRetrievals > 0
172+
? ((count / stats.totalRetrievals) * 100).toFixed(1)
173+
: '0';
174+
const icon =
175+
provider === 'anthropic'
176+
? chalk.green('LLM')
177+
: provider === 'cached'
178+
? chalk.yellow('CACHE')
179+
: chalk.gray('HEUR');
180+
console.log(` ${icon}: ${count} (${pct}%)`);
181+
}
182+
183+
console.log('');
184+
} finally {
185+
db.close();
186+
}
187+
});
188+
189+
// Reasoning subcommand - show detailed reasoning for a specific entry
190+
retrieval
191+
.command('reasoning <id>')
192+
.description('Show detailed reasoning for a retrieval decision')
193+
.action(async (id) => {
194+
const projectRoot = process.cwd();
195+
const dbPath = join(projectRoot, '.stackmemory', 'context.db');
196+
197+
if (!existsSync(dbPath)) {
198+
console.log(
199+
chalk.red('StackMemory not initialized in this directory.')
200+
);
201+
return;
202+
}
203+
204+
const db = new Database(dbPath);
205+
206+
try {
207+
let projectId = 'default';
208+
try {
209+
const row = db
210+
.prepare(`SELECT value FROM metadata WHERE key = 'project_id'`)
211+
.get() as any;
212+
if (row?.value) projectId = row.value;
213+
} catch {
214+
// Use default
215+
}
216+
217+
const auditStore = new RetrievalAuditStore(db, projectId);
218+
219+
// Try to find entry by partial ID
220+
const entries = auditStore.getRecent(100);
221+
const entry = entries.find((e) => e.id.startsWith(id));
222+
223+
if (!entry) {
224+
console.log(
225+
chalk.red(`No audit entry found with ID starting with "${id}"`)
226+
);
227+
return;
228+
}
229+
230+
console.log(chalk.blue('\nRetrieval Decision Details\n'));
231+
232+
console.log(`ID: ${chalk.white(entry.id)}`);
233+
console.log(
234+
`Time: ${chalk.white(new Date(entry.timestamp).toLocaleString())}`
235+
);
236+
console.log(`Provider: ${chalk.white(entry.provider)}`);
237+
console.log(`Query: ${chalk.white(entry.query)}`);
238+
console.log(`Complexity: ${chalk.white(entry.queryComplexity)}`);
239+
console.log(`Confidence: ${formatConfidence(entry.confidenceScore)}`);
240+
console.log(
241+
`Tokens: ${chalk.white(`${entry.tokensUsed}/${entry.tokenBudget}`)}`
242+
);
243+
console.log(
244+
`Analysis Time: ${chalk.white(`${entry.analysisTimeMs}ms`)}`
245+
);
246+
247+
console.log(chalk.blue('\nReasoning:'));
248+
console.log(entry.reasoning);
249+
250+
console.log(chalk.blue('\nFrames Retrieved:'));
251+
if (entry.framesRetrieved.length === 0) {
252+
console.log(chalk.gray(' (none)'));
253+
} else {
254+
for (const frameId of entry.framesRetrieved) {
255+
console.log(` - ${frameId}`);
256+
}
257+
}
258+
259+
console.log('');
260+
} finally {
261+
db.close();
262+
}
263+
});
264+
265+
// Cleanup subcommand
266+
retrieval
267+
.command('cleanup')
268+
.description('Remove old audit entries')
269+
.option('-d, --days <n>', 'Keep entries from last N days', '7')
270+
.action(async (options) => {
271+
const projectRoot = process.cwd();
272+
const dbPath = join(projectRoot, '.stackmemory', 'context.db');
273+
274+
if (!existsSync(dbPath)) {
275+
console.log(
276+
chalk.red('StackMemory not initialized in this directory.')
277+
);
278+
return;
279+
}
280+
281+
const db = new Database(dbPath);
282+
283+
try {
284+
let projectId = 'default';
285+
try {
286+
const row = db
287+
.prepare(`SELECT value FROM metadata WHERE key = 'project_id'`)
288+
.get() as any;
289+
if (row?.value) projectId = row.value;
290+
} catch {
291+
// Use default
292+
}
293+
294+
const auditStore = new RetrievalAuditStore(db, projectId);
295+
const days = parseInt(options.days);
296+
const maxAgeMs = days * 24 * 60 * 60 * 1000;
297+
const deleted = auditStore.cleanup(maxAgeMs);
298+
299+
console.log(
300+
chalk.green(
301+
`Cleaned up ${deleted} old audit entries (older than ${days} days)`
302+
)
303+
);
304+
} finally {
305+
db.close();
306+
}
307+
});
308+
309+
// Status subcommand - show current retrieval configuration
310+
retrieval
311+
.command('status')
312+
.description('Show current retrieval system status')
313+
.action(async () => {
314+
const hasApiKey = !!process.env['ANTHROPIC_API_KEY'];
315+
const model = process.env['ANTHROPIC_MODEL'] || 'claude-3-haiku-20240307';
316+
317+
console.log(chalk.blue('\nRetrieval System Status\n'));
318+
319+
if (hasApiKey) {
320+
console.log(`LLM Provider: ${chalk.green('Anthropic (active)')}`);
321+
console.log(`Model: ${chalk.white(model)}`);
322+
} else {
323+
console.log(
324+
`LLM Provider: ${chalk.yellow('Heuristic fallback (no API key)')}`
325+
);
326+
console.log(
327+
chalk.gray('Set ANTHROPIC_API_KEY to enable LLM-driven retrieval')
328+
);
329+
}
330+
331+
console.log(`Default Token Budget: ${chalk.white('8000')}`);
332+
console.log(`Confidence Threshold: ${chalk.white('0.6')}`);
333+
console.log(`Cache TTL: ${chalk.white('5 minutes')}`);
334+
335+
console.log('');
336+
});
337+
338+
return retrieval;
339+
}
340+
341+
function formatConfidence(score: number): string {
342+
if (score >= 0.8) return chalk.green(`${(score * 100).toFixed(0)}%`);
343+
if (score >= 0.6) return chalk.yellow(`${(score * 100).toFixed(0)}%`);
344+
return chalk.red(`${(score * 100).toFixed(0)}%`);
345+
}

‎src/cli/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ import { createCleanupProcessesCommand } from './commands/cleanup-processes.js';
5656
import { createAutoBackgroundCommand } from './commands/auto-background.js';
5757
import { createSMSNotifyCommand } from './commands/sms-notify.js';
5858
import { createSettingsCommand } from './commands/settings.js';
59+
import { createRetrievalCommands } from './commands/retrieval.js';
5960
import { ProjectManager } from '../core/projects/project-manager.js';
6061
import Database from 'better-sqlite3';
6162
import { join } from 'path';
@@ -674,6 +675,7 @@ program.addCommand(createCleanupProcessesCommand());
674675
program.addCommand(createAutoBackgroundCommand());
675676
program.addCommand(createSMSNotifyCommand());
676677
program.addCommand(createSettingsCommand());
678+
program.addCommand(createRetrievalCommands());
677679

678680
// Register dashboard command
679681
program

0 commit comments

Comments
 (0)