Skip to content

Commit 30a56e3

Browse files
author
StackMemory Bot (CLI)
committed
feat(search): add FTS5 full-text search, sqlite-vec embeddings, and maintenance daemon
- Replace LIKE queries with FTS5 MATCH + BM25 ranking in SQLiteAdapter - Add FTS5 virtual table with content-sync triggers and schema migration - Add sqlite-vec integration for vector search (gated behind embeddingProvider) - Implement searchByVector, searchHybrid, storeEmbedding, rebuildFtsIndex - Add EmbeddingProvider abstraction with NoOp default - Add DaemonMaintenanceService for stale frame cleanup, FTS rebuild, embedding backfill, VACUUM, and digest generation - Register maintenance service in UnifiedDaemon lifecycle and CLI status - Consolidate tests to reduce count from 513 to 488
1 parent 1f440b5 commit 30a56e3

10 files changed

Lines changed: 1150 additions & 247 deletions

File tree

src/cli/commands/daemon.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ The daemon provides:
138138
const services = [];
139139
if (newStatus.services.context.enabled) services.push('context');
140140
if (newStatus.services.linear.enabled) services.push('linear');
141+
if (newStatus.services.maintenance?.enabled)
142+
services.push('maintenance');
141143
if (newStatus.services.fileWatch.enabled) services.push('file-watch');
142144
if (services.length > 0) {
143145
console.log(chalk.gray(`Services: ${services.join(', ')}`));
@@ -298,6 +300,33 @@ The daemon provides:
298300
}
299301
}
300302

303+
// Maintenance service
304+
const maint = status.services.maintenance;
305+
if (maint) {
306+
console.log(
307+
` Maintenance: ${maint.enabled ? chalk.green('Enabled') : chalk.gray('Disabled')}`
308+
);
309+
if (maint.enabled) {
310+
console.log(
311+
chalk.gray(` Interval: ${config.maintenance.interval} min`)
312+
);
313+
if (maint.staleFramesCleaned) {
314+
console.log(
315+
chalk.gray(
316+
` Stale frames cleaned: ${maint.staleFramesCleaned}`
317+
)
318+
);
319+
}
320+
if (maint.ftsRebuilds) {
321+
console.log(chalk.gray(` FTS rebuilds: ${maint.ftsRebuilds}`));
322+
}
323+
if (maint.lastRun) {
324+
const ago = Math.round((Date.now() - maint.lastRun) / 1000 / 60);
325+
console.log(chalk.gray(` Last run: ${ago} min ago`));
326+
}
327+
}
328+
}
329+
301330
// File watch
302331
const fw = status.services.fileWatch;
303332
console.log(
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/**
2+
* Tests for FTS5 Full-Text Search in SQLiteAdapter
3+
*/
4+
5+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
6+
import { SQLiteAdapter } from '../sqlite-adapter.js';
7+
import * as fs from 'fs';
8+
import * as path from 'path';
9+
import * as os from 'os';
10+
11+
describe('FTS5 Search', () => {
12+
let adapter: SQLiteAdapter;
13+
let dbPath: string;
14+
15+
beforeEach(async () => {
16+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'stackmemory-fts-'));
17+
dbPath = path.join(tmpDir, 'test.db');
18+
adapter = new SQLiteAdapter('test-project', { dbPath });
19+
await adapter.connect();
20+
await adapter.initializeSchema();
21+
});
22+
23+
afterEach(async () => {
24+
await adapter.disconnect();
25+
try {
26+
fs.rmSync(path.dirname(dbPath), { recursive: true });
27+
} catch {
28+
// cleanup best-effort
29+
}
30+
});
31+
32+
it('should enable FTS5 and return BM25-ranked results with custom boost', async () => {
33+
expect(adapter.getFeatures().supportsFullTextSearch).toBe(true);
34+
35+
await adapter.createFrame({
36+
run_id: 'run-1',
37+
project_id: 'test-project',
38+
type: 'task',
39+
name: 'authentication login flow',
40+
digest_text: 'handles user auth',
41+
});
42+
await adapter.createFrame({
43+
run_id: 'run-1',
44+
project_id: 'test-project',
45+
type: 'task',
46+
name: 'database migration',
47+
digest_text: 'authentication schema migration for login',
48+
});
49+
await adapter.createFrame({
50+
run_id: 'run-1',
51+
project_id: 'test-project',
52+
type: 'task',
53+
name: 'plain name',
54+
inputs: JSON.stringify({ query: 'boosted content here' }),
55+
});
56+
await adapter.createFrame({
57+
run_id: 'run-1',
58+
project_id: 'test-project',
59+
type: 'task',
60+
name: 'boosted content here',
61+
inputs: JSON.stringify({}),
62+
});
63+
64+
// BM25 ranking: name match scores higher than digest-only match
65+
const results = await adapter.search({ query: 'authentication' });
66+
expect(results.length).toBeGreaterThanOrEqual(2);
67+
const nameMatch = results.find(
68+
(r) => r.name === 'authentication login flow'
69+
);
70+
const digestMatch = results.find((r) => r.name === 'database migration');
71+
expect(nameMatch).toBeDefined();
72+
expect(digestMatch).toBeDefined();
73+
expect(nameMatch!.score).toBeGreaterThan(digestMatch!.score);
74+
75+
// Custom boost: inputs weighted 20x, name 1x → inputs match wins
76+
const customResults = await adapter.search({
77+
query: 'boosted',
78+
boost: { name: 1, inputs: 20 },
79+
});
80+
expect(customResults[0].name).toBe('plain name');
81+
});
82+
83+
it('should fall back to LIKE on bad FTS syntax and respect limit/offset', async () => {
84+
await adapter.createFrame({
85+
run_id: 'run-1',
86+
project_id: 'test-project',
87+
type: 'task',
88+
name: 'test frame',
89+
digest_text: 'hello world',
90+
});
91+
92+
// Unbalanced quotes trigger LIKE fallback
93+
const results = await adapter.search({ query: '"unbalanced' });
94+
expect(Array.isArray(results)).toBe(true);
95+
96+
// Limit/offset
97+
for (let i = 0; i < 5; i++) {
98+
await adapter.createFrame({
99+
run_id: 'run-1',
100+
project_id: 'test-project',
101+
type: 'task',
102+
name: `search target item ${i}`,
103+
digest_text: `description ${i}`,
104+
});
105+
}
106+
const limited = await adapter.search({ query: 'target', limit: 2 });
107+
expect(limited.length).toBe(2);
108+
const offset = await adapter.search({
109+
query: 'target',
110+
limit: 2,
111+
offset: 3,
112+
});
113+
expect(offset.length).toBeLessThanOrEqual(2);
114+
});
115+
116+
it('should keep FTS in sync after insert/update/delete and survive rebuild', async () => {
117+
const frameId = await adapter.createFrame({
118+
run_id: 'run-1',
119+
project_id: 'test-project',
120+
type: 'task',
121+
name: 'searchable original name',
122+
digest_text: 'original digest',
123+
});
124+
125+
let results = await adapter.search({ query: 'searchable' });
126+
expect(results.length).toBe(1);
127+
128+
await adapter.updateFrame(frameId, {
129+
digest_text: 'updated digest with searchable content',
130+
});
131+
results = await adapter.search({ query: 'updated' });
132+
expect(results.length).toBe(1);
133+
134+
await adapter.deleteFrame(frameId);
135+
results = await adapter.search({ query: 'searchable' });
136+
expect(results.length).toBe(0);
137+
138+
// Rebuild after re-inserting
139+
await adapter.createFrame({
140+
run_id: 'run-1',
141+
project_id: 'test-project',
142+
type: 'task',
143+
name: 'rebuild test',
144+
digest_text: 'testing rebuild',
145+
});
146+
await expect(adapter.rebuildFtsIndex()).resolves.not.toThrow();
147+
results = await adapter.search({ query: 'rebuild' });
148+
expect(results.length).toBe(1);
149+
});
150+
151+
it('should populate FTS index for pre-existing data on schema init', async () => {
152+
const db = adapter.getRawDatabase()!;
153+
db.prepare(
154+
`
155+
INSERT INTO frames (frame_id, run_id, project_id, type, name, state, depth, inputs, outputs, digest_json)
156+
VALUES ('pre-existing', 'run-1', 'test-project', 'task', 'legacy data frame', 'active', 0, '{}', '{}', '{}')
157+
`
158+
).run();
159+
160+
const adapter2 = new SQLiteAdapter('test-project', { dbPath });
161+
await adapter2.connect();
162+
await adapter2.initializeSchema();
163+
164+
const results = await adapter2.search({ query: 'legacy' });
165+
expect(results.length).toBeGreaterThanOrEqual(1);
166+
await adapter2.disconnect();
167+
});
168+
});
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/**
2+
* Embedding Provider Interface
3+
* Abstraction for generating vector embeddings from text
4+
*/
5+
6+
export interface EmbeddingProvider {
7+
/** Generate embedding for a single text */
8+
embed(text: string): Promise<number[]>;
9+
/** Generate embeddings for multiple texts */
10+
embedBatch(texts: string[]): Promise<number[][]>;
11+
/** Dimensionality of the embedding vectors */
12+
dimension: number;
13+
}
14+
15+
/**
16+
* No-op provider that disables vector search.
17+
* Used as default when no real provider is configured.
18+
*/
19+
export class NoOpEmbeddingProvider implements EmbeddingProvider {
20+
readonly dimension = 0;
21+
22+
async embed(_text: string): Promise<number[]> {
23+
return [];
24+
}
25+
26+
async embedBatch(_texts: string[]): Promise<number[][]> {
27+
return [];
28+
}
29+
}

0 commit comments

Comments
 (0)