From bfe8da8cffc388ecc97a1e7117cf7c21c27c8a83 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Fri, 12 Jun 2026 14:37:29 +0800 Subject: [PATCH 01/17] perf: skip settled Masyu strong inference probes at higher budgets --- .../masyu/rules/blackPearlStrongInference.ts | 12 ++++ .../lineComponentEndpointStrongInference.ts | 12 ++++ .../masyu/rules/whitePearlStrongInference.ts | 18 +++++- .../tests/blackPearlStrongInference.test.ts | 57 +++++++++++++++++++ ...neComponentEndpointStrongInference.test.ts | 57 +++++++++++++++++++ .../tests/whitePearlStrongInference.test.ts | 57 +++++++++++++++++++ 6 files changed, 212 insertions(+), 1 deletion(-) diff --git a/src/domain/rules/masyu/rules/blackPearlStrongInference.ts b/src/domain/rules/masyu/rules/blackPearlStrongInference.ts index 3b725a8..13691aa 100644 --- a/src/domain/rules/masyu/rules/blackPearlStrongInference.ts +++ b/src/domain/rules/masyu/rules/blackPearlStrongInference.ts @@ -108,8 +108,13 @@ export const createBlackPearlStrongInferenceRule = ( const budgets = deriveMasyuStrongProbeBudgets( options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, ) + let eligibleCandidates: Set | null = null for (const budget of budgets) { + const exhaustedCandidates = new Set() for (const candidate of candidates) { + if (eligibleCandidates !== null && !eligibleCandidates.has(candidate)) { + continue + } if (Date.now() > deadlineMs) { return null } @@ -127,6 +132,9 @@ export const createBlackPearlStrongInferenceRule = ( return null } if (!result.contradiction) { + if (result.exhausted) { + exhaustedCandidates.add(candidate) + } continue } if ( @@ -161,6 +169,10 @@ export const createBlackPearlStrongInferenceRule = ( ), } } + eligibleCandidates = exhaustedCandidates + if (eligibleCandidates.size === 0) { + break + } } return null diff --git a/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts b/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts index 5ca74de..60cf421 100644 --- a/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts +++ b/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts @@ -102,9 +102,14 @@ export const createLineComponentEndpointStrongInferenceRule = ( const budgets = deriveMasyuStrongProbeBudgets( options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, ) + let eligibleCandidates: Set | null = null for (const budget of budgets) { + const exhaustedCandidates = new Set() for (const candidate of candidates) { + if (eligibleCandidates !== null && !eligibleCandidates.has(candidate)) { + continue + } if (Date.now() > deadlineMs) { return null } @@ -124,6 +129,9 @@ export const createLineComponentEndpointStrongInferenceRule = ( return null } if (!result.contradiction) { + if (result.exhausted) { + exhaustedCandidates.add(candidate) + } continue } @@ -156,6 +164,10 @@ export const createLineComponentEndpointStrongInferenceRule = ( ), } } + eligibleCandidates = exhaustedCandidates + if (eligibleCandidates.size === 0) { + break + } } return null diff --git a/src/domain/rules/masyu/rules/whitePearlStrongInference.ts b/src/domain/rules/masyu/rules/whitePearlStrongInference.ts index e6a0242..454d3ae 100644 --- a/src/domain/rules/masyu/rules/whitePearlStrongInference.ts +++ b/src/domain/rules/masyu/rules/whitePearlStrongInference.ts @@ -169,14 +169,23 @@ export const createWhitePearlStrongInferenceRule = ( const budgets = deriveMasyuStrongProbeBudgets( options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, ) + let eligibleTrialIndexes: Set | null = null for (const budget of budgets) { - for (const pearlCandidate of candidates) { + const exhaustedTrialIndexes = new Set() + for (const [candidateIndex, pearlCandidate] of candidates.entries()) { if (Date.now() > deadlineMs) { return null } for (const assumedIndex of [0, 1] as const) { + const trialIndex = candidateIndex * 2 + assumedIndex + if ( + eligibleTrialIndexes !== null && + !eligibleTrialIndexes.has(trialIndex) + ) { + continue + } const assumed = pearlCandidate.candidates[assumedIndex] const forced = pearlCandidate.candidates[assumedIndex === 0 ? 1 : 0] const assumptions = candidateAssumptions(assumed) @@ -196,6 +205,9 @@ export const createWhitePearlStrongInferenceRule = ( return null } if (!result.contradiction) { + if (result.exhausted) { + exhaustedTrialIndexes.add(trialIndex) + } continue } @@ -226,6 +238,10 @@ export const createWhitePearlStrongInferenceRule = ( } } } + eligibleTrialIndexes = exhaustedTrialIndexes + if (eligibleTrialIndexes.size === 0) { + break + } } return null diff --git a/src/domain/rules/masyu/tests/blackPearlStrongInference.test.ts b/src/domain/rules/masyu/tests/blackPearlStrongInference.test.ts index d8f478c..9d30f2f 100644 --- a/src/domain/rules/masyu/tests/blackPearlStrongInference.test.ts +++ b/src/domain/rules/masyu/tests/blackPearlStrongInference.test.ts @@ -125,6 +125,63 @@ describe('Masyu black pearl strong inference', () => { expect(puzzle.lines[unrelated]?.mark).toBe('unknown') }) + it('does not retry a settled trial at higher budgets', () => { + const puzzle = createMasyuPuzzle(5, 5) + addPearl(puzzle, 2, 2, 'black') + let attempts = 0 + const settledRule: Rule = { + id: 'test-settled-trial', + name: 'Test Settled Trial', + apply: () => { + attempts += 1 + return null + }, + } + + const result = createBlackPearlStrongInferenceRule(() => [settledRule], { + maxCandidates: 1, + maxTrialSteps: 13, + }).apply(puzzle) + + expect(result).toBeNull() + expect(attempts).toBe(1) + }) + + it('retries a budget-limited trial at the next budget', () => { + const puzzle = createMasyuPuzzle(5, 5) + addPearl(puzzle, 2, 2, 'black') + const targetCell = cellKey(0, 0) + let attempts = 0 + const progressingRule: Rule = { + id: 'test-budget-limited-trial', + name: 'Test Budget Limited Trial', + apply: (trial) => { + attempts += 1 + const fromFill = trial.cells[targetCell]?.fill ?? null + return { + message: 'Keep the trial progressing', + diffs: [ + { + kind: 'cell', + cellKey: targetCell, + fromFill, + toFill: fromFill === 'a' ? 'b' : 'a', + }, + ], + affectedCells: [targetCell], + } + }, + } + + const result = createBlackPearlStrongInferenceRule( + () => [progressingRule], + { maxCandidates: 1, maxTrialSteps: 13 }, + ).apply(puzzle) + + expect(result).toBeNull() + expect(attempts).toBe(25) + }) + it('returns null when the trial budget times out', () => { const puzzle = createMasyuPuzzle(5, 5) addPearl(puzzle, 2, 2, 'black') diff --git a/src/domain/rules/masyu/tests/lineComponentEndpointStrongInference.test.ts b/src/domain/rules/masyu/tests/lineComponentEndpointStrongInference.test.ts index b7e9b54..02b4278 100644 --- a/src/domain/rules/masyu/tests/lineComponentEndpointStrongInference.test.ts +++ b/src/domain/rules/masyu/tests/lineComponentEndpointStrongInference.test.ts @@ -208,6 +208,63 @@ describe('Masyu line component endpoint strong inference', () => { expect(puzzle.lines[unrelated]?.mark).toBe('unknown') }) + it('does not retry a settled endpoint trial at higher budgets', () => { + const puzzle = createMasyuPuzzle(3, 4) + markLine(puzzle, lineKey([0, 0], [0, 1]), 'line') + let attempts = 0 + const settledRule: Rule = { + id: 'test-settled-endpoint-trial', + name: 'Test Settled Endpoint Trial', + apply: () => { + attempts += 1 + return null + }, + } + + const result = createLineComponentEndpointStrongInferenceRule( + () => [settledRule], + { maxCandidates: 1, maxTrialSteps: 13 }, + ).apply(puzzle) + + expect(result).toBeNull() + expect(attempts).toBe(1) + }) + + it('retries a budget-limited endpoint trial at the next budget', () => { + const puzzle = createMasyuPuzzle(3, 4) + markLine(puzzle, lineKey([0, 0], [0, 1]), 'line') + const targetCell = cellKey(2, 3) + let attempts = 0 + const progressingRule: Rule = { + id: 'test-budget-limited-endpoint-trial', + name: 'Test Budget Limited Endpoint Trial', + apply: (trial) => { + attempts += 1 + const fromFill = trial.cells[targetCell]?.fill ?? null + return { + message: 'Keep the endpoint trial progressing', + diffs: [ + { + kind: 'cell', + cellKey: targetCell, + fromFill, + toFill: fromFill === 'a' ? 'b' : 'a', + }, + ], + affectedCells: [targetCell], + } + }, + } + + const result = createLineComponentEndpointStrongInferenceRule( + () => [progressingRule], + { maxCandidates: 1, maxTrialSteps: 13 }, + ).apply(puzzle) + + expect(result).toBeNull() + expect(attempts).toBe(25) + }) + it('honors timeout and candidate limits', () => { const puzzle = createMasyuPuzzle(3, 4) markLine(puzzle, lineKey([0, 0], [0, 1]), 'line') diff --git a/src/domain/rules/masyu/tests/whitePearlStrongInference.test.ts b/src/domain/rules/masyu/tests/whitePearlStrongInference.test.ts index 8d87243..aac432d 100644 --- a/src/domain/rules/masyu/tests/whitePearlStrongInference.test.ts +++ b/src/domain/rules/masyu/tests/whitePearlStrongInference.test.ts @@ -128,6 +128,63 @@ describe('Masyu white pearl strong inference', () => { expect(puzzle.lines[unrelated]?.mark).toBe('unknown') }) + it('does not retry settled white-axis trials at higher budgets', () => { + const puzzle = createMasyuPuzzle(5, 5) + addPearl(puzzle, 2, 2, 'white') + let attempts = 0 + const settledRule: Rule = { + id: 'test-settled-white-trial', + name: 'Test Settled White Trial', + apply: () => { + attempts += 1 + return null + }, + } + + const result = createWhitePearlStrongInferenceRule(() => [settledRule], { + maxCandidates: 1, + maxTrialSteps: 13, + }).apply(puzzle) + + expect(result).toBeNull() + expect(attempts).toBe(2) + }) + + it('retries budget-limited white-axis trials at the next budget', () => { + const puzzle = createMasyuPuzzle(5, 5) + addPearl(puzzle, 2, 2, 'white') + const targetCell = cellKey(0, 0) + let attempts = 0 + const progressingRule: Rule = { + id: 'test-budget-limited-white-trial', + name: 'Test Budget Limited White Trial', + apply: (trial) => { + attempts += 1 + const fromFill = trial.cells[targetCell]?.fill ?? null + return { + message: 'Keep the white trial progressing', + diffs: [ + { + kind: 'cell', + cellKey: targetCell, + fromFill, + toFill: fromFill === 'a' ? 'b' : 'a', + }, + ], + affectedCells: [targetCell], + } + }, + } + + const result = createWhitePearlStrongInferenceRule( + () => [progressingRule], + { maxCandidates: 1, maxTrialSteps: 13 }, + ).apply(puzzle) + + expect(result).toBeNull() + expect(attempts).toBe(50) + }) + it('returns null when the white trial budget times out', () => { const puzzle = createMasyuPuzzle(5, 5) addPearl(puzzle, 2, 2, 'white') From a1e27cbbd97f30f05efcc966f98c923a3338264f Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Fri, 12 Jun 2026 16:13:56 +0800 Subject: [PATCH 02/17] chore: minimum solver observer API --- src/domain/rules/engine.test.ts | 127 +++++++++++++++++++++++++++++++- src/domain/rules/engine.ts | 22 ++++++ src/domain/rules/types.ts | 13 ++++ 3 files changed, 160 insertions(+), 2 deletions(-) diff --git a/src/domain/rules/engine.test.ts b/src/domain/rules/engine.test.ts index 29710e5..f4cea82 100644 --- a/src/domain/rules/engine.test.ts +++ b/src/domain/rules/engine.test.ts @@ -1,12 +1,16 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { decodeSlitherFromPuzzlink } from '../parsers/puzzlink' import { tileKey, vertexKey } from '../ir/keys' import { createMasyuPuzzle } from '../ir/masyu' import { applyRuleDiffs, revertRuleDiffs, runNextRule } from './engine' import { slitherRules } from './slither/rules' -import type { Rule, RuleDiff } from './types' +import type { Rule, RuleAttemptEvent, RuleDiff } from './types' describe('rule engine', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + it('finds at least one step for simple zero clue puzzle', () => { const puzzle = decodeSlitherFromPuzzlink( 'https://puzz.link/p?slither/3/3/g0h', @@ -58,6 +62,125 @@ describe('rule engine', () => { ]) }) + it('reports ordered miss and hit rule attempts through an optional observer', () => { + vi.spyOn(performance, 'now') + .mockReturnValueOnce(0) + .mockReturnValueOnce(1) + .mockReturnValueOnce(3) + .mockReturnValueOnce(5) + .mockReturnValueOnce(8) + .mockReturnValueOnce(10) + + const puzzle = createMasyuPuzzle(1, 2) + const line = Object.keys(puzzle.lines)[0] + const events: RuleAttemptEvent[] = [] + const rules: Rule[] = [ + { id: 'miss', name: 'Miss', apply: () => null }, + { + id: 'hit', + name: 'Hit', + apply: () => ({ + message: 'hit', + diffs: [ + { kind: 'line', lineKey: line, from: 'unknown', to: 'line' }, + ], + affectedCells: [], + }), + }, + ] + + const result = runNextRule(puzzle, rules, 7, { + observer: { + onRuleAttemptCompleted: (event) => events.push(event), + }, + }) + + expect(result.step?.ruleId).toBe('hit') + expect(events).toEqual([ + { + solverStepNumber: 7, + ruleId: 'miss', + ruleName: 'Miss', + hit: false, + durationMs: 2, + producedDiffCount: 0, + }, + { + solverStepNumber: 7, + ruleId: 'hit', + ruleName: 'Hit', + hit: true, + durationMs: 3, + producedDiffCount: 1, + }, + ]) + }) + + it('reports every rule in a final no-hit scan', () => { + const puzzle = createMasyuPuzzle(1, 2) + const events: RuleAttemptEvent[] = [] + const rules: Rule[] = [ + { id: 'miss-null', name: 'Miss Null', apply: () => null }, + { + id: 'miss-empty', + name: 'Miss Empty', + apply: () => ({ + message: 'empty', + diffs: [], + affectedCells: [], + }), + }, + ] + + const result = runNextRule(puzzle, rules, 9, { + observer: { + onRuleAttemptCompleted: (event) => events.push(event), + }, + }) + + expect(result).toEqual({ nextPuzzle: puzzle, step: null }) + expect(events.map(({ ruleId, hit, producedDiffCount }) => ({ + ruleId, + hit, + producedDiffCount, + }))).toEqual([ + { ruleId: 'miss-null', hit: false, producedDiffCount: 0 }, + { ruleId: 'miss-empty', hit: false, producedDiffCount: 0 }, + ]) + }) + + it('isolates observer callback errors from solver behavior', () => { + const puzzle = createMasyuPuzzle(1, 2) + const line = Object.keys(puzzle.lines)[0] + const rules: Rule[] = [ + { id: 'miss', name: 'Miss', apply: () => null }, + { + id: 'hit', + name: 'Hit', + apply: () => ({ + message: 'hit', + diffs: [ + { kind: 'line', lineKey: line, from: 'unknown', to: 'line' }, + ], + affectedCells: [], + }), + }, + ] + + const baseline = runNextRule(puzzle, rules, 1) + const observed = runNextRule(puzzle, rules, 1, { + observer: { + onRuleAttemptCompleted: () => { + throw new Error('observer failed') + }, + }, + }) + + expect(observed.step?.ruleId).toBe(baseline.step?.ruleId) + expect(observed.step?.diffs).toEqual(baseline.step?.diffs) + expect(observed.nextPuzzle).toEqual(baseline.nextPuzzle) + }) + it('applies and reverts diffs without mutating input puzzle', () => { const puzzle = decodeSlitherFromPuzzlink( 'https://puzz.link/p?slither/3/3/g0h', diff --git a/src/domain/rules/engine.ts b/src/domain/rules/engine.ts index 3763260..736f909 100644 --- a/src/domain/rules/engine.ts +++ b/src/domain/rules/engine.ts @@ -2,7 +2,9 @@ import type { PuzzleIR } from '../ir/types' import type { Rule, RuleAttempt, + RuleAttemptEvent, RuleDiff, + RunNextRuleOptions, RuleRuntimeContext, RuleStep, } from './types' @@ -157,10 +159,22 @@ export const rewindPuzzleByStep = ( return revertRuleDiffs(puzzle, step.diffs) } +const notifyRuleAttemptCompleted = ( + options: RunNextRuleOptions, + event: RuleAttemptEvent, +): void => { + try { + options.observer?.onRuleAttemptCompleted?.(event) + } catch { + // Observability must never affect solver behavior. + } +} + export const runNextRule = ( puzzle: PuzzleIR, rules: Rule[], stepNumber: number, + options: RunNextRuleOptions = {}, ): { nextPuzzle: PuzzleIR; step: RuleStep | null } => { const startedAt = performance.now() const runtimeContext: RuleRuntimeContext = { @@ -178,6 +192,14 @@ export const runNextRule = ( durationMs: ruleApplyMs, hit, }) + notifyRuleAttemptCompleted(options, { + solverStepNumber: stepNumber, + ruleId: rule.id, + ruleName: rule.name, + durationMs: ruleApplyMs, + hit, + producedDiffCount: hit ? result?.diffs.length ?? 0 : 0, + }) if (!result || result.diffs.length === 0) { continue } diff --git a/src/domain/rules/types.ts b/src/domain/rules/types.ts index 2df6ea6..e5cf601 100644 --- a/src/domain/rules/types.ts +++ b/src/domain/rules/types.ts @@ -136,6 +136,19 @@ export type RuleAttempt = { hit: boolean } +export type RuleAttemptEvent = RuleAttempt & { + solverStepNumber: number + producedDiffCount: number +} + +export type SolverObserver = { + onRuleAttemptCompleted?: (event: RuleAttemptEvent) => void +} + +export type RunNextRuleOptions = { + observer?: SolverObserver +} + export type RuleRuntimeContext = { cache: Map } From 983015b1ff251edcd99c46188294e0cf0d239e84 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Fri, 12 Jun 2026 16:14:45 +0800 Subject: [PATCH 03/17] chore: add 3 masyu puzzle --- dataset/public/masyu.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dataset/public/masyu.json b/dataset/public/masyu.json index 4a17353..33164f8 100644 --- a/dataset/public/masyu.json +++ b/dataset/public/masyu.json @@ -498,6 +498,15 @@ "height": 15, "tags": ["easy", "Medium"], "source": "example.txt" + }, + { + "id": "masyu-25x25-1202688", + "puzzleType": "masyu", + "sourceUrl": "https://puzz.link/p?mashu/25/25/00k900o0j0la9a1120000000312la7b1i9036g6ii900110l303600i42102c953163101f179344169f01309119i300aa909221502f080c3f3079m20163l39f10a03p00300400163fi593j031039kj1i01600000c713b0e09190e23632f6099b040c0f999c020909p00", + "width": 25, + "height": 25, + "tags": ["hard", "Large"], + "source": "example.txt" } ] } From ae8f827683a5bb2b58fd7fdde265cce9822cb7e1 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Fri, 12 Jun 2026 16:55:17 +0800 Subject: [PATCH 04/17] feat: add rule-attempt summary collector --- .../rules/ruleAttemptObserverContract.test.ts | 110 +++++++++++++ .../rules/ruleAttemptSummaryCollector.test.ts | 145 ++++++++++++++++++ .../rules/ruleAttemptSummaryCollector.ts | 135 ++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100644 src/domain/rules/ruleAttemptObserverContract.test.ts create mode 100644 src/domain/rules/ruleAttemptSummaryCollector.test.ts create mode 100644 src/domain/rules/ruleAttemptSummaryCollector.ts diff --git a/src/domain/rules/ruleAttemptObserverContract.test.ts b/src/domain/rules/ruleAttemptObserverContract.test.ts new file mode 100644 index 0000000..84d04ff --- /dev/null +++ b/src/domain/rules/ruleAttemptObserverContract.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest' +import masyuRaw from '../../../dataset/public/masyu.json?raw' +import { analyzeMasyuCompletion } from './masyu/completion' +import { deterministicMasyuRules } from './masyu/rules' +import { decodeMasyuFromPuzzlink } from '../parsers/puzzlink' +import { runNextRule } from './engine' +import { createRuleAttemptSummaryCollector } from './ruleAttemptSummaryCollector' +import type { BenchmarkDatasetManifest } from '../benchmark/types' +import type { RuleStep, SolverObserver } from './types' + +const caseIds = ['masyu-20x20-8268975', 'masyu-25x25-988309'] +const manifest = JSON.parse(masyuRaw) as BenchmarkDatasetManifest + +const normalizeAttempts = (step: RuleStep) => + step.ruleAttempts?.map(({ ruleId, ruleName, hit }) => ({ + ruleId, + ruleName, + hit, + })) + +const solveDeterministically = ( + sourceUrl: string, + observer?: SolverObserver, +) => { + let puzzle = decodeMasyuFromPuzzlink(sourceUrl) + const steps: RuleStep[] = [] + const ruleUsage: Record = {} + const ruleSteps: Record = {} + + while (true) { + const result = runNextRule( + puzzle, + deterministicMasyuRules, + steps.length + 1, + { observer }, + ) + if (!result.step) { + return { + puzzle, + steps, + ruleUsage, + ruleSteps, + terminal: analyzeMasyuCompletion(puzzle), + } + } + + puzzle = result.nextPuzzle + steps.push(result.step) + ruleUsage[result.step.ruleId] = (ruleUsage[result.step.ruleId] ?? 0) + 1 + ruleSteps[result.step.ruleId] = [ + ...(ruleSteps[result.step.ruleId] ?? []), + steps.length, + ] + } +} + +describe.each(caseIds)('rule attempt observer contract: %s', (caseId) => { + it('preserves deterministic solve behavior and produces a consistent summary', () => { + const item = manifest.items.find((candidate) => candidate.id === caseId) + expect(item).toBeDefined() + + const baseline = solveDeterministically(item!.sourceUrl) + const collector = createRuleAttemptSummaryCollector() + const observed = solveDeterministically(item!.sourceUrl, collector.observer) + const summary = collector.getSummary() + + expect(observed.puzzle).toEqual(baseline.puzzle) + expect(observed.terminal).toEqual(baseline.terminal) + expect(observed.steps.map((step) => step.ruleId)).toEqual( + baseline.steps.map((step) => step.ruleId), + ) + expect(observed.steps.map((step) => step.diffs)).toEqual( + baseline.steps.map((step) => step.diffs), + ) + expect(observed.steps.map(normalizeAttempts)).toEqual( + baseline.steps.map(normalizeAttempts), + ) + expect(observed.ruleUsage).toEqual(baseline.ruleUsage) + expect(observed.ruleSteps).toEqual(baseline.ruleSteps) + + const ruleSummaries = Object.values(summary.rules) + expect( + ruleSummaries.reduce((total, rule) => total + rule.attemptCount, 0), + ).toBe(summary.totalAttemptCount) + expect( + ruleSummaries.reduce((total, rule) => total + rule.hitCount, 0), + ).toBe(observed.steps.length) + expect( + ruleSummaries.reduce((total, rule) => total + rule.producedDiffCount, 0), + ).toBe(observed.steps.reduce((total, step) => total + step.diffs.length, 0)) + expect( + ruleSummaries.every( + (rule) => + rule.totalDurationMs >= 0 && + rule.hitDurationMs >= 0 && + rule.missDurationMs >= 0 && + rule.averageDurationMs >= 0, + ), + ).toBe(true) + expect(summary.finalNoHitScan?.solverStepNumber).toBe( + observed.steps.length + 1, + ) + expect(summary.finalNoHitScan?.rules.map((rule) => rule.ruleId)).toEqual( + deterministicMasyuRules.map((rule) => rule.id), + ) + expect(summary.finalNoHitScan?.attemptCount).toBe( + deterministicMasyuRules.length, + ) + }, 20_000) +}) diff --git a/src/domain/rules/ruleAttemptSummaryCollector.test.ts b/src/domain/rules/ruleAttemptSummaryCollector.test.ts new file mode 100644 index 0000000..623201f --- /dev/null +++ b/src/domain/rules/ruleAttemptSummaryCollector.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from 'vitest' +import { createRuleAttemptSummaryCollector } from './ruleAttemptSummaryCollector' +import type { RuleAttemptEvent } from './types' + +const emit = ( + collector: ReturnType, + event: RuleAttemptEvent, +): void => { + collector.observer.onRuleAttemptCompleted?.(event) +} + +describe('rule attempt summary collector', () => { + it('summarizes attempts, durations, hit rate, and produced diffs', () => { + const collector = createRuleAttemptSummaryCollector() + + emit(collector, { + solverStepNumber: 1, + ruleId: 'rule-a', + ruleName: 'Rule A', + hit: false, + durationMs: 2, + producedDiffCount: 0, + }) + emit(collector, { + solverStepNumber: 1, + ruleId: 'rule-b', + ruleName: 'Rule B', + hit: true, + durationMs: 6, + producedDiffCount: 3, + }) + emit(collector, { + solverStepNumber: 2, + ruleId: 'rule-a', + ruleName: 'Rule A', + hit: true, + durationMs: 4, + producedDiffCount: 2, + }) + + expect(collector.getSummary()).toEqual({ + totalAttemptCount: 3, + rules: { + 'rule-a': { + ruleId: 'rule-a', + ruleName: 'Rule A', + attemptCount: 2, + hitCount: 1, + missCount: 1, + hitRate: 0.5, + totalDurationMs: 6, + hitDurationMs: 4, + missDurationMs: 2, + averageDurationMs: 3, + producedDiffCount: 2, + }, + 'rule-b': { + ruleId: 'rule-b', + ruleName: 'Rule B', + attemptCount: 1, + hitCount: 1, + missCount: 0, + hitRate: 1, + totalDurationMs: 6, + hitDurationMs: 6, + missDurationMs: 0, + averageDurationMs: 6, + producedDiffCount: 3, + }, + }, + finalNoHitScan: null, + }) + }) + + it('reports the latest all-miss step as the final no-hit scan', () => { + const collector = createRuleAttemptSummaryCollector() + + emit(collector, { + solverStepNumber: 4, + ruleId: 'rule-a', + ruleName: 'Rule A', + hit: false, + durationMs: 2, + producedDiffCount: 0, + }) + emit(collector, { + solverStepNumber: 4, + ruleId: 'rule-b', + ruleName: 'Rule B', + hit: false, + durationMs: 5, + producedDiffCount: 0, + }) + + expect(collector.getSummary().finalNoHitScan).toEqual({ + solverStepNumber: 4, + totalDurationMs: 7, + attemptCount: 2, + rules: [ + { ruleId: 'rule-a', ruleName: 'Rule A', durationMs: 2 }, + { ruleId: 'rule-b', ruleName: 'Rule B', durationMs: 5 }, + ], + }) + }) + + it('returns no final scan when empty or when the latest step hit', () => { + const emptyCollector = createRuleAttemptSummaryCollector() + expect(emptyCollector.getSummary()).toEqual({ + totalAttemptCount: 0, + rules: {}, + finalNoHitScan: null, + }) + + const hitCollector = createRuleAttemptSummaryCollector() + emit(hitCollector, { + solverStepNumber: 1, + ruleId: 'rule-a', + ruleName: 'Rule A', + hit: true, + durationMs: 1, + producedDiffCount: 1, + }) + expect(hitCollector.getSummary().finalNoHitScan).toBeNull() + }) + + it('returns defensive summary snapshots', () => { + const collector = createRuleAttemptSummaryCollector() + emit(collector, { + solverStepNumber: 1, + ruleId: 'rule-a', + ruleName: 'Rule A', + hit: false, + durationMs: 2, + producedDiffCount: 0, + }) + + const first = collector.getSummary() + first.rules['rule-a'].attemptCount = 99 + first.finalNoHitScan!.rules[0].durationMs = 99 + + const second = collector.getSummary() + expect(second.rules['rule-a'].attemptCount).toBe(1) + expect(second.finalNoHitScan?.rules[0].durationMs).toBe(2) + }) +}) diff --git a/src/domain/rules/ruleAttemptSummaryCollector.ts b/src/domain/rules/ruleAttemptSummaryCollector.ts new file mode 100644 index 0000000..4f0b6ca --- /dev/null +++ b/src/domain/rules/ruleAttemptSummaryCollector.ts @@ -0,0 +1,135 @@ +import type { RuleAttemptEvent, SolverObserver } from './types' + +export type RuleAttemptRuleSummary = { + ruleId: string + ruleName: string + attemptCount: number + hitCount: number + missCount: number + hitRate: number + totalDurationMs: number + hitDurationMs: number + missDurationMs: number + averageDurationMs: number + producedDiffCount: number +} + +export type FinalNoHitScanRuleSummary = { + ruleId: string + ruleName: string + durationMs: number +} + +export type FinalNoHitScanSummary = { + solverStepNumber: number + totalDurationMs: number + attemptCount: number + rules: FinalNoHitScanRuleSummary[] +} + +export type RuleAttemptSummary = { + totalAttemptCount: number + rules: Record + finalNoHitScan: FinalNoHitScanSummary | null +} + +export type RuleAttemptSummaryCollector = { + observer: SolverObserver + getSummary: () => RuleAttemptSummary +} + +type MutableRuleSummary = Omit< + RuleAttemptRuleSummary, + 'hitRate' | 'averageDurationMs' +> + +export const createRuleAttemptSummaryCollector = + (): RuleAttemptSummaryCollector => { + let totalAttemptCount = 0 + const rules = new Map() + let currentStepNumber: number | null = null + let currentStepEvents: RuleAttemptEvent[] = [] + + const observer: SolverObserver = { + onRuleAttemptCompleted: (event) => { + totalAttemptCount += 1 + const existing = rules.get(event.ruleId) ?? { + ruleId: event.ruleId, + ruleName: event.ruleName, + attemptCount: 0, + hitCount: 0, + missCount: 0, + totalDurationMs: 0, + hitDurationMs: 0, + missDurationMs: 0, + producedDiffCount: 0, + } + + existing.attemptCount += 1 + existing.totalDurationMs += event.durationMs + existing.producedDiffCount += event.producedDiffCount + if (event.hit) { + existing.hitCount += 1 + existing.hitDurationMs += event.durationMs + } else { + existing.missCount += 1 + existing.missDurationMs += event.durationMs + } + rules.set(event.ruleId, existing) + + if (currentStepNumber !== event.solverStepNumber) { + currentStepNumber = event.solverStepNumber + currentStepEvents = [] + } + currentStepEvents.push({ ...event }) + }, + } + + return { + observer, + getSummary: () => { + const ruleSummaries = Object.fromEntries( + Array.from(rules.entries(), ([ruleId, rule]) => [ + ruleId, + { + ...rule, + hitRate: + rule.attemptCount === 0 ? 0 : rule.hitCount / rule.attemptCount, + averageDurationMs: + rule.attemptCount === 0 + ? 0 + : rule.totalDurationMs / rule.attemptCount, + }, + ]), + ) + const isFinalNoHitScan = + currentStepNumber !== null && + currentStepEvents.length > 0 && + currentStepEvents.every((event) => !event.hit) + let finalNoHitScan: FinalNoHitScanSummary | null = null + if (isFinalNoHitScan && currentStepNumber !== null) { + finalNoHitScan = { + solverStepNumber: currentStepNumber, + totalDurationMs: currentStepEvents.reduce( + (total, event) => total + event.durationMs, + 0, + ), + attemptCount: currentStepEvents.length, + rules: currentStepEvents.map( + ({ ruleId, ruleName, durationMs }) => ({ + ruleId, + ruleName, + durationMs, + }), + ), + } + } + + return { + totalAttemptCount, + rules: ruleSummaries, + finalNoHitScan, + } + }, + } + } From bf19957af8628ce83ddb53df19496d0190a0ae18 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Fri, 12 Jun 2026 17:36:47 +0800 Subject: [PATCH 05/17] feat: implement strong inference tracking and observer for Masyu rules --- src/app/DatasetPage.test.tsx | 24 +- src/domain/rules/engine.test.ts | 48 +++- src/domain/rules/engine.ts | 19 +- .../masyu/rules/blackPearlStrongInference.ts | 187 +++++++------- .../lineComponentEndpointStrongInference.ts | 182 ++++++++------ .../rules/masyu/rules/strongInference.ts | 66 ++++- .../masyu/rules/whitePearlStrongInference.ts | 185 ++++++++------ .../tests/strongInferenceObserver.test.ts | 233 ++++++++++++++++++ .../strongInferenceSummaryCollector.test.ts | 133 ++++++++++ .../rules/strongInferenceSummaryCollector.ts | 105 ++++++++ src/domain/rules/types.ts | 23 +- src/features/dataset/publicDatasets.test.ts | 14 +- 12 files changed, 940 insertions(+), 279 deletions(-) create mode 100644 src/domain/rules/masyu/tests/strongInferenceObserver.test.ts create mode 100644 src/domain/rules/strongInferenceSummaryCollector.test.ts create mode 100644 src/domain/rules/strongInferenceSummaryCollector.ts diff --git a/src/app/DatasetPage.test.tsx b/src/app/DatasetPage.test.tsx index afdf75d..9e19816 100644 --- a/src/app/DatasetPage.test.tsx +++ b/src/app/DatasetPage.test.tsx @@ -79,7 +79,7 @@ describe('DatasetPage', () => { expect( screen.getByLabelText(/slitherlink-10x10-0001 dataset preview/i), ).toHaveClass('dataset-preview-canvas') - expect(screen.getByText(/showing 56 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 56 \/ 112 puzzles/i)).toBeInTheDocument() }) it('filters by search text, size, and tag', () => { @@ -98,7 +98,7 @@ describe('DatasetPage', () => { expect( screen.queryByRole('heading', { name: 'slitherlink-10x10-0001' }), ).not.toBeInTheDocument() - expect(screen.getByText(/showing 1 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 1 \/ 112 puzzles/i)).toBeInTheDocument() fireEvent.change( screen.getByPlaceholderText(/name, tag, size, type, or url/i), @@ -119,13 +119,13 @@ describe('DatasetPage', () => { expect( screen.queryByRole('heading', { name: 'slitherlink-10x10-0001' }), ).not.toBeInTheDocument() - expect(screen.getByText(/showing 2 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 2 \/ 112 puzzles/i)).toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: /auto-imported/i })) expect( screen.getByRole('button', { name: /auto-imported/i }), ).toHaveAttribute('data-active', 'true') - expect(screen.getByText(/showing 2 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 2 \/ 112 puzzles/i)).toBeInTheDocument() }) it('enables Masyu datasets and resets type-specific filters', () => { @@ -143,7 +143,7 @@ describe('DatasetPage', () => { fireEvent.click(screen.getByRole('button', { name: /auto-imported/i })) fireEvent.change(typeSelect, { target: { value: 'masyu' } }) - expect(screen.getByText(/showing 55 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 56 \/ 112 puzzles/i)).toBeInTheDocument() expect( screen.getByRole('heading', { name: 'masyu-25x25-0001' }), ).toBeInTheDocument() @@ -162,7 +162,7 @@ describe('DatasetPage', () => { fireEvent.change(screen.getByLabelText(/^size$/i), { target: { value: '20 x 20' }, }) - expect(screen.getByText(/showing 2 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 2 \/ 112 puzzles/i)).toBeInTheDocument() expect( screen.getByRole('heading', { name: 'masyu-20x20-6613546' }), ).toBeInTheDocument() @@ -179,7 +179,7 @@ describe('DatasetPage', () => { target: { value: 'masyu-20x20-6613546' }, }, ) - expect(screen.getByText(/showing 1 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 1 \/ 112 puzzles/i)).toBeInTheDocument() }) it('filters Masyu datasets by difficulty and area tags', () => { @@ -190,19 +190,19 @@ describe('DatasetPage', () => { }) fireEvent.click(screen.getByRole('button', { name: 'easy' })) - expect(screen.getByText(/showing 18 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 18 \/ 112 puzzles/i)).toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: 'Small' })) - expect(screen.getByText(/showing 7 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 7 \/ 112 puzzles/i)).toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: 'Medium' })) - expect(screen.getByText(/showing 7 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 7 \/ 112 puzzles/i)).toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: 'Large' })) - expect(screen.getByText(/showing 41 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 42 \/ 112 puzzles/i)).toBeInTheDocument() fireEvent.click(screen.getByRole('button', { name: 'hard' })) - expect(screen.getByText(/showing 37 \/ 111 puzzles/i)).toBeInTheDocument() + expect(screen.getByText(/showing 38 \/ 112 puzzles/i)).toBeInTheDocument() }) it('renders compact action links for each dataset puzzle', () => { diff --git a/src/domain/rules/engine.test.ts b/src/domain/rules/engine.test.ts index f4cea82..113c9ea 100644 --- a/src/domain/rules/engine.test.ts +++ b/src/domain/rules/engine.test.ts @@ -4,7 +4,7 @@ import { tileKey, vertexKey } from '../ir/keys' import { createMasyuPuzzle } from '../ir/masyu' import { applyRuleDiffs, revertRuleDiffs, runNextRule } from './engine' import { slitherRules } from './slither/rules' -import type { Rule, RuleAttemptEvent, RuleDiff } from './types' +import type { Rule, RuleAttemptEvent, RuleDiff, SolverObserver } from './types' describe('rule engine', () => { afterEach(() => { @@ -62,6 +62,32 @@ describe('rule engine', () => { ]) }) + it('provides the step number and observer through the runtime context', () => { + const puzzle = createMasyuPuzzle(1, 2) + const observer: SolverObserver = {} + const seen: Array<{ + solverStepNumber: number | undefined + observer: SolverObserver | undefined + }> = [] + const rules: Rule[] = [ + { + id: 'context', + name: 'Context', + apply: (_puzzle, runtimeContext) => { + seen.push({ + solverStepNumber: runtimeContext?.solverStepNumber, + observer: runtimeContext?.observer, + }) + return null + }, + }, + ] + + runNextRule(puzzle, rules, 17, { observer }) + + expect(seen).toEqual([{ solverStepNumber: 17, observer }]) + }) + it('reports ordered miss and hit rule attempts through an optional observer', () => { vi.spyOn(performance, 'now') .mockReturnValueOnce(0) @@ -81,9 +107,7 @@ describe('rule engine', () => { name: 'Hit', apply: () => ({ message: 'hit', - diffs: [ - { kind: 'line', lineKey: line, from: 'unknown', to: 'line' }, - ], + diffs: [{ kind: 'line', lineKey: line, from: 'unknown', to: 'line' }], affectedCells: [], }), }, @@ -139,11 +163,13 @@ describe('rule engine', () => { }) expect(result).toEqual({ nextPuzzle: puzzle, step: null }) - expect(events.map(({ ruleId, hit, producedDiffCount }) => ({ - ruleId, - hit, - producedDiffCount, - }))).toEqual([ + expect( + events.map(({ ruleId, hit, producedDiffCount }) => ({ + ruleId, + hit, + producedDiffCount, + })), + ).toEqual([ { ruleId: 'miss-null', hit: false, producedDiffCount: 0 }, { ruleId: 'miss-empty', hit: false, producedDiffCount: 0 }, ]) @@ -159,9 +185,7 @@ describe('rule engine', () => { name: 'Hit', apply: () => ({ message: 'hit', - diffs: [ - { kind: 'line', lineKey: line, from: 'unknown', to: 'line' }, - ], + diffs: [{ kind: 'line', lineKey: line, from: 'unknown', to: 'line' }], affectedCells: [], }), }, diff --git a/src/domain/rules/engine.ts b/src/domain/rules/engine.ts index 736f909..7c67bd4 100644 --- a/src/domain/rules/engine.ts +++ b/src/domain/rules/engine.ts @@ -7,6 +7,7 @@ import type { RunNextRuleOptions, RuleRuntimeContext, RuleStep, + StrongInferenceCompletedEvent, } from './types' type WritableBuckets = { @@ -170,6 +171,20 @@ const notifyRuleAttemptCompleted = ( } } +export const notifyStrongInferenceCompleted = ( + runtimeContext: RuleRuntimeContext | undefined, + event: Omit, +): void => { + try { + runtimeContext?.observer?.onStrongInferenceCompleted?.({ + solverStepNumber: runtimeContext.solverStepNumber, + ...event, + }) + } catch { + // Observability must never affect solver behavior. + } +} + export const runNextRule = ( puzzle: PuzzleIR, rules: Rule[], @@ -179,6 +194,8 @@ export const runNextRule = ( const startedAt = performance.now() const runtimeContext: RuleRuntimeContext = { cache: new Map(), + solverStepNumber: stepNumber, + observer: options.observer, } const attempts: RuleAttempt[] = [] for (const rule of rules) { @@ -198,7 +215,7 @@ export const runNextRule = ( ruleName: rule.name, durationMs: ruleApplyMs, hit, - producedDiffCount: hit ? result?.diffs.length ?? 0 : 0, + producedDiffCount: hit ? (result?.diffs.length ?? 0) : 0, }) if (!result || result.diffs.length === 0) { continue diff --git a/src/domain/rules/masyu/rules/blackPearlStrongInference.ts b/src/domain/rules/masyu/rules/blackPearlStrongInference.ts index 13691aa..3467a78 100644 --- a/src/domain/rules/masyu/rules/blackPearlStrongInference.ts +++ b/src/domain/rules/masyu/rules/blackPearlStrongInference.ts @@ -1,8 +1,6 @@ import type { PuzzleIR } from '../../../ir/types' -import type { Rule, RuleApplication } from '../../types' -import { - runMasyuTrialUntilFixpoint, -} from './trial' +import type { Rule, RuleApplication, RuleRuntimeContext } from '../../types' +import { runMasyuTrialUntilFixpoint } from './trial' import { getMasyuBlackPearlKeys } from './pearlSelectors' import { formatMasyuCellKeyLabel, @@ -17,6 +15,7 @@ import { import { buildMasyuStrongBranch, buildMasyuInferenceDetails, + createMasyuStrongInferenceTracker, deriveMasyuStrongProbeBudgets, describeMasyuStrongTrialResult, immediateMasyuStrongContradictionResult, @@ -91,90 +90,112 @@ const buildBranch = ( export const createBlackPearlStrongInferenceRule = ( getDeterministicRules: () => Rule[], options: MasyuStrongInferenceOptions = {}, -): Rule => ({ - id: 'masyu-black-pearl-strong-inference', - name: 'Black Pearl Strong Inference', - apply: (puzzle: PuzzleIR): RuleApplication | null => { - const deterministicRules = getDeterministicRules() - const candidates = collectBlackPearlExitCandidates( - puzzle, - options.maxCandidates ?? STRONG_MAX_CANDIDATES, - ) - if (candidates.length === 0) { - return null - } +): Rule => { + const id = 'masyu-black-pearl-strong-inference' + const name = 'Black Pearl Strong Inference' + return { + id, + name, + apply: ( + puzzle: PuzzleIR, + runtimeContext?: RuleRuntimeContext, + ): RuleApplication | null => { + const deterministicRules = getDeterministicRules() + const candidates = collectBlackPearlExitCandidates( + puzzle, + options.maxCandidates ?? STRONG_MAX_CANDIDATES, + ) + const tracker = createMasyuStrongInferenceTracker( + runtimeContext, + id, + name, + candidates.length, + ) + if (candidates.length === 0) { + tracker.complete('miss') + return null + } - const deadlineMs = Date.now() + (options.maxMs ?? STRONG_MAX_MS) - const budgets = deriveMasyuStrongProbeBudgets( - options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, - ) - let eligibleCandidates: Set | null = null - for (const budget of budgets) { - const exhaustedCandidates = new Set() - for (const candidate of candidates) { - if (eligibleCandidates !== null && !eligibleCandidates.has(candidate)) { - continue - } - if (Date.now() > deadlineMs) { - return null - } + const deadlineMs = Date.now() + (options.maxMs ?? STRONG_MAX_MS) + const budgets = deriveMasyuStrongProbeBudgets( + options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, + ) + let eligibleCandidates: Set | null = null + for (const budget of budgets) { + const exhaustedCandidates = new Set() + for (const candidate of candidates) { + if ( + eligibleCandidates !== null && + !eligibleCandidates.has(candidate) + ) { + continue + } + if (Date.now() > deadlineMs) { + tracker.complete('timeout') + return null + } - const branch = buildBranch(puzzle, candidate) - const result = branch.setupOk - ? runMasyuTrialUntilFixpoint( - branch.puzzle, - deterministicRules, - budget, - deadlineMs, - ) - : immediateMasyuStrongContradictionResult(branch.puzzle) - if (result.timedOut) { - return null - } - if (!result.contradiction) { - if (result.exhausted) { - exhaustedCandidates.add(candidate) + const branch = buildBranch(puzzle, candidate) + const result = branch.setupOk + ? runMasyuTrialUntilFixpoint( + branch.puzzle, + deterministicRules, + budget, + deadlineMs, + ) + : immediateMasyuStrongContradictionResult(branch.puzzle) + tracker.recordProbe(result) + if (result.timedOut) { + tracker.complete('timeout') + return null + } + if (!result.contradiction) { + if (result.exhausted) { + exhaustedCandidates.add(candidate) + } + continue + } + if ( + (puzzle.lines[candidate.firstLine]?.mark ?? 'unknown') !== 'unknown' + ) { + continue } - continue - } - if ( - (puzzle.lines[candidate.firstLine]?.mark ?? 'unknown') !== 'unknown' - ) { - continue - } - const diffs: RuleApplication['diffs'] = [ - { - kind: 'line', - lineKey: candidate.firstLine, - from: 'unknown', - to: 'blank', - }, - ] - return { - message: - `Black Pearl Strong Inference: assuming ${formatMasyuCellKeyLabel(candidate.pearlKey)} exits ${candidate.direction} ` + - `(${branch.setupDescription}) leads to ${describeMasyuStrongTrialResult(result)}, so ${formatMasyuLineLabel( - candidate.firstLine, - )} is crossed out.`, - diffs, - affectedCells: [candidate.pearlKey], - affectedLines: [candidate.firstLine], - inferenceDetails: buildMasyuInferenceDetails( - puzzle, - `Assume ${formatMasyuCellKeyLabel(candidate.pearlKey)} exits ${candidate.direction}`, - branch, - result, + const diffs: RuleApplication['diffs'] = [ + { + kind: 'line', + lineKey: candidate.firstLine, + from: 'unknown', + to: 'blank', + }, + ] + tracker.complete('hit', diffs.length) + return { + message: + `Black Pearl Strong Inference: assuming ${formatMasyuCellKeyLabel(candidate.pearlKey)} exits ${candidate.direction} ` + + `(${branch.setupDescription}) leads to ${describeMasyuStrongTrialResult(result)}, so ${formatMasyuLineLabel( + candidate.firstLine, + )} is crossed out.`, diffs, - ), + affectedCells: [candidate.pearlKey], + affectedLines: [candidate.firstLine], + inferenceDetails: buildMasyuInferenceDetails( + puzzle, + `Assume ${formatMasyuCellKeyLabel(candidate.pearlKey)} exits ${candidate.direction}`, + branch, + result, + diffs, + ), + } + } + eligibleCandidates = exhaustedCandidates + if (eligibleCandidates.size === 0) { + break } } - eligibleCandidates = exhaustedCandidates - if (eligibleCandidates.size === 0) { - break - } - } - return null - }, -}) + tracker.complete('miss') + return null + }, + } +} diff --git a/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts b/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts index 60cf421..501e4aa 100644 --- a/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts +++ b/src/domain/rules/masyu/rules/lineComponentEndpointStrongInference.ts @@ -1,6 +1,6 @@ import { parseCellKey } from '../../../ir/keys' import type { PuzzleIR } from '../../../ir/types' -import type { Rule, RuleApplication } from '../../types' +import type { Rule, RuleApplication, RuleRuntimeContext } from '../../types' import { getMasyuOpenLineComponents } from './lineGraph' import { formatMasyuCellKeyLabel, @@ -12,6 +12,7 @@ import { import { buildMasyuInferenceDetails, buildMasyuStrongBranch, + createMasyuStrongInferenceTracker, deriveMasyuStrongProbeBudgets, describeMasyuStrongTrialResult, immediateMasyuStrongContradictionResult, @@ -82,94 +83,115 @@ const buildCandidateAssumptions = ( export const createLineComponentEndpointStrongInferenceRule = ( getDeterministicRules: () => Rule[], options: MasyuStrongInferenceOptions = {}, -): Rule => ({ - id: 'masyu-line-component-endpoint-strong-inference', - name: 'Masyu Line Component Endpoint Strong Inference', - apply: (puzzle: PuzzleIR): RuleApplication | null => { - const candidates = collectLineComponentEndpointCandidates( - puzzle, - options.maxCandidates ?? - Math.min(ENDPOINT_STRONG_MAX_CANDIDATES, STRONG_MAX_CANDIDATES), - ) - if (candidates.length === 0) { - return null - } +): Rule => { + const id = 'masyu-line-component-endpoint-strong-inference' + const name = 'Masyu Line Component Endpoint Strong Inference' + return { + id, + name, + apply: ( + puzzle: PuzzleIR, + runtimeContext?: RuleRuntimeContext, + ): RuleApplication | null => { + const candidates = collectLineComponentEndpointCandidates( + puzzle, + options.maxCandidates ?? + Math.min(ENDPOINT_STRONG_MAX_CANDIDATES, STRONG_MAX_CANDIDATES), + ) + const tracker = createMasyuStrongInferenceTracker( + runtimeContext, + id, + name, + candidates.length, + ) + if (candidates.length === 0) { + tracker.complete('miss') + return null + } - const deterministicRules = getDeterministicRules() - const deadlineMs = - Date.now() + - (options.maxMs ?? Math.min(ENDPOINT_STRONG_MAX_MS, STRONG_MAX_MS)) - const budgets = deriveMasyuStrongProbeBudgets( - options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, - ) - let eligibleCandidates: Set | null = null + const deterministicRules = getDeterministicRules() + const deadlineMs = + Date.now() + + (options.maxMs ?? Math.min(ENDPOINT_STRONG_MAX_MS, STRONG_MAX_MS)) + const budgets = deriveMasyuStrongProbeBudgets( + options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, + ) + let eligibleCandidates: Set | null = null - for (const budget of budgets) { - const exhaustedCandidates = new Set() - for (const candidate of candidates) { - if (eligibleCandidates !== null && !eligibleCandidates.has(candidate)) { - continue - } - if (Date.now() > deadlineMs) { - return null - } - const branch = buildMasyuStrongBranch( - puzzle, - buildCandidateAssumptions(candidate), - ) - const result = branch.setupOk - ? runMasyuTrialUntilFixpoint( - branch.puzzle, - deterministicRules, - budget, - deadlineMs, - ) - : immediateMasyuStrongContradictionResult(branch.puzzle) - if (result.timedOut) { - return null - } - if (!result.contradiction) { - if (result.exhausted) { - exhaustedCandidates.add(candidate) + for (const budget of budgets) { + const exhaustedCandidates = new Set() + for (const candidate of candidates) { + if ( + eligibleCandidates !== null && + !eligibleCandidates.has(candidate) + ) { + continue + } + if (Date.now() > deadlineMs) { + tracker.complete('timeout') + return null + } + const branch = buildMasyuStrongBranch( + puzzle, + buildCandidateAssumptions(candidate), + ) + const result = branch.setupOk + ? runMasyuTrialUntilFixpoint( + branch.puzzle, + deterministicRules, + budget, + deadlineMs, + ) + : immediateMasyuStrongContradictionResult(branch.puzzle) + tracker.recordProbe(result) + if (result.timedOut) { + tracker.complete('timeout') + return null + } + if (!result.contradiction) { + if (result.exhausted) { + exhaustedCandidates.add(candidate) + } + continue } - continue - } - const diffs: RuleApplication['diffs'] = [ - { - kind: 'line', - lineKey: candidate.assumed.lineKey, - from: 'unknown', - to: 'blank', - }, - ] - return { - message: - `Masyu Line Component Endpoint Strong Inference: assuming the ${candidate.componentEdgeCount}-segment component at ${formatMasyuCellKeyLabel( + const diffs: RuleApplication['diffs'] = [ + { + kind: 'line', + lineKey: candidate.assumed.lineKey, + from: 'unknown', + to: 'blank', + }, + ] + tracker.complete('hit', diffs.length) + return { + message: `Masyu Line Component Endpoint Strong Inference: assuming the ${candidate.componentEdgeCount}-segment component at ${formatMasyuCellKeyLabel( candidate.endpointKey, )} continues ${candidate.assumed.direction} among ${candidate.unknowns.length} candidate directions (${branch.setupDescription}) leads to ${describeMasyuStrongTrialResult( result, )}, so ${formatMasyuLineLabel(candidate.assumed.lineKey)} is crossed out.`, - diffs, - affectedCells: [candidate.endpointKey], - affectedLines: [candidate.assumed.lineKey], - inferenceDetails: buildMasyuInferenceDetails( - puzzle, - `Assume the ${candidate.componentEdgeCount}-segment component at ${formatMasyuCellKeyLabel( - candidate.endpointKey, - )} continues ${candidate.assumed.direction}`, - branch, - result, diffs, - ), + affectedCells: [candidate.endpointKey], + affectedLines: [candidate.assumed.lineKey], + inferenceDetails: buildMasyuInferenceDetails( + puzzle, + `Assume the ${candidate.componentEdgeCount}-segment component at ${formatMasyuCellKeyLabel( + candidate.endpointKey, + )} continues ${candidate.assumed.direction}`, + branch, + result, + diffs, + ), + } + } + eligibleCandidates = exhaustedCandidates + if (eligibleCandidates.size === 0) { + break } } - eligibleCandidates = exhaustedCandidates - if (eligibleCandidates.size === 0) { - break - } - } - return null - }, -}) + tracker.complete('miss') + return null + }, + } +} diff --git a/src/domain/rules/masyu/rules/strongInference.ts b/src/domain/rules/masyu/rules/strongInference.ts index e7c9d3f..6f7f3ab 100644 --- a/src/domain/rules/masyu/rules/strongInference.ts +++ b/src/domain/rules/masyu/rules/strongInference.ts @@ -1,10 +1,13 @@ import { clonePuzzle } from '../../../ir/normalize' import type { LineMark, PuzzleIR } from '../../../ir/types' -import type { InferenceDetails, RuleApplication } from '../../types' -import { - applyMasyuLineAssumption, - type MasyuTrialResult, -} from './trial' +import { notifyStrongInferenceCompleted } from '../../engine' +import type { + InferenceDetails, + RuleApplication, + RuleRuntimeContext, + StrongInferenceOutcome, +} from '../../types' +import { applyMasyuLineAssumption, type MasyuTrialResult } from './trial' import { formatMasyuLineLabel } from './shared' export type MasyuStrongInferenceOptions = { @@ -22,10 +25,63 @@ export type MasyuStrongBranch = { export type MasyuLineAssumption = [lineKey: string, mark: LineMark] +export type MasyuStrongInferenceTracker = { + recordProbe: (result: MasyuTrialResult) => void + complete: ( + outcome: StrongInferenceOutcome, + producedDiffCount?: number, + ) => void +} + export const STRONG_MAX_CANDIDATES = 300 export const STRONG_MAX_TRIAL_STEPS = 100 export const STRONG_MAX_MS = 30000 +const NOOP_STRONG_INFERENCE_TRACKER: MasyuStrongInferenceTracker = { + recordProbe: () => {}, + complete: () => {}, +} + +export const createMasyuStrongInferenceTracker = ( + runtimeContext: RuleRuntimeContext | undefined, + ruleId: string, + ruleName: string, + candidateCount: number, +): MasyuStrongInferenceTracker => { + if (!runtimeContext?.observer?.onStrongInferenceCompleted) { + return NOOP_STRONG_INFERENCE_TRACKER + } + + let probeCount = 0 + let trialStepCount = 0 + let probeDurationMs = 0 + let completed = false + + return { + recordProbe: (result) => { + probeCount += 1 + trialStepCount += result.stepsRun + probeDurationMs += result.elapsedMs + }, + complete: (outcome, producedDiffCount = 0) => { + if (completed) { + return + } + completed = true + notifyStrongInferenceCompleted(runtimeContext, { + ruleId, + ruleName, + candidateCount, + probeCount, + trialStepCount, + probeDurationMs, + outcome, + producedDiffCount, + }) + }, + } +} + export const deriveMasyuStrongProbeBudgets = ( maxTrialSteps: number, ): number[] => { diff --git a/src/domain/rules/masyu/rules/whitePearlStrongInference.ts b/src/domain/rules/masyu/rules/whitePearlStrongInference.ts index 454d3ae..29e71e6 100644 --- a/src/domain/rules/masyu/rules/whitePearlStrongInference.ts +++ b/src/domain/rules/masyu/rules/whitePearlStrongInference.ts @@ -1,6 +1,6 @@ import { cellKey, parseCellKey } from '../../../ir/keys' import type { PuzzleIR } from '../../../ir/types' -import type { Rule, RuleApplication } from '../../types' +import type { Rule, RuleApplication, RuleRuntimeContext } from '../../types' import { createMasyuLineDecisionCollector } from './decisionCollector' import { createMasyuLookaheadContext } from './lookahead' import type { WhitePearlCandidate } from './pearlCandidates' @@ -16,6 +16,7 @@ import { import { buildMasyuStrongBranch, buildMasyuInferenceDetails, + createMasyuStrongInferenceTracker, deriveMasyuStrongProbeBudgets, describeMasyuStrongTrialResult, immediateMasyuStrongContradictionResult, @@ -79,7 +80,9 @@ const getDecidedLocalLineCount = ( const localLines = new Set() for (const key of localCells) { - for (const line of Object.values(getMasyuIncidentDirectionalLines(puzzle, key))) { + for (const line of Object.values( + getMasyuIncidentDirectionalLines(puzzle, key), + )) { if (line) { localLines.add(line.lineKey) } @@ -133,7 +136,9 @@ const candidateAssumptions = ( const collectCandidateDiffs = ( puzzle: PuzzleIR, candidate: WhitePearlCandidate, -): ReturnType['diffs']> | null => { +): ReturnType< + ReturnType['diffs'] +> | null => { const overlay = masyuPearlCandidateToOverlay(candidate) if (!overlay) { return null @@ -152,98 +157,116 @@ const collectCandidateDiffs = ( export const createWhitePearlStrongInferenceRule = ( getDeterministicRules: () => Rule[], options: MasyuStrongInferenceOptions = {}, -): Rule => ({ - id: 'masyu-white-pearl-strong-inference', - name: 'White Pearl Strong Inference', - apply: (puzzle: PuzzleIR): RuleApplication | null => { - const deterministicRules = getDeterministicRules() - const candidates = collectWhitePearlAxisCandidates( - puzzle, - options.maxCandidates ?? STRONG_MAX_CANDIDATES, - ) - if (candidates.length === 0) { - return null - } - - const deadlineMs = Date.now() + (options.maxMs ?? STRONG_MAX_MS) - const budgets = deriveMasyuStrongProbeBudgets( - options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, - ) - let eligibleTrialIndexes: Set | null = null +): Rule => { + const id = 'masyu-white-pearl-strong-inference' + const name = 'White Pearl Strong Inference' + return { + id, + name, + apply: ( + puzzle: PuzzleIR, + runtimeContext?: RuleRuntimeContext, + ): RuleApplication | null => { + const deterministicRules = getDeterministicRules() + const candidates = collectWhitePearlAxisCandidates( + puzzle, + options.maxCandidates ?? STRONG_MAX_CANDIDATES, + ) + const tracker = createMasyuStrongInferenceTracker( + runtimeContext, + id, + name, + candidates.length, + ) + if (candidates.length === 0) { + tracker.complete('miss') + return null + } - for (const budget of budgets) { - const exhaustedTrialIndexes = new Set() - for (const [candidateIndex, pearlCandidate] of candidates.entries()) { - if (Date.now() > deadlineMs) { - return null - } + const deadlineMs = Date.now() + (options.maxMs ?? STRONG_MAX_MS) + const budgets = deriveMasyuStrongProbeBudgets( + options.maxTrialSteps ?? STRONG_MAX_TRIAL_STEPS, + ) + let eligibleTrialIndexes: Set | null = null - for (const assumedIndex of [0, 1] as const) { - const trialIndex = candidateIndex * 2 + assumedIndex - if ( - eligibleTrialIndexes !== null && - !eligibleTrialIndexes.has(trialIndex) - ) { - continue - } - const assumed = pearlCandidate.candidates[assumedIndex] - const forced = pearlCandidate.candidates[assumedIndex === 0 ? 1 : 0] - const assumptions = candidateAssumptions(assumed) - if (assumptions.length === 0) { - continue - } - const branch = buildMasyuStrongBranch(puzzle, assumptions) - const result = branch.setupOk - ? runMasyuTrialUntilFixpoint( - branch.puzzle, - deterministicRules, - budget, - deadlineMs, - ) - : immediateMasyuStrongContradictionResult(branch.puzzle) - if (result.timedOut) { + for (const budget of budgets) { + const exhaustedTrialIndexes = new Set() + for (const [candidateIndex, pearlCandidate] of candidates.entries()) { + if (Date.now() > deadlineMs) { + tracker.complete('timeout') return null } - if (!result.contradiction) { - if (result.exhausted) { - exhaustedTrialIndexes.add(trialIndex) + + for (const assumedIndex of [0, 1] as const) { + const trialIndex = candidateIndex * 2 + assumedIndex + if ( + eligibleTrialIndexes !== null && + !eligibleTrialIndexes.has(trialIndex) + ) { + continue + } + const assumed = pearlCandidate.candidates[assumedIndex] + const forced = pearlCandidate.candidates[assumedIndex === 0 ? 1 : 0] + const assumptions = candidateAssumptions(assumed) + if (assumptions.length === 0) { + continue + } + const branch = buildMasyuStrongBranch(puzzle, assumptions) + const result = branch.setupOk + ? runMasyuTrialUntilFixpoint( + branch.puzzle, + deterministicRules, + budget, + deadlineMs, + ) + : immediateMasyuStrongContradictionResult(branch.puzzle) + tracker.recordProbe(result) + if (result.timedOut) { + tracker.complete('timeout') + return null + } + if (!result.contradiction) { + if (result.exhausted) { + exhaustedTrialIndexes.add(trialIndex) + } + continue } - continue - } - const diffs = collectCandidateDiffs(puzzle, forced) - if (!diffs || diffs.length === 0) { - continue - } - const firstLine = diffs[0]?.lineKey - return { - message: - `White Pearl Strong Inference: assuming ${formatMasyuCellKeyLabel(pearlCandidate.pearlKey)} goes ${axisLabel( + const diffs = collectCandidateDiffs(puzzle, forced) + if (!diffs || diffs.length === 0) { + continue + } + const firstLine = diffs[0]?.lineKey + tracker.complete('hit', diffs.length) + return { + message: `White Pearl Strong Inference: assuming ${formatMasyuCellKeyLabel(pearlCandidate.pearlKey)} goes ${axisLabel( assumed.axis, )} (${branch.setupDescription}) leads to ${describeMasyuStrongTrialResult( result, )}, so it must go ${axisLabel(forced.axis)} through ${formatMasyuLineLabel( firstLine, )}${diffs.length > 1 ? ` (${diffs.length} total)` : ''}.`, - diffs, - affectedCells: [pearlCandidate.pearlKey], - affectedLines: diffs.map((diff) => diff.lineKey), - inferenceDetails: buildMasyuInferenceDetails( - puzzle, - `Assume ${formatMasyuCellKeyLabel(pearlCandidate.pearlKey)} goes ${axisLabel(assumed.axis)}`, - branch, - result, diffs, - ), + affectedCells: [pearlCandidate.pearlKey], + affectedLines: diffs.map((diff) => diff.lineKey), + inferenceDetails: buildMasyuInferenceDetails( + puzzle, + `Assume ${formatMasyuCellKeyLabel(pearlCandidate.pearlKey)} goes ${axisLabel(assumed.axis)}`, + branch, + result, + diffs, + ), + } } } + eligibleTrialIndexes = exhaustedTrialIndexes + if (eligibleTrialIndexes.size === 0) { + break + } } - eligibleTrialIndexes = exhaustedTrialIndexes - if (eligibleTrialIndexes.size === 0) { - break - } - } - return null - }, -}) + tracker.complete('miss') + return null + }, + } +} diff --git a/src/domain/rules/masyu/tests/strongInferenceObserver.test.ts b/src/domain/rules/masyu/tests/strongInferenceObserver.test.ts new file mode 100644 index 0000000..d890615 --- /dev/null +++ b/src/domain/rules/masyu/tests/strongInferenceObserver.test.ts @@ -0,0 +1,233 @@ +import { describe, expect, it } from 'vitest' +import { cellKey, lineKey, tileKey } from '../../../ir/keys' +import { createMasyuPuzzle } from '../../../ir/masyu' +import { runNextRule } from '../../engine' +import type { + Rule, + RuleRuntimeContext, + StrongInferenceCompletedEvent, +} from '../../types' +import { createBlackPearlStrongInferenceRule } from '../rules/blackPearlStrongInference' +import { createLineComponentEndpointStrongInferenceRule } from '../rules/lineComponentEndpointStrongInference' +import { createWhitePearlStrongInferenceRule } from '../rules/whitePearlStrongInference' +import { addPearl, markLine } from './testUtils' + +const observe = ( + apply: (runtimeContext: RuleRuntimeContext) => unknown, +): StrongInferenceCompletedEvent[] => { + const events: StrongInferenceCompletedEvent[] = [] + apply({ + cache: new Map(), + solverStepNumber: 7, + observer: { + onStrongInferenceCompleted: (event) => events.push(event), + }, + }) + return events +} + +describe('Masyu strong inference observer', () => { + it('reports one zero-candidate miss for each strong rule', () => { + const puzzle = createMasyuPuzzle(3, 3) + const rules = [ + createBlackPearlStrongInferenceRule(() => []), + createLineComponentEndpointStrongInferenceRule(() => []), + createWhitePearlStrongInferenceRule(() => []), + ] + + for (const rule of rules) { + const events = observe((runtimeContext) => + rule.apply(puzzle, runtimeContext), + ) + expect(events).toEqual([ + { + solverStepNumber: 7, + ruleId: rule.id, + ruleName: rule.name, + candidateCount: 0, + probeCount: 0, + trialStepCount: 0, + probeDurationMs: 0, + outcome: 'miss', + producedDiffCount: 0, + }, + ]) + } + }) + + it('reports black-pearl hit, timeout, setup contradiction, and gradient retries', () => { + const hitPuzzle = createMasyuPuzzle(5, 5) + addPearl(hitPuzzle, 2, 2, 'black') + markLine(hitPuzzle, lineKey([1, 1], [1, 2]), 'line') + markLine(hitPuzzle, lineKey([1, 2], [1, 3]), 'line') + const hitRule = createBlackPearlStrongInferenceRule(() => []) + const hitEvents = observe((runtimeContext) => + hitRule.apply(hitPuzzle, runtimeContext), + ) + expect(hitEvents).toHaveLength(1) + expect(hitEvents[0]).toMatchObject({ + outcome: 'hit', + probeCount: 1, + trialStepCount: 0, + producedDiffCount: 1, + }) + + const setupPuzzle = createMasyuPuzzle(5, 5) + addPearl(setupPuzzle, 2, 2, 'black') + markLine(setupPuzzle, lineKey([0, 2], [1, 2]), 'blank') + const setupRule = createBlackPearlStrongInferenceRule(() => [], { + maxCandidates: 1, + }) + const setupEvents = observe((runtimeContext) => + setupRule.apply(setupPuzzle, runtimeContext), + ) + expect(setupEvents[0]).toMatchObject({ + outcome: 'hit', + probeCount: 1, + trialStepCount: 0, + probeDurationMs: 0, + }) + + const timeoutPuzzle = createMasyuPuzzle(5, 5) + addPearl(timeoutPuzzle, 2, 2, 'black') + const timeoutRule = createBlackPearlStrongInferenceRule(() => [], { + maxMs: -1, + }) + const timeoutEvents = observe((runtimeContext) => + timeoutRule.apply(timeoutPuzzle, runtimeContext), + ) + expect(timeoutEvents[0]).toMatchObject({ + outcome: 'timeout', + probeCount: 0, + producedDiffCount: 0, + }) + + const retryPuzzle = createMasyuPuzzle(5, 5) + addPearl(retryPuzzle, 2, 2, 'black') + const targetCell = cellKey(0, 0) + const progressingRule: Rule = { + id: 'test-progress', + name: 'Test Progress', + apply: (trial) => { + const fromFill = trial.cells[targetCell]?.fill ?? null + return { + message: 'progress', + diffs: [ + { + kind: 'cell', + cellKey: targetCell, + fromFill, + toFill: fromFill === 'a' ? 'b' : 'a', + }, + ], + affectedCells: [targetCell], + } + }, + } + const retryRule = createBlackPearlStrongInferenceRule( + () => [progressingRule], + { maxCandidates: 1, maxTrialSteps: 13 }, + ) + const retryEvents = observe((runtimeContext) => + retryRule.apply(retryPuzzle, runtimeContext), + ) + expect(retryEvents[0]).toMatchObject({ + outcome: 'miss', + candidateCount: 1, + probeCount: 2, + trialStepCount: 25, + producedDiffCount: 0, + }) + }) + + it('reports endpoint and white-pearl hits with formal diff counts', () => { + const endpointPuzzle = createMasyuPuzzle(3, 4) + markLine(endpointPuzzle, lineKey([0, 0], [0, 1]), 'line') + markLine(endpointPuzzle, lineKey([0, 2], [0, 3]), 'line') + markLine(endpointPuzzle, lineKey([0, 2], [1, 2]), 'line') + const endpointRule = createLineComponentEndpointStrongInferenceRule( + () => [], + ) + const endpointEvents = observe((runtimeContext) => + endpointRule.apply(endpointPuzzle, runtimeContext), + ) + expect(endpointEvents).toHaveLength(1) + expect(endpointEvents[0]).toMatchObject({ + outcome: 'hit', + probeCount: 1, + producedDiffCount: 1, + }) + + const whitePuzzle = createMasyuPuzzle(5, 5) + addPearl(whitePuzzle, 2, 2, 'white') + whitePuzzle.tiles[tileKey(2, 2)] = { fill: 'yellow' } + whitePuzzle.tiles[tileKey(2, 3)] = { fill: 'yellow' } + const whiteRule = createWhitePearlStrongInferenceRule(() => []) + const whiteEvents = observe((runtimeContext) => + whiteRule.apply(whitePuzzle, runtimeContext), + ) + expect(whiteEvents).toHaveLength(1) + expect(whiteEvents[0]).toMatchObject({ + outcome: 'hit', + candidateCount: 1, + probeCount: 1, + producedDiffCount: 4, + }) + }) + + it('keeps internal trial attempts unobserved and isolates observer errors', () => { + const puzzle = createMasyuPuzzle(5, 5) + addPearl(puzzle, 2, 2, 'black') + const north = lineKey([1, 2], [2, 2]) + const west = lineKey([1, 1], [1, 2]) + const east = lineKey([1, 2], [1, 3]) + const downstreamRule: Rule = { + id: 'test-downstream', + name: 'Test Downstream', + apply: (trial) => + trial.lines[north]?.mark === 'line' + ? { + message: 'contradiction', + diffs: [ + { kind: 'line', lineKey: west, from: 'unknown', to: 'line' }, + { kind: 'line', lineKey: east, from: 'unknown', to: 'line' }, + ], + affectedCells: [], + } + : null, + } + const strongRule = createBlackPearlStrongInferenceRule(() => [ + downstreamRule, + ]) + const ruleAttempts: string[] = [] + const strongEvents: StrongInferenceCompletedEvent[] = [] + + const observed = runNextRule(puzzle, [strongRule], 9, { + observer: { + onRuleAttemptCompleted: (event) => ruleAttempts.push(event.ruleId), + onStrongInferenceCompleted: (event) => strongEvents.push(event), + }, + }) + const throwing = runNextRule(puzzle, [strongRule], 9, { + observer: { + onRuleAttemptCompleted: () => { + throw new Error('rule observer failed') + }, + onStrongInferenceCompleted: () => { + throw new Error('strong observer failed') + }, + }, + }) + + expect(ruleAttempts).toEqual([strongRule.id]) + expect(strongEvents).toHaveLength(1) + expect(strongEvents[0]).toMatchObject({ + solverStepNumber: 9, + ruleId: strongRule.id, + outcome: 'hit', + trialStepCount: 1, + }) + expect(throwing.step?.diffs).toEqual(observed.step?.diffs) + expect(throwing.nextPuzzle).toEqual(observed.nextPuzzle) + }) +}) diff --git a/src/domain/rules/strongInferenceSummaryCollector.test.ts b/src/domain/rules/strongInferenceSummaryCollector.test.ts new file mode 100644 index 0000000..0dc68e2 --- /dev/null +++ b/src/domain/rules/strongInferenceSummaryCollector.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest' +import { createStrongInferenceSummaryCollector } from './strongInferenceSummaryCollector' +import type { StrongInferenceCompletedEvent } from './types' + +const emit = ( + collector: ReturnType, + event: StrongInferenceCompletedEvent, +): void => { + collector.observer.onStrongInferenceCompleted?.(event) +} + +describe('strong inference summary collector', () => { + it('summarizes outcomes and work while preserving first-observed rule order', () => { + const collector = createStrongInferenceSummaryCollector() + + emit(collector, { + solverStepNumber: 1, + ruleId: 'rule-b', + ruleName: 'Rule B', + candidateCount: 2, + probeCount: 3, + trialStepCount: 5, + probeDurationMs: 7, + outcome: 'miss', + producedDiffCount: 0, + }) + emit(collector, { + solverStepNumber: 2, + ruleId: 'rule-a', + ruleName: 'Rule A', + candidateCount: 1, + probeCount: 1, + trialStepCount: 2, + probeDurationMs: 4, + outcome: 'hit', + producedDiffCount: 3, + }) + emit(collector, { + solverStepNumber: 3, + ruleId: 'rule-b', + ruleName: 'Rule B', + candidateCount: 4, + probeCount: 2, + trialStepCount: 1, + probeDurationMs: 6, + outcome: 'timeout', + producedDiffCount: 0, + }) + + expect(collector.getSummary()).toEqual({ + totals: { + attemptCount: 3, + hitCount: 1, + missCount: 1, + timeoutCount: 1, + hitRate: 1 / 3, + candidateCount: 7, + probeCount: 6, + trialStepCount: 8, + probeDurationMs: 17, + producedDiffCount: 3, + }, + rules: [ + { + ruleId: 'rule-b', + ruleName: 'Rule B', + attemptCount: 2, + hitCount: 0, + missCount: 1, + timeoutCount: 1, + hitRate: 0, + candidateCount: 6, + probeCount: 5, + trialStepCount: 6, + probeDurationMs: 13, + producedDiffCount: 0, + }, + { + ruleId: 'rule-a', + ruleName: 'Rule A', + attemptCount: 1, + hitCount: 1, + missCount: 0, + timeoutCount: 0, + hitRate: 1, + candidateCount: 1, + probeCount: 1, + trialStepCount: 2, + probeDurationMs: 4, + producedDiffCount: 3, + }, + ], + }) + }) + + it('returns empty and defensive snapshots', () => { + const collector = createStrongInferenceSummaryCollector() + expect(collector.getSummary()).toEqual({ + totals: { + attemptCount: 0, + hitCount: 0, + missCount: 0, + timeoutCount: 0, + hitRate: 0, + candidateCount: 0, + probeCount: 0, + trialStepCount: 0, + probeDurationMs: 0, + producedDiffCount: 0, + }, + rules: [], + }) + + emit(collector, { + solverStepNumber: 1, + ruleId: 'rule-a', + ruleName: 'Rule A', + candidateCount: 1, + probeCount: 2, + trialStepCount: 3, + probeDurationMs: 4, + outcome: 'hit', + producedDiffCount: 1, + }) + const first = collector.getSummary() + first.totals.attemptCount = 99 + first.rules[0].probeCount = 99 + + const second = collector.getSummary() + expect(second.totals.attemptCount).toBe(1) + expect(second.rules[0].probeCount).toBe(2) + }) +}) diff --git a/src/domain/rules/strongInferenceSummaryCollector.ts b/src/domain/rules/strongInferenceSummaryCollector.ts new file mode 100644 index 0000000..ce93c50 --- /dev/null +++ b/src/domain/rules/strongInferenceSummaryCollector.ts @@ -0,0 +1,105 @@ +import type { SolverObserver, StrongInferenceCompletedEvent } from './types' + +export type StrongInferenceTotals = { + attemptCount: number + hitCount: number + missCount: number + timeoutCount: number + hitRate: number + candidateCount: number + probeCount: number + trialStepCount: number + probeDurationMs: number + producedDiffCount: number +} + +export type StrongInferenceRuleSummary = StrongInferenceTotals & { + ruleId: string + ruleName: string +} + +export type StrongInferenceSummary = { + totals: StrongInferenceTotals + rules: StrongInferenceRuleSummary[] +} + +export type StrongInferenceSummaryCollector = { + observer: SolverObserver + getSummary: () => StrongInferenceSummary +} + +type MutableStrongInferenceTotals = Omit +type MutableStrongInferenceRuleSummary = MutableStrongInferenceTotals & { + ruleId: string + ruleName: string +} + +const emptyTotals = (): MutableStrongInferenceTotals => ({ + attemptCount: 0, + hitCount: 0, + missCount: 0, + timeoutCount: 0, + candidateCount: 0, + probeCount: 0, + trialStepCount: 0, + probeDurationMs: 0, + producedDiffCount: 0, +}) + +const addEvent = ( + summary: MutableStrongInferenceTotals, + event: StrongInferenceCompletedEvent, +): void => { + summary.attemptCount += 1 + summary.candidateCount += event.candidateCount + summary.probeCount += event.probeCount + summary.trialStepCount += event.trialStepCount + summary.probeDurationMs += event.probeDurationMs + summary.producedDiffCount += event.producedDiffCount + if (event.outcome === 'hit') { + summary.hitCount += 1 + } else if (event.outcome === 'miss') { + summary.missCount += 1 + } else { + summary.timeoutCount += 1 + } +} + +const snapshotTotals = ( + totals: MutableStrongInferenceTotals, +): StrongInferenceTotals => ({ + ...totals, + hitRate: + totals.attemptCount === 0 ? 0 : totals.hitCount / totals.attemptCount, +}) + +export const createStrongInferenceSummaryCollector = + (): StrongInferenceSummaryCollector => { + const totals = emptyTotals() + const rules = new Map() + + const observer: SolverObserver = { + onStrongInferenceCompleted: (event) => { + addEvent(totals, event) + const rule = rules.get(event.ruleId) ?? { + ...emptyTotals(), + ruleId: event.ruleId, + ruleName: event.ruleName, + } + addEvent(rule, event) + rules.set(event.ruleId, rule) + }, + } + + return { + observer, + getSummary: () => ({ + totals: snapshotTotals(totals), + rules: Array.from(rules.values(), (rule) => ({ + ruleId: rule.ruleId, + ruleName: rule.ruleName, + ...snapshotTotals(rule), + })), + }), + } + } diff --git a/src/domain/rules/types.ts b/src/domain/rules/types.ts index e5cf601..e8d5bb2 100644 --- a/src/domain/rules/types.ts +++ b/src/domain/rules/types.ts @@ -93,7 +93,11 @@ export type InferenceBranch = { } export type InferenceDetails = { - kind: 'slither-strong' | 'slither-color-assumption' | 'slither-sector-parity' | 'masyu-strong' + kind: + | 'slither-strong' + | 'slither-color-assumption' + | 'slither-sector-parity' + | 'masyu-strong' conclusion: 'opposite-branch' | 'shared-consequence' basePuzzle: PuzzleIR defaultBranchId: string @@ -141,8 +145,23 @@ export type RuleAttemptEvent = RuleAttempt & { producedDiffCount: number } +export type StrongInferenceOutcome = 'hit' | 'miss' | 'timeout' + +export type StrongInferenceCompletedEvent = { + solverStepNumber: number + ruleId: string + ruleName: string + candidateCount: number + probeCount: number + trialStepCount: number + probeDurationMs: number + outcome: StrongInferenceOutcome + producedDiffCount: number +} + export type SolverObserver = { onRuleAttemptCompleted?: (event: RuleAttemptEvent) => void + onStrongInferenceCompleted?: (event: StrongInferenceCompletedEvent) => void } export type RunNextRuleOptions = { @@ -151,6 +170,8 @@ export type RunNextRuleOptions = { export type RuleRuntimeContext = { cache: Map + solverStepNumber: number + observer?: SolverObserver } export type Rule = { diff --git a/src/features/dataset/publicDatasets.test.ts b/src/features/dataset/publicDatasets.test.ts index 608ef59..a3fe0dc 100644 --- a/src/features/dataset/publicDatasets.test.ts +++ b/src/features/dataset/publicDatasets.test.ts @@ -13,14 +13,20 @@ describe('public datasets', () => { const validated = validateBenchmarkManifest(manifest) expect(validated.puzzleType).toBe('masyu') expect(validated.title).toBe('Masyu Dataset') - expect(validated.items).toHaveLength(55) - expect(new Set(validated.items.map((item) => item.id)).size).toBe(55) + expect(validated.items).toHaveLength(56) + expect(new Set(validated.items.map((item) => item.id)).size).toBe(56) const existingItems = validated.items.slice(0, 37) - const newItems = validated.items.slice(37) + const newItems = validated.items.slice(37, -1) + const newestItem = validated.items.at(-1) expect(existingItems.every((item) => item.tags.includes('hard'))).toBe(true) expect(newItems.every((item) => item.tags.includes('easy'))).toBe(true) expect(newItems.every((item) => item.source === 'example.txt')).toBe(true) + expect(newestItem).toMatchObject({ + id: 'masyu-25x25-1202688', + tags: ['hard', 'Large'], + source: 'example.txt', + }) expect(newItems.map((item) => item.id)).toEqual([ 'masyu-15x21-0005', 'masyu-10x10-0006', @@ -52,7 +58,7 @@ describe('public datasets', () => { expect(item.puzzleType).toBe('masyu') expect(item.tags).toEqual([difficultyTag, sizeTag]) expect(item.id).toMatch(/^masyu-\d+x\d+-\d+$/) - if (difficultyTag === 'hard') { + if (difficultyTag === 'hard' && item.id !== newestItem?.id) { expect(item.source).toBe( item.width === 25 ? 'https://zh.puzzle-masyu.com/?size=18' From 0a9b1d1cffd9ff7db00b52ca9d699033c3a02438 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Fri, 12 Jun 2026 20:23:20 +0800 Subject: [PATCH 06/17] feat: new CLI benchmark with great pipeline update --- README.md | 38 ++ docs/PROJECT_GUIDE_EN.md | 34 +- scripts/benchmark-solve.test.ts | 158 ++++++++ scripts/benchmark-solve.ts | 355 ++++++++++++++++-- src/domain/benchmark/aggregation.test.ts | 75 ++++ src/domain/benchmark/aggregation.ts | 206 ++++++++++ src/domain/benchmark/index.ts | 3 + src/domain/benchmark/runner.test.ts | 69 +++- src/domain/benchmark/runner.ts | 117 ++++-- src/domain/benchmark/textFormatter.test.ts | 98 +++++ src/domain/benchmark/textFormatter.ts | 72 ++++ src/domain/benchmark/tsvFormatter.test.ts | 70 ++++ src/domain/benchmark/tsvFormatter.ts | 293 +++++++++++++++ src/domain/benchmark/types.ts | 56 ++- src/domain/plugins/masyuPlugin.ts | 25 +- src/domain/plugins/slitherPlugin.ts | 19 + src/domain/plugins/types.ts | 11 + .../rules/composeSolverObservers.test.ts | 66 ++++ src/domain/rules/composeSolverObservers.ts | 30 ++ 19 files changed, 1716 insertions(+), 79 deletions(-) create mode 100644 scripts/benchmark-solve.test.ts create mode 100644 src/domain/benchmark/aggregation.test.ts create mode 100644 src/domain/benchmark/aggregation.ts create mode 100644 src/domain/benchmark/textFormatter.test.ts create mode 100644 src/domain/benchmark/textFormatter.ts create mode 100644 src/domain/benchmark/tsvFormatter.test.ts create mode 100644 src/domain/benchmark/tsvFormatter.ts create mode 100644 src/domain/rules/composeSolverObservers.test.ts create mode 100644 src/domain/rules/composeSolverObservers.ts diff --git a/README.md b/README.md index c5dd4c4..a1c60df 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,44 @@ pnpm test:e2e # Playwright end-to-end tests pnpm benchmark:solve ``` +## Solver Analysis CLI + +`pnpm benchmark:solve` is the unified batch solver-analysis command. It scans +`dataset/public` and local `dataset/private` manifests by default, runs each +puzzle once, and writes timestamped text and TSV analysis artifacts under +`benchmark-results///`. + +Target specific datasets or puzzle IDs while developing: + +```bash +pnpm benchmark:solve --dataset dataset/public/masyu.json --ids masyu-20x20-8268975,masyu-25x25-988309 +``` + +Useful options: + +```text +--dataset Repeatable manifest path +--ids Repeatable puzzle-ID filter +--max-steps Default: 2000 +--timeout-ms Per-puzzle timeout, default: 60000 +--telemetry off | summary, default: summary +--format text | tsv | both | json, default: both +--out-dir Default: benchmark-results +``` + +The default `both` mode prints and saves `summary.txt`, then writes +`puzzles.tsv`, `rule-attempts.tsv`, and `strong-inference.tsv`. These long-form +tables are intended for spreadsheets, pivot tables, and scripts. With +`--telemetry off`, only `puzzles.tsv` is generated. JSON is reserved for +complete debugging output and is written only when explicitly requested with +`--format json`. + +Summary telemetry reports rule attempts, misses, durations, and supported +strong-inference work. Reports also list strong-inference rules whose detailed +telemetry interface has not been implemented, so empty Strong data is not +mistaken for a rule that never ran. Generated artifacts have no compatibility +guarantee. + ## App Surfaces - **Solver** (`/`): import a puzzle, run one deduction at a time, jump through the replay timeline, inspect explanations and strong-inference branches, view stats, and export supported states. diff --git a/docs/PROJECT_GUIDE_EN.md b/docs/PROJECT_GUIDE_EN.md index f66ebff..b39b343 100644 --- a/docs/PROJECT_GUIDE_EN.md +++ b/docs/PROJECT_GUIDE_EN.md @@ -142,15 +142,45 @@ Run: pnpm benchmark:solve ``` -The benchmark runner scans public/private manifests, runs each puzzle with the default plugin rule order, and writes reports to `benchmark-results/.report.json`. +The unified solver-analysis runner scans public/private manifests, runs each +puzzle once with the default plugin rule order, and writes timestamped analysis +artifacts under `benchmark-results///`. Default benchmark settings: - `maxSteps = 2000` - `timeoutMs = 60000` - `ruleProfile = "default"` +- `telemetry = "summary"` -Reports include per-puzzle status, step count, duration, terminal completion report, `ruleUsage`, and compact `ruleSteps`. Full `steps` are intentionally omitted to keep reports small. +Targeted development runs can select explicit manifests and puzzle IDs: + +```bash +pnpm benchmark:solve --dataset dataset/public/masyu.json --ids masyu-20x20-8268975,masyu-25x25-988309 +``` + +CLI options: + +- `--dataset ` is repeatable; without it, public/private manifests are scanned. +- `--ids ` is repeatable and filters selected manifests. +- `--max-steps ` and `--timeout-ms ` set per-puzzle limits. +- `--telemetry off | summary` controls observer collectors. +- `--format text | tsv | both | json` controls generated artifacts; default + `both` means text plus TSV and never includes JSON. +- `--out-dir ` changes the artifact output directory. + +The default run prints and saves `summary.txt`, plus three long-form tables: +`puzzles.tsv` for one-row-per-puzzle outcomes, `rule-attempts.tsv` for per-rule +cost and hit behavior, and `strong-inference.tsv` for detailed Strong work. +When telemetry is off, only the puzzle table is generated. JSON retains the +complete schema-v2 report for urgent debugging and is written only by explicit +`--format json`. + +Rule attempt telemetry includes misses and the final no-hit scan. Strong +inference telemetry includes explicit plugin-declared coverage, so unsupported +collection interfaces are distinguishable from a run that produced no Strong +events. Generated artifacts are development outputs with no compatibility +guarantee. Full replay steps and rule-step locations are intentionally omitted. ## 8. AI Agent Quick Start diff --git a/scripts/benchmark-solve.test.ts b/scripts/benchmark-solve.test.ts new file mode 100644 index 0000000..6b22a84 --- /dev/null +++ b/scripts/benchmark-solve.test.ts @@ -0,0 +1,158 @@ +import { mkdtemp, readdir } from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { describe, expect, it } from 'vitest' +import { runBenchmarkManifest } from '../src/domain/benchmark' +import { + createRunDirectory, + formatRunTimestamp, + parseBenchmarkCliArgs, + selectManifestItems, + writeBenchmarkOutputs, +} from './benchmark-solve' +import type { BenchmarkDatasetManifest } from '../src/domain/benchmark' + +const manifest: BenchmarkDatasetManifest = { + schemaVersion: 1, + id: 'sample', + title: 'Sample', + puzzleType: 'slitherlink', + items: [ + { + id: 'a', + puzzleType: 'slitherlink', + sourceUrl: 'https://puzz.link/p?slither/1/1/0', + width: 1, + height: 1, + tags: [], + }, + { + id: 'b', + puzzleType: 'slitherlink', + sourceUrl: 'https://puzz.link/p?slither/1/1/1', + width: 1, + height: 1, + tags: [], + }, + ], +} + +describe('benchmark solve CLI', () => { + it('defaults to text plus TSV output', () => { + expect(parseBenchmarkCliArgs([]).format).toBe('both') + }) + + it('parses repeatable datasets and comma-separated IDs', () => { + expect( + parseBenchmarkCliArgs([ + '--dataset', + 'a.json', + '--dataset', + 'b.json', + '--ids', + 'a,b', + '--ids', + 'c', + '--max-steps', + '12', + '--timeout-ms', + '34', + '--telemetry', + 'off', + '--format', + 'both', + '--out-dir', + 'out', + ]), + ).toEqual({ + datasetPaths: ['a.json', 'b.json'], + ids: ['a', 'b', 'c'], + maxSteps: 12, + timeoutMs: 34, + telemetry: 'off', + format: 'both', + outDir: 'out', + }) + }) + + it('rejects invalid values and unknown options', () => { + expect(() => parseBenchmarkCliArgs(['--max-steps', '0'])).toThrow( + '--max-steps must be a positive integer', + ) + expect(() => parseBenchmarkCliArgs(['--telemetry', 'details'])).toThrow( + '--telemetry must be one of off, summary', + ) + expect(() => parseBenchmarkCliArgs(['--unknown'])).toThrow( + 'Invalid benchmark arguments', + ) + }) + + it('filters manifests and rejects duplicate or unmatched selections', () => { + expect( + selectManifestItems([manifest], ['b'])[0].items.map((item) => item.id), + ).toEqual(['b']) + expect(() => selectManifestItems([manifest], ['missing'])).toThrow( + 'No puzzles matched', + ) + expect(() => selectManifestItems([manifest, { ...manifest }], [])).toThrow( + 'Duplicate dataset ID', + ) + }) + + it('formats local timestamps and avoids overwriting same-second runs', async () => { + const outDir = await mkdtemp(path.join(os.tmpdir(), 'benchmark-output-')) + const timestamp = formatRunTimestamp(new Date(2026, 5, 12, 19, 45, 7)) + + expect(timestamp).toBe('20260612-194507') + expect( + path.basename(await createRunDirectory(outDir, 'sample', timestamp)), + ).toBe(timestamp) + expect( + path.basename(await createRunDirectory(outDir, 'sample', timestamp)), + ).toBe(`${timestamp}-2`) + }) + + it('writes only the files selected by each format', async () => { + const outDir = await mkdtemp(path.join(os.tmpdir(), 'benchmark-output-')) + const report = runBenchmarkManifest(manifest, { + maxSteps: 1, + telemetry: 'summary', + }) + const expected = { + both: [ + 'puzzles.tsv', + 'rule-attempts.tsv', + 'strong-inference.tsv', + 'summary.txt', + ], + text: ['summary.txt'], + tsv: ['puzzles.tsv', 'rule-attempts.tsv', 'strong-inference.tsv'], + json: ['report.json'], + } as const + + for (const format of ['both', 'text', 'tsv', 'json'] as const) { + const timestamp = `20260612-19450${Object.keys(expected).indexOf(format)}` + await writeBenchmarkOutputs(report, { format, outDir }, timestamp) + expect( + ( + await readdir(path.join(outDir, report.run.datasetId, timestamp)) + ).sort(), + ).toEqual([...expected[format]].sort()) + } + }) + + it('writes only puzzles.tsv when TSV telemetry is off', async () => { + const outDir = await mkdtemp(path.join(os.tmpdir(), 'benchmark-output-')) + const report = runBenchmarkManifest(manifest, { + maxSteps: 1, + telemetry: 'off', + }) + const timestamp = '20260612-194510' + + await writeBenchmarkOutputs(report, { format: 'tsv', outDir }, timestamp) + + expect( + await readdir(path.join(outDir, report.run.datasetId, timestamp)), + ).toEqual(['puzzles.tsv']) + }) +}) diff --git a/scripts/benchmark-solve.ts b/scripts/benchmark-solve.ts index 13a8a18..0b37e64 100644 --- a/scripts/benchmark-solve.ts +++ b/scripts/benchmark-solve.ts @@ -1,22 +1,151 @@ import type { Dirent } from 'node:fs' import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises' import path from 'node:path' -import { fileURLToPath } from 'node:url' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { parseArgs } from 'node:util' import { + formatBenchmarkReportText, + formatBenchmarkReportTsv, runBenchmarkManifest, validateBenchmarkManifest, } from '../src/domain/benchmark' +import type { + BenchmarkDatasetManifest, + BenchmarkReport, + BenchmarkTelemetryLevel, +} from '../src/domain/benchmark' const DATASET_DIRS = ['dataset/public', 'dataset/private'] -const OUTPUT_DIR = 'benchmark-results' +const DEFAULT_OUTPUT_DIR = 'benchmark-results' + +export type BenchmarkOutputFormat = 'json' | 'text' | 'tsv' | 'both' -const projectRoot = path.resolve( +export type BenchmarkCliOptions = { + datasetPaths: string[] + ids: string[] + maxSteps: number + timeoutMs: number + telemetry: BenchmarkTelemetryLevel + format: BenchmarkOutputFormat + outDir: string +} + +export const projectRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), '..', ) -const findJsonFiles = async (dir: string): Promise => { - const absoluteDir = path.join(projectRoot, dir) +const parsePositiveInteger = ( + name: string, + value: string | undefined, + fallback: number, +) => { + if (value === undefined) return fallback + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error( + `--${name} must be a positive integer; received "${value}".`, + ) + } + return parsed +} + +const parseEnum = ( + name: string, + value: string | undefined, + allowed: readonly T[], + fallback: T, +): T => { + if (value === undefined) return fallback + if (!allowed.includes(value as T)) { + throw new Error( + `--${name} must be one of ${allowed.join(', ')}; received "${value}".`, + ) + } + return value as T +} + +const flattenCsvValues = (values: string[] | undefined): string[] => + (values ?? []) + .flatMap((value) => value.split(',')) + .map((value) => value.trim()) + .filter(Boolean) + +const getString = ( + values: ReturnType['values'], + key: string, +): string | undefined => { + const value = values[key] + return typeof value === 'string' ? value : undefined +} + +const getStrings = ( + values: ReturnType['values'], + key: string, +): string[] | undefined => { + const value = values[key] + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : undefined +} + +export const parseBenchmarkCliArgs = (args: string[]): BenchmarkCliOptions => { + let values: ReturnType['values'] + try { + ;({ values } = parseArgs({ + args, + strict: true, + allowPositionals: false, + options: { + dataset: { type: 'string', multiple: true }, + ids: { type: 'string', multiple: true }, + 'max-steps': { type: 'string' }, + 'timeout-ms': { type: 'string' }, + telemetry: { type: 'string' }, + format: { type: 'string' }, + 'out-dir': { type: 'string' }, + }, + })) + } catch (error) { + throw new Error( + `Invalid benchmark arguments: ${error instanceof Error ? error.message : String(error)}`, + ) + } + + return { + datasetPaths: getStrings(values, 'dataset') ?? [], + ids: flattenCsvValues(getStrings(values, 'ids')), + maxSteps: parsePositiveInteger( + 'max-steps', + getString(values, 'max-steps'), + 2000, + ), + timeoutMs: parsePositiveInteger( + 'timeout-ms', + getString(values, 'timeout-ms'), + 60_000, + ), + telemetry: parseEnum( + 'telemetry', + getString(values, 'telemetry'), + ['off', 'summary'] as const, + 'summary', + ), + format: parseEnum( + 'format', + getString(values, 'format'), + ['json', 'text', 'tsv', 'both'] as const, + 'both', + ), + outDir: getString(values, 'out-dir') ?? DEFAULT_OUTPUT_DIR, + } +} + +const resolveProjectPath = (targetPath: string): string => + path.isAbsolute(targetPath) ? targetPath : path.join(projectRoot, targetPath) + +export const findJsonFiles = async (dir: string): Promise => { + const absoluteDir = resolveProjectPath(dir) let entries: Dirent[] try { entries = await readdir(absoluteDir, { withFileTypes: true }) @@ -29,54 +158,216 @@ const findJsonFiles = async (dir: string): Promise => { const files: string[] = [] for (const entry of entries) { - const relativePath = path.join(dir, entry.name) + const entryPath = path.join(absoluteDir, entry.name) if (entry.isDirectory()) { - files.push(...(await findJsonFiles(relativePath))) + files.push(...(await findJsonFiles(entryPath))) } else if (entry.isFile() && entry.name.endsWith('.json')) { - files.push(relativePath) + files.push(entryPath) } } return files.sort() } -const readManifest = async (manifestPath: string) => { - const raw = await readFile(path.join(projectRoot, manifestPath), 'utf8') - return validateBenchmarkManifest(JSON.parse(raw)) +export const readManifest = async ( + manifestPath: string, +): Promise => { + const absolutePath = resolveProjectPath(manifestPath) + let raw: string + try { + raw = await readFile(absolutePath, 'utf8') + } catch (error) { + throw new Error( + `Unable to read dataset "${manifestPath}": ${error instanceof Error ? error.message : String(error)}`, + ) + } + try { + return validateBenchmarkManifest(JSON.parse(raw)) + } catch (error) { + throw new Error( + `Invalid dataset "${manifestPath}": ${error instanceof Error ? error.message : String(error)}`, + ) + } +} + +export const selectManifestItems = ( + manifests: BenchmarkDatasetManifest[], + ids: string[], +): BenchmarkDatasetManifest[] => { + const duplicateIds = manifests + .map((manifest) => manifest.id) + .filter((id, index, all) => all.indexOf(id) !== index) + if (duplicateIds.length > 0) { + throw new Error( + `Duplicate dataset ID(s): ${Array.from(new Set(duplicateIds)).join(', ')}.`, + ) + } + if (ids.length === 0) return manifests + + const selectedIds = new Set(ids) + const filtered = manifests.flatMap((manifest) => { + const items = manifest.items.filter((item) => selectedIds.has(item.id)) + return items.length > 0 ? [{ ...manifest, items }] : [] + }) + if (filtered.length === 0) { + throw new Error(`No puzzles matched --ids ${ids.join(', ')}.`) + } + return filtered +} + +export const formatRunTimestamp = (date: Date): string => { + const pad = (value: number): string => String(value).padStart(2, '0') + return [ + date.getFullYear(), + pad(date.getMonth() + 1), + pad(date.getDate()), + '-', + pad(date.getHours()), + pad(date.getMinutes()), + pad(date.getSeconds()), + ].join('') } -const writeReport = async ( +export const createRunDirectory = async ( + outDir: string, datasetId: string, - report: unknown, + timestamp: string, +): Promise => { + const datasetDir = path.join(resolveProjectPath(outDir), datasetId) + await mkdir(datasetDir, { recursive: true }) + for (let suffix = 1; ; suffix += 1) { + const name = suffix === 1 ? timestamp : `${timestamp}-${suffix}` + const runDir = path.join(datasetDir, name) + try { + await mkdir(runDir) + return runDir + } catch (error) { + if ( + error instanceof Error && + 'code' in error && + error.code === 'EEXIST' + ) { + continue + } + throw error + } + } +} + +const writeArtifact = async ( + runDir: string, + filename: string, + content: string, ): Promise => { - const outPath = path.join(OUTPUT_DIR, `${datasetId}.report.json`) - const absoluteOutPath = path.join(projectRoot, outPath) - await mkdir(path.dirname(absoluteOutPath), { recursive: true }) - await writeFile(absoluteOutPath, `${JSON.stringify(report, null, 2)}\n`) - return outPath + const absolutePath = path.join(runDir, filename) + await writeFile(absolutePath, content) + return path.relative(projectRoot, absolutePath) } -const main = async (): Promise => { - const manifestPaths = ( - await Promise.all(DATASET_DIRS.map((dir) => findJsonFiles(dir))) - ).flat() +export const writeBenchmarkOutputs = async ( + report: BenchmarkReport, + options: Pick, + timestamp: string, +): Promise => { + const runDir = await createRunDirectory( + options.outDir, + report.run.datasetId, + timestamp, + ) + if (options.format === 'json') { + return [ + await writeArtifact( + runDir, + 'report.json', + `${JSON.stringify(report, null, 2)}\n`, + ), + ] + } + + const outputPaths: string[] = [] + if (options.format === 'text' || options.format === 'both') { + outputPaths.push( + await writeArtifact( + runDir, + 'summary.txt', + `${formatBenchmarkReportText(report)}\n`, + ), + ) + } + if (options.format === 'tsv' || options.format === 'both') { + const tables = formatBenchmarkReportTsv(report) + outputPaths.push(await writeArtifact(runDir, 'puzzles.tsv', tables.puzzles)) + if (tables.ruleAttempts && tables.strongInference) { + outputPaths.push( + await writeArtifact(runDir, 'rule-attempts.tsv', tables.ruleAttempts), + await writeArtifact( + runDir, + 'strong-inference.tsv', + tables.strongInference, + ), + ) + } + } + return outputPaths +} +export const runBenchmarkCli = async ( + options: BenchmarkCliOptions, +): Promise => { + const manifestPaths = + options.datasetPaths.length > 0 + ? options.datasetPaths + : ( + await Promise.all(DATASET_DIRS.map((dir) => findJsonFiles(dir))) + ).flat() if (manifestPaths.length === 0) { throw new Error( `No benchmark manifests found under ${DATASET_DIRS.join(' or ')}.`, ) } - for (const manifestPath of manifestPaths) { - const manifest = await readManifest(manifestPath) - const report = runBenchmarkManifest(manifest) - const outPath = await writeReport(manifest.id, report) - console.log( - `Wrote ${manifest.id}: ${report.summary.total} puzzle(s), ${outPath}`, + const manifests = selectManifestItems( + await Promise.all(manifestPaths.map(readManifest)), + options.ids, + ) + const runTimestamp = formatRunTimestamp(new Date()) + for (const manifest of manifests) { + const report = runBenchmarkManifest(manifest, { + maxSteps: options.maxSteps, + timeoutMs: options.timeoutMs, + telemetry: options.telemetry, + }) + if (options.format === 'text' || options.format === 'both') { + console.log(formatBenchmarkReportText(report)) + } + if ( + (options.format === 'tsv' || options.format === 'both') && + options.telemetry === 'off' + ) { + console.log( + `${manifest.id}: telemetry is off; rule-attempts.tsv and strong-inference.tsv were not generated.`, + ) + } + const outputPaths = await writeBenchmarkOutputs( + report, + options, + runTimestamp, ) + console.log(`Wrote ${manifest.id}: ${report.summary.total} puzzle(s)`) + for (const outputPath of outputPaths) { + console.log(` ${outputPath}`) + } } } -main().catch((error: unknown) => { - console.error(error instanceof Error ? error.message : String(error)) - process.exitCode = 1 -}) +const isMain = process.argv[1] + ? import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href + : false + +if (isMain) { + runBenchmarkCli(parseBenchmarkCliArgs(process.argv.slice(2))).catch( + (error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + }, + ) +} diff --git a/src/domain/benchmark/aggregation.test.ts b/src/domain/benchmark/aggregation.test.ts new file mode 100644 index 0000000..cfb7c26 --- /dev/null +++ b/src/domain/benchmark/aggregation.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest' +import { + aggregateRuleAttemptSummaries, + aggregateStrongCoverage, + aggregateStrongInferenceSummaries, +} from './aggregation' + +describe('benchmark telemetry aggregation', () => { + it('sums rule attempts and drops per-item final scans', () => { + const summary = aggregateRuleAttemptSummaries([ + { + totalAttemptCount: 2, + rules: { + a: { + ruleId: 'a', + ruleName: 'A', + attemptCount: 2, + hitCount: 1, + missCount: 1, + hitRate: 0.5, + totalDurationMs: 6, + hitDurationMs: 4, + missDurationMs: 2, + averageDurationMs: 3, + producedDiffCount: 2, + }, + }, + finalNoHitScan: { + solverStepNumber: 2, + totalDurationMs: 2, + attemptCount: 1, + rules: [{ ruleId: 'a', ruleName: 'A', durationMs: 2 }], + }, + }, + ]) + + expect(summary.rules.a.hitRate).toBe(0.5) + expect(summary.finalNoHitScan).toBeNull() + }) + + it('aggregates strong work and coverage', () => { + const summary = aggregateStrongInferenceSummaries([ + { + totals: { + attemptCount: 1, + hitCount: 1, + missCount: 0, + timeoutCount: 0, + hitRate: 1, + candidateCount: 2, + probeCount: 3, + trialStepCount: 4, + probeDurationMs: 5, + producedDiffCount: 1, + }, + rules: [], + }, + ]) + const coverage = aggregateStrongCoverage([ + { + status: 'full', + supportedRules: [{ ruleId: 'a', ruleName: 'A', supported: true }], + unsupportedRules: [], + }, + { + status: 'none', + supportedRules: [], + unsupportedRules: [{ ruleId: 'b', ruleName: 'B', supported: false }], + }, + ]) + + expect(summary.totals.probeCount).toBe(3) + expect(coverage.status).toBe('partial') + }) +}) diff --git a/src/domain/benchmark/aggregation.ts b/src/domain/benchmark/aggregation.ts new file mode 100644 index 0000000..bf711bf --- /dev/null +++ b/src/domain/benchmark/aggregation.ts @@ -0,0 +1,206 @@ +import type { + RuleAttemptRuleSummary, + RuleAttemptSummary, +} from '../rules/ruleAttemptSummaryCollector' +import type { + StrongInferenceRuleSummary, + StrongInferenceSummary, + StrongInferenceTotals, +} from '../rules/strongInferenceSummaryCollector' +import type { + BenchmarkStrongInferenceTelemetry, + BenchmarkTelemetrySummary, + StrongTelemetryCoverage, +} from './types' + +const addRecordCounts = ( + target: Record, + source: Record, +): void => { + for (const [key, value] of Object.entries(source)) { + target[key] = (target[key] ?? 0) + value + } +} + +export const aggregateRuleUsage = ( + records: Array>, +): Record => { + const result: Record = {} + for (const record of records) addRecordCounts(result, record) + return result +} + +export const ruleUsageFromAttempts = ( + summary: RuleAttemptSummary, +): Record => + Object.fromEntries( + Object.values(summary.rules) + .filter((rule) => rule.hitCount > 0) + .map((rule) => [rule.ruleId, rule.hitCount]), + ) + +export const aggregateRuleAttemptSummaries = ( + summaries: RuleAttemptSummary[], +): RuleAttemptSummary => { + const mutable = new Map< + string, + Omit + >() + let totalAttemptCount = 0 + for (const summary of summaries) { + totalAttemptCount += summary.totalAttemptCount + for (const rule of Object.values(summary.rules)) { + const target = mutable.get(rule.ruleId) ?? { + ruleId: rule.ruleId, + ruleName: rule.ruleName, + attemptCount: 0, + hitCount: 0, + missCount: 0, + totalDurationMs: 0, + hitDurationMs: 0, + missDurationMs: 0, + producedDiffCount: 0, + } + target.attemptCount += rule.attemptCount + target.hitCount += rule.hitCount + target.missCount += rule.missCount + target.totalDurationMs += rule.totalDurationMs + target.hitDurationMs += rule.hitDurationMs + target.missDurationMs += rule.missDurationMs + target.producedDiffCount += rule.producedDiffCount + mutable.set(rule.ruleId, target) + } + } + return { + totalAttemptCount, + rules: Object.fromEntries( + Array.from(mutable, ([ruleId, rule]) => [ + ruleId, + { + ...rule, + hitRate: + rule.attemptCount === 0 ? 0 : rule.hitCount / rule.attemptCount, + averageDurationMs: + rule.attemptCount === 0 + ? 0 + : rule.totalDurationMs / rule.attemptCount, + }, + ]), + ), + finalNoHitScan: null, + } +} + +const emptyStrongTotals = (): Omit => ({ + attemptCount: 0, + hitCount: 0, + missCount: 0, + timeoutCount: 0, + candidateCount: 0, + probeCount: 0, + trialStepCount: 0, + probeDurationMs: 0, + producedDiffCount: 0, +}) + +const addStrongTotals = ( + target: Omit, + source: StrongInferenceTotals, +): void => { + target.attemptCount += source.attemptCount + target.hitCount += source.hitCount + target.missCount += source.missCount + target.timeoutCount += source.timeoutCount + target.candidateCount += source.candidateCount + target.probeCount += source.probeCount + target.trialStepCount += source.trialStepCount + target.probeDurationMs += source.probeDurationMs + target.producedDiffCount += source.producedDiffCount +} + +const snapshotStrongTotals = ( + totals: Omit, +): StrongInferenceTotals => ({ + ...totals, + hitRate: + totals.attemptCount === 0 ? 0 : totals.hitCount / totals.attemptCount, +}) + +export const aggregateStrongInferenceSummaries = ( + summaries: StrongInferenceSummary[], +): StrongInferenceSummary => { + const totals = emptyStrongTotals() + const rules = new Map>() + for (const summary of summaries) { + addStrongTotals(totals, summary.totals) + for (const rule of summary.rules) { + const target = rules.get(rule.ruleId) ?? { + ...emptyStrongTotals(), + ruleId: rule.ruleId, + ruleName: rule.ruleName, + } + addStrongTotals(target, rule) + rules.set(rule.ruleId, target) + } + } + return { + totals: snapshotStrongTotals(totals), + rules: Array.from(rules.values(), (rule) => ({ + ruleId: rule.ruleId, + ruleName: rule.ruleName, + ...snapshotStrongTotals(rule), + })), + } +} + +export const aggregateStrongCoverage = ( + coverages: StrongTelemetryCoverage[], +): StrongTelemetryCoverage => { + const rules = new Map< + string, + StrongTelemetryCoverage['supportedRules'][number] + >() + for (const coverage of coverages) { + for (const rule of [ + ...coverage.supportedRules, + ...coverage.unsupportedRules, + ]) { + const existing = rules.get(rule.ruleId) + rules.set(rule.ruleId, { + ...rule, + supported: Boolean(existing?.supported || rule.supported), + }) + } + } + const supportedRules = Array.from(rules.values()).filter( + (rule) => rule.supported, + ) + const unsupportedRules = Array.from(rules.values()).filter( + (rule) => !rule.supported, + ) + const status = + rules.size === 0 + ? 'not-applicable' + : unsupportedRules.length === 0 + ? 'full' + : supportedRules.length === 0 + ? 'none' + : 'partial' + return { status, supportedRules, unsupportedRules } +} + +export const aggregateTelemetrySummaries = ( + summaries: BenchmarkTelemetrySummary[], +): BenchmarkTelemetrySummary => ({ + ruleAttempts: aggregateRuleAttemptSummaries( + summaries.map((summary) => summary.ruleAttempts), + ), + strongInference: { + coverage: aggregateStrongCoverage( + summaries.map((summary) => summary.strongInference.coverage), + ), + summary: aggregateStrongInferenceSummaries( + summaries.map((summary) => summary.strongInference.summary), + ), + } satisfies BenchmarkStrongInferenceTelemetry, +}) diff --git a/src/domain/benchmark/index.ts b/src/domain/benchmark/index.ts index 95db03e..3a5ba4c 100644 --- a/src/domain/benchmark/index.ts +++ b/src/domain/benchmark/index.ts @@ -1,5 +1,7 @@ export { validateBenchmarkManifest } from './manifest' export { runBenchmarkItem, runBenchmarkManifest } from './runner' +export { formatBenchmarkReportText } from './textFormatter' +export { formatBenchmarkReportTsv, serializeTsv } from './tsvFormatter' export type { BenchmarkDatasetItem, BenchmarkDatasetManifest, @@ -7,4 +9,5 @@ export type { BenchmarkPuzzleStatus, BenchmarkReport, BenchmarkRunnerOptions, + BenchmarkTelemetryLevel, } from './types' diff --git a/src/domain/benchmark/runner.test.ts b/src/domain/benchmark/runner.test.ts index 561b0c0..d633c77 100644 --- a/src/domain/benchmark/runner.test.ts +++ b/src/domain/benchmark/runner.test.ts @@ -16,18 +16,36 @@ describe('benchmark runner', () => { vi.restoreAllMocks() }) - it('reports step-capped runs when maxSteps is reached before completion', () => { - const result = runBenchmarkItem(validItem, { + it('emits schema-v2 compact results and preserves off/summary outcomes', () => { + const off = runBenchmarkItem(validItem, { maxSteps: 1, timeoutMs: 60_000, + telemetry: 'off', }) - - expect(result.status).toBe('step-capped') - expect(result.stepCount).toBe(1) - expect(result.steps).toEqual([]) - expect(result.ruleSteps).toEqual({ - [Object.keys(result.ruleUsage)[0]]: [1], + const summary = runBenchmarkItem(validItem, { + maxSteps: 1, + timeoutMs: 60_000, + telemetry: 'summary', }) + + expect(summary.status).toBe(off.status) + expect(summary.stepCount).toBe(off.stepCount) + expect(summary.terminal).toEqual(off.terminal) + expect(summary.ruleUsage).toEqual(off.ruleUsage) + expect(summary).not.toHaveProperty('steps') + expect(summary).not.toHaveProperty('ruleSteps') + expect(off.telemetry).toBeUndefined() + expect(summary.telemetry?.ruleAttempts.totalAttemptCount).toBeGreaterThan(0) + expect(summary.telemetry?.strongInference.coverage.status).toBe('none') + expect( + summary.telemetry?.strongInference.coverage.unsupportedRules.map( + (rule) => rule.ruleId, + ), + ).toEqual([ + 'color-assumption-inference', + 'sector-parity-inference', + 'strong-inference', + ]) }) it('reports time-capped runs without crashing', () => { @@ -36,12 +54,44 @@ describe('benchmark runner', () => { .mockReturnValueOnce(2) .mockReturnValueOnce(2) - const result = runBenchmarkItem(validItem, { maxSteps: 2000, timeoutMs: 1 }) + const result = runBenchmarkItem(validItem, { + maxSteps: 2000, + timeoutMs: 1, + telemetry: 'summary', + }) expect(result.status).toBe('time-capped') expect(result.stepCount).toBe(0) }) + it('keeps collectors isolated and aggregates item summaries', () => { + const manifest: BenchmarkDatasetManifest = { + schemaVersion: 1, + id: 'two-items', + title: 'Two Items', + puzzleType: 'slitherlink', + items: [validItem, { ...validItem, id: 'second' }], + } + const report = runBenchmarkManifest(manifest, { + maxSteps: 1, + timeoutMs: 60_000, + telemetry: 'summary', + }) + const [first, second] = report.items + + expect(report.schemaVersion).toBe(2) + expect(Object.keys(first.telemetry?.ruleAttempts.rules ?? {})).toEqual( + Object.keys(second.telemetry?.ruleAttempts.rules ?? {}), + ) + expect(first.telemetry?.ruleAttempts.totalAttemptCount).toBe( + second.telemetry?.ruleAttempts.totalAttemptCount, + ) + expect(report.summary.telemetry?.ruleAttempts.totalAttemptCount).toBe( + (first.telemetry?.ruleAttempts.totalAttemptCount ?? 0) * 2, + ) + expect(report.summary.telemetry?.ruleAttempts.finalNoHitScan).toBeNull() + }) + it('keeps running after per-item parse errors', () => { const manifest: BenchmarkDatasetManifest = { schemaVersion: 1, @@ -61,6 +111,7 @@ describe('benchmark runner', () => { const report = runBenchmarkManifest(manifest, { maxSteps: 1, timeoutMs: 60_000, + telemetry: 'summary', }) expect(report.summary.total).toBe(2) diff --git a/src/domain/benchmark/runner.ts b/src/domain/benchmark/runner.ts index 7d49701..5a64230 100644 --- a/src/domain/benchmark/runner.ts +++ b/src/domain/benchmark/runner.ts @@ -1,8 +1,15 @@ import type { PuzzleIR } from '../ir/types' import { puzzleRegistry } from '../plugins/registry' import { analyzePuzzleCompletion } from '../rules/completion' +import { composeSolverObservers } from '../rules/composeSolverObservers' import { runNextRule } from '../rules/engine' -import { addRuleUsage } from '../difficulty/traceStats' +import { createRuleAttemptSummaryCollector } from '../rules/ruleAttemptSummaryCollector' +import { createStrongInferenceSummaryCollector } from '../rules/strongInferenceSummaryCollector' +import { + aggregateRuleUsage, + aggregateTelemetrySummaries, + ruleUsageFromAttempts, +} from './aggregation' import type { BenchmarkDatasetItem, BenchmarkDatasetManifest, @@ -11,10 +18,13 @@ import type { BenchmarkReport, BenchmarkRunnerOptions, BenchmarkSummary, + BenchmarkTelemetryLevel, } from './types' +import { getStrongTelemetryCoverage } from './types' const DEFAULT_MAX_STEPS = 2000 const DEFAULT_TIMEOUT_MS = 60_000 +const DEFAULT_TELEMETRY: BenchmarkTelemetryLevel = 'summary' const normalizeLimit = ( value: number | undefined, @@ -26,11 +36,15 @@ const normalizeLimit = ( return Math.max(1, Math.floor(value)) } -const getTerminal = (puzzleType: string, puzzle: PuzzleIR) => analyzePuzzleCompletion(puzzleType, puzzle) +const getTerminal = (puzzleType: string, puzzle: PuzzleIR) => + analyzePuzzleCompletion(puzzleType, puzzle) const getStatusCountKey = ( status: BenchmarkPuzzleStatus, -): keyof Omit => { +): keyof Omit< + BenchmarkSummary, + 'total' | 'totalDurationMs' | 'ruleUsage' | 'telemetry' +> => { if (status === 'parse-error') return 'parseError' if (status === 'runtime-error') return 'runtimeError' if (status === 'step-capped') return 'stepCapped' @@ -40,33 +54,63 @@ const getStatusCountKey = ( export const runBenchmarkItem = ( item: BenchmarkDatasetItem, - options: Required, + options: BenchmarkRunnerOptions = {}, ): BenchmarkPuzzleResult => { + const maxSteps = normalizeLimit(options.maxSteps, DEFAULT_MAX_STEPS) + const timeoutMs = normalizeLimit(options.timeoutMs, DEFAULT_TIMEOUT_MS) + const telemetry = options.telemetry ?? DEFAULT_TELEMETRY const startedAt = performance.now() - const ruleUsage: Record = {} - const ruleSteps: Record = {} + const plugin = puzzleRegistry.get(item.puzzleType) + const ruleAttemptCollector = + telemetry === 'summary' ? createRuleAttemptSummaryCollector() : undefined + const strongInferenceCollector = + telemetry === 'summary' + ? createStrongInferenceSummaryCollector() + : undefined + const observer = + telemetry === 'summary' + ? composeSolverObservers([ + ruleAttemptCollector?.observer, + strongInferenceCollector?.observer, + ]) + : undefined + const offRuleUsage: Record = {} let stepCount = 0 + const finish = ( status: BenchmarkPuzzleStatus, terminal: BenchmarkPuzzleResult['terminal'], error?: string, - ): BenchmarkPuzzleResult => ({ - id: item.id, - puzzleType: item.puzzleType, - sourceUrl: item.sourceUrl, - width: item.width, - height: item.height, - status, - stepCount, - durationMs: Math.max(0, performance.now() - startedAt), - ruleUsage, - ruleSteps, - terminal, - steps: [], - ...(error ? { error } : {}), - }) + ): BenchmarkPuzzleResult => { + const ruleAttempts = ruleAttemptCollector?.getSummary() + const telemetry = + ruleAttempts && strongInferenceCollector + ? { + ruleAttempts, + strongInference: { + coverage: getStrongTelemetryCoverage(plugin?.strongTelemetry), + summary: strongInferenceCollector.getSummary(), + }, + } + : undefined + return { + id: item.id, + puzzleType: item.puzzleType, + sourceUrl: item.sourceUrl, + width: item.width, + height: item.height, + status, + stepCount, + durationMs: Math.max(0, performance.now() - startedAt), + ruleUsage: ruleAttempts + ? ruleUsageFromAttempts(ruleAttempts) + : offRuleUsage, + terminal, + ...(telemetry ? { telemetry } : {}), + ...(error ? { error } : {}), + } + } - const plugin = puzzleRegistry.get(item.puzzleType) if (!plugin) { return finish('parse-error', null, `Plugin "${item.puzzleType}" not found.`) } @@ -84,11 +128,11 @@ export const runBenchmarkItem = ( const rules = plugin.getRules() while (true) { - if (performance.now() - startedAt >= options.timeoutMs) { + if (performance.now() - startedAt >= timeoutMs) { return finish('time-capped', getTerminal(item.puzzleType, puzzle)) } - if (stepCount >= options.maxSteps) { + if (stepCount >= maxSteps) { const terminal = getTerminal(item.puzzleType, puzzle) return finish( terminal?.status === 'solved' ? 'solved' : 'step-capped', @@ -98,7 +142,7 @@ export const runBenchmarkItem = ( let result: ReturnType try { - result = runNextRule(puzzle, rules, stepCount + 1) + result = runNextRule(puzzle, rules, stepCount + 1, { observer }) } catch (error) { return finish( 'runtime-error', @@ -114,7 +158,10 @@ export const runBenchmarkItem = ( puzzle = result.nextPuzzle stepCount += 1 - addRuleUsage(ruleUsage, ruleSteps, result.step, stepCount) + if (telemetry === 'off') { + offRuleUsage[result.step.ruleId] = + (offRuleUsage[result.step.ruleId] ?? 0) + 1 + } } } @@ -124,9 +171,10 @@ export const runBenchmarkManifest = ( ): BenchmarkReport => { const maxSteps = normalizeLimit(options.maxSteps, DEFAULT_MAX_STEPS) const timeoutMs = normalizeLimit(options.timeoutMs, DEFAULT_TIMEOUT_MS) + const telemetry = options.telemetry ?? DEFAULT_TELEMETRY const startedAt = new Date().toISOString() const items = manifest.items.map((item) => - runBenchmarkItem(item, { maxSteps, timeoutMs }), + runBenchmarkItem(item, { maxSteps, timeoutMs, telemetry }), ) const summary: BenchmarkSummary = { total: items.length, @@ -137,19 +185,23 @@ export const runBenchmarkManifest = ( stepCapped: 0, timeCapped: 0, totalDurationMs: 0, - ruleUsage: {}, + ruleUsage: aggregateRuleUsage(items.map((item) => item.ruleUsage)), + ...(telemetry === 'summary' + ? { + telemetry: aggregateTelemetrySummaries( + items.flatMap((item) => (item.telemetry ? [item.telemetry] : [])), + ), + } + : {}), } for (const item of items) { summary[getStatusCountKey(item.status)] += 1 summary.totalDurationMs += item.durationMs - for (const [ruleId, count] of Object.entries(item.ruleUsage)) { - summary.ruleUsage[ruleId] = (summary.ruleUsage[ruleId] ?? 0) + count - } } return { - schemaVersion: 1, + schemaVersion: 2, run: { datasetId: manifest.id, startedAt, @@ -157,6 +209,7 @@ export const runBenchmarkManifest = ( maxSteps, timeoutMs, ruleProfile: 'default', + telemetry, }, summary, items, diff --git a/src/domain/benchmark/textFormatter.test.ts b/src/domain/benchmark/textFormatter.test.ts new file mode 100644 index 0000000..427d089 --- /dev/null +++ b/src/domain/benchmark/textFormatter.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import { formatBenchmarkReportText } from './textFormatter' +import type { BenchmarkReport } from './types' + +const report: BenchmarkReport = { + schemaVersion: 2, + run: { + datasetId: 'sample', + startedAt: '2026-01-01T00:00:00.000Z', + completedAt: '2026-01-01T00:00:01.000Z', + maxSteps: 10, + timeoutMs: 1000, + ruleProfile: 'default', + telemetry: 'summary', + }, + summary: { + total: 1, + solved: 1, + stalled: 0, + parseError: 0, + runtimeError: 0, + stepCapped: 0, + timeCapped: 0, + totalDurationMs: 10, + ruleUsage: { a: 1 }, + telemetry: { + ruleAttempts: { + totalAttemptCount: 2, + rules: { + a: { + ruleId: 'a', + ruleName: 'A', + attemptCount: 2, + hitCount: 1, + missCount: 1, + hitRate: 0.5, + totalDurationMs: 5, + hitDurationMs: 3, + missDurationMs: 2, + averageDurationMs: 2.5, + producedDiffCount: 1, + }, + }, + finalNoHitScan: null, + }, + strongInference: { + coverage: { + status: 'none', + supportedRules: [], + unsupportedRules: [ + { ruleId: 'strong', ruleName: 'Strong', supported: false }, + ], + }, + summary: { + totals: { + attemptCount: 0, + hitCount: 0, + missCount: 0, + timeoutCount: 0, + hitRate: 0, + candidateCount: 0, + probeCount: 0, + trialStepCount: 0, + probeDurationMs: 0, + producedDiffCount: 0, + }, + rules: [], + }, + }, + }, + }, + items: [ + { + id: 'puzzle', + puzzleType: 'slitherlink', + sourceUrl: 'url', + width: 1, + height: 1, + status: 'solved', + stepCount: 1, + durationMs: 10, + ruleUsage: { a: 1 }, + terminal: null, + }, + ], +} + +describe('benchmark text formatter', () => { + it('highlights status, expensive rules, and missing strong telemetry', () => { + const text = formatBenchmarkReportText(report) + + expect(text).toContain('1 solved') + expect(text).toContain('Most expensive rules') + expect(text).toContain('Strong inference (none coverage)') + expect(text).toContain('Telemetry not implemented') + expect(text).toContain('strong (Strong)') + }) +}) diff --git a/src/domain/benchmark/textFormatter.ts b/src/domain/benchmark/textFormatter.ts new file mode 100644 index 0000000..49ecd50 --- /dev/null +++ b/src/domain/benchmark/textFormatter.ts @@ -0,0 +1,72 @@ +import type { BenchmarkReport } from './types' + +const formatDuration = (durationMs: number): string => + durationMs >= 1000 + ? `${(durationMs / 1000).toFixed(2)}s` + : `${durationMs.toFixed(1)}ms` + +const formatRuleRows = ( + report: BenchmarkReport, + kind: 'duration' | 'misses', +): string[] => { + const rules = Object.values( + report.summary.telemetry?.ruleAttempts.rules ?? {}, + ) + const sorted = [...rules] + .sort((left, right) => + kind === 'duration' + ? right.totalDurationMs - left.totalDurationMs + : right.missCount - left.missCount || + right.missDurationMs - left.missDurationMs, + ) + .slice(0, 5) + if (sorted.length === 0) return [' (no rule-attempt telemetry)'] + return sorted.map( + (rule) => + ` ${rule.ruleId}: ${formatDuration(rule.totalDurationMs)}, ${rule.hitCount}/${rule.attemptCount} hits, ${rule.missCount} misses`, + ) +} + +export const formatBenchmarkReportText = (report: BenchmarkReport): string => { + const { summary } = report + const lines = [ + `${report.run.datasetId} (schema v${report.schemaVersion}, telemetry ${report.run.telemetry})`, + `Status: ${summary.solved} solved, ${summary.stalled} stalled, ${summary.parseError} parse errors, ${summary.runtimeError} runtime errors, ${summary.stepCapped} step capped, ${summary.timeCapped} time capped`, + `Total: ${summary.total} puzzles, ${formatDuration(summary.totalDurationMs)}`, + '', + 'Puzzles:', + ...report.items.map( + (item) => + ` ${item.id}: ${item.status}, ${item.stepCount} steps, ${formatDuration(item.durationMs)}`, + ), + ] + + if (!summary.telemetry) return lines.join('\n') + + lines.push( + '', + 'Most expensive rules:', + ...formatRuleRows(report, 'duration'), + '', + 'Most frequently missed rules:', + ...formatRuleRows(report, 'misses'), + ) + + const strong = summary.telemetry.strongInference + const totals = strong.summary.totals + lines.push( + '', + `Strong inference (${strong.coverage.status} coverage): ${totals.attemptCount} attempts, ${totals.hitCount} hits, ${totals.missCount} misses, ${totals.timeoutCount} timeouts`, + ` ${totals.probeCount} probes, ${totals.trialStepCount} trial steps, ${formatDuration(totals.probeDurationMs)} probe time`, + ) + if (strong.coverage.unsupportedRules.length > 0) { + lines.push( + ' Telemetry not implemented:', + ...strong.coverage.unsupportedRules.map( + (rule) => ` ${rule.ruleId} (${rule.ruleName})`, + ), + ) + } + + return lines.join('\n') +} diff --git a/src/domain/benchmark/tsvFormatter.test.ts b/src/domain/benchmark/tsvFormatter.test.ts new file mode 100644 index 0000000..690048f --- /dev/null +++ b/src/domain/benchmark/tsvFormatter.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import { runBenchmarkManifest } from './runner' +import { formatBenchmarkReportTsv, serializeTsv } from './tsvFormatter' +import type { BenchmarkDatasetManifest } from './types' + +const manifest: BenchmarkDatasetManifest = { + schemaVersion: 1, + id: 'sample', + title: 'Sample', + puzzleType: 'slitherlink', + items: [ + { + id: 'slitherlink-3x3-0001', + puzzleType: 'slitherlink', + sourceUrl: 'https://puzz.link/p?slither/3/3/g0h', + width: 3, + height: 3, + tags: [], + }, + ], +} + +describe('benchmark TSV formatter', () => { + it('serializes stable columns, empty values, booleans, tabs, and newlines', () => { + expect( + serializeTsv( + ['a', 'b', 'c', 'd'], + [ + { + a: true, + b: null, + c: 'tab\tand\n"quoted" line', + d: 1.23456789, + }, + ], + ), + ).toBe('a\tb\tc\td\ntrue\t\ttab and "quoted" line\t1.234568\n') + }) + + it('builds puzzle and long rule tables with unattempted rules', () => { + const report = runBenchmarkManifest(manifest, { + maxSteps: 1, + telemetry: 'summary', + }) + const tables = formatBenchmarkReportTsv(report) + const puzzleLines = tables.puzzles.trimEnd().split('\n') + const ruleLines = tables.ruleAttempts!.trimEnd().split('\n') + const strongLines = tables.strongInference!.trimEnd().split('\n') + + expect(puzzleLines).toHaveLength(2) + expect(puzzleLines[0]).toContain('strong_coverage') + expect( + ruleLines.some((line) => line.includes('\tfalse\t0\t0\t0\t0\t')), + ).toBe(true) + expect(strongLines).toHaveLength(4) + expect(strongLines[1]).toContain('\tfalse\t\t\t\t\t\t\t') + }) + + it('only builds the puzzle table when telemetry is off', () => { + const report = runBenchmarkManifest(manifest, { + maxSteps: 1, + telemetry: 'off', + }) + const tables = formatBenchmarkReportTsv(report) + + expect(tables.ruleAttempts).toBeUndefined() + expect(tables.strongInference).toBeUndefined() + expect(tables.puzzles.split('\n')[1]).toContain('\toff\t') + }) +}) diff --git a/src/domain/benchmark/tsvFormatter.ts b/src/domain/benchmark/tsvFormatter.ts new file mode 100644 index 0000000..b0b13c9 --- /dev/null +++ b/src/domain/benchmark/tsvFormatter.ts @@ -0,0 +1,293 @@ +import { puzzleRegistry } from '../plugins/registry' +import type { RuleAttemptRuleSummary } from '../rules/ruleAttemptSummaryCollector' +import type { StrongInferenceRuleSummary } from '../rules/strongInferenceSummaryCollector' +import type { BenchmarkPuzzleResult, BenchmarkReport } from './types' + +type TsvValue = string | number | boolean | null | undefined +type TsvRow = Record + +export type BenchmarkTsvTables = { + puzzles: string + ruleAttempts?: string + strongInference?: string +} + +const PUZZLE_HEADERS = [ + 'dataset_id', + 'run_started_at', + 'puzzle_id', + 'puzzle_type', + 'source_url', + 'width', + 'height', + 'status', + 'solved', + 'step_count', + 'duration_ms', + 'terminal_status', + 'terminal_decided_ratio', + 'terminal_unknown_units', + 'error', + 'telemetry_level', + 'total_rule_attempts', + 'total_rule_hits', + 'total_rule_misses', + 'total_rule_duration_ms', + 'final_no_hit_scan_attempts', + 'final_no_hit_scan_duration_ms', + 'strong_coverage', + 'strong_attempts', + 'strong_hits', + 'strong_misses', + 'strong_timeouts', + 'strong_candidates', + 'strong_probes', + 'strong_trial_steps', + 'strong_probe_duration_ms', + 'strong_produced_diffs', +] as const + +const RULE_ATTEMPT_HEADERS = [ + 'dataset_id', + 'puzzle_id', + 'puzzle_type', + 'puzzle_status', + 'rule_id', + 'rule_name', + 'attempted', + 'attempt_count', + 'hit_count', + 'miss_count', + 'hit_rate', + 'total_duration_ms', + 'hit_duration_ms', + 'miss_duration_ms', + 'average_duration_ms', + 'produced_diff_count', +] as const + +const STRONG_INFERENCE_HEADERS = [ + 'dataset_id', + 'puzzle_id', + 'puzzle_type', + 'puzzle_status', + 'coverage_status', + 'rule_id', + 'rule_name', + 'telemetry_supported', + 'attempted', + 'attempt_count', + 'hit_count', + 'miss_count', + 'timeout_count', + 'hit_rate', + 'candidate_count', + 'probe_count', + 'trial_step_count', + 'probe_duration_ms', + 'produced_diff_count', +] as const + +const serializeTsvValue = (value: TsvValue): string => { + if (value === null || value === undefined) return '' + if (typeof value === 'number' && !Number.isInteger(value)) { + return String(Number(value.toFixed(6))) + } + return String(value).replaceAll('\t', ' ').replaceAll(/\r?\n/g, ' ') +} + +export const serializeTsv = ( + headers: readonly string[], + rows: TsvRow[], +): string => + [ + headers.join('\t'), + ...rows.map((row) => + headers.map((header) => serializeTsvValue(row[header])).join('\t'), + ), + ].join('\n') + '\n' + +const sumRuleMetric = ( + item: BenchmarkPuzzleResult, + metric: keyof Pick< + RuleAttemptRuleSummary, + 'hitCount' | 'missCount' | 'totalDurationMs' + >, +): number | undefined => { + if (!item.telemetry) return undefined + return Object.values(item.telemetry.ruleAttempts.rules).reduce( + (total, rule) => total + rule[metric], + 0, + ) +} + +const buildPuzzleRow = ( + report: BenchmarkReport, + item: BenchmarkPuzzleResult, +): TsvRow => { + const strong = item.telemetry?.strongInference + const strongTotals = strong?.summary.totals + const finalScan = item.telemetry?.ruleAttempts.finalNoHitScan + return { + dataset_id: report.run.datasetId, + run_started_at: report.run.startedAt, + puzzle_id: item.id, + puzzle_type: item.puzzleType, + source_url: item.sourceUrl, + width: item.width, + height: item.height, + status: item.status, + solved: item.status === 'solved', + step_count: item.stepCount, + duration_ms: item.durationMs, + terminal_status: item.terminal?.status, + terminal_decided_ratio: item.terminal?.stats.decidedRatio, + terminal_unknown_units: item.terminal?.stats.unknownUnits, + error: item.error, + telemetry_level: report.run.telemetry, + total_rule_attempts: item.telemetry?.ruleAttempts.totalAttemptCount, + total_rule_hits: sumRuleMetric(item, 'hitCount'), + total_rule_misses: sumRuleMetric(item, 'missCount'), + total_rule_duration_ms: sumRuleMetric(item, 'totalDurationMs'), + final_no_hit_scan_attempts: finalScan?.attemptCount, + final_no_hit_scan_duration_ms: finalScan?.totalDurationMs, + strong_coverage: strong?.coverage.status, + strong_attempts: strongTotals?.attemptCount, + strong_hits: strongTotals?.hitCount, + strong_misses: strongTotals?.missCount, + strong_timeouts: strongTotals?.timeoutCount, + strong_candidates: strongTotals?.candidateCount, + strong_probes: strongTotals?.probeCount, + strong_trial_steps: strongTotals?.trialStepCount, + strong_probe_duration_ms: strongTotals?.probeDurationMs, + strong_produced_diffs: strongTotals?.producedDiffCount, + } +} + +const emptyRuleSummary = ( + ruleId: string, + ruleName: string, +): RuleAttemptRuleSummary => ({ + ruleId, + ruleName, + attemptCount: 0, + hitCount: 0, + missCount: 0, + hitRate: 0, + totalDurationMs: 0, + hitDurationMs: 0, + missDurationMs: 0, + averageDurationMs: 0, + producedDiffCount: 0, +}) + +const buildRuleAttemptRows = ( + report: BenchmarkReport, + item: BenchmarkPuzzleResult, +): TsvRow[] => { + if (!item.telemetry) return [] + const pluginRules = puzzleRegistry.get(item.puzzleType)?.getRules() ?? [] + const observedRules = item.telemetry.ruleAttempts.rules + const ruleOrder = new Map(pluginRules.map((rule, index) => [rule.id, index])) + const rules = new Map( + pluginRules.map((rule) => [ + rule.id, + observedRules[rule.id] ?? emptyRuleSummary(rule.id, rule.name), + ]), + ) + for (const rule of Object.values(observedRules)) { + if (!rules.has(rule.ruleId)) rules.set(rule.ruleId, rule) + } + + return Array.from(rules.values()) + .sort( + (left, right) => + (ruleOrder.get(left.ruleId) ?? Number.MAX_SAFE_INTEGER) - + (ruleOrder.get(right.ruleId) ?? Number.MAX_SAFE_INTEGER) || + left.ruleId.localeCompare(right.ruleId), + ) + .map((rule) => ({ + dataset_id: report.run.datasetId, + puzzle_id: item.id, + puzzle_type: item.puzzleType, + puzzle_status: item.status, + rule_id: rule.ruleId, + rule_name: rule.ruleName, + attempted: rule.attemptCount > 0, + attempt_count: rule.attemptCount, + hit_count: rule.hitCount, + miss_count: rule.missCount, + hit_rate: rule.hitRate, + total_duration_ms: rule.totalDurationMs, + hit_duration_ms: rule.hitDurationMs, + miss_duration_ms: rule.missDurationMs, + average_duration_ms: rule.averageDurationMs, + produced_diff_count: rule.producedDiffCount, + })) +} + +const buildStrongInferenceRows = ( + report: BenchmarkReport, + item: BenchmarkPuzzleResult, +): TsvRow[] => { + const strong = item.telemetry?.strongInference + if (!strong) return [] + const observed = new Map( + strong.summary.rules.map((rule) => [rule.ruleId, rule]), + ) + const declaredRules = [ + ...strong.coverage.supportedRules, + ...strong.coverage.unsupportedRules, + ] + return declaredRules.map((declared) => { + const rule: StrongInferenceRuleSummary | undefined = observed.get( + declared.ruleId, + ) + const supported = declared.supported + return { + dataset_id: report.run.datasetId, + puzzle_id: item.id, + puzzle_type: item.puzzleType, + puzzle_status: item.status, + coverage_status: strong.coverage.status, + rule_id: declared.ruleId, + rule_name: declared.ruleName, + telemetry_supported: supported, + attempted: supported ? (rule?.attemptCount ?? 0) > 0 : undefined, + attempt_count: supported ? (rule?.attemptCount ?? 0) : undefined, + hit_count: supported ? (rule?.hitCount ?? 0) : undefined, + miss_count: supported ? (rule?.missCount ?? 0) : undefined, + timeout_count: supported ? (rule?.timeoutCount ?? 0) : undefined, + hit_rate: supported ? (rule?.hitRate ?? 0) : undefined, + candidate_count: supported ? (rule?.candidateCount ?? 0) : undefined, + probe_count: supported ? (rule?.probeCount ?? 0) : undefined, + trial_step_count: supported ? (rule?.trialStepCount ?? 0) : undefined, + probe_duration_ms: supported ? (rule?.probeDurationMs ?? 0) : undefined, + produced_diff_count: supported + ? (rule?.producedDiffCount ?? 0) + : undefined, + } + }) +} + +export const formatBenchmarkReportTsv = ( + report: BenchmarkReport, +): BenchmarkTsvTables => { + const puzzles = serializeTsv( + PUZZLE_HEADERS, + report.items.map((item) => buildPuzzleRow(report, item)), + ) + if (report.run.telemetry === 'off') return { puzzles } + + return { + puzzles, + ruleAttempts: serializeTsv( + RULE_ATTEMPT_HEADERS, + report.items.flatMap((item) => buildRuleAttemptRows(report, item)), + ), + strongInference: serializeTsv( + STRONG_INFERENCE_HEADERS, + report.items.flatMap((item) => buildStrongInferenceRows(report, item)), + ), + } +} diff --git a/src/domain/benchmark/types.ts b/src/domain/benchmark/types.ts index de662ce..2a3975e 100644 --- a/src/domain/benchmark/types.ts +++ b/src/domain/benchmark/types.ts @@ -1,4 +1,10 @@ +import type { + PuzzleStrongTelemetryConfig, + StrongTelemetryRule, +} from '../plugins/types' import type { CompletionReport } from '../rules/completion' +import type { RuleAttemptSummary } from '../rules/ruleAttemptSummaryCollector' +import type { StrongInferenceSummary } from '../rules/strongInferenceSummaryCollector' export type BenchmarkDatasetItem = { id: string @@ -26,6 +32,30 @@ export type BenchmarkPuzzleStatus = | 'step-capped' | 'time-capped' +export type BenchmarkTelemetryLevel = 'off' | 'summary' + +export type StrongTelemetryCoverageStatus = + | 'full' + | 'partial' + | 'none' + | 'not-applicable' + +export type StrongTelemetryCoverage = { + status: StrongTelemetryCoverageStatus + supportedRules: StrongTelemetryRule[] + unsupportedRules: StrongTelemetryRule[] +} + +export type BenchmarkStrongInferenceTelemetry = { + coverage: StrongTelemetryCoverage + summary: StrongInferenceSummary +} + +export type BenchmarkTelemetrySummary = { + ruleAttempts: RuleAttemptSummary + strongInference: BenchmarkStrongInferenceTelemetry +} + export type BenchmarkPuzzleResult = { id: string puzzleType: string @@ -36,9 +66,8 @@ export type BenchmarkPuzzleResult = { stepCount: number durationMs: number ruleUsage: Record - ruleSteps: Record terminal: CompletionReport | null - steps: [] + telemetry?: BenchmarkTelemetrySummary error?: string } @@ -49,6 +78,7 @@ export type BenchmarkRunConfig = { maxSteps: number timeoutMs: number ruleProfile: 'default' + telemetry: BenchmarkTelemetryLevel } export type BenchmarkSummary = { @@ -61,10 +91,11 @@ export type BenchmarkSummary = { timeCapped: number totalDurationMs: number ruleUsage: Record + telemetry?: BenchmarkTelemetrySummary } export type BenchmarkReport = { - schemaVersion: 1 + schemaVersion: 2 run: BenchmarkRunConfig summary: BenchmarkSummary items: BenchmarkPuzzleResult[] @@ -73,4 +104,23 @@ export type BenchmarkReport = { export type BenchmarkRunnerOptions = { maxSteps?: number timeoutMs?: number + telemetry?: BenchmarkTelemetryLevel +} + +export const getStrongTelemetryCoverage = ( + config: PuzzleStrongTelemetryConfig | undefined, +): StrongTelemetryCoverage => { + const rules = config?.rules ?? [] + const supportedRules = rules.filter((rule) => rule.supported) + const unsupportedRules = rules.filter((rule) => !rule.supported) + const status: StrongTelemetryCoverageStatus = + rules.length === 0 + ? 'not-applicable' + : unsupportedRules.length === 0 + ? 'full' + : supportedRules.length === 0 + ? 'none' + : 'partial' + + return { status, supportedRules, unsupportedRules } } diff --git a/src/domain/plugins/masyuPlugin.ts b/src/domain/plugins/masyuPlugin.ts index 55564d1..bc1ad29 100644 --- a/src/domain/plugins/masyuPlugin.ts +++ b/src/domain/plugins/masyuPlugin.ts @@ -127,6 +127,25 @@ export const getMasyuStats = (puzzle: PuzzleIR): PuzzleStatsContent => { export const masyuPlugin: PuzzlePlugin = { id: 'masyu', displayName: 'Masyu', + strongTelemetry: { + rules: [ + { + ruleId: 'masyu-black-pearl-strong-inference', + ruleName: 'Black Pearl Strong Inference', + supported: true, + }, + { + ruleId: 'masyu-line-component-endpoint-strong-inference', + ruleName: 'Masyu Line Component Endpoint Strong Inference', + supported: true, + }, + { + ruleId: 'masyu-white-pearl-strong-inference', + ruleName: 'White Pearl Strong Inference', + supported: true, + }, + ], + }, liveStats: { coverageTitle: 'Inference Coverage', coverageDescription: 'Decided or colored Masyu state', @@ -139,7 +158,11 @@ export const masyuPlugin: PuzzlePlugin = { legend: masyuLegend, displayOptions: [ { id: 'showTiles', label: 'Show Tiles', enabledByDefault: true }, - { id: 'showLineCrosses', label: 'Show Line Crosses', enabledByDefault: true }, + { + id: 'showLineCrosses', + label: 'Show Line Crosses', + enabledByDefault: true, + }, { id: 'showHighlights', label: 'Show Highlights', enabledByDefault: true }, { id: 'showGridLabels', label: 'Show Grid Labels', enabledByDefault: true }, { id: 'showGrid', label: 'Show Grid', enabledByDefault: true }, diff --git a/src/domain/plugins/slitherPlugin.ts b/src/domain/plugins/slitherPlugin.ts index 0ed0001..b4a3160 100644 --- a/src/domain/plugins/slitherPlugin.ts +++ b/src/domain/plugins/slitherPlugin.ts @@ -261,6 +261,25 @@ export const getSlitherStats = (puzzle: PuzzleIR): PuzzleStatsContent => { export const slitherPlugin: PuzzlePlugin = { id: 'slitherlink', displayName: 'Slitherlink', + strongTelemetry: { + rules: [ + { + ruleId: 'color-assumption-inference', + ruleName: 'Color Assumption Inference', + supported: false, + }, + { + ruleId: 'sector-parity-inference', + ruleName: 'Sector Parity Inference', + supported: false, + }, + { + ruleId: 'strong-inference', + ruleName: 'Strong Inference (Conservative)', + supported: false, + }, + ], + }, liveStats: { coverageTitle: 'Inference Coverage', coverageDescription: 'Decided or narrowed Slitherlink state', diff --git a/src/domain/plugins/types.ts b/src/domain/plugins/types.ts index 0677cac..ee8751a 100644 --- a/src/domain/plugins/types.ts +++ b/src/domain/plugins/types.ts @@ -98,6 +98,16 @@ export type PuzzleLiveStatsConfig = { coverageSeries: LiveStatsCoverageSeries[] } +export type StrongTelemetryRule = { + ruleId: string + ruleName: string + supported: boolean +} + +export type PuzzleStrongTelemetryConfig = { + rules: StrongTelemetryRule[] +} + export interface PuzzlePlugin { id: string displayName: string @@ -105,6 +115,7 @@ export interface PuzzlePlugin { legend?: PuzzleLegendContent displayOptions?: PuzzleDisplayOption[] liveStats?: PuzzleLiveStatsConfig + strongTelemetry?: PuzzleStrongTelemetryConfig getStats?: (puzzle: PuzzleIR) => PuzzleStatsContent | null parse: (input: string) => PuzzleIR encode: (puzzle: PuzzleIR) => string diff --git a/src/domain/rules/composeSolverObservers.test.ts b/src/domain/rules/composeSolverObservers.test.ts new file mode 100644 index 0000000..a086003 --- /dev/null +++ b/src/domain/rules/composeSolverObservers.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from 'vitest' +import { composeSolverObservers } from './composeSolverObservers' +import type { RuleAttemptEvent, StrongInferenceCompletedEvent } from './types' + +const ruleAttempt: RuleAttemptEvent = { + solverStepNumber: 1, + ruleId: 'rule-a', + ruleName: 'Rule A', + durationMs: 2, + hit: true, + producedDiffCount: 1, +} + +const strongInference: StrongInferenceCompletedEvent = { + solverStepNumber: 1, + ruleId: 'rule-a', + ruleName: 'Rule A', + candidateCount: 1, + probeCount: 2, + trialStepCount: 3, + probeDurationMs: 4, + outcome: 'hit', + producedDiffCount: 1, +} + +describe('compose solver observers', () => { + it('fans out events in registration order', () => { + const calls: string[] = [] + const observer = composeSolverObservers([ + { + onRuleAttemptCompleted: () => calls.push('first-rule'), + onStrongInferenceCompleted: () => calls.push('first-strong'), + }, + undefined, + { + onRuleAttemptCompleted: () => calls.push('second-rule'), + onStrongInferenceCompleted: () => calls.push('second-strong'), + }, + ]) + + observer.onRuleAttemptCompleted?.(ruleAttempt) + observer.onStrongInferenceCompleted?.(strongInference) + + expect(calls).toEqual([ + 'first-rule', + 'second-rule', + 'first-strong', + 'second-strong', + ]) + }) + + it('isolates observer failures', () => { + const laterObserver = vi.fn() + const observer = composeSolverObservers([ + { + onRuleAttemptCompleted: () => { + throw new Error('collector failed') + }, + }, + { onRuleAttemptCompleted: laterObserver }, + ]) + + expect(() => observer.onRuleAttemptCompleted?.(ruleAttempt)).not.toThrow() + expect(laterObserver).toHaveBeenCalledWith(ruleAttempt) + }) +}) diff --git a/src/domain/rules/composeSolverObservers.ts b/src/domain/rules/composeSolverObservers.ts new file mode 100644 index 0000000..09af76e --- /dev/null +++ b/src/domain/rules/composeSolverObservers.ts @@ -0,0 +1,30 @@ +import type { SolverObserver } from './types' + +export const composeSolverObservers = ( + observers: Array, +): SolverObserver => { + const activeObservers = observers.filter( + (observer): observer is SolverObserver => observer !== undefined, + ) + + return { + onRuleAttemptCompleted: (event) => { + for (const observer of activeObservers) { + try { + observer.onRuleAttemptCompleted?.(event) + } catch { + // Observability must never affect solver behavior. + } + } + }, + onStrongInferenceCompleted: (event) => { + for (const observer of activeObservers) { + try { + observer.onStrongInferenceCompleted?.(event) + } catch { + // Observability must never affect solver behavior. + } + } + }, + } +} From b91466a3e8a0f4c5f36635b32f883edba9229838 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Mon, 15 Jun 2026 15:51:01 +0800 Subject: [PATCH 07/17] feat: add rule documentation workflow and new Slitherlink rules[docs] --- docs/PROJECT_GUIDE_EN.md | 1 + docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md | 259 ++++++++++++++++++ docs/content/masyu/rules/black-pearl-rule.mdx | 18 +- docs/content/masyu/rules/white-pearl-rule.mdx | 18 +- .../slitherlink/rules/adjacent-threes.mdx | 21 ++ .../slitherlink/rules/diagonal-threes.mdx | 18 ++ .../slitherlink/rules/vertex-degree.mdx | 20 +- src/domain/rules/slither/rules/patterns.ts | 77 ++++-- .../rules/slither/tests/patterns.test.ts | 8 +- src/features/docs/RuleExample.tsx | 30 +- src/features/docs/ruleDocRegistry.tsx | 11 + src/features/docs/ruleExamples.ts | 54 +++- 12 files changed, 455 insertions(+), 80 deletions(-) create mode 100644 docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md create mode 100644 docs/content/slitherlink/rules/adjacent-threes.mdx create mode 100644 docs/content/slitherlink/rules/diagonal-threes.mdx diff --git a/docs/PROJECT_GUIDE_EN.md b/docs/PROJECT_GUIDE_EN.md index b39b343..377694a 100644 --- a/docs/PROJECT_GUIDE_EN.md +++ b/docs/PROJECT_GUIDE_EN.md @@ -198,6 +198,7 @@ For targeted work: - Slitherlink rules: start at `src/domain/rules/slither/rules.ts`. - Masyu rules: start at `docs/MASYU_AGENT_BRIEF.md`, then inspect `src/domain/rules/masyu/rules.ts`. +- Rule documentation: follow `docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md`. - Editor/UI work: inspect the relevant `src/features/*` component plus page tests. - Benchmark work: read `src/domain/benchmark/runner.ts` and `scripts/benchmark-solve.ts`. - Historical Masyu plans: check `docs/legacy/` only when old design context is useful. diff --git a/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md b/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md new file mode 100644 index 0000000..018e309 --- /dev/null +++ b/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md @@ -0,0 +1,259 @@ +# Rule Documentation Agent Workflow + +Use this workflow when documenting registered PuzzleKit solver rules. It is +designed for AI agents that must read the production implementation, explain the +deduction accurately, and add an in-app rule page without changing solver +behavior. + +## Goal + +For each selected rule, produce documentation that lets a player answer three +questions: + +1. **What it does:** What decisions can this rule add to the board? +2. **When it triggers:** What exact board state allows the deduction? +3. **Why it works:** Why would every alternative violate a puzzle rule? + +The production rule implementation is the source of truth. Existing names, +messages, tests, technique notes, and external terminology are supporting +evidence only. + +## Required Reading + +Read these files before documenting a rule: + +1. The rule factory and every helper that materially affects its trigger or + conclusion. +2. The rule registration file to understand its order and puzzle family. +3. Existing focused tests to see supported orientations, boundary cases, and + multi-match behavior. +4. `src/features/docs/ruleDocRegistry.tsx` +5. `src/features/docs/ruleExamples.ts` +6. Existing MDX documents under `docs/content//rules/` + +Do not infer behavior from the rule name alone. + +## Workflow + +### 1. Build a Rule Fact Sheet + +Before editing, write a private fact sheet from the implementation: + +```text +Rule ID: +Current display name: +Factory function: +Puzzle family: +Trigger: +Conclusions: +Orientations or mirrored forms: +Existing-state requirements: +Boundary behavior: +Multiple matches in one application: +Reason the deduction is valid: +Related implementation helpers: +``` + +Resolve discrepancies by trusting current production code. If the code appears +logically incorrect or broader than the intended technique, stop documenting +and report the suspected rule issue separately. + +### 2. Review the User-Facing Name + +Prefer a short, recognizable technique name that describes the pattern or +reasoning concept. + +- Change the display name when the current name is overly implementation-driven, + verbose, or ambiguous. +- Preserve the rule ID and factory function by default. They may be referenced + by deep links, traces, tests, benchmarks, or external notes. +- Update the solver-step message when the existing wording no longer matches the + improved name or explanation. +- Keep the message concise and state the trigger plus conclusion. Do not attempt + to reproduce the full MDX explanation in the solver trace. + +### 3. Write the MDX Document + +Write in clear English. Every rule document must contain exactly these three +sections: + +```md +## What it does + +State the visible deductions the rule makes. Include distinct conclusion types +in this section. + +## When it triggers + +State the minimum board conditions required before the rule can act. Explicitly +say when no prior decisions are required. + +## Why it works + +Give the shortest complete logical argument. Explain which puzzle constraint +would be violated by the alternatives. +``` + +Writing rules: + +- Use player-facing puzzle terms, not implementation variables. +- Be precise about `line`, `crossed out`, `inside`, `outside`, clue values, + pearls, vertices, and cells. +- Describe rotations and mirrored forms only when they materially clarify the + trigger. +- Mention multiple subcases only when they share one compact explanation. +- Avoid restating the same sentence in more than one section. +- Do not add `Conclusions`, `Limits and related techniques`, or generic + background sections. +- Do not claim a deduction is unconditional when it relies on the single-loop + rule, a non-empty remainder, uniqueness, or another global assumption. + +### 4. Decide Whether to Add a Canvas Example + +Classify the rule before creating example data. + +#### A. Simple deterministic rule: add an example + +Add a Before/After Canvas example when all of these are true: + +- The trigger fits on one small board. +- One static Before state makes the reason recognizable. +- The complete conclusion is understandable in one After state. +- The example does not need branch history, hidden candidates, or several + sequential deductions. + +Example requirements: + +- Use the smallest board that leaves comfortable visual padding. +- Center the relevant pattern when practical. +- Put prerequisite decisions in `before`. +- Put only this rule's new decisions in `after`. +- Do not highlight clue cells or add colored cell backgrounds. +- Do not use inference overlays or dashed conclusion marks. +- In the After view, highlight only newly decided edges or lines through the + standard board decision rendering. New lines remain solid; new crosses remain + visually distinct. +- The caption should state the deduction, not repeat the full proof. + +#### B. Strong-inference rule: do not add an example yet + +Do not create a Before/After Canvas example for strong inference, assumption +inference, contradiction probing, or branch-comparison rules. A static pair +hides the reasoning chain and can misrepresent why the conclusion is valid. + +Document the rule in MDX and report: + +```text +Canvas example deferred: this is a strong-inference rule whose explanation +requires branch or contradiction history. +``` + +#### C. Complex or multi-case rule: defer when unclear + +Defer the Canvas example when the rule has several materially different trigger +paths, produces different conclusion types, or needs a sequence of intermediate +states to be understood. + +Do not invent an oversimplified example. Report: + +```text +Canvas example needs author input: the rule covers . +Please choose the representative case or approve a multi-stage explanation. +``` + +When uncertain between A and C, choose C. + +### 5. Register the Documentation + +For a completed rule: + +1. Add the MDX file under `docs/content//rules/`. +2. Import it in `src/features/docs/ruleDocRegistry.tsx`. +3. Add a concise one-sentence summary describing the rule's practical result. +4. Add `RuleExampleData` only when the Canvas decision is category A. +5. Keep the existing rule ID as the registry key and documentation deep-link + identifier. + +## Quality Review + +Before finishing, verify: + +- The MDX trigger matches the exact implementation conditions. +- Every diff type the rule can produce is described. +- The proof explains necessity, not merely observed behavior. +- The display name and solver message use the same terminology as the MDX. +- The document has exactly `What it does`, `When it triggers`, and + `Why it works`. +- A Canvas example, if present, shows real board state rather than decorative + overlays. +- Strong-inference and deferred examples are explicitly reported to the author. +- No solver logic, rule order, rule ID, or factory name changed unintentionally. + +Run focused existing tests, formatting, lint, and build. Do not add tests solely +for prose. Update existing assertions only when an approved display name or +solver message changes. + +## Delivery Report + +End the task with a compact report: + +```text +Documented: +- (``) + +Naming: +- +- Rule ID and factory function preserved. + +Canvas: +- Added: +or +- Deferred: + +Verification: +- + +Needs author input: +- +``` + +## Reusable Prompt Chain + +Use these prompts sequentially when assigning the work to an AI agent. + +### Prompt 1: Analyze + +```text +Read the production implementation, registration, helpers, and focused tests +for . Produce a rule fact sheet using +docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md. Do not edit files yet. Flag any +disagreement between the implementation and the apparent intended technique. +``` + +### Prompt 2: Propose + +```text +Using the approved fact sheet, propose the user-facing display name, solver-step +message, three-section English MDX outline, and Canvas classification (simple, +strong inference, or complex/multi-case). Preserve the rule ID and factory +function unless explicitly approved otherwise. +``` + +### Prompt 3: Implement + +```text +Implement the approved rule documentation according to +docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md. Add a Canvas example only for a +simple deterministic rule. For strong-inference or complex rules, document the +rule and explicitly report why the Canvas example was deferred. Run focused +existing tests, formatting, lint, and build. +``` + +### Prompt 4: Review + +```text +Review the completed rule documentation against the production implementation +and docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md. Prioritize logical inaccuracies, +ambiguous trigger wording, misleading Canvas states, duplicated prose, and +unintentional compatibility changes. +``` diff --git a/docs/content/masyu/rules/black-pearl-rule.mdx b/docs/content/masyu/rules/black-pearl-rule.mdx index 964c3df..f1481dd 100644 --- a/docs/content/masyu/rules/black-pearl-rule.mdx +++ b/docs/content/masyu/rules/black-pearl-rule.mdx @@ -1,25 +1,17 @@ -## What this technique does +## What it does A black pearl must be a turn in the loop. Each of the two exits from the pearl -must continue straight through the next cell. +must continue straight through the next cell. This can force the required +straight extensions, cross out an opposite exit, or complete the only remaining +turn around the pearl. -## Trigger conditions +## When it triggers The technique acts when an exit is known, a direction is blocked, or only one valid turning pair remains. -## Conclusions - -It can force the required straight extensions, cross out an opposite exit, or -complete the only remaining turn around the pearl. - ## Why it works Using opposite exits would pass straight through the black pearl. Turning away immediately in a neighboring cell would violate the required straight continuation. - -## Limits and related techniques - -This technique checks direct pearl geometry. Black Pearl Candidate Pruning -combines that geometry with broader local feasibility checks. diff --git a/docs/content/masyu/rules/white-pearl-rule.mdx b/docs/content/masyu/rules/white-pearl-rule.mdx index 8d3745e..05993f3 100644 --- a/docs/content/masyu/rules/white-pearl-rule.mdx +++ b/docs/content/masyu/rules/white-pearl-rule.mdx @@ -1,25 +1,17 @@ -## What this technique does +## What it does A white pearl must be crossed by a straight section of the loop. The loop must also turn in at least one of the cells immediately before or after the pearl. +This can force the opposite line through the pearl, cross out perpendicular +exits, or reject a straight continuation that would make the required nearby +turn impossible. -## Trigger conditions +## When it triggers The technique acts when a known line, blocked direction, or impossible adjacent turn leaves only one valid straight axis through the pearl. -## Conclusions - -It can force the opposite line through the pearl, cross out perpendicular exits, -or reject a straight continuation that would make the required nearby turn -impossible. - ## Why it works Any alternative would either turn on the white pearl or continue straight on both adjacent cells. Both outcomes violate the puzzle's white-pearl rule. - -## Limits and related techniques - -This is local deterministic reasoning. Candidate pruning and strong inference -can resolve cases where more than one locally valid axis remains. diff --git a/docs/content/slitherlink/rules/adjacent-threes.mdx b/docs/content/slitherlink/rules/adjacent-threes.mdx new file mode 100644 index 0000000..b54ab6d --- /dev/null +++ b/docs/content/slitherlink/rules/adjacent-threes.mdx @@ -0,0 +1,21 @@ +## What it does + +Two or more horizontally or vertically adjacent `3` clues force a repeating +line pattern along the run. Every edge perpendicular to the run is a line, +including the shared edges between neighboring `3` clues. Each shared edge is +crossed out where it would continue straight beyond either side of the run. + +## When it triggers + +The technique acts whenever at least two `3` clues form one uninterrupted +horizontal or vertical run. It does not require any edges to be decided first. + +## Why it works + +For a pair of adjacent `3` clues, leaving their shared edge unused would force +the other six surrounding edges to form a closed loop around only those two +cells, preventing any required loop elsewhere from joining it. In a standard +Slitherlink puzzle, the shared edge must therefore be a line. Each clue then +needs both of its edges perpendicular to the run. At either end of a shared +edge, the loop already has two incident lines, so it cannot continue straight +beyond the run. diff --git a/docs/content/slitherlink/rules/diagonal-threes.mdx b/docs/content/slitherlink/rules/diagonal-threes.mdx new file mode 100644 index 0000000..f667fac --- /dev/null +++ b/docs/content/slitherlink/rules/diagonal-threes.mdx @@ -0,0 +1,18 @@ +## What it does + +Two diagonally adjacent `3` clues force the two edges at each clue's outer +corner. The result is four lines: two meeting at the corner of each `3` that +faces away from their shared vertex. + +## When it triggers + +The technique acts whenever two `3` clues occupy opposite cells of the same +`2 x 2` block. It does not require any edges to be decided first. + +## Why it works + +A `3` has exactly one unused edge. If either outer-corner edge of one clue were +unused, its other three edges would be lines. At the vertex shared by the two +clues, that forces the diagonal clue to leave too many of its surrounding edges +unused to reach three lines. Therefore neither clue can use an outer-corner +edge as its single unused edge. diff --git a/docs/content/slitherlink/rules/vertex-degree.mdx b/docs/content/slitherlink/rules/vertex-degree.mdx index 249718e..449f0a7 100644 --- a/docs/content/slitherlink/rules/vertex-degree.mdx +++ b/docs/content/slitherlink/rules/vertex-degree.mdx @@ -1,25 +1,17 @@ -## What this technique does +## What it does Every Slitherlink vertex belongs to either zero or two loop edges. A vertex may -never become a dead end or a branch. +never become a dead end or a branch. If two lines already meet at a vertex, +every other incident edge is crossed out. If one line and only one unknown edge +remain, that unknown edge is forced. If no lines and only one unknown edge +remain, it is crossed out. -## Trigger conditions +## When it triggers The technique acts when the decided edges around a vertex determine the remaining unknown edges. -## Conclusions - -If two lines already meet at a vertex, every other incident edge is crossed out. -If one line and only one unknown edge remain, that unknown edge is forced. -If no lines and only one unknown edge remain, it is crossed out. - ## Why it works A single used edge creates a dead end, while three or more used edges create a branch. Neither can be part of one continuous loop. - -## Limits and related techniques - -This rule enforces local degree only. Prevent Premature Loop and connectivity -rules handle larger loop structure. diff --git a/src/domain/rules/slither/rules/patterns.ts b/src/domain/rules/slither/rules/patterns.ts index 06a991a..c7e878d 100644 --- a/src/domain/rules/slither/rules/patterns.ts +++ b/src/domain/rules/slither/rules/patterns.ts @@ -5,7 +5,7 @@ import { formatCellRunLabel, isClueThree } from './shared' export const createContiguousThreeRunBoundariesRule = (): Rule => ({ id: 'contiguous-three-run-boundaries', - name: 'Contiguous 3-Run Boundaries', + name: 'Adjacent 3s', apply: (puzzle: PuzzleIR): RuleApplication | null => { const decidedEdges = new Map() const allAffectedCells = new Set() @@ -41,7 +41,11 @@ export const createContiguousThreeRunBoundariesRule = (): Rule => ({ } let runAddedAny = false - for (let boundaryCol = cStart; boundaryCol <= cEnd + 1; boundaryCol += 1) { + for ( + let boundaryCol = cStart; + boundaryCol <= cEnd + 1; + boundaryCol += 1 + ) { const key = edgeKey([r, boundaryCol], [r + 1, boundaryCol]) runAddedAny = decideUnknownEdge(key, 'line') || runAddedAny } @@ -58,8 +62,10 @@ export const createContiguousThreeRunBoundariesRule = (): Rule => ({ } if (runAddedAny) { - for (let col = cStart; col <= cEnd; col += 1) allAffectedCells.add(cellKey(r, col)) - if (firstExample === null) firstExample = formatCellRunLabel('row', r, cStart, cEnd) + for (let col = cStart; col <= cEnd; col += 1) + allAffectedCells.add(cellKey(r, col)) + if (firstExample === null) + firstExample = formatCellRunLabel('row', r, cStart, cEnd) } } } @@ -81,7 +87,11 @@ export const createContiguousThreeRunBoundariesRule = (): Rule => ({ } let runAddedAny = false - for (let boundaryRow = rStart; boundaryRow <= rEnd + 1; boundaryRow += 1) { + for ( + let boundaryRow = rStart; + boundaryRow <= rEnd + 1; + boundaryRow += 1 + ) { const key = edgeKey([boundaryRow, c], [boundaryRow, c + 1]) runAddedAny = decideUnknownEdge(key, 'line') || runAddedAny } @@ -98,8 +108,10 @@ export const createContiguousThreeRunBoundariesRule = (): Rule => ({ } if (runAddedAny) { - for (let row = rStart; row <= rEnd; row += 1) allAffectedCells.add(cellKey(row, c)) - if (firstExample === null) firstExample = formatCellRunLabel('col', c, rStart, rEnd) + for (let row = rStart; row <= rEnd; row += 1) + allAffectedCells.add(cellKey(row, c)) + if (firstExample === null) + firstExample = formatCellRunLabel('col', c, rStart, rEnd) } } } @@ -109,8 +121,8 @@ export const createContiguousThreeRunBoundariesRule = (): Rule => ({ return { message: firstExample !== null - ? `Contiguous 3-run at ${firstExample}: the 3-clues need the run boundary lines, so straight extensions outside the run are blank.` - : 'Contiguous 3-run: the 3-clues need the run boundary lines, so straight extensions outside the run are blank.', + ? `Adjacent 3s at ${firstExample}: every edge perpendicular to the run is a line, and each shared edge cannot continue straight beyond the run.` + : 'Adjacent 3s: every edge perpendicular to the run is a line, and each shared edge cannot continue straight beyond the run.', diffs: [...decidedEdges.entries()].map(([k, to]) => ({ kind: 'edge' as const, edgeKey: k, @@ -124,15 +136,17 @@ export const createContiguousThreeRunBoundariesRule = (): Rule => ({ export const createDiagonalAdjacentThreeOuterCornersRule = (): Rule => ({ id: 'diagonal-adjacent-three-outer-corners', - name: 'Diagonal Adjacent 3 Outer Corners', + name: 'Diagonal 3s', apply: (puzzle: PuzzleIR): RuleApplication | null => { const decidedEdges = new Map() const allAffectedCells = new Set() for (let r = 0; r < puzzle.rows - 1; r += 1) { for (let c = 0; c < puzzle.cols - 1; c += 1) { - const mainDiagonal = isClueThree(puzzle, r, c) && isClueThree(puzzle, r + 1, c + 1) - const antiDiagonal = isClueThree(puzzle, r, c + 1) && isClueThree(puzzle, r + 1, c) + const mainDiagonal = + isClueThree(puzzle, r, c) && isClueThree(puzzle, r + 1, c + 1) + const antiDiagonal = + isClueThree(puzzle, r, c + 1) && isClueThree(puzzle, r + 1, c) if (!mainDiagonal && !antiDiagonal) { continue } @@ -155,7 +169,10 @@ export const createDiagonalAdjacentThreeOuterCornersRule = (): Rule => ({ let positionAddedAny = false for (const key of candidateEdgeKeys) { - if ((puzzle.edges[key]?.mark ?? 'unknown') === 'unknown' && !decidedEdges.has(key)) { + if ( + (puzzle.edges[key]?.mark ?? 'unknown') === 'unknown' && + !decidedEdges.has(key) + ) { decidedEdges.set(key, 'line') positionAddedAny = true } @@ -177,7 +194,8 @@ export const createDiagonalAdjacentThreeOuterCornersRule = (): Rule => ({ if (decidedEdges.size === 0) return null return { - message: 'Diagonal adjacent 3s force their outside corner edges to be lines; otherwise one of the 3-clues cannot reach three lines.', + message: + 'Diagonal 3s force the two outer-corner edges of each clue to be lines; otherwise one of the clues cannot reach three lines.', diffs: [...decidedEdges.entries()].map(([k, to]) => ({ kind: 'edge' as const, edgeKey: k, @@ -189,9 +207,15 @@ export const createDiagonalAdjacentThreeOuterCornersRule = (): Rule => ({ }, }) -const getNumberClueValue = (puzzle: PuzzleIR, row: number, col: number): number | null => { +const getNumberClueValue = ( + puzzle: PuzzleIR, + row: number, + col: number, +): number | null => { const clue = puzzle.cells[cellKey(row, col)]?.clue - return clue?.kind === 'number' && clue.value !== '?' ? Number(clue.value) : null + return clue?.kind === 'number' && clue.value !== '?' + ? Number(clue.value) + : null } export const createAdjacentTwoThreeOppositeCrossRule = (): Rule => ({ @@ -216,8 +240,10 @@ export const createAdjacentTwoThreeOppositeCrossRule = (): Rule => ({ return true } - const verticalEdge = (row: number, col: number): string => edgeKey([row, col], [row + 1, col]) - const horizontalEdge = (row: number, col: number): string => edgeKey([row, col], [row, col + 1]) + const verticalEdge = (row: number, col: number): string => + edgeKey([row, col], [row + 1, col]) + const horizontalEdge = (row: number, col: number): string => + edgeKey([row, col], [row, col + 1]) const applyPair = ( twoRow: number, @@ -234,7 +260,10 @@ export const createAdjacentTwoThreeOppositeCrossRule = (): Rule => ({ if (rowDelta === 0) { const sharedCol = colDelta === 1 ? twoCol + 1 : twoCol twoOpposite = verticalEdge(twoRow, colDelta === 1 ? twoCol : twoCol + 1) - threeOpposite = verticalEdge(threeRow, colDelta === 1 ? threeCol + 1 : threeCol) + threeOpposite = verticalEdge( + threeRow, + colDelta === 1 ? threeCol + 1 : threeCol, + ) if (twoRow > 0) { extensionEdges.push(verticalEdge(twoRow - 1, sharedCol)) } @@ -243,8 +272,14 @@ export const createAdjacentTwoThreeOppositeCrossRule = (): Rule => ({ } } else { const sharedRow = rowDelta === 1 ? twoRow + 1 : twoRow - twoOpposite = horizontalEdge(rowDelta === 1 ? twoRow : twoRow + 1, twoCol) - threeOpposite = horizontalEdge(rowDelta === 1 ? threeRow + 1 : threeRow, threeCol) + twoOpposite = horizontalEdge( + rowDelta === 1 ? twoRow : twoRow + 1, + twoCol, + ) + threeOpposite = horizontalEdge( + rowDelta === 1 ? threeRow + 1 : threeRow, + threeCol, + ) if (twoCol > 0) { extensionEdges.push(horizontalEdge(sharedRow, twoCol - 1)) } diff --git a/src/domain/rules/slither/tests/patterns.test.ts b/src/domain/rules/slither/tests/patterns.test.ts index a003a7c..38e12cf 100644 --- a/src/domain/rules/slither/tests/patterns.test.ts +++ b/src/domain/rules/slither/tests/patterns.test.ts @@ -23,7 +23,7 @@ describe('slither contiguous 3-run boundaries rule', () => { const result = threeRunRule.apply(puzzle) expect(result).not.toBeNull() - expect(result?.message).toContain('Contiguous 3-run') + expect(result?.message).toContain('Adjacent 3s') expect(result?.affectedCells).toEqual(['1,1', '1,2', '1,3']) expect(result?.diffs).toEqual([ { @@ -86,7 +86,7 @@ describe('slither contiguous 3-run boundaries rule', () => { const result = threeRunRule.apply(puzzle) expect(result).not.toBeNull() - expect(result?.message).toContain('Contiguous 3-run') + expect(result?.message).toContain('Adjacent 3s') expect(result?.affectedCells).toEqual(['1,2', '2,2', '3,2']) expect(result?.diffs).toEqual([ { @@ -264,7 +264,7 @@ describe('slither diagonal adjacent 3 outer corners rule', () => { expect(result).not.toBeNull() expect(result?.message).toContain( - 'Diagonal adjacent 3s force their outside corner edges', + 'Diagonal 3s force the two outer-corner edges', ) expect(result?.affectedCells).toEqual(['0,0', '1,1']) expect(getEdgeDiffKeys(result)).toEqual([ @@ -284,7 +284,7 @@ describe('slither diagonal adjacent 3 outer corners rule', () => { expect(result).not.toBeNull() expect(result?.message).toContain( - 'Diagonal adjacent 3s force their outside corner edges', + 'Diagonal 3s force the two outer-corner edges', ) expect(result?.affectedCells).toEqual(['0,1', '1,0']) expect(getEdgeDiffKeys(result)).toEqual([ diff --git a/src/features/docs/RuleExample.tsx b/src/features/docs/RuleExample.tsx index d9297ad..19bcfec 100644 --- a/src/features/docs/RuleExample.tsx +++ b/src/features/docs/RuleExample.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from 'react' import type { PuzzleIR } from '../../domain/ir/types' import { applyRuleDiffs } from '../../domain/rules/engine' -import type { InferenceFocus, RuleDiff } from '../../domain/rules/types' +import type { RuleDiff } from '../../domain/rules/types' import { CanvasBoard } from '../board/CanvasBoard' import type { DisplaySettings } from '../solver/solverStore' @@ -9,7 +9,6 @@ type Props = { puzzle: PuzzleIR before?: RuleDiff[] after: RuleDiff[] - highlights?: InferenceFocus explanation?: string } @@ -30,7 +29,6 @@ export const RuleExample = ({ puzzle, before = [], after, - highlights = {}, explanation, }: Props) => { const [view, setView] = useState<'before' | 'after'>('before') @@ -43,6 +41,20 @@ export const RuleExample = ({ () => applyRuleDiffs(beforePuzzle, after), [after, beforePuzzle], ) + const changedEdges = useMemo( + () => + view === 'after' + ? after.flatMap((diff) => (diff.kind === 'edge' ? [diff.edgeKey] : [])) + : [], + [after, view], + ) + const changedLines = useMemo( + () => + view === 'after' + ? after.flatMap((diff) => (diff.kind === 'line' ? [diff.lineKey] : [])) + : [], + [after, view], + ) useEffect(() => { if (!isPlaying) { @@ -88,16 +100,14 @@ export const RuleExample = ({ undefined} variant="surface" - inferenceDiffs={view === 'after' ? after : []} - inferenceDiffRole="conclusion" ariaLabel="Rule documentation example" /> {explanation ?
{explanation}
: null} diff --git a/src/features/docs/ruleDocRegistry.tsx b/src/features/docs/ruleDocRegistry.tsx index d00ece6..76c4419 100644 --- a/src/features/docs/ruleDocRegistry.tsx +++ b/src/features/docs/ruleDocRegistry.tsx @@ -1,6 +1,8 @@ import type { ComponentType } from 'react' import BlackPearlRuleDoc from '../../../docs/content/masyu/rules/black-pearl-rule.mdx' import WhitePearlRuleDoc from '../../../docs/content/masyu/rules/white-pearl-rule.mdx' +import AdjacentThreesRuleDoc from '../../../docs/content/slitherlink/rules/adjacent-threes.mdx' +import DiagonalThreesRuleDoc from '../../../docs/content/slitherlink/rules/diagonal-threes.mdx' import VertexDegreeRuleDoc from '../../../docs/content/slitherlink/rules/vertex-degree.mdx' import { puzzleRegistry } from '../../domain/plugins/registry' import type { Rule } from '../../domain/rules/types' @@ -39,6 +41,15 @@ const documentedContent: Record< summary: 'Maintains the loop degree of zero or two at every Slitherlink vertex.', }, + 'slitherlink:contiguous-three-run-boundaries': { + content: AdjacentThreesRuleDoc, + summary: 'Draws the forced repeating line pattern around adjacent 3 clues.', + }, + 'slitherlink:diagonal-adjacent-three-outer-corners': { + content: DiagonalThreesRuleDoc, + summary: + 'Draws the four forced outer-corner edges of diagonally adjacent 3 clues.', + }, } const getCategory = (rule: Rule): string => { diff --git a/src/features/docs/ruleExamples.ts b/src/features/docs/ruleExamples.ts index ee5dddc..9719391 100644 --- a/src/features/docs/ruleExamples.ts +++ b/src/features/docs/ruleExamples.ts @@ -2,13 +2,12 @@ import { cellKey, edgeKey, lineKey } from '../../domain/ir/keys' import { createMasyuPuzzle } from '../../domain/ir/masyu' import { createSlitherPuzzle } from '../../domain/ir/slither' import type { PuzzleIR } from '../../domain/ir/types' -import type { InferenceFocus, RuleDiff } from '../../domain/rules/types' +import type { RuleDiff } from '../../domain/rules/types' export type RuleExampleData = { puzzle: PuzzleIR before?: RuleDiff[] after: RuleDiff[] - highlights?: InferenceFocus explanation: string } @@ -27,7 +26,6 @@ const createWhitePearlExample = (): RuleExampleData => { { kind: 'line', lineKey: up, from: 'unknown', to: 'blank' }, { kind: 'line', lineKey: down, from: 'unknown', to: 'blank' }, ], - highlights: { cells: [cellKey(1, 1)], lines: [right, up, down] }, explanation: 'A known horizontal exit forces the white pearl to continue horizontally and rejects both vertical exits.', } @@ -46,7 +44,6 @@ const createBlackPearlExample = (): RuleExampleData => { { kind: 'line', lineKey: upExtension, from: 'unknown', to: 'line' }, { kind: 'line', lineKey: down, from: 'unknown', to: 'blank' }, ], - highlights: { cells: [cellKey(2, 1)], lines: [upExtension, down] }, explanation: 'Once the loop exits upward from a black pearl, it must continue straight for another cell and cannot also exit downward.', } @@ -68,14 +65,61 @@ const createVertexDegreeExample = (): RuleExampleData => { { kind: 'edge', edgeKey: right, from: 'unknown', to: 'blank' }, { kind: 'edge', edgeKey: bottom, from: 'unknown', to: 'blank' }, ], - highlights: { edges: [right, bottom], vertices: ['1,1'] }, explanation: 'A loop vertex already containing two used edges cannot accept either remaining edge.', } } +const createAdjacentThreesExample = (): RuleExampleData => { + const puzzle = createSlitherPuzzle(3, 4) + puzzle.cells[cellKey(1, 1)] = { clue: { kind: 'number', value: 3 } } + puzzle.cells[cellKey(1, 2)] = { clue: { kind: 'number', value: 3 } } + const left = edgeKey([1, 1], [2, 1]) + const shared = edgeKey([1, 2], [2, 2]) + const right = edgeKey([1, 3], [2, 3]) + const sharedAbove = edgeKey([0, 2], [1, 2]) + const sharedBelow = edgeKey([2, 2], [3, 2]) + return { + puzzle, + after: [ + { kind: 'edge', edgeKey: left, from: 'unknown', to: 'line' }, + { kind: 'edge', edgeKey: shared, from: 'unknown', to: 'line' }, + { kind: 'edge', edgeKey: right, from: 'unknown', to: 'line' }, + { kind: 'edge', edgeKey: sharedAbove, from: 'unknown', to: 'blank' }, + { kind: 'edge', edgeKey: sharedBelow, from: 'unknown', to: 'blank' }, + ], + explanation: + 'Adjacent 3s force every perpendicular edge to be a line and cross out both straight extensions of their shared edge. The vertical pattern is the rotated equivalent, and longer runs apply the same deduction pairwise.', + } +} + +const createDiagonalThreesExample = (): RuleExampleData => { + const puzzle = createSlitherPuzzle(4, 4) + puzzle.cells[cellKey(1, 1)] = { clue: { kind: 'number', value: 3 } } + puzzle.cells[cellKey(2, 2)] = { clue: { kind: 'number', value: 3 } } + const firstLeft = edgeKey([1, 1], [2, 1]) + const firstTop = edgeKey([1, 1], [1, 2]) + const secondRight = edgeKey([2, 3], [3, 3]) + const secondBottom = edgeKey([3, 2], [3, 3]) + const decidedEdges = [firstLeft, firstTop, secondRight, secondBottom] + return { + puzzle, + after: decidedEdges.map((key) => ({ + kind: 'edge' as const, + edgeKey: key, + from: 'unknown' as const, + to: 'line' as const, + })), + explanation: + 'Diagonal 3s force the two edges at each outer corner, giving four lines in total.', + } +} + export const ruleExamples: Record = { 'masyu:white-pearl-rule': createWhitePearlExample(), 'masyu:black-pearl-rule': createBlackPearlExample(), 'slitherlink:vertex-degree': createVertexDegreeExample(), + 'slitherlink:contiguous-three-run-boundaries': createAdjacentThreesExample(), + 'slitherlink:diagonal-adjacent-three-outer-corners': + createDiagonalThreesExample(), } From 2d9fa05514d1f72252d0c99dc1f7a4b8732589c5 Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Mon, 15 Jun 2026 16:24:43 +0800 Subject: [PATCH 08/17] feat: add new Slitherlink rules for adjacent two-three opposite cross and cell clue completion with examples and documentation --- .../adjacent-two-three-opposite-cross.mdx | 24 ++++++++++ .../rules/cell-clue-completion.mdx | 22 ++++++++++ src/features/docs/ruleDocRegistry.tsx | 12 +++++ src/features/docs/ruleExamples.ts | 44 +++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 docs/content/slitherlink/rules/adjacent-two-three-opposite-cross.mdx create mode 100644 docs/content/slitherlink/rules/cell-clue-completion.mdx diff --git a/docs/content/slitherlink/rules/adjacent-two-three-opposite-cross.mdx b/docs/content/slitherlink/rules/adjacent-two-three-opposite-cross.mdx new file mode 100644 index 0000000..d66bdd9 --- /dev/null +++ b/docs/content/slitherlink/rules/adjacent-two-three-opposite-cross.mdx @@ -0,0 +1,24 @@ +## What it does + +When a `2` and a `3` are horizontally or vertically adjacent, and the edge on +the `2` opposite their shared side is already crossed out, the edge on the `3` +opposite their shared side is forced to be a line. The edges that continue +straight along their shared side beyond the pair are crossed out. At a board +edge, only the in-bounds extension is crossed out. + +## When it triggers + +The technique acts on a horizontally or vertically adjacent pair of numbered +clues with values `2` and `3`, in either order. The edge on the `2` that lies +on the side away from the `3` must already be crossed out. Unknown edges do not +satisfy this prerequisite. + +## Why it works + +With its far-side edge crossed out, the `2` must take both of its lines from +its shared edge with the `3` and from one of the two edges perpendicular to +that shared side. That fixes how the loop passes through the pair, so the +`3` must use its own far-side edge to reach three lines. If the loop also +continued straight along the shared side beyond the pair, a vertex on that +side would receive too many incident lines or the `3` could not reach its +required count, so those extension edges must be crossed out. diff --git a/docs/content/slitherlink/rules/cell-clue-completion.mdx b/docs/content/slitherlink/rules/cell-clue-completion.mdx new file mode 100644 index 0000000..bad5fe3 --- /dev/null +++ b/docs/content/slitherlink/rules/cell-clue-completion.mdx @@ -0,0 +1,22 @@ +## What it does + +When a numbered clue already has enough lines among its four surrounding +edges, every remaining unknown edge is crossed out. When every remaining +unknown edge is needed to reach the clue count, all of them are marked as +lines. + +## When it triggers + +The technique acts on a numbered cell that still has at least one unknown +surrounding edge. Either the number of existing lines already equals the clue, +or the number of existing lines plus the number of unknown edges equals the +clue. No other prior decisions are required beyond the partial edge state +around that cell. + +## Why it works + +A numbered cell must have exactly that many surrounding lines. If the clue is +already satisfied, any additional line would exceed it, so every remaining +unknown edge must be crossed out. If only as many unknown edges remain as are +still needed, crossing out any one of them would leave the clue unsatisfied, +so all of them must be lines. diff --git a/src/features/docs/ruleDocRegistry.tsx b/src/features/docs/ruleDocRegistry.tsx index 76c4419..c54f599 100644 --- a/src/features/docs/ruleDocRegistry.tsx +++ b/src/features/docs/ruleDocRegistry.tsx @@ -2,6 +2,8 @@ import type { ComponentType } from 'react' import BlackPearlRuleDoc from '../../../docs/content/masyu/rules/black-pearl-rule.mdx' import WhitePearlRuleDoc from '../../../docs/content/masyu/rules/white-pearl-rule.mdx' import AdjacentThreesRuleDoc from '../../../docs/content/slitherlink/rules/adjacent-threes.mdx' +import AdjacentTwoThreeOppositeCrossRuleDoc from '../../../docs/content/slitherlink/rules/adjacent-two-three-opposite-cross.mdx' +import CellClueCompletionRuleDoc from '../../../docs/content/slitherlink/rules/cell-clue-completion.mdx' import DiagonalThreesRuleDoc from '../../../docs/content/slitherlink/rules/diagonal-threes.mdx' import VertexDegreeRuleDoc from '../../../docs/content/slitherlink/rules/vertex-degree.mdx' import { puzzleRegistry } from '../../domain/plugins/registry' @@ -50,6 +52,16 @@ const documentedContent: Record< summary: 'Draws the four forced outer-corner edges of diagonally adjacent 3 clues.', }, + 'slitherlink:cell-count-completion': { + content: CellClueCompletionRuleDoc, + summary: + 'Completes or blanks every remaining unknown edge around a numbered clue when the clue count is already determined.', + }, + 'slitherlink:adjacent-two-three-opposite-cross': { + content: AdjacentTwoThreeOppositeCrossRuleDoc, + summary: + "When a 2's far-side edge is crossed out beside a 3, forces the 3's opposite line and blanks the shared-side extensions.", + }, } const getCategory = (rule: Rule): string => { diff --git a/src/features/docs/ruleExamples.ts b/src/features/docs/ruleExamples.ts index 9719391..db6b31f 100644 --- a/src/features/docs/ruleExamples.ts +++ b/src/features/docs/ruleExamples.ts @@ -93,6 +93,47 @@ const createAdjacentThreesExample = (): RuleExampleData => { } } +const createCellClueCompletionExample = (): RuleExampleData => { + const puzzle = createSlitherPuzzle(3, 3) + puzzle.cells[cellKey(1, 1)] = { clue: { kind: 'number', value: 1 } } + const top = edgeKey([1, 1], [1, 2]) + const bottom = edgeKey([2, 1], [2, 2]) + const left = edgeKey([1, 1], [2, 1]) + const right = edgeKey([1, 2], [2, 2]) + return { + puzzle, + before: [{ kind: 'edge', edgeKey: top, from: 'unknown', to: 'line' }], + after: [ + { kind: 'edge', edgeKey: bottom, from: 'unknown', to: 'blank' }, + { kind: 'edge', edgeKey: left, from: 'unknown', to: 'blank' }, + { kind: 'edge', edgeKey: right, from: 'unknown', to: 'blank' }, + ], + explanation: + 'Once the clue already has one line, every remaining unknown edge must be crossed out.', + } +} + +const createAdjacentTwoThreeOppositeCrossExample = (): RuleExampleData => { + const puzzle = createSlitherPuzzle(4, 4) + puzzle.cells[cellKey(1, 1)] = { clue: { kind: 'number', value: 2 } } + puzzle.cells[cellKey(1, 2)] = { clue: { kind: 'number', value: 3 } } + const twoOpposite = edgeKey([1, 1], [2, 1]) + const threeOpposite = edgeKey([1, 3], [2, 3]) + const extensionAbove = edgeKey([0, 2], [1, 2]) + const extensionBelow = edgeKey([2, 2], [3, 2]) + return { + puzzle, + before: [{ kind: 'edge', edgeKey: twoOpposite, from: 'unknown', to: 'blank' }], + after: [ + { kind: 'edge', edgeKey: threeOpposite, from: 'unknown', to: 'line' }, + { kind: 'edge', edgeKey: extensionAbove, from: 'unknown', to: 'blank' }, + { kind: 'edge', edgeKey: extensionBelow, from: 'unknown', to: 'blank' }, + ], + explanation: + "With the 2's far-side edge crossed out, the 3's opposite edge is a line and the shared-side extensions are blank.", + } +} + const createDiagonalThreesExample = (): RuleExampleData => { const puzzle = createSlitherPuzzle(4, 4) puzzle.cells[cellKey(1, 1)] = { clue: { kind: 'number', value: 3 } } @@ -122,4 +163,7 @@ export const ruleExamples: Record = { 'slitherlink:contiguous-three-run-boundaries': createAdjacentThreesExample(), 'slitherlink:diagonal-adjacent-three-outer-corners': createDiagonalThreesExample(), + 'slitherlink:cell-count-completion': createCellClueCompletionExample(), + 'slitherlink:adjacent-two-three-opposite-cross': + createAdjacentTwoThreeOppositeCrossExample(), } From c5f0f20d09a7a1e47fe282a29062b4bb4fca23bb Mon Sep 17 00:00:00 2001 From: xiaoxiao Date: Mon, 15 Jun 2026 17:28:17 +0800 Subject: [PATCH 09/17] feat: enhance rule documentation with multi-canvas struct and styling updates for better clarity --- docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md | 220 ++++++++++++++++------ src/app/DocsPage.test.tsx | 2 +- src/app/workspace.css | 67 ++++++- src/features/docs/RuleExample.test.tsx | 87 +++++++++ src/features/docs/RuleExample.tsx | 89 +++++---- src/features/docs/ruleDocRegistry.test.ts | 12 ++ src/features/docs/ruleExamples.ts | 95 +++++++--- 7 files changed, 450 insertions(+), 122 deletions(-) create mode 100644 src/features/docs/RuleExample.test.tsx diff --git a/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md b/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md index 018e309..06b0626 100644 --- a/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md +++ b/docs/RULE_DOCUMENTATION_AGENT_WORKFLOW.md @@ -29,7 +29,9 @@ Read these files before documenting a rule: multi-match behavior. 4. `src/features/docs/ruleDocRegistry.tsx` 5. `src/features/docs/ruleExamples.ts` -6. Existing MDX documents under `docs/content//rules/` +6. `src/features/docs/RuleExample.tsx` +7. The `.rule-example*` styles in `src/app/workspace.css` +8. Existing MDX documents under `docs/content//rules/` Do not infer behavior from the rule name alone. @@ -110,7 +112,17 @@ Writing rules: ### 4. Decide Whether to Add a Canvas Example -Classify the rule before creating example data. +Classify the rule before creating example data. A rule does not need to fit into +one Canvas to qualify. Prefer a small multi-Canvas group over omitting a useful +deterministic explanation merely because the rule has several separable cases. + +The classifications are: + +- **Simple deterministic:** use one Canvas case. +- **Separable deterministic:** use multiple Canvas cases. +- **Strong inference:** never add Canvas cases. +- **Too complex or too coupled:** defer the Canvas and leave an explicit + reminder for the author. #### A. Simple deterministic rule: add an example @@ -135,33 +147,157 @@ Example requirements: visually distinct. - The caption should state the deduction, not repeat the full proof. -#### B. Strong-inference rule: do not add an example yet +#### B. Separable deterministic rule: add multiple Canvas cases + +A somewhat complex rule should still receive Canvas documentation when its +important triggers or conclusions can be divided into independent, +direct Before/After cases. Do not defer merely because one board cannot express +the whole rule clearly. + +Use one case per distinct point that a reader needs to compare, such as: + +- Two equally important completion branches. +- Distinct deterministic trigger shapes with the same underlying rule. +- Different direct conclusion types that would be confusing on one board. + +Case-selection rules: + +- Prefer an even number of cases, normally `2` or `4`, because the desktop + layout uses two columns. +- Do not invent a redundant or weak case solely to make the count even. A clear + three-case group is better than four cases with filler. +- Normally stop at four representative cases. More than four is a strong signal + that the rule should be deferred or redesigned with author input. +- Give every case a stable, descriptive `id`. +- Give every case in a multi-case group a short, parallel title. +- Use similarly sized puzzles where practical. Prefer matching dimensions; small + differences are acceptable when the Canvas cards still feel balanced. +- Avoid grouping boards when the largest is roughly more than twice the width + or height of the smallest. +- Keep every case independent. A reader must not need to understand one case as + an intermediate step or hidden prerequisite of another. + +Each case must still satisfy all visual and diff requirements from category A. + +#### C. Strong-inference rule: never add a Canvas example Do not create a Before/After Canvas example for strong inference, assumption inference, contradiction probing, or branch-comparison rules. A static pair hides the reasoning chain and can misrepresent why the conclusion is valid. -Document the rule in MDX and report: +This restriction overrides every other Canvas guideline. Even if a +strong-inference rule appears easy to illustrate or can be divided into several +small boards, leave its Canvas example empty. Document the rule in MDX and +report: ```text Canvas example deferred: this is a strong-inference rule whose explanation requires branch or contradiction history. ``` -#### C. Complex or multi-case rule: defer when unclear +#### D. Complex rule: defer when unclear + +Defer the Canvas example when cases require intermediate history, branch +reasoning, hidden candidate state, more than four representative cases, or +cannot be understood as independent Before/After states. Also defer when the +cases are too coupled, the necessary split is unclear, or implementing the +group confidently would be disproportionately difficult. + +Do not invent an oversimplified example. Leave the rule's example registration +empty and make the missing Canvas obvious: -Defer the Canvas example when the rule has several materially different trigger -paths, produces different conclusion types, or needs a sequence of intermediate -states to be understood. +- Add a concise `TODO(rule-doc-canvas)` comment beside the relevant location in + `src/features/docs/ruleExamples.ts`. Include the puzzle ID, rule ID, and exact + blocker so a future author can find and resume the work. +- Always state the deferral and exact blocker in the delivery report under + `Needs author input`. -Do not invent an oversimplified example. Report: +Use this wording: ```text -Canvas example needs author input: the rule covers . -Please choose the representative case or approve a multi-stage explanation. +Canvas example deferred: needs , +but