diff --git a/src/@types/types.d.ts b/src/@types/types.d.ts index 663d74d..2d621c4 100644 --- a/src/@types/types.d.ts +++ b/src/@types/types.d.ts @@ -188,6 +188,10 @@ export interface ActiveSession { skillState: SkillFireState; lastUsedAt: number; lastUrl?: string; + lastElements?: Map; + lastSnapshotElements?: SnapshotElement[]; + // Active tab the cache above belongs to; a change means re-baseline (new tab). + lastActiveTargetId?: string | null; } /* ------------------------------------------------------------------ */ diff --git a/src/lib/agent-format.ts b/src/lib/agent-format.ts index 2e7229d..2157b50 100644 --- a/src/lib/agent-format.ts +++ b/src/lib/agent-format.ts @@ -176,24 +176,8 @@ export const formatSnapshot = (snapshot: SnapshotResult): string => { } } - // Label cross-origin iframes (frame#1, …) and list them so the agent knows - // which elements live in a frame and that their deep-ref selectors pierce it. - const frameLabels = new Map(); - if (snapshot.frames?.length) { - snapshot.frames.forEach((frame, i) => - frameLabels.set(frame.frameId, `frame#${i + 1}`), - ); - lines.push(`Frames (${snapshot.frames.length} iframes):`); - for (const frame of snapshot.frames) { - const origin = frame.crossOrigin ? 'cross-origin' : 'same-origin'; - lines.push( - ` ${frameLabels.get(frame.frameId)} ${frame.url} (${origin})`, - ); - } - lines.push( - 'Elements tagged [frame#N] live in that iframe; their deep-ref selectors pierce it — pass as-is to click/type/hover.', - ); - } + const frameLabels = frameLabelsOf(snapshot); + lines.push(...frameLegend(snapshot, frameLabels)); lines.push(''); @@ -204,3 +188,178 @@ export const formatSnapshot = (snapshot: SnapshotResult): string => { lines.push('--- END SNAPSHOT ---'); return lines.join('\n'); }; + +const SIGNATURE_FIELDS: Array = [ + 'role', + 'name', + 'text', + 'value', + 'type', + 'placeholder', + 'href', + 'disabled', + 'checked', + 'focused', + 'required', + 'ariaLabel', + 'tag', + 'frameId', +]; + +const elementSignature = (el: SnapshotElement): string => + JSON.stringify(SIGNATURE_FIELDS.map((f) => el[f])); + +// Framework-generated ids (Radix/useId/Headless/MUI) churn every render, so a +// selector or id built from one is not a stable diff key — excluded below. +const FRAMEWORK_ID = /(radix-|headlessui-|mui-[a-z]|«r|:r[0-9a-z]|_r_)/i; + +// Stable cross-snapshot identity: clean id → clean selector → semantic composite +// (never the positional ref). Degrades gracefully when selectors churn on SPAs. +export const elementKey = (el: SnapshotElement): string => { + if (el.id && !FRAMEWORK_ID.test(el.id)) return `#${el.id}`; + if (el.selector && !FRAMEWORK_ID.test(el.selector)) + return `sel:${el.selector}`; + // name often holds the volatile value (a stat number) and would churn the key; + // aria-label is the stable descriptor, so the value shows as `changed` instead. + const label = el.ariaLabel || el.name; + return `sem:${el.tag}|${el.role}|${label}|${el.href ?? ''}`; +}; + +// Occurrence-suffix the identity key so duplicate-keyed elements (two unlabeled +// buttons, repeated links) are each kept instead of collapsing to last-wins. +const identityEntries = ( + elements: SnapshotElement[], +): Array<[string, SnapshotElement]> => { + const seen = new Map(); + return elements.map((el) => { + const base = elementKey(el); + const n = seen.get(base) ?? 0; + seen.set(base, n + 1); + return [`${base}${n}`, el]; + }); +}; + +export const indexByIdentity = ( + snapshot: SnapshotResult, +): Map => new Map(identityEntries(snapshot.elements)); + +const frameLabelsOf = (snapshot: SnapshotResult): Map => { + const labels = new Map(); + snapshot.frames?.forEach((f, i) => labels.set(f.frameId, `frame#${i + 1}`)); + return labels; +}; + +// Frame legend + deep-ref guidance, shared by full and diff snapshots so framed +// elements never arrive without their [frame#N] → url mapping. +const frameLegend = ( + snapshot: SnapshotResult, + frameLabels: Map, +): string[] => { + if (!snapshot.frames?.length) return []; + const lines = [`Frames (${snapshot.frames.length} iframes):`]; + for (const frame of snapshot.frames) { + const origin = frame.crossOrigin ? 'cross-origin' : 'same-origin'; + lines.push(` ${frameLabels.get(frame.frameId)} ${frame.url} (${origin})`); + } + lines.push( + 'Elements tagged [frame#N] live in that iframe; their deep-ref selectors pierce it — pass as-is to click/type/hover.', + ); + return lines; +}; + +// Shared header/footer for both diffs so the rendered shape stays one thing. +const assembleDiff = ( + snapshot: SnapshotResult, + countsLine: string, + changeLines: string[], + unchanged: number, +): string => { + const lines = [ + '--- PAGE SNAPSHOT (diff vs previous; unchanged elements omitted) ---', + `${snapshot.url} | ${snapshot.title}`, + countsLine, + ...(snapshot.detectedChallenges ?? []).map( + (t) => `! Detected challenge: ${t}`, + ), + ...frameLegend(snapshot, frameLabelsOf(snapshot)), + ]; + if (changeLines.length === 0) { + lines.push('', 'No changes since last snapshot.'); + } else { + lines.push('', ...changeLines); + if (unchanged > 0) + lines.push( + `… ${unchanged} unchanged elements omitted (still valid from the previous snapshot).`, + ); + } + lines.push('--- END SNAPSHOT ---'); + return lines.join('\n'); +}; + +// Delta vs the previous snapshot: +new / ~changed / -removed, unchanged collapsed +// to a count. Assumes the prior snapshot is still in context (caller sends full otherwise). +export const formatSnapshotDiff = ( + snapshot: SnapshotResult, + prev: Map, +): string => { + const frameLabels = frameLabelsOf(snapshot); + const seen = new Set(); + const added: SnapshotElement[] = []; + const changed: Array<{ before: SnapshotElement; after: SnapshotElement }> = + []; + for (const [key, el] of identityEntries(snapshot.elements)) { + seen.add(key); + const before = prev.get(key); + if (!before) added.push(el); + else if (elementSignature(before) !== elementSignature(el)) + changed.push({ before, after: el }); + } + // Removed elements print by selector (the actionable handle), not the key. + const removed = [...prev.entries()] + .filter(([key]) => !seen.has(key)) + .map(([, el]) => el.selector); + const unchanged = snapshot.elements.length - added.length - changed.length; + + const changeLines = [ + ...added.map((el) => `+ ${formatElement(el, frameLabels)}`), + ...changed.map(({ before, after }) => + formatChange(before, after, frameLabels), + ), + ...removed.map((sel) => `- ref=${sel} (removed)`), + ]; + const counts = `Changes: ${added.length} new, ${changed.length} changed, ${removed.length} removed, ${unchanged} unchanged (${snapshot.elements.length} total)`; + return assembleDiff(snapshot, counts, changeLines, unchanged); +}; + +// `~` line with the field-level old→new reason, so a detected change is always +// visible. Reads SIGNATURE_FIELDS — the same fields change detection keys on. +const formatChange = ( + before: SnapshotElement, + after: SnapshotElement, + frameLabels?: Map, +): string => { + const deltas = SIGNATURE_FIELDS.filter((f) => before[f] !== after[f]).map( + (f) => + `${f}: ${JSON.stringify(before[f] ?? '')}→${JSON.stringify(after[f] ?? '')}`, + ); + const suffix = deltas.length ? ` (${deltas.join(', ')})` : ''; + return `~ ${formatElement(after, frameLabels)}${suffix}`; +}; + +// Diff equal-length snapshots by DOM position — order survives an in-place SPA +// re-render when ids churn, surfacing moved values. Caller gates on equal length + shortest-wins. +export const formatSnapshotDiffPositional = ( + prev: SnapshotElement[], + snapshot: SnapshotResult, +): string => { + const frameLabels = frameLabelsOf(snapshot); + const changeLines: string[] = []; + snapshot.elements.forEach((after, i) => { + const before = prev[i]; + if (before && elementSignature(before) !== elementSignature(after)) + changeLines.push(formatChange(before, after, frameLabels)); + }); + const unchanged = snapshot.elements.length - changeLines.length; + const counts = `Changes: ${changeLines.length} changed, ${unchanged} unchanged (${snapshot.elements.length} total)`; + return assembleDiff(snapshot, counts, changeLines, unchanged); +}; diff --git a/src/skills/system-prompt.ts b/src/skills/system-prompt.ts index 8e69007..35f5f46 100644 --- a/src/skills/system-prompt.ts +++ b/src/skills/system-prompt.ts @@ -46,6 +46,7 @@ Load manually via **browserless_skill** if suspected but not injected: - Snapshot VALID after: type, hover, scroll, evaluate - Expect new content? → re-snapshot - Element roles in snapshot (link, button, textbox, combobox, checkbox, heading) tell you what each does +- Snapshots after the first return a **diff** vs. your previous snapshot: only \`+\` new / \`~\` changed / \`-\` removed elements, plus a count of unchanged ones omitted. Unchanged elements stay valid — keep using their refs from the earlier snapshot. If that earlier snapshot is no longer in your context (summarized/trimmed away), request \`snapshot { full: true }\` to get the complete element list again. ## Selectors - Use **ref=** (CSS) or **deep-ref=** (starts \`< \`) exactly as shown in snapshot diff --git a/src/tools/agent.ts b/src/tools/agent.ts index 42a42c4..5888bb7 100644 --- a/src/tools/agent.ts +++ b/src/tools/agent.ts @@ -44,6 +44,9 @@ import { formatConnectError, formatErrorMessage, formatSnapshot, + formatSnapshotDiff, + formatSnapshotDiffPositional, + indexByIdentity, } from '../lib/agent-format.js'; // export schemas, system prompt, and formatters @@ -707,14 +710,65 @@ export function registerAgentTools( ); const noticeBlock = notice ? `${notice}\n\n` : ''; if (lastSnapshot.url) agentSession.lastUrl = lastSnapshot.url; + // A peek (targetId ≠ active tab) reads a different tab; it must not + // diff against or overwrite the active tab's cache. + const targetId = ( + lastCmd?.params as { targetId?: string } | undefined + )?.targetId; + const peeking = + !!targetId && targetId !== lastSnapshot.activeTargetId; + // Active tab changed (switch/create/close) → prior cache is a different + // tab; re-baseline with a full snapshot. + const switchedTab = + lastSnapshot.activeTargetId != null && + agentSession.lastActiveTargetId != null && + lastSnapshot.activeTargetId !== agentSession.lastActiveTargetId; + // Send full on first snapshot / cross-origin nav / mid-hydration + // baseline / explicit full / peek / tab switch; else diff (shortest-wins). + const prevElements = agentSession.lastElements; + const prevArr = agentSession.lastSnapshotElements; + const hydrating = + !!prevElements && + lastSnapshot.elements.length > 0 && + prevElements.size < lastSnapshot.elements.length * 0.25; + const forceFull = + (lastCmd?.params as { full?: boolean } | undefined)?.full === true; + const full = formatSnapshot(lastSnapshot); + let snapshotText = full; + if ( + prevElements && + !notice && + !hydrating && + !forceFull && + !peeking && + !switchedTab + ) { + // Shortest of full / identity-diff / positional-diff wins, so a bad + // positional guess (only tried at equal count) can never cost more. + const candidates = [ + full, + formatSnapshotDiff(lastSnapshot, prevElements), + ]; + if (prevArr && prevArr.length === lastSnapshot.elements.length) { + candidates.push( + formatSnapshotDiffPositional(prevArr, lastSnapshot), + ); + } + snapshotText = candidates.reduce((a, b) => + b.length < a.length ? b : a, + ); + } + // Never let a peek's other-tab elements become the active baseline. + if (!peeking) { + agentSession.lastElements = indexByIdentity(lastSnapshot); + agentSession.lastSnapshotElements = lastSnapshot.elements; + agentSession.lastActiveTargetId = lastSnapshot.activeTargetId; + } baseContent = [ { type: 'text' as const, text: appendSkills( - batchPrefix + - noticeBlock + - formatSnapshot(lastSnapshot) + - closedSuffix, + batchPrefix + noticeBlock + snapshotText + closedSuffix, triggered, ), }, diff --git a/src/tools/schemas.ts b/src/tools/schemas.ts index 75bfc3c..bb7de62 100644 --- a/src/tools/schemas.ts +++ b/src/tools/schemas.ts @@ -60,6 +60,15 @@ const SnapshotCommandSchema = z.object({ 'Optional tab targetId to peek at without switching the active tab. ' + 'Obtain via getTabs or a prior snapshot response. Omit to snapshot the active tab.', ), + full: z + .boolean() + .optional() + .describe( + 'Force a complete snapshot instead of a diff. Snapshots normally return ' + + 'only what changed since your previous one; set full:true when you no ' + + 'longer have that previous snapshot in context (e.g. it was summarized ' + + 'away) and need the entire element list again.', + ), }) .optional() .default({}), diff --git a/test/lib/agent-format.diff.spec.ts b/test/lib/agent-format.diff.spec.ts new file mode 100644 index 0000000..55fa483 --- /dev/null +++ b/test/lib/agent-format.diff.spec.ts @@ -0,0 +1,165 @@ +import { expect } from 'chai'; + +import { + formatSnapshotDiff, + formatSnapshotDiffPositional, + indexByIdentity, +} from '../../src/lib/agent-format.js'; +import type { + SnapshotElement, + SnapshotResult, +} from '../../src/@types/types.js'; + +const el = (over: Partial): SnapshotElement => ({ + ref: 0, + role: 'button', + name: '', + selector: '', + tag: 'button', + ...over, +}); + +const snap = ( + elements: SnapshotElement[], + extra: Partial = {}, +): SnapshotResult => ({ + url: 'https://example.com', + title: 'Example', + elements, + time: 1, + ...extra, +}); + +describe('formatSnapshotDiff', () => { + it('omits unchanged elements and reports counts', () => { + const prev = indexByIdentity( + snap([ + el({ ref: 1, name: 'Home', selector: 'a#home' }), + el({ ref: 2, name: 'Login', selector: 'a#login' }), + ]), + ); + // refs renumber, one unchanged, one new; identity is the selector. + const out = formatSnapshotDiff( + snap([ + el({ ref: 5, name: 'Home', selector: 'a#home' }), + el({ ref: 6, name: 'Sign up', selector: 'a#signup' }), + ]), + prev, + ); + expect(out).to.match(/1 new, 0 changed, 1 removed, 1 unchanged/); + expect(out).to.include('+ [6] button button "Sign up" ref=a#signup'); + expect(out).to.include('- ref=a#login (removed)'); + expect(out).to.include('1 unchanged elements omitted'); + // Unchanged element must NOT be re-listed — that's the whole point. + expect(out).to.not.include('"Home"'); + }); + + it('preserves duplicate identity keys (removal of one is not hidden)', () => { + // Two elements collapse to the same semantic key; removing one must show. + const dup = { role: 'button', tag: 'button', name: 'Go', selector: '' }; + const prev = indexByIdentity(snap([el(dup), el(dup)])); + const out = formatSnapshotDiff(snap([el(dup)]), prev); + expect(out).to.match(/0 new, 0 changed, 1 removed/); + }); + + it('includes the frame legend in a diff when frames are present', () => { + const frames = [ + { frameId: 'f1', url: 'https://ad.example', crossOrigin: true }, + ]; + const prev = indexByIdentity(snap([el({ selector: '#a', name: 'A' })])); + const out = formatSnapshotDiff( + snap( + [ + el({ selector: '#a', name: 'A' }), + el({ selector: '#b', name: 'B', frameId: 'f1' }), + ], + { frames }, + ), + prev, + ); + expect(out).to.include('Frames (1 iframes):'); + expect(out).to.include('frame#1 https://ad.example (cross-origin)'); + }); + + it('flags an element whose state changed, ignoring ref renumbering', () => { + const prev = indexByIdentity( + snap([el({ ref: 1, selector: '#agree', name: 'Agree', checked: false })]), + ); + const out = formatSnapshotDiff( + snap([el({ ref: 9, selector: '#agree', name: 'Agree', checked: true })]), + prev, + ); + expect(out).to.match(/0 new, 1 changed, 0 removed, 0 unchanged/); + expect(out).to.include('~ [9]'); + expect(out).to.include('(checked)'); + }); + + it('shows a reason for changes formatElement does not render (e.g. placeholder)', () => { + const prev = indexByIdentity( + snap([el({ selector: '#q', name: 'Search', placeholder: 'old' })]), + ); + const out = formatSnapshotDiff( + snap([el({ selector: '#q', name: 'Search', placeholder: 'new' })]), + prev, + ); + expect(out).to.match(/1 changed/); + expect(out).to.include('placeholder: "old"→"new"'); + }); + + it('reports no changes when nothing moved', () => { + const prev = indexByIdentity( + snap([el({ ref: 1, selector: '#x', name: 'X' })]), + ); + const out = formatSnapshotDiff( + snap([el({ ref: 2, selector: '#x', name: 'X' })]), + prev, + ); + expect(out).to.include('No changes since last snapshot.'); + }); + + it('positional diff surfaces value changes when ids churn but order holds', () => { + // In-place SPA re-render: stat tiles keep DOM order, ids churn, numbers move. + const prev = [ + el({ ref: 1, role: 'text', name: '0', selector: 'div#radix-«r1»' }), + el({ ref: 2, role: 'text', name: '5', selector: 'div#radix-«r2»' }), + ]; + const out = formatSnapshotDiffPositional( + prev, + snap([ + el({ ref: 8, role: 'text', name: '1,158', selector: 'div#radix-«r7»' }), + el({ ref: 9, role: 'text', name: '5', selector: 'div#radix-«r8»' }), + ]), + ); + expect(out).to.match(/1 changed, 1 unchanged/); + expect(out).to.include('name: "0"→"1,158"'); + // Unchanged slot (value 5) must not be listed. + expect(out).to.not.include('"5"'); + }); + + it('treats a churned framework-id selector as the same element via semantic key', () => { + // Radix/useId selectors regenerate per render on SPAs — semantic identity + // (role+name+tag) must still match so it is not counted as new+removed. + const prev = indexByIdentity( + snap([ + el({ + ref: 1, + role: 'menuitem', + name: 'Profile', + selector: 'div#radix-«r1»', + }), + ]), + ); + const out = formatSnapshotDiff( + snap([ + el({ + ref: 7, + role: 'menuitem', + name: 'Profile', + selector: 'div#radix-«r9»', + }), + ]), + prev, + ); + expect(out).to.include('No changes since last snapshot.'); + }); +});