Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions src/daemon/handlers/__tests__/interaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -935,6 +935,67 @@ test('press coordinates appends touch-visualization events while recording', asy
}
});

test('press coordinates on iOS recording captures a non-compact snapshot for the touch reference frame', async () => {
const sessionStore = makeSessionStore();
const sessionName = 'ios-direct-press-frame';
const session = makeSession(sessionName);
session.snapshot = undefined;
session.recording = {
platform: 'ios',
outPath: '/tmp/demo.mp4',
startedAt: Date.now() - 1_000,
showTouches: true,
gestureEvents: [],
child: { kill: () => {} } as any,
wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }),
};
sessionStore.set(sessionName, session);

mockDispatch.mockResolvedValue({ x: 220, y: 600 });
// Regression: a compact snapshot has no Application/Window node, so viewport inference would
// return a leaf-element bounding box and the recording overlay would misplace tap markers.
mockCaptureSnapshotForSession.mockResolvedValueOnce({
nodes: attachRefs([
{
index: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 440, height: 956 },
},
{
index: 1,
type: 'XCUIElementTypeCell',
rect: { x: 16, y: 156, width: 370, height: 52 },
hittable: true,
},
]),
createdAt: Date.now(),
backend: 'xctest',
});

const response = await handleInteractionCommands({
req: {
token: 't',
session: sessionName,
command: 'press',
positionals: ['220', '600'],
flags: {},
},
sessionName,
sessionStore,
contextFromFlags,
});

expect(response?.ok).toBe(true);
expect(mockCaptureSnapshotForSession.mock.calls[0]?.[4]).toEqual({
interactiveOnly: true,
compact: false,
});
const event = sessionStore.get(sessionName)?.recording?.gestureEvents[0];
expect(event?.kind).toBe('tap');
expect(event?.referenceWidth).toBe(440);
expect(event?.referenceHeight).toBe(956);
});

test('press coordinates on Android recording uses physical screen size when no snapshot exists', async () => {
const sessionStore = makeSessionStore();
const sessionName = 'android-direct-press-frame';
Expand Down
6 changes: 3 additions & 3 deletions src/daemon/handlers/interaction-snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,20 @@ export type CaptureSnapshotForSession = (
flags: CommandFlags | undefined,
sessionStore: SessionStore,
contextFromFlags: ContextFromFlags,
options: { interactiveOnly: boolean; androidFreshnessMode?: 'ref-refresh' },
options: { interactiveOnly: boolean; compact?: boolean; androidFreshnessMode?: 'ref-refresh' },
) => Promise<SnapshotState>;

export async function captureSnapshotForSession(
session: SessionState,
flags: CommandFlags | undefined,
sessionStore: SessionStore,
contextFromFlags: ContextFromFlags,
options: { interactiveOnly: boolean; androidFreshnessMode?: 'ref-refresh' },
options: { interactiveOnly: boolean; compact?: boolean; androidFreshnessMode?: 'ref-refresh' },
): Promise<SnapshotState> {
const effectiveFlags = {
...(flags ?? {}),
snapshotInteractiveOnly: options.interactiveOnly,
snapshotCompact: options.interactiveOnly,
snapshotCompact: options.compact ?? options.interactiveOnly,
};
const dispatchContext = contextFromFlags(
effectiveFlags,
Expand Down
3 changes: 3 additions & 0 deletions src/daemon/handlers/interaction-touch-reference-frame.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,11 @@ async function resolveDirectTouchReferenceFrame(params: {
return undefined;
}

// Compact snapshots prune Application/Window containers, leaving viewport inference to fall
// back to a bounding box of leaf elements — a garbage reference frame for screen-point touches.
const snapshot = await captureSnapshotForSession(session, flags, sessionStore, contextFromFlags, {
interactiveOnly: true,
compact: false,
});
const referenceFrame = getSnapshotReferenceFrame(snapshot);
if (referenceFrame && session.recording) {
Expand Down
Loading