From 5d9ff0ee025530897e16892fc4ba3c6753a548e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 25 Jun 2026 18:52:47 +0200 Subject: [PATCH 01/18] feat: add web audio probe --- examples/test-app/app/(tabs)/_layout.tsx | 4 + examples/test-app/app/(tabs)/audio.tsx | 10 + examples/test-app/src/screens/AudioScreen.tsx | 184 +++++++++++++ src/batch-policy.ts | 1 + src/client-types.ts | 9 + src/client.ts | 1 + src/command-catalog.ts | 1 + src/commands/observability/index.test.ts | 29 ++ src/commands/observability/index.ts | 84 +++++- src/commands/observability/output.ts | 52 ++++ src/core/capabilities.ts | 6 + .../__tests__/daemon-command-registry.test.ts | 1 + src/daemon/daemon-command-registry.ts | 1 + src/daemon/handlers/session-observability.ts | 112 ++++++++ .../web/agent-browser-provider.test.ts | 63 +++++ src/platforms/web/agent-browser-provider.ts | 256 +++++++++++++++++- src/platforms/web/provider.ts | 31 +++ src/utils/cli-help.ts | 6 +- .../provider-scenarios/web-provider.test.ts | 31 +++ 19 files changed, 877 insertions(+), 5 deletions(-) create mode 100644 examples/test-app/app/(tabs)/audio.tsx create mode 100644 examples/test-app/src/screens/AudioScreen.tsx diff --git a/examples/test-app/app/(tabs)/_layout.tsx b/examples/test-app/app/(tabs)/_layout.tsx index 9b3067361..06bbbe54d 100644 --- a/examples/test-app/app/(tabs)/_layout.tsx +++ b/examples/test-app/app/(tabs)/_layout.tsx @@ -33,6 +33,10 @@ export default function TabsLayout() { Form + + + Audio + Settings diff --git a/examples/test-app/app/(tabs)/audio.tsx b/examples/test-app/app/(tabs)/audio.tsx new file mode 100644 index 000000000..a3df116c7 --- /dev/null +++ b/examples/test-app/app/(tabs)/audio.tsx @@ -0,0 +1,10 @@ +import { AppFrame } from '../../src/components'; +import { AudioScreen } from '../../src/screens/AudioScreen'; + +export default function AudioRoute() { + return ( + + + + ); +} diff --git a/examples/test-app/src/screens/AudioScreen.tsx b/examples/test-app/src/screens/AudioScreen.tsx new file mode 100644 index 000000000..1763a1340 --- /dev/null +++ b/examples/test-app/src/screens/AudioScreen.tsx @@ -0,0 +1,184 @@ +import { createElement, useEffect, useRef, useState } from 'react'; +import { Platform, ScrollView, StyleSheet, Text, View } from 'react-native'; + +import { ActionButton, InlineBadge, ScreenTitle, SectionCard } from '../components'; +import { useAppColors, type AppColors } from '../theme'; + +type BrowserAudioElement = { + currentTime: number; + pause: () => void; + play: () => Promise; +}; + +export function AudioScreen() { + const colors = useAppColors(); + const styles = createStyles(colors); + const audioRef = useRef(null); + const [audioSrc, setAudioSrc] = useState(); + const [playbackState, setPlaybackState] = useState<'ready' | 'playing' | 'paused' | 'ended'>( + 'ready', + ); + + useEffect(() => { + if (Platform.OS !== 'web') return; + const url = createClassicLoopWavUrl(); + setAudioSrc(url); + return () => { + URL.revokeObjectURL(url); + }; + }, []); + + function playSample() { + const audio = audioRef.current; + if (!audio) return; + audio.currentTime = 0; + void audio.play().then(() => setPlaybackState('playing')); + } + + function pauseSample() { + const audio = audioRef.current; + if (!audio) return; + audio.pause(); + setPlaybackState('paused'); + } + + return ( + + + + + {Platform.OS === 'web' && audioSrc ? ( + + {createElement('audio', { + 'aria-label': 'Classic sample audio', + controls: true, + loop: false, + onEnded: () => setPlaybackState('ended'), + onPause: () => { + if (playbackState === 'playing') setPlaybackState('paused'); + }, + onPlay: () => setPlaybackState('playing'), + ref: (node: BrowserAudioElement | null) => { + audioRef.current = node; + }, + src: audioSrc, + style: { width: '100%' }, + 'data-testid': 'classic-audio', + })} + + + + + {playbackState} + + + + + + + + + ) : ( + + Browser audio sample + + )} + + + ); +} + +function createClassicLoopWavUrl(): string { + const sampleRate = 11_025; + const durationSeconds = 8; + const sampleCount = sampleRate * durationSeconds; + const channels = 1; + const bytesPerSample = 2; + const dataBytes = sampleCount * channels * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataBytes); + const view = new DataView(buffer); + + writeAscii(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataBytes, true); + writeAscii(view, 8, 'WAVE'); + writeAscii(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, channels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * channels * bytesPerSample, true); + view.setUint16(32, channels * bytesPerSample, true); + view.setUint16(34, bytesPerSample * 8, true); + writeAscii(view, 36, 'data'); + view.setUint32(40, dataBytes, true); + + const melody = [196, 247, 294, 330, 392, 330, 294, 247, 220, 262, 330, 392, 494, 392, 330, 262]; + for (let index = 0; index < sampleCount; index += 1) { + const t = index / sampleRate; + const step = Math.floor(t * 4) % melody.length; + const beatPhase = (t * 4) % 1; + const envelope = Math.max(0.18, 1 - beatPhase * 0.72); + const bass = squareWave(98, t) * 0.18; + const lead = squareWave(melody[step] ?? 220, t) * 0.42 * envelope; + const hat = ((index * 17) % 31 < 2 ? 0.12 : 0) * (beatPhase < 0.08 ? 1 : 0); + const sample = Math.max(-0.82, Math.min(0.82, lead + bass + hat)); + view.setInt16(44 + index * 2, Math.round(sample * 32767), true); + } + + return URL.createObjectURL(new Blob([buffer], { type: 'audio/wav' })); +} + +function squareWave(frequency: number, t: number): number { + return Math.sin(Math.PI * 2 * frequency * t) >= 0 ? 1 : -1; +} + +function writeAscii(view: DataView, offset: number, value: string): void { + for (let index = 0; index < value.length; index += 1) { + view.setUint8(offset + index, value.charCodeAt(index)); + } +} + +function createStyles(colors: AppColors) { + return StyleSheet.create({ + content: { + paddingBottom: 28, + }, + player: { + gap: 14, + }, + statusRow: { + alignItems: 'center', + flexDirection: 'row', + gap: 10, + }, + statusText: { + color: colors.text, + fontSize: 15, + fontWeight: '600', + textTransform: 'capitalize', + }, + actionRow: { + gap: 10, + }, + nativeFallback: { + backgroundColor: colors.cardStrong, + borderColor: colors.line, + borderRadius: 4, + borderWidth: StyleSheet.hairlineWidth, + padding: 14, + }, + }); +} diff --git a/src/batch-policy.ts b/src/batch-policy.ts index 3c690c476..8dd5f4a7d 100644 --- a/src/batch-policy.ts +++ b/src/batch-policy.ts @@ -31,6 +31,7 @@ export const STRUCTURED_BATCH_COMMAND_NAMES = [ PUBLIC_COMMANDS.gesture, PUBLIC_COMMANDS.is, PUBLIC_COMMANDS.find, + PUBLIC_COMMANDS.audio, PUBLIC_COMMANDS.perf, PUBLIC_COMMANDS.logs, PUBLIC_COMMANDS.network, diff --git a/src/client-types.ts b/src/client-types.ts index ef62703eb..800e59563 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -787,6 +787,14 @@ export type NetworkOptions = AgentDeviceRequestOverrides & { include?: NetworkIncludeMode; }; +export type AudioOptions = AgentDeviceRequestOverrides & { + action?: 'probe'; + probeAction?: 'start' | 'status' | 'stop'; + durationMs?: number; + bucketMs?: number; + source?: 'media-elements'; +}; + export type RecordOptions = AgentDeviceRequestOverrides & { action: 'start' | 'stop'; path?: string; @@ -1013,6 +1021,7 @@ export type AgentDeviceClient = { perf: (options?: PerfOptions) => Promise; logs: (options?: LogsOptions) => Promise; network: (options?: NetworkOptions) => Promise; + audio: (options?: AudioOptions) => Promise; }; debug: { symbols: (options: DebugSymbolsOptions) => Promise; diff --git a/src/client.ts b/src/client.ts index 9f5d6d6c3..7f0dae4ba 100644 --- a/src/client.ts +++ b/src/client.ts @@ -303,6 +303,7 @@ export function createAgentDeviceClient( perf: async (options = {}) => await executeCommand('perf', options), logs: async (options = {}) => await executeCommand('logs', options), network: async (options = {}) => await executeCommand('network', options), + audio: async (options = {}) => await executeCommand('audio', options), }, debug: { symbols: async (options) => diff --git a/src/command-catalog.ts b/src/command-catalog.ts index 8d5ccf331..b51c19f6c 100644 --- a/src/command-catalog.ts +++ b/src/command-catalog.ts @@ -1,5 +1,6 @@ export const PUBLIC_COMMANDS = { alert: 'alert', + audio: 'audio', appState: 'appstate', appSwitcher: 'app-switcher', apps: 'apps', diff --git a/src/commands/observability/index.test.ts b/src/commands/observability/index.test.ts index b23682b5d..762bf6f6b 100644 --- a/src/commands/observability/index.test.ts +++ b/src/commands/observability/index.test.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '../../utils/cli-flags.ts'; import { + audioCliReader, + audioCommandDefinition, + audioCommandMetadata, + audioDaemonWriter, logsCliReader, logsCommandDefinition, logsCommandMetadata, @@ -24,12 +28,35 @@ function expectInvalidArgs(fn: () => unknown, messageFragment: string) { describe('observability command interface', () => { test('owns logs and network public metadata', () => { + expect(audioCommandMetadata.name).toBe('audio'); + expect(audioCommandDefinition.name).toBe('audio'); expect(logsCommandMetadata.name).toBe('logs'); expect(logsCommandDefinition.name).toBe('logs'); expect(networkCommandMetadata.name).toBe('network'); expect(networkCommandDefinition.name).toBe('network'); }); + test('reads audio probe timing as compact daemon positionals', () => { + expect(audioCliReader(['probe', 'start', '7.5', '500'], NO_FLAGS)).toEqual({ + action: 'probe', + probeAction: 'start', + durationMs: 7500, + bucketMs: 500, + source: 'media-elements', + }); + expect( + audioDaemonWriter({ + action: 'probe', + probeAction: 'start', + durationMs: 7500, + bucketMs: 500, + }), + ).toMatchObject({ + command: 'audio', + positionals: ['probe', 'start', '7500', '500'], + }); + }); + test('reads logs action and message', () => { expect(logsCliReader(['mark', 'checkout', 'started'], NO_FLAGS)).toEqual({ action: 'mark', @@ -66,6 +93,8 @@ describe('observability command interface', () => { test('rejects invalid observability positionals', () => { expectInvalidArgs(() => logsCliReader(['explode'], NO_FLAGS), 'logs requires'); expectInvalidArgs(() => networkCliReader(['explode'], NO_FLAGS), 'network requires'); + expectInvalidArgs(() => audioCliReader(['explode'], NO_FLAGS), 'audio requires probe'); + expectInvalidArgs(() => audioCliReader(['probe', 'explode'], NO_FLAGS), 'audio probe requires'); expectInvalidArgs( () => networkCliReader(['dump', '25', 'explode'], NO_FLAGS), 'network include', diff --git a/src/commands/observability/index.ts b/src/commands/observability/index.ts index bfb7b865b..7fe36214a 100644 --- a/src/commands/observability/index.ts +++ b/src/commands/observability/index.ts @@ -1,4 +1,4 @@ -import type { LogsOptions, NetworkOptions } from '../../client-types.ts'; +import type { AudioOptions, LogsOptions, NetworkOptions } from '../../client-types.ts'; import { NETWORK_INCLUDE_MODES, type NetworkIncludeMode } from '../../contracts.ts'; import { AppError } from '../../utils/errors.ts'; import { parseStringMember } from '../../utils/string-enum.ts'; @@ -21,10 +21,14 @@ import { observabilityCliOutputFormatters } from './output.ts'; const LOGS_COMMAND_NAME = 'logs'; const NETWORK_COMMAND_NAME = 'network'; +const AUDIO_COMMAND_NAME = 'audio'; const NETWORK_ACTION_VALUES = ['dump', 'log'] as const; +const AUDIO_ACTION_VALUES = ['probe'] as const; +const AUDIO_PROBE_ACTION_VALUES = ['start', 'status', 'stop'] as const; const logsCommandDescription = 'Manage session app logs.'; const networkCommandDescription = 'Show recent HTTP traffic.'; +const audioCommandDescription = 'Probe browser page audio levels.'; export const logsCommandMetadata = defineFieldCommandMetadata( LOGS_COMMAND_NAME, @@ -46,6 +50,18 @@ export const networkCommandMetadata = defineFieldCommandMetadata( }, ); +export const audioCommandMetadata = defineFieldCommandMetadata( + AUDIO_COMMAND_NAME, + audioCommandDescription, + { + action: enumField(AUDIO_ACTION_VALUES), + probeAction: enumField(AUDIO_PROBE_ACTION_VALUES), + durationMs: integerField('Probe duration in milliseconds.'), + bucketMs: integerField('Audio level bucket size in milliseconds.'), + source: enumField(['media-elements'] as const), + }, +); + export const logsCommandDefinition = defineExecutableCommand(logsCommandMetadata, (client, input) => client.observability.logs(input), ); @@ -55,6 +71,11 @@ export const networkCommandDefinition = defineExecutableCommand( (client, input) => client.observability.network(input), ); +export const audioCommandDefinition = defineExecutableCommand( + audioCommandMetadata, + (client, input) => client.observability.audio(input), +); + const logsCliSchema = { usageOverride: 'logs path | logs start | logs stop | logs clear [--restart] | logs doctor | logs mark [message...]', @@ -76,6 +97,15 @@ const networkCliSchema = { allowedFlags: ['networkInclude'], } as const satisfies CommandSchemaOverride; +const audioCliSchema = { + usageOverride: + 'audio probe start [durationSeconds] [bucketMs] | audio probe status | audio probe stop', + listUsageOverride: 'audio', + helpDescription: 'Probe browser page audio levels as compact dBFS buckets', + summary: 'Probe browser page audio levels', + positionalArgs: ['probe', 'start|status|stop', 'durationSeconds?', 'bucketMs?'], +} as const satisfies CommandSchemaOverride; + export const logsCliReader: CliReader = (positionals, flags) => ({ ...commonInputFromFlags(flags), action: readLogsAction(positionals[0]), @@ -90,6 +120,15 @@ export const networkCliReader: CliReader = (positionals, flags) => ({ include: flags.networkInclude ?? readNetworkInclude(positionals[2]), }); +export const audioCliReader: CliReader = (positionals, flags) => ({ + ...commonInputFromFlags(flags), + action: readAudioAction(positionals[0]), + probeAction: readAudioProbeAction(positionals[1]), + durationMs: readAudioDurationMs(positionals[2]), + bucketMs: optionalCliNumber(positionals[3]), + source: 'media-elements', +}); + export const logsDaemonWriter: DaemonWriter = direct(LOGS_COMMAND_NAME, (input) => logsPositionals(input as LogsOptions), ); @@ -100,6 +139,9 @@ export const networkDaemonWriter: DaemonWriter = (input) => networkInclude: input.include, }); +export const audioDaemonWriter: DaemonWriter = (input) => + request(AUDIO_COMMAND_NAME, audioPositionals(input as AudioOptions), input); + const logsCommandFacet = defineCommandFacet({ name: LOGS_COMMAND_NAME, metadata: logsCommandMetadata, @@ -120,9 +162,19 @@ const networkCommandFacet = defineCommandFacet({ cliOutputFormatter: observabilityCliOutputFormatters.network, }); +const audioCommandFacet = defineCommandFacet({ + name: AUDIO_COMMAND_NAME, + metadata: audioCommandMetadata, + definition: audioCommandDefinition, + cliSchema: audioCliSchema, + cliReader: audioCliReader, + daemonWriter: audioDaemonWriter, + cliOutputFormatter: observabilityCliOutputFormatters.audio, +}); + export const observabilityCommandFamily = defineCommandFamilyFromFacets({ name: 'observability', - commands: [logsCommandFacet, networkCommandFacet], + commands: [logsCommandFacet, networkCommandFacet, audioCommandFacet], }); function logsPositionals(input: { action?: string; message?: string }): string[] { @@ -133,6 +185,15 @@ function networkPositionals(input: NetworkOptions): string[] { return [...(input.action ? [input.action] : []), ...optionalNumber(input.limit)]; } +function audioPositionals(input: AudioOptions): string[] { + return [ + input.action ?? 'probe', + input.probeAction ?? 'status', + ...optionalNumber(input.durationMs), + ...optionalNumber(input.bucketMs), + ]; +} + function readLogsAction(value: string | undefined): LogAction | undefined { if (value === undefined) return undefined; return parseStringMember(LOG_ACTION_VALUES, value, { @@ -152,3 +213,22 @@ function readNetworkInclude(value: string | undefined): NetworkIncludeMode | und message: 'network include must be summary, headers, body, or all', }); } + +function readAudioAction(value: string | undefined): 'probe' | undefined { + if (value === undefined) return undefined; + return parseStringMember(AUDIO_ACTION_VALUES, value, { + message: 'audio requires probe', + }); +} + +function readAudioProbeAction(value: string | undefined): 'start' | 'status' | 'stop' | undefined { + if (value === undefined) return undefined; + return parseStringMember(AUDIO_PROBE_ACTION_VALUES, value, { + message: 'audio probe requires start, status, or stop', + }); +} + +function readAudioDurationMs(value: string | undefined): number | undefined { + const durationSeconds = optionalCliNumber(value); + return durationSeconds === undefined ? undefined : Math.round(durationSeconds * 1000); +} diff --git a/src/commands/observability/output.ts b/src/commands/observability/output.ts index a6cdb6169..61dbee7bc 100644 --- a/src/commands/observability/output.ts +++ b/src/commands/observability/output.ts @@ -50,6 +50,23 @@ type NetworkCliResult = { notes?: readonly string[]; }; +type AudioCliResult = { + active?: boolean; + backend?: string; + source?: string; + state?: string; + heard?: boolean; + durationMs?: number; + elapsedMs?: number; + bucketMs?: number; + sampleCount?: number; + sourceCount?: number; + mediaElementCount?: number; + rmsDbfs?: readonly number[]; + peakDbfs?: readonly number[]; + notes?: readonly string[]; +}; + function logsCliOutput(data: LogsCliResult): CliOutput { return { data, @@ -91,11 +108,46 @@ function networkCliOutput(data: NetworkCliResult): CliOutput { }; } +function audioCliOutput(data: AudioCliResult): CliOutput { + const lines = [ + `Audio probe: ${String(data.state ?? 'stopped')} heard=${String(data.heard === true)}`, + formatAudioArray('rmsDbfs', data.rmsDbfs), + formatAudioArray('peakDbfs', data.peakDbfs), + ].filter((line): line is string => Boolean(line)); + return { + data, + text: lines.join('\n'), + stderr: joinDefinedLines([ + formatKeyValueFields(data, [ + 'active', + 'backend', + 'source', + 'durationMs', + 'elapsedMs', + 'bucketMs', + 'sampleCount', + 'sourceCount', + 'mediaElementCount', + ] as const), + formatNotes(data.notes), + ]), + }; +} + export const observabilityCliOutputFormatters = { logs: resultOutput(logsCliOutput), network: resultOutput(networkCliOutput), + audio: resultOutput(audioCliOutput), } as const satisfies Record; +function formatAudioArray(label: string, value: readonly number[] | undefined): string | undefined { + if (!Array.isArray(value)) return undefined; + const numbers = value.filter( + (item): item is number => typeof item === 'number' && Number.isFinite(item), + ); + return numbers.length > 0 ? `${label}: [${numbers.join(', ')}]` : undefined; +} + function formatActionFields(data: LogsActionFields): string | undefined { return ( LOG_ACTION_FIELD_KEYS.map((key) => formatActionField(key, data[key])) diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index e3c8f82e3..48c782985 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -44,6 +44,7 @@ const WEB_DEVICE: KindMatrix = { device: true }; const WEB_RUNTIME_COMMANDS = ['open', 'close'] as const; const WEB_RECORDING_COMMANDS = ['record'] as const; const WEB_QUERY_COMMANDS = [ + 'audio', 'find', 'get', 'is', @@ -198,6 +199,11 @@ const BASE_COMMAND_CAPABILITY_MATRIX: Record = { android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, }, + audio: { + apple: {}, + android: {}, + linux: LINUX_NONE, + }, network: { apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, diff --git a/src/daemon/__tests__/daemon-command-registry.test.ts b/src/daemon/__tests__/daemon-command-registry.test.ts index 269e39a6b..ee07752ce 100644 --- a/src/daemon/__tests__/daemon-command-registry.test.ts +++ b/src/daemon/__tests__/daemon-command-registry.test.ts @@ -44,6 +44,7 @@ test('daemon command registry owns session handler subroutes', () => { assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.boot), 'state'); assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.shutdown), 'state'); assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.appState), 'state'); + assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.audio), 'observability'); assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.logs), 'observability'); assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.test), 'replay'); assert.equal(getSessionCommandKind(PUBLIC_COMMANDS.open), undefined); diff --git a/src/daemon/daemon-command-registry.ts b/src/daemon/daemon-command-registry.ts index c089b9f32..0a60f993d 100644 --- a/src/daemon/daemon-command-registry.ts +++ b/src/daemon/daemon-command-registry.ts @@ -82,6 +82,7 @@ const DAEMON_COMMAND_DESCRIPTORS = [ PUBLIC_COMMANDS.perf, PUBLIC_COMMANDS.logs, PUBLIC_COMMANDS.network, + PUBLIC_COMMANDS.audio, ), ...descriptors( 'session', diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index 36df9f286..fcbb87907 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -44,8 +44,12 @@ import { const LOG_ACTIONS_MESSAGE = `logs requires ${LOG_ACTIONS.slice(0, -1).join(', ')}, or ${LOG_ACTIONS.at(-1)}`; const NETWORK_ACTIONS = ['dump', 'log'] as const; +const AUDIO_ACTIONS = ['probe'] as const; +const AUDIO_PROBE_ACTIONS = ['start', 'status', 'stop'] as const; const NETWORK_ACTIONS_MESSAGE = `network requires ${NETWORK_ACTIONS.join(' or ')}`; const NETWORK_INCLUDE_MESSAGE = `network include mode must be one of: ${NETWORK_INCLUDE_MODES.join(', ')}`; +const AUDIO_ACTIONS_MESSAGE = 'audio requires probe'; +const AUDIO_PROBE_ACTIONS_MESSAGE = `audio probe requires ${AUDIO_PROBE_ACTIONS.join(', ')}`; type ObservabilityParams = { req: DaemonRequest; @@ -93,6 +97,9 @@ export async function handleSessionObservabilityCommands( if (req.command === 'network') { return handleNetworkCommand(params); } + if (req.command === 'audio') { + return await handleAudioCommand(params); + } return null; } @@ -605,3 +612,108 @@ function resolveNetworkIncludeMode( } return { ok: true, include: requestedInclude as NetworkIncludeMode }; } + +// --------------------------------------------------------------------------- +// audio +// --------------------------------------------------------------------------- + +async function handleAudioCommand(params: ObservabilityParams): Promise { + const request = resolveAudioCommandRequest(params); + if (!request.ok) return request; + const provider = resolveWebProvider(); + if (!provider.probeAudio) { + return errorResponse('UNSUPPORTED_OPERATION', 'audio is not supported by this web provider'); + } + + try { + return { + ok: true, + data: await provider.probeAudio({ + action: request.probeAction, + durationMs: request.durationMs, + bucketMs: request.bucketMs, + source: 'media-elements', + }), + }; + } catch (error) { + return { ok: false, error: normalizeError(error) }; + } +} + +function resolveAudioCommandRequest(params: ObservabilityParams): + | { + ok: true; + probeAction: 'start' | 'status' | 'stop'; + durationMs: number; + bucketMs: number; + } + | DaemonFailureResponse { + const { req, sessionName, sessionStore } = params; + const session = sessionStore.get(sessionName); + if (!session) { + return errorResponse('SESSION_NOT_FOUND', 'audio requires an active session'); + } + if (!isCommandSupportedOnDevice('audio', session.device)) { + return errorResponse( + 'UNSUPPORTED_OPERATION', + 'audio is currently supported for web browser sessions only', + ); + } + + const action = (req.positionals?.[0] ?? 'probe').toLowerCase(); + if (!AUDIO_ACTIONS.includes(action as (typeof AUDIO_ACTIONS)[number])) { + return errorResponse('INVALID_ARGS', AUDIO_ACTIONS_MESSAGE); + } + + const probeAction = (req.positionals?.[1] ?? 'status').toLowerCase(); + if (!AUDIO_PROBE_ACTIONS.includes(probeAction as (typeof AUDIO_PROBE_ACTIONS)[number])) { + return errorResponse('INVALID_ARGS', AUDIO_PROBE_ACTIONS_MESSAGE); + } + + const durationMs = readBoundedInteger(req.positionals?.[2], { + defaultValue: 10_000, + min: 100, + max: 120_000, + message: 'audio probe duration must be an integer in range 100..120000 ms', + }); + if (durationMs instanceof Error) return errorResponse('INVALID_ARGS', durationMs.message); + + const bucketMs = readBoundedInteger(req.positionals?.[3], { + defaultValue: 1_000, + min: 100, + max: 10_000, + message: 'audio probe bucket must be an integer in range 100..10000 ms', + }); + if (bucketMs instanceof Error) return errorResponse('INVALID_ARGS', bucketMs.message); + + if (probeAction !== 'start' && req.positionals && req.positionals.length > 2) { + return errorResponse( + 'INVALID_ARGS', + 'audio probe duration and bucket are only supported with audio probe start', + ); + } + + return { + ok: true, + probeAction: probeAction as 'start' | 'status' | 'stop', + durationMs, + bucketMs, + }; +} + +function readBoundedInteger( + value: string | undefined, + params: { defaultValue: number; min: number; max: number; message: string }, +): number | Error { + if (value === undefined) return params.defaultValue; + const parsed = Number.parseInt(value, 10); + if ( + !Number.isInteger(parsed) || + String(parsed) !== value || + parsed < params.min || + parsed > params.max + ) { + return new Error(params.message); + } + return parsed; +} diff --git a/src/platforms/web/agent-browser-provider.test.ts b/src/platforms/web/agent-browser-provider.test.ts index b5d702611..449e92b05 100644 --- a/src/platforms/web/agent-browser-provider.test.ts +++ b/src/platforms/web/agent-browser-provider.test.ts @@ -188,6 +188,69 @@ test('agent-browser provider dumps session network requests', async () => { }); }); +test('agent-browser provider probes page audio through eval', async () => { + await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => { + const calls: AgentBrowserCall[] = []; + const audio = await withCommandExecutorOverride( + async (cmd, args) => { + calls.push({ cmd, args }); + return jsonResult({ + success: true, + data: { + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'media-elements', + backend: 'agent-browser', + durationMs: 7500, + elapsedMs: 1000, + bucketMs: 500, + sampleCount: 2, + mediaElementCount: 1, + sourceCount: 1, + rmsDbfs: [-30, -24], + peakDbfs: [-18, -12], + notes: ['Audio probe samples HTML media elements exposed by captureStream().'], + }, + }); + }, + async () => + await provider.probeAudio?.({ + action: 'start', + durationMs: 7500, + bucketMs: 500, + source: 'media-elements', + }), + ); + + assert.equal(calls.length, 1); + assert.equal(calls[0]?.args[0], 'eval'); + assert.match(calls[0]?.args[1] ?? '', /__agentDeviceAudioProbe/); + assert.deepEqual(calls[0]?.args.slice(2), ['--json', '--session', 'web-session']); + assert.deepEqual(audio, { + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'media-elements', + backend: 'agent-browser', + durationMs: 7500, + elapsedMs: 1000, + bucketMs: 500, + sampleCount: 2, + mediaElementCount: 1, + sourceCount: 1, + rmsDbfs: [-30, -24], + peakDbfs: [-18, -12], + startedAt: undefined, + stoppedAt: undefined, + reason: undefined, + notes: ['Audio probe samples HTML media elements exposed by captureStream().'], + }); + }); +}); + test('agent-browser provider surfaces stale ref failures during requested snapshot geometry lookup', async () => { await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => { await assert.rejects( diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index ab1fcb675..e483b6fda 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -6,11 +6,18 @@ import { normalizeAgentBrowserNetworkRequests } from './agent-browser-network.ts import { normalizeAgentBrowserSnapshot } from './agent-browser-snapshot.ts'; import { isJsonObject, + readBooleanProperty, readNumberProperty, readStringProperty, type JsonObject, } from './json-utils.ts'; -import type { WebProvider, WebSnapshotOptions, WebSnapshotResult } from './provider.ts'; +import type { + WebAudioProbeOptions, + WebAudioProbeResult, + WebProvider, + WebSnapshotOptions, + WebSnapshotResult, +} from './provider.ts'; import { mapManagedAgentBrowserError, resolveAgentBrowserTool } from './agent-browser-tool.ts'; const AGENT_BROWSER = 'agent-browser'; @@ -80,9 +87,256 @@ export function createAgentBrowserWebProvider( async dumpNetwork(options) { return normalizeAgentBrowserNetworkRequests(await runJson(['network', 'requests']), options); }, + async probeAudio(options) { + return normalizeAgentBrowserAudioProbeResult( + await runJson(['eval', buildAudioProbeEvalScript(options)]), + ); + }, + }; +} + +function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { + return `(() => { + const options = ${JSON.stringify({ + action: options.action, + durationMs: options.durationMs, + bucketMs: options.bucketMs, + source: options.source ?? 'media-elements', + })}; + const key = '__agentDeviceAudioProbe'; + const silenceDb = -90; + const now = () => Date.now(); + const dbfs = (value) => { + if (!Number.isFinite(value) || value <= 0) return silenceDb; + return Math.max(silenceDb, Math.min(0, Math.round(20 * Math.log10(value)))); + }; + const note = (probe, message) => { + if (!probe.notes.includes(message)) probe.notes.push(message); + }; + const mediaElements = () => Array.from(document.querySelectorAll('audio,video')); + const stopProbe = (probe, reason) => { + if (!probe || probe.state === 'stopped') return probe; + clearInterval(probe.timer); + clearTimeout(probe.timeout); + probe.timer = undefined; + probe.timeout = undefined; + probe.state = 'stopped'; + probe.active = false; + probe.reason = reason; + probe.stoppedAt = now(); + try { + void probe.context.close(); + } catch {} + return probe; + }; + const discover = (probe) => { + const elements = mediaElements(); + probe.mediaElementCount = elements.length; + for (const element of elements) { + if (probe.seen.has(element)) continue; + probe.seen.add(element); + if (typeof element.captureStream !== 'function') { + note(probe, 'Some media elements do not expose captureStream(), so they cannot be sampled.'); + continue; + } + const stream = element.captureStream(); + if (!stream || stream.getAudioTracks().length === 0) { + note(probe, 'Some media elements have no exposed audio track.'); + continue; + } + const analyser = probe.context.createAnalyser(); + analyser.fftSize = 2048; + const source = probe.context.createMediaStreamSource(stream); + source.connect(analyser); + analyser.connect(probe.sink); + probe.analysers.push({ analyser, buffer: new Float32Array(analyser.fftSize) }); + } + probe.sourceCount = probe.analysers.length; + if (probe.sourceCount === 0) { + note(probe, 'No capturable page media audio sources were found yet.'); + } + }; + const sample = (probe) => { + if (!probe || probe.state !== 'running') return; + discover(probe); + let totalSquares = 0; + let totalSamples = 0; + let peak = 0; + for (const entry of probe.analysers) { + entry.analyser.getFloatTimeDomainData(entry.buffer); + for (const value of entry.buffer) { + totalSquares += value * value; + totalSamples += 1; + const abs = Math.abs(value); + if (abs > peak) peak = abs; + } + } + const rms = totalSamples > 0 ? Math.sqrt(totalSquares / totalSamples) : 0; + const rmsDb = dbfs(rms); + const peakDb = dbfs(peak); + probe.rmsDbfs.push(rmsDb); + probe.peakDbfs.push(peakDb); + probe.heard = probe.heard || rmsDb > silenceDb || peakDb > silenceDb; + const maxSamples = Math.ceil(probe.durationMs / probe.bucketMs) + 2; + if (probe.rmsDbfs.length > maxSamples) probe.rmsDbfs.splice(0, probe.rmsDbfs.length - maxSamples); + if (probe.peakDbfs.length > maxSamples) probe.peakDbfs.splice(0, probe.peakDbfs.length - maxSamples); + if (now() - probe.startedAt >= probe.durationMs) stopProbe(probe, 'duration'); + }; + const result = (probe) => { + const mediaCount = mediaElements().length; + if (!probe) { + return { + audio: 'probe', + state: 'stopped', + active: false, + heard: false, + source: 'media-elements', + backend: 'agent-browser', + durationMs: Number(options.durationMs) || 10000, + elapsedMs: 0, + bucketMs: Number(options.bucketMs) || 1000, + sampleCount: 0, + mediaElementCount: mediaCount, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + notes: [ + 'Audio probe samples HTML media elements exposed by captureStream(); it is not whole-tab or system audio capture.' + ], + }; + } + return { + audio: 'probe', + state: probe.state, + active: probe.state === 'running', + heard: probe.heard, + source: 'media-elements', + backend: 'agent-browser', + durationMs: probe.durationMs, + elapsedMs: Math.max(0, Math.min(now() - probe.startedAt, probe.durationMs)), + bucketMs: probe.bucketMs, + sampleCount: probe.rmsDbfs.length, + mediaElementCount: mediaCount, + sourceCount: probe.sourceCount, + rmsDbfs: probe.rmsDbfs.slice(), + peakDbfs: probe.peakDbfs.slice(), + startedAt: new Date(probe.startedAt).toISOString(), + stoppedAt: probe.stoppedAt ? new Date(probe.stoppedAt).toISOString() : undefined, + reason: probe.reason, + notes: [ + 'Audio probe samples HTML media elements exposed by captureStream(); it is not whole-tab or system audio capture.', + ...probe.notes, + ], + }; + }; + const action = options.action || 'status'; + let probe = window[key]; + if (probe && probe.state === 'running' && now() - probe.startedAt >= probe.durationMs) { + sample(probe); + stopProbe(probe, 'duration'); + } + if (action === 'start') { + if (probe) stopProbe(probe, 'restarted'); + const AudioContextCtor = window.AudioContext || window.webkitAudioContext; + if (!AudioContextCtor) { + return { + audio: 'probe', + state: 'stopped', + active: false, + heard: false, + source: 'media-elements', + backend: 'agent-browser', + durationMs: Number(options.durationMs) || 10000, + elapsedMs: 0, + bucketMs: Number(options.bucketMs) || 1000, + sampleCount: 0, + mediaElementCount: mediaElements().length, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + notes: ['Web Audio API is not available in this browser context.'], + }; + } + const context = new AudioContextCtor(); + const sink = context.createGain(); + sink.gain.value = 0; + sink.connect(context.destination); + probe = { + state: 'running', + active: true, + context, + sink, + seen: new WeakSet(), + analysers: [], + mediaElementCount: 0, + sourceCount: 0, + durationMs: Math.max(100, Number(options.durationMs) || 10000), + bucketMs: Math.max(100, Number(options.bucketMs) || 1000), + startedAt: now(), + stoppedAt: undefined, + reason: undefined, + heard: false, + rmsDbfs: [], + peakDbfs: [], + notes: [], + }; + window[key] = probe; + try { + void context.resume(); + } catch { + note(probe, 'AudioContext could not be resumed by the probe.'); + } + discover(probe); + probe.timer = setInterval(() => sample(probe), probe.bucketMs); + probe.timeout = setTimeout(() => stopProbe(probe, 'duration'), probe.durationMs); + return result(probe); + } + if (action === 'stop') { + if (probe) sample(probe); + stopProbe(probe, 'manual'); + return result(probe); + } + if (probe) sample(probe); + return result(probe); +})()`; +} + +function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { + const record = isJsonObject(data) ? data : {}; + return { + audio: 'probe', + state: readStringProperty(record, 'state') === 'running' ? 'running' : 'stopped', + active: readBooleanProperty(record, 'active') === true, + heard: readBooleanProperty(record, 'heard') === true, + source: 'media-elements', + backend: readStringProperty(record, 'backend') ?? 'agent-browser', + durationMs: readNumberProperty(record, 'durationMs') ?? 0, + elapsedMs: readNumberProperty(record, 'elapsedMs') ?? 0, + bucketMs: readNumberProperty(record, 'bucketMs') ?? 1000, + sampleCount: readNumberProperty(record, 'sampleCount') ?? 0, + mediaElementCount: readNumberProperty(record, 'mediaElementCount') ?? 0, + sourceCount: readNumberProperty(record, 'sourceCount') ?? 0, + rmsDbfs: readNumberArray(record.rmsDbfs), + peakDbfs: readNumberArray(record.peakDbfs), + startedAt: readStringProperty(record, 'startedAt'), + stoppedAt: readStringProperty(record, 'stoppedAt'), + reason: readStringProperty(record, 'reason'), + notes: readStringArray(record.notes), }; } +function readNumberArray(value: unknown): number[] { + return Array.isArray(value) + ? value.filter((item): item is number => typeof item === 'number' && Number.isFinite(item)) + : []; +} + +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + const items = value.filter((item): item is string => typeof item === 'string' && item.length > 0); + return items.length > 0 ? items : undefined; +} + async function runPacedScroll( runJson: (args: string[]) => Promise, direction: string, diff --git a/src/platforms/web/provider.ts b/src/platforms/web/provider.ts index 860d27e89..b35ad8992 100644 --- a/src/platforms/web/provider.ts +++ b/src/platforms/web/provider.ts @@ -29,6 +29,36 @@ export type WebSnapshotResult = { truncated?: boolean; }; +export type WebAudioProbeAction = 'start' | 'status' | 'stop'; + +export type WebAudioProbeOptions = { + action: WebAudioProbeAction; + durationMs?: number; + bucketMs?: number; + source?: 'media-elements'; +}; + +export type WebAudioProbeResult = { + audio: 'probe'; + state: 'running' | 'stopped'; + active: boolean; + heard: boolean; + source: 'media-elements'; + backend?: string; + durationMs: number; + elapsedMs: number; + bucketMs: number; + sampleCount: number; + mediaElementCount: number; + sourceCount: number; + rmsDbfs: number[]; + peakDbfs: number[]; + startedAt?: string; + stoppedAt?: string; + reason?: string; + notes?: string[]; +}; + export type WebProvider = { open(target: string, options?: WebOpenOptions): Promise; close(target?: string): Promise; @@ -48,6 +78,7 @@ export type WebProvider = { ): Promise | void>; readText?(x: number, y: number): Promise; dumpNetwork?(options?: BackendDumpNetworkOptions): Promise; + probeAudio?(options: WebAudioProbeOptions): Promise; }; const localWebProvider: WebProvider = createAgentBrowserWebProvider(); diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 62ccf53f5..7f25d7f13 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -283,12 +283,13 @@ Validation and evidence: agent-device wait text "Welcome" 3000 --platform web agent-device record start ./artifacts/web-flow.webm --platform web agent-device network dump 25 --include headers --platform web + agent-device audio probe start 10 1000 --platform web agent-device screenshot ./artifacts/web-home.png --platform web agent-device screenshot ./artifacts/web-full.png --platform web --fullscreen agent-device viewport 1280 900 --platform web agent-device record stop --platform web agent-device close --platform web - Minimal web support is for browser sessions with open, snapshot, find, get, is, click/press, fill/type, wait, network dump, screenshot, record start/stop with WebM output, close, and replay over those commands. Use agent-browser directly for browser-specific features that agent-device does not surface, such as tab/devtools management, advanced page scripting, network routing/HAR, or raw browser debugging. + Minimal web support is for browser sessions with open, snapshot, find, get, is, click/press, fill/type, wait, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay over those commands. Use agent-browser directly for browser-specific features that agent-device does not surface, such as tab/devtools management, advanced page scripting, network routing/HAR, or raw browser debugging. macOS menu bar: open ... --platform macos --surface menubar; snapshot -i --platform macos --surface menubar. Maestro full-suite validation on explicit connected devices uses one test command with a comma-separated --device list and --shard-all. Use --shard-split only when splitting suite entries across devices: agent-device test ./e2e/maestro --maestro --device udid1,emulator-5554 --shard-all 2 @@ -766,6 +767,7 @@ First-slice loop: agent-device wait text "Welcome" 3000 --platform web agent-device record start ./artifacts/web-flow.webm --platform web agent-device network dump 25 --include headers --platform web + agent-device audio probe start 10 1000 --platform web agent-device screenshot ./artifacts/web-home.png --platform web agent-device screenshot ./artifacts/web-full.png --platform web --fullscreen agent-device viewport 1280 900 --platform web @@ -773,7 +775,7 @@ First-slice loop: agent-device close --platform web Supported in agent-device web sessions: - open , snapshot -i, get text/attrs, is visible/exists/text, find text/selector, click/press @ref or selector, fill/type @ref or selector, wait text/selector, network dump, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. + open , snapshot -i, get text/attrs, is visible/exists/text, find text/selector, click/press @ref or selector, fill/type @ref or selector, wait text/selector, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. Out of scope for agent-device web support: Browser runtime debugging, tabs/windows/devtools control, network routing/interception/HAR, storage/cookie management, arbitrary page scripting, downloads/uploads, multi-page orchestration, and agent-browser-specific diagnostics. Use agent-browser directly for those browser-specific workflows. diff --git a/test/integration/provider-scenarios/web-provider.test.ts b/test/integration/provider-scenarios/web-provider.test.ts index 4096d9ccb..746e7241d 100644 --- a/test/integration/provider-scenarios/web-provider.test.ts +++ b/test/integration/provider-scenarios/web-provider.test.ts @@ -74,6 +74,26 @@ test('web provider is scoped through the request router and dispatch path', asyn redacted: false, }; }, + async probeAudio(options) { + calls.push(`audio:${options.action}:${options.durationMs ?? ''}:${options.bucketMs ?? ''}`); + return { + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'media-elements', + backend: 'agent-browser', + durationMs: options.durationMs ?? 10_000, + elapsedMs: 1000, + bucketMs: options.bucketMs ?? 1000, + sampleCount: 1, + mediaElementCount: 1, + sourceCount: 1, + rmsDbfs: [-24], + peakDbfs: [-12], + notes: ['HTML media element probe'], + }; + }, }; const harness = await createProviderScenarioHarness({ @@ -143,6 +163,15 @@ test('web provider is scoped through the request router and dispatch path', asyn ]); assert.equal(network.json.result.data.backend, 'agent-browser'); assert.equal(network.json.result.data.include, 'headers'); + const audio = await harness.callCommand( + 'audio', + ['probe', 'start', '10000', '1000'], + { platform: 'web' }, + { meta: { requestId: 'req-web-audio' } }, + ); + assert.deepEqual(audio.json.result.data.rmsDbfs, [-24]); + assert.equal(audio.json.result.data.heard, true); + const viewport = await harness.callCommand( 'viewport', ['1280', '900'], @@ -162,6 +191,8 @@ test('web provider is scoped through the request router and dispatch path', asyn 'scope:default:agent-browser-chrome', 'network:5:headers', 'scope:default:agent-browser-chrome', + 'audio:start:10000:1000', + 'scope:default:agent-browser-chrome', 'viewport:1280:900', ]); } finally { From 392b057e6cd8cd9fe3ef73b93f36d8ef5562dc26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 25 Jun 2026 19:30:11 +0200 Subject: [PATCH 02/18] fix: stabilize web audio probe --- examples/test-app/package.json | 4 +- examples/test-app/pnpm-lock.yaml | 314 ++++++++++++++---- examples/test-app/src/screens/AudioScreen.tsx | 172 ++++++---- src/daemon/handlers/session-observability.ts | 79 +++-- .../web/agent-browser-provider.test.ts | 33 +- src/platforms/web/agent-browser-provider.ts | 137 ++++++-- 6 files changed, 530 insertions(+), 209 deletions(-) diff --git a/examples/test-app/package.json b/examples/test-app/package.json index 97ca6028a..72e64bdaa 100644 --- a/examples/test-app/package.json +++ b/examples/test-app/package.json @@ -19,10 +19,12 @@ "expo-router": "~56.2.11", "expo-status-bar": "~56.0.4", "react": "19.2.3", + "react-dom": "19.2.3", "react-native": "0.85.3", "react-native-gesture-handler": "^2.31.2", "react-native-safe-area-context": "~5.7.0", - "react-native-screens": "~4.25.2" + "react-native-screens": "~4.25.2", + "react-native-web": "^0.21.2" }, "devDependencies": { "@types/react": "~19.2.2", diff --git a/examples/test-app/pnpm-lock.yaml b/examples/test-app/pnpm-lock.yaml index e11a31e32..c5dfd7811 100644 --- a/examples/test-app/pnpm-lock.yaml +++ b/examples/test-app/pnpm-lock.yaml @@ -21,10 +21,10 @@ importers: version: 56.0.5(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) '@expo/metro-runtime': specifier: ~56.0.15 - version: 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + version: 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) expo: specifier: ~56.0.12 - version: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + version: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-constants: specifier: 56.0.18 version: 56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) @@ -36,13 +36,16 @@ importers: version: 56.0.14(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) expo-router: specifier: ~56.2.11 - version: 56.2.11(73fd217edc861e03b88af57871d8ab89) + version: 56.2.11(4e54386c19a86f18d3fced032d4d30ae) expo-status-bar: specifier: ~56.0.4 version: 56.0.4(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) react: specifier: 19.2.3 version: 19.2.3 + react-dom: + specifier: 19.2.3 + version: 19.2.3(react@19.2.3) react-native: specifier: 0.85.3 version: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) @@ -55,6 +58,9 @@ importers: react-native-screens: specifier: ~4.25.2 version: 4.25.2(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + react-native-web: + specifier: ^0.21.2 + version: 0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3) devDependencies: '@types/react': specifier: ~19.2.2 @@ -984,6 +990,9 @@ packages: resolution: {integrity: sha512-7v+xbTeEci9ZcQ/Z1OqI4RXcqN69wSMDYL5BAMvOReZ7U04+aDQ0/SQhClYPn6x2/RxM4WzMKSAuNyLKqvYVtw==} engines: {node: ^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0} + '@react-native/normalize-colors@0.74.89': + resolution: {integrity: sha512-qoMMXddVKVhZ8PA1AbUCk83trpd6N+1nF2A6k1i6LsQObyS92fELuk8kU/lQs6M7BsMHwqyLCpQJ1uFgNvIQXg==} + '@react-native/normalize-colors@0.85.3': resolution: {integrity: sha512-hj0PScZEhIbcOvQV5yMKX3ha4XEIOy/SVE1Rrpp0beW0dpNLOgSC7KDxGewmDnIHK9YdQUXGY9eMEfShUMIaZw==} @@ -1314,10 +1323,16 @@ packages: core-js-compat@3.49.0: resolution: {integrity: sha512-VQXt1jr9cBz03b331DFDCCP90b3fanciLkgiOoy8SBHy06gNf+vQ1A3WFLqG7I8TipYIKeYK9wxd0tUrvHcOZA==} + cross-fetch@3.2.0: + resolution: {integrity: sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + css-in-js-utils@3.1.0: + resolution: {integrity: sha512-fJAcud6B3rRu+KHYk+Bwf+WFL2MDCJJ1XG9x137tJQ0xYxor7XziQtuGFbWNdqrvF4Tk26O3H73nfVqXt/fW1A==} + css.escape@1.5.1: resolution: {integrity: sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==} @@ -1618,6 +1633,12 @@ packages: fb-watchman@2.0.2: resolution: {integrity: sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==} + fbjs-css-vars@1.0.2: + resolution: {integrity: sha512-b2XGFAFdWZWg0phtAWLHCk836A1Xann+I+Dgd3Gk64MHKZO44FfoD1KxyvbSh0qZsIoXQGGlVztIY+oitJPpRQ==} + + fbjs@3.0.5: + resolution: {integrity: sha512-ztsSx77JBtkuMrEypfhgc3cI0+0h+svqeie7xHbh1k/IKdcydnvadp/mUaGgjAOXQmQSxsqgaRhS3q9fy+1kxg==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -1726,6 +1747,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + hyphenate-style-name@1.1.0: + resolution: {integrity: sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw==} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -1742,6 +1766,9 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + inline-style-prefixer@7.0.1: + resolution: {integrity: sha512-lhYo5qNTQp3EvSSp3sRvXMbVQTLrvGV6DycRMJ5dm2BLMiJ30wpXKdDdgX+GmJZ5uQMucwRKHamXSst3Sj/Giw==} + invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} @@ -1937,6 +1964,9 @@ packages: memoize-one@5.2.1: resolution: {integrity: sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==} + memoize-one@6.0.0: + resolution: {integrity: sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==} + merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -2070,6 +2100,15 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-forge@1.4.0: resolution: {integrity: sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==} engines: {node: '>= 6.13.0'} @@ -2091,6 +2130,10 @@ packages: resolution: {integrity: sha512-eJXMpz4aQHXF/YBB9ddqZDIS+ooO91hObo9FoW/xBkr54/zCwYYCDqT/O54vNo8kOkWs5Ou/y28NgdrV0edQNA==} engines: {node: ^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0} + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + on-finished@2.3.0: resolution: {integrity: sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==} engines: {node: '>= 0.8'} @@ -2153,6 +2196,9 @@ packages: resolution: {integrity: sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==} engines: {node: '>=4.0.0'} + postcss-value-parser@4.2.0: + resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} + postcss@8.5.12: resolution: {integrity: sha512-W62t/Se6rA0Az3DfCL0AqJwXuKwBeYg6nOaIgzP+xZ7N5BFCI7DYi1qs6ygUYT6rvfi6t9k65UMLJC+PHZpDAA==} engines: {node: ^10 || ^12 || >=14} @@ -2173,6 +2219,9 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + promise@7.3.1: + resolution: {integrity: sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==} + promise@8.3.0: resolution: {integrity: sha512-rZPNPKTOYVNEEKFaq1HqTgOwZD+4/YHS5ukLzQCypkj+OkYx7iv0mA91lJlpPPZ8vMau3IIGj5Qlwrx+8iiSmg==} @@ -2194,6 +2243,11 @@ packages: react-devtools-core@6.1.5: resolution: {integrity: sha512-ePrwPfxAnB+7hgnEr8vpKxL9cmnp7F322t8oqcPshbIQQhDKgFDW4tjhF2wjVbdXF9O/nyuy3sQWd9JGpiLPvA==} + react-dom@19.2.3: + resolution: {integrity: sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==} + peerDependencies: + react: ^19.2.3 + react-fast-compare@3.2.2: resolution: {integrity: sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==} @@ -2254,6 +2308,12 @@ packages: react: '*' react-native: '>=0.82.0' + react-native-web@0.21.2: + resolution: {integrity: sha512-SO2t9/17zM4iEnFvlu2DA9jqNbzNhoUP+AItkoCOyFmDMOhUnBBznBDCYN92fGdfAkfQlWzPoez6+zLxFNsZEg==} + peerDependencies: + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + react-native-worklets@0.10.0: resolution: {integrity: sha512-JhE6IxDf6iabC0qu3+TAKA4v9RlluXmoIngPQX7/QUByf75lfrsHZ6/dQhyjEWnp1EEQiwzz8Cpew140ZcewDw==} peerDependencies: @@ -2393,6 +2453,9 @@ packages: server-only@0.0.1: resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} + setimmediate@1.0.5: + resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -2495,6 +2558,9 @@ packages: structured-headers@0.4.1: resolution: {integrity: sha512-0MP/Cxx5SzeeZ10p/bZI0S6MpgD+yxAhi1BOQ34jgnMXsCq3j1t6tQnZu+KdlL7dvJTLT3g9xN8tl10TqgFMcg==} + styleq@0.1.3: + resolution: {integrity: sha512-3ZUifmCDCQanjeej1f6kyl/BeP/Vae5EYkQ9iJfUm/QwZvlgnZzyflqAsAWYURdtea8Vkvswu2GrC57h3qffcA==} + supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -2545,6 +2611,9 @@ packages: toqr@0.1.1: resolution: {integrity: sha512-FWAPzCIHZHnrE/5/w9MPk0kK25hSQSH2IKhYh9PyjS3SG/+IEMvlwIHbhz+oF7xl54I+ueZlVnMjyzdSwLmAwA==} + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} @@ -2561,6 +2630,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + ua-parser-js@1.0.41: + resolution: {integrity: sha512-LbBDqdIC5s8iROCUjMbW1f5dJQTEFB1+KO9ogbvlb3nm9n4YHa5p4KTvFPWvh2Hs8gZMBuiB1/8+pdfe/tDPug==} + hasBin: true + undici-types@7.18.2: resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} @@ -2649,12 +2722,18 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + whatwg-fetch@3.6.20: resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} whatwg-url-minimum@0.1.2: resolution: {integrity: sha512-XPEm0XFQWNVG292lII1PrRRJl3sItrs7CettZ4ncYxuDVpLyy+NwlGyut2hXI0JswcJUxeCH+CyOJK0ZzAXD6A==} + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -3223,7 +3302,7 @@ snapshots: '@expo-google-fonts/material-symbols@0.4.29': {} - '@expo/cli@56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3)': + '@expo/cli@56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3)': dependencies: '@expo/code-signing-certificates': 0.0.6 '@expo/config': 56.0.9(typescript@6.0.3) @@ -3242,7 +3321,7 @@ snapshots: '@expo/plist': 0.7.0 '@expo/prebuild-config': 56.0.16(typescript@6.0.3) '@expo/require-utils': 56.1.3(typescript@6.0.3) - '@expo/router-server': 56.0.14(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo-server@56.0.5)(expo@56.0.12)(react@19.2.3) + '@expo/router-server': 56.0.14(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo-server@56.0.5)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@expo/schema-utils': 56.0.1 '@expo/spawn-async': 1.8.0 '@expo/ws-tunnel': 2.0.0(ws@8.21.0) @@ -3258,7 +3337,7 @@ snapshots: connect: 3.7.0 debug: 4.4.3 dnssd-advertise: 1.1.4 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-server: 56.0.5 fetch-nodeshim: 0.4.10 getenv: 2.0.0 @@ -3284,7 +3363,7 @@ snapshots: ws: 8.21.0 zod: 3.25.76 optionalDependencies: - expo-router: 56.2.11(73fd217edc861e03b88af57871d8ab89) + expo-router: 56.2.11(4e54386c19a86f18d3fced032d4d30ae) react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) transitivePeerDependencies: - '@expo/dom-webview' @@ -3356,7 +3435,7 @@ snapshots: '@expo/dom-webview@56.0.5(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)': dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) @@ -3423,7 +3502,7 @@ snapshots: dependencies: '@expo/dom-webview': 56.0.5(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) anser: 1.4.10 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) stacktrace-parser: 0.1.11 @@ -3454,7 +3533,7 @@ snapshots: postcss: 8.5.12 resolve-from: 5.0.0 optionalDependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) transitivePeerDependencies: - bufferutil - supports-color @@ -3472,16 +3551,18 @@ snapshots: transitivePeerDependencies: - supports-color - '@expo/metro-runtime@56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)': + '@expo/metro-runtime@56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)': dependencies: '@expo/log-box': 56.0.13(@expo/dom-webview@56.0.5)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) anser: 1.4.10 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) pretty-format: 29.7.0 react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) stacktrace-parser: 0.1.11 whatwg-fetch: 3.6.20 + optionalDependencies: + react-dom: 19.2.3(react@19.2.3) '@expo/metro@56.0.0': dependencies: @@ -3549,17 +3630,18 @@ snapshots: transitivePeerDependencies: - supports-color - '@expo/router-server@56.0.14(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo-server@56.0.5)(expo@56.0.12)(react@19.2.3)': + '@expo/router-server@56.0.14(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo-server@56.0.5)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: debug: 4.4.3 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-constants: 56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) expo-font: 56.0.7(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) expo-server: 56.0.5 react: 19.2.3 optionalDependencies: - '@expo/metro-runtime': 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) - expo-router: 56.2.11(73fd217edc861e03b88af57871d8ab89) + '@expo/metro-runtime': 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + expo-router: 56.2.11(4e54386c19a86f18d3fced032d4d30ae) + react-dom: 19.2.3(react@19.2.3) transitivePeerDependencies: - supports-color @@ -3573,15 +3655,16 @@ snapshots: '@expo/sudo-prompt@9.3.2': {} - '@expo/ui@56.0.18(@babel/core@7.29.0)(@types/react@19.2.14)(expo@56.0.12)(react-native-reanimated@4.5.0(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)': + '@expo/ui@56.0.18(@babel/core@7.29.0)(@types/react@19.2.14)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native-reanimated@4.5.0(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)': dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) sf-symbols-typescript: 2.2.0 - vaul: 1.1.2(@types/react@19.2.14)(react@19.2.3) + vaul: 1.1.2(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) optionalDependencies: '@babel/core': 7.29.0 + react-dom: 19.2.3(react@19.2.3) react-native-reanimated: 4.5.0(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) react-native-worklets: 0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) transitivePeerDependencies: @@ -3639,13 +3722,14 @@ snapshots: '@radix-ui/primitive@1.1.3': {} - '@radix-ui/react-collection@1.1.7(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-collection@1.1.7(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 @@ -3661,22 +3745,23 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-dialog@1.1.15(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-dialog@1.1.15(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/primitive': 1.1.3 '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-dismissable-layer': 1.1.11(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-dismissable-layer': 1.1.11(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-focus-guards': 1.1.3(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-focus-scope': 1.1.7(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-focus-scope': 1.1.7(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-portal': 1.1.9(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-presence': 1.1.5(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-portal': 1.1.9(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-presence': 1.1.5(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.3) aria-hidden: 1.2.6 react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) react-remove-scroll: 2.7.2(@types/react@19.2.14)(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 @@ -3687,14 +3772,15 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-dismissable-layer@1.1.11(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-dismissable-layer@1.1.11(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/primitive': 1.1.3 '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-use-escape-keydown': 1.1.1(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 @@ -3704,12 +3790,13 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-focus-scope@1.1.7(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-focus-scope@1.1.7(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 @@ -3720,41 +3807,45 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-portal@1.1.9(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-portal@1.1.9(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-presence@1.1.5(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-presence@1.1.5(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-primitive@2.1.3(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-primitive@2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-roving-focus@1.1.11(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-roving-focus@1.1.11(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-collection': 1.1.7(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-collection': 1.1.7(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 @@ -3765,17 +3856,18 @@ snapshots: optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-tabs@1.1.13(@types/react@19.2.14)(react@19.2.3)': + '@radix-ui/react-tabs@1.1.13(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: '@radix-ui/primitive': 1.1.3 '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-direction': 1.1.1(@types/react@19.2.14)(react@19.2.3) '@radix-ui/react-id': 1.1.1(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-presence': 1.1.5(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-roving-focus': 1.1.11(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-presence': 1.1.5(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-primitive': 2.1.3(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@radix-ui/react-roving-focus': 1.1.11(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) optionalDependencies: '@types/react': 19.2.14 @@ -3966,6 +4058,8 @@ snapshots: - supports-color - utf-8-validate + '@react-native/normalize-colors@0.74.89': {} + '@react-native/normalize-colors@0.85.3': {} '@react-native/virtualized-lists@0.85.3(@types/react@19.2.14)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)': @@ -4184,7 +4278,7 @@ snapshots: react-refresh: 0.14.2 optionalDependencies: '@babel/runtime': 7.29.2 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) transitivePeerDependencies: - '@babel/core' - supports-color @@ -4346,12 +4440,22 @@ snapshots: dependencies: browserslist: 4.28.2 + cross-fetch@3.2.0: + dependencies: + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 shebang-command: 2.0.0 which: 2.0.2 + css-in-js-utils@3.1.0: + dependencies: + hyphenate-style-name: 1.1.0 + css.escape@1.5.1: {} csstype@3.2.3: {} @@ -4421,7 +4525,7 @@ snapshots: expo-asset@56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3): dependencies: '@expo/image-utils': 0.10.1(typescript@6.0.3) - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-constants: 56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) @@ -4432,14 +4536,14 @@ snapshots: expo-constants@56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: '@expo/env': 2.3.0 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) transitivePeerDependencies: - supports-color expo-dev-client@56.0.20(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-dev-launcher: 56.0.20(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) expo-dev-menu: 56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) expo-dev-menu-interface: 56.0.1(expo@56.0.12) @@ -4451,36 +4555,36 @@ snapshots: expo-dev-launcher@56.0.20(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: '@expo/schema-utils': 56.0.1 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-dev-menu: 56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) expo-manifests: 56.0.4(expo@56.0.12) react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) expo-dev-menu-interface@56.0.1(expo@56.0.12): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-dev-menu@56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-dev-menu-interface: 56.0.1(expo@56.0.12) react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) expo-file-system@56.0.8(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) expo-font@56.0.7(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) fontfaceobserver: 2.3.0 react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) expo-glass-effect@56.0.4(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) @@ -4488,7 +4592,7 @@ snapshots: expo-keep-awake@56.0.3(expo@56.0.12)(react@19.2.3): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react: 19.2.3 expo-linking@56.0.14(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): @@ -4503,7 +4607,7 @@ snapshots: expo-manifests@56.0.4(expo@56.0.12): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-json-utils: 56.0.0 expo-modules-autolinking@56.0.16(typescript@6.0.3): @@ -4530,14 +4634,14 @@ snapshots: dependencies: react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) - expo-router@56.2.11(73fd217edc861e03b88af57871d8ab89): + expo-router@56.2.11(4e54386c19a86f18d3fced032d4d30ae): dependencies: '@expo/log-box': 56.0.13(@expo/dom-webview@56.0.5)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) - '@expo/metro-runtime': 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + '@expo/metro-runtime': 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) '@expo/schema-utils': 56.0.1 - '@expo/ui': 56.0.18(@babel/core@7.29.0)(@types/react@19.2.14)(expo@56.0.12)(react-native-reanimated@4.5.0(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + '@expo/ui': 56.0.18(@babel/core@7.29.0)(@types/react@19.2.14)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native-reanimated@4.5.0(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.3) - '@radix-ui/react-tabs': 1.1.13(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-tabs': 1.1.13(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@react-native-masked-view/masked-view': 0.3.2(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) '@testing-library/jest-dom': 6.9.1 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.1) @@ -4545,7 +4649,7 @@ snapshots: color: 4.2.3 debug: 4.4.3 escape-string-regexp: 4.0.0 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-constants: 56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) expo-glass-effect: 56.0.4(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) expo-linking: 56.0.14(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) @@ -4566,10 +4670,12 @@ snapshots: sf-symbols-typescript: 2.2.0 shallowequal: 1.1.0 standard-navigation: 0.0.5 - vaul: 1.1.2(@types/react@19.2.14)(react@19.2.3) + vaul: 1.1.2(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) optionalDependencies: + react-dom: 19.2.3(react@19.2.3) react-native-gesture-handler: 2.31.2(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) react-native-reanimated: 4.5.0(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + react-native-web: 0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3) transitivePeerDependencies: - '@babel/core' - '@testing-library/dom' @@ -4583,14 +4689,14 @@ snapshots: expo-status-bar@56.0.4(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) expo-symbols@56.0.6(expo-font@56.0.7)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): dependencies: '@expo-google-fonts/material-symbols': 0.4.29 - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) expo-font: 56.0.7(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) react: 19.2.3 react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) @@ -4598,12 +4704,12 @@ snapshots: expo-updates-interface@56.0.2(expo@56.0.12): dependencies: - expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) - expo@56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3): + expo@56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3): dependencies: '@babel/runtime': 7.29.2 - '@expo/cli': 56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + '@expo/cli': 56.1.16(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-constants@56.0.18)(expo-font@56.0.7)(expo-router@56.2.11)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) '@expo/config': 56.0.9(typescript@6.0.3) '@expo/config-plugins': 56.0.9(typescript@6.0.3) '@expo/devtools': 56.0.2(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) @@ -4628,7 +4734,9 @@ snapshots: whatwg-url-minimum: 0.1.2 optionalDependencies: '@expo/dom-webview': 56.0.5(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) - '@expo/metro-runtime': 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + '@expo/metro-runtime': 56.0.15(@expo/log-box@56.0.13)(expo@56.0.12)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) + react-dom: 19.2.3(react@19.2.3) + react-native-web: 0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3) transitivePeerDependencies: - '@babel/core' - bufferutil @@ -4650,6 +4758,20 @@ snapshots: dependencies: bser: 2.1.1 + fbjs-css-vars@1.0.2: {} + + fbjs@3.0.5: + dependencies: + cross-fetch: 3.2.0 + fbjs-css-vars: 1.0.2 + loose-envify: 1.4.0 + object-assign: 4.1.1 + promise: 7.3.1 + setimmediate: 1.0.5 + ua-parser-js: 1.0.41 + transitivePeerDependencies: + - encoding + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -4749,6 +4871,8 @@ snapshots: transitivePeerDependencies: - supports-color + hyphenate-style-name@1.1.0: {} + ignore@5.3.2: {} image-size@1.2.1: @@ -4759,6 +4883,10 @@ snapshots: inherits@2.0.4: {} + inline-style-prefixer@7.0.1: + dependencies: + css-in-js-utils: 3.1.0 + invariant@2.2.4: dependencies: loose-envify: 1.4.0 @@ -4914,6 +5042,8 @@ snapshots: memoize-one@5.2.1: {} + memoize-one@6.0.0: {} + merge-stream@2.0.0: {} metro-babel-transformer@0.84.4: @@ -5135,6 +5265,10 @@ snapshots: negotiator@1.0.0: {} + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-forge@1.4.0: {} node-int64@0.4.0: {} @@ -5154,6 +5288,8 @@ snapshots: dependencies: flow-enums-runtime: 0.0.6 + object-assign@4.1.1: {} + on-finished@2.3.0: dependencies: ee-first: 1.1.1 @@ -5211,6 +5347,8 @@ snapshots: pngjs@3.4.0: {} + postcss-value-parser@4.2.0: {} + postcss@8.5.12: dependencies: nanoid: 3.3.11 @@ -5233,6 +5371,10 @@ snapshots: progress@2.0.3: {} + promise@7.3.1: + dependencies: + asap: 2.0.6 + promise@8.3.0: dependencies: asap: 2.0.6 @@ -5263,6 +5405,11 @@ snapshots: - bufferutil - utf-8-validate + react-dom@19.2.3(react@19.2.3): + dependencies: + react: 19.2.3 + scheduler: 0.27.0 + react-fast-compare@3.2.2: {} react-freeze@1.0.4(react@19.2.3): @@ -5320,6 +5467,21 @@ snapshots: react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) warn-once: 0.1.1 + react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3): + dependencies: + '@babel/runtime': 7.29.2 + '@react-native/normalize-colors': 0.74.89 + fbjs: 3.0.5 + inline-style-prefixer: 7.0.1 + memoize-one: 6.0.0 + nullthrows: 1.1.1 + postcss-value-parser: 4.2.0 + react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) + styleq: 0.1.3 + transitivePeerDependencies: + - encoding + react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): dependencies: '@babel/core': 7.29.0 @@ -5503,6 +5665,8 @@ snapshots: server-only@0.0.1: {} + setimmediate@1.0.5: {} + setprototypeof@1.2.0: {} sf-symbols-typescript@2.2.0: {} @@ -5582,6 +5746,8 @@ snapshots: structured-headers@0.4.1: {} + styleq@0.1.3: {} + supports-color@5.5.0: dependencies: has-flag: 3.0.0 @@ -5630,6 +5796,8 @@ snapshots: toqr@0.1.1: {} + tr46@0.0.3: {} + tslib@2.8.1: {} type-fest@0.21.3: {} @@ -5638,6 +5806,8 @@ snapshots: typescript@6.0.3: {} + ua-parser-js@1.0.41: {} + undici-types@7.18.2: {} unicode-canonical-property-names-ecmascript@2.0.1: {} @@ -5686,10 +5856,11 @@ snapshots: vary@1.1.2: {} - vaul@1.1.2(@types/react@19.2.14)(react@19.2.3): + vaul@1.1.2(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: - '@radix-ui/react-dialog': 1.1.15(@types/react@19.2.14)(react@19.2.3) + '@radix-ui/react-dialog': 1.1.15(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) react: 19.2.3 + react-dom: 19.2.3(react@19.2.3) transitivePeerDependencies: - '@types/react' - '@types/react-dom' @@ -5706,10 +5877,17 @@ snapshots: dependencies: defaults: 1.0.4 + webidl-conversions@3.0.1: {} + whatwg-fetch@3.6.20: {} whatwg-url-minimum@0.1.2: {} + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which@2.0.2: dependencies: isexe: 2.0.0 diff --git a/examples/test-app/src/screens/AudioScreen.tsx b/examples/test-app/src/screens/AudioScreen.tsx index 1763a1340..a67a40334 100644 --- a/examples/test-app/src/screens/AudioScreen.tsx +++ b/examples/test-app/src/screens/AudioScreen.tsx @@ -1,45 +1,58 @@ import { createElement, useEffect, useRef, useState } from 'react'; import { Platform, ScrollView, StyleSheet, Text, View } from 'react-native'; -import { ActionButton, InlineBadge, ScreenTitle, SectionCard } from '../components'; +import { InlineBadge, ScreenTitle, SectionCard } from '../components'; import { useAppColors, type AppColors } from '../theme'; type BrowserAudioElement = { - currentTime: number; + srcObject: MediaStream | null; pause: () => void; play: () => Promise; }; +type SamplePlayback = { + stream: MediaStream; + stop: () => void; +}; + export function AudioScreen() { const colors = useAppColors(); const styles = createStyles(colors); const audioRef = useRef(null); - const [audioSrc, setAudioSrc] = useState(); + const playbackRef = useRef(null); const [playbackState, setPlaybackState] = useState<'ready' | 'playing' | 'paused' | 'ended'>( 'ready', ); useEffect(() => { - if (Platform.OS !== 'web') return; - const url = createClassicLoopWavUrl(); - setAudioSrc(url); - return () => { - URL.revokeObjectURL(url); - }; + return () => stopSample('ended'); }, []); function playSample() { const audio = audioRef.current; if (!audio) return; - audio.currentTime = 0; + stopSample('ready'); + const playback = createClassicLoopStream(() => { + stopSample('ended'); + }); + playbackRef.current = playback; + audio.srcObject = playback.stream; void audio.play().then(() => setPlaybackState('playing')); } function pauseSample() { + stopSample('paused'); + } + + function stopSample(nextState: 'ready' | 'paused' | 'ended') { const audio = audioRef.current; - if (!audio) return; - audio.pause(); - setPlaybackState('paused'); + playbackRef.current?.stop(); + playbackRef.current = null; + if (audio) { + audio.pause(); + audio.srcObject = null; + } + setPlaybackState(nextState); } return ( @@ -52,13 +65,12 @@ export function AudioScreen() { /> - {Platform.OS === 'web' && audioSrc ? ( + {Platform.OS === 'web' ? ( {createElement('audio', { 'aria-label': 'Classic sample audio', controls: true, loop: false, - onEnded: () => setPlaybackState('ended'), onPause: () => { if (playbackState === 'playing') setPlaybackState('paused'); }, @@ -66,7 +78,6 @@ export function AudioScreen() { ref: (node: BrowserAudioElement | null) => { audioRef.current = node; }, - src: audioSrc, style: { width: '100%' }, 'data-testid': 'classic-audio', })} @@ -82,13 +93,26 @@ export function AudioScreen() { - - + {createElement( + 'button', + { + 'aria-label': 'Start sample', + onClick: playSample, + style: webButtonStyle(colors, 'primary'), + 'data-testid': 'start-audio', + }, + 'Start sample', + )} + {createElement( + 'button', + { + 'aria-label': 'Pause', + onClick: pauseSample, + style: webButtonStyle(colors, 'secondary'), + 'data-testid': 'pause-audio', + }, + 'Pause', + )} ) : ( @@ -101,54 +125,72 @@ export function AudioScreen() { ); } -function createClassicLoopWavUrl(): string { - const sampleRate = 11_025; - const durationSeconds = 8; - const sampleCount = sampleRate * durationSeconds; - const channels = 1; - const bytesPerSample = 2; - const dataBytes = sampleCount * channels * bytesPerSample; - const buffer = new ArrayBuffer(44 + dataBytes); - const view = new DataView(buffer); - - writeAscii(view, 0, 'RIFF'); - view.setUint32(4, 36 + dataBytes, true); - writeAscii(view, 8, 'WAVE'); - writeAscii(view, 12, 'fmt '); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, channels, true); - view.setUint32(24, sampleRate, true); - view.setUint32(28, sampleRate * channels * bytesPerSample, true); - view.setUint16(32, channels * bytesPerSample, true); - view.setUint16(34, bytesPerSample * 8, true); - writeAscii(view, 36, 'data'); - view.setUint32(40, dataBytes, true); +function webButtonStyle(colors: AppColors, kind: 'primary' | 'secondary') { + return { + backgroundColor: kind === 'primary' ? colors.text : 'transparent', + border: `1px solid ${kind === 'primary' ? colors.text : colors.lineStrong}`, + borderRadius: 4, + color: kind === 'primary' ? colors.surface : colors.text, + cursor: 'pointer', + fontSize: 15, + fontWeight: 600, + padding: '13px 16px', + }; +} +function createClassicLoopStream(onEnded: () => void): SamplePlayback { + const webkitAudio = window as Window & { webkitAudioContext?: typeof AudioContext }; + const AudioContextCtor = window.AudioContext ?? webkitAudio.webkitAudioContext; + if (!AudioContextCtor) throw new Error('Web Audio API is not available.'); + const context = new AudioContextCtor(); + const destination = context.createMediaStreamDestination(); + const master = context.createGain(); + const durationSeconds = 8; + const startAt = context.currentTime + 0.03; const melody = [196, 247, 294, 330, 392, 330, 294, 247, 220, 262, 330, 392, 494, 392, 330, 262]; - for (let index = 0; index < sampleCount; index += 1) { - const t = index / sampleRate; - const step = Math.floor(t * 4) % melody.length; - const beatPhase = (t * 4) % 1; - const envelope = Math.max(0.18, 1 - beatPhase * 0.72); - const bass = squareWave(98, t) * 0.18; - const lead = squareWave(melody[step] ?? 220, t) * 0.42 * envelope; - const hat = ((index * 17) % 31 < 2 ? 0.12 : 0) * (beatPhase < 0.08 ? 1 : 0); - const sample = Math.max(-0.82, Math.min(0.82, lead + bass + hat)); - view.setInt16(44 + index * 2, Math.round(sample * 32767), true); - } - return URL.createObjectURL(new Blob([buffer], { type: 'audio/wav' })); -} + master.gain.value = 0.55; + master.connect(destination); + void context.resume(); -function squareWave(frequency: number, t: number): number { - return Math.sin(Math.PI * 2 * frequency * t) >= 0 ? 1 : -1; + for (let step = 0; step < durationSeconds * 4; step += 1) { + const noteStart = startAt + step * 0.25; + const frequency = melody[step % melody.length] ?? 220; + scheduleTone(context, master, frequency, noteStart, 0.2, 0.42); + if (step % 4 === 0) scheduleTone(context, master, 98, noteStart, 0.22, 0.25); + if (step % 2 === 0) scheduleTone(context, master, 1760, noteStart, 0.04, 0.08); + } + + const endTimer = window.setTimeout(onEnded, durationSeconds * 1000); + return { + stream: destination.stream, + stop: () => { + window.clearTimeout(endTimer); + destination.stream.getTracks().forEach((track) => track.stop()); + void context.close(); + }, + }; } -function writeAscii(view: DataView, offset: number, value: string): void { - for (let index = 0; index < value.length; index += 1) { - view.setUint8(offset + index, value.charCodeAt(index)); - } +function scheduleTone( + context: AudioContext, + output: AudioNode, + frequency: number, + startAt: number, + duration: number, + volume: number, +): void { + const oscillator = context.createOscillator(); + const gain = context.createGain(); + oscillator.type = 'square'; + oscillator.frequency.setValueAtTime(frequency, startAt); + gain.gain.setValueAtTime(0.0001, startAt); + gain.gain.exponentialRampToValueAtTime(volume, startAt + 0.015); + gain.gain.exponentialRampToValueAtTime(0.0001, startAt + duration); + oscillator.connect(gain); + gain.connect(output); + oscillator.start(startAt); + oscillator.stop(startAt + duration + 0.01); } function createStyles(colors: AppColors) { diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index fcbb87907..1f7d98f2d 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -648,28 +648,46 @@ function resolveAudioCommandRequest(params: ObservabilityParams): bucketMs: number; } | DaemonFailureResponse { - const { req, sessionName, sessionStore } = params; - const session = sessionStore.get(sessionName); - if (!session) { - return errorResponse('SESSION_NOT_FOUND', 'audio requires an active session'); - } - if (!isCommandSupportedOnDevice('audio', session.device)) { - return errorResponse( - 'UNSUPPORTED_OPERATION', - 'audio is currently supported for web browser sessions only', - ); - } + const sessionResult = resolveAudioSession(params); + if (!sessionResult.ok) return sessionResult; + const actionResult = resolveAudioProbeAction(params.req); + if (!actionResult.ok) return actionResult; + const timingResult = resolveAudioProbeTiming(params.req, actionResult.probeAction); + if (!timingResult.ok) return timingResult; + return { ok: true, probeAction: actionResult.probeAction, ...timingResult.timing }; +} - const action = (req.positionals?.[0] ?? 'probe').toLowerCase(); - if (!AUDIO_ACTIONS.includes(action as (typeof AUDIO_ACTIONS)[number])) { - return errorResponse('INVALID_ARGS', AUDIO_ACTIONS_MESSAGE); - } +function resolveAudioSession(params: ObservabilityParams): { ok: true } | DaemonFailureResponse { + const session = params.sessionStore.get(params.sessionName); + if (!session) return errorResponse('SESSION_NOT_FOUND', 'audio requires an active session'); + return isCommandSupportedOnDevice('audio', session.device) + ? { ok: true } + : errorResponse( + 'UNSUPPORTED_OPERATION', + 'audio is currently supported for web browser sessions only', + ); +} - const probeAction = (req.positionals?.[1] ?? 'status').toLowerCase(); - if (!AUDIO_PROBE_ACTIONS.includes(probeAction as (typeof AUDIO_PROBE_ACTIONS)[number])) { - return errorResponse('INVALID_ARGS', AUDIO_PROBE_ACTIONS_MESSAGE); - } +function resolveAudioProbeAction( + req: DaemonRequest, +): { ok: true; probeAction: 'start' | 'status' | 'stop' } | DaemonFailureResponse { + const audioAction = readAudioAction(req.positionals?.[0]); + if (!audioAction) return errorResponse('INVALID_ARGS', AUDIO_ACTIONS_MESSAGE); + const probeAction = readAudioProbeAction(req.positionals?.[1]); + if (!probeAction) return errorResponse('INVALID_ARGS', AUDIO_PROBE_ACTIONS_MESSAGE); + return { ok: true, probeAction }; +} +function resolveAudioProbeTiming( + req: DaemonRequest, + probeAction: 'start' | 'status' | 'stop', +): { ok: true; timing: { durationMs: number; bucketMs: number } } | DaemonFailureResponse { + if (probeAction !== 'start' && req.positionals && req.positionals.length > 2) { + return errorResponse( + 'INVALID_ARGS', + 'audio probe duration and bucket are only supported with audio probe start', + ); + } const durationMs = readBoundedInteger(req.positionals?.[2], { defaultValue: 10_000, min: 100, @@ -685,20 +703,19 @@ function resolveAudioCommandRequest(params: ObservabilityParams): message: 'audio probe bucket must be an integer in range 100..10000 ms', }); if (bucketMs instanceof Error) return errorResponse('INVALID_ARGS', bucketMs.message); + return { ok: true, timing: { durationMs, bucketMs } }; +} - if (probeAction !== 'start' && req.positionals && req.positionals.length > 2) { - return errorResponse( - 'INVALID_ARGS', - 'audio probe duration and bucket are only supported with audio probe start', - ); - } +function readAudioAction(value: string | undefined): 'probe' | undefined { + const action = (value ?? 'probe').toLowerCase(); + return AUDIO_ACTIONS.includes(action as (typeof AUDIO_ACTIONS)[number]) ? 'probe' : undefined; +} - return { - ok: true, - probeAction: probeAction as 'start' | 'status' | 'stop', - durationMs, - bucketMs, - }; +function readAudioProbeAction(value: string | undefined): 'start' | 'status' | 'stop' | undefined { + const probeAction = (value ?? 'status').toLowerCase(); + return AUDIO_PROBE_ACTIONS.includes(probeAction as (typeof AUDIO_PROBE_ACTIONS)[number]) + ? (probeAction as 'start' | 'status' | 'stop') + : undefined; } function readBoundedInteger( diff --git a/src/platforms/web/agent-browser-provider.test.ts b/src/platforms/web/agent-browser-provider.test.ts index 449e92b05..7cc068db4 100644 --- a/src/platforms/web/agent-browser-provider.test.ts +++ b/src/platforms/web/agent-browser-provider.test.ts @@ -197,21 +197,24 @@ test('agent-browser provider probes page audio through eval', async () => { return jsonResult({ success: true, data: { - audio: 'probe', - state: 'running', - active: true, - heard: true, - source: 'media-elements', - backend: 'agent-browser', - durationMs: 7500, - elapsedMs: 1000, - bucketMs: 500, - sampleCount: 2, - mediaElementCount: 1, - sourceCount: 1, - rmsDbfs: [-30, -24], - peakDbfs: [-18, -12], - notes: ['Audio probe samples HTML media elements exposed by captureStream().'], + origin: 'http://localhost:19006', + result: { + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'media-elements', + backend: 'agent-browser', + durationMs: 7500, + elapsedMs: 1000, + bucketMs: 500, + sampleCount: 2, + mediaElementCount: 1, + sourceCount: 1, + rmsDbfs: [-30, -24], + peakDbfs: [-18, -12], + notes: ['Audio probe samples HTML media elements exposed by captureStream().'], + }, }, }); }, diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index e483b6fda..ef37de8e5 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -104,6 +104,8 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { source: options.source ?? 'media-elements', })}; const key = '__agentDeviceAudioProbe'; + const contextKey = '__agentDeviceAudioProbeContext'; + const sourceKey = '__agentDeviceAudioProbeSources'; const silenceDb = -90; const now = () => Date.now(); const dbfs = (value) => { @@ -124,32 +126,78 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { probe.active = false; probe.reason = reason; probe.stoppedAt = now(); - try { - void probe.context.close(); - } catch {} + for (const entry of probe.analysers) { + try { + entry.analyser.disconnect(); + entry.source.disconnect(); + if (entry.audible) entry.source.connect(probe.context.destination); + } catch {} + } + if (probe.resumeOnGesture) { + for (const eventName of ['click', 'pointerdown', 'keydown']) { + window.removeEventListener(eventName, probe.resumeOnGesture, true); + } + probe.resumeOnGesture = undefined; + } return probe; }; + const getContext = () => { + const AudioContextCtor = window.AudioContext || window.webkitAudioContext; + if (!AudioContextCtor) return undefined; + const existing = window[contextKey]; + if (existing && existing.state !== 'closed') return existing; + const context = new AudioContextCtor(); + window[contextKey] = context; + return context; + }; + const createElementAudioSource = (probe, element) => { + if (!element.currentSrc && !element.src && element.readyState === 0) return undefined; + try { + if (!window[sourceKey]) window[sourceKey] = new WeakMap(); + let source = window[sourceKey].get(element); + if (!source) { + source = probe.context.createMediaElementSource(element); + window[sourceKey].set(element, source); + } + source.disconnect(); + return { source, audible: true }; + } catch { + return undefined; + } + }; + const createMediaStreamAudioSource = (probe, stream) => { + if (!stream || typeof stream.getAudioTracks !== 'function') return undefined; + if (stream.getAudioTracks().length === 0) return undefined; + return { source: probe.context.createMediaStreamSource(stream), audible: false }; + }; + const createCaptureStreamAudioSource = (probe, element) => { + if (typeof element.captureStream !== 'function') return undefined; + const stream = element.captureStream(); + return createMediaStreamAudioSource(probe, stream); + }; const discover = (probe) => { const elements = mediaElements(); probe.mediaElementCount = elements.length; for (const element of elements) { if (probe.seen.has(element)) continue; - probe.seen.add(element); - if (typeof element.captureStream !== 'function') { - note(probe, 'Some media elements do not expose captureStream(), so they cannot be sampled.'); - continue; - } - const stream = element.captureStream(); - if (!stream || stream.getAudioTracks().length === 0) { - note(probe, 'Some media elements have no exposed audio track.'); + const sourceEntry = + createMediaStreamAudioSource(probe, element.srcObject) ?? + createCaptureStreamAudioSource(probe, element) ?? + createElementAudioSource(probe, element); + if (!sourceEntry) { + note(probe, 'Some media elements do not expose capturable audio to Web Audio.'); continue; } + probe.seen.add(element); const analyser = probe.context.createAnalyser(); analyser.fftSize = 2048; - const source = probe.context.createMediaStreamSource(stream); - source.connect(analyser); - analyser.connect(probe.sink); - probe.analysers.push({ analyser, buffer: new Float32Array(analyser.fftSize) }); + sourceEntry.source.connect(analyser); + analyser.connect(sourceEntry.audible ? probe.context.destination : probe.sink); + probe.analysers.push({ + ...sourceEntry, + analyser, + buffer: new Float32Array(analyser.fftSize) + }); } probe.sourceCount = probe.analysers.length; if (probe.sourceCount === 0) { @@ -201,7 +249,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { rmsDbfs: [], peakDbfs: [], notes: [ - 'Audio probe samples HTML media elements exposed by captureStream(); it is not whole-tab or system audio capture.' + 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.' ], }; } @@ -224,7 +272,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { stoppedAt: probe.stoppedAt ? new Date(probe.stoppedAt).toISOString() : undefined, reason: probe.reason, notes: [ - 'Audio probe samples HTML media elements exposed by captureStream(); it is not whole-tab or system audio capture.', + 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.', ...probe.notes, ], }; @@ -237,8 +285,8 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { } if (action === 'start') { if (probe) stopProbe(probe, 'restarted'); - const AudioContextCtor = window.AudioContext || window.webkitAudioContext; - if (!AudioContextCtor) { + const context = getContext(); + if (!context) { return { audio: 'probe', state: 'stopped', @@ -257,7 +305,6 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { notes: ['Web Audio API is not available in this browser context.'], }; } - const context = new AudioContextCtor(); const sink = context.createGain(); sink.gain.value = 0; sink.connect(context.destination); @@ -286,6 +333,19 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { } catch { note(probe, 'AudioContext could not be resumed by the probe.'); } + probe.resumeOnGesture = () => { + try { + void context.resume(); + } catch { + note(probe, 'AudioContext could not be resumed from a user gesture.'); + } + }; + for (const eventName of ['click', 'pointerdown', 'keydown']) { + window.addEventListener(eventName, probe.resumeOnGesture, { + capture: true, + once: true, + }); + } discover(probe); probe.timer = setInterval(() => sample(probe), probe.bucketMs); probe.timeout = setTimeout(() => stopProbe(probe, 'duration'), probe.durationMs); @@ -302,20 +362,20 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { } function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { - const record = isJsonObject(data) ? data : {}; + const record = readAgentBrowserEvalResultRecord(data); return { audio: 'probe', - state: readStringProperty(record, 'state') === 'running' ? 'running' : 'stopped', + state: readAudioProbeState(record), active: readBooleanProperty(record, 'active') === true, heard: readBooleanProperty(record, 'heard') === true, source: 'media-elements', - backend: readStringProperty(record, 'backend') ?? 'agent-browser', - durationMs: readNumberProperty(record, 'durationMs') ?? 0, - elapsedMs: readNumberProperty(record, 'elapsedMs') ?? 0, - bucketMs: readNumberProperty(record, 'bucketMs') ?? 1000, - sampleCount: readNumberProperty(record, 'sampleCount') ?? 0, - mediaElementCount: readNumberProperty(record, 'mediaElementCount') ?? 0, - sourceCount: readNumberProperty(record, 'sourceCount') ?? 0, + backend: readStringPropertyWithDefault(record, 'backend', 'agent-browser'), + durationMs: readNumberPropertyWithDefault(record, 'durationMs', 0), + elapsedMs: readNumberPropertyWithDefault(record, 'elapsedMs', 0), + bucketMs: readNumberPropertyWithDefault(record, 'bucketMs', 1000), + sampleCount: readNumberPropertyWithDefault(record, 'sampleCount', 0), + mediaElementCount: readNumberPropertyWithDefault(record, 'mediaElementCount', 0), + sourceCount: readNumberPropertyWithDefault(record, 'sourceCount', 0), rmsDbfs: readNumberArray(record.rmsDbfs), peakDbfs: readNumberArray(record.peakDbfs), startedAt: readStringProperty(record, 'startedAt'), @@ -325,6 +385,25 @@ function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResu }; } +function readAgentBrowserEvalResultRecord(data: unknown): JsonObject { + if (!isJsonObject(data)) return {}; + return isJsonObject(data.result) ? data.result : data; +} + +function readAudioProbeState(record: JsonObject): 'running' | 'stopped' { + return readStringProperty(record, 'state') === 'running' ? 'running' : 'stopped'; +} + +function readStringPropertyWithDefault(record: JsonObject, key: string, fallback: string): string { + const value = readStringProperty(record, key); + return value === undefined ? fallback : value; +} + +function readNumberPropertyWithDefault(record: JsonObject, key: string, fallback: number): number { + const value = readNumberProperty(record, key); + return value === undefined ? fallback : value; +} + function readNumberArray(value: unknown): number[] { return Array.isArray(value) ? value.filter((item): item is number => typeof item === 'number' && Number.isFinite(item)) From dcdc3d874e48e89b017c18c29b15d88fabf6d747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 13:20:32 +0200 Subject: [PATCH 03/18] test: cover audio probe review gaps --- .../__tests__/session-observability.test.ts | 109 +++++++++- .../web/agent-browser-provider.test.ts | 191 ++++++++++++++++++ src/platforms/web/agent-browser-provider.ts | 8 +- src/utils/cli-help.ts | 3 +- 4 files changed, 308 insertions(+), 3 deletions(-) diff --git a/src/daemon/handlers/__tests__/session-observability.test.ts b/src/daemon/handlers/__tests__/session-observability.test.ts index b48494575..12e760807 100644 --- a/src/daemon/handlers/__tests__/session-observability.test.ts +++ b/src/daemon/handlers/__tests__/session-observability.test.ts @@ -5,10 +5,16 @@ import path from 'node:path'; import { beforeEach, test, vi } from 'vitest'; import type { AndroidAdbExecutor } from '../../../platforms/android/adb-executor.ts'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { makeAndroidSession, makeIosSession } from '../../../__tests__/test-utils/index.ts'; +import { + makeAndroidSession, + makeIosSession, + makeSession, + WEB_DESKTOP_DEVICE, +} from '../../../__tests__/test-utils/index.ts'; import { AppError } from '../../../utils/errors.ts'; import type { AppleXctracePerfCapture } from '../../../platforms/ios/perf-xctrace.ts'; import type { DaemonResponse } from '../../types.ts'; +import { withWebProvider, type WebProvider } from '../../../platforms/web/provider.ts'; const applePerfMocks = vi.hoisted(() => ({ startAppleXctracePerfCapture: vi.fn(), @@ -411,6 +417,43 @@ test('network dump accepts explicit include flag and rejects conflicting values' } }); +test('audio probe validates daemon duration bounds', async () => { + const provider = makeAudioWebProvider(); + const response = await runAudioCommand(['probe', 'start', '99', '1000'], provider); + + assertInvalidArgs(response, /duration must be an integer in range 100..120000/); + assert.equal(provider.probeAudio.mock.calls.length, 0); +}); + +test('audio probe validates daemon bucket bounds', async () => { + const provider = makeAudioWebProvider(); + const response = await runAudioCommand(['probe', 'start', '1000', '99'], provider); + + assertInvalidArgs(response, /bucket must be an integer in range 100..10000/); + assert.equal(provider.probeAudio.mock.calls.length, 0); +}); + +test('audio probe rejects timing positionals for status', async () => { + const provider = makeAudioWebProvider(); + const response = await runAudioCommand(['probe', 'status', '1000'], provider); + + assertInvalidArgs(response, /only supported with audio probe start/); + assert.equal(provider.probeAudio.mock.calls.length, 0); +}); + +test('audio probe forwards daemon millisecond timing to web provider', async () => { + const provider = makeAudioWebProvider(); + const response = await runAudioCommand(['probe', 'start', '7500', '500'], provider); + + assert.equal(response?.ok, true); + assert.deepEqual(provider.probeAudio.mock.calls[0]?.[0], { + action: 'start', + durationMs: 7500, + bucketMs: 500, + source: 'media-elements', + }); +}); + test('perf memory sample routes to memory-only Android sampler', async () => { const sessionStore = makeSessionStore('agent-device-session-observability-'); sessionStore.set('android', { @@ -937,6 +980,70 @@ function readAndroidNativePerfState( return sessionStore.get(sessionName)?.nativePerf?.android?.state; } +async function runAudioCommand( + positionals: string[], + provider: WebProvider = makeAudioWebProvider(), +): Promise { + const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); + sessionStore.set('web', makeSession('web', { device: WEB_DESKTOP_DEVICE })); + return await withWebProvider( + provider, + async () => + await handleSessionObservabilityCommands({ + req: { + token: 't', + session: 'web', + command: 'audio', + positionals, + flags: {}, + }, + sessionName: 'web', + sessionStore, + }), + ); +} + +function assertInvalidArgs(response: DaemonResponse | null, message: RegExp): void { + assert.equal(response?.ok, false); + if (response && !response.ok) { + assert.equal(response.error.code, 'INVALID_ARGS'); + assert.match(response.error.message, message); + } +} + +function makeAudioWebProvider(): WebProvider & { + probeAudio: ReturnType>>; +} { + const probeAudio = vi.fn>(async (options) => ({ + audio: 'probe', + state: options.action === 'start' ? 'running' : 'stopped', + active: options.action === 'start', + heard: false, + source: 'media-elements', + backend: 'test', + durationMs: options.durationMs ?? 10_000, + elapsedMs: 0, + bucketMs: options.bucketMs ?? 1_000, + sampleCount: 0, + mediaElementCount: 0, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + })); + return { + open: async () => {}, + close: async () => {}, + snapshot: async () => ({ nodes: [] }), + screenshot: async () => {}, + setViewport: async () => {}, + click: async () => {}, + fill: async () => {}, + typeText: async () => {}, + scroll: async () => {}, + probeAudio, + }; +} + type MockAdbResult = Awaited>; type MockAdbResponder = { diff --git a/src/platforms/web/agent-browser-provider.test.ts b/src/platforms/web/agent-browser-provider.test.ts index 7cc068db4..ef51ff5f5 100644 --- a/src/platforms/web/agent-browser-provider.test.ts +++ b/src/platforms/web/agent-browser-provider.test.ts @@ -254,6 +254,63 @@ test('agent-browser provider probes page audio through eval', async () => { }); }); +test('agent-browser provider generated audio probe script samples streams discovered after start', async () => { + await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => { + const calls: AgentBrowserCall[] = []; + const page = createAudioProbeScriptPage(); + const executor = async (cmd: string, args: string[]): Promise => { + calls.push({ cmd, args }); + assert.equal(args[0], 'eval'); + const script = args[1]; + if (typeof script !== 'string') throw new Error('Expected generated eval script'); + if (calls.length === 2) page.audio.srcObject = page.stream; + return jsonResult({ + success: true, + data: { + origin: 'http://localhost:19006', + result: page.evaluate(script), + }, + }); + }; + + const start = await withCommandExecutorOverride( + executor, + async () => + await provider.probeAudio?.({ + action: 'start', + durationMs: 5000, + bucketMs: 1000, + source: 'media-elements', + }), + ); + const status = await withCommandExecutorOverride( + executor, + async () => + await provider.probeAudio?.({ + action: 'status', + durationMs: 5000, + bucketMs: 1000, + source: 'media-elements', + }), + ); + + assert.equal(start?.sourceCount, 0); + assert.equal(start?.sampleCount, 0); + assert.equal(status?.heard, true); + assert.equal(status?.sourceCount, 1); + assert.deepEqual(status?.rmsDbfs, [-6]); + assert.deepEqual(status?.peakDbfs, [-6]); + assert.equal(page.createdStreamSources, 1); + assert.deepEqual( + calls.map((call) => call.args.slice(0, 3)), + [ + ['eval', calls[0]?.args[1] ?? '', '--json'], + ['eval', calls[1]?.args[1] ?? '', '--json'], + ], + ); + }); +}); + test('agent-browser provider surfaces stale ref failures during requested snapshot geometry lookup', async () => { await withManagedAgentBrowserProvider({ session: 'web-session' }, async (provider) => { await assert.rejects( @@ -483,3 +540,137 @@ function jsonResult(value: unknown, exitCode = 0): ExecResult { function expectedSelectAllShortcut(): string { return process.platform === 'darwin' ? 'Meta+a' : 'Control+a'; } + +type AudioProbeScriptPage = { + audio: { currentSrc: string; readyState: number; src: string; srcObject: FakeMediaStream | null }; + createdStreamSources: number; + evaluate: (script: string) => unknown; + stream: FakeMediaStream; +}; + +type FakeMediaStream = { + getAudioTracks: () => Array>; +}; + +type FakeTimer = { + active: boolean; +}; + +class FakeAudioNode { + readonly connections: FakeAudioNode[] = []; + + connect(target: FakeAudioNode): FakeAudioNode { + this.connections.push(target); + return target; + } + + disconnect(): void { + this.connections.length = 0; + } +} + +class FakeAnalyserNode extends FakeAudioNode { + fftSize = 0; + + getFloatTimeDomainData(buffer: Float32Array): void { + buffer.fill(0.5); + } +} + +class FakeAudioContext { + readonly destination = new FakeAudioNode(); + private readonly onMediaStreamSource: () => void; + state: 'running' | 'closed' = 'running'; + + constructor(onMediaStreamSource: () => void) { + this.onMediaStreamSource = onMediaStreamSource; + } + + createAnalyser(): FakeAnalyserNode { + return new FakeAnalyserNode(); + } + + createGain(): FakeAudioNode & { gain: { value: number } } { + return Object.assign(new FakeAudioNode(), { gain: { value: 1 } }); + } + + createMediaElementSource(): FakeAudioNode { + return new FakeAudioNode(); + } + + createMediaStreamSource(): FakeAudioNode { + this.onMediaStreamSource(); + return new FakeAudioNode(); + } + + close(): Promise { + this.state = 'closed'; + return Promise.resolve(); + } + + resume(): Promise { + this.state = 'running'; + return Promise.resolve(); + } +} + +function createAudioProbeScriptPage(): AudioProbeScriptPage { + const timers = new Set(); + const audio = { + currentSrc: '', + readyState: 0, + src: '', + srcObject: null as FakeMediaStream | null, + }; + const stream = { getAudioTracks: () => [{}] }; + let createdStreamSources = 0; + const windowObject: Record = { + addEventListener: () => {}, + removeEventListener: () => {}, + }; + windowObject.AudioContext = class extends FakeAudioContext { + constructor() { + super(() => { + createdStreamSources += 1; + }); + } + }; + const documentObject = { + querySelectorAll: (selector: string) => (selector === 'audio,video' ? [audio] : []), + }; + const setTimer = (): FakeTimer => { + const timer = { active: true }; + timers.add(timer); + return timer; + }; + const clearTimer = (timer: FakeTimer | undefined): void => { + if (timer) timer.active = false; + }; + + return { + audio, + get createdStreamSources() { + return createdStreamSources; + }, + stream, + evaluate(script: string): unknown { + const run = new Function( + 'window', + 'document', + 'setInterval', + 'clearInterval', + 'setTimeout', + 'clearTimeout', + `return ${script};`, + ) as ( + window: Record, + document: typeof documentObject, + setInterval: () => FakeTimer, + clearInterval: (timer: FakeTimer | undefined) => void, + setTimeout: () => FakeTimer, + clearTimeout: (timer: FakeTimer | undefined) => void, + ) => unknown; + return run(windowObject, documentObject, setTimer, clearTimer, setTimer, clearTimer); + }, + }; +} diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index ef37de8e5..2875d0498 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -156,6 +156,9 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { if (!window[sourceKey]) window[sourceKey] = new WeakMap(); let source = window[sourceKey].get(element); if (!source) { + // createMediaElementSource permanently moves this element through the + // shared probe AudioContext. We keep that context open and reconnect the + // source to destination on stop so audible playback survives the probe. source = probe.context.createMediaElementSource(element); window[sourceKey].set(element, source); } @@ -261,7 +264,10 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { source: 'media-elements', backend: 'agent-browser', durationMs: probe.durationMs, - elapsedMs: Math.max(0, Math.min(now() - probe.startedAt, probe.durationMs)), + elapsedMs: Math.max( + 0, + Math.min((probe.stoppedAt || now()) - probe.startedAt, probe.durationMs) + ), bucketMs: probe.bucketMs, sampleCount: probe.rmsDbfs.length, mediaElementCount: mediaCount, diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 7f25d7f13..733958796 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -270,7 +270,7 @@ Validation and evidence: Remote lifecycle: cloud, remote-config, direct proxy, and limrun use the same flow: connect, open, commands, close, disconnect. Remote config profile: agent-device connect --remote-config ./remote-config.json; then run normal commands and disconnect. Direct proxy to a Mac you control: cloud/Linux clients can use local/proxy iOS devices through the proxied Mac. Run agent-device connect proxy --daemon-base-url first. Device leases are automatic on open and expire after five minutes of inactivity. - Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. + Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. agent-device web setup agent-device web doctor agent-device open https://example.com --platform web @@ -755,6 +755,7 @@ Planning rule: For web command plans, output only agent-device command lines. Do not add prose, numbering, Markdown fences, shell pipes, or agent-browser commands unless the task is explicitly standalone browser automation outside agent-device. First-slice loop: + Audio probe start uses duration seconds first, then bucket milliseconds. agent-device web setup agent-device web doctor agent-device open https://example.com --platform web From ce7eefd7cf14058e2cc9aa97ae0d10b0bb7adcb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 13:37:49 +0200 Subject: [PATCH 04/18] fix: address audio probe review feedback --- examples/test-app/src/screens/AudioScreen.tsx | 149 ++++++++---------- .../__tests__/session-observability.test.ts | 22 +++ src/platforms/web/agent-browser-provider.ts | 13 +- src/utils/cli-help.ts | 2 +- 4 files changed, 93 insertions(+), 93 deletions(-) diff --git a/examples/test-app/src/screens/AudioScreen.tsx b/examples/test-app/src/screens/AudioScreen.tsx index a67a40334..ba360045e 100644 --- a/examples/test-app/src/screens/AudioScreen.tsx +++ b/examples/test-app/src/screens/AudioScreen.tsx @@ -1,7 +1,7 @@ import { createElement, useEffect, useRef, useState } from 'react'; import { Platform, ScrollView, StyleSheet, Text, View } from 'react-native'; -import { InlineBadge, ScreenTitle, SectionCard } from '../components'; +import { ActionButton, InlineBadge, ScreenTitle, SectionCard } from '../components'; import { useAppColors, type AppColors } from '../theme'; type BrowserAudioElement = { @@ -20,31 +20,44 @@ export function AudioScreen() { const styles = createStyles(colors); const audioRef = useRef(null); const playbackRef = useRef(null); - const [playbackState, setPlaybackState] = useState<'ready' | 'playing' | 'paused' | 'ended'>( - 'ready', - ); + const [playbackState, setPlaybackState] = useState< + 'ready' | 'playing' | 'paused' | 'ended' | 'error' + >('ready'); useEffect(() => { - return () => stopSample('ended'); + return () => { + playbackRef.current?.stop(); + playbackRef.current = null; + const audio = audioRef.current; + if (audio) { + audio.pause(); + audio.srcObject = null; + } + }; }, []); function playSample() { const audio = audioRef.current; if (!audio) return; stopSample('ready'); - const playback = createClassicLoopStream(() => { + const playback = createBeepStream(() => { stopSample('ended'); }); playbackRef.current = playback; audio.srcObject = playback.stream; - void audio.play().then(() => setPlaybackState('playing')); + void audio + .play() + .then(() => setPlaybackState('playing')) + .catch(() => { + stopSample('error'); + }); } function pauseSample() { stopSample('paused'); } - function stopSample(nextState: 'ready' | 'paused' | 'ended') { + function stopSample(nextState: 'ready' | 'paused' | 'ended' | 'error') { const audio = audioRef.current; playbackRef.current?.stop(); playbackRef.current = null; @@ -64,55 +77,45 @@ export function AudioScreen() { testID="audio-title" /> - + {Platform.OS === 'web' ? ( - + {createElement('audio', { - 'aria-label': 'Classic sample audio', + 'aria-label': 'Sample audio', controls: true, loop: false, onPause: () => { - if (playbackState === 'playing') setPlaybackState('paused'); + if (playbackRef.current) setPlaybackState('paused'); }, onPlay: () => setPlaybackState('playing'), ref: (node: BrowserAudioElement | null) => { audioRef.current = node; }, style: { width: '100%' }, - 'data-testid': 'classic-audio', + 'data-testid': 'sample-audio', })} - + - - {playbackState} - - {createElement( - 'button', - { - 'aria-label': 'Start sample', - onClick: playSample, - style: webButtonStyle(colors, 'primary'), - 'data-testid': 'start-audio', - }, - 'Start sample', - )} - {createElement( - 'button', - { - 'aria-label': 'Pause', - onClick: pauseSample, - style: webButtonStyle(colors, 'secondary'), - 'data-testid': 'pause-audio', - }, - 'Pause', - )} + + ) : ( @@ -125,74 +128,50 @@ export function AudioScreen() { ); } -function webButtonStyle(colors: AppColors, kind: 'primary' | 'secondary') { - return { - backgroundColor: kind === 'primary' ? colors.text : 'transparent', - border: `1px solid ${kind === 'primary' ? colors.text : colors.lineStrong}`, - borderRadius: 4, - color: kind === 'primary' ? colors.surface : colors.text, - cursor: 'pointer', - fontSize: 15, - fontWeight: 600, - padding: '13px 16px', - }; +function playbackLabel(state: 'ready' | 'playing' | 'paused' | 'ended' | 'error'): string { + return state === 'error' ? 'Playback blocked' : state === 'playing' ? 'Playing' : state; } -function createClassicLoopStream(onEnded: () => void): SamplePlayback { +function createBeepStream(onEnded: () => void): SamplePlayback { const webkitAudio = window as Window & { webkitAudioContext?: typeof AudioContext }; const AudioContextCtor = window.AudioContext ?? webkitAudio.webkitAudioContext; if (!AudioContextCtor) throw new Error('Web Audio API is not available.'); const context = new AudioContextCtor(); const destination = context.createMediaStreamDestination(); - const master = context.createGain(); - const durationSeconds = 8; + const oscillator = context.createOscillator(); + const gain = context.createGain(); + const durationSeconds = 6; const startAt = context.currentTime + 0.03; - const melody = [196, 247, 294, 330, 392, 330, 294, 247, 220, 262, 330, 392, 494, 392, 330, 262]; + const endAt = startAt + durationSeconds; - master.gain.value = 0.55; - master.connect(destination); + oscillator.type = 'square'; + oscillator.frequency.setValueAtTime(440, startAt); + gain.gain.setValueAtTime(0.0001, startAt); + gain.gain.exponentialRampToValueAtTime(0.35, startAt + 0.05); + gain.gain.setValueAtTime(0.35, endAt - 0.08); + gain.gain.exponentialRampToValueAtTime(0.0001, endAt); + oscillator.connect(gain); + gain.connect(destination); + oscillator.start(startAt); + oscillator.stop(endAt + 0.01); void context.resume(); - for (let step = 0; step < durationSeconds * 4; step += 1) { - const noteStart = startAt + step * 0.25; - const frequency = melody[step % melody.length] ?? 220; - scheduleTone(context, master, frequency, noteStart, 0.2, 0.42); - if (step % 4 === 0) scheduleTone(context, master, 98, noteStart, 0.22, 0.25); - if (step % 2 === 0) scheduleTone(context, master, 1760, noteStart, 0.04, 0.08); - } - const endTimer = window.setTimeout(onEnded, durationSeconds * 1000); return { stream: destination.stream, stop: () => { window.clearTimeout(endTimer); + try { + oscillator.stop(); + } catch { + // The scheduled stop may already have fired. + } destination.stream.getTracks().forEach((track) => track.stop()); void context.close(); }, }; } -function scheduleTone( - context: AudioContext, - output: AudioNode, - frequency: number, - startAt: number, - duration: number, - volume: number, -): void { - const oscillator = context.createOscillator(); - const gain = context.createGain(); - oscillator.type = 'square'; - oscillator.frequency.setValueAtTime(frequency, startAt); - gain.gain.setValueAtTime(0.0001, startAt); - gain.gain.exponentialRampToValueAtTime(volume, startAt + 0.015); - gain.gain.exponentialRampToValueAtTime(0.0001, startAt + duration); - oscillator.connect(gain); - gain.connect(output); - oscillator.start(startAt); - oscillator.stop(startAt + duration + 0.01); -} - function createStyles(colors: AppColors) { return StyleSheet.create({ content: { diff --git a/src/daemon/handlers/__tests__/session-observability.test.ts b/src/daemon/handlers/__tests__/session-observability.test.ts index 12e760807..0c237880b 100644 --- a/src/daemon/handlers/__tests__/session-observability.test.ts +++ b/src/daemon/handlers/__tests__/session-observability.test.ts @@ -425,6 +425,28 @@ test('audio probe validates daemon duration bounds', async () => { assert.equal(provider.probeAudio.mock.calls.length, 0); }); +test('audio probe rejects non-web sessions in daemon handler', async () => { + const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); + sessionStore.set('android', makeAndroidSession('android')); + const response = await handleSessionObservabilityCommands({ + req: { + token: 't', + session: 'android', + command: 'audio', + positionals: ['probe', 'status'], + flags: {}, + }, + sessionName: 'android', + sessionStore, + }); + + assert.equal(response?.ok, false); + if (response && !response.ok) { + assert.equal(response.error.code, 'UNSUPPORTED_OPERATION'); + assert.match(response.error.message, /web browser sessions only/); + } +}); + test('audio probe validates daemon bucket bounds', async () => { const provider = makeAudioWebProvider(); const response = await runAudioCommand(['probe', 'start', '1000', '99'], provider); diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index 2875d0498..28a3bc6ed 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -115,6 +115,10 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const note = (probe, message) => { if (!probe.notes.includes(message)) probe.notes.push(message); }; + const scopeNote = + 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.'; + const routingNote = + 'URL-backed media elements may be routed through the probe AudioContext while they are observed.'; const mediaElements = () => Array.from(document.querySelectorAll('audio,video')); const stopProbe = (probe, reason) => { if (!probe || probe.state === 'stopped') return probe; @@ -251,9 +255,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { sourceCount: 0, rmsDbfs: [], peakDbfs: [], - notes: [ - 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.' - ], + notes: [scopeNote, routingNote], }; } return { @@ -277,10 +279,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { startedAt: new Date(probe.startedAt).toISOString(), stoppedAt: probe.stoppedAt ? new Date(probe.stoppedAt).toISOString() : undefined, reason: probe.reason, - notes: [ - 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.', - ...probe.notes, - ], + notes: [scopeNote, routingNote, ...probe.notes], }; }; const action = options.action || 'status'; diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 733958796..e88bce6a0 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -270,7 +270,7 @@ Validation and evidence: Remote lifecycle: cloud, remote-config, direct proxy, and limrun use the same flow: connect, open, commands, close, disconnect. Remote config profile: agent-device connect --remote-config ./remote-config.json; then run normal commands and disconnect. Direct proxy to a Mac you control: cloud/Linux clients can use local/proxy iOS devices through the proxied Mac. Run agent-device connect proxy --daemon-base-url first. Device leases are automatic on open and expire after five minutes of inactivity. - Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. + Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. Audio probe samples HTML media elements, and URL-backed media may be routed through the probe AudioContext while observed. agent-device web setup agent-device web doctor agent-device open https://example.com --platform web From 51c0422e075f32ec818dc183081e968c491a9a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 14:24:39 +0200 Subject: [PATCH 05/18] feat: support macOS audio probe --- .../AgentDeviceMacOSHelper/AudioProbe.swift | 302 ++++++++++++++++++ .../Sources/AgentDeviceMacOSHelper/main.swift | 30 ++ src/client-types.ts | 2 +- src/commands/observability/index.ts | 8 +- src/core/capabilities.ts | 3 +- .../__tests__/session-observability.test.ts | 126 +++++++- src/daemon/handlers/session-close.ts | 2 + src/daemon/handlers/session-observability.ts | 263 ++++++++++++++- src/daemon/session-teardown.ts | 9 + src/daemon/types.ts | 9 + src/platforms/ios/macos-helper.ts | 27 +- src/utils/cli-help.ts | 3 +- 12 files changed, 771 insertions(+), 13 deletions(-) create mode 100644 macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift diff --git a/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift b/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift new file mode 100644 index 000000000..c3d34b790 --- /dev/null +++ b/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift @@ -0,0 +1,302 @@ +import AVFoundation +import AudioToolbox +import CoreMedia +import Foundation +import ScreenCaptureKit + +private let audioProbeSilenceDb = -90 + +struct AudioProbeResponse: Codable { + let audio: String + let state: String + let active: Bool + let heard: Bool + let source: String + let backend: String + let durationMs: Int + let elapsedMs: Int + let bucketMs: Int + let sampleCount: Int + let sourceCount: Int + let rmsDbfs: [Int] + let peakDbfs: [Int] + let startedAt: String + let stoppedAt: String? + let reason: String? + let notes: [String] +} + +private struct AudioProbeBucket { + var totalSquares: Double = 0 + var totalSamples: Int = 0 + var peak: Double = 0 +} + +private final class AudioProbeStatusWriter { + private let outPath: String + private let startedAt = Date() + private let durationMs: Int + private let bucketMs: Int + private let lock = NSLock() + private var current = AudioProbeBucket() + private var rmsDbfs: [Int] = [] + private var peakDbfs: [Int] = [] + private var heard = false + private var stoppedAt: Date? + private var reason: String? + + init(outPath: String, durationMs: Int, bucketMs: Int) { + self.outPath = outPath + self.durationMs = durationMs + self.bucketMs = bucketMs + } + + func add(sampleBuffer: CMSampleBuffer) { + guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else { + return + } + let format = AVAudioFormat(cmAudioFormatDescription: formatDescription) + let frameCount = AVAudioFrameCount(CMSampleBufferGetNumSamples(sampleBuffer)) + guard frameCount > 0, + let pcmBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) + else { + return + } + let status = CMSampleBufferCopyPCMDataIntoAudioBufferList( + sampleBuffer, + at: 0, + frameCount: Int32(frameCount), + into: pcmBuffer.mutableAudioBufferList + ) + guard status == noErr else { + return + } + pcmBuffer.frameLength = frameCount + add(audioBufferList: pcmBuffer.mutableAudioBufferList, format: format) + } + + func flushRunning() throws { + lock.lock() + appendCurrentBucket() + let response = buildResponse(state: "running", active: true) + lock.unlock() + try write(response) + } + + func finish(reason: String) throws { + lock.lock() + if current.totalSamples > 0 || rmsDbfs.isEmpty { + appendCurrentBucket() + } + self.stoppedAt = Date() + self.reason = reason + let response = buildResponse(state: "stopped", active: false) + lock.unlock() + try write(response) + } + + private func add(audioBufferList: UnsafeMutablePointer, format: AVAudioFormat) { + let streamDescription = format.streamDescription.pointee + let bitsPerChannel = Int(streamDescription.mBitsPerChannel) + guard bitsPerChannel > 0 else { + return + } + let bytesPerSample = max(1, bitsPerChannel / 8) + let isFloat = (streamDescription.mFormatFlags & kAudioFormatFlagIsFloat) != 0 + let isSignedInteger = (streamDescription.mFormatFlags & kAudioFormatFlagIsSignedInteger) != 0 + var bucket = AudioProbeBucket() + for buffer in UnsafeMutableAudioBufferListPointer(audioBufferList) { + guard let data = buffer.mData else { + continue + } + let sampleCount = Int(buffer.mDataByteSize) / bytesPerSample + if isFloat && bytesPerSample == 4 { + let samples = data.bindMemory(to: Float.self, capacity: sampleCount) + for index in 0.. 0 else { + return + } + lock.lock() + current.totalSquares += bucket.totalSquares + current.totalSamples += bucket.totalSamples + current.peak = max(current.peak, bucket.peak) + lock.unlock() + } + + private func appendCurrentBucket() { + let rms = current.totalSamples > 0 + ? sqrt(current.totalSquares / Double(current.totalSamples)) + : 0 + let rmsDb = dbfs(rms) + let peakDb = dbfs(current.peak) + rmsDbfs.append(rmsDb) + peakDbfs.append(peakDb) + if rmsDb > audioProbeSilenceDb || peakDb > audioProbeSilenceDb { + heard = true + } + current = AudioProbeBucket() + } + + private func buildResponse(state: String, active: Bool) -> AudioProbeResponse { + let end = stoppedAt ?? Date() + let elapsed = min(durationMs, max(0, Int(end.timeIntervalSince(startedAt) * 1000))) + return AudioProbeResponse( + audio: "probe", + state: state, + active: active, + heard: heard, + source: "system-audio", + backend: "macos-screencapturekit", + durationMs: durationMs, + elapsedMs: elapsed, + bucketMs: bucketMs, + sampleCount: rmsDbfs.count, + sourceCount: 1, + rmsDbfs: rmsDbfs, + peakDbfs: peakDbfs, + startedAt: iso8601(startedAt), + stoppedAt: stoppedAt.map(iso8601), + reason: reason, + notes: [ + "Audio probe samples macOS system audio through ScreenCaptureKit; it is not app-instrumented audio.", + "Screen Recording permission is required for macOS system audio capture.", + ] + ) + } + + private func write(_ response: AudioProbeResponse) throws { + let outputURL = URL(fileURLWithPath: outPath) + try FileManager.default.createDirectory( + at: outputURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + let data = try JSONEncoder().encode(response) + try data.write(to: outputURL, options: .atomic) + } +} + +private final class AudioProbeStreamOutput: NSObject, SCStreamOutput { + private let writer: AudioProbeStatusWriter + + init(writer: AudioProbeStatusWriter) { + self.writer = writer + } + + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard type == .audio else { + return + } + writer.add(sampleBuffer: sampleBuffer) + } +} + +extension AudioProbeBucket { + fileprivate mutating func add(_ value: Double) { + guard value.isFinite else { + return + } + let clipped = min(1, max(-1, value)) + totalSquares += clipped * clipped + totalSamples += 1 + peak = max(peak, abs(clipped)) + } +} + +func runAudioProbe(durationMs: Int, bucketMs: Int, outPath: String) throws -> AudioProbeResponse { + guard #available(macOS 13.0, *) else { + throw HelperError.commandFailed("audio probe requires macOS 13 or newer") + } + let semaphore = DispatchSemaphore(value: 0) + var response: AudioProbeResponse? + var runError: Error? + Task { + do { + response = try await runAudioProbeAsync(durationMs: durationMs, bucketMs: bucketMs, outPath: outPath) + } catch { + runError = error + } + semaphore.signal() + } + semaphore.wait() + if let runError { + throw runError + } + guard let response else { + throw HelperError.commandFailed("audio probe failed") + } + return response +} + +@available(macOS 13.0, *) +private func runAudioProbeAsync(durationMs: Int, bucketMs: Int, outPath: String) async throws -> AudioProbeResponse { + let content: SCShareableContent + do { + content = try await SCShareableContent.current + } catch { + throw HelperError.commandFailed( + "audio probe requires Screen Recording permission on macOS", + details: ["permission": "screen-recording", "error": error.localizedDescription] + ) + } + guard let display = content.displays.first else { + throw HelperError.commandFailed("audio probe could not resolve a macOS display") + } + + let configuration = SCStreamConfiguration() + configuration.width = max(2, display.width) + configuration.height = max(2, display.height) + configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1) + configuration.queueDepth = 1 + configuration.capturesAudio = true + configuration.sampleRate = 48_000 + configuration.channelCount = 2 + configuration.excludesCurrentProcessAudio = true + + let filter = SCContentFilter(display: display, excludingWindows: []) + let writer = AudioProbeStatusWriter(outPath: outPath, durationMs: durationMs, bucketMs: bucketMs) + let output = AudioProbeStreamOutput(writer: writer) + let stream = SCStream(filter: filter, configuration: configuration, delegate: nil) + let queue = DispatchQueue(label: "com.callstack.agent-device.audio-probe") + try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: queue) + try await stream.startCapture() + try writer.flushRunning() + + let deadline = Date().addingTimeInterval(Double(durationMs) / 1000) + while Date() < deadline { + let remainingMs = Int(deadline.timeIntervalSinceNow * 1000) + let sleepMs = max(1, min(bucketMs, remainingMs)) + try await Task.sleep(nanoseconds: UInt64(sleepMs) * 1_000_000) + try writer.flushRunning() + } + + try await stream.stopCapture() + try writer.finish(reason: "completed") + let data = try Data(contentsOf: URL(fileURLWithPath: outPath)) + return try JSONDecoder().decode(AudioProbeResponse.self, from: data) +} + +private func dbfs(_ value: Double) -> Int { + if !value.isFinite || value <= 0 { + return audioProbeSilenceDb + } + let db = Int((20 * log10(value)).rounded()) + return max(audioProbeSilenceDb, min(0, db)) +} + +private func iso8601(_ date: Date) -> String { + ISO8601DateFormatter().string(from: date) +} diff --git a/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 12f02ca66..e2069072c 100644 --- a/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -120,6 +120,8 @@ struct AgentDeviceMacOSHelper { return try handlePress(arguments: Array(arguments.dropFirst())) case "screenshot": return try handleScreenshot(arguments: Array(arguments.dropFirst())) + case "audio-probe": + return try handleAudioProbe(arguments: Array(arguments.dropFirst())) default: throw HelperError.invalidArgs("unknown command: \(command)") } @@ -390,6 +392,27 @@ struct AgentDeviceMacOSHelper { try captureSurfaceScreenshot(surface: surface, outPath: outPath, fullscreen: fullscreen) return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface, fullscreen: fullscreen)) } + + static func handleAudioProbe(arguments: [String]) throws -> any Encodable { + let durationMs = intOption(arguments: arguments, name: "--duration-ms") ?? 10_000 + let bucketMs = intOption(arguments: arguments, name: "--bucket-ms") ?? 1_000 + guard durationMs >= 100, durationMs <= 120_000 else { + throw HelperError.invalidArgs("audio-probe --duration-ms must be in range 100..120000") + } + guard bucketMs >= 100, bucketMs <= 10_000 else { + throw HelperError.invalidArgs("audio-probe --bucket-ms must be in range 100..10000") + } + guard let outPath = optionValue(arguments: arguments, name: "--out")? + .trimmingCharacters(in: .whitespacesAndNewlines), + !outPath.isEmpty + else { + throw HelperError.invalidArgs("audio-probe requires --out ") + } + + return SuccessEnvelope( + data: try runAudioProbe(durationMs: durationMs, bucketMs: bucketMs, outPath: outPath) + ) + } } private func optionValue(arguments: [String], name: String) -> String? { @@ -399,6 +422,13 @@ private func optionValue(arguments: [String], name: String) -> String? { return arguments[index + 1] } +private func intOption(arguments: [String], name: String) -> Int? { + guard let value = optionValue(arguments: arguments, name: name) else { + return nil + } + return Int(value) +} + private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String { let targetApp: NSRunningApplication? if surface == "frontmost-app" || (surface == nil && bundleId != nil) { diff --git a/src/client-types.ts b/src/client-types.ts index 800e59563..fdfae9bcd 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -792,7 +792,7 @@ export type AudioOptions = AgentDeviceRequestOverrides & { probeAction?: 'start' | 'status' | 'stop'; durationMs?: number; bucketMs?: number; - source?: 'media-elements'; + source?: 'media-elements' | 'system-audio'; }; export type RecordOptions = AgentDeviceRequestOverrides & { diff --git a/src/commands/observability/index.ts b/src/commands/observability/index.ts index 7fe36214a..0ad6137f7 100644 --- a/src/commands/observability/index.ts +++ b/src/commands/observability/index.ts @@ -28,7 +28,7 @@ const AUDIO_PROBE_ACTION_VALUES = ['start', 'status', 'stop'] as const; const logsCommandDescription = 'Manage session app logs.'; const networkCommandDescription = 'Show recent HTTP traffic.'; -const audioCommandDescription = 'Probe browser page audio levels.'; +const audioCommandDescription = 'Probe audio levels.'; export const logsCommandMetadata = defineFieldCommandMetadata( LOGS_COMMAND_NAME, @@ -58,7 +58,7 @@ export const audioCommandMetadata = defineFieldCommandMetadata( probeAction: enumField(AUDIO_PROBE_ACTION_VALUES), durationMs: integerField('Probe duration in milliseconds.'), bucketMs: integerField('Audio level bucket size in milliseconds.'), - source: enumField(['media-elements'] as const), + source: enumField(['media-elements', 'system-audio'] as const), }, ); @@ -101,8 +101,8 @@ const audioCliSchema = { usageOverride: 'audio probe start [durationSeconds] [bucketMs] | audio probe status | audio probe stop', listUsageOverride: 'audio', - helpDescription: 'Probe browser page audio levels as compact dBFS buckets', - summary: 'Probe browser page audio levels', + helpDescription: 'Probe browser or macOS audio levels as compact dBFS buckets', + summary: 'Probe audio levels', positionalArgs: ['probe', 'start|status|stop', 'durationSeconds?', 'bucketMs?'], } as const satisfies CommandSchemaOverride; diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 48c782985..58131647a 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -200,9 +200,10 @@ const BASE_COMMAND_CAPABILITY_MATRIX: Record = { linux: LINUX_NONE, }, audio: { - apple: {}, + apple: { device: true }, android: {}, linux: LINUX_NONE, + supports: (device) => device.platform === 'macos' || device.platform === 'web', }, network: { apple: { simulator: true, device: true }, diff --git a/src/daemon/handlers/__tests__/session-observability.test.ts b/src/daemon/handlers/__tests__/session-observability.test.ts index 0c237880b..a40fea364 100644 --- a/src/daemon/handlers/__tests__/session-observability.test.ts +++ b/src/daemon/handlers/__tests__/session-observability.test.ts @@ -8,6 +8,7 @@ import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts import { makeAndroidSession, makeIosSession, + makeMacOsSession, makeSession, WEB_DESKTOP_DEVICE, } from '../../../__tests__/test-utils/index.ts'; @@ -21,6 +22,9 @@ const applePerfMocks = vi.hoisted(() => ({ stopAppleXctracePerfCapture: vi.fn(), writeAppleXctracePerfReport: vi.fn(), })); +const macosAudioMocks = vi.hoisted(() => ({ + startMacOsAudioProbeProcess: vi.fn(), +})); vi.mock('../../../platforms/ios/perf-xctrace.ts', async (importOriginal) => { const actual = await importOriginal(); @@ -31,6 +35,13 @@ vi.mock('../../../platforms/ios/perf-xctrace.ts', async (importOriginal) => { writeAppleXctracePerfReport: applePerfMocks.writeAppleXctracePerfReport, }; }); +vi.mock('../../../platforms/ios/macos-helper.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + startMacOsAudioProbeProcess: macosAudioMocks.startMacOsAudioProbeProcess, + }; +}); import { handleSessionObservabilityCommands } from '../session-observability.ts'; beforeEach(() => { @@ -443,7 +454,120 @@ test('audio probe rejects non-web sessions in daemon handler', async () => { assert.equal(response?.ok, false); if (response && !response.ok) { assert.equal(response.error.code, 'UNSUPPORTED_OPERATION'); - assert.match(response.error.message, /web browser sessions only/); + assert.match(response.error.message, /web browser and macOS sessions only/); + } +}); + +test('audio probe starts macOS ScreenCaptureKit helper and reads status', async () => { + const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); + sessionStore.set('macos', makeMacOsSession('macos')); + const kill = vi.fn(); + macosAudioMocks.startMacOsAudioProbeProcess.mockImplementation( + async (options: { durationMs: number; bucketMs: number; statusPath: string }) => { + await fsPromises.mkdir(path.dirname(options.statusPath), { recursive: true }); + await fsPromises.writeFile( + options.statusPath, + JSON.stringify({ + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: options.durationMs, + elapsedMs: 1000, + bucketMs: options.bucketMs, + sampleCount: 1, + sourceCount: 1, + rmsDbfs: [-12], + peakDbfs: [-8], + notes: ['macOS status'], + }), + ); + return { + child: { kill, pid: 1234 }, + wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), + }; + }, + ); + + const response = await handleSessionObservabilityCommands({ + req: { + token: 't', + session: 'macos', + command: 'audio', + positionals: ['probe', 'start', '1000', '500'], + flags: {}, + }, + sessionName: 'macos', + sessionStore, + }); + + assert.equal(response?.ok, true); + if (response?.ok) { + assert.equal(response.data?.backend, 'macos-screencapturekit'); + assert.equal(response.data?.source, 'system-audio'); + assert.deepEqual(response.data?.rmsDbfs, [-12]); + } + assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls.length, 1); + assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls[0]?.[0].durationMs, 1000); + assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls[0]?.[0].bucketMs, 500); + assert.equal(kill.mock.calls.length, 0); +}); + +test('audio probe stop kills active macOS helper and returns stopped status', async () => { + const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); + const session = makeMacOsSession('macos'); + const statusPath = path.join(sessionStore.ensureSessionDir('macos'), 'audio-probe.json'); + await fsPromises.writeFile( + statusPath, + JSON.stringify({ + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: 10000, + elapsedMs: 2000, + bucketMs: 1000, + sampleCount: 2, + sourceCount: 1, + rmsDbfs: [-15, -14], + peakDbfs: [-9, -8], + }), + ); + const kill = vi.fn(); + session.audioProbe = { + platform: 'macos', + child: { kill, pid: 1234 }, + wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), + statusPath, + startedAt: Date.now() - 2000, + durationMs: 10000, + bucketMs: 1000, + }; + sessionStore.set('macos', session); + + const response = await handleSessionObservabilityCommands({ + req: { + token: 't', + session: 'macos', + command: 'audio', + positionals: ['probe', 'stop'], + flags: {}, + }, + sessionName: 'macos', + sessionStore, + }); + + assert.equal(response?.ok, true); + assert.equal(kill.mock.calls[0]?.[0], 'SIGTERM'); + assert.equal(sessionStore.get('macos')?.audioProbe, undefined); + if (response?.ok) { + assert.equal(response.data?.state, 'stopped'); + assert.equal(response.data?.active, false); + assert.deepEqual(response.data?.peakDbfs, [-9, -8]); } }); diff --git a/src/daemon/handlers/session-close.ts b/src/daemon/handlers/session-close.ts index ecd307000..0292fd296 100644 --- a/src/daemon/handlers/session-close.ts +++ b/src/daemon/handlers/session-close.ts @@ -27,6 +27,7 @@ import { stopSessionAndroidSnapshotHelper, stopSessionAppLog, stopSessionApplePerfCapture, + stopSessionAudioProbe, } from '../session-teardown.ts'; async function maybeShutdownSessionTarget(params: { @@ -61,6 +62,7 @@ export async function handleCloseCommand(params: { } try { await stopSessionAppLog(session); + await stopSessionAudioProbe(session); await stopSessionApplePerfCapture(session); await stopSessionAndroidNativePerfCapture(session); await stopSessionAndroidSnapshotHelper(session); diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index 1f7d98f2d..0326d66b8 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -1,3 +1,5 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; import { isPerfAction, @@ -14,6 +16,7 @@ import { } from '../../contracts/perf.ts'; import { AppError, normalizeError } from '../../utils/errors.ts'; import { resolveWebProvider } from '../../platforms/web/provider.ts'; +import { startMacOsAudioProbeProcess } from '../../platforms/ios/macos-helper.ts'; import type { AndroidAdbExecutor } from '../../platforms/android/adb-executor.ts'; import type { DaemonRequest, DaemonResponse, DaemonResponseData, SessionState } from '../types.ts'; import { SessionStore } from '../session-store.ts'; @@ -620,6 +623,9 @@ function resolveNetworkIncludeMode( async function handleAudioCommand(params: ObservabilityParams): Promise { const request = resolveAudioCommandRequest(params); if (!request.ok) return request; + if (request.session.device.platform === 'macos') { + return await handleMacOsAudioCommand(params, request); + } const provider = resolveWebProvider(); if (!provider.probeAudio) { return errorResponse('UNSUPPORTED_OPERATION', 'audio is not supported by this web provider'); @@ -643,6 +649,7 @@ async function handleAudioCommand(params: ObservabilityParams): Promise, + { ok: true } +>; + +type MacOsAudioProbeData = { + audio: 'probe'; + state: 'running' | 'stopped'; + active: boolean; + heard: boolean; + source: 'system-audio'; + backend: 'macos-screencapturekit'; + durationMs: number; + elapsedMs: number; + bucketMs: number; + sampleCount: number; + sourceCount: number; + rmsDbfs: number[]; + peakDbfs: number[]; + startedAt?: string; + stoppedAt?: string; + reason?: string; + notes?: string[]; +}; + +async function handleMacOsAudioCommand( + params: ObservabilityParams, + request: ResolvedAudioCommandRequest, +): Promise { + const { session, probeAction } = request; + try { + if (probeAction === 'start') { + await stopMacOsAudioProbe(session); + const statusPath = path.join( + params.sessionStore.ensureSessionDir(params.sessionName), + 'audio-probe.json', + ); + const probe = await startMacOsAudioProbeProcess({ + durationMs: request.durationMs, + bucketMs: request.bucketMs, + statusPath, + }); + session.audioProbe = { + platform: 'macos', + child: probe.child, + wait: probe.wait, + statusPath, + startedAt: Date.now(), + durationMs: request.durationMs, + bucketMs: request.bucketMs, + }; + void probe.wait.catch(() => {}); + return { ok: true, data: await waitForMacOsAudioProbeStatus(session) }; + } + + if (probeAction === 'stop') { + const data = await stopMacOsAudioProbe(session); + return { + ok: true, + data: data ?? buildMacOsAudioProbeFallback(request, 'stopped', 'not-started'), + }; + } + + const data = await readMacOsAudioProbeStatus(session); + if (data) { + if (data.state === 'stopped') session.audioProbe = undefined; + return { ok: true, data }; + } + return { ok: true, data: buildMacOsAudioProbeFallback(request, 'stopped', 'not-started') }; + } catch (error) { + return { ok: false, error: normalizeError(error) }; + } +} + +async function waitForMacOsAudioProbeStatus(session: SessionState): Promise { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const status = await readMacOsAudioProbeStatus(session); + if (status) return status; + const exit = await Promise.race([ + session.audioProbe?.wait.then( + (result) => result, + (error: unknown) => error, + ), + sleep(100).then(() => undefined), + ]); + if (exit instanceof Error) throw exit; + if (exit) { + const result = exit as { stdout?: string; stderr?: string; exitCode?: number }; + const message = + result.stderr?.trim() || + result.stdout?.trim() || + `macOS audio probe helper exited with code ${result.exitCode ?? 1}`; + throw new AppError('COMMAND_FAILED', `failed to start macOS audio probe: ${message}`); + } + } + throw new AppError('COMMAND_FAILED', 'failed to start macOS audio probe'); +} + +async function readMacOsAudioProbeStatus( + session: SessionState, +): Promise { + const probe = session.audioProbe; + if (!probe) return undefined; + try { + const raw = await fs.readFile(probe.statusPath, 'utf8'); + return normalizeMacOsAudioProbeData(JSON.parse(raw), probe); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + +async function stopMacOsAudioProbe( + session: SessionState, +): Promise { + const probe = session.audioProbe; + if (!probe) return undefined; + const beforeStop = await readMacOsAudioProbeStatus(session); + probe.child.kill('SIGTERM'); + await probe.wait.catch(() => {}); + session.audioProbe = undefined; + return finalizeMacOsAudioProbeStatus(beforeStop, probe, 'stopped'); +} + +function normalizeMacOsAudioProbeData( + value: unknown, + probe: NonNullable, +): MacOsAudioProbeData { + const record = value && typeof value === 'object' ? (value as Record) : {}; + const state = record.state === 'stopped' ? 'stopped' : 'running'; + const rmsDbfs = readNumberArray(record.rmsDbfs); + const peakDbfs = readNumberArray(record.peakDbfs); + return { + audio: 'probe', + state, + active: state === 'running' && record.active !== false, + heard: record.heard === true, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: readFiniteNumber(record.durationMs, probe.durationMs), + elapsedMs: readFiniteNumber(record.elapsedMs, Date.now() - probe.startedAt), + bucketMs: readFiniteNumber(record.bucketMs, probe.bucketMs), + sampleCount: readFiniteNumber(record.sampleCount, rmsDbfs.length), + sourceCount: readFiniteNumber(record.sourceCount, 1), + rmsDbfs, + peakDbfs, + startedAt: typeof record.startedAt === 'string' ? record.startedAt : undefined, + stoppedAt: typeof record.stoppedAt === 'string' ? record.stoppedAt : undefined, + reason: typeof record.reason === 'string' ? record.reason : undefined, + notes: readStringArray(record.notes), + }; +} + +function finalizeMacOsAudioProbeStatus( + status: MacOsAudioProbeData | undefined, + probe: NonNullable, + reason: string, +): MacOsAudioProbeData { + const elapsedMs = Math.min(probe.durationMs, Math.max(0, Date.now() - probe.startedAt)); + const base = + status ?? + ({ + audio: 'probe', + state: 'stopped', + active: false, + heard: false, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: probe.durationMs, + elapsedMs: 0, + bucketMs: probe.bucketMs, + sampleCount: 0, + sourceCount: 1, + rmsDbfs: [], + peakDbfs: [], + notes: [ + 'Audio probe samples macOS system audio through ScreenCaptureKit; it is not app-instrumented audio.', + 'Screen Recording permission is required for macOS system audio capture.', + ], + } as MacOsAudioProbeData); + return { + ...base, + state: 'stopped', + active: false, + elapsedMs, + stoppedAt: new Date().toISOString(), + reason, + }; +} + +function buildMacOsAudioProbeFallback( + request: ResolvedAudioCommandRequest, + state: 'running' | 'stopped', + reason?: string, +): MacOsAudioProbeData { + return { + audio: 'probe', + state, + active: state === 'running', + heard: false, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: request.durationMs, + elapsedMs: 0, + bucketMs: request.bucketMs, + sampleCount: 0, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + reason, + notes: [ + 'Audio probe samples macOS system audio through ScreenCaptureKit; it is not app-instrumented audio.', + 'Screen Recording permission is required for macOS system audio capture.', + 'No active macOS audio probe is running.', + ], + }; +} + +function readFiniteNumber(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +function readNumberArray(value: unknown): number[] { + if (!Array.isArray(value)) return []; + const numbers: number[] = []; + for (const item of value) { + if (typeof item === 'number' && Number.isFinite(item)) numbers.push(item); + } + return numbers; +} + +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + return value.filter((item): item is string => typeof item === 'string'); +} + +async function sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + function resolveAudioProbeAction( req: DaemonRequest, ): { ok: true; probeAction: 'start' | 'status' | 'stop' } | DaemonFailureResponse { diff --git a/src/daemon/session-teardown.ts b/src/daemon/session-teardown.ts index 25c35517f..6e41146d0 100644 --- a/src/daemon/session-teardown.ts +++ b/src/daemon/session-teardown.ts @@ -56,11 +56,20 @@ export async function stopSessionAndroidSnapshotHelper(session: SessionState): P await stopAndroidSnapshotHelperSessionForDevice(session.device); } +export async function stopSessionAudioProbe(session: SessionState): Promise { + const probe = session.audioProbe; + if (!probe) return; + probe.child.kill('SIGTERM'); + await probe.wait.catch(() => {}); + session.audioProbe = undefined; +} + export async function teardownSessionResources( session: SessionState, sessionName: string, ): Promise { await stopSessionAppLog(session); + await stopSessionAudioProbe(session); await stopSessionApplePerfCapture(session); await stopSessionAndroidNativePerfCapture(session); await stopSessionAndroidSnapshotHelper(session); diff --git a/src/daemon/types.ts b/src/daemon/types.ts index d209a687e..5677ea9c5 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -267,6 +267,15 @@ export type SessionState = { nativePerf?: { android?: AndroidNativePerfSession; }; + audioProbe?: { + platform: 'macos'; + child: SessionRecordingProcessChild; + wait: Promise; + statusPath: string; + startedAt: number; + durationMs: number; + bucketMs: number; + }; /** Session was created by record start and should be released when recording stops. */ recordOnlySession?: boolean; recordSession?: boolean; diff --git a/src/platforms/ios/macos-helper.ts b/src/platforms/ios/macos-helper.ts index 024238f88..758fc06e0 100644 --- a/src/platforms/ios/macos-helper.ts +++ b/src/platforms/ios/macos-helper.ts @@ -4,7 +4,11 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { AppError } from '../../utils/errors.ts'; -import { resolveExecutableOverridePath } from '../../utils/exec.ts'; +import { + resolveExecutableOverridePath, + runCmdBackground, + type ExecBackgroundResult, +} from '../../utils/exec.ts'; import type { SessionSurface } from '../../core/session-surface.ts'; import { hasScopedAppleToolProvider, @@ -227,6 +231,27 @@ async function resolveMacOsHelperCommandPath(): Promise { return await ensureMacOsHelperBinary(); } +export async function startMacOsAudioProbeProcess(options: { + durationMs: number; + bucketMs: number; + statusPath: string; +}): Promise { + const helperPath = await resolveMacOsHelperCommandPath(); + return runCmdBackground( + helperPath, + [ + 'audio-probe', + '--duration-ms', + String(options.durationMs), + '--bucket-ms', + String(options.bucketMs), + '--out', + options.statusPath, + ], + { allowFailure: true, captureOutput: true }, + ); +} + async function runMacOsHelper>(args: string[]): Promise { const helperOptions = { allowFailure: true, diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index e88bce6a0..b79e4e8e0 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -270,7 +270,7 @@ Validation and evidence: Remote lifecycle: cloud, remote-config, direct proxy, and limrun use the same flow: connect, open, commands, close, disconnect. Remote config profile: agent-device connect --remote-config ./remote-config.json; then run normal commands and disconnect. Direct proxy to a Mac you control: cloud/Linux clients can use local/proxy iOS devices through the proxied Mac. Run agent-device connect proxy --daemon-base-url first. Device leases are automatic on open and expire after five minutes of inactivity. - Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. Audio probe samples HTML media elements, and URL-backed media may be routed through the probe AudioContext while observed. + Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. On web, audio probe samples HTML media elements, and URL-backed media may be routed through the probe AudioContext while observed. On macOS, audio probe samples system audio through ScreenCaptureKit and requires Screen Recording permission. agent-device web setup agent-device web doctor agent-device open https://example.com --platform web @@ -291,6 +291,7 @@ Validation and evidence: agent-device close --platform web Minimal web support is for browser sessions with open, snapshot, find, get, is, click/press, fill/type, wait, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay over those commands. Use agent-browser directly for browser-specific features that agent-device does not surface, such as tab/devtools management, advanced page scripting, network routing/HAR, or raw browser debugging. macOS menu bar: open ... --platform macos --surface menubar; snapshot -i --platform macos --surface menubar. + macOS audio: audio probe start 10 1000 --platform macos samples host system audio through ScreenCaptureKit; grant Screen Recording permission first. Maestro full-suite validation on explicit connected devices uses one test command with a comma-separated --device list and --shard-all. Use --shard-split only when splitting suite entries across devices: agent-device test ./e2e/maestro --maestro --device udid1,emulator-5554 --shard-all 2 From 7d9eeae9cf47ae116bc5d44f1fd487df97f82120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 14:41:16 +0200 Subject: [PATCH 06/18] docs: document audio probe help --- README.md | 4 +-- src/utils/__tests__/args.test.ts | 10 ++++++- src/utils/cli-help.ts | 15 +++++++++-- website/docs/docs/agent-setup.md | 8 +++--- website/docs/docs/client-api.md | 33 ++++++++++++++++++++++-- website/docs/docs/commands.md | 14 +++++++--- website/docs/docs/debugging-profiling.md | 17 +++++++++++- website/docs/docs/introduction.md | 2 +- website/docs/docs/security-trust.md | 4 +-- 9 files changed, 88 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 0e2ca7104..4f5a06cf2 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ It works with native iOS and Android apps, plus apps built with Expo, Flutter, a - **Inspect** real app UI through structured accessibility snapshots, interactive refs like `@e3`, selectors, and React Native component trees. - **Interact** by opening apps, tapping, typing, scrolling, performing gestures, waiting, asserting state, handling alerts, and closing sessions. -- **Capture evidence** with screenshots, videos, logs, traces, network traffic, performance samples, crash context, and React profiles. +- **Capture evidence** with screenshots, videos, logs, traces, network traffic, audio-level probes, performance samples, crash context, and React profiles. - **Replay workflows** by recording `.ad` scripts for local runs, CI, repeatable e2e checks, and strict Maestro YAML export when a flow needs to run in Maestro. - **Run across platforms** with iOS Simulator automation, Android Emulator automation, physical devices, tvOS, Android TV, macOS, Linux, and desktop app automation, so agents can see and feel the app they work on. @@ -39,7 +39,7 @@ It works with native iOS and Android apps, plus apps built with Expo, Flutter, a - Verify mobile changes on real devices, simulators, and emulators before review or merge. - Give AI coding agents a real app feedback loop while they implement features. -- Debug regressions with screenshots, logs, traces, network evidence, and crash context. +- Debug regressions with screenshots, logs, traces, network/audio evidence, and crash context. - Profile performance issues with CPU/memory samples and React render profiles when needed. - Turn exploratory app interactions into replayable e2e checks for CI. - Use one agent workflow across native iOS, Android, Expo, Flutter, React Native, TV, and desktop apps. diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index b89d30b8e..40ec99da7 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -1267,7 +1267,7 @@ test('usage includes agent workflows, config, environment, and examples footers' ); assert.match( usageText, - /agent-device help debugging\s+Use when logs, network, perf memory, traces, alerts, or diagnostics matter/, + /agent-device help debugging\s+Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter/, ); assert.match( usageText, @@ -1493,12 +1493,15 @@ test('usageForCommand resolves web help topic', () => { assert.match(help, /agent-device fill @e13 "qa@example\.com" --platform web/); assert.match(help, /agent-device wait text "Welcome" 3000 --platform web/); assert.match(help, /agent-device network dump 25 --include headers --platform web/); + assert.match(help, /agent-device audio probe start 10 1000 --platform web/); + assert.match(help, /Audio probe start uses duration seconds first, then bucket milliseconds/); assert.match(help, /agent-device screenshot \.\/artifacts\/web-home\.png --platform web/); assert.match(help, /agent-device close --platform web/); assert.match(help, /open , snapshot -i, get text\/attrs/); assert.match(help, /is visible\/exists\/text, find text\/selector/); assert.match(help, /click\/press @ref or selector/); assert.match(help, /network dump/); + assert.match(help, /audio probe/); assert.match(help, /network routing\/interception\/HAR/); assert.match(help, /Use agent-browser directly for those browser-specific workflows/); assert.match(help, /Do not claim web e2e CI exists/); @@ -1526,6 +1529,7 @@ test('usageForCommand resolves debugging help topic', () => { assert.match(help, /Use Xcode\/LLDB when you need live state/); assert.match(help, /debug symbols --artifact crash\.ips --search-path \.\/build/); assert.match(help, /Android Java\/R8 mapping\.txt and native ndk-stack\/addr2line/); + assert.match(help, /network\/audio evidence/); assert.match(help, /agent-device alert wait 3000/); assert.match(help, /iOS support is runner-derived/); assert.match(help, /resolved app executable/); @@ -1534,6 +1538,10 @@ test('usageForCommand resolves debugging help topic', () => { assert.match(help, /requests\/\.ndjson holds daemon request diagnostics/); assert.match(help, /daemon\.log is global daemon lifecycle evidence/); assert.match(help, /agent-device perf memory sample --json/); + assert.match(help, /agent-device audio probe start 10 1000 --platform web/); + assert.match(help, /agent-device audio probe start 10 1000 --platform macos/); + assert.match(help, /compact rmsDbfs and peakDbfs arrays/); + assert.match(help, /requires Screen Recording permission/); assert.match(help, /Memory artifact \(android-hprof\): \/tmp\/app\.hprof \(42MB\)/); assert.match(help, /Prefer perf memory sample over raw dumpsys\/leaks output/); assert.match(help, /Unsupported platforms return artifact\.available=false with reason\/hint/); diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index b79e4e8e0..09c8d2a98 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -16,7 +16,8 @@ const AGENT_WORKFLOWS = [ }, { label: 'agent-device help debugging', - description: 'Use when logs, network, perf memory, traces, alerts, or diagnostics matter', + description: + 'Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter', }, { label: 'agent-device help react-native', @@ -349,6 +350,16 @@ Network: Use this instead of logs path when the question is request/response metadata. network log is a supported alias, but network dump --include headers is the clearest plan form. Do not write network log headers. +Audio: + Use audio probe when the question is whether a browser page or macOS host produced audible output during a short observation window. + agent-device audio probe start 10 1000 --platform web + agent-device audio probe status --platform web + agent-device audio probe stop --platform web + agent-device audio probe start 10 1000 --platform macos + audio probe start uses duration seconds first, then bucket milliseconds. Results are compact rmsDbfs and peakDbfs arrays so agents can correlate audible moments with screenshots, actions, network entries, or frame samples. + On web, audio probe samples HTML media elements and URL-backed media may be routed through the probe AudioContext while observed. + On macOS, audio probe samples host system audio through ScreenCaptureKit and requires Screen Recording permission. It is system-audio evidence, not app-instrumented audio. + Crash symbolication: Crash routing: Use logs when you need the lead-up timeline before a failure. @@ -357,7 +368,7 @@ Crash symbolication: Use debug symbols when you already have an Apple crash artifact and local dSYMs and need the failing code path, not a full log dump: agent-device debug symbols --artifact crash.log --dsym MyApp.dSYM --out crash-symbolicated.log agent-device debug symbols --artifact crash.ips --search-path ./build --out crash-symbolicated.ips - debug is intentionally narrow. Do not use it for logs, network evidence, performance samples, recordings, traces, or React Native internals. + debug is intentionally narrow. Do not use it for logs, network/audio evidence, performance samples, recordings, traces, or React Native internals. Apple support matches crash Binary Images / IPS usedImages UUIDs against dwarfdump --uuid output from .dSYM bundles, then writes a symbolicated artifact path and compact crash report: app/thread, exception or termination, top symbolicated frames, and first-frame finding. This is better than pasting crash logs because it keeps agent context small while preserving the artifact on disk for inspection. Android Java/R8 mapping.txt and native ndk-stack/addr2line symbolication are not in this first debug symbols workflow; capture crash evidence with logs and use the Android toolchain externally for now. diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index 78b07ba27..e97b0172f 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -47,9 +47,9 @@ The bundled [agent-device skill](https://github.com/callstack/agent-device/blob/ Add this as a project rule, custom instruction, or skill equivalent when your agent client supports it: ```text -Use agent-device only for app/device automation tasks. Before planning commands, run `agent-device --version` and read `agent-device help workflow`. For exploratory QA, read `agent-device help dogfood`. For logs, network, traces, or runtime failures, read `agent-device help debugging`. For React Native component trees, props/state/hooks, slow renders, or rerenders, read `agent-device help react-devtools`. For React Native JavaScript heap growth, heap snapshots, or retained-object leaks, read `agent-device help cdp`. For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, read `agent-device help react-native`. +Use agent-device only for app/device automation tasks. Before planning commands, run `agent-device --version` and read `agent-device help workflow`. For exploratory QA, read `agent-device help dogfood`. For logs, network, audio, traces, or runtime failures, read `agent-device help debugging`. For React Native component trees, props/state/hooks, slow renders, or rerenders, read `agent-device help react-devtools`. For React Native JavaScript heap growth, heap snapshots, or retained-object leaks, read `agent-device help cdp`. For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, read `agent-device help react-native`. -Use MCP tools or the CLI in the integrated terminal. If `agent-device` is not on PATH but the user installed it globally in another shell, resolve the command the same way the user would from a normal terminal session and run that absolute path instead. This may require inspecting shell startup behavior or package-manager/global bin locations; do not assume the agent process `PATH` is the user's `PATH`. Do not silently fall back to `npx -y agent-device@latest`; ask or use an exact version. MCP exposes structured tools backed by the agent-device client; it does not expose generic shell execution. Prefer `open -> snapshot -i -> act -> re-snapshot -> verify -> close`. Use current refs such as `@e3` for exploration and selectors for durable replay. Keep mutating commands against one session serial. Capture screenshots, logs, network, perf, traces, recordings, and `.ad` replay scripts only when they add evidence. +Use MCP tools or the CLI in the integrated terminal. If `agent-device` is not on PATH but the user installed it globally in another shell, resolve the command the same way the user would from a normal terminal session and run that absolute path instead. This may require inspecting shell startup behavior or package-manager/global bin locations; do not assume the agent process `PATH` is the user's `PATH`. Do not silently fall back to `npx -y agent-device@latest`; ask or use an exact version. MCP exposes structured tools backed by the agent-device client; it does not expose generic shell execution. Prefer `open -> snapshot -i -> act -> re-snapshot -> verify -> close`. Use current refs such as `@e3` for exploration and selectors for durable replay. Keep mutating commands against one session serial. Capture screenshots, logs, network, audio, perf, traces, recordings, and `.ad` replay scripts only when they add evidence. ``` ## MCP server @@ -109,7 +109,7 @@ alwaysApply: true Use agent-device only for app/device automation tasks. Before planning device work, run `agent-device --version` and read `agent-device help workflow`. For exploratory QA, read `agent-device help dogfood`. -For logs, network, traces, or runtime failures, read `agent-device help debugging`. +For logs, network, audio, traces, or runtime failures, read `agent-device help debugging`. For React Native component trees, props/state/hooks, slow renders, or rerenders, read `agent-device help react-devtools`. For React Native JavaScript heap growth, heap snapshots, or retained-object leaks, read `agent-device help cdp`. For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, read `agent-device help react-native`. @@ -198,7 +198,7 @@ cat > CLAUDE.md <<'EOF' Use agent-device only for app/device automation tasks. Before planning device work, run `agent-device --version` and read `agent-device help workflow`. For exploratory QA, read `agent-device help dogfood`. -For logs, network, traces, or runtime failures, read `agent-device help debugging`. +For logs, network, audio, traces, or runtime failures, read `agent-device help debugging`. For React Native component trees, props/state/hooks, slow renders, or rerenders, read `agent-device help react-devtools`. For React Native JavaScript heap growth, heap snapshots, or retained-object leaks, read `agent-device help cdp`. For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, read `agent-device help react-native`. diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 6fde028e4..5d0fd3430 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -139,13 +139,21 @@ await client.capture.snapshot({ platform: 'web', interactiveOnly: true }); await client.interactions.fill({ platform: 'web', ref: '@e12', text: 'test@example.com' }); await client.command.wait({ platform: 'web', text: 'Welcome' }); await client.observability.network({ platform: 'web', include: 'headers' }); +await client.observability.audio({ + platform: 'web', + action: 'probe', + probeAction: 'start', + durationMs: 10_000, + bucketMs: 1_000, +}); await client.sessions.close(); ``` Web automation requires Node 24+. MCP tools use the same command contracts, so they can target `platform: 'web'` after setup, but local setup/doctor remains a CLI-only workflow. Web network inspection adapts managed `agent-browser` request history to the existing network result shape; -request and response bodies are not exposed by that backend path. +request and response bodies are not exposed by that backend path. Web audio probes sample HTML +media elements and return compact dBFS buckets. ## Android snapshot helper providers @@ -280,10 +288,31 @@ Additional CLI-backed methods are exposed on their domain groups with typed opti - `client.interactions.click()`, `press()`, `longPress()`, `swipe()`, `pan()`, `fling()`, `focus()`, `type()`, `fill()`, `scroll()`, `pinch()`, `rotateGesture()`, `transformGesture()`, `get()`, `is()`, `find()` - `client.replay.run()` and `client.replay.test()` - `client.batch.run()` -- `client.observability.perf()`, `logs()`, and `network()` +- `client.observability.perf()`, `logs()`, `network()`, and `audio()` - `client.recording.record()` and `client.recording.trace()` - `client.settings.update()` +`client.observability.audio()` mirrors `audio probe start|status|stop`. Use it to collect compact RMS/peak dBFS buckets while other session actions continue: + +```ts +await client.observability.audio({ + platform: 'web', + action: 'probe', + probeAction: 'start', + durationMs: 10_000, + bucketMs: 1_000, +}); +await client.interactions.click({ platform: 'web', ref: '@e4' }); +const audio = await client.observability.audio({ + platform: 'web', + action: 'probe', + probeAction: 'status', +}); +await client.observability.audio({ platform: 'web', action: 'probe', probeAction: 'stop' }); +``` + +Web probes sample HTML media elements. macOS probes use `platform: 'macos'`, sample host system audio through ScreenCaptureKit, and require Screen Recording permission. iOS and Android app audio are not exposed by this command. + `client.observability.perf()` returns daemon-shaped JSON so local and remote transports expose the same metrics payload. Pass `{ area: 'metrics' }` for the broad startup/CPU/memory/frame first pass, `{ area: 'frames' }` for a focused frame/jank-health payload, or `{ area: 'memory', action: 'sample' }` for a compact memory-only sample. Use `{ area: 'memory', action: 'snapshot', kind: 'android-hprof', out: 'app.hprof' }` on Android or `{ area: 'memory', action: 'snapshot', kind: 'memgraph', out: 'app.memgraph' }` on supported Apple simulator/macOS app sessions to write large memory artifacts to disk. Android native artifacts use `{ area: 'cpu', subject: 'profile', action: 'start' | 'stop' | 'report', kind: 'simpleperf', out }` and `{ area: 'trace', action: 'start' | 'stop', kind: 'perfetto', out }`; these Android-only commands return artifact paths and compact summaries, not trace/profile contents. Physical iOS device memgraph capture reports unavailable with a reason/hint. heapprofd allocation tracing is deferred until Perfetto plumbing is available. On Android and supported Apple targets, `data.metrics.fps.droppedFramePercent` is the primary frame-smoothness value. Android derives it from the current `adb shell dumpsys gfxinfo framestats` window; connected iOS devices derive it from `xcrun xctrace` Animation Hitches for the active app process. Frame samples include `windowStartedAt`, `windowEndedAt`, and `worstWindows` so agents can correlate dropped-frame clusters with logs, network entries, and their own session actions. A successful Android read resets Android frame stats; `open ` resets the Android frame window too, so agents can call `perf({ area: 'frames' })`, perform a transition or gesture, then call it again to inspect that focused window. iOS simulator and macOS app sessions report frame health as unavailable rather than inventing FPS or dropped-frame values. For Apple native profiling, call `perf({ area: 'cpu', subject: 'profile', action: 'start', kind: 'xctrace', template: 'Time Profiler', out: 'app.trace' })`, then stop with the same trace path and write a compact report with `action: 'report'`. `area: 'trace'` supports xctrace templates such as `Animation Hitches`. Responses include artifact paths and compact metadata only. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cd6f7b087..cd4a69364 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -123,6 +123,9 @@ agent-device click @e12 --platform web agent-device fill @e13 "test@example.com" --platform web agent-device wait text "Welcome" --platform web agent-device network dump 25 --include headers --platform web +agent-device audio probe start 10 1000 --platform web +agent-device audio probe status --platform web +agent-device audio probe stop --platform web agent-device screenshot ./artifacts/web-home.png --platform web agent-device screenshot ./artifacts/web-full.png --platform web --fullscreen agent-device viewport 1280 900 --platform web @@ -136,7 +139,9 @@ agent-device close --platform web - `web doctor` verifies the managed backend after setup. - The managed install respects `--state-dir` and `AGENT_DEVICE_STATE_DIR`. - Web automation requires Node 24+. -- Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/exists/text`, `find text/selector`, click/press, fill/type, wait, `network dump`, screenshot, close, and replay scripts composed from those commands. +- Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/exists/text`, `find text/selector`, click/press, fill/type, wait, `network dump`, `audio probe`, screenshot, close, and replay scripts composed from those commands. +- `audio probe start [durationSeconds] [bucketMs]` samples HTML media elements into compact RMS/peak dBFS buckets while the page keeps running. The first timing positional is seconds; the second is milliseconds. +- URL-backed web media may be routed through the probe `AudioContext` while observed. Use `audio probe status` to poll partial buckets and `audio probe stop` to end the probe early. - Out of scope for `agent-device` web support: tab/window/devtools control, network routing/interception/HAR, cookies/storage, downloads/uploads, arbitrary page scripting, multi-page orchestration, and raw browser diagnostics. Use `agent-browser` directly for those browser-specific workflows. ## Device isolation scopes @@ -217,7 +222,8 @@ agent-device snapshot -i --platform apple --target desktop - Status-item apps often expose little or no useful UI through the default macOS `app` surface. Prefer `--surface menubar` for discovery when the app lives in the top menu bar. - Use `frontmost-app`, `desktop`, and `menubar` mainly for `snapshot`, `get`, `is`, and `wait`. - If you inspect with `desktop` or `menubar` and then need to click or fill inside one app, open that app in a normal `app` session. -- macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `alert`, `settings appearance`, and `settings permission `. +- macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `audio probe`, `alert`, `settings appearance`, and `settings permission `. +- macOS `audio probe start 10 1000 --platform macos` samples host system audio through ScreenCaptureKit. Grant Screen Recording permission before relying on it in a run. - In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop. - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. @@ -706,7 +712,7 @@ agent-device react-devtools profile report @c5 - Use it when a React Native workflow needs component hierarchy, props, state, hooks, render causes, slow components, or re-render counts. - For profiling, keep the window narrow and make one bounded first-pass survey: use the `profile stop` summary, run `profile slow --limit 5` and `profile rerenders --limit 5` once, add `profile timeline --limit 20` only when commit timing matters, then drill into a specific `@c` ref with `profile report`. - Do not repeatedly raise broad `profile slow` limits such as `--limit 50`, `--limit 200`, or `--limit 500` unless you have a specific target that needs more rows. -- Keep using `snapshot`, `press`, `fill`, `logs`, `network`, `perf metrics`, and `perf frames` for device/app runtime evidence. Use `react-devtools` for React internals. +- Keep using `snapshot`, `press`, `fill`, `logs`, `network`, `audio probe`, `perf metrics`, and `perf frames` for device/app runtime evidence. Use `react-devtools` for React internals. - For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, start with `agent-device help react-native`. - On Android, use `alert get`, `alert wait `, `alert accept`, and `alert dismiss` for runtime permission prompts and native alerts. On iOS, use the same alert commands for XCTest alerts, app-owned modal popups with native blocking markers, and blocking system dialogs. Do not use `settings permission` to answer a dialog already on screen; reserve it for setup or resetting permission state before a flow. - React Native development builds can connect to the DevTools daemon on port 8097. For Android emulators or physical devices, run `adb reverse tcp:8097 tcp:8097` if the app cannot reach the host. @@ -832,7 +838,7 @@ agent-device debug symbols --artifact crash.log --dsym MyApp.dSYM --out crash-sy agent-device debug symbols --artifact crash.ips --search-path ./build --out crash-symbolicated.ips ``` -- `debug` is intentionally narrow: do not use it for app logs, network evidence, performance samples, recordings, traces, or React Native internals. +- `debug` is intentionally narrow: do not use it for app logs, network/audio evidence, performance samples, recordings, traces, or React Native internals. - Android Java/R8 `mapping.txt` and native `ndk-stack`/`addr2line` symbolication are deferred; capture Android crash evidence with `logs` and symbolicate externally for now. - The crash artifact body is written to `--out`; it is not dumped into agent context or default JSON. diff --git a/website/docs/docs/debugging-profiling.md b/website/docs/docs/debugging-profiling.md index b8fd8fc15..caf72e044 100644 --- a/website/docs/docs/debugging-profiling.md +++ b/website/docs/docs/debugging-profiling.md @@ -10,6 +10,7 @@ Use `agent-device` when the task moves past UI automation and you need runtime e - Session app logs for targeted debugging windows - Network inspection from recent HTTP(s) entries in app logs via `network dump` +- Audio-level probes for browser media elements and macOS system audio - Performance snapshots with `perf metrics` / `perf frames` - Apple crash symbolication with `debug symbols` - Screenshots, recordings, and replayable repro flows @@ -33,7 +34,7 @@ agent-device react-devtools profile report @c5 `agent-device` remains centered on the device and app runtime layer. The `react-devtools` command dynamically runs pinned `agent-react-devtools` commands for React internals. -For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, start with `agent-device help react-native`. For slow-flow investigations, combine `help react-devtools` for the narrow React profile window with `help debugging` for log markers, network evidence, traces, and perf samples. Make one bounded first-pass survey with the `profile stop` summary, bounded `slow` and `rerenders` tables, and `timeline` only when commit timing matters; then drill into a specific `@c` ref with `profile report` instead of repeatedly raising broad `profile slow` limits. +For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, start with `agent-device help react-native`. For slow-flow investigations, combine `help react-devtools` for the narrow React profile window with `help debugging` for log markers, network/audio evidence, traces, and perf samples. Make one bounded first-pass survey with the `profile stop` summary, bounded `slow` and `rerenders` tables, and `timeline` only when commit timing matters; then drill into a specific `@c` ref with `profile report` instead of repeatedly raising broad `profile slow` limits. React Native warning/error overlays belong to the app run. Treat them as findings or blockers: capture them, check `react-devtools errors` when connected, run `agent-device react-native dismiss-overlay` when the overlay is unrelated, then re-snapshot and report the overlay. @@ -138,6 +139,20 @@ agent-device network dump 25 --include all - Parsed results depend on what the app emits into the platform log backend. - Web `network dump` includes request and response headers when requested, but the current `agent-browser network requests` backend does not expose request or response bodies. +### Audio probes + +```bash +agent-device audio probe start 10 1000 --platform web +agent-device audio probe status --platform web +agent-device audio probe stop --platform web +agent-device audio probe start 10 1000 --platform macos +``` + +- `audio probe start [durationSeconds] [bucketMs]` samples live audio while the session keeps running, then exposes compact `rmsDbfs` and `peakDbfs` buckets. The first timing positional is seconds; the second is milliseconds. +- On web, the probe samples HTML media elements through Web Audio. URL-backed media may be routed through the probe `AudioContext` while observed. +- On macOS, the probe samples host system audio through ScreenCaptureKit and requires Screen Recording permission. It is system-audio evidence, not app-instrumented audio. +- Use `status` to poll partial buckets during a 10-20 second observation window, and `stop` to end the probe early. + ### Performance snapshots ```bash diff --git a/website/docs/docs/introduction.md b/website/docs/docs/introduction.md index 3171da3fa..d78faf44d 100644 --- a/website/docs/docs/introduction.md +++ b/website/docs/docs/introduction.md @@ -15,7 +15,7 @@ Use it when an agent needs to inspect and operate a real app, not just reason ab - **App verification for agents**: run the app, inspect visible UI, act through refs/selectors, and verify expected state. - **Token-efficient UI context**: accessibility snapshots give agents structured UI state instead of screenshot-only reasoning. -- **Runtime evidence**: capture screenshots, recordings, logs, network traffic, traces, CPU/memory/perf snapshots, and crash-related logs when the happy path breaks. +- **Runtime evidence**: capture screenshots, recordings, logs, network traffic, audio-level probes, traces, CPU/memory/perf snapshots, and crash-related logs when the happy path breaks. - **Replayable checks**: turn stable exploratory sessions into `.ad` replay scripts that can run again without AI. - **React Native and Expo workflows**: pair device automation with optional React DevTools profiling for component trees, props/state/hooks, slow renders, and rerenders. - **Local devices and app surfaces**: drive simulators, emulators, physical devices, TV targets, desktop apps, and browser sessions through one CLI. diff --git a/website/docs/docs/security-trust.md b/website/docs/docs/security-trust.md index 77caa0d8a..f20eefc23 100644 --- a/website/docs/docs/security-trust.md +++ b/website/docs/docs/security-trust.md @@ -1,6 +1,6 @@ --- title: Security & Trust -description: Security and trust guidance for agent-device local app automation, device permissions, screenshots, recordings, logs, network dumps, traces, and reports. +description: Security and trust guidance for agent-device local app automation, device permissions, screenshots, recordings, logs, network dumps, audio probes, traces, and reports. --- # Security & Trust @@ -26,7 +26,7 @@ For remote or cloud deployments, the daemon supports a custom auth hook: `AGENT_ ## Sensitive artifacts -Screenshots, recordings, traces, logs, network dumps, replay files, and reports can contain private UI state, credentials, tokens, request data, or customer information. Store them in a controlled directory, review before sharing, and avoid committing artifacts unless they are intentionally sanitized fixtures. +Screenshots, recordings, traces, logs, network dumps, audio probes, replay files, and reports can contain private UI state, credentials, tokens, request data, timing signals, or customer information. Store them in a controlled directory, review before sharing, and avoid committing artifacts unless they are intentionally sanitized fixtures. ## Permissions From 21d8beb7ba73ea7581abbc8ce4c9cdc8f79ca46e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 15:29:38 +0200 Subject: [PATCH 07/18] feat: support simulator audio probe --- README.md | 2 +- examples/test-app/package.json | 1 + examples/test-app/pnpm-lock.yaml | 32 +++- examples/test-app/src/screens/AudioScreen.tsx | 150 +++++++++++++++++- src/commands/observability/index.ts | 3 +- src/core/__tests__/capabilities.test.ts | 17 ++ src/core/capabilities.ts | 12 +- .../__tests__/session-observability.test.ts | 127 ++++++++++++++- src/daemon/handlers/session-observability.ts | 109 ++++++++----- src/daemon/types.ts | 2 +- src/utils/__tests__/args.test.ts | 3 + src/utils/cli-help.ts | 10 +- website/docs/docs/client-api.md | 2 +- website/docs/docs/commands.md | 2 +- website/docs/docs/debugging-profiling.md | 6 +- website/docs/docs/introduction.md | 2 +- 16 files changed, 406 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 4f5a06cf2..aecbb46a8 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ It works with native iOS and Android apps, plus apps built with Expo, Flutter, a - **Inspect** real app UI through structured accessibility snapshots, interactive refs like `@e3`, selectors, and React Native component trees. - **Interact** by opening apps, tapping, typing, scrolling, performing gestures, waiting, asserting state, handling alerts, and closing sessions. -- **Capture evidence** with screenshots, videos, logs, traces, network traffic, audio-level probes, performance samples, crash context, and React profiles. +- **Capture evidence** with screenshots, videos, logs, traces, network traffic, audio-level probes for browser and host-rendered simulator/emulator audio, performance samples, crash context, and React profiles. - **Replay workflows** by recording `.ad` scripts for local runs, CI, repeatable e2e checks, and strict Maestro YAML export when a flow needs to run in Maestro. - **Run across platforms** with iOS Simulator automation, Android Emulator automation, physical devices, tvOS, Android TV, macOS, Linux, and desktop app automation, so agents can see and feel the app they work on. diff --git a/examples/test-app/package.json b/examples/test-app/package.json index 72e64bdaa..128d7329d 100644 --- a/examples/test-app/package.json +++ b/examples/test-app/package.json @@ -13,6 +13,7 @@ "@expo/dom-webview": "~56.0.5", "@expo/metro-runtime": "~56.0.15", "expo": "~56.0.12", + "expo-audio": "~56.0.12", "expo-constants": "56.0.18", "expo-dev-client": "~56.0.20", "expo-linking": "56.0.14", diff --git a/examples/test-app/pnpm-lock.yaml b/examples/test-app/pnpm-lock.yaml index c5dfd7811..31d27dd60 100644 --- a/examples/test-app/pnpm-lock.yaml +++ b/examples/test-app/pnpm-lock.yaml @@ -25,6 +25,9 @@ importers: expo: specifier: ~56.0.12 version: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo-audio: + specifier: ~56.0.12 + version: 56.0.12(expo-asset@56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3) expo-constants: specifier: 56.0.18 version: 56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)) @@ -1453,6 +1456,14 @@ packages: react: '*' react-native: '*' + expo-audio@56.0.12: + resolution: {integrity: sha512-ne2UIO/HsQoBL9e+tGs5N9Sf3NyW5sJMm4sDkexbSJRc2IchLDG+9Msu/+l5N4RlZ8SiF42wRyWsh/Usg+SwOw==} + peerDependencies: + expo: '*' + expo-asset: '*' + react: '*' + react-native: '*' + expo-constants@56.0.18: resolution: {integrity: sha512-8AMtbDGl/WVPnWlmbpGmvcdnNCy9E4PFnwdVwj600vljkMDPSxcAcjw8GVXEPk3PpZ+ngTqsrkltWyj0UKYAxw==} peerDependencies: @@ -3007,7 +3018,7 @@ snapshots: '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 '@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.29.0)': dependencies: @@ -3056,7 +3067,7 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/helper-create-class-features-plugin': 7.28.6(@babel/core@7.29.0) - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 transitivePeerDependencies: - supports-color @@ -3074,7 +3085,7 @@ snapshots: '@babel/helper-annotate-as-pure': 7.27.3 '@babel/helper-compilation-targets': 7.28.6 '@babel/helper-globals': 7.28.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0) '@babel/traverse': 7.29.0 transitivePeerDependencies: @@ -3129,7 +3140,7 @@ snapshots: '@babel/plugin-transform-nullish-coalescing-operator@7.28.6(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 '@babel/plugin-transform-object-rest-spread@7.28.6(@babel/core@7.29.0)': dependencies: @@ -3150,7 +3161,7 @@ snapshots: '@babel/plugin-transform-optional-chaining@7.28.6(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 transitivePeerDependencies: - supports-color @@ -3258,12 +3269,12 @@ snapshots: dependencies: '@babel/core': 7.29.0 '@babel/helper-create-regexp-features-plugin': 7.28.5(@babel/core@7.29.0) - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 '@babel/preset-typescript@7.28.5(@babel/core@7.29.0)': dependencies: '@babel/core': 7.29.0 - '@babel/helper-plugin-utils': 7.28.6 + '@babel/helper-plugin-utils': 7.29.7 '@babel/helper-validator-option': 7.27.1 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0) @@ -4533,6 +4544,13 @@ snapshots: - supports-color - typescript + expo-audio@56.0.12(expo-asset@56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3))(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3): + dependencies: + expo: 56.0.12(@babel/core@7.29.0)(@expo/dom-webview@56.0.5)(@expo/metro-runtime@56.0.15)(expo-router@56.2.11)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo-asset: 56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + react: 19.2.3 + react-native: 0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3) + expo-constants@56.0.18(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.0)(@react-native/metro-config@0.86.0(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.3)): dependencies: '@expo/env': 2.3.0 diff --git a/examples/test-app/src/screens/AudioScreen.tsx b/examples/test-app/src/screens/AudioScreen.tsx index ba360045e..3cb908d93 100644 --- a/examples/test-app/src/screens/AudioScreen.tsx +++ b/examples/test-app/src/screens/AudioScreen.tsx @@ -1,4 +1,5 @@ import { createElement, useEffect, useRef, useState } from 'react'; +import { createAudioPlayer, type AudioPlayer } from 'expo-audio'; import { Platform, ScrollView, StyleSheet, Text, View } from 'react-native'; import { ActionButton, InlineBadge, ScreenTitle, SectionCard } from '../components'; @@ -15,11 +16,16 @@ type SamplePlayback = { stop: () => void; }; +const SAMPLE_DURATION_SECONDS = 6; +const SAMPLE_FREQUENCY_HZ = 440; + export function AudioScreen() { const colors = useAppColors(); const styles = createStyles(colors); const audioRef = useRef(null); const playbackRef = useRef(null); + const nativePlayerRef = useRef(null); + const nativeEndTimerRef = useRef | null>(null); const [playbackState, setPlaybackState] = useState< 'ready' | 'playing' | 'paused' | 'ended' | 'error' >('ready'); @@ -33,10 +39,15 @@ export function AudioScreen() { audio.pause(); audio.srcObject = null; } + stopNativeSample(); }; }, []); function playSample() { + if (Platform.OS !== 'web') { + playNativeSample(); + return; + } const audio = audioRef.current; if (!audio) return; stopSample('ready'); @@ -54,6 +65,10 @@ export function AudioScreen() { } function pauseSample() { + if (Platform.OS !== 'web') { + pauseNativeSample(); + return; + } stopSample('paused'); } @@ -68,16 +83,54 @@ export function AudioScreen() { setPlaybackState(nextState); } + function playNativeSample() { + stopNativeSample(); + try { + const player = createAudioPlayer({ uri: createNativeBeepDataUri(), name: 'Agent Device beep' }); + nativePlayerRef.current = player; + player.play(); + setPlaybackState('playing'); + nativeEndTimerRef.current = setTimeout(() => { + stopNativeSample(); + setPlaybackState('ended'); + }, SAMPLE_DURATION_SECONDS * 1000); + } catch { + stopNativeSample(); + setPlaybackState('error'); + } + } + + function pauseNativeSample() { + stopNativeSample(); + setPlaybackState('paused'); + } + + function stopNativeSample() { + if (nativeEndTimerRef.current) { + clearTimeout(nativeEndTimerRef.current); + nativeEndTimerRef.current = null; + } + const player = nativePlayerRef.current; + nativePlayerRef.current = null; + if (!player) return; + try { + player.pause(); + } catch { + // Playback may already have finished. + } + player.remove(); + } + return ( - + {Platform.OS === 'web' ? ( {createElement('audio', { @@ -119,8 +172,33 @@ export function AudioScreen() { ) : ( - - Browser audio sample + + + Native audio sample + + + + + + + + + + )} @@ -140,12 +218,12 @@ function createBeepStream(onEnded: () => void): SamplePlayback { const destination = context.createMediaStreamDestination(); const oscillator = context.createOscillator(); const gain = context.createGain(); - const durationSeconds = 6; + const durationSeconds = SAMPLE_DURATION_SECONDS; const startAt = context.currentTime + 0.03; const endAt = startAt + durationSeconds; oscillator.type = 'square'; - oscillator.frequency.setValueAtTime(440, startAt); + oscillator.frequency.setValueAtTime(SAMPLE_FREQUENCY_HZ, startAt); gain.gain.setValueAtTime(0.0001, startAt); gain.gain.exponentialRampToValueAtTime(0.35, startAt + 0.05); gain.gain.setValueAtTime(0.35, endAt - 0.08); @@ -172,6 +250,66 @@ function createBeepStream(onEnded: () => void): SamplePlayback { }; } +function createNativeBeepDataUri(): string { + const sampleRate = 8000; + const dataSize = sampleRate * SAMPLE_DURATION_SECONDS; + const headerSize = 44; + const bytes = new Uint8Array(headerSize + dataSize); + writeAscii(bytes, 0, 'RIFF'); + writeUint32(bytes, 4, 36 + dataSize); + writeAscii(bytes, 8, 'WAVE'); + writeAscii(bytes, 12, 'fmt '); + writeUint32(bytes, 16, 16); + writeUint16(bytes, 20, 1); + writeUint16(bytes, 22, 1); + writeUint32(bytes, 24, sampleRate); + writeUint32(bytes, 28, sampleRate); + writeUint16(bytes, 32, 1); + writeUint16(bytes, 34, 8); + writeAscii(bytes, 36, 'data'); + writeUint32(bytes, 40, dataSize); + + for (let index = 0; index < dataSize; index += 1) { + const cycle = Math.sin((2 * Math.PI * SAMPLE_FREQUENCY_HZ * index) / sampleRate); + bytes[headerSize + index] = Math.round(128 + cycle * 96); + } + + return `data:audio/wav;base64,${base64Encode(bytes)}`; +} + +function writeAscii(bytes: Uint8Array, offset: number, value: string) { + for (let index = 0; index < value.length; index += 1) { + bytes[offset + index] = value.charCodeAt(index); + } +} + +function writeUint16(bytes: Uint8Array, offset: number, value: number) { + bytes[offset] = value & 0xff; + bytes[offset + 1] = (value >> 8) & 0xff; +} + +function writeUint32(bytes: Uint8Array, offset: number, value: number) { + bytes[offset] = value & 0xff; + bytes[offset + 1] = (value >> 8) & 0xff; + bytes[offset + 2] = (value >> 16) & 0xff; + bytes[offset + 3] = (value >> 24) & 0xff; +} + +function base64Encode(bytes: Uint8Array): string { + const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + let output = ''; + for (let index = 0; index < bytes.length; index += 3) { + const first = bytes[index]; + const second = bytes[index + 1]; + const third = bytes[index + 2]; + output += alphabet[first >> 2]; + output += alphabet[((first & 0x03) << 4) | ((second ?? 0) >> 4)]; + output += index + 1 < bytes.length ? alphabet[((second & 0x0f) << 2) | ((third ?? 0) >> 6)] : '='; + output += index + 2 < bytes.length ? alphabet[(third ?? 0) & 0x3f] : '='; + } + return output; +} + function createStyles(colors: AppColors) { return StyleSheet.create({ content: { diff --git a/src/commands/observability/index.ts b/src/commands/observability/index.ts index 0ad6137f7..9b8621b8c 100644 --- a/src/commands/observability/index.ts +++ b/src/commands/observability/index.ts @@ -101,7 +101,8 @@ const audioCliSchema = { usageOverride: 'audio probe start [durationSeconds] [bucketMs] | audio probe status | audio probe stop', listUsageOverride: 'audio', - helpDescription: 'Probe browser or macOS audio levels as compact dBFS buckets', + helpDescription: + 'Probe browser or host-rendered simulator/emulator audio as compact dBFS buckets', summary: 'Probe audio levels', positionalArgs: ['probe', 'start|status|stop', 'durationSeconds?', 'bucketMs?'], } as const satisfies CommandSchemaOverride; diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index 062f6546f..8b858d0ce 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -390,6 +390,7 @@ test('Linux supports desktop interaction commands and blocks mobile/unsupported test('web supports only the initial browser interaction slice', () => { assertCommandSupport( [ + 'audio', 'click', 'close', 'fill', @@ -440,6 +441,22 @@ test('web supports only the initial browser interaction slice', () => { ); }); +test('audio probe support is limited to browser and host-rendered audio targets', () => { + const hostAudioSupported = process.platform === 'darwin'; + assertCommandSupport( + ['audio'], + [ + { device: webDevice, expected: true, label: 'on web' }, + { device: macOsDevice, expected: hostAudioSupported, label: 'on macOS host' }, + { device: iosSimulator, expected: hostAudioSupported, label: 'on iOS simulator' }, + { device: androidEmulator, expected: hostAudioSupported, label: 'on Android emulator' }, + { device: iosDevice, expected: false, label: 'on iOS physical device' }, + { device: androidDevice, expected: false, label: 'on Android physical device' }, + { device: linuxDevice, expected: false, label: 'on Linux desktop' }, + ], + ); +}); + test('apple selector does not match web platform', () => { assert.equal(matchesPlatformSelector(webDevice.platform, 'apple'), false); assert.equal(matchesPlatformSelector(webDevice.platform, 'web'), true); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 58131647a..b5f77126a 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -22,6 +22,12 @@ const isMacOsOrAppleSimulator = (device: DeviceInfo): boolean => device.platform === 'macos' || device.kind === 'simulator'; const isIosMobileSimulator = (device: DeviceInfo): boolean => device.platform === 'ios' && device.kind === 'simulator' && device.target !== 'tv'; +const isHostSystemAudioProbeDevice = (device: DeviceInfo): boolean => + device.platform === 'web' || + (process.platform === 'darwin' && + (device.platform === 'macos' || + (device.platform === 'ios' && device.kind === 'simulator') || + (device.platform === 'android' && device.kind === 'emulator'))); // Two-finger gesture synthesis (RunnerSynthesizedGesture) is iOS-simulator-only (plus Android). // When such a gesture is rejected at admission, explain where it IS available so an agent can @@ -200,10 +206,10 @@ const BASE_COMMAND_CAPABILITY_MATRIX: Record = { linux: LINUX_NONE, }, audio: { - apple: { device: true }, - android: {}, + apple: { simulator: true, device: true }, + android: { emulator: true }, linux: LINUX_NONE, - supports: (device) => device.platform === 'macos' || device.platform === 'web', + supports: isHostSystemAudioProbeDevice, }, network: { apple: { simulator: true, device: true }, diff --git a/src/daemon/handlers/__tests__/session-observability.test.ts b/src/daemon/handlers/__tests__/session-observability.test.ts index a40fea364..61932dd0f 100644 --- a/src/daemon/handlers/__tests__/session-observability.test.ts +++ b/src/daemon/handlers/__tests__/session-observability.test.ts @@ -10,6 +10,7 @@ import { makeIosSession, makeMacOsSession, makeSession, + IOS_DEVICE, WEB_DESKTOP_DEVICE, } from '../../../__tests__/test-utils/index.ts'; import { AppError } from '../../../utils/errors.ts'; @@ -438,23 +439,23 @@ test('audio probe validates daemon duration bounds', async () => { test('audio probe rejects non-web sessions in daemon handler', async () => { const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); - sessionStore.set('android', makeAndroidSession('android')); + sessionStore.set('ios-device', makeIosSession('ios-device', { device: IOS_DEVICE })); const response = await handleSessionObservabilityCommands({ req: { token: 't', - session: 'android', + session: 'ios-device', command: 'audio', positionals: ['probe', 'status'], flags: {}, }, - sessionName: 'android', + sessionName: 'ios-device', sessionStore, }); assert.equal(response?.ok, false); if (response && !response.ok) { assert.equal(response.error.code, 'UNSUPPORTED_OPERATION'); - assert.match(response.error.message, /web browser and macOS sessions only/); + assert.match(response.error.message, /web browser sessions, macOS sessions, iOS simulators/); } }); @@ -481,7 +482,7 @@ test('audio probe starts macOS ScreenCaptureKit helper and reads status', async sourceCount: 1, rmsDbfs: [-12], peakDbfs: [-8], - notes: ['macOS status'], + notes: ['helper status'], }), ); return { @@ -508,6 +509,12 @@ test('audio probe starts macOS ScreenCaptureKit helper and reads status', async assert.equal(response.data?.backend, 'macos-screencapturekit'); assert.equal(response.data?.source, 'system-audio'); assert.deepEqual(response.data?.rmsDbfs, [-12]); + assert.deepEqual(response.data?.notes, [ + 'helper status', + 'Audio probe samples host system audio through ScreenCaptureKit for this macOS session; it is not app-instrumented audio.', + 'Screen Recording permission is required for host system audio capture.', + 'Other audible host apps can contribute to the measured buckets.', + ]); } assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls.length, 1); assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls[0]?.[0].durationMs, 1000); @@ -539,7 +546,7 @@ test('audio probe stop kills active macOS helper and returns stopped status', as ); const kill = vi.fn(); session.audioProbe = { - platform: 'macos', + platform: 'host-system-audio', child: { kill, pid: 1234 }, wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), statusPath, @@ -571,6 +578,114 @@ test('audio probe stop kills active macOS helper and returns stopped status', as } }); +test('audio probe starts host helper for iOS simulator audio', async () => { + const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); + sessionStore.set('ios', makeIosSession('ios')); + macosAudioMocks.startMacOsAudioProbeProcess.mockImplementation( + async (options: { durationMs: number; bucketMs: number; statusPath: string }) => { + await fsPromises.mkdir(path.dirname(options.statusPath), { recursive: true }); + await fsPromises.writeFile( + options.statusPath, + JSON.stringify({ + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: options.durationMs, + elapsedMs: 500, + bucketMs: options.bucketMs, + sampleCount: 1, + sourceCount: 1, + rmsDbfs: [-18], + peakDbfs: [-12], + }), + ); + return { + child: { kill: vi.fn(), pid: 1234 }, + wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), + }; + }, + ); + + const response = await handleSessionObservabilityCommands({ + req: { + token: 't', + session: 'ios', + command: 'audio', + positionals: ['probe', 'start', '1000', '500'], + flags: {}, + }, + sessionName: 'ios', + sessionStore, + }); + + assert.equal(response?.ok, true); + assert.equal(sessionStore.get('ios')?.audioProbe?.platform, 'host-system-audio'); + if (response?.ok) { + assert.equal(response.data?.source, 'system-audio'); + assert.deepEqual(response.data?.rmsDbfs, [-18]); + const notes = response.data?.notes; + assert.ok(Array.isArray(notes)); + assert.match(String(notes[0]), /iOS simulator/); + } +}); + +test('audio probe starts host helper for Android emulator audio', async () => { + const sessionStore = makeSessionStore('agent-device-session-observability-audio-'); + sessionStore.set('android', makeAndroidSession('android')); + macosAudioMocks.startMacOsAudioProbeProcess.mockImplementation( + async (options: { durationMs: number; bucketMs: number; statusPath: string }) => { + await fsPromises.mkdir(path.dirname(options.statusPath), { recursive: true }); + await fsPromises.writeFile( + options.statusPath, + JSON.stringify({ + audio: 'probe', + state: 'running', + active: true, + heard: true, + source: 'system-audio', + backend: 'macos-screencapturekit', + durationMs: options.durationMs, + elapsedMs: 500, + bucketMs: options.bucketMs, + sampleCount: 1, + sourceCount: 1, + rmsDbfs: [-20], + peakDbfs: [-13], + }), + ); + return { + child: { kill: vi.fn(), pid: 1234 }, + wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), + }; + }, + ); + + const response = await handleSessionObservabilityCommands({ + req: { + token: 't', + session: 'android', + command: 'audio', + positionals: ['probe', 'start', '1000', '500'], + flags: {}, + }, + sessionName: 'android', + sessionStore, + }); + + assert.equal(response?.ok, true); + assert.equal(sessionStore.get('android')?.audioProbe?.platform, 'host-system-audio'); + if (response?.ok) { + assert.equal(response.data?.source, 'system-audio'); + assert.deepEqual(response.data?.peakDbfs, [-13]); + const notes = response.data?.notes; + assert.ok(Array.isArray(notes)); + assert.match(String(notes[0]), /Android emulator/); + } +}); + test('audio probe validates daemon bucket bounds', async () => { const provider = makeAudioWebProvider(); const response = await runAudioCommand(['probe', 'start', '1000', '99'], provider); diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index 0326d66b8..c0f0690f2 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -623,8 +623,8 @@ function resolveNetworkIncludeMode( async function handleAudioCommand(params: ObservabilityParams): Promise { const request = resolveAudioCommandRequest(params); if (!request.ok) return request; - if (request.session.device.platform === 'macos') { - return await handleMacOsAudioCommand(params, request); + if (usesHostSystemAudioProbe(request.session.device)) { + return await handleHostSystemAudioCommand(params, request); } const provider = resolveWebProvider(); if (!provider.probeAudio) { @@ -678,7 +678,7 @@ function resolveAudioSession( ? { ok: true, session } : errorResponse( 'UNSUPPORTED_OPERATION', - 'audio is currently supported for web browser and macOS sessions only', + 'audio is supported for web browser sessions, macOS sessions, iOS simulators, and Android emulators on macOS hosts', ); } @@ -687,7 +687,7 @@ type ResolvedAudioCommandRequest = Extract< { ok: true } >; -type MacOsAudioProbeData = { +type HostSystemAudioProbeData = { audio: 'probe'; state: 'running' | 'stopped'; active: boolean; @@ -707,14 +707,22 @@ type MacOsAudioProbeData = { notes?: string[]; }; -async function handleMacOsAudioCommand( +function usesHostSystemAudioProbe(device: SessionState['device']): boolean { + return ( + device.platform === 'macos' || + (device.platform === 'ios' && device.kind === 'simulator') || + (device.platform === 'android' && device.kind === 'emulator') + ); +} + +async function handleHostSystemAudioCommand( params: ObservabilityParams, request: ResolvedAudioCommandRequest, ): Promise { const { session, probeAction } = request; try { if (probeAction === 'start') { - await stopMacOsAudioProbe(session); + await stopHostSystemAudioProbe(session); const statusPath = path.join( params.sessionStore.ensureSessionDir(params.sessionName), 'audio-probe.json', @@ -725,7 +733,7 @@ async function handleMacOsAudioCommand( statusPath, }); session.audioProbe = { - platform: 'macos', + platform: 'host-system-audio', child: probe.child, wait: probe.wait, statusPath, @@ -734,32 +742,34 @@ async function handleMacOsAudioCommand( bucketMs: request.bucketMs, }; void probe.wait.catch(() => {}); - return { ok: true, data: await waitForMacOsAudioProbeStatus(session) }; + return { ok: true, data: await waitForHostSystemAudioProbeStatus(session) }; } if (probeAction === 'stop') { - const data = await stopMacOsAudioProbe(session); + const data = await stopHostSystemAudioProbe(session); return { ok: true, - data: data ?? buildMacOsAudioProbeFallback(request, 'stopped', 'not-started'), + data: data ?? buildHostSystemAudioProbeFallback(request, 'stopped', 'not-started'), }; } - const data = await readMacOsAudioProbeStatus(session); + const data = await readHostSystemAudioProbeStatus(session); if (data) { if (data.state === 'stopped') session.audioProbe = undefined; return { ok: true, data }; } - return { ok: true, data: buildMacOsAudioProbeFallback(request, 'stopped', 'not-started') }; + return { ok: true, data: buildHostSystemAudioProbeFallback(request, 'stopped', 'not-started') }; } catch (error) { return { ok: false, error: normalizeError(error) }; } } -async function waitForMacOsAudioProbeStatus(session: SessionState): Promise { +async function waitForHostSystemAudioProbeStatus( + session: SessionState, +): Promise { const deadline = Date.now() + 5_000; while (Date.now() < deadline) { - const status = await readMacOsAudioProbeStatus(session); + const status = await readHostSystemAudioProbeStatus(session); if (status) return status; const exit = await Promise.race([ session.audioProbe?.wait.then( @@ -774,43 +784,44 @@ async function waitForMacOsAudioProbeStatus(session: SessionState): Promise { +): Promise { const probe = session.audioProbe; if (!probe) return undefined; try { const raw = await fs.readFile(probe.statusPath, 'utf8'); - return normalizeMacOsAudioProbeData(JSON.parse(raw), probe); + return normalizeHostSystemAudioProbeData(JSON.parse(raw), probe, session.device); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; throw error; } } -async function stopMacOsAudioProbe( +async function stopHostSystemAudioProbe( session: SessionState, -): Promise { +): Promise { const probe = session.audioProbe; if (!probe) return undefined; - const beforeStop = await readMacOsAudioProbeStatus(session); + const beforeStop = await readHostSystemAudioProbeStatus(session); probe.child.kill('SIGTERM'); await probe.wait.catch(() => {}); session.audioProbe = undefined; - return finalizeMacOsAudioProbeStatus(beforeStop, probe, 'stopped'); + return finalizeHostSystemAudioProbeStatus(beforeStop, probe, session.device, 'stopped'); } -function normalizeMacOsAudioProbeData( +function normalizeHostSystemAudioProbeData( value: unknown, probe: NonNullable, -): MacOsAudioProbeData { + device: SessionState['device'], +): HostSystemAudioProbeData { const record = value && typeof value === 'object' ? (value as Record) : {}; const state = record.state === 'stopped' ? 'stopped' : 'running'; const rmsDbfs = readNumberArray(record.rmsDbfs); @@ -832,15 +843,16 @@ function normalizeMacOsAudioProbeData( startedAt: typeof record.startedAt === 'string' ? record.startedAt : undefined, stoppedAt: typeof record.stoppedAt === 'string' ? record.stoppedAt : undefined, reason: typeof record.reason === 'string' ? record.reason : undefined, - notes: readStringArray(record.notes), + notes: mergeHostSystemAudioProbeNotes(readStringArray(record.notes), device), }; } -function finalizeMacOsAudioProbeStatus( - status: MacOsAudioProbeData | undefined, +function finalizeHostSystemAudioProbeStatus( + status: HostSystemAudioProbeData | undefined, probe: NonNullable, + device: SessionState['device'], reason: string, -): MacOsAudioProbeData { +): HostSystemAudioProbeData { const elapsedMs = Math.min(probe.durationMs, Math.max(0, Date.now() - probe.startedAt)); const base = status ?? @@ -858,11 +870,8 @@ function finalizeMacOsAudioProbeStatus( sourceCount: 1, rmsDbfs: [], peakDbfs: [], - notes: [ - 'Audio probe samples macOS system audio through ScreenCaptureKit; it is not app-instrumented audio.', - 'Screen Recording permission is required for macOS system audio capture.', - ], - } as MacOsAudioProbeData); + notes: hostSystemAudioProbeNotes(device), + } as HostSystemAudioProbeData); return { ...base, state: 'stopped', @@ -873,11 +882,11 @@ function finalizeMacOsAudioProbeStatus( }; } -function buildMacOsAudioProbeFallback( +function buildHostSystemAudioProbeFallback( request: ResolvedAudioCommandRequest, state: 'running' | 'stopped', reason?: string, -): MacOsAudioProbeData { +): HostSystemAudioProbeData { return { audio: 'probe', state, @@ -894,13 +903,33 @@ function buildMacOsAudioProbeFallback( peakDbfs: [], reason, notes: [ - 'Audio probe samples macOS system audio through ScreenCaptureKit; it is not app-instrumented audio.', - 'Screen Recording permission is required for macOS system audio capture.', - 'No active macOS audio probe is running.', + ...hostSystemAudioProbeNotes(request.session.device), + 'No active host audio probe is running.', ], }; } +function mergeHostSystemAudioProbeNotes( + notes: string[] | undefined, + device: SessionState['device'], +): string[] { + return [...(notes ?? []), ...hostSystemAudioProbeNotes(device)]; +} + +function hostSystemAudioProbeNotes(device: SessionState['device']): string[] { + const target = + device.platform === 'ios' + ? 'iOS simulator' + : device.platform === 'android' + ? 'Android emulator' + : 'macOS session'; + return [ + `Audio probe samples host system audio through ScreenCaptureKit for this ${target}; it is not app-instrumented audio.`, + 'Screen Recording permission is required for host system audio capture.', + 'Other audible host apps can contribute to the measured buckets.', + ]; +} + function readFiniteNumber(value: unknown, fallback: number): number { return typeof value === 'number' && Number.isFinite(value) ? value : fallback; } diff --git a/src/daemon/types.ts b/src/daemon/types.ts index 5677ea9c5..543f31993 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -268,7 +268,7 @@ export type SessionState = { android?: AndroidNativePerfSession; }; audioProbe?: { - platform: 'macos'; + platform: 'host-system-audio'; child: SessionRecordingProcessChild; wait: Promise; statusPath: string; diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 40ec99da7..396eaa07b 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -1540,8 +1540,11 @@ test('usageForCommand resolves debugging help topic', () => { assert.match(help, /agent-device perf memory sample --json/); assert.match(help, /agent-device audio probe start 10 1000 --platform web/); assert.match(help, /agent-device audio probe start 10 1000 --platform macos/); + assert.match(help, /agent-device audio probe start 10 1000 --platform ios/); + assert.match(help, /agent-device audio probe start 10 1000 --platform android/); assert.match(help, /compact rmsDbfs and peakDbfs arrays/); assert.match(help, /requires Screen Recording permission/); + assert.match(help, /Physical iOS and Android devices are not supported/); assert.match(help, /Memory artifact \(android-hprof\): \/tmp\/app\.hprof \(42MB\)/); assert.match(help, /Prefer perf memory sample over raw dumpsys\/leaks output/); assert.match(help, /Unsupported platforms return artifact\.available=false with reason\/hint/); diff --git a/src/utils/cli-help.ts b/src/utils/cli-help.ts index 09c8d2a98..57d7ee043 100644 --- a/src/utils/cli-help.ts +++ b/src/utils/cli-help.ts @@ -271,7 +271,7 @@ Validation and evidence: Remote lifecycle: cloud, remote-config, direct proxy, and limrun use the same flow: connect, open, commands, close, disconnect. Remote config profile: agent-device connect --remote-config ./remote-config.json; then run normal commands and disconnect. Direct proxy to a Mac you control: cloud/Linux clients can use local/proxy iOS devices through the proxied Mac. Run agent-device connect proxy --daemon-base-url first. Device leases are automatic on open and expire after five minutes of inactivity. - Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. On web, audio probe samples HTML media elements, and URL-backed media may be routed through the probe AudioContext while observed. On macOS, audio probe samples system audio through ScreenCaptureKit and requires Screen Recording permission. + Web: agent-device uses a managed, pinned agent-browser backend as an implementation detail. Use --platform web when a browser step belongs inside an agent-device session, replay, batch, MCP, or typed-client flow; use agent-browser directly for standalone web automation. Run agent-device web setup before first use, then agent-device web doctor for backend health checks. Web automation requires Node 24+. For audio probe start, the first timing positional is duration in seconds and the second is bucket size in milliseconds. On web, audio probe samples HTML media elements, and URL-backed media may be routed through the probe AudioContext while observed. On macOS hosts, audio probe samples host system audio through ScreenCaptureKit for macOS sessions, iOS simulators, and Android emulators; Screen Recording permission is required. agent-device web setup agent-device web doctor agent-device open https://example.com --platform web @@ -292,7 +292,7 @@ Validation and evidence: agent-device close --platform web Minimal web support is for browser sessions with open, snapshot, find, get, is, click/press, fill/type, wait, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay over those commands. Use agent-browser directly for browser-specific features that agent-device does not surface, such as tab/devtools management, advanced page scripting, network routing/HAR, or raw browser debugging. macOS menu bar: open ... --platform macos --surface menubar; snapshot -i --platform macos --surface menubar. - macOS audio: audio probe start 10 1000 --platform macos samples host system audio through ScreenCaptureKit; grant Screen Recording permission first. + Host audio: audio probe start 10 1000 --platform macos|ios|android samples host system audio through ScreenCaptureKit for macOS sessions, iOS simulators, and Android emulators on macOS hosts; grant Screen Recording permission first. Maestro full-suite validation on explicit connected devices uses one test command with a comma-separated --device list and --shard-all. Use --shard-split only when splitting suite entries across devices: agent-device test ./e2e/maestro --maestro --device udid1,emulator-5554 --shard-all 2 @@ -351,14 +351,16 @@ Network: network log is a supported alias, but network dump --include headers is the clearest plan form. Do not write network log headers. Audio: - Use audio probe when the question is whether a browser page or macOS host produced audible output during a short observation window. + Use audio probe when the question is whether a browser page, macOS session, iOS simulator, or Android emulator produced audible output during a short observation window. agent-device audio probe start 10 1000 --platform web agent-device audio probe status --platform web agent-device audio probe stop --platform web agent-device audio probe start 10 1000 --platform macos + agent-device audio probe start 10 1000 --platform ios + agent-device audio probe start 10 1000 --platform android audio probe start uses duration seconds first, then bucket milliseconds. Results are compact rmsDbfs and peakDbfs arrays so agents can correlate audible moments with screenshots, actions, network entries, or frame samples. On web, audio probe samples HTML media elements and URL-backed media may be routed through the probe AudioContext while observed. - On macOS, audio probe samples host system audio through ScreenCaptureKit and requires Screen Recording permission. It is system-audio evidence, not app-instrumented audio. + On macOS hosts, audio probe samples host system audio through ScreenCaptureKit for macOS sessions, iOS simulators, and Android emulators. It requires Screen Recording permission and is system-audio evidence, not app-instrumented audio. Physical iOS and Android devices are not supported. Crash symbolication: Crash routing: diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 5d0fd3430..5e830c262 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -311,7 +311,7 @@ const audio = await client.observability.audio({ await client.observability.audio({ platform: 'web', action: 'probe', probeAction: 'stop' }); ``` -Web probes sample HTML media elements. macOS probes use `platform: 'macos'`, sample host system audio through ScreenCaptureKit, and require Screen Recording permission. iOS and Android app audio are not exposed by this command. +Web probes sample HTML media elements. Host-system probes use `platform: 'macos'`, `platform: 'ios'` for iOS simulators, or `platform: 'android'` for Android emulators on macOS hosts. They sample host system audio through ScreenCaptureKit and require Screen Recording permission. Physical iOS and Android app audio are not exposed by this command. `client.observability.perf()` returns daemon-shaped JSON so local and remote transports expose the same metrics payload. Pass `{ area: 'metrics' }` for the broad startup/CPU/memory/frame first pass, `{ area: 'frames' }` for a focused frame/jank-health payload, or `{ area: 'memory', action: 'sample' }` for a compact memory-only sample. Use `{ area: 'memory', action: 'snapshot', kind: 'android-hprof', out: 'app.hprof' }` on Android or `{ area: 'memory', action: 'snapshot', kind: 'memgraph', out: 'app.memgraph' }` on supported Apple simulator/macOS app sessions to write large memory artifacts to disk. Android native artifacts use `{ area: 'cpu', subject: 'profile', action: 'start' | 'stop' | 'report', kind: 'simpleperf', out }` and `{ area: 'trace', action: 'start' | 'stop', kind: 'perfetto', out }`; these Android-only commands return artifact paths and compact summaries, not trace/profile contents. Physical iOS device memgraph capture reports unavailable with a reason/hint. heapprofd allocation tracing is deferred until Perfetto plumbing is available. On Android and supported Apple targets, `data.metrics.fps.droppedFramePercent` is the primary frame-smoothness value. Android derives it from the current `adb shell dumpsys gfxinfo framestats` window; connected iOS devices derive it from `xcrun xctrace` Animation Hitches for the active app process. Frame samples include `windowStartedAt`, `windowEndedAt`, and `worstWindows` so agents can correlate dropped-frame clusters with logs, network entries, and their own session actions. A successful Android read resets Android frame stats; `open ` resets the Android frame window too, so agents can call `perf({ area: 'frames' })`, perform a transition or gesture, then call it again to inspect that focused window. iOS simulator and macOS app sessions report frame health as unavailable rather than inventing FPS or dropped-frame values. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cd4a69364..86625cfc1 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -223,7 +223,7 @@ agent-device snapshot -i --platform apple --target desktop - Use `frontmost-app`, `desktop`, and `menubar` mainly for `snapshot`, `get`, `is`, and `wait`. - If you inspect with `desktop` or `menubar` and then need to click or fill inside one app, open that app in a normal `app` session. - macOS also supports `clipboard read|write`, `trigger-app-event`, `logs`, `network dump`, `audio probe`, `alert`, `settings appearance`, and `settings permission `. -- macOS `audio probe start 10 1000 --platform macos` samples host system audio through ScreenCaptureKit. Grant Screen Recording permission before relying on it in a run. +- `audio probe start 10 1000 --platform macos` samples host system audio through ScreenCaptureKit. The same host-system audio backend is used for iOS simulators and Android emulators on macOS hosts; grant Screen Recording permission before relying on it in a run. - In macOS app sessions, `screenshot` captures the target app window bounds rather than the full desktop. - Prefer selector or `@ref`-driven interactions on macOS. Window position can shift between runs, so raw x/y point commands are less stable than snapshot-derived targets. - Use `click --button secondary` for context menus on macOS, then run `snapshot -i` again. diff --git a/website/docs/docs/debugging-profiling.md b/website/docs/docs/debugging-profiling.md index caf72e044..0af55a43c 100644 --- a/website/docs/docs/debugging-profiling.md +++ b/website/docs/docs/debugging-profiling.md @@ -10,7 +10,7 @@ Use `agent-device` when the task moves past UI automation and you need runtime e - Session app logs for targeted debugging windows - Network inspection from recent HTTP(s) entries in app logs via `network dump` -- Audio-level probes for browser media elements and macOS system audio +- Audio-level probes for browser media elements and host-rendered simulator/emulator audio - Performance snapshots with `perf metrics` / `perf frames` - Apple crash symbolication with `debug symbols` - Screenshots, recordings, and replayable repro flows @@ -146,11 +146,13 @@ agent-device audio probe start 10 1000 --platform web agent-device audio probe status --platform web agent-device audio probe stop --platform web agent-device audio probe start 10 1000 --platform macos +agent-device audio probe start 10 1000 --platform ios +agent-device audio probe start 10 1000 --platform android ``` - `audio probe start [durationSeconds] [bucketMs]` samples live audio while the session keeps running, then exposes compact `rmsDbfs` and `peakDbfs` buckets. The first timing positional is seconds; the second is milliseconds. - On web, the probe samples HTML media elements through Web Audio. URL-backed media may be routed through the probe `AudioContext` while observed. -- On macOS, the probe samples host system audio through ScreenCaptureKit and requires Screen Recording permission. It is system-audio evidence, not app-instrumented audio. +- On macOS hosts, the probe samples host system audio through ScreenCaptureKit for macOS sessions, iOS simulators, and Android emulators. It requires Screen Recording permission and is system-audio evidence, not app-instrumented audio. Physical iOS and Android devices are not supported. - Use `status` to poll partial buckets during a 10-20 second observation window, and `stop` to end the probe early. ### Performance snapshots diff --git a/website/docs/docs/introduction.md b/website/docs/docs/introduction.md index d78faf44d..b16c2fd0a 100644 --- a/website/docs/docs/introduction.md +++ b/website/docs/docs/introduction.md @@ -15,7 +15,7 @@ Use it when an agent needs to inspect and operate a real app, not just reason ab - **App verification for agents**: run the app, inspect visible UI, act through refs/selectors, and verify expected state. - **Token-efficient UI context**: accessibility snapshots give agents structured UI state instead of screenshot-only reasoning. -- **Runtime evidence**: capture screenshots, recordings, logs, network traffic, audio-level probes, traces, CPU/memory/perf snapshots, and crash-related logs when the happy path breaks. +- **Runtime evidence**: capture screenshots, recordings, logs, network traffic, audio-level probes for browser and host-rendered simulator/emulator audio, traces, CPU/memory/perf snapshots, and crash-related logs when the happy path breaks. - **Replayable checks**: turn stable exploratory sessions into `.ad` replay scripts that can run again without AI. - **React Native and Expo workflows**: pair device automation with optional React DevTools profiling for component trees, props/state/hooks, slow renders, and rerenders. - **Local devices and app surfaces**: drive simulators, emulators, physical devices, TV targets, desktop apps, and browser sessions through one CLI. From d023734b3eceee5d522a07834b1d217706394d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 15:33:49 +0200 Subject: [PATCH 08/18] test: account for host audio platform support --- .../__tests__/session-observability.test.ts | 91 ++++++++++++------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/src/daemon/handlers/__tests__/session-observability.test.ts b/src/daemon/handlers/__tests__/session-observability.test.ts index 61932dd0f..e99d942b5 100644 --- a/src/daemon/handlers/__tests__/session-observability.test.ts +++ b/src/daemon/handlers/__tests__/session-observability.test.ts @@ -504,18 +504,22 @@ test('audio probe starts macOS ScreenCaptureKit helper and reads status', async sessionStore, }); - assert.equal(response?.ok, true); - if (response?.ok) { - assert.equal(response.data?.backend, 'macos-screencapturekit'); - assert.equal(response.data?.source, 'system-audio'); - assert.deepEqual(response.data?.rmsDbfs, [-12]); - assert.deepEqual(response.data?.notes, [ - 'helper status', - 'Audio probe samples host system audio through ScreenCaptureKit for this macOS session; it is not app-instrumented audio.', - 'Screen Recording permission is required for host system audio capture.', - 'Other audible host apps can contribute to the measured buckets.', - ]); + if (process.platform !== 'darwin') { + assertHostAudioUnsupportedResponse(response); + assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls.length, 0); + return; } + + assert.ok(response?.ok); + assert.equal(response.data?.backend, 'macos-screencapturekit'); + assert.equal(response.data?.source, 'system-audio'); + assert.deepEqual(response.data?.rmsDbfs, [-12]); + assert.deepEqual(response.data?.notes, [ + 'helper status', + 'Audio probe samples host system audio through ScreenCaptureKit for this macOS session; it is not app-instrumented audio.', + 'Screen Recording permission is required for host system audio capture.', + 'Other audible host apps can contribute to the measured buckets.', + ]); assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls.length, 1); assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls[0]?.[0].durationMs, 1000); assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls[0]?.[0].bucketMs, 500); @@ -568,14 +572,18 @@ test('audio probe stop kills active macOS helper and returns stopped status', as sessionStore, }); - assert.equal(response?.ok, true); + if (process.platform !== 'darwin') { + assertHostAudioUnsupportedResponse(response); + assert.equal(kill.mock.calls.length, 0); + return; + } + + assert.ok(response?.ok); assert.equal(kill.mock.calls[0]?.[0], 'SIGTERM'); assert.equal(sessionStore.get('macos')?.audioProbe, undefined); - if (response?.ok) { - assert.equal(response.data?.state, 'stopped'); - assert.equal(response.data?.active, false); - assert.deepEqual(response.data?.peakDbfs, [-9, -8]); - } + assert.equal(response.data?.state, 'stopped'); + assert.equal(response.data?.active, false); + assert.deepEqual(response.data?.peakDbfs, [-9, -8]); }); test('audio probe starts host helper for iOS simulator audio', async () => { @@ -621,15 +629,18 @@ test('audio probe starts host helper for iOS simulator audio', async () => { sessionStore, }); - assert.equal(response?.ok, true); - assert.equal(sessionStore.get('ios')?.audioProbe?.platform, 'host-system-audio'); - if (response?.ok) { - assert.equal(response.data?.source, 'system-audio'); - assert.deepEqual(response.data?.rmsDbfs, [-18]); - const notes = response.data?.notes; - assert.ok(Array.isArray(notes)); - assert.match(String(notes[0]), /iOS simulator/); + if (process.platform !== 'darwin') { + assertHostAudioUnsupportedResponse(response); + assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls.length, 0); + return; } + + assert.ok(response?.ok); + assert.equal(sessionStore.get('ios')?.audioProbe?.platform, 'host-system-audio'); + assert.equal(response.data?.source, 'system-audio'); + assert.deepEqual(response.data?.rmsDbfs, [-18]); + assert.ok(Array.isArray(response.data?.notes)); + assert.match(String(response.data.notes[0]), /iOS simulator/); }); test('audio probe starts host helper for Android emulator audio', async () => { @@ -675,15 +686,18 @@ test('audio probe starts host helper for Android emulator audio', async () => { sessionStore, }); - assert.equal(response?.ok, true); - assert.equal(sessionStore.get('android')?.audioProbe?.platform, 'host-system-audio'); - if (response?.ok) { - assert.equal(response.data?.source, 'system-audio'); - assert.deepEqual(response.data?.peakDbfs, [-13]); - const notes = response.data?.notes; - assert.ok(Array.isArray(notes)); - assert.match(String(notes[0]), /Android emulator/); + if (process.platform !== 'darwin') { + assertHostAudioUnsupportedResponse(response); + assert.equal(macosAudioMocks.startMacOsAudioProbeProcess.mock.calls.length, 0); + return; } + + assert.ok(response?.ok); + assert.equal(sessionStore.get('android')?.audioProbe?.platform, 'host-system-audio'); + assert.equal(response.data?.source, 'system-audio'); + assert.deepEqual(response.data?.peakDbfs, [-13]); + assert.ok(Array.isArray(response.data?.notes)); + assert.match(String(response.data.notes[0]), /Android emulator/); }); test('audio probe validates daemon bucket bounds', async () => { @@ -1272,6 +1286,17 @@ function assertInvalidArgs(response: DaemonResponse | null, message: RegExp): vo } } +function assertHostAudioUnsupportedResponse(response: DaemonResponse | null): void { + assert.equal(response?.ok, false); + if (response && !response.ok) { + assert.equal(response.error.code, 'UNSUPPORTED_OPERATION'); + assert.match( + response.error.message, + /web browser sessions, macOS sessions, iOS simulators, and Android emulators on macOS hosts/, + ); + } +} + function makeAudioWebProvider(): WebProvider & { probeAudio: ReturnType>>; } { From 71dd363c202c40c082755c9b8d0aacde7f9e6d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 16:46:12 +0200 Subject: [PATCH 09/18] refactor: deepen audio probe lifecycle --- src/audio-probe-result.ts | 108 +++++++ src/daemon/audio-probe.ts | 215 ++++++++++++++ .../__tests__/session-close-shutdown.test.ts | 44 +++ src/daemon/handlers/session-close.ts | 2 +- src/daemon/handlers/session-observability.ts | 266 +----------------- src/daemon/session-teardown.ts | 13 +- src/platforms/host-system-audio.ts | 14 + src/platforms/web/agent-browser-provider.ts | 56 +--- src/platforms/web/provider.ts | 19 +- 9 files changed, 408 insertions(+), 329 deletions(-) create mode 100644 src/audio-probe-result.ts create mode 100644 src/daemon/audio-probe.ts create mode 100644 src/platforms/host-system-audio.ts diff --git a/src/audio-probe-result.ts b/src/audio-probe-result.ts new file mode 100644 index 000000000..2229f9596 --- /dev/null +++ b/src/audio-probe-result.ts @@ -0,0 +1,108 @@ +export type AudioProbeSource = 'media-elements' | 'system-audio'; + +export type AudioProbeResult = { + audio: 'probe'; + state: 'running' | 'stopped'; + active: boolean; + heard: boolean; + source: AudioProbeSource; + backend?: string; + durationMs: number; + elapsedMs: number; + bucketMs: number; + sampleCount: number; + mediaElementCount?: number; + sourceCount: number; + rmsDbfs: number[]; + peakDbfs: number[]; + startedAt?: string; + stoppedAt?: string; + reason?: string; + notes?: string[]; +}; + +export type NormalizeAudioProbeRecordOptions = { + source: AudioProbeSource; + backend: string; + durationMs: number; + elapsedMs: number; + bucketMs: number; + activeFallback?: boolean; + mediaElementCount?: number; + sourceCount?: number; + notes?: string[]; +}; + +export function normalizeAudioProbeRecord( + value: unknown, + options: NormalizeAudioProbeRecordOptions, +): AudioProbeResult { + const record = readRecord(value); + const state = readAudioProbeState(record); + const rmsDbfs = readNumberArray(record.rmsDbfs); + const peakDbfs = readNumberArray(record.peakDbfs); + const notes = [...(readStringArray(record.notes) ?? []), ...(options.notes ?? [])]; + return { + audio: 'probe', + state, + active: state === 'running' && readBoolean(record.active, options.activeFallback ?? true), + heard: record.heard === true, + source: options.source, + backend: readString(record.backend) ?? options.backend, + durationMs: readFiniteNumber(record.durationMs, options.durationMs), + elapsedMs: readFiniteNumber(record.elapsedMs, options.elapsedMs), + bucketMs: readFiniteNumber(record.bucketMs, options.bucketMs), + sampleCount: readFiniteNumber(record.sampleCount, rmsDbfs.length), + mediaElementCount: + options.mediaElementCount === undefined + ? readOptionalFiniteNumber(record.mediaElementCount) + : readFiniteNumber(record.mediaElementCount, options.mediaElementCount), + sourceCount: readFiniteNumber(record.sourceCount, options.sourceCount ?? 0), + rmsDbfs, + peakDbfs, + startedAt: readString(record.startedAt), + stoppedAt: readString(record.stoppedAt), + reason: readString(record.reason), + notes: notes.length > 0 ? notes : undefined, + }; +} + +function readAudioProbeState(record: Record): 'running' | 'stopped' { + return record.state === 'running' ? 'running' : 'stopped'; +} + +function readFiniteNumber(value: unknown, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +function readNumberArray(value: unknown): number[] { + if (!Array.isArray(value)) return []; + const numbers: number[] = []; + for (const item of value) { + if (typeof item === 'number' && Number.isFinite(item)) numbers.push(item); + } + return numbers; +} + +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + return value.filter((item): item is string => typeof item === 'string'); +} + +function readRecord(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function readBoolean(value: unknown, fallback: boolean): boolean { + return typeof value === 'boolean' ? value : fallback; +} + +function readString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function readOptionalFiniteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} diff --git a/src/daemon/audio-probe.ts b/src/daemon/audio-probe.ts new file mode 100644 index 000000000..36b0022c7 --- /dev/null +++ b/src/daemon/audio-probe.ts @@ -0,0 +1,215 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { normalizeAudioProbeRecord, type AudioProbeResult } from '../audio-probe-result.ts'; +import { startHostSystemAudioProbeProcess } from '../platforms/host-system-audio.ts'; +import { AppError } from '../utils/errors.ts'; +import { sleep } from '../utils/timeouts.ts'; +import type { SessionStore } from './session-store.ts'; +import type { SessionState } from './types.ts'; + +const HOST_AUDIO_BACKEND = 'macos-screencapturekit'; + +export type HostAudioProbeCommand = { + session: SessionState; + sessionName: string; + sessionStore: SessionStore; + probeAction: 'start' | 'status' | 'stop'; + durationMs: number; + bucketMs: number; +}; + +export function usesHostSystemAudioProbe(device: SessionState['device']): boolean { + return ( + device.platform === 'macos' || + (device.platform === 'ios' && device.kind === 'simulator') || + (device.platform === 'android' && device.kind === 'emulator') + ); +} + +export async function runHostSystemAudioProbeCommand( + request: HostAudioProbeCommand, +): Promise { + const { session, probeAction } = request; + if (probeAction === 'start') { + await stopSessionAudioProbe(session, 'restarted'); + const statusPath = path.join( + request.sessionStore.ensureSessionDir(request.sessionName), + 'audio-probe.json', + ); + const probe = await startHostSystemAudioProbeProcess({ + durationMs: request.durationMs, + bucketMs: request.bucketMs, + statusPath, + }); + session.audioProbe = { + platform: 'host-system-audio', + child: probe.child, + wait: probe.wait, + statusPath, + startedAt: Date.now(), + durationMs: request.durationMs, + bucketMs: request.bucketMs, + }; + void probe.wait.catch(() => {}); + return await waitForHostSystemAudioProbeStatus(session); + } + + if (probeAction === 'stop') { + return ( + (await stopSessionAudioProbe(session, 'stopped')) ?? + buildHostSystemAudioProbeFallback(request, 'stopped', 'not-started') + ); + } + + const data = await readHostSystemAudioProbeStatus(session); + if (data) { + if (data.state === 'stopped') session.audioProbe = undefined; + return data; + } + return buildHostSystemAudioProbeFallback(request, 'stopped', 'not-started'); +} + +export async function stopSessionAudioProbe( + session: SessionState, + reason = 'session-cleanup', +): Promise { + const probe = session.audioProbe; + if (!probe) return undefined; + const beforeStop = await readHostSystemAudioProbeStatus(session); + probe.child.kill('SIGTERM'); + await probe.wait.catch(() => {}); + session.audioProbe = undefined; + return finalizeHostSystemAudioProbeStatus(beforeStop, probe, session.device, reason); +} + +async function waitForHostSystemAudioProbeStatus(session: SessionState): Promise { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const status = await readHostSystemAudioProbeStatus(session); + if (status) return status; + const exit = await Promise.race([ + session.audioProbe?.wait.then( + (result) => result, + (error: unknown) => error, + ), + sleep(100).then(() => undefined), + ]); + if (exit instanceof Error) throw exit; + if (exit) { + const result = exit as { stdout?: string; stderr?: string; exitCode?: number }; + const message = + result.stderr?.trim() || + result.stdout?.trim() || + `host audio probe helper exited with code ${result.exitCode ?? 1}`; + throw new AppError('COMMAND_FAILED', `failed to start host audio probe: ${message}`); + } + } + throw new AppError('COMMAND_FAILED', 'failed to start host audio probe'); +} + +async function readHostSystemAudioProbeStatus( + session: SessionState, +): Promise { + const probe = session.audioProbe; + if (!probe) return undefined; + try { + const raw = await fs.readFile(probe.statusPath, 'utf8'); + return normalizeHostSystemAudioProbeData(JSON.parse(raw), probe, session.device); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + +function normalizeHostSystemAudioProbeData( + value: unknown, + probe: NonNullable, + device: SessionState['device'], +): AudioProbeResult { + return normalizeAudioProbeRecord(value, { + source: 'system-audio', + backend: HOST_AUDIO_BACKEND, + durationMs: probe.durationMs, + elapsedMs: Date.now() - probe.startedAt, + bucketMs: probe.bucketMs, + activeFallback: true, + sourceCount: 1, + notes: hostSystemAudioProbeNotes(device), + }); +} + +function finalizeHostSystemAudioProbeStatus( + status: AudioProbeResult | undefined, + probe: NonNullable, + device: SessionState['device'], + reason: string, +): AudioProbeResult { + const elapsedMs = Math.min(probe.durationMs, Math.max(0, Date.now() - probe.startedAt)); + const base = + status ?? + ({ + audio: 'probe', + state: 'stopped', + active: false, + heard: false, + source: 'system-audio', + backend: HOST_AUDIO_BACKEND, + durationMs: probe.durationMs, + elapsedMs: 0, + bucketMs: probe.bucketMs, + sampleCount: 0, + sourceCount: 1, + rmsDbfs: [], + peakDbfs: [], + notes: hostSystemAudioProbeNotes(device), + } as AudioProbeResult); + return { + ...base, + state: 'stopped', + active: false, + elapsedMs, + stoppedAt: new Date().toISOString(), + reason, + }; +} + +function buildHostSystemAudioProbeFallback( + request: HostAudioProbeCommand, + state: 'running' | 'stopped', + reason?: string, +): AudioProbeResult { + return { + audio: 'probe', + state, + active: state === 'running', + heard: false, + source: 'system-audio', + backend: HOST_AUDIO_BACKEND, + durationMs: request.durationMs, + elapsedMs: 0, + bucketMs: request.bucketMs, + sampleCount: 0, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + reason, + notes: [ + ...hostSystemAudioProbeNotes(request.session.device), + 'No active host audio probe is running.', + ], + }; +} + +function hostSystemAudioProbeNotes(device: SessionState['device']): string[] { + const target = + device.platform === 'ios' + ? 'iOS simulator' + : device.platform === 'android' + ? 'Android emulator' + : 'macOS session'; + return [ + `Audio probe samples host system audio through ScreenCaptureKit for this ${target}; it is not app-instrumented audio.`, + 'Screen Recording permission is required for host system audio capture.', + 'Other audible host apps can contribute to the measured buckets.', + ]; +} diff --git a/src/daemon/handlers/__tests__/session-close-shutdown.test.ts b/src/daemon/handlers/__tests__/session-close-shutdown.test.ts index f3953f628..ebb99b2ef 100644 --- a/src/daemon/handlers/__tests__/session-close-shutdown.test.ts +++ b/src/daemon/handlers/__tests__/session-close-shutdown.test.ts @@ -391,6 +391,50 @@ test('close stops active Android native perf capture before deleting session', a expect(sessionStore.get(sessionName)).toBeUndefined(); }); +test('close stops active host audio probe before deleting session', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'macos-active-audio-probe-session'; + const kill = vi.fn(); + const session = { + ...makeSession(sessionName, { + platform: 'macos', + id: 'macos', + name: 'Mac', + kind: 'device', + booted: true, + }), + audioProbe: { + platform: 'host-system-audio', + child: { kill, pid: 1234 }, + wait: Promise.resolve({ stdout: '', stderr: '', exitCode: 0 }), + statusPath: path.join(os.tmpdir(), 'missing-audio-probe.json'), + startedAt: Date.now() - 2000, + durationMs: 10000, + bucketMs: 1000, + }, + } as SessionState; + sessionStore.set(sessionName, session); + + const response = await handleSessionCommands({ + req: { + token: 't', + session: sessionName, + command: 'close', + positionals: [], + flags: {}, + }, + sessionName, + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + }); + + expect(response?.ok).toBe(true); + expect(kill).toHaveBeenCalledWith('SIGTERM'); + expect(session.audioProbe).toBeUndefined(); + expect(sessionStore.get(sessionName)).toBeUndefined(); +}); + test('close dispatches web session cleanup without a positional target', async () => { const sessionStore = makeSessionStore(); const sessionName = 'web-close-session'; diff --git a/src/daemon/handlers/session-close.ts b/src/daemon/handlers/session-close.ts index 0292fd296..1166ce039 100644 --- a/src/daemon/handlers/session-close.ts +++ b/src/daemon/handlers/session-close.ts @@ -62,7 +62,7 @@ export async function handleCloseCommand(params: { } try { await stopSessionAppLog(session); - await stopSessionAudioProbe(session); + await stopSessionAudioProbe(session, 'session-close'); await stopSessionApplePerfCapture(session); await stopSessionAndroidNativePerfCapture(session); await stopSessionAndroidSnapshotHelper(session); diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index c0f0690f2..c5954896e 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -1,5 +1,3 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; import { isPerfAction, @@ -16,10 +14,10 @@ import { } from '../../contracts/perf.ts'; import { AppError, normalizeError } from '../../utils/errors.ts'; import { resolveWebProvider } from '../../platforms/web/provider.ts'; -import { startMacOsAudioProbeProcess } from '../../platforms/ios/macos-helper.ts'; import type { AndroidAdbExecutor } from '../../platforms/android/adb-executor.ts'; import type { DaemonRequest, DaemonResponse, DaemonResponseData, SessionState } from '../types.ts'; import { SessionStore } from '../session-store.ts'; +import { runHostSystemAudioProbeCommand, usesHostSystemAudioProbe } from '../audio-probe.ts'; import { appendAppLogMarker, clearAppLogFiles, @@ -687,271 +685,27 @@ type ResolvedAudioCommandRequest = Extract< { ok: true } >; -type HostSystemAudioProbeData = { - audio: 'probe'; - state: 'running' | 'stopped'; - active: boolean; - heard: boolean; - source: 'system-audio'; - backend: 'macos-screencapturekit'; - durationMs: number; - elapsedMs: number; - bucketMs: number; - sampleCount: number; - sourceCount: number; - rmsDbfs: number[]; - peakDbfs: number[]; - startedAt?: string; - stoppedAt?: string; - reason?: string; - notes?: string[]; -}; - -function usesHostSystemAudioProbe(device: SessionState['device']): boolean { - return ( - device.platform === 'macos' || - (device.platform === 'ios' && device.kind === 'simulator') || - (device.platform === 'android' && device.kind === 'emulator') - ); -} - async function handleHostSystemAudioCommand( params: ObservabilityParams, request: ResolvedAudioCommandRequest, ): Promise { - const { session, probeAction } = request; try { - if (probeAction === 'start') { - await stopHostSystemAudioProbe(session); - const statusPath = path.join( - params.sessionStore.ensureSessionDir(params.sessionName), - 'audio-probe.json', - ); - const probe = await startMacOsAudioProbeProcess({ - durationMs: request.durationMs, - bucketMs: request.bucketMs, - statusPath, - }); - session.audioProbe = { - platform: 'host-system-audio', - child: probe.child, - wait: probe.wait, - statusPath, - startedAt: Date.now(), + return { + ok: true, + data: await runHostSystemAudioProbeCommand({ + session: request.session, + sessionName: params.sessionName, + sessionStore: params.sessionStore, + probeAction: request.probeAction, durationMs: request.durationMs, bucketMs: request.bucketMs, - }; - void probe.wait.catch(() => {}); - return { ok: true, data: await waitForHostSystemAudioProbeStatus(session) }; - } - - if (probeAction === 'stop') { - const data = await stopHostSystemAudioProbe(session); - return { - ok: true, - data: data ?? buildHostSystemAudioProbeFallback(request, 'stopped', 'not-started'), - }; - } - - const data = await readHostSystemAudioProbeStatus(session); - if (data) { - if (data.state === 'stopped') session.audioProbe = undefined; - return { ok: true, data }; - } - return { ok: true, data: buildHostSystemAudioProbeFallback(request, 'stopped', 'not-started') }; + }), + }; } catch (error) { return { ok: false, error: normalizeError(error) }; } } -async function waitForHostSystemAudioProbeStatus( - session: SessionState, -): Promise { - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - const status = await readHostSystemAudioProbeStatus(session); - if (status) return status; - const exit = await Promise.race([ - session.audioProbe?.wait.then( - (result) => result, - (error: unknown) => error, - ), - sleep(100).then(() => undefined), - ]); - if (exit instanceof Error) throw exit; - if (exit) { - const result = exit as { stdout?: string; stderr?: string; exitCode?: number }; - const message = - result.stderr?.trim() || - result.stdout?.trim() || - `host audio probe helper exited with code ${result.exitCode ?? 1}`; - throw new AppError('COMMAND_FAILED', `failed to start host audio probe: ${message}`); - } - } - throw new AppError('COMMAND_FAILED', 'failed to start host audio probe'); -} - -async function readHostSystemAudioProbeStatus( - session: SessionState, -): Promise { - const probe = session.audioProbe; - if (!probe) return undefined; - try { - const raw = await fs.readFile(probe.statusPath, 'utf8'); - return normalizeHostSystemAudioProbeData(JSON.parse(raw), probe, session.device); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } -} - -async function stopHostSystemAudioProbe( - session: SessionState, -): Promise { - const probe = session.audioProbe; - if (!probe) return undefined; - const beforeStop = await readHostSystemAudioProbeStatus(session); - probe.child.kill('SIGTERM'); - await probe.wait.catch(() => {}); - session.audioProbe = undefined; - return finalizeHostSystemAudioProbeStatus(beforeStop, probe, session.device, 'stopped'); -} - -function normalizeHostSystemAudioProbeData( - value: unknown, - probe: NonNullable, - device: SessionState['device'], -): HostSystemAudioProbeData { - const record = value && typeof value === 'object' ? (value as Record) : {}; - const state = record.state === 'stopped' ? 'stopped' : 'running'; - const rmsDbfs = readNumberArray(record.rmsDbfs); - const peakDbfs = readNumberArray(record.peakDbfs); - return { - audio: 'probe', - state, - active: state === 'running' && record.active !== false, - heard: record.heard === true, - source: 'system-audio', - backend: 'macos-screencapturekit', - durationMs: readFiniteNumber(record.durationMs, probe.durationMs), - elapsedMs: readFiniteNumber(record.elapsedMs, Date.now() - probe.startedAt), - bucketMs: readFiniteNumber(record.bucketMs, probe.bucketMs), - sampleCount: readFiniteNumber(record.sampleCount, rmsDbfs.length), - sourceCount: readFiniteNumber(record.sourceCount, 1), - rmsDbfs, - peakDbfs, - startedAt: typeof record.startedAt === 'string' ? record.startedAt : undefined, - stoppedAt: typeof record.stoppedAt === 'string' ? record.stoppedAt : undefined, - reason: typeof record.reason === 'string' ? record.reason : undefined, - notes: mergeHostSystemAudioProbeNotes(readStringArray(record.notes), device), - }; -} - -function finalizeHostSystemAudioProbeStatus( - status: HostSystemAudioProbeData | undefined, - probe: NonNullable, - device: SessionState['device'], - reason: string, -): HostSystemAudioProbeData { - const elapsedMs = Math.min(probe.durationMs, Math.max(0, Date.now() - probe.startedAt)); - const base = - status ?? - ({ - audio: 'probe', - state: 'stopped', - active: false, - heard: false, - source: 'system-audio', - backend: 'macos-screencapturekit', - durationMs: probe.durationMs, - elapsedMs: 0, - bucketMs: probe.bucketMs, - sampleCount: 0, - sourceCount: 1, - rmsDbfs: [], - peakDbfs: [], - notes: hostSystemAudioProbeNotes(device), - } as HostSystemAudioProbeData); - return { - ...base, - state: 'stopped', - active: false, - elapsedMs, - stoppedAt: new Date().toISOString(), - reason, - }; -} - -function buildHostSystemAudioProbeFallback( - request: ResolvedAudioCommandRequest, - state: 'running' | 'stopped', - reason?: string, -): HostSystemAudioProbeData { - return { - audio: 'probe', - state, - active: state === 'running', - heard: false, - source: 'system-audio', - backend: 'macos-screencapturekit', - durationMs: request.durationMs, - elapsedMs: 0, - bucketMs: request.bucketMs, - sampleCount: 0, - sourceCount: 0, - rmsDbfs: [], - peakDbfs: [], - reason, - notes: [ - ...hostSystemAudioProbeNotes(request.session.device), - 'No active host audio probe is running.', - ], - }; -} - -function mergeHostSystemAudioProbeNotes( - notes: string[] | undefined, - device: SessionState['device'], -): string[] { - return [...(notes ?? []), ...hostSystemAudioProbeNotes(device)]; -} - -function hostSystemAudioProbeNotes(device: SessionState['device']): string[] { - const target = - device.platform === 'ios' - ? 'iOS simulator' - : device.platform === 'android' - ? 'Android emulator' - : 'macOS session'; - return [ - `Audio probe samples host system audio through ScreenCaptureKit for this ${target}; it is not app-instrumented audio.`, - 'Screen Recording permission is required for host system audio capture.', - 'Other audible host apps can contribute to the measured buckets.', - ]; -} - -function readFiniteNumber(value: unknown, fallback: number): number { - return typeof value === 'number' && Number.isFinite(value) ? value : fallback; -} - -function readNumberArray(value: unknown): number[] { - if (!Array.isArray(value)) return []; - const numbers: number[] = []; - for (const item of value) { - if (typeof item === 'number' && Number.isFinite(item)) numbers.push(item); - } - return numbers; -} - -function readStringArray(value: unknown): string[] | undefined { - if (!Array.isArray(value)) return undefined; - return value.filter((item): item is string => typeof item === 'string'); -} - -async function sleep(ms: number): Promise { - await new Promise((resolve) => setTimeout(resolve, ms)); -} - function resolveAudioProbeAction( req: DaemonRequest, ): { ok: true; probeAction: 'start' | 'status' | 'stop' } | DaemonFailureResponse { diff --git a/src/daemon/session-teardown.ts b/src/daemon/session-teardown.ts index 6e41146d0..027d15ff0 100644 --- a/src/daemon/session-teardown.ts +++ b/src/daemon/session-teardown.ts @@ -7,8 +7,11 @@ import { cleanupAppleXctracePerfCapture } from '../platforms/ios/perf-xctrace.ts import { cleanupAndroidNativePerfSession } from '../platforms/android/perf.ts'; import { stopAndroidSnapshotHelperSessionForDevice } from '../platforms/android/snapshot-helper.ts'; import { cleanupRetainedMaterializedPathsForSession } from './materialized-path-registry.ts'; +import { stopSessionAudioProbe } from './audio-probe.ts'; import type { SessionState } from './types.ts'; +export { stopSessionAudioProbe } from './audio-probe.ts'; + export async function stopAppleRunnerForClose(session: SessionState): Promise { await stopIosRunnerSession(session.device.id); if (session.device.platform !== 'macos') { @@ -56,20 +59,12 @@ export async function stopSessionAndroidSnapshotHelper(session: SessionState): P await stopAndroidSnapshotHelperSessionForDevice(session.device); } -export async function stopSessionAudioProbe(session: SessionState): Promise { - const probe = session.audioProbe; - if (!probe) return; - probe.child.kill('SIGTERM'); - await probe.wait.catch(() => {}); - session.audioProbe = undefined; -} - export async function teardownSessionResources( session: SessionState, sessionName: string, ): Promise { await stopSessionAppLog(session); - await stopSessionAudioProbe(session); + await stopSessionAudioProbe(session, 'session-teardown'); await stopSessionApplePerfCapture(session); await stopSessionAndroidNativePerfCapture(session); await stopSessionAndroidSnapshotHelper(session); diff --git a/src/platforms/host-system-audio.ts b/src/platforms/host-system-audio.ts new file mode 100644 index 000000000..ee6f5f965 --- /dev/null +++ b/src/platforms/host-system-audio.ts @@ -0,0 +1,14 @@ +import type { ExecBackgroundResult } from '../utils/exec.ts'; +import { startMacOsAudioProbeProcess } from './ios/macos-helper.ts'; + +export type HostSystemAudioProbeProcessOptions = { + durationMs: number; + bucketMs: number; + statusPath: string; +}; + +export async function startHostSystemAudioProbeProcess( + options: HostSystemAudioProbeProcessOptions, +): Promise { + return await startMacOsAudioProbeProcess(options); +} diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index 28a3bc6ed..4bff6be3a 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -2,11 +2,11 @@ import { runCmd } from '../../utils/exec.ts'; import { AppError } from '../../utils/errors.ts'; import { sleep } from '../../utils/timeouts.ts'; import type { Rect } from '../../utils/snapshot.ts'; +import { normalizeAudioProbeRecord } from '../../audio-probe-result.ts'; import { normalizeAgentBrowserNetworkRequests } from './agent-browser-network.ts'; import { normalizeAgentBrowserSnapshot } from './agent-browser-snapshot.ts'; import { isJsonObject, - readBooleanProperty, readNumberProperty, readStringProperty, type JsonObject, @@ -368,26 +368,16 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { const record = readAgentBrowserEvalResultRecord(data); - return { - audio: 'probe', - state: readAudioProbeState(record), - active: readBooleanProperty(record, 'active') === true, - heard: readBooleanProperty(record, 'heard') === true, + return normalizeAudioProbeRecord(record, { source: 'media-elements', - backend: readStringPropertyWithDefault(record, 'backend', 'agent-browser'), - durationMs: readNumberPropertyWithDefault(record, 'durationMs', 0), - elapsedMs: readNumberPropertyWithDefault(record, 'elapsedMs', 0), - bucketMs: readNumberPropertyWithDefault(record, 'bucketMs', 1000), - sampleCount: readNumberPropertyWithDefault(record, 'sampleCount', 0), - mediaElementCount: readNumberPropertyWithDefault(record, 'mediaElementCount', 0), - sourceCount: readNumberPropertyWithDefault(record, 'sourceCount', 0), - rmsDbfs: readNumberArray(record.rmsDbfs), - peakDbfs: readNumberArray(record.peakDbfs), - startedAt: readStringProperty(record, 'startedAt'), - stoppedAt: readStringProperty(record, 'stoppedAt'), - reason: readStringProperty(record, 'reason'), - notes: readStringArray(record.notes), - }; + backend: 'agent-browser', + durationMs: 0, + elapsedMs: 0, + bucketMs: 1000, + activeFallback: false, + mediaElementCount: 0, + sourceCount: 0, + }) as WebAudioProbeResult; } function readAgentBrowserEvalResultRecord(data: unknown): JsonObject { @@ -395,32 +385,6 @@ function readAgentBrowserEvalResultRecord(data: unknown): JsonObject { return isJsonObject(data.result) ? data.result : data; } -function readAudioProbeState(record: JsonObject): 'running' | 'stopped' { - return readStringProperty(record, 'state') === 'running' ? 'running' : 'stopped'; -} - -function readStringPropertyWithDefault(record: JsonObject, key: string, fallback: string): string { - const value = readStringProperty(record, key); - return value === undefined ? fallback : value; -} - -function readNumberPropertyWithDefault(record: JsonObject, key: string, fallback: number): number { - const value = readNumberProperty(record, key); - return value === undefined ? fallback : value; -} - -function readNumberArray(value: unknown): number[] { - return Array.isArray(value) - ? value.filter((item): item is number => typeof item === 'number' && Number.isFinite(item)) - : []; -} - -function readStringArray(value: unknown): string[] | undefined { - if (!Array.isArray(value)) return undefined; - const items = value.filter((item): item is string => typeof item === 'string' && item.length > 0); - return items.length > 0 ? items : undefined; -} - async function runPacedScroll( runJson: (args: string[]) => Promise, direction: string, diff --git a/src/platforms/web/provider.ts b/src/platforms/web/provider.ts index b35ad8992..398e549ee 100644 --- a/src/platforms/web/provider.ts +++ b/src/platforms/web/provider.ts @@ -3,6 +3,7 @@ import type { SessionSurface } from '../../core/session-surface.ts'; import { createScopedProvider } from '../../utils/scoped-provider.ts'; import type { RawSnapshotNode } from '../../utils/snapshot.ts'; import type { BackendDumpNetworkOptions, BackendDumpNetworkResult } from '../../backend.ts'; +import type { AudioProbeResult } from '../../audio-probe-result.ts'; import { createAgentBrowserWebProvider } from './agent-browser-provider.ts'; export type WebOpenOptions = { @@ -38,25 +39,9 @@ export type WebAudioProbeOptions = { source?: 'media-elements'; }; -export type WebAudioProbeResult = { - audio: 'probe'; - state: 'running' | 'stopped'; - active: boolean; - heard: boolean; +export type WebAudioProbeResult = AudioProbeResult & { source: 'media-elements'; - backend?: string; - durationMs: number; - elapsedMs: number; - bucketMs: number; - sampleCount: number; mediaElementCount: number; - sourceCount: number; - rmsDbfs: number[]; - peakDbfs: number[]; - startedAt?: string; - stoppedAt?: string; - reason?: string; - notes?: string[]; }; export type WebProvider = { From b3eef51a07cbb00063fb88f6fbf121e118a2cdac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:02:33 +0200 Subject: [PATCH 10/18] perf: trim audio probe package size --- .../AgentDeviceMacOSHelper/AudioProbe.swift | 7 +- src/platforms/web/agent-browser-provider.ts | 103 +++++++++--------- 2 files changed, 51 insertions(+), 59 deletions(-) diff --git a/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift b/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift index c3d34b790..78d9c5911 100644 --- a/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift +++ b/macos-helper/Sources/AgentDeviceMacOSHelper/AudioProbe.swift @@ -23,7 +23,6 @@ struct AudioProbeResponse: Codable { let startedAt: String let stoppedAt: String? let reason: String? - let notes: [String] } private struct AudioProbeBucket { @@ -170,11 +169,7 @@ private final class AudioProbeStatusWriter { peakDbfs: peakDbfs, startedAt: iso8601(startedAt), stoppedAt: stoppedAt.map(iso8601), - reason: reason, - notes: [ - "Audio probe samples macOS system audio through ScreenCaptureKit; it is not app-instrumented audio.", - "Screen Recording permission is required for macOS system audio capture.", - ] + reason: reason ) } diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index 4bff6be3a..034b3c70a 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -96,23 +96,30 @@ export function createAgentBrowserWebProvider( } function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { - return `(() => { - const options = ${JSON.stringify({ + return `(${audioProbeEvalScript.toString()})(${JSON.stringify({ action: options.action, durationMs: options.durationMs, bucketMs: options.bucketMs, source: options.source ?? 'media-elements', - })}; + })})`; +} + +type AudioProbePageRecord = Record; + +declare const window: AudioProbePageRecord; +declare const document: { querySelectorAll(selector: string): any[] }; + +function audioProbeEvalScript(options: AudioProbePageRecord): unknown { const key = '__agentDeviceAudioProbe'; const contextKey = '__agentDeviceAudioProbeContext'; const sourceKey = '__agentDeviceAudioProbeSources'; const silenceDb = -90; const now = () => Date.now(); - const dbfs = (value) => { + const dbfs = (value: number) => { if (!Number.isFinite(value) || value <= 0) return silenceDb; return Math.max(silenceDb, Math.min(0, Math.round(20 * Math.log10(value)))); }; - const note = (probe, message) => { + const note = (probe: AudioProbePageRecord, message: string) => { if (!probe.notes.includes(message)) probe.notes.push(message); }; const scopeNote = @@ -120,7 +127,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const routingNote = 'URL-backed media elements may be routed through the probe AudioContext while they are observed.'; const mediaElements = () => Array.from(document.querySelectorAll('audio,video')); - const stopProbe = (probe, reason) => { + const stopProbe = (probe: AudioProbePageRecord | undefined, reason: string) => { if (!probe || probe.state === 'stopped') return probe; clearInterval(probe.timer); clearTimeout(probe.timeout); @@ -154,7 +161,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { window[contextKey] = context; return context; }; - const createElementAudioSource = (probe, element) => { + const createElementAudioSource = (probe: AudioProbePageRecord, element: AudioProbePageRecord) => { if (!element.currentSrc && !element.src && element.readyState === 0) return undefined; try { if (!window[sourceKey]) window[sourceKey] = new WeakMap(); @@ -172,17 +179,23 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { return undefined; } }; - const createMediaStreamAudioSource = (probe, stream) => { + const createMediaStreamAudioSource = ( + probe: AudioProbePageRecord, + stream: AudioProbePageRecord | undefined, + ) => { if (!stream || typeof stream.getAudioTracks !== 'function') return undefined; if (stream.getAudioTracks().length === 0) return undefined; return { source: probe.context.createMediaStreamSource(stream), audible: false }; }; - const createCaptureStreamAudioSource = (probe, element) => { + const createCaptureStreamAudioSource = ( + probe: AudioProbePageRecord, + element: AudioProbePageRecord, + ) => { if (typeof element.captureStream !== 'function') return undefined; const stream = element.captureStream(); return createMediaStreamAudioSource(probe, stream); }; - const discover = (probe) => { + const discover = (probe: AudioProbePageRecord) => { const elements = mediaElements(); probe.mediaElementCount = elements.length; for (const element of elements) { @@ -203,7 +216,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { probe.analysers.push({ ...sourceEntry, analyser, - buffer: new Float32Array(analyser.fftSize) + buffer: new Float32Array(analyser.fftSize), }); } probe.sourceCount = probe.analysers.length; @@ -211,7 +224,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { note(probe, 'No capturable page media audio sources were found yet.'); } }; - const sample = (probe) => { + const sample = (probe: AudioProbePageRecord | undefined) => { if (!probe || probe.state !== 'running') return; discover(probe); let totalSquares = 0; @@ -233,31 +246,15 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { probe.peakDbfs.push(peakDb); probe.heard = probe.heard || rmsDb > silenceDb || peakDb > silenceDb; const maxSamples = Math.ceil(probe.durationMs / probe.bucketMs) + 2; - if (probe.rmsDbfs.length > maxSamples) probe.rmsDbfs.splice(0, probe.rmsDbfs.length - maxSamples); - if (probe.peakDbfs.length > maxSamples) probe.peakDbfs.splice(0, probe.peakDbfs.length - maxSamples); + if (probe.rmsDbfs.length > maxSamples) + probe.rmsDbfs.splice(0, probe.rmsDbfs.length - maxSamples); + if (probe.peakDbfs.length > maxSamples) + probe.peakDbfs.splice(0, probe.peakDbfs.length - maxSamples); if (now() - probe.startedAt >= probe.durationMs) stopProbe(probe, 'duration'); }; - const result = (probe) => { + const result = (probe: AudioProbePageRecord | undefined) => { const mediaCount = mediaElements().length; - if (!probe) { - return { - audio: 'probe', - state: 'stopped', - active: false, - heard: false, - source: 'media-elements', - backend: 'agent-browser', - durationMs: Number(options.durationMs) || 10000, - elapsedMs: 0, - bucketMs: Number(options.bucketMs) || 1000, - sampleCount: 0, - mediaElementCount: mediaCount, - sourceCount: 0, - rmsDbfs: [], - peakDbfs: [], - notes: [scopeNote, routingNote], - }; - } + if (!probe) return stoppedResult([scopeNote, routingNote]); return { audio: 'probe', state: probe.state, @@ -268,7 +265,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { durationMs: probe.durationMs, elapsedMs: Math.max( 0, - Math.min((probe.stoppedAt || now()) - probe.startedAt, probe.durationMs) + Math.min((probe.stoppedAt || now()) - probe.startedAt, probe.durationMs), ), bucketMs: probe.bucketMs, sampleCount: probe.rmsDbfs.length, @@ -282,6 +279,23 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { notes: [scopeNote, routingNote, ...probe.notes], }; }; + const stoppedResult = (notes: string[]) => ({ + audio: 'probe', + state: 'stopped', + active: false, + heard: false, + source: 'media-elements', + backend: 'agent-browser', + durationMs: Number(options.durationMs) || 10000, + elapsedMs: 0, + bucketMs: Number(options.bucketMs) || 1000, + sampleCount: 0, + mediaElementCount: mediaElements().length, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + notes, + }); const action = options.action || 'status'; let probe = window[key]; if (probe && probe.state === 'running' && now() - probe.startedAt >= probe.durationMs) { @@ -292,23 +306,7 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { if (probe) stopProbe(probe, 'restarted'); const context = getContext(); if (!context) { - return { - audio: 'probe', - state: 'stopped', - active: false, - heard: false, - source: 'media-elements', - backend: 'agent-browser', - durationMs: Number(options.durationMs) || 10000, - elapsedMs: 0, - bucketMs: Number(options.bucketMs) || 1000, - sampleCount: 0, - mediaElementCount: mediaElements().length, - sourceCount: 0, - rmsDbfs: [], - peakDbfs: [], - notes: ['Web Audio API is not available in this browser context.'], - }; + return stoppedResult(['Web Audio API is not available in this browser context.']); } const sink = context.createGain(); sink.gain.value = 0; @@ -363,7 +361,6 @@ function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { } if (probe) sample(probe); return result(probe); -})()`; } function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { From 879dd48822566968ad71bba3da58dded26117ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:17:17 +0200 Subject: [PATCH 11/18] refactor: address audio probe review comments --- src/audio-probe-result.ts | 35 ++ src/core/capabilities.ts | 14 +- src/daemon/audio-probe.ts | 42 +- src/platforms/host-system-audio.ts | 14 - .../web/agent-browser-audio-probe.ts | 375 ++++++++++++++++++ src/platforms/web/agent-browser-provider.ts | 300 +------------- 6 files changed, 436 insertions(+), 344 deletions(-) delete mode 100644 src/platforms/host-system-audio.ts create mode 100644 src/platforms/web/agent-browser-audio-probe.ts diff --git a/src/audio-probe-result.ts b/src/audio-probe-result.ts index 2229f9596..de70b0506 100644 --- a/src/audio-probe-result.ts +++ b/src/audio-probe-result.ts @@ -33,6 +33,41 @@ export type NormalizeAudioProbeRecordOptions = { notes?: string[]; }; +export type EmptyAudioProbeResultOptions = { + source: AudioProbeSource; + backend: string; + durationMs: number; + bucketMs: number; + state?: 'running' | 'stopped'; + elapsedMs?: number; + mediaElementCount?: number; + sourceCount?: number; + reason?: string; + notes?: string[]; +}; + +export function emptyAudioProbeResult(options: EmptyAudioProbeResultOptions): AudioProbeResult { + const state = options.state ?? 'stopped'; + return { + audio: 'probe', + state, + active: state === 'running', + heard: false, + source: options.source, + backend: options.backend, + durationMs: options.durationMs, + elapsedMs: options.elapsedMs ?? 0, + bucketMs: options.bucketMs, + sampleCount: 0, + mediaElementCount: options.mediaElementCount, + sourceCount: options.sourceCount ?? 0, + rmsDbfs: [], + peakDbfs: [], + reason: options.reason, + notes: options.notes, + }; +} + export function normalizeAudioProbeRecord( value: unknown, options: NormalizeAudioProbeRecordOptions, diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index b5f77126a..875283f37 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -22,12 +22,14 @@ const isMacOsOrAppleSimulator = (device: DeviceInfo): boolean => device.platform === 'macos' || device.kind === 'simulator'; const isIosMobileSimulator = (device: DeviceInfo): boolean => device.platform === 'ios' && device.kind === 'simulator' && device.target !== 'tv'; -const isHostSystemAudioProbeDevice = (device: DeviceInfo): boolean => +export const isHostSystemAudioProbeDevice = (device: DeviceInfo): boolean => + device.platform === 'macos' || + (device.platform === 'ios' && device.kind === 'simulator') || + (device.platform === 'android' && device.kind === 'emulator'); + +const isAudioProbeSupportedDevice = (device: DeviceInfo): boolean => device.platform === 'web' || - (process.platform === 'darwin' && - (device.platform === 'macos' || - (device.platform === 'ios' && device.kind === 'simulator') || - (device.platform === 'android' && device.kind === 'emulator'))); + (process.platform === 'darwin' && isHostSystemAudioProbeDevice(device)); // Two-finger gesture synthesis (RunnerSynthesizedGesture) is iOS-simulator-only (plus Android). // When such a gesture is rejected at admission, explain where it IS available so an agent can @@ -209,7 +211,7 @@ const BASE_COMMAND_CAPABILITY_MATRIX: Record = { apple: { simulator: true, device: true }, android: { emulator: true }, linux: LINUX_NONE, - supports: isHostSystemAudioProbeDevice, + supports: isAudioProbeSupportedDevice, }, network: { apple: { simulator: true, device: true }, diff --git a/src/daemon/audio-probe.ts b/src/daemon/audio-probe.ts index 36b0022c7..605a20868 100644 --- a/src/daemon/audio-probe.ts +++ b/src/daemon/audio-probe.ts @@ -1,7 +1,12 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { normalizeAudioProbeRecord, type AudioProbeResult } from '../audio-probe-result.ts'; -import { startHostSystemAudioProbeProcess } from '../platforms/host-system-audio.ts'; +import { + emptyAudioProbeResult, + normalizeAudioProbeRecord, + type AudioProbeResult, +} from '../audio-probe-result.ts'; +import { isHostSystemAudioProbeDevice } from '../core/capabilities.ts'; +import { startMacOsAudioProbeProcess } from '../platforms/ios/macos-helper.ts'; import { AppError } from '../utils/errors.ts'; import { sleep } from '../utils/timeouts.ts'; import type { SessionStore } from './session-store.ts'; @@ -18,13 +23,7 @@ export type HostAudioProbeCommand = { bucketMs: number; }; -export function usesHostSystemAudioProbe(device: SessionState['device']): boolean { - return ( - device.platform === 'macos' || - (device.platform === 'ios' && device.kind === 'simulator') || - (device.platform === 'android' && device.kind === 'emulator') - ); -} +export const usesHostSystemAudioProbe = isHostSystemAudioProbeDevice; export async function runHostSystemAudioProbeCommand( request: HostAudioProbeCommand, @@ -36,7 +35,7 @@ export async function runHostSystemAudioProbeCommand( request.sessionStore.ensureSessionDir(request.sessionName), 'audio-probe.json', ); - const probe = await startHostSystemAudioProbeProcess({ + const probe = await startMacOsAudioProbeProcess({ durationMs: request.durationMs, bucketMs: request.bucketMs, statusPath, @@ -147,22 +146,14 @@ function finalizeHostSystemAudioProbeStatus( const elapsedMs = Math.min(probe.durationMs, Math.max(0, Date.now() - probe.startedAt)); const base = status ?? - ({ - audio: 'probe', - state: 'stopped', - active: false, - heard: false, + emptyAudioProbeResult({ source: 'system-audio', backend: HOST_AUDIO_BACKEND, durationMs: probe.durationMs, - elapsedMs: 0, bucketMs: probe.bucketMs, - sampleCount: 0, sourceCount: 1, - rmsDbfs: [], - peakDbfs: [], notes: hostSystemAudioProbeNotes(device), - } as AudioProbeResult); + }); return { ...base, state: 'stopped', @@ -178,26 +169,19 @@ function buildHostSystemAudioProbeFallback( state: 'running' | 'stopped', reason?: string, ): AudioProbeResult { - return { - audio: 'probe', + return emptyAudioProbeResult({ state, - active: state === 'running', - heard: false, source: 'system-audio', backend: HOST_AUDIO_BACKEND, durationMs: request.durationMs, - elapsedMs: 0, bucketMs: request.bucketMs, - sampleCount: 0, sourceCount: 0, - rmsDbfs: [], - peakDbfs: [], reason, notes: [ ...hostSystemAudioProbeNotes(request.session.device), 'No active host audio probe is running.', ], - }; + }); } function hostSystemAudioProbeNotes(device: SessionState['device']): string[] { diff --git a/src/platforms/host-system-audio.ts b/src/platforms/host-system-audio.ts deleted file mode 100644 index ee6f5f965..000000000 --- a/src/platforms/host-system-audio.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { ExecBackgroundResult } from '../utils/exec.ts'; -import { startMacOsAudioProbeProcess } from './ios/macos-helper.ts'; - -export type HostSystemAudioProbeProcessOptions = { - durationMs: number; - bucketMs: number; - statusPath: string; -}; - -export async function startHostSystemAudioProbeProcess( - options: HostSystemAudioProbeProcessOptions, -): Promise { - return await startMacOsAudioProbeProcess(options); -} diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts new file mode 100644 index 000000000..04250bdf1 --- /dev/null +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -0,0 +1,375 @@ +import { normalizeAudioProbeRecord, type AudioProbeResult } from '../../audio-probe-result.ts'; +import { isJsonObject, type JsonObject } from './json-utils.ts'; +import type { WebAudioProbeOptions, WebAudioProbeResult } from './provider.ts'; + +const audioProbePageScriptFunctions = [ + audioProbeDbfs, + audioProbeNote, + audioProbeMediaElements, + audioProbeStop, + audioProbeGetContext, + audioProbeCreateElementAudioSource, + audioProbeCreateMediaStreamAudioSource, + audioProbeCreateCaptureStreamAudioSource, + audioProbeConnectSource, + audioProbeDiscover, + audioProbeReadStats, + audioProbeTrimSamples, + audioProbeSample, + audioProbeStoppedResult, + audioProbeResult, + audioProbeStart, + audioProbeEvalScript, +] as const; + +export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { + const scriptBody = audioProbePageScriptFunctions.map((fn) => `${fn.toString()};`).join(''); + return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}(${JSON.stringify({ + action: options.action, + durationMs: options.durationMs, + bucketMs: options.bucketMs, + source: options.source ?? 'media-elements', + })})})()`; +} + +export function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { + const result: AudioProbeResult = normalizeAudioProbeRecord( + readAgentBrowserEvalResultRecord(data), + { + source: 'media-elements', + backend: 'agent-browser', + durationMs: 0, + elapsedMs: 0, + bucketMs: 1000, + activeFallback: false, + mediaElementCount: 0, + sourceCount: 0, + }, + ); + return { + ...result, + source: 'media-elements', + mediaElementCount: result.mediaElementCount ?? 0, + }; +} + +type AudioProbePageRecord = Record; +type AudioProbePageSource = { source: AudioProbePageRecord; audible: boolean }; +type AudioProbePageStats = { rms: number; peak: number }; + +declare const window: AudioProbePageRecord; +declare const document: { querySelectorAll(selector: string): any[] }; + +function audioProbeDbfs(value: number): number { + const silenceDb = -90; + if (!Number.isFinite(value) || value <= 0) return silenceDb; + return Math.max(silenceDb, Math.min(0, Math.round(20 * Math.log10(value)))); +} + +function audioProbeNote(probe: AudioProbePageRecord, message: string): void { + if (!probe.notes.includes(message)) probe.notes.push(message); +} + +function audioProbeMediaElements(): AudioProbePageRecord[] { + return Array.from(document.querySelectorAll('audio,video')); +} + +function audioProbeStop( + probe: AudioProbePageRecord | undefined, + reason: string, +): AudioProbePageRecord | undefined { + if (!probe || probe.state === 'stopped') return probe; + clearInterval(probe.timer); + clearTimeout(probe.timeout); + probe.timer = undefined; + probe.timeout = undefined; + probe.state = 'stopped'; + probe.active = false; + probe.reason = reason; + probe.stoppedAt = Date.now(); + for (const entry of probe.analysers) { + try { + entry.analyser.disconnect(); + entry.source.disconnect(); + if (entry.audible) entry.source.connect(probe.context.destination); + } catch {} + } + if (probe.resumeOnGesture) { + for (const eventName of ['click', 'pointerdown', 'keydown']) { + window.removeEventListener(eventName, probe.resumeOnGesture, true); + } + probe.resumeOnGesture = undefined; + } + return probe; +} + +function audioProbeGetContext(contextKey: string): AudioProbePageRecord | undefined { + const AudioContextCtor = window.AudioContext || window.webkitAudioContext; + if (!AudioContextCtor) return undefined; + const existing = window[contextKey]; + if (existing && existing.state !== 'closed') return existing; + const context = new AudioContextCtor(); + window[contextKey] = context; + return context; +} + +function audioProbeCreateElementAudioSource( + probe: AudioProbePageRecord, + element: AudioProbePageRecord, + sourceKey: string, +): AudioProbePageSource | undefined { + if (!element.currentSrc && !element.src && element.readyState === 0) return undefined; + try { + if (!window[sourceKey]) window[sourceKey] = new WeakMap(); + let source = window[sourceKey].get(element); + if (!source) { + // createMediaElementSource permanently moves this element through the + // shared probe AudioContext. We keep that context open and reconnect the + // source to destination on stop so audible playback survives the probe. + source = probe.context.createMediaElementSource(element); + window[sourceKey].set(element, source); + } + source.disconnect(); + return { source, audible: true }; + } catch { + return undefined; + } +} + +function audioProbeCreateMediaStreamAudioSource( + probe: AudioProbePageRecord, + stream: AudioProbePageRecord | undefined, +): AudioProbePageSource | undefined { + if (!stream || typeof stream.getAudioTracks !== 'function') return undefined; + if (stream.getAudioTracks().length === 0) return undefined; + return { source: probe.context.createMediaStreamSource(stream), audible: false }; +} + +function audioProbeCreateCaptureStreamAudioSource( + probe: AudioProbePageRecord, + element: AudioProbePageRecord, +): AudioProbePageSource | undefined { + if (typeof element.captureStream !== 'function') return undefined; + const stream = element.captureStream(); + return audioProbeCreateMediaStreamAudioSource(probe, stream); +} + +function audioProbeConnectSource( + probe: AudioProbePageRecord, + sourceEntry: AudioProbePageSource, +): void { + const analyser = probe.context.createAnalyser(); + analyser.fftSize = 2048; + sourceEntry.source.connect(analyser); + analyser.connect(sourceEntry.audible ? probe.context.destination : probe.sink); + probe.analysers.push({ + ...sourceEntry, + analyser, + buffer: new Float32Array(analyser.fftSize), + }); +} + +function audioProbeDiscover(probe: AudioProbePageRecord, sourceKey: string): void { + const elements = audioProbeMediaElements(); + probe.mediaElementCount = elements.length; + for (const element of elements) { + if (probe.seen.has(element)) continue; + const sourceEntry = + audioProbeCreateMediaStreamAudioSource(probe, element.srcObject) ?? + audioProbeCreateCaptureStreamAudioSource(probe, element) ?? + audioProbeCreateElementAudioSource(probe, element, sourceKey); + if (!sourceEntry) { + audioProbeNote(probe, 'Some media elements do not expose capturable audio to Web Audio.'); + continue; + } + probe.seen.add(element); + audioProbeConnectSource(probe, sourceEntry); + } + probe.sourceCount = probe.analysers.length; + if (probe.sourceCount === 0) { + audioProbeNote(probe, 'No capturable page media audio sources were found yet.'); + } +} + +function audioProbeReadStats(probe: AudioProbePageRecord): AudioProbePageStats { + let totalSquares = 0; + let totalSamples = 0; + let peak = 0; + for (const entry of probe.analysers) { + entry.analyser.getFloatTimeDomainData(entry.buffer); + for (const value of entry.buffer) { + totalSquares += value * value; + totalSamples += 1; + peak = Math.max(peak, Math.abs(value)); + } + } + return { + rms: totalSamples > 0 ? Math.sqrt(totalSquares / totalSamples) : 0, + peak, + }; +} + +function audioProbeTrimSamples(probe: AudioProbePageRecord): void { + const maxSamples = Math.ceil(probe.durationMs / probe.bucketMs) + 2; + if (probe.rmsDbfs.length > maxSamples) probe.rmsDbfs.splice(0, probe.rmsDbfs.length - maxSamples); + if (probe.peakDbfs.length > maxSamples) + probe.peakDbfs.splice(0, probe.peakDbfs.length - maxSamples); +} + +function audioProbeSample(probe: AudioProbePageRecord | undefined, sourceKey: string): void { + if (!probe || probe.state !== 'running') return; + audioProbeDiscover(probe, sourceKey); + const stats = audioProbeReadStats(probe); + const rmsDb = audioProbeDbfs(stats.rms); + const peakDb = audioProbeDbfs(stats.peak); + probe.rmsDbfs.push(rmsDb); + probe.peakDbfs.push(peakDb); + probe.heard = probe.heard || rmsDb > -90 || peakDb > -90; + audioProbeTrimSamples(probe); + if (Date.now() - probe.startedAt >= probe.durationMs) audioProbeStop(probe, 'duration'); +} + +function audioProbeStoppedResult( + options: AudioProbePageRecord, + notes: string[], +): AudioProbePageRecord { + return { + audio: 'probe', + state: 'stopped', + active: false, + heard: false, + source: 'media-elements', + backend: 'agent-browser', + durationMs: Number(options.durationMs) || 10000, + elapsedMs: 0, + bucketMs: Number(options.bucketMs) || 1000, + sampleCount: 0, + mediaElementCount: audioProbeMediaElements().length, + sourceCount: 0, + rmsDbfs: [], + peakDbfs: [], + notes, + }; +} + +function audioProbeResult( + probe: AudioProbePageRecord | undefined, + options: AudioProbePageRecord, + scopeNote: string, + routingNote: string, +): AudioProbePageRecord { + if (!probe) return audioProbeStoppedResult(options, [scopeNote, routingNote]); + return { + audio: 'probe', + state: probe.state, + active: probe.state === 'running', + heard: probe.heard, + source: 'media-elements', + backend: 'agent-browser', + durationMs: probe.durationMs, + elapsedMs: Math.max( + 0, + Math.min((probe.stoppedAt || Date.now()) - probe.startedAt, probe.durationMs), + ), + bucketMs: probe.bucketMs, + sampleCount: probe.rmsDbfs.length, + mediaElementCount: audioProbeMediaElements().length, + sourceCount: probe.sourceCount, + rmsDbfs: probe.rmsDbfs.slice(), + peakDbfs: probe.peakDbfs.slice(), + startedAt: new Date(probe.startedAt).toISOString(), + stoppedAt: probe.stoppedAt ? new Date(probe.stoppedAt).toISOString() : undefined, + reason: probe.reason, + notes: [scopeNote, routingNote, ...probe.notes], + }; +} + +function audioProbeStart( + options: AudioProbePageRecord, + existingProbe: AudioProbePageRecord | undefined, + probeKey: string, + contextKey: string, + sourceKey: string, + scopeNote: string, + routingNote: string, +): AudioProbePageRecord { + if (existingProbe) audioProbeStop(existingProbe, 'restarted'); + const context = audioProbeGetContext(contextKey); + if (!context) { + return audioProbeStoppedResult(options, [ + 'Web Audio API is not available in this browser context.', + ]); + } + const sink = context.createGain(); + sink.gain.value = 0; + sink.connect(context.destination); + const probe: AudioProbePageRecord = { + state: 'running', + active: true, + context, + sink, + seen: new WeakSet(), + analysers: [], + mediaElementCount: 0, + sourceCount: 0, + durationMs: Math.max(100, Number(options.durationMs) || 10000), + bucketMs: Math.max(100, Number(options.bucketMs) || 1000), + startedAt: Date.now(), + stoppedAt: undefined, + reason: undefined, + heard: false, + rmsDbfs: [], + peakDbfs: [], + notes: [], + }; + try { + void context.resume(); + } catch { + audioProbeNote(probe, 'AudioContext could not be resumed by the probe.'); + } + probe.resumeOnGesture = () => { + try { + void context.resume(); + } catch { + audioProbeNote(probe, 'AudioContext could not be resumed from a user gesture.'); + } + }; + for (const eventName of ['click', 'pointerdown', 'keydown']) { + window.addEventListener(eventName, probe.resumeOnGesture, { capture: true, once: true }); + } + audioProbeDiscover(probe, sourceKey); + probe.timer = setInterval(() => audioProbeSample(probe, sourceKey), probe.bucketMs); + probe.timeout = setTimeout(() => audioProbeStop(probe, 'duration'), probe.durationMs); + window[probeKey] = probe; + return audioProbeResult(probe, options, scopeNote, routingNote); +} + +function audioProbeEvalScript(options: AudioProbePageRecord): unknown { + const key = '__agentDeviceAudioProbe'; + const contextKey = '__agentDeviceAudioProbeContext'; + const sourceKey = '__agentDeviceAudioProbeSources'; + const scopeNote = + 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.'; + const routingNote = + 'URL-backed media elements may be routed through the probe AudioContext while they are observed.'; + const probe = window[key]; + if (probe && probe.state === 'running' && Date.now() - probe.startedAt >= probe.durationMs) { + audioProbeSample(probe, sourceKey); + audioProbeStop(probe, 'duration'); + } + if (options.action === 'start') { + return audioProbeStart(options, probe, key, contextKey, sourceKey, scopeNote, routingNote); + } + if (options.action === 'stop') { + if (probe) audioProbeSample(probe, sourceKey); + audioProbeStop(probe, 'manual'); + return audioProbeResult(probe, options, scopeNote, routingNote); + } + if (probe) audioProbeSample(probe, sourceKey); + return audioProbeResult(probe, options, scopeNote, routingNote); +} + +function readAgentBrowserEvalResultRecord(data: unknown): JsonObject { + if (!isJsonObject(data)) return {}; + return isJsonObject(data.result) ? data.result : data; +} diff --git a/src/platforms/web/agent-browser-provider.ts b/src/platforms/web/agent-browser-provider.ts index 034b3c70a..d725a1cd6 100644 --- a/src/platforms/web/agent-browser-provider.ts +++ b/src/platforms/web/agent-browser-provider.ts @@ -2,7 +2,10 @@ import { runCmd } from '../../utils/exec.ts'; import { AppError } from '../../utils/errors.ts'; import { sleep } from '../../utils/timeouts.ts'; import type { Rect } from '../../utils/snapshot.ts'; -import { normalizeAudioProbeRecord } from '../../audio-probe-result.ts'; +import { + buildAudioProbeEvalScript, + normalizeAgentBrowserAudioProbeResult, +} from './agent-browser-audio-probe.ts'; import { normalizeAgentBrowserNetworkRequests } from './agent-browser-network.ts'; import { normalizeAgentBrowserSnapshot } from './agent-browser-snapshot.ts'; import { @@ -11,13 +14,7 @@ import { readStringProperty, type JsonObject, } from './json-utils.ts'; -import type { - WebAudioProbeOptions, - WebAudioProbeResult, - WebProvider, - WebSnapshotOptions, - WebSnapshotResult, -} from './provider.ts'; +import type { WebProvider, WebSnapshotOptions, WebSnapshotResult } from './provider.ts'; import { mapManagedAgentBrowserError, resolveAgentBrowserTool } from './agent-browser-tool.ts'; const AGENT_BROWSER = 'agent-browser'; @@ -95,293 +92,6 @@ export function createAgentBrowserWebProvider( }; } -function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { - return `(${audioProbeEvalScript.toString()})(${JSON.stringify({ - action: options.action, - durationMs: options.durationMs, - bucketMs: options.bucketMs, - source: options.source ?? 'media-elements', - })})`; -} - -type AudioProbePageRecord = Record; - -declare const window: AudioProbePageRecord; -declare const document: { querySelectorAll(selector: string): any[] }; - -function audioProbeEvalScript(options: AudioProbePageRecord): unknown { - const key = '__agentDeviceAudioProbe'; - const contextKey = '__agentDeviceAudioProbeContext'; - const sourceKey = '__agentDeviceAudioProbeSources'; - const silenceDb = -90; - const now = () => Date.now(); - const dbfs = (value: number) => { - if (!Number.isFinite(value) || value <= 0) return silenceDb; - return Math.max(silenceDb, Math.min(0, Math.round(20 * Math.log10(value)))); - }; - const note = (probe: AudioProbePageRecord, message: string) => { - if (!probe.notes.includes(message)) probe.notes.push(message); - }; - const scopeNote = - 'Audio probe samples HTML media elements exposed to Web Audio; it is not whole-tab or system audio capture.'; - const routingNote = - 'URL-backed media elements may be routed through the probe AudioContext while they are observed.'; - const mediaElements = () => Array.from(document.querySelectorAll('audio,video')); - const stopProbe = (probe: AudioProbePageRecord | undefined, reason: string) => { - if (!probe || probe.state === 'stopped') return probe; - clearInterval(probe.timer); - clearTimeout(probe.timeout); - probe.timer = undefined; - probe.timeout = undefined; - probe.state = 'stopped'; - probe.active = false; - probe.reason = reason; - probe.stoppedAt = now(); - for (const entry of probe.analysers) { - try { - entry.analyser.disconnect(); - entry.source.disconnect(); - if (entry.audible) entry.source.connect(probe.context.destination); - } catch {} - } - if (probe.resumeOnGesture) { - for (const eventName of ['click', 'pointerdown', 'keydown']) { - window.removeEventListener(eventName, probe.resumeOnGesture, true); - } - probe.resumeOnGesture = undefined; - } - return probe; - }; - const getContext = () => { - const AudioContextCtor = window.AudioContext || window.webkitAudioContext; - if (!AudioContextCtor) return undefined; - const existing = window[contextKey]; - if (existing && existing.state !== 'closed') return existing; - const context = new AudioContextCtor(); - window[contextKey] = context; - return context; - }; - const createElementAudioSource = (probe: AudioProbePageRecord, element: AudioProbePageRecord) => { - if (!element.currentSrc && !element.src && element.readyState === 0) return undefined; - try { - if (!window[sourceKey]) window[sourceKey] = new WeakMap(); - let source = window[sourceKey].get(element); - if (!source) { - // createMediaElementSource permanently moves this element through the - // shared probe AudioContext. We keep that context open and reconnect the - // source to destination on stop so audible playback survives the probe. - source = probe.context.createMediaElementSource(element); - window[sourceKey].set(element, source); - } - source.disconnect(); - return { source, audible: true }; - } catch { - return undefined; - } - }; - const createMediaStreamAudioSource = ( - probe: AudioProbePageRecord, - stream: AudioProbePageRecord | undefined, - ) => { - if (!stream || typeof stream.getAudioTracks !== 'function') return undefined; - if (stream.getAudioTracks().length === 0) return undefined; - return { source: probe.context.createMediaStreamSource(stream), audible: false }; - }; - const createCaptureStreamAudioSource = ( - probe: AudioProbePageRecord, - element: AudioProbePageRecord, - ) => { - if (typeof element.captureStream !== 'function') return undefined; - const stream = element.captureStream(); - return createMediaStreamAudioSource(probe, stream); - }; - const discover = (probe: AudioProbePageRecord) => { - const elements = mediaElements(); - probe.mediaElementCount = elements.length; - for (const element of elements) { - if (probe.seen.has(element)) continue; - const sourceEntry = - createMediaStreamAudioSource(probe, element.srcObject) ?? - createCaptureStreamAudioSource(probe, element) ?? - createElementAudioSource(probe, element); - if (!sourceEntry) { - note(probe, 'Some media elements do not expose capturable audio to Web Audio.'); - continue; - } - probe.seen.add(element); - const analyser = probe.context.createAnalyser(); - analyser.fftSize = 2048; - sourceEntry.source.connect(analyser); - analyser.connect(sourceEntry.audible ? probe.context.destination : probe.sink); - probe.analysers.push({ - ...sourceEntry, - analyser, - buffer: new Float32Array(analyser.fftSize), - }); - } - probe.sourceCount = probe.analysers.length; - if (probe.sourceCount === 0) { - note(probe, 'No capturable page media audio sources were found yet.'); - } - }; - const sample = (probe: AudioProbePageRecord | undefined) => { - if (!probe || probe.state !== 'running') return; - discover(probe); - let totalSquares = 0; - let totalSamples = 0; - let peak = 0; - for (const entry of probe.analysers) { - entry.analyser.getFloatTimeDomainData(entry.buffer); - for (const value of entry.buffer) { - totalSquares += value * value; - totalSamples += 1; - const abs = Math.abs(value); - if (abs > peak) peak = abs; - } - } - const rms = totalSamples > 0 ? Math.sqrt(totalSquares / totalSamples) : 0; - const rmsDb = dbfs(rms); - const peakDb = dbfs(peak); - probe.rmsDbfs.push(rmsDb); - probe.peakDbfs.push(peakDb); - probe.heard = probe.heard || rmsDb > silenceDb || peakDb > silenceDb; - const maxSamples = Math.ceil(probe.durationMs / probe.bucketMs) + 2; - if (probe.rmsDbfs.length > maxSamples) - probe.rmsDbfs.splice(0, probe.rmsDbfs.length - maxSamples); - if (probe.peakDbfs.length > maxSamples) - probe.peakDbfs.splice(0, probe.peakDbfs.length - maxSamples); - if (now() - probe.startedAt >= probe.durationMs) stopProbe(probe, 'duration'); - }; - const result = (probe: AudioProbePageRecord | undefined) => { - const mediaCount = mediaElements().length; - if (!probe) return stoppedResult([scopeNote, routingNote]); - return { - audio: 'probe', - state: probe.state, - active: probe.state === 'running', - heard: probe.heard, - source: 'media-elements', - backend: 'agent-browser', - durationMs: probe.durationMs, - elapsedMs: Math.max( - 0, - Math.min((probe.stoppedAt || now()) - probe.startedAt, probe.durationMs), - ), - bucketMs: probe.bucketMs, - sampleCount: probe.rmsDbfs.length, - mediaElementCount: mediaCount, - sourceCount: probe.sourceCount, - rmsDbfs: probe.rmsDbfs.slice(), - peakDbfs: probe.peakDbfs.slice(), - startedAt: new Date(probe.startedAt).toISOString(), - stoppedAt: probe.stoppedAt ? new Date(probe.stoppedAt).toISOString() : undefined, - reason: probe.reason, - notes: [scopeNote, routingNote, ...probe.notes], - }; - }; - const stoppedResult = (notes: string[]) => ({ - audio: 'probe', - state: 'stopped', - active: false, - heard: false, - source: 'media-elements', - backend: 'agent-browser', - durationMs: Number(options.durationMs) || 10000, - elapsedMs: 0, - bucketMs: Number(options.bucketMs) || 1000, - sampleCount: 0, - mediaElementCount: mediaElements().length, - sourceCount: 0, - rmsDbfs: [], - peakDbfs: [], - notes, - }); - const action = options.action || 'status'; - let probe = window[key]; - if (probe && probe.state === 'running' && now() - probe.startedAt >= probe.durationMs) { - sample(probe); - stopProbe(probe, 'duration'); - } - if (action === 'start') { - if (probe) stopProbe(probe, 'restarted'); - const context = getContext(); - if (!context) { - return stoppedResult(['Web Audio API is not available in this browser context.']); - } - const sink = context.createGain(); - sink.gain.value = 0; - sink.connect(context.destination); - probe = { - state: 'running', - active: true, - context, - sink, - seen: new WeakSet(), - analysers: [], - mediaElementCount: 0, - sourceCount: 0, - durationMs: Math.max(100, Number(options.durationMs) || 10000), - bucketMs: Math.max(100, Number(options.bucketMs) || 1000), - startedAt: now(), - stoppedAt: undefined, - reason: undefined, - heard: false, - rmsDbfs: [], - peakDbfs: [], - notes: [], - }; - window[key] = probe; - try { - void context.resume(); - } catch { - note(probe, 'AudioContext could not be resumed by the probe.'); - } - probe.resumeOnGesture = () => { - try { - void context.resume(); - } catch { - note(probe, 'AudioContext could not be resumed from a user gesture.'); - } - }; - for (const eventName of ['click', 'pointerdown', 'keydown']) { - window.addEventListener(eventName, probe.resumeOnGesture, { - capture: true, - once: true, - }); - } - discover(probe); - probe.timer = setInterval(() => sample(probe), probe.bucketMs); - probe.timeout = setTimeout(() => stopProbe(probe, 'duration'), probe.durationMs); - return result(probe); - } - if (action === 'stop') { - if (probe) sample(probe); - stopProbe(probe, 'manual'); - return result(probe); - } - if (probe) sample(probe); - return result(probe); -} - -function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { - const record = readAgentBrowserEvalResultRecord(data); - return normalizeAudioProbeRecord(record, { - source: 'media-elements', - backend: 'agent-browser', - durationMs: 0, - elapsedMs: 0, - bucketMs: 1000, - activeFallback: false, - mediaElementCount: 0, - sourceCount: 0, - }) as WebAudioProbeResult; -} - -function readAgentBrowserEvalResultRecord(data: unknown): JsonObject { - if (!isJsonObject(data)) return {}; - return isJsonObject(data.result) ? data.result : data; -} - async function runPacedScroll( runJson: (args: string[]) => Promise, direction: string, From a96fe16406779baad76795adf4cfbf828670a247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:33:53 +0200 Subject: [PATCH 12/18] refactor: remove audio probe leftovers --- src/daemon/audio-probe.ts | 3 --- src/daemon/handlers/session-observability.ts | 9 ++++++--- src/platforms/web/agent-browser-audio-probe.ts | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/daemon/audio-probe.ts b/src/daemon/audio-probe.ts index 605a20868..d1db975d4 100644 --- a/src/daemon/audio-probe.ts +++ b/src/daemon/audio-probe.ts @@ -5,7 +5,6 @@ import { normalizeAudioProbeRecord, type AudioProbeResult, } from '../audio-probe-result.ts'; -import { isHostSystemAudioProbeDevice } from '../core/capabilities.ts'; import { startMacOsAudioProbeProcess } from '../platforms/ios/macos-helper.ts'; import { AppError } from '../utils/errors.ts'; import { sleep } from '../utils/timeouts.ts'; @@ -23,8 +22,6 @@ export type HostAudioProbeCommand = { bucketMs: number; }; -export const usesHostSystemAudioProbe = isHostSystemAudioProbeDevice; - export async function runHostSystemAudioProbeCommand( request: HostAudioProbeCommand, ): Promise { diff --git a/src/daemon/handlers/session-observability.ts b/src/daemon/handlers/session-observability.ts index c5954896e..ea54d4880 100644 --- a/src/daemon/handlers/session-observability.ts +++ b/src/daemon/handlers/session-observability.ts @@ -1,4 +1,7 @@ -import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { + isCommandSupportedOnDevice, + isHostSystemAudioProbeDevice, +} from '../../core/capabilities.ts'; import { isPerfAction, isPerfArea, @@ -17,7 +20,7 @@ import { resolveWebProvider } from '../../platforms/web/provider.ts'; import type { AndroidAdbExecutor } from '../../platforms/android/adb-executor.ts'; import type { DaemonRequest, DaemonResponse, DaemonResponseData, SessionState } from '../types.ts'; import { SessionStore } from '../session-store.ts'; -import { runHostSystemAudioProbeCommand, usesHostSystemAudioProbe } from '../audio-probe.ts'; +import { runHostSystemAudioProbeCommand } from '../audio-probe.ts'; import { appendAppLogMarker, clearAppLogFiles, @@ -621,7 +624,7 @@ function resolveNetworkIncludeMode( async function handleAudioCommand(params: ObservabilityParams): Promise { const request = resolveAudioCommandRequest(params); if (!request.ok) return request; - if (usesHostSystemAudioProbe(request.session.device)) { + if (isHostSystemAudioProbeDevice(request.session.device)) { return await handleHostSystemAudioCommand(params, request); } const provider = resolveWebProvider(); diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index 04250bdf1..d371a7d96 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -28,7 +28,6 @@ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string action: options.action, durationMs: options.durationMs, bucketMs: options.bucketMs, - source: options.source ?? 'media-elements', })})})()`; } From 36038857334a2f51fccd53be05f438d9a8def8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:36:15 +0200 Subject: [PATCH 13/18] fix: encode audio probe eval options as data --- src/platforms/web/agent-browser-audio-probe.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index d371a7d96..7995e3665 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -24,11 +24,14 @@ const audioProbePageScriptFunctions = [ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const scriptBody = audioProbePageScriptFunctions.map((fn) => `${fn.toString()};`).join(''); - return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}(${JSON.stringify({ - action: options.action, - durationMs: options.durationMs, - bucketMs: options.bucketMs, - })})})()`; + const optionsJsonLiteral = JSON.stringify( + JSON.stringify({ + action: options.action, + durationMs: finiteNumberOrUndefined(options.durationMs), + bucketMs: finiteNumberOrUndefined(options.bucketMs), + }), + ); + return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}(JSON.parse(${optionsJsonLiteral}))})()`; } export function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { @@ -59,6 +62,10 @@ type AudioProbePageStats = { rms: number; peak: number }; declare const window: AudioProbePageRecord; declare const document: { querySelectorAll(selector: string): any[] }; +function finiteNumberOrUndefined(value: number | undefined): number | undefined { + return Number.isFinite(value) ? value : undefined; +} + function audioProbeDbfs(value: number): number { const silenceDb = -90; if (!Number.isFinite(value) || value <= 0) return silenceDb; From 0b85cb3c3fe8ae2197020ab8231a13a5eef96e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:39:54 +0200 Subject: [PATCH 14/18] fix: document audio probe eval sanitization --- src/platforms/web/agent-browser-audio-probe.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index 7995e3665..3e5014bc7 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -31,6 +31,8 @@ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string bucketMs: finiteNumberOrUndefined(options.bucketMs), }), ); + // lgtm[js/code-injection] agent-browser eval requires a code string; scriptBody is built + // from local static functions, and runtime options are parsed from a JSON string literal. return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}(JSON.parse(${optionsJsonLiteral}))})()`; } From d919d6276b7cd474ce46613158f801370c2ccf8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:44:17 +0200 Subject: [PATCH 15/18] fix: sanitize audio probe eval options --- .../web/agent-browser-audio-probe.ts | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index 3e5014bc7..3b232a462 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -24,15 +24,15 @@ const audioProbePageScriptFunctions = [ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const scriptBody = audioProbePageScriptFunctions.map((fn) => `${fn.toString()};`).join(''); - const optionsJsonLiteral = JSON.stringify( - JSON.stringify({ - action: options.action, - durationMs: finiteNumberOrUndefined(options.durationMs), - bucketMs: finiteNumberOrUndefined(options.bucketMs), - }), + const optionsJsonLiteral = escapeUnsafeCodeString( + JSON.stringify( + JSON.stringify({ + action: options.action, + durationMs: finiteNumberOrUndefined(options.durationMs), + bucketMs: finiteNumberOrUndefined(options.bucketMs), + }), + ), ); - // lgtm[js/code-injection] agent-browser eval requires a code string; scriptBody is built - // from local static functions, and runtime options are parsed from a JSON string literal. return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}(JSON.parse(${optionsJsonLiteral}))})()`; } @@ -64,6 +64,28 @@ type AudioProbePageStats = { rms: number; peak: number }; declare const window: AudioProbePageRecord; declare const document: { querySelectorAll(selector: string): any[] }; +const unsafeCodeStringCharacters: Record = { + '<': '\\u003C', + '>': '\\u003E', + '/': '\\u002F', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', + '\u0000': '\\0', + '\u2028': '\\u2028', + '\u2029': '\\u2029', +}; + +function escapeUnsafeCodeString(value: string): string { + let escaped = ''; + for (const character of value) { + escaped += unsafeCodeStringCharacters[character] ?? character; + } + return escaped; +} + function finiteNumberOrUndefined(value: number | undefined): number | undefined { return Number.isFinite(value) ? value : undefined; } From 99a2f4ec42158600b49aaf00e9d01ddaad0fc187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:47:57 +0200 Subject: [PATCH 16/18] fix: use codeql-recognized eval option sanitizer --- src/platforms/web/agent-browser-audio-probe.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index 3b232a462..9bb65f3cc 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -24,7 +24,7 @@ const audioProbePageScriptFunctions = [ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const scriptBody = audioProbePageScriptFunctions.map((fn) => `${fn.toString()};`).join(''); - const optionsJsonLiteral = escapeUnsafeCodeString( + const optionsJsonLiteral = escapeUnsafeChars( JSON.stringify( JSON.stringify({ action: options.action, @@ -67,7 +67,6 @@ declare const document: { querySelectorAll(selector: string): any[] }; const unsafeCodeStringCharacters: Record = { '<': '\\u003C', '>': '\\u003E', - '/': '\\u002F', '\b': '\\b', '\f': '\\f', '\n': '\\n', @@ -78,12 +77,12 @@ const unsafeCodeStringCharacters: Record = { '\u2029': '\\u2029', }; -function escapeUnsafeCodeString(value: string): string { - let escaped = ''; - for (const character of value) { - escaped += unsafeCodeStringCharacters[character] ?? character; - } - return escaped; +function escapeUnsafeChars(value: string): string { + return value.replace( + // eslint-disable-next-line no-control-regex -- CodeQL js/bad-code-sanitization recommends this sanitizer shape for generated code strings. + /[<>\b\f\n\r\t\0\u2028\u2029]/g, + (character) => unsafeCodeStringCharacters[character] ?? character, + ); } function finiteNumberOrUndefined(value: number | undefined): number | undefined { From e92ccdd0f03fa783eed248bb42a4038117b85fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 17:51:44 +0200 Subject: [PATCH 17/18] fix: allowlist audio probe eval options --- .../web/agent-browser-audio-probe.ts | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index 9bb65f3cc..445473563 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -24,16 +24,10 @@ const audioProbePageScriptFunctions = [ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const scriptBody = audioProbePageScriptFunctions.map((fn) => `${fn.toString()};`).join(''); - const optionsJsonLiteral = escapeUnsafeChars( - JSON.stringify( - JSON.stringify({ - action: options.action, - durationMs: finiteNumberOrUndefined(options.durationMs), - bucketMs: finiteNumberOrUndefined(options.bucketMs), - }), - ), - ); - return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}(JSON.parse(${optionsJsonLiteral}))})()`; + const action = readAudioProbeEvalAction(options.action); + const durationMs = finiteNumberLiteralOrUndefined(options.durationMs); + const bucketMs = finiteNumberLiteralOrUndefined(options.bucketMs); + return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}({action:${JSON.stringify(action)},durationMs:${durationMs},bucketMs:${bucketMs}})})()`; } export function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { @@ -64,29 +58,21 @@ type AudioProbePageStats = { rms: number; peak: number }; declare const window: AudioProbePageRecord; declare const document: { querySelectorAll(selector: string): any[] }; -const unsafeCodeStringCharacters: Record = { - '<': '\\u003C', - '>': '\\u003E', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', - '\u0000': '\\0', - '\u2028': '\\u2028', - '\u2029': '\\u2029', -}; - -function escapeUnsafeChars(value: string): string { - return value.replace( - // eslint-disable-next-line no-control-regex -- CodeQL js/bad-code-sanitization recommends this sanitizer shape for generated code strings. - /[<>\b\f\n\r\t\0\u2028\u2029]/g, - (character) => unsafeCodeStringCharacters[character] ?? character, - ); +function readAudioProbeEvalAction( + action: WebAudioProbeOptions['action'], +): WebAudioProbeOptions['action'] { + switch (action) { + case 'start': + return 'start'; + case 'stop': + return 'stop'; + default: + return 'status'; + } } -function finiteNumberOrUndefined(value: number | undefined): number | undefined { - return Number.isFinite(value) ? value : undefined; +function finiteNumberLiteralOrUndefined(value: number | undefined): string { + return value === undefined || !Number.isFinite(value) ? 'undefined' : String(Math.trunc(value)); } function audioProbeDbfs(value: number): number { From b982e4b366ea75489786128950aa3060ecdc1304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 26 Jun 2026 20:01:29 +0200 Subject: [PATCH 18/18] fix: avoid json-stringified audio eval action --- src/platforms/web/agent-browser-audio-probe.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/platforms/web/agent-browser-audio-probe.ts b/src/platforms/web/agent-browser-audio-probe.ts index 445473563..0139bf58e 100644 --- a/src/platforms/web/agent-browser-audio-probe.ts +++ b/src/platforms/web/agent-browser-audio-probe.ts @@ -24,10 +24,10 @@ const audioProbePageScriptFunctions = [ export function buildAudioProbeEvalScript(options: WebAudioProbeOptions): string { const scriptBody = audioProbePageScriptFunctions.map((fn) => `${fn.toString()};`).join(''); - const action = readAudioProbeEvalAction(options.action); + const action = audioProbeEvalActionLiteral(options.action); const durationMs = finiteNumberLiteralOrUndefined(options.durationMs); const bucketMs = finiteNumberLiteralOrUndefined(options.bucketMs); - return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}({action:${JSON.stringify(action)},durationMs:${durationMs},bucketMs:${bucketMs}})})()`; + return `(()=>{${scriptBody}return ${audioProbeEvalScript.name}({action:${action},durationMs:${durationMs},bucketMs:${bucketMs}})})()`; } export function normalizeAgentBrowserAudioProbeResult(data: unknown): WebAudioProbeResult { @@ -58,16 +58,16 @@ type AudioProbePageStats = { rms: number; peak: number }; declare const window: AudioProbePageRecord; declare const document: { querySelectorAll(selector: string): any[] }; -function readAudioProbeEvalAction( +function audioProbeEvalActionLiteral( action: WebAudioProbeOptions['action'], -): WebAudioProbeOptions['action'] { +): "'start'" | "'stop'" | "'status'" { switch (action) { case 'start': - return 'start'; + return "'start'"; case 'stop': - return 'stop'; + return "'stop'"; default: - return 'status'; + return "'status'"; } }