From 55213fd3884f24eab75a540145e5663a766f90e8 Mon Sep 17 00:00:00 2001 From: AdzeB <59929333+AdzeB@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:22:48 +0200 Subject: [PATCH 1/2] feat(ios): add non-activating observe-only snapshots Squashed from the PR #3108 history through 1e75ce4 and rebased onto main. Co-Authored-By: Claude Fable 5.1 --- .../RunnerTests+CommandDispatch.swift | 38 +++++- .../RunnerTests+Models.swift | 10 ++ .../RunnerTests+SnapshotCaptureTarget.swift | 2 +- .../RunnerTests+SnapshotExecution.swift | 34 ++++- .../RunnerTests+CommandDispatchTests.swift | 75 +++++++++++ contracts/fixtures/runner-requests.json | 2 + .../src/flag-definitions-workflow.ts | 12 ++ packages/contracts/src/cli-flags.ts | 1 + packages/contracts/src/facades/capture.ts | 1 + .../src/snapshot-capture-annotations.ts | 24 +++- packages/contracts/src/snapshot-runtime.ts | 14 ++ packages/contracts/src/snapshot-types.ts | 2 + packages/kernel/src/device.ts | 12 ++ packages/kernel/src/snapshot.ts | 13 ++ .../src/__tests__/device-fixtures.ts | 9 ++ .../interactor-target-activation.test.ts | 122 +++++++++++++++++- .../src/__tests__/runner-requests.test.ts | 27 ++++ packages/platform-apple/src/interactor.ts | 61 ++++++++- .../__tests__/runner-command-traits.test.ts | 15 ++- .../runner/__tests__/runner-exchange.test.ts | 21 +++ .../__tests__/runner-recovery-wiring.test.ts | 34 +++++ .../src/runner/runner-command-traits.ts | 12 +- .../src/runner/runner-contract.ts | 2 + .../src/runner/runner-lifecycle.ts | 22 +++- .../src/runner/snapshot-presentation.ts | 5 + .../platform-apple/src/runtime-snapshot.ts | 1 + scripts/integration-progress-model.ts | 8 ++ .../contracts/apple-os-resolution.test.ts | 19 ++- src/backend-snapshot-options.ts | 1 + src/commands/capture/runtime/snapshot.ts | 4 + src/commands/capture/snapshot.ts | 4 +- src/commands/schema/usage.test.ts | 2 +- src/daemon/response-views.ts | 1 + src/daemon/snapshot-runtime-binding.ts | 11 ++ website/docs/docs/commands.md | 2 +- website/docs/docs/snapshots.md | 31 +++++ 36 files changed, 632 insertions(+), 22 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index 959df239de..ab27de9ef2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -42,9 +42,14 @@ extension RunnerTests { func executeUptime() -> Response { // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a // fresher send-time stamp on every ok response; kept so direct callers still get a value. - Response( +#if os(iOS) + let supportsObserveOnlySnapshot = true +#else + let supportsObserveOnlySnapshot = false +#endif + return Response( ok: true, - data: DataPayload(currentUptimeMs: currentUptimeMs()) + data: DataPayload(currentUptimeMs: currentUptimeMs(), supportsObserveOnlySnapshot: supportsObserveOnlySnapshot) ) } @@ -114,6 +119,7 @@ extension RunnerTests { let app: XCUIApplication /// Set when `app` is a system surface served in place over the still-bound session app (#2438). var systemSurface: SystemSurfaceHost? = nil + var observation: SnapshotObservationPayload? = nil } enum ActiveCommandPreparation { @@ -494,6 +500,26 @@ extension RunnerTests { command: Command, routeToSpringboard: Bool = false ) -> ActiveCommandPreparation { + if command.command == .snapshot, command.observeOnly == true { +#if os(iOS) + guard command.appBundleId?.trimmedNonEmpty != nil else { + return .response(observeOnlyUnavailableResponse()) + } + let observedApp = resolveAppWithoutActivation(command: command) + let reportedState = observedApp.state + guard reportedState == .runningForeground else { + return .response(observeOnlyUnavailableResponse()) + } + return .context(ActiveCommandContext( + app: observedApp, + observation: SnapshotObservationPayload(appState: Self.applicationStateName(reportedState)) + )) +#else + return .response(Response(ok: false, error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", message: "observe-only snapshot is supported on iOS and iPadOS only" + ))) +#endif + } if routeToSpringboard { return .context(ActiveCommandContext(app: springboard)) } @@ -644,6 +670,14 @@ extension RunnerTests { return XCUIApplication(bundleIdentifier: host.bundleId).state } + func observeOnlyUnavailableResponse() -> Response { + Response(ok: false, error: ErrorPayload( + code: "OBSERVATION_UNAVAILABLE", + message: "The session app cannot be observed without activation.", + hint: "XCUIApplication must report runningForeground before and after capture. This state report does not prove screen ownership (#2696)." + )) + } + func currentXCTestFailureCount() -> Int { return testRun?.failureCount ?? 0 } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index c2f4d8ea44..1e36d85447 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -239,6 +239,7 @@ struct Command: Codable { let interactiveOnly: Bool? let preferredBackend: String? let customActions: Bool? + let observeOnly: Bool? let depth: Int? let scope: String? let raw: Bool? @@ -447,6 +448,7 @@ struct DataPayload: Codable { var referenceWidth: Double? var referenceHeight: Double? var currentUptimeMs: Double? + var supportsObserveOnlySnapshot: Bool? var applicationState: String? var commandId: String? var lifecycleState: String? @@ -488,6 +490,7 @@ struct DataPayload: Codable { var failedStepIndex: Int? var sequenceResults: [SequenceStepResult]? var targetActivation: TargetActivationFactPayload? + var observation: SnapshotObservationPayload? /// Present on a screenshot the runner captured from a display it resolved, alongside the /// `message` path or `imageBase64` payload that carries the image itself (#2728). var screenshotMetadata: ScreenshotMetadataPayload? @@ -529,6 +532,13 @@ struct SystemSurfaceProvenancePayload: Codable { let kind: String } +struct SnapshotObservationPayload: Codable { + var mode = "observe-only" + var activationPerformed = false + let appState: String + var appStateSource = "xcuiapplication-state" +} + struct SnapshotQualityPayload: Codable { let nodes: [PresentedNode] let truncated: Bool diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift index af106bb702..0096953434 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift @@ -17,7 +17,7 @@ struct SnapshotCaptureTarget { /// What snapshot command preparation hands the off-main capture. enum SnapshotCommandPreparation { case response(Response) - case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?) + case capture(SnapshotCaptureTarget, systemSurface: SystemSurfaceHost?, observation: SnapshotObservationPayload?) } /// The target a bounded XCTest probe arms its abandonment penalty with. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift index 42d1bfc562..b4063d9b0e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift @@ -19,19 +19,27 @@ extension RunnerTests { return .response(response) case .context(let context): return .capture( - self.takeSnapshotCaptureTarget(app: context.app), - systemSurface: context.systemSurface + context.observation == nil + ? self.takeSnapshotCaptureTarget(app: context.app) + : SnapshotCaptureTarget( + app: context.app, + bundleId: command.appBundleId?.trimmedNonEmpty, + processIdentifier: Self.processIdentifier(of: context.app) + ), + systemSurface: context.systemSurface, + observation: context.observation ) } } switch preparation { case .response(let response): return response - case .capture(let target, let systemSurface): + case .capture(let target, let systemSurface, let observation): return try executeSnapshotPrepared( command: command, target: target, - systemSurface: systemSurface + systemSurface: systemSurface, + observation: observation ) } } @@ -86,7 +94,8 @@ extension RunnerTests { private func executeSnapshotPrepared( command: Command, target: SnapshotCaptureTarget, - systemSurface: SystemSurfaceHost? + systemSurface: SystemSurfaceHost?, + observation: SnapshotObservationPayload? ) throws -> Response { let options = Self.presentationOptions(from: command) do { @@ -102,6 +111,21 @@ extension RunnerTests { kind: systemSurface.kind.rawValue ) } + if let observation { + guard let capturedProcessIdentifier = target.processIdentifier else { + return observeOnlyUnavailableResponse() + } + let unchanged = try runMainThreadWork( + "observe_only_state", + timeout: Self.mainThreadExecutionTimeout, + timeoutError: Self.mainThreadExecutionTimeoutError + ) { + target.app.state == .runningForeground + && Self.processIdentifier(of: target.app) == capturedProcessIdentifier + } + guard unchanged else { return observeOnlyUnavailableResponse() } + payload.observation = observation + } setNeedsPostSnapshotInteractionDelay() return Response(ok: true, data: payload) } catch let failure as SnapshotCaptureFailure { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift index 76c6a642a9..bd3e94a526 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -3,6 +3,14 @@ import AgentDeviceSnapshotPresentation #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { + func testUptimeAdvertisesObserveOnlyOnIosOnly() { +#if os(iOS) + XCTAssertEqual(executeUptime().data?.supportsObserveOnlySnapshot, true) +#else + XCTAssertEqual(executeUptime().data?.supportsObserveOnlySnapshot, false) +#endif + } + func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { // The seam's recording side cannot run in-bundle (a real XCTIssue would // fail this very test run — same constraint the record(_:) suppression @@ -66,6 +74,73 @@ extension RunnerTests { // `os(iOS)` regions in this file are pure runner decisions and also run on the macOS host // lane (ci.yml) — see the classification convention in RunnerTests.swift. #if os(iOS) + @MainActor + func testObserveOnlySnapshotRefusesBackgroundWithoutActivationOrDisclosure() throws { + app.launch() + XCUIDevice.shared.press(.home) + XCTAssertTrue(app.wait(for: .runningBackground, timeout: 10)) + pendingTargetActivation = nil + defer { app.terminate() } + let command = try runnerCommandFixture( + "{\"command\":\"snapshot\",\"appBundleId\":\"com.callstack.agentdevice.runner\",\"observeOnly\":true}" + ) + guard case .response(let response) = prepareActiveCommandContext(command: command) else { + return XCTFail("observe-only must refuse a background target") + } + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "OBSERVATION_UNAVAILABLE") + XCTAssertEqual(app.state, .runningBackground) + XCTAssertNil(pendingTargetActivation) + XCTAssertNil(response.data?.targetActivation) + } + + @MainActor + func testObserveOnlySnapshotReportsStateWithoutBindingOrActivatingTarget() throws { + app.launch() + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + invalidateCachedTarget(reason: "unit_test_setup") + pendingTargetActivation = nil + defer { app.terminate() } + let command = try runnerCommandFixture( + "{\"command\":\"snapshot\",\"appBundleId\":\"com.callstack.agentdevice.runner\",\"observeOnly\":true}" + ) + guard case .context(let context) = prepareActiveCommandContext(command: command) else { + return XCTFail("observe-only must accept the reported foreground state") + } + XCTAssertEqual(context.observation?.mode, "observe-only") + XCTAssertEqual(context.observation?.activationPerformed, false) + XCTAssertEqual(context.observation?.appState, "runningForeground") + XCTAssertEqual(context.observation?.appStateSource, "xcuiapplication-state") + XCTAssertNil(context.systemSurface) + XCTAssertNil(mainOwned.bundleId) + XCTAssertNil(pendingTargetActivation) + } + + @MainActor + func testRegularSnapshotStillRepairsBackgroundAndDisclosesPriorState() throws { + app.launch() + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + XCUIDevice.shared.press(.home) + XCTAssertTrue(app.wait(for: .runningBackground, timeout: 10)) + pendingTargetActivation = nil + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + pendingTargetActivation = nil + app.terminate() + } + let command = try runnerCommandFixture( + "{\"command\":\"snapshot\",\"appBundleId\":\"com.callstack.agentdevice.runner\"}" + ) + guard case .context(let context) = prepareActiveCommandContext(command: command) else { + return XCTFail("regular snapshot must keep the foreground repair") + } + XCTAssertTrue(app.wait(for: .runningForeground, timeout: 10)) + XCTAssertNil(context.observation) + XCTAssertEqual(pendingTargetActivation?.priorState, Int(XCUIApplication.State.runningBackground.rawValue)) + XCTAssertEqual(pendingTargetActivation?.reason, "stale_target") + } + @MainActor func testMissingBundleCommandInvalidatesCompleteCachedTargetState() throws { app.launch() diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index cf337eab32..b875630453 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -43,6 +43,8 @@ {"name": "ios-simulator.interactor-read-text.point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-set-orientation.rotate", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "rotate", "orientation": "landscape-left", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-snapshot.every-option", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "preferredBackend": "tree", "customActions": true, "depth": 3, "scope": "Go", "raw": true, "commandId": ""}}, + {"name": "ios-simulator.interactor-snapshot.observe-only", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "observeOnly": true, "commandId": ""}}, + {"name": "ios-simulator.interactor-snapshot.observe-only.capability", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "uptime", "observeOnly": true, "commandId": ""}}, {"name": "ios-simulator.interactor-write-clipboard.text", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "pasteboardWrite", "text": "one-time code 246810", "commandId": ""}}, {"name": "ios-simulator.recording-clock-anchor.snapshot", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "depth": 1}}, {"name": "ios-simulator.recording-start.default", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "recordStart", "outPath": "capture.mp4", "appBundleId": "com.example.app"}}, diff --git a/packages/command-registry/src/flag-definitions-workflow.ts b/packages/command-registry/src/flag-definitions-workflow.ts index 5474413767..d1e82d5b67 100644 --- a/packages/command-registry/src/flag-definitions-workflow.ts +++ b/packages/command-registry/src/flag-definitions-workflow.ts @@ -273,6 +273,18 @@ export const WORKFLOW_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ projectConfig: true, recorded: true, }, + { + key: 'snapshotObserveOnly', + names: ['--observe-only'], + type: 'boolean', + usageLabel: '--observe-only', + usageDescription: + 'Snapshot: observe the iOS session app without activation or foreground repair', + inputDescription: + 'iOS only. Refuse instead of activating or repairing the session app. Reports XCUIApplication state, which does not prove screen ownership (#2696).', + projectConfig: false, + recorded: false, + }, { key: 'snapshotForceFull', names: ['--force-full'], diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index 8cd4412922..01e8162e90 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -86,6 +86,7 @@ export type CliFlags = CloudProviderProfileFields & snapshotScope?: string; snapshotRaw?: boolean; snapshotCustomActions?: boolean; + snapshotObserveOnly?: boolean; snapshotForceFull?: boolean; artifact?: string; dsym?: string; diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index a25a0c2258..0598b68f5e 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -28,6 +28,7 @@ export type { export { publicSnapshotCaptureAnnotations, readSerializedSnapshotCaptureAnnotations, + readSnapshotObservation, snapshotCaptureAnnotationsFrom, } from '../snapshot-capture-annotations.ts'; export type { diff --git a/packages/contracts/src/snapshot-capture-annotations.ts b/packages/contracts/src/snapshot-capture-annotations.ts index 7ad86290cd..6a7f592461 100644 --- a/packages/contracts/src/snapshot-capture-annotations.ts +++ b/packages/contracts/src/snapshot-capture-annotations.ts @@ -1,5 +1,6 @@ import type { IosTargetActivation, + IosSnapshotObservation, SnapshotCaptureBackend, SnapshotQualityState, SnapshotQualityVerdict, @@ -43,11 +44,12 @@ export type SnapshotCaptureAnnotations = { warnings?: string[]; /** The Apple runner re-activated the session app while serving this capture (#2682). */ targetActivation?: IosTargetActivation; + observation?: IosSnapshotObservation; }; export type PublicSnapshotCaptureAnnotations = Pick< SnapshotCaptureAnnotations, - 'androidSnapshot' | 'warnings' | 'targetActivation' + 'androidSnapshot' | 'warnings' | 'targetActivation' | 'observation' > & { snapshotQuality?: SnapshotQualityVerdict; }; @@ -63,6 +65,7 @@ export function snapshotCaptureAnnotationsFrom( ...(quality ? { quality } : {}), ...(source.warnings ? { warnings: source.warnings } : {}), ...(source.targetActivation ? { targetActivation: source.targetActivation } : {}), + ...(source.observation ? { observation: source.observation } : {}), }; } @@ -76,6 +79,7 @@ export function publicSnapshotCaptureAnnotations( ? { warnings: annotations.warnings } : {}), ...(annotations.targetActivation ? { targetActivation: annotations.targetActivation } : {}), + ...(annotations.observation ? { observation: annotations.observation } : {}), }; } @@ -91,6 +95,7 @@ export function readSerializedSnapshotCaptureAnnotations( : undefined; const quality = readPublishedSnapshotQualityVerdict(data.snapshotQuality); const targetActivation = readTargetActivation(data.targetActivation); + const observation = readSnapshotObservation(data.observation); return publicSnapshotCaptureAnnotations({ ...(androidSnapshot ? { androidSnapshot: androidSnapshot as AndroidSnapshotBackendMetadata } @@ -98,9 +103,26 @@ export function readSerializedSnapshotCaptureAnnotations( ...(quality ? { quality } : {}), ...(warnings ? { warnings } : {}), ...(targetActivation ? { targetActivation } : {}), + ...(observation ? { observation } : {}), }); } +export function readSnapshotObservation(value: unknown): IosSnapshotObservation | undefined { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined; + const raw = value as Record; + return raw.mode === 'observe-only' && + raw.activationPerformed === false && + raw.appState === 'runningForeground' && + raw.appStateSource === 'xcuiapplication-state' + ? { + mode: 'observe-only', + activationPerformed: false, + appState: raw.appState, + appStateSource: raw.appStateSource, + } + : undefined; +} + /** Re-read of a fact this module projected; the declared keys are the only ones it publishes. */ function readTargetActivation(value: unknown): IosTargetActivation | undefined { if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined; diff --git a/packages/contracts/src/snapshot-runtime.ts b/packages/contracts/src/snapshot-runtime.ts index 4289c26e88..93df5d1be6 100644 --- a/packages/contracts/src/snapshot-runtime.ts +++ b/packages/contracts/src/snapshot-runtime.ts @@ -116,6 +116,7 @@ function bindSnapshotInteractor( params: SnapshotInteractorBindingParams, ): SnapshotRuntimeOperations { const captureSnapshot = async (input: CaptureSnapshotInput) => { + await assertSnapshotObservationOwner(params, input); const signal = captureSnapshotSignal(params.signal, input); const runner: RunnerContext = { ...input.execution, @@ -151,6 +152,19 @@ function bindSnapshotInteractor( }); } +async function assertSnapshotObservationOwner( + params: SnapshotInteractorBindingParams, + input: CaptureSnapshotInput, +): Promise { + if (input.options?.observeOnly !== true) return; + const { isHandheldAppleDevice } = await import('@agent-device/kernel/device'); + if (params.ownership === 'local' && isHandheldAppleDevice(params.device)) return; + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'observe-only snapshot requires a local iOS or iPadOS runner.', + ); +} + export function bindLocalSnapshotInteractor( params: Readonly<{ device: DeviceInfo; diff --git a/packages/contracts/src/snapshot-types.ts b/packages/contracts/src/snapshot-types.ts index bb0f7ba345..3c0c56e093 100644 --- a/packages/contracts/src/snapshot-types.ts +++ b/packages/contracts/src/snapshot-types.ts @@ -3,6 +3,7 @@ import type { SnapshotState, SnapshotOptions, SnapshotQualityVerdict, + IosSnapshotObservation, ScreenshotOverlayRef, SnapshotViewportSize, } from '@agent-device/kernel/snapshot'; @@ -44,6 +45,7 @@ export type BackendSnapshotResult = { }; quality?: SnapshotQualityVerdict; warnings?: string[]; + observation?: IosSnapshotObservation; }; export type BackendSnapshotOptions = SnapshotOptions & { includeRects?: boolean; diff --git a/packages/kernel/src/device.ts b/packages/kernel/src/device.ts index 8b5f172118..3d098dda7a 100644 --- a/packages/kernel/src/device.ts +++ b/packages/kernel/src/device.ts @@ -129,6 +129,18 @@ export function isHandheldAppleSimulator( return appleOs === 'ios' || appleOs === 'ipados'; } +/** + * The iPhone/iPad leaf on a simulator or hardware, including legacy records that predate the + * stored `appleOs` discriminant. Excludes tvOS, visionOS, the macOS host, and non-Apple platforms. + */ +export function isHandheldAppleDevice( + device: Pick, +): boolean { + if (!isIosFamily(device)) return false; + const appleOs = resolveDeviceAppleOs(device); + return appleOs === 'ios' || appleOs === 'ipados'; +} + export function isMobilePlatform(device: Pick): boolean { // Phone/tablet device family: Android plus every Apple OS except the macOS desktop // host. Preserves the pre-collapse `platform === 'ios' || platform === 'android'` diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 684e9e55d1..01117acd61 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -81,7 +81,16 @@ export type Point = { y: number; }; +export type IosSnapshotObservation = { + mode: 'observe-only'; + activationPerformed: false; + /** XCTest's report, not proof of foreground ownership (#2696). */ + appState: 'runningForeground'; + appStateSource: 'xcuiapplication-state'; +}; + export type SnapshotOptions = { + observeOnly?: boolean; interactiveOnly?: boolean; depth?: number; scope?: string; @@ -129,6 +138,7 @@ export const SNAPSHOT_OPTION_FLAGS = { depth: 'snapshotDepth', scope: 'snapshotScope', raw: 'snapshotRaw', + observeOnly: 'snapshotObserveOnly', customActions: 'snapshotCustomActions', forceFull: 'snapshotForceFull', includeHiddenContentHints: 'snapshotIncludeHiddenContentHints', @@ -138,6 +148,7 @@ export const SNAPSHOT_OPTION_FLAGS = { export type SnapshotOptionKey = keyof typeof SNAPSHOT_OPTION_FLAGS; type SnapshotOptionValues = { + observeOnly: boolean; interactiveOnly: boolean; depth: number; scope: string; @@ -164,6 +175,7 @@ export type SnapshotOptionFlagFields) { const runnerProvider: AppleRunnerProvider = { hasLiveSession: () => true, - runCommand: async () => payload, + runCommand: async () => ({ + ...payload, + supportsObserveOnlySnapshot: true, + runnerSessionId: 'observation-runner', + }), }; return createAppleInteractor(IOS_SIMULATOR, { appBundleId: 'com.example.app' }, runnerProvider); } @@ -50,6 +55,117 @@ test('a capture whose own command repaired foreground discloses it and keeps the ); }); +test.each([ + ['iOS', IOS_SIMULATOR], + ['iPadOS', IPADOS_SIMULATOR], +] as const)( + 'observe-only on %s forwards the policy and publishes sourced state without activation disclosure', + async (_leaf, device) => { + const observation = { + mode: 'observe-only', + activationPerformed: false, + appState: 'runningForeground', + appStateSource: 'xcuiapplication-state', + }; + const commands: RunnerCommand[] = []; + const runnerProvider: AppleRunnerProvider = { + hasLiveSession: () => true, + runCommand: async (_device, command, options) => { + commands.push(command); + if (command.command === 'snapshot') { + assert.equal(options?.expectedRunnerSessionId, 'observation-runner'); + } + return { + ...HEALTHY_TREE, + observation, + supportsObserveOnlySnapshot: true, + runnerSessionId: 'observation-runner', + }; + }, + }; + const snapshot = (await createAppleInteractor(device, {}, runnerProvider).snapshot({ + appBundleId: 'com.example.app', + observeOnly: true, + })) as SnapshotResult; + const runnerCommand = (name: string) => commands.find((command) => command.command === name); + assert.equal(runnerCommand('uptime')?.observeOnly, true); + assert.equal(runnerCommand('snapshot')?.observeOnly, true); + assert.deepEqual(snapshot.observation, observation); + assert.equal('targetActivation' in snapshot, false); + }, +); + +test('observe-only refuses a tvOS session before any runner command', async () => { + const commands: string[] = []; + const runnerProvider: AppleRunnerProvider = { + hasLiveSession: () => true, + runCommand: async (_device, command) => { + commands.push(command.command); + return {}; + }, + }; + await assert.rejects( + createAppleInteractor( + { ...IOS_SIMULATOR, appleOs: 'tvos', target: 'tv' }, + {}, + runnerProvider, + ).snapshot({ appBundleId: 'com.example.app', observeOnly: true }), + (error: unknown) => error instanceof AppError && error.code === 'UNSUPPORTED_OPERATION', + ); + assert.deepEqual(commands, []); +}); + +test.each([ + { supportsObserveOnlySnapshot: false, runnerSessionId: 'observation-runner' }, + { supportsObserveOnlySnapshot: true }, +])( + 'observe-only rejects an unbound capability response before snapshot dispatch: %j', + async (capabilities) => { + const commands: string[] = []; + const runnerProvider: AppleRunnerProvider = { + hasLiveSession: () => true, + runCommand: async (_device, command) => { + commands.push(command.command); + return capabilities; + }, + }; + await assert.rejects( + createAppleInteractor(IOS_SIMULATOR, {}, runnerProvider).snapshot({ + appBundleId: 'com.example.app', + observeOnly: true, + }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'observation-unavailable', + ); + assert.deepEqual(commands, ['uptime']); + }, +); + +test('observe-only refuses absent provenance rather than crediting a legacy activating runner', async () => { + await assert.rejects( + interactorServing(HEALTHY_TREE).snapshot({ appBundleId: 'com.example.app', observeOnly: true }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'observation-unavailable', + ); +}); + +test('observe-only refuses a contradictory activation fact', async () => { + await assert.rejects( + interactorServing({ + ...HEALTHY_TREE, + observation: { + mode: 'observe-only', + activationPerformed: false, + appState: 'runningForeground', + appStateSource: 'xcuiapplication-state', + }, + targetActivation: { reason: 'stale_target', priorState: 3 }, + }).snapshot({ appBundleId: 'com.example.app', observeOnly: true }), + (error: unknown) => + error instanceof AppError && error.details?.reason === 'observation-unavailable', + ); +}); + test('an untouched capture stays silent and carries no activation fact', async () => { const snapshot = (await interactorServing({ ...HEALTHY_TREE, diff --git a/packages/platform-apple/src/__tests__/runner-requests.test.ts b/packages/platform-apple/src/__tests__/runner-requests.test.ts index d3767da208..319d05a0da 100644 --- a/packages/platform-apple/src/__tests__/runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/runner-requests.test.ts @@ -1,3 +1,4 @@ +import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; @@ -224,6 +225,32 @@ test('every Apple request site builds exactly its golden runner request', async ); captured.push([name, onlyRequest(name, calls)]); } + const observationCalls: RecordedRunnerCall[] = []; + await createAppleInteractor( + IOS_SIMULATOR, + { appBundleId: APP }, + recordingRunnerProvider(observationCalls, { + uptime: { supportsObserveOnlySnapshot: true, runnerSessionId: 'observe-only-runner' }, + snapshot: { + nodes: [{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }], + observation: { + mode: 'observe-only', + activationPerformed: false, + appState: 'runningForeground', + appStateSource: 'xcuiapplication-state', + }, + }, + }), + ).snapshot({ appBundleId: APP, observeOnly: true }); + assert.deepEqual( + observationCalls.map((call) => call.command.command), + ['uptime', 'snapshot'], + ); + assert.equal(observationCalls[1]!.options.expectedRunnerSessionId, 'observe-only-runner'); + captured.push( + ['ios-simulator.interactor-snapshot.observe-only.capability', observationCalls[0]!.command], + ['ios-simulator.interactor-snapshot.observe-only', observationCalls[1]!.command], + ); const dir = await mkdtempForTest('agent-device-runner-requests-'); const runnerScreenshot = path.join(dir, 'runner.png'); fs.writeFileSync(runnerScreenshot, ''); diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 4e33efa95e..daec761e34 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -7,6 +7,7 @@ import { appleRemotePressCommand } from './os/tvos/remote.ts'; import { runMacOsScreenshotAction } from './os/macos/helper.ts'; import { actOnAppleAlert, awaitAppleAlert, readAppleAlert } from './alert.ts'; import { runAppleRunnerCommand } from './core/runner-client.ts'; +import type { AppleRunnerCommandOptions } from './runner/runner-provider.ts'; import { withAppleRunnerProvider, type AppleRunnerCommandExecutor, @@ -22,7 +23,12 @@ import { import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; -import { isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device'; +import { + isHandheldAppleDevice, + isMacOs, + isTvOsDevice, + type DeviceInfo, +} from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { withMethodScope } from '@agent-device/kernel/scoped-provider'; import type { Point, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; @@ -227,6 +233,12 @@ async function captureAppleSnapshot( runnerOpts: RunnerCallOptions, helper: MacOsHelperSurface | undefined, ) { + if (options?.observeOnly === true && !isHandheldAppleDevice(device)) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'observe-only snapshot is supported on iOS and iPadOS only.', + ); + } if (helper) { return await captureMacOsSurfaceSnapshot({ ...options, surface: helper }, options?.signal); } @@ -238,6 +250,7 @@ async function captureAppleRunnerSnapshot( options: SnapshotOptions | undefined, runnerOpts: RunnerCallOptions, ) { + const observationPolicy = await snapshotObservationPolicy(device, options, runnerOpts); const result = readAppleSnapshotResult( await withDiagnosticTimer( 'snapshot_capture', @@ -250,15 +263,17 @@ async function captureAppleRunnerSnapshot( interactiveOnly: options?.interactiveOnly, preferredBackend: options?.preferredBackend, customActions: options?.customActions, + ...observationPolicy.request, depth: options?.depth, scope: options?.scope, raw: options?.raw, }, - mergeRunnerCallSignal(runnerOpts, options?.signal), + mergeRunnerCallSignal(observationPolicy.runnerOpts, options?.signal), ), { backend: 'xctest' }, ), ); + assertSnapshotObservationResponse(options, result); assertReportedRunnerSnapshotNodes(device, options, result); const warnings = runnerSnapshotWarnings(result); const presentation = presentRunnerSnapshotForDevice(device, options, result); @@ -272,10 +287,52 @@ async function captureAppleRunnerSnapshot( ...(result.systemSurface ? { systemSurface: result.systemSurface } : {}), ...(result.keyboard ? { keyboard: result.keyboard } : {}), ...(result.targetActivation ? { targetActivation: result.targetActivation } : {}), + ...(result.observation ? { observation: result.observation } : {}), ...(warnings.length > 0 ? { warnings } : {}), }; } +async function snapshotObservationPolicy( + device: DeviceInfo, + options: SnapshotOptions | undefined, + runnerOpts: RunnerCallOptions, +): Promise<{ + request: { observeOnly?: true }; + runnerOpts: RunnerCallOptions & Pick; +}> { + if (options?.observeOnly !== true) return { request: {}, runnerOpts }; + const capabilities = await runAppleRunnerCommand( + device, + { command: 'uptime', observeOnly: true }, + mergeRunnerCallSignal(runnerOpts, options.signal), + ); + if ( + capabilities.supportsObserveOnlySnapshot !== true || + typeof capabilities.runnerSessionId !== 'string' || + capabilities.runnerSessionId.length === 0 + ) { + throw new AppError('COMMAND_FAILED', 'Runner does not support observe-only snapshots.', { + reason: 'observation-unavailable', + dispatched: 'no', + }); + } + return { + request: { observeOnly: true }, + runnerOpts: { ...runnerOpts, expectedRunnerSessionId: capabilities.runnerSessionId }, + }; +} + +function assertSnapshotObservationResponse( + options: SnapshotOptions | undefined, + result: AppleRunnerSnapshotResult, +): void { + if (options?.observeOnly !== true) return; + if (result.observation && !result.targetActivation) return; + throw new AppError('COMMAND_FAILED', 'Runner did not return an observe-only snapshot.', { + reason: 'observation-unavailable', + }); +} + /** * A runner capture reporting no nodes is a failure unless the caller's own scope justifies it: an * empty scoped capture is what `--scope` asked for, and a sparse-quality capture already says so. diff --git a/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts b/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts index 60974e424b..fb8734b7cd 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts @@ -39,9 +39,8 @@ test('runner command trait table pins lifecycle-sensitive command groups', () => 'querySelector', 'readText', 'screenshot', - 'snapshot', ], - payloadDependent: ['alert'], + payloadDependent: ['alert', 'snapshot'], readOnlyReadinessProbe: ['status', 'uptime'], readinessPreflightExemptMutation: ['activate', 'targetReset', 'terminate'], default: [ @@ -65,6 +64,18 @@ test('runner command trait table pins lifecycle-sensitive command groups', () => assert.deepEqual(Object.values(groups).flat().sort(), [...RUNNER_COMMANDS].sort()); }); +test('only an observe-only snapshot is exempt from the readiness preflight', () => { + assert.deepEqual(readRunnerCommandTraits({ command: 'snapshot' }), readOnly()); + assert.deepEqual( + readRunnerCommandTraits({ command: 'snapshot', observeOnly: false }), + readOnly(), + ); + assert.deepEqual(readRunnerCommandTraits({ command: 'snapshot', observeOnly: true }), { + ...readOnly(), + readinessPreflightExempt: true, + }); +}); + test('alert actions match the native read-only golden table', () => { // The fixture's `query` column records whether the alert request changes anything — `get` is the // one action that is side-effect-free — and each side consumes it under its own name: `readOnly` diff --git a/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts b/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts index e6bf4b125d..8fb8aa86f2 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts @@ -57,6 +57,27 @@ test('the exchange preflights a mutation and settles only its answer', async () assert.deepEqual(invalidations, []); }); +test('the exchange sends an observe-only snapshot without a readiness preflight', async () => { + server = await startFakeRunnerServer({ + snapshot: [{ kind: 'ok', data: { nodes: [] } }], + }); + const session = sessionFor(server.port); + + await executeRunnerExchange( + IOS_SIMULATOR, + session, + { command: 'snapshot', observeOnly: true, appBundleId: 'com.example.app' }, + undefined, + 10_000, + async () => {}, + ); + + assert.deepEqual( + server.requests.map(({ command }) => command), + ['snapshot'], + ); +}); + test('the readiness probe preserves its main-thread busy report when the command omits it', async () => { server = await startFakeRunnerServer({ uptime: [{ kind: 'ok', data: { runnerMainThreadBusy: true } }], diff --git a/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts b/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts index 4d8e4106fe..1e883c564c 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts @@ -597,6 +597,40 @@ test('an exact-session command never dispatches to a replacement runner', async assert.deepEqual(server.requests, []); }); +test('observe-only refuses an unavailable runner without creating a session', async () => { + await expect( + runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot', observeOnly: true }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'observation-unavailable', dispatched: 'no' }, + }); + expect(ensureRunnerSessionMock).not.toHaveBeenCalled(); + expect(invalidateRunnerSessionMock).not.toHaveBeenCalled(); +}); + +test('observe-only keeps the capability-probed identity when a replacement runner is ready', async () => { + server = await startFakeRunnerServer({ snapshot: [{ kind: 'ok', data: {} }] }); + const replacement = seedSession(server.port); + const sessionModule = await import('../runner-session.ts'); + const liveness = vi.spyOn(sessionModule, 'readRunnerSessionLiveness').mockReturnValue({ + sessionId: replacement.sessionId, + liveness: 'ready', + }); + try { + await expect( + runAppleRunnerCommand( + IOS_SIMULATOR, + { command: 'snapshot', observeOnly: true }, + { expectedRunnerSessionId: `${replacement.sessionId}:capability-probed` }, + ), + ).rejects.toThrow('runner session ownership changed'); + assert.deepEqual(server.requests, []); + expect(invalidateRunnerSessionMock).not.toHaveBeenCalled(); + } finally { + liveness.mockRestore(); + } +}); + test.each( (['accept', 'dismiss'] as const).flatMap((action) => (['accepted', 'started', 'completed'] as const).map((lifecycleState) => ({ diff --git a/packages/platform-apple/src/runner/runner-command-traits.ts b/packages/platform-apple/src/runner/runner-command-traits.ts index 17d5582cb7..32a2774db1 100644 --- a/packages/platform-apple/src/runner/runner-command-traits.ts +++ b/packages/platform-apple/src/runner/runner-command-traits.ts @@ -24,6 +24,11 @@ const READ_ONLY_TRAITS: RunnerCommandTraits = { readOnly: true, }; +const READ_ONLY_READINESS_PREFLIGHT_EXEMPT_TRAITS: RunnerCommandTraits = { + ...READ_ONLY_TRAITS, + readinessPreflightExempt: true, +}; + const READ_ONLY_READINESS_PROBE_TRAITS: RunnerCommandTraits = { ...READ_ONLY_TRAITS, readinessProbe: true, @@ -47,6 +52,11 @@ type RunnerCommandTraitsEntry = const readAlertActionTraits = (command: RunnerCommand): RunnerCommandTraits => (command.action ?? 'get').toLowerCase() === 'get' ? READ_ONLY_TRAITS : DEFAULT_TRAITS; +// An observe-only snapshot is pinned to the session its own observe-only uptime probe just proved +// ready, so a second readiness probe would add a request without adding evidence. +const readSnapshotTraits = (command: RunnerCommand): RunnerCommandTraits => + command.observeOnly === true ? READ_ONLY_READINESS_PREFLIGHT_EXEMPT_TRAITS : READ_ONLY_TRAITS; + /** * Traits of every runner command the daemon can send. A command whose traits depend on its * payload maps to a function of the command instead of a fixed trait set. @@ -64,7 +74,7 @@ export const RUNNER_COMMAND_TRAITS = { findText: READ_ONLY_TRAITS, querySelector: READ_ONLY_TRAITS, readText: READ_ONLY_TRAITS, - snapshot: READ_ONLY_TRAITS, + snapshot: readSnapshotTraits, screenshot: READ_ONLY_TRAITS, backInApp: DEFAULT_TRAITS, backSystem: DEFAULT_TRAITS, diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index e4a2d6ff4b..3616d505b3 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -134,6 +134,7 @@ export type RunnerCommand = { * because no other backend can carry them. */ customActions?: boolean; + observeOnly?: boolean; depth?: number; scope?: string; raw?: boolean; @@ -289,6 +290,7 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new [RUNNER_BUSY_RUNNER_CODE, 'no'], [RUNNER_WEDGED_RUNNER_CODE, 'no'], [APP_NOT_RUNNING_RUNNER_CODE, 'no'], + ['OBSERVATION_UNAVAILABLE', 'no'], [SCROLL_KEYBOARD_OCCLUDES_SURFACE_RUNNER_CODE, 'no'], [ALERT_NOT_FOUND_RUNNER_CODE, 'no'], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, 'no'] as const), diff --git a/packages/platform-apple/src/runner/runner-lifecycle.ts b/packages/platform-apple/src/runner/runner-lifecycle.ts index 76c5218e4d..46fd54ba6c 100644 --- a/packages/platform-apple/src/runner/runner-lifecycle.ts +++ b/packages/platform-apple/src/runner/runner-lifecycle.ts @@ -286,9 +286,29 @@ export async function executeRunnerCommand( command: RunnerCommand, options: AppleRunnerCommandOptions, ): Promise> { + if (command.observeOnly === true) { + const registration = readRunnerSessionLiveness(device.id); + if (registration?.liveness !== 'ready') { + throw new AppError( + 'COMMAND_FAILED', + 'observe-only snapshot requires an already-ready runner.', + { + reason: 'observation-unavailable', + dispatched: 'no', + }, + ); + } + options = { + ...options, + expectedRunnerSessionId: options.expectedRunnerSessionId ?? registration.sessionId, + }; + } const exchange = { entered: false }; try { - return await executeRunnerCommandAttempt(device, command, options, exchange); + const result = await executeRunnerCommandAttempt(device, command, options, exchange); + return command.observeOnly === true && command.command === 'uptime' + ? { ...result, runnerSessionId: options.expectedRunnerSessionId } + : result; } catch (error) { const failure = asAppError(error); if (!exchange.entered) throw discloseDispatch(failure, 'no'); diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts index 4d300d45b0..7bb2d67f69 100644 --- a/packages/platform-apple/src/runner/snapshot-presentation.ts +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -28,6 +28,7 @@ import type { SnapshotQualityVerdict, SnapshotViewportSize, IosTargetActivation, + IosSnapshotObservation, } from '@agent-device/kernel/snapshot'; import { iosSystemSurfaceHost, @@ -35,6 +36,7 @@ import { } from '@agent-device/contracts/ios-system-surface'; import { emitDiagnostic } from './host.ts'; import { TARGET_ACTIVATION_WIRE_KEY, readTargetActivationFact } from './target-activation.ts'; +import { readSnapshotObservation } from '@agent-device/contracts/capture'; export type AppleRunnerSnapshotResult = Readonly<{ nodes?: RawSnapshotNode[]; @@ -47,6 +49,7 @@ export type AppleRunnerSnapshotResult = Readonly<{ keyboard?: SnapshotKeyboardBandFact; /** Foreground repair this capture's own command had to perform (#2682). */ targetActivation?: IosTargetActivation; + observation?: IosSnapshotObservation; }>; export function readAppleSnapshotResult( @@ -54,6 +57,7 @@ export function readAppleSnapshotResult( ): AppleRunnerSnapshotResult { const systemSurface = readSystemSurfaceProvenance(result.systemSurface); const keyboard = readSnapshotKeyboardBandFact(result.keyboard); + const observation = readSnapshotObservation(result.observation); const targetActivation = readTargetActivationFact(result[TARGET_ACTIVATION_WIRE_KEY], (detail) => emitDiagnostic({ level: 'debug', @@ -70,6 +74,7 @@ export function readAppleSnapshotResult( ...(keyboard ? { keyboard } : {}), ...(systemSurface ? { systemSurface } : {}), ...(targetActivation ? { targetActivation } : {}), + ...(observation ? { observation } : {}), message: typeof result.message === 'string' && result.message.trim().length > 0 ? result.message diff --git a/packages/platform-apple/src/runtime-snapshot.ts b/packages/platform-apple/src/runtime-snapshot.ts index 2f7159fd49..06548f2542 100644 --- a/packages/platform-apple/src/runtime-snapshot.ts +++ b/packages/platform-apple/src/runtime-snapshot.ts @@ -40,6 +40,7 @@ export function bindAppleSnapshotRuntime( resolveInteractor: host.localInteractors.resolve, }); const captureSnapshot = async (input: CaptureSnapshotInput) => { + if (input.options?.observeOnly === true) return await appSnapshot.captureSnapshot(input); const helperSurface = isMacOs(request.device) ? macOsHelperSurface(input.options?.surface, request.appBackend) : undefined; diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 2d9eb9a809..bfd8156cc1 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -372,6 +372,14 @@ function summarizeProviderScenarioFlagExclusions() { owner: 'runner XCTest unit, snapshot-lines, and snapshot-quality tests', keys: ['snapshotCustomActions'], }, + { + // Observe-only capture is admitted only for a local iOS runner session; provider-owned + // sessions refuse it before capture, so no provider scenario can reach the observed path. + name: 'local iOS observe-only snapshot', + owner: + 'Apple interactor observe-only, runner lifecycle wiring, and runner XCTest dispatch tests', + keys: ['snapshotObserveOnly'], + }, { // The crop is daemon-level post-processing: the platform write happens first, then the // daemon crops the PNG against a fresh snapshot whose pixel/tree identity the fake diff --git a/src/__tests__/contracts/apple-os-resolution.test.ts b/src/__tests__/contracts/apple-os-resolution.test.ts index 24009f74d2..4d2db78544 100644 --- a/src/__tests__/contracts/apple-os-resolution.test.ts +++ b/src/__tests__/contracts/apple-os-resolution.test.ts @@ -1,7 +1,12 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { isHandheldAppleSimulator, resolveDeviceAppleOs } from '@agent-device/kernel/device'; import { + isHandheldAppleDevice, + isHandheldAppleSimulator, + resolveDeviceAppleOs, +} from '@agent-device/kernel/device'; +import { + ANDROID_EMULATOR, IOS_DEVICE, IOS_SIMULATOR, IPADOS_SIMULATOR, @@ -29,3 +34,15 @@ test('isHandheldAppleSimulator admits only an iPhone or iPad simulator leaf', () assert.equal(isHandheldAppleSimulator(IOS_DEVICE), false); assert.equal(isHandheldAppleSimulator(MACOS_DEVICE), false); }); + +test('isHandheldAppleDevice admits iPhone and iPad leaves on simulators, hardware, and legacy records', () => { + const { appleOs: _appleOs, ...legacyIosDevice } = IOS_DEVICE; + assert.equal(isHandheldAppleDevice(IOS_SIMULATOR), true); + assert.equal(isHandheldAppleDevice(IPADOS_SIMULATOR), true); + assert.equal(isHandheldAppleDevice(IOS_DEVICE), true); + assert.equal(isHandheldAppleDevice(legacyIosDevice), true); + assert.equal(isHandheldAppleDevice(TVOS_SIMULATOR), false); + assert.equal(isHandheldAppleDevice(VISIONOS_SIMULATOR), false); + assert.equal(isHandheldAppleDevice(MACOS_DEVICE), false); + assert.equal(isHandheldAppleDevice(ANDROID_EMULATOR), false); +}); diff --git a/src/backend-snapshot-options.ts b/src/backend-snapshot-options.ts index e60f192310..194413172b 100644 --- a/src/backend-snapshot-options.ts +++ b/src/backend-snapshot-options.ts @@ -15,6 +15,7 @@ const ROUTED_SNAPSHOT_OPTIONS = { depth: true, raw: true, customActions: true, + observeOnly: true, includeHiddenContentHints: true, preferredBackend: true, } as const satisfies Record; diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index de306a185e..db847b3b66 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -116,6 +116,9 @@ export const diffSnapshotCommand: RuntimeCommand< DiffSnapshotCommandOptions, DiffSnapshotCommandResult > = async (runtime, options): Promise => { + if (options.observeOnly === true) { + throw new AppError('INVALID_ARGS', 'observeOnly is supported by snapshot, not diff.'); + } const capture = await captureRuntimeSnapshot(runtime, options); const flattenForDiff = options.interactiveOnly === true; const previousSnapshot = capture.session?.snapshot; @@ -177,6 +180,7 @@ async function captureRuntimeSnapshot( scope: options.scope, raw: options.raw, customActions: options.customActions, + ...(options.observeOnly === undefined ? {} : { observeOnly: options.observeOnly }), }, ); const normalizedSnapshot = normalizeBackendSnapshot(result, runtime); diff --git a/src/commands/capture/snapshot.ts b/src/commands/capture/snapshot.ts index 093a9c2056..838b50aa3a 100644 --- a/src/commands/capture/snapshot.ts +++ b/src/commands/capture/snapshot.ts @@ -37,6 +37,7 @@ const snapshotCommandMetadata = defineFieldCommandMetadata( scope: stringField(), raw: booleanField(), customActions: optionField('snapshotCustomActions'), + observeOnly: optionField('snapshotObserveOnly'), forceFull: booleanField(), timeoutMs: integerField('Maximum wall-clock time for the snapshot command.'), // #1271 stage 2: `snapshot` is observation-only, so a repair-armed heal @@ -55,6 +56,7 @@ const snapshotCliSchema = { 'snapshotDiff', ...SNAPSHOT_FLAGS, 'snapshotCustomActions', + 'snapshotObserveOnly', 'snapshotForceFull', 'timeoutMs', 'record', @@ -74,7 +76,7 @@ export const snapshotCommandFacet = defineCommandFacet({ name: SNAPSHOT_COMMAND_NAME, text: { summary: 'Capture or diff the accessibility tree', - cliDetail: `Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. For iOS raw-coordinate fallback after a no-op ref press, inspect rects with snapshot -i --json, press the rect center, then verify with diff snapshot -i or snapshot --diff. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, + cliDetail: `--observe-only is local iOS snapshot-only: an already-ready compatible runner reads the session app without activation or foreground repair, refuses unavailable observation, and reports XCUIApplication state rather than verified foreground ownership (#2696). Regular targetActivation disclosure stays unchanged (#2682, #2694). Repeated equivalent unfiltered Android snapshots return a compact unchanged acknowledgement. Use --force-full to re-emit the tree; --json and --raw retain full output. For iOS raw-coordinate fallback after a no-op ref press, inspect rects with snapshot -i --json, press the rect center, then verify with diff snapshot -i or snapshot --diff. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`, }, metadata: snapshotCommandMetadata, run: (client, input) => client.capture.snapshot(input), diff --git a/src/commands/schema/usage.test.ts b/src/commands/schema/usage.test.ts index e0420cfebd..d4415a87ea 100644 --- a/src/commands/schema/usage.test.ts +++ b/src/commands/schema/usage.test.ts @@ -76,7 +76,7 @@ test('snapshot synopsis is generated from its allowed flags', () => { assert.equal(schema.usageOverride, undefined); assert.equal( buildCommandUsage('snapshot', schema), - 'snapshot [--diff] [-i] [--depth, -d ] [--scope, -s ] [--raw] [--actions] [--force-full] [--timeout ]', + 'snapshot [--diff] [-i] [--depth, -d ] [--scope, -s ] [--raw] [--actions] [--observe-only] [--force-full] [--timeout ]', ); }); diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 2aebbe2ccf..a46286e8d4 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -37,6 +37,7 @@ function snapshotView(data: DaemonResponseData, level: ResponseLevel): DaemonRes 'viewport', 'snapshotQuality', 'targetActivation', + 'observation', 'warnings', 'fallbackScreenshotPath', 'artifacts', diff --git a/src/daemon/snapshot-runtime-binding.ts b/src/daemon/snapshot-runtime-binding.ts index 961dd40fd8..f88d502159 100644 --- a/src/daemon/snapshot-runtime-binding.ts +++ b/src/daemon/snapshot-runtime-binding.ts @@ -37,6 +37,8 @@ import { type BoundNativeTextRead, } from './selector-operation-binding.ts'; import { errorResponse } from '@agent-device/kernel/contracts'; +import { AppError } from '@agent-device/kernel/errors'; +import { isHandheldAppleDevice } from '@agent-device/kernel/device'; export type SnapshotRuntimeRouteParams = { req: DaemonRequest; @@ -142,6 +144,15 @@ export async function resolveBoundSnapshotCaptureRuntime( req.flags, params.sessionRef, ); + if ( + req.flags?.snapshotObserveOnly === true && + (command !== 'snapshot' || !isHandheldAppleDevice(device) || !session?.appBundleId) + ) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'observe-only snapshot requires an iOS or iPadOS app session.', + ); + } const resolvedScope = resolveSnapshotScope(req.flags?.snapshotScope, session); if (!resolvedScope.ok) return { ok: false, response: resolvedScope }; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index c3b6bd1398..8561afd7ee 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -382,7 +382,7 @@ agent-device close ## Snapshot and inspect ```bash -agent-device snapshot [--diff] [-i] [--depth, -d ] [--scope, -s ] [--raw] [--actions] [--force-full] [--timeout ] +agent-device snapshot [--diff] [-i] [--depth, -d ] [--scope, -s ] [--raw] [--actions] [--observe-only] [--force-full] [--timeout ] agent-device diff snapshot [-i] [-d ] [-s ] [--raw] agent-device get text @e1 agent-device get attrs @e1 diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 766f4770f3..eb8ff1210a 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -25,6 +25,7 @@ agent-device diff snapshot # Structural diff vs previous session b | `-s ` | Scope to label, value, identifier, or `@ref` ([Scope a snapshot](#scope-a-snapshot)) | | `--raw` | Full provider tree instead of the visible-first agent view | | `--actions` | Name the custom accessibility actions merged inside an element (iOS simulator) | +| `--observe-only` | Read the session app without activating it or repairing foreground (local iOS and iPadOS sessions only) | | `--force-full` | Re-emit the full tree even when it is unchanged since the previous snapshot | | `--timeout ` | Maximum wall-clock time for the snapshot command | @@ -43,6 +44,36 @@ agent-device diff snapshot # Structural diff vs previous session b - You can't combine it with `--raw`. The CLI, the Node client, and MCP all reject the pair with `INVALID_ARGS` before touching the device. +## iOS observe-only snapshots + +`agent-device snapshot --observe-only` reads the bound iOS or iPadOS app without activating it or +repairing its foreground state. An ordinary snapshot brings a backgrounded session app back and +discloses that with `targetActivation`; an observe-only snapshot refuses instead, so the screen is +left exactly as it was. Other platforms, provider-owned sessions, and `--diff` are unsupported. + +The runner refuses with `OBSERVATION_UNAVAILABLE` (also named by `details.runnerErrorCode`) unless +the app reports `runningForeground` before and after the capture under one process identity. It does +not substitute SpringBoard or a presented system surface. The host refuses with `COMMAND_FAILED` and +`details.reason: "observation-unavailable"` when a runner answers an observe-only request without +`observation` or with a `targetActivation` fact. Successful JSON includes: + +```json +{ + "observation": { + "mode": "observe-only", + "activationPerformed": false, + "appState": "runningForeground", + "appStateSource": "xcuiapplication-state" + } +} +``` + +This reports **XCUIApplication state, not verified foreground ownership**: XCTest can report that +state incorrectly when another app owns the screen ([#2696](https://github.com/callstack/agent-device/issues/2696)). +Observe-only results never carry `targetActivation`. Regular snapshots retain their existing repair +and disclosure behavior ([#2682](https://github.com/callstack/agent-device/issues/2682), +[#2694](https://github.com/callstack/agent-device/issues/2694)). + ## Keep snapshots small and current - iOS and Android produce the same kind of snapshot: visible elements first, refs you can act on From 10bc4f7e48286ffce64982a7ed9e27dbca12c2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 8 Oct 2026 15:22:48 +0200 Subject: [PATCH 2/2] refactor(ios): drop the observe-only capability probe and runner pinning An observe-only snapshot now sends the `observeOnly` field on the snapshot request alone. The runner enforces the no-activation contract and returns `observation`; the host asserts that shape on the response. The separate `uptime` capability probe, `supportsObserveOnlySnapshot`, the ready-runner-only rule, runner session pinning and the readiness-preflight exemption existed to survive a CLI and runner that disagree about the field, which the repo ships together and already guards through the protocol fixture. Co-Authored-By: Claude Fable 5.1 --- .../RunnerTests+CommandDispatch.swift | 9 +-- .../RunnerTests+Models.swift | 1 - .../RunnerTests+CommandDispatchTests.swift | 8 --- contracts/fixtures/runner-requests.json | 1 - .../interactor-target-activation.test.ts | 71 ++----------------- .../src/__tests__/runner-requests.test.ts | 15 ++-- packages/platform-apple/src/interactor.ts | 68 +++++------------- .../__tests__/runner-command-traits.test.ts | 15 +--- .../runner/__tests__/runner-exchange.test.ts | 21 ------ .../__tests__/runner-recovery-wiring.test.ts | 34 --------- .../src/runner/runner-command-traits.ts | 12 +--- .../src/runner/runner-lifecycle.ts | 22 +----- scripts/integration-progress-model.ts | 3 +- 13 files changed, 34 insertions(+), 246 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift index ab27de9ef2..c2aa672ebf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift @@ -42,14 +42,9 @@ extension RunnerTests { func executeUptime() -> Response { // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a // fresher send-time stamp on every ok response; kept so direct callers still get a value. -#if os(iOS) - let supportsObserveOnlySnapshot = true -#else - let supportsObserveOnlySnapshot = false -#endif - return Response( + Response( ok: true, - data: DataPayload(currentUptimeMs: currentUptimeMs(), supportsObserveOnlySnapshot: supportsObserveOnlySnapshot) + data: DataPayload(currentUptimeMs: currentUptimeMs()) ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 1e36d85447..604bb2b255 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -448,7 +448,6 @@ struct DataPayload: Codable { var referenceWidth: Double? var referenceHeight: Double? var currentUptimeMs: Double? - var supportsObserveOnlySnapshot: Bool? var applicationState: String? var commandId: String? var lifecycleState: String? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift index bd3e94a526..20f81f8dd3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CommandDispatchTests.swift @@ -3,14 +3,6 @@ import AgentDeviceSnapshotPresentation #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { - func testUptimeAdvertisesObserveOnlyOnIosOnly() { -#if os(iOS) - XCTAssertEqual(executeUptime().data?.supportsObserveOnlySnapshot, true) -#else - XCTAssertEqual(executeUptime().data?.supportsObserveOnlySnapshot, false) -#endif - } - func testInjectedTapRecordedFailureGateIsTapOnlyAndCountGated() { // The seam's recording side cannot run in-bundle (a real XCTIssue would // fail this very test run — same constraint the record(_:) suppression diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index b875630453..c5586e3c26 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -44,7 +44,6 @@ {"name": "ios-simulator.interactor-set-orientation.rotate", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "rotate", "orientation": "landscape-left", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-snapshot.every-option", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "preferredBackend": "tree", "customActions": true, "depth": 3, "scope": "Go", "raw": true, "commandId": ""}}, {"name": "ios-simulator.interactor-snapshot.observe-only", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "observeOnly": true, "commandId": ""}}, - {"name": "ios-simulator.interactor-snapshot.observe-only.capability", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "uptime", "observeOnly": true, "commandId": ""}}, {"name": "ios-simulator.interactor-write-clipboard.text", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "pasteboardWrite", "text": "one-time code 246810", "commandId": ""}}, {"name": "ios-simulator.recording-clock-anchor.snapshot", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "snapshot", "appBundleId": "com.example.app", "interactiveOnly": true, "depth": 1}}, {"name": "ios-simulator.recording-start.default", "producer": "src/__tests__/screen-recording-runner-requests.test.ts", "request": {"command": "recordStart", "outPath": "capture.mp4", "appBundleId": "com.example.app"}}, diff --git a/packages/platform-apple/src/__tests__/interactor-target-activation.test.ts b/packages/platform-apple/src/__tests__/interactor-target-activation.test.ts index 8352992be0..f6cd4ba331 100644 --- a/packages/platform-apple/src/__tests__/interactor-target-activation.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-target-activation.test.ts @@ -29,11 +29,7 @@ const HEALTHY_TREE = { function interactorServing(payload: Record) { const runnerProvider: AppleRunnerProvider = { hasLiveSession: () => true, - runCommand: async () => ({ - ...payload, - supportsObserveOnlySnapshot: true, - runnerSessionId: 'observation-runner', - }), + runCommand: async () => payload, }; return createAppleInteractor(IOS_SIMULATOR, { appBundleId: 'com.example.app' }, runnerProvider); } @@ -70,77 +66,24 @@ test.each([ const commands: RunnerCommand[] = []; const runnerProvider: AppleRunnerProvider = { hasLiveSession: () => true, - runCommand: async (_device, command, options) => { + runCommand: async (_device, command) => { commands.push(command); - if (command.command === 'snapshot') { - assert.equal(options?.expectedRunnerSessionId, 'observation-runner'); - } - return { - ...HEALTHY_TREE, - observation, - supportsObserveOnlySnapshot: true, - runnerSessionId: 'observation-runner', - }; + return { ...HEALTHY_TREE, observation }; }, }; const snapshot = (await createAppleInteractor(device, {}, runnerProvider).snapshot({ appBundleId: 'com.example.app', observeOnly: true, })) as SnapshotResult; - const runnerCommand = (name: string) => commands.find((command) => command.command === name); - assert.equal(runnerCommand('uptime')?.observeOnly, true); - assert.equal(runnerCommand('snapshot')?.observeOnly, true); + assert.deepEqual( + commands.map((command) => [command.command, command.observeOnly]), + [['snapshot', true]], + ); assert.deepEqual(snapshot.observation, observation); assert.equal('targetActivation' in snapshot, false); }, ); -test('observe-only refuses a tvOS session before any runner command', async () => { - const commands: string[] = []; - const runnerProvider: AppleRunnerProvider = { - hasLiveSession: () => true, - runCommand: async (_device, command) => { - commands.push(command.command); - return {}; - }, - }; - await assert.rejects( - createAppleInteractor( - { ...IOS_SIMULATOR, appleOs: 'tvos', target: 'tv' }, - {}, - runnerProvider, - ).snapshot({ appBundleId: 'com.example.app', observeOnly: true }), - (error: unknown) => error instanceof AppError && error.code === 'UNSUPPORTED_OPERATION', - ); - assert.deepEqual(commands, []); -}); - -test.each([ - { supportsObserveOnlySnapshot: false, runnerSessionId: 'observation-runner' }, - { supportsObserveOnlySnapshot: true }, -])( - 'observe-only rejects an unbound capability response before snapshot dispatch: %j', - async (capabilities) => { - const commands: string[] = []; - const runnerProvider: AppleRunnerProvider = { - hasLiveSession: () => true, - runCommand: async (_device, command) => { - commands.push(command.command); - return capabilities; - }, - }; - await assert.rejects( - createAppleInteractor(IOS_SIMULATOR, {}, runnerProvider).snapshot({ - appBundleId: 'com.example.app', - observeOnly: true, - }), - (error: unknown) => - error instanceof AppError && error.details?.reason === 'observation-unavailable', - ); - assert.deepEqual(commands, ['uptime']); - }, -); - test('observe-only refuses absent provenance rather than crediting a legacy activating runner', async () => { await assert.rejects( interactorServing(HEALTHY_TREE).snapshot({ appBundleId: 'com.example.app', observeOnly: true }), diff --git a/packages/platform-apple/src/__tests__/runner-requests.test.ts b/packages/platform-apple/src/__tests__/runner-requests.test.ts index 319d05a0da..c90ce9f2ce 100644 --- a/packages/platform-apple/src/__tests__/runner-requests.test.ts +++ b/packages/platform-apple/src/__tests__/runner-requests.test.ts @@ -1,4 +1,3 @@ -import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; @@ -230,7 +229,6 @@ test('every Apple request site builds exactly its golden runner request', async IOS_SIMULATOR, { appBundleId: APP }, recordingRunnerProvider(observationCalls, { - uptime: { supportsObserveOnlySnapshot: true, runnerSessionId: 'observe-only-runner' }, snapshot: { nodes: [{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }], observation: { @@ -242,15 +240,10 @@ test('every Apple request site builds exactly its golden runner request', async }, }), ).snapshot({ appBundleId: APP, observeOnly: true }); - assert.deepEqual( - observationCalls.map((call) => call.command.command), - ['uptime', 'snapshot'], - ); - assert.equal(observationCalls[1]!.options.expectedRunnerSessionId, 'observe-only-runner'); - captured.push( - ['ios-simulator.interactor-snapshot.observe-only.capability', observationCalls[0]!.command], - ['ios-simulator.interactor-snapshot.observe-only', observationCalls[1]!.command], - ); + captured.push([ + 'ios-simulator.interactor-snapshot.observe-only', + onlyRequest('ios-simulator.interactor-snapshot.observe-only', observationCalls), + ]); const dir = await mkdtempForTest('agent-device-runner-requests-'); const runnerScreenshot = path.join(dir, 'runner.png'); fs.writeFileSync(runnerScreenshot, ''); diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index daec761e34..3ca2da7d96 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -7,11 +7,11 @@ import { appleRemotePressCommand } from './os/tvos/remote.ts'; import { runMacOsScreenshotAction } from './os/macos/helper.ts'; import { actOnAppleAlert, awaitAppleAlert, readAppleAlert } from './alert.ts'; import { runAppleRunnerCommand } from './core/runner-client.ts'; -import type { AppleRunnerCommandOptions } from './runner/runner-provider.ts'; import { withAppleRunnerProvider, type AppleRunnerCommandExecutor, type AppleRunnerProvider, + type RunnerCommand, } from './runner/index.ts'; import { toAppleTvRemoteButton } from '@agent-device/contracts/tv-remote'; import { SCREENSHOT_FULLSCREEN_REASONS } from '@agent-device/contracts/capture'; @@ -23,12 +23,7 @@ import { import { DEVICE_ROTATIONS, type DeviceRotation } from '@agent-device/contracts/device'; import { normalizeSnapshotScope } from '@agent-device/contracts/snapshot'; import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; -import { - isHandheldAppleDevice, - isMacOs, - isTvOsDevice, - type DeviceInfo, -} from '@agent-device/kernel/device'; +import { isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { withMethodScope } from '@agent-device/kernel/scoped-provider'; import type { Point, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; @@ -233,12 +228,6 @@ async function captureAppleSnapshot( runnerOpts: RunnerCallOptions, helper: MacOsHelperSurface | undefined, ) { - if (options?.observeOnly === true && !isHandheldAppleDevice(device)) { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'observe-only snapshot is supported on iOS and iPadOS only.', - ); - } if (helper) { return await captureMacOsSurfaceSnapshot({ ...options, surface: helper }, options?.signal); } @@ -250,25 +239,14 @@ async function captureAppleRunnerSnapshot( options: SnapshotOptions | undefined, runnerOpts: RunnerCallOptions, ) { - const observationPolicy = await snapshotObservationPolicy(device, options, runnerOpts); const result = readAppleSnapshotResult( await withDiagnosticTimer( 'snapshot_capture', async () => await runAppleRunnerCommand( device, - { - command: 'snapshot', - appBundleId: options?.appBundleId, - interactiveOnly: options?.interactiveOnly, - preferredBackend: options?.preferredBackend, - customActions: options?.customActions, - ...observationPolicy.request, - depth: options?.depth, - scope: options?.scope, - raw: options?.raw, - }, - mergeRunnerCallSignal(observationPolicy.runnerOpts, options?.signal), + runnerSnapshotCommand(options), + mergeRunnerCallSignal(runnerOpts, options?.signal), ), { backend: 'xctest' }, ), @@ -292,33 +270,19 @@ async function captureAppleRunnerSnapshot( }; } -async function snapshotObservationPolicy( - device: DeviceInfo, - options: SnapshotOptions | undefined, - runnerOpts: RunnerCallOptions, -): Promise<{ - request: { observeOnly?: true }; - runnerOpts: RunnerCallOptions & Pick; -}> { - if (options?.observeOnly !== true) return { request: {}, runnerOpts }; - const capabilities = await runAppleRunnerCommand( - device, - { command: 'uptime', observeOnly: true }, - mergeRunnerCallSignal(runnerOpts, options.signal), - ); - if ( - capabilities.supportsObserveOnlySnapshot !== true || - typeof capabilities.runnerSessionId !== 'string' || - capabilities.runnerSessionId.length === 0 - ) { - throw new AppError('COMMAND_FAILED', 'Runner does not support observe-only snapshots.', { - reason: 'observation-unavailable', - dispatched: 'no', - }); - } +function runnerSnapshotCommand(options: SnapshotOptions | undefined): RunnerCommand { + const { appBundleId, interactiveOnly, preferredBackend, customActions, depth, scope, raw } = + options ?? {}; return { - request: { observeOnly: true }, - runnerOpts: { ...runnerOpts, expectedRunnerSessionId: capabilities.runnerSessionId }, + command: 'snapshot', + appBundleId, + interactiveOnly, + preferredBackend, + customActions, + ...(options?.observeOnly === true ? { observeOnly: true } : {}), + depth, + scope, + raw, }; } diff --git a/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts b/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts index fb8734b7cd..60974e424b 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-command-traits.test.ts @@ -39,8 +39,9 @@ test('runner command trait table pins lifecycle-sensitive command groups', () => 'querySelector', 'readText', 'screenshot', + 'snapshot', ], - payloadDependent: ['alert', 'snapshot'], + payloadDependent: ['alert'], readOnlyReadinessProbe: ['status', 'uptime'], readinessPreflightExemptMutation: ['activate', 'targetReset', 'terminate'], default: [ @@ -64,18 +65,6 @@ test('runner command trait table pins lifecycle-sensitive command groups', () => assert.deepEqual(Object.values(groups).flat().sort(), [...RUNNER_COMMANDS].sort()); }); -test('only an observe-only snapshot is exempt from the readiness preflight', () => { - assert.deepEqual(readRunnerCommandTraits({ command: 'snapshot' }), readOnly()); - assert.deepEqual( - readRunnerCommandTraits({ command: 'snapshot', observeOnly: false }), - readOnly(), - ); - assert.deepEqual(readRunnerCommandTraits({ command: 'snapshot', observeOnly: true }), { - ...readOnly(), - readinessPreflightExempt: true, - }); -}); - test('alert actions match the native read-only golden table', () => { // The fixture's `query` column records whether the alert request changes anything — `get` is the // one action that is side-effect-free — and each side consumes it under its own name: `readOnly` diff --git a/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts b/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts index 8fb8aa86f2..e6bf4b125d 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-exchange.test.ts @@ -57,27 +57,6 @@ test('the exchange preflights a mutation and settles only its answer', async () assert.deepEqual(invalidations, []); }); -test('the exchange sends an observe-only snapshot without a readiness preflight', async () => { - server = await startFakeRunnerServer({ - snapshot: [{ kind: 'ok', data: { nodes: [] } }], - }); - const session = sessionFor(server.port); - - await executeRunnerExchange( - IOS_SIMULATOR, - session, - { command: 'snapshot', observeOnly: true, appBundleId: 'com.example.app' }, - undefined, - 10_000, - async () => {}, - ); - - assert.deepEqual( - server.requests.map(({ command }) => command), - ['snapshot'], - ); -}); - test('the readiness probe preserves its main-thread busy report when the command omits it', async () => { server = await startFakeRunnerServer({ uptime: [{ kind: 'ok', data: { runnerMainThreadBusy: true } }], diff --git a/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts b/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts index 1e883c564c..4d8e4106fe 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-recovery-wiring.test.ts @@ -597,40 +597,6 @@ test('an exact-session command never dispatches to a replacement runner', async assert.deepEqual(server.requests, []); }); -test('observe-only refuses an unavailable runner without creating a session', async () => { - await expect( - runAppleRunnerCommand(IOS_SIMULATOR, { command: 'snapshot', observeOnly: true }), - ).rejects.toMatchObject({ - code: 'COMMAND_FAILED', - details: { reason: 'observation-unavailable', dispatched: 'no' }, - }); - expect(ensureRunnerSessionMock).not.toHaveBeenCalled(); - expect(invalidateRunnerSessionMock).not.toHaveBeenCalled(); -}); - -test('observe-only keeps the capability-probed identity when a replacement runner is ready', async () => { - server = await startFakeRunnerServer({ snapshot: [{ kind: 'ok', data: {} }] }); - const replacement = seedSession(server.port); - const sessionModule = await import('../runner-session.ts'); - const liveness = vi.spyOn(sessionModule, 'readRunnerSessionLiveness').mockReturnValue({ - sessionId: replacement.sessionId, - liveness: 'ready', - }); - try { - await expect( - runAppleRunnerCommand( - IOS_SIMULATOR, - { command: 'snapshot', observeOnly: true }, - { expectedRunnerSessionId: `${replacement.sessionId}:capability-probed` }, - ), - ).rejects.toThrow('runner session ownership changed'); - assert.deepEqual(server.requests, []); - expect(invalidateRunnerSessionMock).not.toHaveBeenCalled(); - } finally { - liveness.mockRestore(); - } -}); - test.each( (['accept', 'dismiss'] as const).flatMap((action) => (['accepted', 'started', 'completed'] as const).map((lifecycleState) => ({ diff --git a/packages/platform-apple/src/runner/runner-command-traits.ts b/packages/platform-apple/src/runner/runner-command-traits.ts index 32a2774db1..17d5582cb7 100644 --- a/packages/platform-apple/src/runner/runner-command-traits.ts +++ b/packages/platform-apple/src/runner/runner-command-traits.ts @@ -24,11 +24,6 @@ const READ_ONLY_TRAITS: RunnerCommandTraits = { readOnly: true, }; -const READ_ONLY_READINESS_PREFLIGHT_EXEMPT_TRAITS: RunnerCommandTraits = { - ...READ_ONLY_TRAITS, - readinessPreflightExempt: true, -}; - const READ_ONLY_READINESS_PROBE_TRAITS: RunnerCommandTraits = { ...READ_ONLY_TRAITS, readinessProbe: true, @@ -52,11 +47,6 @@ type RunnerCommandTraitsEntry = const readAlertActionTraits = (command: RunnerCommand): RunnerCommandTraits => (command.action ?? 'get').toLowerCase() === 'get' ? READ_ONLY_TRAITS : DEFAULT_TRAITS; -// An observe-only snapshot is pinned to the session its own observe-only uptime probe just proved -// ready, so a second readiness probe would add a request without adding evidence. -const readSnapshotTraits = (command: RunnerCommand): RunnerCommandTraits => - command.observeOnly === true ? READ_ONLY_READINESS_PREFLIGHT_EXEMPT_TRAITS : READ_ONLY_TRAITS; - /** * Traits of every runner command the daemon can send. A command whose traits depend on its * payload maps to a function of the command instead of a fixed trait set. @@ -74,7 +64,7 @@ export const RUNNER_COMMAND_TRAITS = { findText: READ_ONLY_TRAITS, querySelector: READ_ONLY_TRAITS, readText: READ_ONLY_TRAITS, - snapshot: readSnapshotTraits, + snapshot: READ_ONLY_TRAITS, screenshot: READ_ONLY_TRAITS, backInApp: DEFAULT_TRAITS, backSystem: DEFAULT_TRAITS, diff --git a/packages/platform-apple/src/runner/runner-lifecycle.ts b/packages/platform-apple/src/runner/runner-lifecycle.ts index 46fd54ba6c..76c5218e4d 100644 --- a/packages/platform-apple/src/runner/runner-lifecycle.ts +++ b/packages/platform-apple/src/runner/runner-lifecycle.ts @@ -286,29 +286,9 @@ export async function executeRunnerCommand( command: RunnerCommand, options: AppleRunnerCommandOptions, ): Promise> { - if (command.observeOnly === true) { - const registration = readRunnerSessionLiveness(device.id); - if (registration?.liveness !== 'ready') { - throw new AppError( - 'COMMAND_FAILED', - 'observe-only snapshot requires an already-ready runner.', - { - reason: 'observation-unavailable', - dispatched: 'no', - }, - ); - } - options = { - ...options, - expectedRunnerSessionId: options.expectedRunnerSessionId ?? registration.sessionId, - }; - } const exchange = { entered: false }; try { - const result = await executeRunnerCommandAttempt(device, command, options, exchange); - return command.observeOnly === true && command.command === 'uptime' - ? { ...result, runnerSessionId: options.expectedRunnerSessionId } - : result; + return await executeRunnerCommandAttempt(device, command, options, exchange); } catch (error) { const failure = asAppError(error); if (!exchange.entered) throw discloseDispatch(failure, 'no'); diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index bfd8156cc1..bc7575bdb3 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -376,8 +376,7 @@ function summarizeProviderScenarioFlagExclusions() { // Observe-only capture is admitted only for a local iOS runner session; provider-owned // sessions refuse it before capture, so no provider scenario can reach the observed path. name: 'local iOS observe-only snapshot', - owner: - 'Apple interactor observe-only, runner lifecycle wiring, and runner XCTest dispatch tests', + owner: 'Apple interactor observe-only and runner XCTest dispatch tests', keys: ['snapshotObserveOnly'], }, {