From 86a9c656bdddc71b3ac5e2e4d3d092c10470491e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 21:42:32 +0200 Subject: [PATCH] feat(macos-helper): drive app sessions through accessibility actions Add an `app` surface to the macOS helper so an app session can be served without XCTest Automation Mode while the app stays behind other windows: - snapshot targets the session app by bundle id and turns on the Chromium/Electron accessibility tree before traversal - press, fill, type, and scroll act through AXPress, focus, value, selected text, and scroll bar values; pointer actions with no accessibility equivalent are refused with a typed reason instead of posting process events an inactive app drops - screenshot captures the app's front window alone via ScreenCaptureKit Co-Authored-By: Claude Opus 5.5 (1M context) --- .../AppWindowScreenshot.swift | 73 +++ .../BackgroundInteraction.swift | 496 ++++++++++++++++++ .../SnapshotTraversal.swift | 110 +++- .../Sources/AgentDeviceMacOSHelper/main.swift | 141 ++++- .../BackgroundInteractionTests.swift | 88 ++++ .../macos-native-helper-outcomes.json | 18 + 6 files changed, 901 insertions(+), 25 deletions(-) create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift create mode 100644 apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift create mode 100644 contracts/fixtures/macos-native-helper-outcomes.json diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift new file mode 100644 index 0000000000..da4c6247c7 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -0,0 +1,73 @@ +import AppKit +import CoreGraphics +import Foundation +import ScreenCaptureKit + +/// Captures the session app's front window by itself: windows above it, the ghost cursor, and +/// the rest of the desktop are not in the image, so the app may sit behind the user's work. +func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) throws { + guard #available(macOS 14.0, *) else { + throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") + } + let pid = app.processIdentifier + let frontWindowID = frontWindowNumber(pid: pid) + let content = try awaitShareableContent() + let candidates = content.windows.filter { + $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 + && $0.frame.height > 0 + } + guard + let window = candidates.first(where: { Int($0.windowID) == frontWindowID }) + ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }) + else { + throw HelperError.commandFailed( + "screenshot could not find a window for the app", + details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] + ) + } + + let filter = SCContentFilter(desktopIndependentWindow: window) + let configuration = SCStreamConfiguration() + let scale = CGFloat(filter.pointPixelScale) + configuration.width = Int(filter.contentRect.width * scale) + configuration.height = Int(filter.contentRect.height * scale) + configuration.showsCursor = false + configuration.ignoreShadowsSingleWindow = true + + let semaphore = DispatchSemaphore(value: 0) + var capturedImage: CGImage? + var capturedError: Error? + SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration) { image, error in + capturedImage = image + capturedError = error + semaphore.signal() + } + semaphore.wait() + if let error = capturedError as NSError? { + throw screenshotFailure(error, surface: "app") + } + guard let capturedImage else { + throw HelperError.commandFailed("screenshot failed") + } + try writePNG(capturedImage, to: outPath) +} + +@available(macOS 14.0, *) +private func awaitShareableContent() throws -> SCShareableContent { + let semaphore = DispatchSemaphore(value: 0) + var shareable: SCShareableContent? + var failure: Error? + SCShareableContent.getExcludingDesktopWindows(true, onScreenWindowsOnly: true) { content, error in + shareable = content + failure = error + semaphore.signal() + } + semaphore.wait() + if let error = failure as NSError? { + throw screenshotFailure(error, surface: "app") + } + guard let shareable else { + throw HelperError.commandFailed("screenshot could not list shareable windows") + } + return shareable +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift new file mode 100644 index 0000000000..30e9e69573 --- /dev/null +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -0,0 +1,496 @@ +import AgentDeviceMacOSInput +import AppKit +import ApplicationServices +import CoreGraphics +import Foundation + +/// How an app-surface action reached the app. Every pointer action is an accessibility action on +/// the element under the point: an inactive app drops pointer events posted to its process, so a +/// pointer action with no accessibility equivalent is refused rather than reported as delivered. +enum BackgroundDeliveryMechanism: String, Encodable, CaseIterable { + case axPress = "ax-press" + case axFocus = "ax-focus" + case axValue = "ax-value" + case axSelectedText = "ax-selected-text" + case axConfirm = "ax-confirm" + case axScrollBar = "ax-scroll-bar" + /// Keyboard events posted to the app's process, which inactive apps do accept. + case keyEvents = "key-events" +} + +/// Typed refusals the host maps to `UNSUPPORTED_OPERATION`. Both vocabularies are pinned by +/// `contracts/fixtures/macos-native-helper-outcomes.json`. +enum BackgroundRefusal: String, CaseIterable { + case pointerGesture = "background-pointer-gesture" + case noAccessibleTarget = "no-accessible-target" + case noTextInput = "no-settable-text-input" + case noScrollBar = "no-scroll-bar" +} + +struct BackgroundPressResponse: Encodable { + let x: Double + let y: Double + let clicks: Int + let bundleId: String? + let surface: String + let mechanism: BackgroundDeliveryMechanism + let role: String? + let windowTitle: String? +} + +struct BackgroundTextResponse: Encodable { + let bundleId: String? + let mechanism: BackgroundDeliveryMechanism + let role: String? + let windowTitle: String? +} + +struct BackgroundScrollResponse: Encodable { + let x: Double + let y: Double + let x2: Double + let y2: Double + let referenceWidth: Double + let referenceHeight: Double + let travelPixels: Double + let mechanism: BackgroundDeliveryMechanism +} + +/// Roles whose ancestors are containers, never the control a click meant. +private let pressSearchBoundaryRoles: Set = [ + "AXWindow", "AXApplication", "AXScrollArea", "AXTable", "AXOutline", "AXList", "AXWebArea", + "AXSheet", "AXDialog", +] +private let textInputRoles: Set = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField"] +/// Roles a click means to activate. Chromium marks most wrapper groups pressable too, so a group +/// is only the target when nothing more specific contains the point. +private let pressableControlRoles: Set = [ + "AXButton", "AXLink", "AXCheckBox", "AXRadioButton", "AXPopUpButton", "AXMenuButton", + "AXMenuItem", "AXMenuBarItem", "AXCell", "AXRow", "AXTab", "AXDisclosureTriangle", + "AXIncrementor", "AXSlider", "AXColorWell", "AXDockItem", +] +private let pressSearchMaxAncestors = 4 + +func refusal(_ reason: BackgroundRefusal, _ message: String, app: NSRunningApplication) -> HelperError { + .commandFailed(message, details: ["reason": reason.rawValue, "bundleId": app.bundleIdentifier ?? ""]) +} + +func requireSessionApplication(bundleId: String?) throws -> NSRunningApplication { + guard let bundleId, !bundleId.isEmpty else { + throw HelperError.invalidArgs("the app surface requires --bundle-id ") + } + return try resolveTargetApplication(bundleId: bundleId, surface: "app") +} + +func pressInBackground( + _ request: MouseClickRequest, + app: NSRunningApplication +) throws -> BackgroundPressResponse { + guard !request.doubleClick, request.holdMs <= 0 else { + throw refusal( + .pointerGesture, + "double-click and press-and-hold have no accessibility equivalent in a background app", + app: app + ) + } + let point = CGPoint(x: request.x, y: request.y) + guard let target = resolvePressTarget(app: app, point: point), + let mechanism = perform(target) + else { + throw refusal(.noAccessibleTarget, "no pressable accessibility element at the point", app: app) + } + var clicks = 1 + while clicks < request.clicks { + Thread.sleep(forTimeInterval: Double(max(request.intervalMs, 0)) / 1000) + guard perform(target) != nil else { break } + clicks += 1 + } + return BackgroundPressResponse( + x: request.x, + y: request.y, + clicks: clicks, + bundleId: app.bundleIdentifier, + surface: "app", + mechanism: mechanism, + role: role(of: target.element), + windowTitle: windowTitle(of: target.element) + ) +} + +/// Inserts text at the focused element's caret, the way typing does. +func typeInBackground( + text: String, + delayMs: Int, + app: NSRunningApplication +) throws -> BackgroundTextResponse { + let appElement = AXUIElementCreateApplication(app.processIdentifier) + let focused = elementAttribute(appElement, attribute: kAXFocusedUIElementAttribute as String) + let focusedRole = focused.map(role(of:)) + if text == "\n", let focused, actionNames(of: focused).contains(kAXConfirmAction as String), + AXUIElementPerformAction(focused, kAXConfirmAction as CFString) == .success + { + return BackgroundTextResponse( + bundleId: app.bundleIdentifier, mechanism: .axConfirm, role: focusedRole, + windowTitle: windowTitle(of: focused)) + } + if !text.contains("\n"), let focused, + isAttributeSettable(focused, attribute: kAXSelectedTextAttribute as String), + AXUIElementSetAttributeValue(focused, kAXSelectedTextAttribute as CFString, text as CFString) + == .success + { + return BackgroundTextResponse( + bundleId: app.bundleIdentifier, mechanism: .axSelectedText, role: focusedRole, + windowTitle: windowTitle(of: focused)) + } + try postText(text, delayMs: delayMs, pid: app.processIdentifier) + return BackgroundTextResponse( + bundleId: app.bundleIdentifier, mechanism: .keyEvents, role: focusedRole, + windowTitle: focused.flatMap(windowTitle(of:))) +} + +/// Replaces the value of the text input at a point. +func fillInBackground( + point: CGPoint, + text: String, + app: NSRunningApplication +) throws -> BackgroundTextResponse { + let hit = elementAtPoint(in: app, point: point) + let chain = hit.map(pressSearchChain) ?? [] + let fallback = actionWindow(app: app, hit: hit).flatMap { window in + smallestElement(in: window, containing: point, where: isTextInput) + } + guard let input = chain.first(where: isTextInput) ?? fallback, + isAttributeSettable(input, attribute: kAXValueAttribute as String) + else { + throw refusal(.noTextInput, "no settable text input at the point", app: app) + } + AXUIElementSetAttributeValue(input, kAXFocusedAttribute as CFString, kCFBooleanTrue) + guard AXUIElementSetAttributeValue(input, kAXValueAttribute as CFString, text as CFString) == .success + else { + throw refusal(.noTextInput, "the text input refused the new value", app: app) + } + return BackgroundTextResponse( + bundleId: app.bundleIdentifier, mechanism: .axValue, role: role(of: input), + windowTitle: windowTitle(of: input)) +} + +/// Scrolls the scroll area at the center of the app's front window. +func scrollInBackground( + direction: String, + amount: Double?, + pixels: Double?, + app: NSRunningApplication +) throws -> BackgroundScrollResponse { + let isVertical = direction == "up" || direction == "down" + guard isVertical || direction == "left" || direction == "right" else { + throw HelperError.invalidArgs("scroll requires --direction ") + } + guard let frame = frontWindowFrame(pid: app.processIdentifier) else { + throw HelperError.commandFailed( + "scroll could not resolve an on-screen window", + details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] + ) + } + let travel = scrollTravelPixels( + axisLength: Double(isVertical ? frame.height : frame.width), + amount: amount, + pixels: pixels + ) + let center = CGPoint(x: frame.midX, y: frame.midY) + let revealsLaterContent = direction == "down" || direction == "right" + guard + performScrollBarScroll( + app: app, + at: center, + isVertical: isVertical, + signedTravel: revealsLaterContent ? travel : -travel + ) + else { + throw refusal(.noScrollBar, "no scroll area with a settable scroll bar under the window center", app: app) + } + // Reported as the equivalent drag, start to end, the way the runner reports a desktop scroll. + let half = travel / 2 + let sign: Double = revealsLaterContent ? 1 : -1 + return BackgroundScrollResponse( + x: isVertical ? center.x : center.x + sign * half, + y: isVertical ? center.y + sign * half : center.y, + x2: isVertical ? center.x : center.x - sign * half, + y2: isVertical ? center.y - sign * half : center.y, + referenceWidth: Double(frame.width), + referenceHeight: Double(frame.height), + travelPixels: travel, + mechanism: .axScrollBar + ) +} + +/// Mirrors `runnerScrollGesturePlan`'s travel so both macOS backends scroll the same distance: +/// the requested travel, kept clear of the outer tenth of the axis on both sides. +func scrollTravelPixels(axisLength: Double, amount: Double?, pixels: Double?) -> Double { + let requested = pixels.map { max(1, $0.rounded()) } ?? (axisLength * (amount ?? 0.6)).rounded() + let edgePadding = max(1, (axisLength * 0.1).rounded()) + return max(1, min(requested, axisLength - edgePadding * 2)) +} + +/// The scroll bar value that moves the content by `signedTravel` points, clamped to its ends. +func scrollBarValue(current: Double, signedTravel: Double, overflow: Double) -> Double { + min(1, max(0, current + signedTravel / overflow)) +} + +func isPressableControlRole(_ role: String) -> Bool { + pressableControlRoles.contains(role) +} + +private enum PressTarget { + case focus(AXUIElement) + case press(AXUIElement) + + var element: AXUIElement { + switch self { + case .focus(let element), .press(let element): element + } + } +} + +/// What a click at a point means. The app's own hit test answers first; web content often answers +/// with a wrapper group, so the snapshot geometry decides when the hit names no control. +private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) -> PressTarget? { + let hit = elementAtPoint(in: app, point: point) + let chain = hit.map(pressSearchChain) ?? [] + if let input = chain.first(where: isTextInput) { return .focus(input) } + if let control = chain.first(where: isPressableControl) { return .press(control) } + if let found = actionWindow(app: app, hit: hit).flatMap({ + smallestElement(in: $0, containing: point, where: { isTextInput($0) || isPressableControl($0) }) + }) { + return isTextInput(found) ? .focus(found) : .press(found) + } + return chain.first(where: { actionNames(of: $0).contains(kAXPressAction as String) }).map { + .press($0) + } +} + +private func perform(_ target: PressTarget) -> BackgroundDeliveryMechanism? { + switch target { + case .focus(let element): + return AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue) + == .success ? .axFocus : nil + case .press(let element): + return AXUIElementPerformAction(element, kAXPressAction as CFString) == .success ? .axPress : nil + } +} + +/// The app's own hit test, scoped to the app so windows of other apps above it do not answer. +private func elementAtPoint(in app: NSRunningApplication, point: CGPoint) -> AXUIElement? { + let appElement = AXUIElementCreateApplication(app.processIdentifier) + var hit: AXUIElement? + guard AXUIElementCopyElementAtPosition(appElement, Float(point.x), Float(point.y), &hit) == .success + else { + return nil + } + return hit +} + +private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] { + var chain: [AXUIElement] = [] + var current: AXUIElement? = hit + while let element = current, chain.count <= pressSearchMaxAncestors { + if pressSearchBoundaryRoles.contains(role(of: element)) { break } + chain.append(element) + current = elementAttribute(element, attribute: kAXParentAttribute as String) + } + return chain +} + +/// The one window a point action may act in: the window the app's hit test landed in, else the +/// app's front on-screen window. A fallback target never comes from a window behind it, and the +/// response names the window so the host can check it. +private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? { + if let hit { + if role(of: hit) == "AXWindow" { return hit } + if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window } + } + guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil } + return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in + guard let rect = rectAttribute(window) else { return false } + return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1 + && abs(rect.width - front.width) < 1 && abs(rect.height - front.height) < 1 + } +} + +private func windowTitle(of element: AXUIElement) -> String? { + let window = role(of: element) == "AXWindow" + ? element : elementAttribute(element, attribute: kAXWindowAttribute as String) + return window.flatMap { stringAttribute($0, attribute: kAXTitleAttribute as String) } +} + +/// The smallest matching element in a window whose frame contains the point, skipping subtrees +/// whose frame excludes the point. +private func smallestElement( + in window: AXUIElement, + containing point: CGPoint, + where matches: (AXUIElement) -> Bool +) -> AXUIElement? { + var best: (element: AXUIElement, area: Double)? + var budget = 6000 + func visit(_ element: AXUIElement, depth: Int) { + guard budget > 0, depth < 64 else { return } + budget -= 1 + if let rect = rectAttribute(element), rect.width > 0, rect.height > 0 { + guard CGRect(x: rect.x, y: rect.y, width: rect.width, height: rect.height).contains(point) + else { return } + let area = rect.width * rect.height + if area < (best?.area ?? .infinity), matches(element) { + best = (element, area) + } + } + for child in children(of: element) { + visit(child, depth: depth + 1) + } + } + visit(window, depth: 0) + return best?.element +} + +/// Moves the scroll bar of the scroll area under a point by the travel's share of the content's +/// overflow. An inactive app honors a scroll bar change where it drops wheel events. +private func performScrollBarScroll( + app: NSRunningApplication, + at point: CGPoint, + isVertical: Bool, + signedTravel: Double +) -> Bool { + var current = elementAtPoint(in: app, point: point) + for _ in 0..<8 { + guard let element = current, role(of: element) != "AXWindow" else { return false } + current = elementAttribute(element, attribute: kAXParentAttribute as String) + guard role(of: element) == "AXScrollArea" else { continue } + let barAttribute = isVertical ? kAXVerticalScrollBarAttribute : kAXHorizontalScrollBarAttribute + guard let bar = elementAttribute(element, attribute: barAttribute as String), + isAttributeSettable(bar, attribute: kAXValueAttribute as String), + let value = numberAttribute(bar, attribute: kAXValueAttribute as String), + let area = rectAttribute(element), + let content = children(of: element) + .filter({ role(of: $0) != "AXScrollBar" }) + .compactMap(rectAttribute) + .max(by: { $0.width * $0.height < $1.width * $1.height }) + else { + continue + } + let overflow = isVertical ? content.height - area.height : content.width - area.width + guard overflow > 0 else { continue } + let next = scrollBarValue(current: value, signedTravel: signedTravel, overflow: overflow) + return AXUIElementSetAttributeValue(bar, kAXValueAttribute as CFString, NSNumber(value: next)) + == .success + } + return false +} + +private func role(of element: AXUIElement) -> String { + stringAttribute(element, attribute: kAXRoleAttribute as String) ?? "" +} + +private func isTextInput(_ element: AXUIElement) -> Bool { + textInputRoles.contains(role(of: element)) +} + +private func isPressableControl(_ element: AXUIElement) -> Bool { + isPressableControlRole(role(of: element)) + && actionNames(of: element).contains(kAXPressAction as String) +} + +private func actionNames(of element: AXUIElement) -> [String] { + var names: CFArray? + guard AXUIElementCopyActionNames(element, &names) == .success, let names = names as? [String] else { + return [] + } + return names +} + +private func isAttributeSettable(_ element: AXUIElement, attribute: String) -> Bool { + var settable = DarwinBoolean(false) + return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success + && settable.boolValue +} + +private func numberAttribute(_ element: AXUIElement, attribute: String) -> Double? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let number = value as? NSNumber + else { + return nil + } + return number.doubleValue +} + +private struct OnScreenWindow { + let number: Int + let bounds: CGRect +} + +/// The pid's normal-layer windows, front to back. Bounds and owners need no Screen Recording +/// permission; only titles do. +private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] { + guard + let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) + as? [[String: Any]] + else { + return [] + } + return info.compactMap { entry in + guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid, + (entry[kCGWindowLayer as String] as? Int) == 0, + let number = entry[kCGWindowNumber as String] as? Int, + let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary, + let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary) + else { + return nil + } + return OnScreenWindow(number: number, bounds: bounds) + } +} + +func frontWindowNumber(pid: pid_t) -> Int? { + onScreenWindows(pid: pid).first?.number +} + +private func frontWindowFrame(pid: pid_t) -> CGRect? { + onScreenWindows(pid: pid).first?.bounds +} + +private let returnVirtualKey: CGKeyCode = 36 +private let tabVirtualKey: CGKeyCode = 48 + +private func postKey(virtualKey: CGKeyCode, pid: pid_t) throws { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: virtualKey, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: virtualKey, keyDown: false) + else { + throw HelperError.commandFailed("key event creation failed", details: ["reason": "event_creation_failed"]) + } + down.postToPid(pid) + up.postToPid(pid) +} + +private func postText(_ text: String, delayMs: Int, pid: pid_t) throws { + for character in text { + switch character { + case "\n", "\r": + try postKey(virtualKey: returnVirtualKey, pid: pid) + case "\t": + try postKey(virtualKey: tabVirtualKey, pid: pid) + default: + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) + else { + throw HelperError.commandFailed( + "key event creation failed", details: ["reason": "event_creation_failed"]) + } + let units = Array(String(character).utf16) + down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: units) + up.keyboardSetUnicodeString(stringLength: units.count, unicodeString: units) + down.postToPid(pid) + up.postToPid(pid) + } + if delayMs > 0 { + Thread.sleep(forTimeInterval: Double(delayMs) / 1000) + } + } +} diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift index 537566a8a6..1121eaa7f3 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift @@ -7,6 +7,8 @@ private enum SnapshotTraversalLimits { static let maxDesktopApps = 24 static let maxNodes = 1500 static let maxDepth = 12 + /// An app session is one app's tree; Electron alone wraps its page in about ten groups. + static let maxAppDepth = 48 static let maxMenuBarBandY = 64.0 static let maxMenuBarBandHeight = 64.0 static let maxMenuBarExtraWidth = 256.0 @@ -45,11 +47,13 @@ struct SnapshotResponse: Encodable { let nodes: [SnapshotNodeResponse] let truncated: Bool let backend = "macos-helper" + let warnings: [String]? } private struct SnapshotBuildResult { let nodes: [SnapshotNodeResponse] let truncated: Bool + var warnings: [String]? = nil } private struct SnapshotContext { @@ -64,6 +68,8 @@ private struct SnapshotTraversalState { var nodes: [SnapshotNodeResponse] = [] var visited: [AXUIElement] = [] var truncated = false + /// An element sat at the depth cap with children the walk left out. + var depthCapped = false } private struct MenuBarWindowFallbackCandidate { @@ -79,6 +85,8 @@ private struct MenuBarWindowFallbackCandidate { func captureSnapshotResponse(surface: String, bundleId: String? = nil) throws -> SnapshotResponse { let result: SnapshotBuildResult switch surface { + case "app": + result = try snapshotSessionApp(bundleId: bundleId) case "frontmost-app": result = try snapshotFrontmostApp() case "desktop": @@ -86,10 +94,101 @@ func captureSnapshotResponse(surface: String, bundleId: String? = nil) throws -> case "menubar": result = try snapshotMenuBar(bundleId: bundleId) default: - throw HelperError.invalidArgs("snapshot requires --surface ") + throw HelperError.invalidArgs("snapshot requires --surface ") } - return SnapshotResponse(surface: surface, nodes: result.nodes, truncated: result.truncated) + return SnapshotResponse( + surface: surface, nodes: result.nodes, truncated: result.truncated, warnings: result.warnings) +} + +/// The session's own app, whether or not it is frontmost. +private func snapshotSessionApp(bundleId: String?) throws -> SnapshotBuildResult { + let app = try requireSessionApplication(bundleId: bundleId) + let webContentReady = enableRemoteAccessibilityTree(app) + var state = SnapshotTraversalState() + _ = appendApplicationSnapshot( + app, + depth: 0, + parentIndex: nil, + surface: "app", + maxDepth: SnapshotTraversalLimits.maxAppDepth, + state: &state + ) + return SnapshotBuildResult( + nodes: state.nodes, + truncated: state.truncated || state.depthCapped, + warnings: webContentReady + ? nil + : [ + "The app's Chromium accessibility tree did not populate, so web content may be missing from this snapshot." + ] + ) +} + +/// Chromium and Electron build their accessibility tree only once a client asks for it, so an +/// unannounced reader sees a window of empty groups. Electron answers `AXManualAccessibility`; +/// other Chromium shells answer the VoiceOver-style `AXEnhancedUserInterface`, which native apps +/// also honor by changing window behavior, so it is set only on a Chromium app. The tree stays on +/// for the app's lifetime, as it does for any assistive client. +/// +/// Whether the app's web content is readable: true for an app that is not Chromium-based or whose +/// tree is already on. The wait is paid only by the call that turns the tree on. +private func enableRemoteAccessibilityTree(_ app: NSRunningApplication) -> Bool { + guard isChromiumApplication(app) else { return true } + let appElement = AXUIElementCreateApplication(app.processIdentifier) + let attributes = ["AXManualAccessibility", "AXEnhancedUserInterface"] + if attributes.contains(where: { boolAttribute(appElement, attribute: $0) == true }) { + return true + } + guard + attributes.contains(where: { + AXUIElementSetAttributeValue(appElement, $0 as CFString, kCFBooleanTrue) == .success + }) + else { + return false + } + return awaitPopulatedWebContent(appElement) +} + +/// Chromium builds the tree after the request returns, so a walk right after enabling it would see +/// only the wrapper groups. Bounded well inside the helper's own deadline. +private func awaitPopulatedWebContent(_ appElement: AXUIElement) -> Bool { + let deadline = Date().addingTimeInterval(1.0) + while !hasPopulatedWebArea(appElement) { + guard Date() < deadline else { return false } + Thread.sleep(forTimeInterval: 0.05) + } + return true +} + +private func hasPopulatedWebArea(_ appElement: AXUIElement) -> Bool { + var budget = 400 + func visit(_ element: AXUIElement, depth: Int) -> Bool { + guard budget > 0, depth < 16 else { return false } + budget -= 1 + let elementChildren = children(of: element) + if stringAttribute(element, attribute: kAXRoleAttribute as String) == "AXWebArea" { + return !elementChildren.isEmpty + } + return elementChildren.contains { visit($0, depth: depth + 1) } + } + return windows(of: appElement).contains { visit($0, depth: 0) } +} + +/// Every Chromium embedder, Electron or a renamed framework, ships Chromium's resource pack. +private func isChromiumApplication(_ app: NSRunningApplication) -> Bool { + guard let frameworksURL = app.bundleURL?.appendingPathComponent("Contents/Frameworks"), + let frameworks = try? FileManager.default.contentsOfDirectory(atPath: frameworksURL.path) + else { + return false + } + return frameworks.contains { framework in + framework.hasSuffix(".framework") + && FileManager.default.fileExists( + atPath: frameworksURL.appendingPathComponent(framework) + .appendingPathComponent("Resources/chrome_100_percent.pak").path + ) + } } private func snapshotFrontmostApp() throws -> SnapshotBuildResult { @@ -166,6 +265,7 @@ private func appendApplicationSnapshot( depth: Int, parentIndex: Int?, surface: String, + maxDepth: Int = SnapshotTraversalLimits.maxDepth, state: inout SnapshotTraversalState ) -> Bool { let appElement = AXUIElementCreateApplication(app.processIdentifier) @@ -207,7 +307,8 @@ private func appendApplicationSnapshot( appName: app.localizedName, windowTitle: windowTitle ), - state: &state + state: &state, + maxDepth: maxDepth ) } @@ -542,6 +643,9 @@ private func appendElementSnapshot( ) guard depth < maxDepth, !state.truncated else { + if depth >= maxDepth, !snapshotChildren(of: element, role: role).isEmpty { + state.depthCapped = true + } return index } diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift index 1dc3445312..9d0aa0c545 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift @@ -122,6 +122,12 @@ struct AgentDeviceMacOSHelper { return try handleRead(arguments: Array(arguments.dropFirst())) case "press": return try handlePress(arguments: Array(arguments.dropFirst())) + case "type": + return try handleType(arguments: Array(arguments.dropFirst())) + case "fill": + return try handleFill(arguments: Array(arguments.dropFirst())) + case "scroll": + return try handleScroll(arguments: Array(arguments.dropFirst())) case "screenshot": return try handleScreenshot(arguments: Array(arguments.dropFirst())) case "audio-probe": @@ -343,18 +349,16 @@ struct AgentDeviceMacOSHelper { .lowercased(), !surface.isEmpty else { - throw HelperError.invalidArgs("snapshot requires --surface ") + throw HelperError.invalidArgs("snapshot requires --surface ") } let bundleId = try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) switch surface { - case "frontmost-app": - return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface, bundleId: bundleId)) - case "desktop", "menubar": + case "app", "frontmost-app", "desktop", "menubar": return SuccessEnvelope(data: try captureSnapshotResponse(surface: surface, bundleId: bundleId)) default: - throw HelperError.invalidArgs("snapshot requires --surface ") + throw HelperError.invalidArgs("snapshot requires --surface ") } } @@ -376,25 +380,25 @@ struct AgentDeviceMacOSHelper { static func handlePress(arguments: [String]) throws -> any Encodable { guard let rawX = optionValue(arguments: arguments, name: "--x"), let rawY = optionValue(arguments: arguments, name: "--y"), - let x = Double(rawX), - let y = Double(rawY) + let x = Double(rawX), x.isFinite, + let y = Double(rawY), y.isFinite else { throw HelperError.invalidArgs("press requires --x --y ") } - let holdMs = try validatedPressInt( + let holdMs = try validatedIntOption( optionValue(arguments: arguments, name: "--hold-ms"), name: "--hold-ms", minimum: 0, default: 0 ) - let clicks = try validatedPressInt( + let clicks = try validatedIntOption( optionValue(arguments: arguments, name: "--clicks"), name: "--clicks", minimum: 1, default: 1 ) - let intervalMs = try validatedPressInt( + let intervalMs = try validatedIntOption( optionValue(arguments: arguments, name: "--interval-ms"), name: "--interval-ms", minimum: 0, @@ -412,6 +416,12 @@ struct AgentDeviceMacOSHelper { doubleClick: doubleClick, intervalMs: intervalMs ) + if surface == "app" { + let app = try requireSessionApplication(bundleId: bundleId) + return SuccessEnvelope( + data: try pressInBackground(request, app: app) + ) + } try pressAtPosition(request) return SuccessEnvelope( data: PressResponse( @@ -426,6 +436,63 @@ struct AgentDeviceMacOSHelper { ) } + static func handleType(arguments: [String]) throws -> any Encodable { + guard let text = optionValue(arguments: arguments, name: "--text") else { + throw HelperError.invalidArgs("type requires --text ") + } + let delayMs = try validatedIntOption( + optionValue(arguments: arguments, name: "--delay-ms"), + name: "--delay-ms", + minimum: 0, + default: 0 + ) + let app = try requireSessionApplication( + bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) + ) + return SuccessEnvelope( + data: try typeInBackground(text: text, delayMs: delayMs, app: app) + ) + } + + static func handleFill(arguments: [String]) throws -> any Encodable { + guard let rawX = optionValue(arguments: arguments, name: "--x"), + let rawY = optionValue(arguments: arguments, name: "--y"), + let x = Double(rawX), x.isFinite, + let y = Double(rawY), y.isFinite, + let text = optionValue(arguments: arguments, name: "--text") + else { + throw HelperError.invalidArgs("fill requires --x --y --text ") + } + let app = try requireSessionApplication( + bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) + ) + return SuccessEnvelope( + data: try fillInBackground(point: CGPoint(x: x, y: y), text: text, app: app) + ) + } + + static func handleScroll(arguments: [String]) throws -> any Encodable { + guard let direction = optionValue(arguments: arguments, name: "--direction") else { + throw HelperError.invalidArgs("scroll requires --direction ") + } + let amount = try positiveDoubleOption(arguments: arguments, name: "--amount") + let pixels = try positiveDoubleOption(arguments: arguments, name: "--pixels") + if amount != nil, pixels != nil { + throw HelperError.invalidArgs("scroll accepts --amount or --pixels, not both") + } + let app = try requireSessionApplication( + bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) + ) + return SuccessEnvelope( + data: try scrollInBackground( + direction: direction, + amount: amount, + pixels: pixels, + app: app + ) + ) + } + static func handleScreenshot(arguments: [String]) throws -> any Encodable { guard let outPath = optionValue(arguments: arguments, name: "--out")? .trimmingCharacters(in: .whitespacesAndNewlines), @@ -435,6 +502,13 @@ struct AgentDeviceMacOSHelper { } let surface = optionValue(arguments: arguments, name: "--surface") + if surface == "app" { + let app = try requireSessionApplication( + bundleId: try optionValue(arguments: arguments, name: "--bundle-id").map(validatedBundleId) + ) + try captureAppWindowScreenshot(app: app, outPath: outPath) + return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface)) + } try captureSurfaceScreenshot(surface: surface, outPath: outPath) return SuccessEnvelope(data: ScreenshotResponse(path: outPath, surface: surface)) } @@ -468,6 +542,14 @@ private func optionValue(arguments: [String], name: String) -> String? { return arguments[index + 1] } +private func positiveDoubleOption(arguments: [String], name: String) throws -> Double? { + guard let raw = optionValue(arguments: arguments, name: name) else { return nil } + guard let value = Double(raw), value.isFinite, value > 0 else { + throw HelperError.invalidArgs("\(name) must be a positive number") + } + return value +} + private func intOption(arguments: [String], name: String) -> Int? { guard let value = optionValue(arguments: arguments, name: name) else { return nil @@ -475,7 +557,7 @@ private func intOption(arguments: [String], name: String) -> Int? { return Int(value) } -private func validatedPressInt( +private func validatedIntOption( _ raw: String?, name: String, minimum: Int, @@ -485,22 +567,30 @@ private func validatedPressInt( return fallback } guard let value = Int(raw), value >= minimum else { - throw HelperError.invalidArgs("press \(name) must be an integer of at least \(minimum)") + throw HelperError.invalidArgs("\(name) must be an integer of at least \(minimum)") } return value } private func readTextAtPosition(bundleId: String?, surface: String?, x: Double, y: Double) throws -> String { let targetApp: NSRunningApplication? - if surface == "frontmost-app" || (surface == nil && bundleId != nil) { + if surface == "app" { + targetApp = try requireSessionApplication(bundleId: bundleId) + } else if surface == "frontmost-app" || (surface == nil && bundleId != nil) { targetApp = try resolveTargetApplication(bundleId: bundleId, surface: surface) } else { targetApp = nil } - let systemWide = AXUIElementCreateSystemWide() + // An app session's window may sit behind other apps, so it is hit-tested on its own. + let hitRoot = + if surface == "app", let targetApp { + AXUIElementCreateApplication(targetApp.processIdentifier) + } else { + AXUIElementCreateSystemWide() + } var hitElement: AXUIElement? - guard AXUIElementCopyElementAtPosition(systemWide, Float(x), Float(y), &hitElement) == .success, + guard AXUIElementCopyElementAtPosition(hitRoot, Float(x), Float(y), &hitElement) == .success, let hitElement else { throw HelperError.commandFailed("read did not resolve an accessibility element") @@ -567,18 +657,25 @@ private func captureSurfaceScreenshot(surface: String?, outPath: String) throws semaphore.wait() if let error = capturedError as NSError? { - if error.domain == "com.apple.ScreenCaptureKit.SCStreamErrorDomain", error.code == -3801 { - throw HelperError.commandFailed( - "screenshot requires Screen Recording permission on macOS desktop and menubar surfaces", - details: ["surface": surface ?? "", "permission": "screen-recording"] - ) - } - throw HelperError.commandFailed("screenshot failed", details: ["error": error.localizedDescription]) + throw screenshotFailure(error, surface: surface) } guard let capturedImage else { throw HelperError.commandFailed("screenshot failed") } + try writePNG(capturedImage, to: outPath) +} + +func screenshotFailure(_ error: NSError, surface: String?) -> HelperError { + if error.domain == "com.apple.ScreenCaptureKit.SCStreamErrorDomain", error.code == -3801 { + return HelperError.commandFailed( + "screenshot requires Screen Recording permission on the macOS \(surface ?? "desktop") surface", + details: ["surface": surface ?? "", "permission": "screen-recording"] + ) + } + return HelperError.commandFailed("screenshot failed", details: ["error": error.localizedDescription]) +} +func writePNG(_ capturedImage: CGImage, to outPath: String) throws { let outputURL = URL(fileURLWithPath: outPath) if let parent = outputURL.deletingLastPathComponent().path.removingPercentEncoding, !parent.isEmpty { try FileManager.default.createDirectory(atPath: parent, withIntermediateDirectories: true) diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift new file mode 100644 index 0000000000..7b61be5fc3 --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift @@ -0,0 +1,88 @@ +import XCTest + +@testable import AgentDeviceMacOSHelper + +private struct ScrollGestureFixture: Decodable { + struct Constants: Decodable { + let defaultScrollAmount: Double + } + struct Expected: Decodable { + let pixels: Double + } + struct Case: Decodable { + let name: String + let direction: String + let amount: Double? + let pixels: Double? + let referenceWidth: Double + let referenceHeight: Double + let expected: Expected + } + + let constants: Constants + let cases: [Case] +} + +private struct HelperOutcomesFixture: Decodable { + let refusalReasons: [String] + let deliveryMechanisms: [String] +} + +/// `#filePath` is `/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/`. +private let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent().deletingLastPathComponent() + +final class BackgroundInteractionTests: XCTestCase { + /// The host keys refusals and mechanisms on these strings; the table is their one declaration. + func testOutcomeVocabularyMatchesTheSharedTable() throws { + let fixture = try JSONDecoder().decode( + HelperOutcomesFixture.self, + from: Data( + contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json")) + ) + XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons) + XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms) + } + + /// The native backend scrolls as far as the runner does: its travel agrees with every case of + /// the cross-language table the runner and the TypeScript planner are held to. + func testScrollTravelMatchesTheSharedScrollGestureTable() throws { + let fixtureURL = repoRoot.appendingPathComponent("contracts/fixtures/scroll-gesture.json") + let fixture = try JSONDecoder().decode( + ScrollGestureFixture.self, from: Data(contentsOf: fixtureURL)) + XCTAssertFalse(fixture.cases.isEmpty) + for testCase in fixture.cases { + let isVertical = testCase.direction == "up" || testCase.direction == "down" + let travel = scrollTravelPixels( + axisLength: isVertical ? testCase.referenceHeight : testCase.referenceWidth, + amount: testCase.amount, + pixels: testCase.pixels + ) + XCTAssertEqual(travel, testCase.expected.pixels, testCase.name) + } + XCTAssertEqual( + scrollTravelPixels(axisLength: 1000, amount: nil, pixels: nil), + (1000 * fixture.constants.defaultScrollAmount).rounded(), + "an unspecified distance travels the table's default amount" + ) + } + + func testScrollBarValueMovesByTheTravelShareOfTheOverflow() { + XCTAssertEqual(scrollBarValue(current: 0, signedTravel: 300, overflow: 1200), 0.25) + XCTAssertEqual(scrollBarValue(current: 0.5, signedTravel: -300, overflow: 1200), 0.25) + } + + func testScrollBarValueStopsAtTheEnds() { + XCTAssertEqual(scrollBarValue(current: 0.9, signedTravel: 600, overflow: 1200), 1) + XCTAssertEqual(scrollBarValue(current: 0.1, signedTravel: -600, overflow: 1200), 0) + } + + /// Chromium marks wrapper groups pressable; only a control role may win the press outright. + func testWrapperGroupsAreNotPressableControls() { + XCTAssertTrue(isPressableControlRole("AXButton")) + XCTAssertTrue(isPressableControlRole("AXLink")) + XCTAssertFalse(isPressableControlRole("AXGroup")) + XCTAssertFalse(isPressableControlRole("AXWebArea")) + } +} diff --git a/contracts/fixtures/macos-native-helper-outcomes.json b/contracts/fixtures/macos-native-helper-outcomes.json new file mode 100644 index 0000000000..be60f68c9b --- /dev/null +++ b/contracts/fixtures/macos-native-helper-outcomes.json @@ -0,0 +1,18 @@ +{ + "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal and BackgroundDeliveryMechanism in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", + "refusalReasons": [ + "background-pointer-gesture", + "no-accessible-target", + "no-settable-text-input", + "no-scroll-bar" + ], + "deliveryMechanisms": [ + "ax-press", + "ax-focus", + "ax-value", + "ax-selected-text", + "ax-confirm", + "ax-scroll-bar", + "key-events" + ] +}