fix(ios): report capture refusals as typed reasons in back-fallback and recording

#2741 routed screenshots through the resolved app display but left two
failure contracts papered over (#2728):

- The in-app `back` coordinate fallback read a nil screenshot sample as "no
  visual change" and sampled SpringBoard when the app resolved no window, so
  a two-home-screen comparison reported "in-app back control is not
  available" after a real leading tap had already run. The check now observes
  only the app's own resolved window and reports the display refusal as an
  unverified outcome; an observable no-change still reads as unavailable.
- `record start` collapsed every no-frame bootstrap into an untyped NSError.
  It now surfaces the shared APP_SCREEN_* reason, so a runtime with no
  resolvable window fails closed with a typed code.

Refuse a zero-pixel capture at the capture type so a required consumer cannot
mistake it for a usable frame. macOS/tvOS keep their host-display capture and
generic bootstrap error. Focused runner unit tests pin the observation split,
the refusal-selection, and the typed-vs-generic error mapping.

Author: thymikee

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift

@@ -140,6 +140,12 @@ extension RunnerTests {
     guard let cgImage = runnerCGImage(from: upright) else {
       return .failure(.unrenderableImage)
     }
+    // A zero-pixel image is a capture that did not happen, not a tiny one. Refusing it here — at the
+    // type that owns the fact — keeps a required consumer (a recording sizing its writer from this
+    // frame) from mistaking it for a usable frame and falling back to an untyped error (#2728).
+    guard cgImage.width > 0, cgImage.height > 0 else {
+      return .failure(.unrenderableImage)
+    }
     return .success(
       CapturedAppScreen(
         image: upright,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -1466,13 +1466,14 @@ extension RunnerTests {
           fps: command.fps.map { Int32($0) }
         )
         try recorder.start { [weak self] in
-          return self?.captureRunnerFrame(app: activeApp)
+          guard let self else { return .failure(.unresolvedScreen) }
+          return self.captureRunnerFrameResult(app: activeApp)
         }
         activeRecording = recorder
         return Response(ok: true, data: DataPayload(message: "recording started"))
       } catch {
         activeRecording = nil
-        return Response(ok: false, error: ErrorPayload(message: "failed to start recording: \(error.localizedDescription)"))
+        return Response(ok: false, error: Self.recordingStartErrorPayload(for: error))
       }
     case .recordStop:
       guard let recorder = activeRecording else {
@@ -2120,11 +2121,20 @@ extension RunnerTests {
       )
 #endif
     case .back, .backInApp:
-      if tapInAppBackControl(app: activeApp) {
+      switch tapInAppBackControl(app: activeApp) {
+      case .performed:
         let message = command.command == .back ? "back" : "backInApp"
         return Response(ok: true, data: DataPayload(message: message))
+      case .unavailable:
+        return Response(
+          ok: false,
+          error: ErrorPayload(message: "in-app back control is not available")
+        )
+      case .unverified(let error):
+        // The fallback gesture ran but the display refused to be sampled. Reporting the typed refusal
+        // keeps an unknown outcome from being laundered into a definitive "no back control" (#2728).
+        return Response(ok: false, error: error)
       }
-      return Response(ok: false, error: ErrorPayload(message: "in-app back control is not available"))
     case .backSystem:
       if performSystemBackAction(app: activeApp) {
         return Response(ok: true, data: DataPayload(message: "backSystem"))

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -42,31 +42,59 @@ enum RunnerInteractionIdleWaits {
 extension RunnerTests {
   // MARK: - Recording
 
-  /// One frame for the recording pump and the keyboard settle sample.
+  /// One frame for a caller that tolerates a dropped one — keyboard settling, which skips a sample it
+  /// cannot take and keeps polling. A frame that must exist goes through `captureRunnerFrameResult`,
+  /// which says why it refused.
   ///
   /// On iOS the frame comes from the display owning a window, because a foldable's
   /// `XCUIScreen.main` can be the dark outer panel while the app runs on the inner one — a stream of
-  /// identical black frames would then read as a settled screen and as a finished recording (#2728).
-  /// An observation with no session window falls to the system surface's window, which is what the
-  /// home screen is. macOS keeps recording the host display the way it always has.
+  /// identical black frames would then read as a settled screen (#2728). An observation with no
+  /// session window falls to the system surface's window, which is what the home screen is. macOS
+  /// keeps the host display it always recorded.
   func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? {
-#if os(iOS)
-    guard case .success(let captured) = captureObservedScreen(app: app) else {
+    switch captureRunnerFrameResult(app: app) {
+    case .success(let captured):
+      return captured.image
+    case .failure:
       return nil
     }
-    return captured.image
+  }
+
+  /// The same frame as `captureRunnerFrame`, but carrying the reason it refused, so a required first
+  /// frame — a recording's bootstrap, which sizes the whole writer from it — fails closed with a
+  /// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal
+  /// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728).
+  func captureRunnerFrameResult(
+    app: XCUIApplication
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+#if os(iOS)
+    return captureObservedScreen(app: app)
 #else
-    var image: RunnerImage?
+    var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
+      .unrenderableImage
+    )
     let capture = {
-      let screenshot = XCUIScreen.main.screenshot()
-      image = screenshot.image
+      let image = XCUIScreen.main.screenshot().image
+      if let cgImage = runnerCGImage(from: image) {
+        // The host display has no resolved-panel facts to report; the recorder reads only the image
+        // and its pixel size, so these two are inert placeholders, not measurements the host scales by.
+        outcome = .success(
+          CapturedAppScreen(
+            image: image,
+            displayID: 0,
+            pixelWidth: cgImage.width,
+            pixelHeight: cgImage.height,
+            pixelsPerPoint: 1
+          )
+        )
+      }
     }
     if Thread.isMainThread {
       capture()
     } else {
       DispatchQueue.main.sync(execute: capture)
     }
-    return image
+    return outcome
 #endif
   }
 

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -4,27 +4,66 @@ extension RunnerTests {
   static let navigationBackKeywords = ["back", "close", "cancel"]
   static let navigationFallbackVerificationDelay: TimeInterval = 0.25
 
-  func tapInAppBackControl(app: XCUIApplication) -> Bool {
+  /// What one in-app `back` attempt concluded. `.unverified` carries the typed capture failure the
+  /// display refused with, so an optional visual check that could not run reports an unknown instead
+  /// of the false "no back control exists" that a `nil` sample would otherwise become (#2728).
+  enum InAppBackOutcome {
+    case performed
+    case unavailable
+    case unverified(ErrorPayload)
+  }
+
+  /// The three answers a before/after visual comparison can give. `unobserved` is not evidence of
+  /// "no change": a display that refuses to be sampled proves nothing either way (#2728).
+  enum NavigationVisualObservation: Equatable {
+    case changed
+    case unchanged
+    case unobserved
+
+    var logToken: String {
+      switch self {
+      case .changed: return "yes"
+      case .unchanged: return "no"
+      case .unobserved: return "unknown"
+      }
+    }
+  }
+
+  /// One navigation-fallback capture: the encoded frame when the display answered, and the typed
+  /// reason it refused otherwise. Only iOS produces a refusal; other platforms capture nothing here.
+  struct NavigationVisualSample {
+    let data: Data?
+    let refusalCode: String?
+    let refusalHint: String?
+
+    init(data: Data?, refusalCode: String? = nil, refusalHint: String? = nil) {
+      self.data = data
+      self.refusalCode = refusalCode
+      self.refusalHint = refusalHint
+    }
+  }
+
+  func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome {
 #if os(macOS)
     if let back = macOSNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
-      return true
+      return .performed
     }
-    return false
+    return .unavailable
 #elseif os(tvOS)
     _ = pressTvRemote(.menu)
-    return true
+    return .performed
 #else
     let buttons = app.navigationBars.buttons.allElementsBoundByIndex
     if let back = buttons.first(where: { $0.isHittable }) {
       back.tap()
-      return true
+      return .performed
     }
     if isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) {
       NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", currentBundleId ?? "")
     } else if let back = topNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
-      return true
+      return .performed
     }
     return tapTopLeadingNavigationFallback(app: app)
 #endif
@@ -109,11 +148,11 @@ extension RunnerTests {
     return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
   }
 
-  private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> Bool {
+  private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> InAppBackOutcome {
 #if os(iOS)
     let frame = onScreenWindowFrame(app: app)
     guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else {
-      return false
+      return .unavailable
     }
     let before = captureNavigationFallbackVisualState(app: app)
     let context = synthesizedCoordinateContext(
@@ -124,54 +163,96 @@ extension RunnerTests {
       synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
     }
     if case .performed = synthesized.outcome {
-      return didNavigationFallbackChangeVisualState(app: app, before: before)
+      return verifyNavigationFallbackOutcome(app: app, before: before)
     }
     let fallback = performGesture(app) {
       tapAt(app: app, x: point.x, y: point.y)
     }
     if case .performed = fallback.outcome {
-      return didNavigationFallbackChangeVisualState(app: app, before: before)
+      return verifyNavigationFallbackOutcome(app: app, before: before)
     }
 #endif
-    return false
+    return .unavailable
   }
 
-  private func captureNavigationFallbackVisualState(app: XCUIApplication) -> Data? {
+  private func captureNavigationFallbackVisualState(app: XCUIApplication) -> NavigationVisualSample {
 #if os(iOS)
-    // A visual check is optional evidence: a display that owns no window reports unknown by returning
-    // no sample, which the comparison below already reads as "nothing proved". It never reaches for
-    // a screen nobody is on, whose stable black would then be read as evidence that the tap did
-    // nothing (#2728).
-    guard case .success(let captured) = captureObservedScreen(app: app) else {
-      return nil
+    // A visual check is optional evidence, and the check is about THIS app's screen after the tap.
+    // It uses the app's own resolved screen, never the system-surface fallback: sampling SpringBoard's
+    // home screen twice would read as "unchanged" for an app that resolved no window, laundering a
+    // wrong-process sample into the same false "no back control" verdict a `nil` sample produces (#2728).
+    switch captureResolvedAppScreen(app: app) {
+    case .success(let captured):
+      guard let png = runnerPngData(for: captured.image) else {
+        // The display resolved but the image would not encode: name it as the capture failure it is,
+        // not as an unnamed no-sample that would default to "no display resolved" (#2728).
+        let refusal = RunnerAppScreenCaptureFailure.unrenderableImage
+        return NavigationVisualSample(data: nil, refusalCode: refusal.rawValue, refusalHint: refusal.hint)
+      }
+      return NavigationVisualSample(data: png)
+    case .failure(let failure):
+      return NavigationVisualSample(
+        data: nil,
+        refusalCode: failure.rawValue,
+        refusalHint: failure.hint
+      )
     }
-    return runnerPngData(for: captured.image)
 #else
-    return nil
+    return NavigationVisualSample(data: nil)
 #endif
   }
 
-  private func didNavigationFallbackChangeVisualState(
+  private func verifyNavigationFallbackOutcome(
     app: XCUIApplication,
-    before: Data?
-  ) -> Bool {
+    before: NavigationVisualSample
+  ) -> InAppBackOutcome {
     sleepFor(Self.navigationFallbackVerificationDelay)
     let after = captureNavigationFallbackVisualState(app: app)
-    let changed = Self.didNavigationFallbackChangeVisualState(before: before, after: after)
-    // The sample sizes are what tells a refused capture apart from an unchanged screen, and the
-    // fallback is rare enough that saying so every time costs nothing (#2728).
+    let observation = Self.navigationVisualObservation(before: before.data, after: after.data)
+    // The sample sizes and the observation name together tell a refused capture apart from an
+    // unchanged screen, and the fallback is rare enough that saying so every time costs nothing.
     NSLog(
       "AGENT_DEVICE_RUNNER_IN_APP_BACK_VISUAL_VERIFICATION beforeBytes=%ld afterBytes=%ld changed=%@",
-      before?.count ?? -1,
-      after?.count ?? -1,
-      changed ? "yes" : "no"
+      before.data?.count ?? -1,
+      after.data?.count ?? -1,
+      observation.logToken
     )
-    return changed
+    switch observation {
+    case .changed:
+      return .performed
+    case .unchanged:
+      return .unavailable
+    case .unobserved:
+      // No sample is not evidence of no navigation change: the fallback ran, so report the display
+      // refusal it hit rather than laundering an unobservable result into "back is not available".
+      return .unverified(Self.navigationFallbackErrorPayload(after: after, before: before))
+    }
+  }
+
+  static func navigationVisualObservation(
+    before: Data?,
+    after: Data?
+  ) -> NavigationVisualObservation {
+    guard let before, let after else { return .unobserved }
+    return before != after ? .changed : .unchanged
   }
 
-  static func didNavigationFallbackChangeVisualState(before: Data?, after: Data?) -> Bool {
-    guard let before, let after else { return false }
-    return before != after
+  /// The refusal the fallback most recently hit wins, so the code names the last thing it looked at
+  /// before giving up. A missing reason on both sides is unreachable on iOS (every nil sample carries
+  /// one) and defaults to the plain display-unresolved code.
+  static func navigationFallbackErrorPayload(
+    after: NavigationVisualSample,
+    before: NavigationVisualSample
+  ) -> ErrorPayload {
+    ErrorPayload(
+      code:
+        after.refusalCode
+        ?? before.refusalCode
+        ?? RunnerAppScreenCaptureFailure.unresolvedScreen.rawValue,
+      message:
+        "The in-app back fallback was dispatched, but no display could be sampled to confirm the result. This is an unknown outcome, not evidence that a back control is absent.",
+      hint: after.refusalHint ?? before.refusalHint
+    )
   }
 
   private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
@@ -234,21 +315,53 @@ extension RunnerTests {
     XCTAssertFalse(Self.isTopNavigationControlFrame(.infinite, in: window))
   }
 
-  func testNavigationFallbackRequiresObservedVisualChange() {
-    XCTAssertTrue(
-      Self.didNavigationFallbackChangeVisualState(
-        before: Data([1, 2, 3]),
-        after: Data([1, 2, 4])
-      )
+  func testNavigationVisualVerificationSeparatesNoChangeFromNoSample() {
+    XCTAssertEqual(
+      Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 4])),
+      .changed
     )
-    XCTAssertFalse(
-      Self.didNavigationFallbackChangeVisualState(
-        before: Data([1, 2, 3]),
-        after: Data([1, 2, 3])
-      )
+    XCTAssertEqual(
+      Self.navigationVisualObservation(before: Data([1, 2, 3]), after: Data([1, 2, 3])),
+      .unchanged
+    )
+    // A missing sample is neither a change nor a no-change; treating it as "unchanged" would let a
+    // capture that refused become the reason the `back` command claims no control exists (#2728).
+    XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: Data([1])), .unobserved)
+    XCTAssertEqual(Self.navigationVisualObservation(before: Data([1]), after: nil), .unobserved)
+    XCTAssertEqual(Self.navigationVisualObservation(before: nil, after: nil), .unobserved)
+  }
+
+  func testNavigationFallbackReportsTheRefusalItHitNotADefaultCode() {
+    // The refusal from the most recent sample wins, so the code names what the fallback last looked at
+    // before giving up; an earlier refusal is reported only when the later sample carried none (#2728).
+    let after = NavigationVisualSample(
+      data: nil,
+      refusalCode: "APP_SCREEN_WINDOW_UNRESOLVED",
+      refusalHint: "after hint"
+    )
+    let before = NavigationVisualSample(
+      data: nil,
+      refusalCode: "APP_SCREEN_UNRESOLVED",
+      refusalHint: "before hint"
+    )
+    let laterWins = Self.navigationFallbackErrorPayload(after: after, before: before)
+    XCTAssertEqual(laterWins.code, "APP_SCREEN_WINDOW_UNRESOLVED")
+    XCTAssertEqual(laterWins.hint, "after hint")
+
+    let onlyBefore = Self.navigationFallbackErrorPayload(
+      after: NavigationVisualSample(data: nil),
+      before: before
+    )
+    XCTAssertEqual(onlyBefore.code, "APP_SCREEN_UNRESOLVED")
+    XCTAssertEqual(onlyBefore.hint, "before hint")
+
+    // Neither side named a reason (unreachable on iOS): a real capture code, never a bare failure.
+    let unnamed = Self.navigationFallbackErrorPayload(
+      after: NavigationVisualSample(data: nil),
+      before: NavigationVisualSample(data: nil)
     )
-    XCTAssertFalse(Self.didNavigationFallbackChangeVisualState(before: nil, after: Data([1])))
-    XCTAssertFalse(Self.didNavigationFallbackChangeVisualState(before: Data([1]), after: nil))
+    XCTAssertEqual(unnamed.code, "APP_SCREEN_UNRESOLVED")
+    XCTAssertTrue(unnamed.message.contains("unknown outcome"))
   }
 #endif
 }