fix(ios): report capture refusals as typed reasons in back-fallback and recording

.github/workflows/ios.yml

@@ -154,6 +154,7 @@ jobs:
             -xctestrun "$XCTESTRUN_PATH" \
             -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSinglePointerFlingFallsBackToXCTestCoordinateDragWhenPrivateSynthesisFails \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRecordStartThrowsTheCaptureRefusalItReceived \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertDispatchResolvesItsOwnModalWithoutCoordinateTapRoutingProbe \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testAlertResolutionCannotBypassRequestedDeadline \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTypeWithoutResolvedInputReturnsTypedFailureBeforeDispatchingText \

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift

@@ -54,6 +54,21 @@ enum RunnerAppScreenCaptureFailure: String, Error {
   }
 }
 
+extension RunnerTests {
+  /// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved
+  /// window asks the system surface, and nothing else does.
+  func selectObservedScreenCapture(
+    resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+    fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    let outcome = resolving()
+    if case .failure(.unresolvedWindow) = outcome {
+      return fallingBack()
+    }
+    return outcome
+  }
+}
+
 #if canImport(UIKit) && os(iOS)
 extension RunnerTests {
   /// Captures the display hosting `app` instead of `XCUIScreen.main`.
@@ -103,19 +118,6 @@ extension RunnerTests {
     )
   }
 
-  /// The target rule, kept apart from the two queries so the rule itself is testable: an unresolved
-  /// window asks the system surface, and nothing else does.
-  func selectObservedScreenCapture(
-    resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
-    fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
-  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
-    let outcome = resolving()
-    if case .failure(.unresolvedWindow) = outcome {
-      return fallingBack()
-    }
-    return outcome
-  }
-
   private static func resolveCapturedAppScreen(
     app: XCUIApplication
   ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
@@ -140,6 +142,12 @@ extension RunnerTests {
     guard let cgImage = runnerCGImage(from: upright) else {
       return .failure(.unrenderableImage)
     }
+    // A zero-pixel image is a capture that did not happen, not a tiny one. Refusing it here — at the
+    // type that owns the fact — keeps a required consumer (a recording sizing its writer from this
+    // frame) from mistaking it for a usable frame and falling back to an untyped error (#2728).
+    guard cgImage.width > 0, cgImage.height > 0 else {
+      return .failure(.unrenderableImage)
+    }
     return .success(
       CapturedAppScreen(
         image: upright,

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -1466,13 +1466,14 @@ extension RunnerTests {
           fps: command.fps.map { Int32($0) }
         )
         try recorder.start { [weak self] in
-          return self?.captureRunnerFrame(app: activeApp)
+          guard let self else { return .failure(.unresolvedScreen) }
+          return self.captureRunnerFrameResult(app: activeApp)
         }
         activeRecording = recorder
         return Response(ok: true, data: DataPayload(message: "recording started"))
       } catch {
         activeRecording = nil
-        return Response(ok: false, error: ErrorPayload(message: "failed to start recording: \(error.localizedDescription)"))
+        return Response(ok: false, error: Self.recordingStartErrorPayload(for: error))
       }
     case .recordStop:
       guard let recorder = activeRecording else {
@@ -2120,11 +2121,20 @@ extension RunnerTests {
       )
 #endif
     case .back, .backInApp:
-      if tapInAppBackControl(app: activeApp) {
+      switch tapInAppBackControl(app: activeApp) {
+      case .performed:
         let message = command.command == .back ? "back" : "backInApp"
         return Response(ok: true, data: DataPayload(message: message))
+      case .unavailable:
+        return Response(
+          ok: false,
+          error: ErrorPayload(message: "in-app back control is not available")
+        )
+      case .unverified(let error):
+        // The fallback gesture ran but the display refused to be sampled. Reporting the typed refusal
+        // keeps an unknown outcome from being laundered into a definitive "no back control" (#2728).
+        return Response(ok: false, error: error)
       }
-      return Response(ok: false, error: ErrorPayload(message: "in-app back control is not available"))
     case .backSystem:
       if performSystemBackAction(app: activeApp) {
         return Response(ok: true, data: DataPayload(message: "backSystem"))

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -42,31 +42,59 @@ enum RunnerInteractionIdleWaits {
 extension RunnerTests {
   // MARK: - Recording
 
-  /// One frame for the recording pump and the keyboard settle sample.
+  /// One frame for a caller that tolerates a dropped one — keyboard settling, which skips a sample it
+  /// cannot take and keeps polling. A frame that must exist goes through `captureRunnerFrameResult`,
+  /// which says why it refused.
   ///
   /// On iOS the frame comes from the display owning a window, because a foldable's
   /// `XCUIScreen.main` can be the dark outer panel while the app runs on the inner one — a stream of
-  /// identical black frames would then read as a settled screen and as a finished recording (#2728).
-  /// An observation with no session window falls to the system surface's window, which is what the
-  /// home screen is. macOS keeps recording the host display the way it always has.
+  /// identical black frames would then read as a settled screen (#2728). An observation with no
+  /// session window falls to the system surface's window, which is what the home screen is. macOS
+  /// keeps the host display it always recorded.
   func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? {
-#if os(iOS)
-    guard case .success(let captured) = captureObservedScreen(app: app) else {
+    switch captureRunnerFrameResult(app: app) {
+    case .success(let captured):
+      return captured.image
+    case .failure:
       return nil
     }
-    return captured.image
+  }
+
+  /// The same frame as `captureRunnerFrame`, but carrying the reason it refused, so a required first
+  /// frame — a recording's bootstrap, which sizes the whole writer from it — fails closed with a
+  /// typed code rather than a message. The ongoing pump reads the same result and ignores a refusal
+  /// the way it ignored the `nil` it used to get; only a frame that must exist owes a reason (#2728).
+  func captureRunnerFrameResult(
+    app: XCUIApplication
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+#if os(iOS)
+    return captureObservedScreen(app: app)
 #else
-    var image: RunnerImage?
+    var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
+      .unrenderableImage
+    )
     let capture = {
-      let screenshot = XCUIScreen.main.screenshot()
-      image = screenshot.image
+      let image = XCUIScreen.main.screenshot().image
+      if let cgImage = runnerCGImage(from: image) {
+        // The host display has no resolved-panel facts to report; the recorder reads only the image
+        // and its pixel size, so these two are inert placeholders, not measurements the host scales by.
+        outcome = .success(
+          CapturedAppScreen(
+            image: image,
+            displayID: 0,
+            pixelWidth: cgImage.width,
+            pixelHeight: cgImage.height,
+            pixelsPerPoint: 1
+          )
+        )
+      }
     }
     if Thread.isMainThread {
       capture()
     } else {
       DispatchQueue.main.sync(execute: capture)
     }
-    return image
+    return outcome
 #endif
   }