From 92b8faed36f4c8b263c7dd4baa88d03da831b47d Mon Sep 17 00:00:00 2001 From: dev Date: Tue, 22 Sep 2026 20:00:49 +0200 Subject: [PATCH] fix(recording): export touch overlay at capture size, never black (#2707) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overlay burn-in re-encoded through AVAssetExportPresetMediumQuality, a fixed-canvas preset that caps the long edge at 480px, collapsing any touch recording to ~220x480 (480x220 landscape) and going all-black on long clips, always with exit 0. --hide-touches and an empty gesture list copy the raw capture through, so only the composited path was affected. Both quality tiers now export through the one geometry-preserving preset (HighestQuality); the hardware encoder makes the full-resolution re-encode cheap. The compositor now verifies its own output against the raw before the caller adopts it — resolved track size must match, and a uniformly black track whose raw had visible content is rejected — and throws on failure so the overlay is dropped, the raw kept, and the choice reported as overlayWarning rather than a broken file. Adds an mp4 transform-aware track-size reader and a device-lane assertion that a touched export equals its --hide-touches control. --- .../RecordingScripts/recording-overlay.swift | 170 +++++++++++++++--- docs/adr/0025-foldable-apple-panels.md | 68 +++++-- docs/agents/device-verification.md | 15 +- .../src/flag-definitions-action.ts | 2 +- ...me-screen-recording-finalizer-host.test.ts | 74 +++++++- test/integration/recording-overlay.test.ts | 101 +++++++++++ .../support/recording-inspect.swift | 4 + test/integration/test-helpers.ts | 2 + website/docs/docs/commands.md | 2 +- 9 files changed, 388 insertions(+), 50 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift index 60f036921f..9c8077fb09 100644 --- a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift @@ -11,6 +11,13 @@ let minimumPinchVisibility: CFTimeInterval = 0.5 let swipeVisibilityTail: CFTimeInterval = 0.16 let trailOpacityKeyTimes: [NSNumber] = [0.0, 0.08, 0.62, 1.0] +// A compositor that cannot render frames still completes with an empty track, so a whole frame at +// or below this mean luma counts as black. +let compositedBlackFrameLuma: Double = 1.0 +// Above this mean luma the raw capture is taken to hold visible content worth preserving. Below it +// the recording itself is dark, so a dark composite is the content and not a compositor failure. +let sourceVisibleFrameLuma: Double = 2.0 + struct GestureEnvelope: Decodable { let events: [GestureEvent] } @@ -126,10 +133,10 @@ func run() throws { in: parentLayer ) - // Overlay burn-in forces a full re-encode; medium quality keeps simulator videos readable - // while avoiding very slow highest-quality exports. Pass --quality high to opt into - // the slower highest-quality export. - let presetName = exportPresetName(for: parsedArgs.exportQuality, compatibleWith: composition) + // Overlay burn-in forces a full re-encode. The export has to keep the captured track's own + // dimensions, so it uses the one preset that preserves arbitrary capture geometry; the hardware + // encoder makes the full-resolution re-encode cheap (measured ~1-2s for a 90s 1206x2622 clip). + let presetName = try exportPresetName(compatibleWith: composition) let exporter = try makeRecordingExporter( composition, presetName: presetName, @@ -141,17 +148,19 @@ func run() throws { timeoutMessage: "Touch overlay export timed out.", failureMessage: "Touch overlay export failed." ) + try verifyCompositedOverlay( + input: inputURL, + output: outputURL, + expectedRenderSize: renderSize + ) } func parseArguments( _ arguments: [String] -) throws -> (inputPath: String, outputPath: String, eventsPath: String, exportQuality: ExportQuality) { +) throws -> (inputPath: String, outputPath: String, eventsPath: String) { var inputPath: String? var outputPath: String? var eventsPath: String? - // Export quality defaults to medium so existing callers keep the fast, simulator-friendly - // export. Pass --quality high to opt into a slower highest-quality export. - var exportQuality: ExportQuality = .medium var index = 0 while index < arguments.count { @@ -168,11 +177,13 @@ func parseArguments( eventsPath = try recordingOptionValue(arguments, nextIndex, "--events") index += 2 case "--quality": + // Accepted for CLI parity with the other backends and still validated, but the composited + // export always preserves the captured geometry (see `exportPresetName`), so the tier never + // picks a resolution-capping preset and is not retained. let rawValue = try recordingOptionValue(arguments, nextIndex, "--quality") - guard let parsed = ExportQuality(rawValue: rawValue) else { + guard ExportQuality(rawValue: rawValue) != nil else { throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high") } - exportQuality = parsed index += 2 default: throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)") @@ -184,24 +195,22 @@ func parseArguments( "Usage: recording-overlay.swift --input