fix: speed up web snapshots (#842)

* fix: speed up web snapshots

* ci: stabilize iOS simulator smoke boot

Author: thymikee

.github/actions/boot-ios-test-simulator/action.yml

@@ -9,14 +9,25 @@ inputs:
     description: "Preferred simulator device name"
     required: false
     default: "iPhone 17 Pro"
+  boot-timeout-seconds:
+    description: "Maximum time to wait for simulator bootstatus"
+    required: false
+    default: "300"
+
+outputs:
+  simulator-udid:
+    description: "UDID of the booted iOS simulator"
+    value: ${{ steps.boot.outputs.simulator-udid }}
 
 runs:
   using: "composite"
   steps:
     - name: Resolve and boot iOS test simulator
+      id: boot
       run: |
         set -euo pipefail
-        RUNTIME_TOKEN="SimRuntime.iOS-${{ inputs.runtime-version }}"
+        RUNTIME_VERSION="${{ inputs.runtime-version }}"
+        RUNTIME_TOKEN="SimRuntime.iOS-${RUNTIME_VERSION//./-}"
         export RUNTIME_TOKEN
         export PREFERRED_DEVICE_NAME="${{ inputs.preferred-device-name }}"
         UDID="$(
@@ -40,10 +51,37 @@ runs:
               available.find((device) => device.state === "Booted") ??
               available[0];
             if (!preferred?.udid) process.exit(1);
+            console.error(`Selected ${preferred.name} (${preferred.udid}) from ${preferred.runtime}`);
             process.stdout.write(preferred.udid);
           '
         )"
+        echo "simulator-udid=$UDID" >> "$GITHUB_OUTPUT"
         xcrun simctl shutdown all || true
         xcrun simctl boot "$UDID" || true
-        xcrun simctl bootstatus "$UDID" -b
+        xcrun simctl bootstatus "$UDID" -b &
+        BOOTSTATUS_PID="$!"
+        TIMEOUT_SECONDS="${{ inputs.boot-timeout-seconds }}"
+        BOOT_TIMEOUT_FILE="$(mktemp)"
+        (
+          sleep "$TIMEOUT_SECONDS"
+          if kill -0 "$BOOTSTATUS_PID" 2>/dev/null; then
+            echo "Timed out waiting ${TIMEOUT_SECONDS}s for simulator $UDID to boot" >&2
+            touch "$BOOT_TIMEOUT_FILE"
+            kill "$BOOTSTATUS_PID" 2>/dev/null || true
+          fi
+        ) &
+        BOOT_WATCHDOG_PID="$!"
+        set +e
+        wait "$BOOTSTATUS_PID"
+        BOOTSTATUS_EXIT="$?"
+        set -e
+        kill "$BOOT_WATCHDOG_PID" 2>/dev/null || true
+        wait "$BOOT_WATCHDOG_PID" 2>/dev/null || true
+        if [ -f "$BOOT_TIMEOUT_FILE" ]; then
+          rm -f "$BOOT_TIMEOUT_FILE"
+          xcrun simctl list devices || true
+          exit 1
+        fi
+        rm -f "$BOOT_TIMEOUT_FILE"
+        exit "$BOOTSTATUS_EXIT"
       shell: bash

.github/workflows/ios.yml

@@ -50,6 +50,7 @@ jobs:
           xcuitest-destination: generic/platform=iOS Simulator
 
       - name: Boot iOS test simulator
+        id: ios-simulator
         uses: ./.github/actions/boot-ios-test-simulator
         with:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
@@ -58,12 +59,12 @@ jobs:
       - name: Prepare iOS runner
         run: |
           pnpm clean:daemon
-          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json
+          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json
           pnpm clean:daemon
 
       - name: Run iOS simulator smoke replay
         run: |
-          node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
+          node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml
 
       - name: Run iOS physical device smoke replay
         if: env.IOS_UDID != ''

src/backend.ts

@@ -55,6 +55,7 @@ export type BackendSnapshotResult = {
 } & SnapshotCaptureAnnotations;
 
 export type BackendSnapshotOptions = SnapshotOptions & {
+  includeRects?: boolean;
   outPath?: string;
 };
 

src/cli/commands/web.ts

@@ -13,10 +13,11 @@ export async function runWebCommand(
   const action = positionals[0];
   switch (action) {
     case 'setup': {
+      printWebSetupStart(options.flags.json);
       const status = await setupManagedAgentBrowser({
         stateDir: options.stateDir,
       });
-      printWebResult(options.flags.json, 'Managed web backend installed.', { status });
+      printWebSetupResult(options.flags.json, status);
       return 0;
     }
     case 'doctor': {
@@ -35,6 +36,24 @@ export async function runWebCommand(
   }
 }
 
+function printWebSetupStart(json: boolean | undefined): void {
+  if (json) return;
+  process.stdout.write('Setting up managed agent-browser backend (downloads if needed)...\n');
+}
+
+function printWebSetupResult(
+  json: boolean | undefined,
+  status: Awaited<ReturnType<typeof setupManagedAgentBrowser>>,
+): void {
+  if (json) {
+    printJson({ success: true, data: { status } });
+    return;
+  }
+  process.stdout.write(
+    `Managed web backend installed.\nagent-browser available at: ${status.binaryPath}\n`,
+  );
+}
+
 function printWebResult(json: boolean | undefined, message: string, data: Record<string, unknown>) {
   if (json) {
     printJson({ success: true, data });

src/commands/interaction/runtime/interactions.test.ts

@@ -106,7 +106,10 @@ test('runtime selector interactions fall back to a full snapshot when interactiv
   assert.equal(result.kind, 'selector');
   assert.equal(result.node?.label, 'General');
   assert.deepEqual(calls, [{ x: 160, y: 122 }]);
-  assert.deepEqual(captureOptions, [{ interactiveOnly: true }, { interactiveOnly: false }]);
+  assert.deepEqual(captureOptions, [
+    { interactiveOnly: true, includeRects: true },
+    { interactiveOnly: false, includeRects: true },
+  ]);
 });
 
 test('runtime click keeps distinct tab button centers when iOS reports the tab bar as hittable', async () => {

src/commands/interaction/runtime/resolution.ts

@@ -238,6 +238,7 @@ export async function captureInteractionSnapshot(
   if (!session) throw new AppError('SESSION_NOT_FOUND', 'No active session. Run open first.');
   const result = await runtime.backend.captureSnapshot(toBackendContext(runtime, options), {
     interactiveOnly,
+    includeRects: true,
   });
   const snapshot =
     result.snapshot ??

src/commands/interaction/runtime/selector-read-shared.ts

@@ -36,7 +36,9 @@ export async function requireSnapshotSession(
 export async function captureSelectorSnapshot(
   runtime: AgentDeviceRuntime,
   options: CommandContext & SelectorSnapshotOptions,
-  captureOptions: { updateSession: boolean; scope?: string } = { updateSession: true },
+  captureOptions: { updateSession: boolean; scope?: string; includeRects?: boolean } = {
+    updateSession: true,
+  },
 ): Promise<CapturedSnapshot> {
   const captureSnapshot = runtime.backend.captureSnapshot;
   if (!captureSnapshot) {
@@ -49,6 +51,7 @@ export async function captureSelectorSnapshot(
     depth: options.depth,
     scope: captureOptions.scope ?? options.scope,
     raw: options.raw,
+    includeRects: captureOptions.includeRects,
   });
   const snapshot =
     result.snapshot ??

src/commands/interaction/runtime/selector-read.test.ts

@@ -145,9 +145,33 @@ test('runtime selectors forward public snapshot options to backend capture', asy
     depth: 2,
     scope: 'Login',
     raw: true,
+    includeRects: false,
   });
 });
 
+test('runtime visibility predicates request snapshot rects', async () => {
+  const snapshot = selectorSnapshot();
+  let captureOptions: BackendSnapshotOptions | undefined;
+  const device = createAgentDevice({
+    backend: {
+      platform: 'web',
+      captureSnapshot: async (_context, options) => {
+        captureOptions = options;
+        return { snapshot };
+      },
+    } satisfies AgentDeviceBackend,
+    artifacts: createLocalArtifactAdapter(),
+    sessions: createMemorySessionStore([{ name: 'default', snapshot }]),
+    policy: localCommandPolicy(),
+  });
+
+  await device.selectors.isVisible(selector('label=Continue'), {
+    session: 'default',
+  });
+
+  assert.equal(captureOptions?.includeRects, true);
+});
+
 test('runtime is validates selector predicates', async () => {
   const device = createSelectorDevice(selectorSnapshot());
 

src/commands/interaction/runtime/selector-read.ts

@@ -18,6 +18,7 @@ import { buildSelectorChainForNode } from '../../../utils/selector-build.ts';
 import {
   evaluateIsPredicate,
   isSupportedPredicate,
+  type IsPredicate,
 } from '../../../utils/selector-is-predicates.ts';
 import type {
   ElementTarget,
@@ -256,7 +257,11 @@ export const isCommand: RuntimeCommand<IsCommandOptions, IsCommandResult> = asyn
   if (options.predicate === 'text' && !options.expectedText) {
     throw new AppError('INVALID_ARGS', 'is text requires expected text value');
   }
-  const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true });
+  const includeRects = predicateNeedsRects(options.predicate);
+  const capture = await captureSelectorSnapshot(runtime, options, {
+    updateSession: true,
+    includeRects,
+  });
   const chain = parseSelectorChain(options.selector);
 
   if (options.predicate === 'exists') {
@@ -423,6 +428,10 @@ function sparseSelectorSnapshotError(verdict: SnapshotQualityVerdict): AppError
   });
 }
 
+function predicateNeedsRects(predicate: IsPredicate): boolean {
+  return predicate === 'visible' || predicate === 'hidden';
+}
+
 async function waitForSelector(
   runtime: AgentDeviceRuntime,
   options: WaitCommandOptions,

src/core/dispatch-context.ts

@@ -47,6 +47,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
   snapshotDepth?: number;
   snapshotScope?: string;
   snapshotRaw?: boolean;
+  snapshotIncludeRects?: boolean;
   count?: number;
   intervalMs?: number;
   delayMs?: number;