Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions examples/test-app/app.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@
"version": "1.0.0",
"orientation": "default",
"userInterfaceStyle": "automatic",
"newArchEnabled": true,
"plugins": ["expo-router"],
"ios": {
"supportsTablet": true,
"bundleIdentifier": "com.callstack.agentdevicelab"
},
"android": {
"package": "com.callstack.agentdevicelab",
"edgeToEdgeEnabled": true,
"predictiveBackGestureEnabled": false
}
}
Expand Down
2 changes: 1 addition & 1 deletion examples/test-app/app/_layout.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ThemeProvider } from '@react-navigation/native';
import { Stack } from 'expo-router';
import { ThemeProvider } from 'expo-router/react-navigation';
import { StatusBar } from 'expo-status-bar';
import { StyleSheet } from 'react-native';
import { GestureHandlerRootView } from 'react-native-gesture-handler';
Expand Down
23 changes: 11 additions & 12 deletions examples/test-app/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,20 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@expo/metro-runtime": "~55.0.9",
"@react-navigation/native": "^7.2.2",
"expo": "~55.0.12",
"expo-constants": "55.0.12",
"expo-linking": "55.0.11",
"expo-router": "~55.0.11",
"expo-status-bar": "~55.0.5",
"react": "19.2.0",
"react-native": "0.83.4",
"@expo/metro-runtime": "~56.0.15",
"expo": "~56.0.12",
"expo-constants": "56.0.18",
"expo-linking": "56.0.14",
"expo-router": "~56.2.11",
"expo-status-bar": "~56.0.4",
"react": "19.2.3",
"react-native": "0.85.3",
"react-native-gesture-handler": "^2.31.2",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0"
"react-native-safe-area-context": "~5.7.0",
"react-native-screens": "~4.25.2"
},
"devDependencies": {
"@types/react": "~19.2.2",
"typescript": "~5.9.2"
"typescript": "~6.0.3"
}
}
2,788 changes: 1,113 additions & 1,675 deletions examples/test-app/pnpm-lock.yaml

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion examples/test-app/src/screens/SettingsScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ export function SettingsScreen(props: SettingsScreenProps) {

{props.diagnosticsExpanded ? (
<View style={styles.diagnosticsBody} testID="diagnostics-body">
<Text style={styles.diagnosticsText}>Build: expo-sdk-55 / lab-fixture-1</Text>
<Text style={styles.diagnosticsText}>Build: expo-sdk-56 / lab-fixture-1</Text>
<Text style={styles.diagnosticsText}>API mode: mock network with retry simulation</Text>
<Text style={styles.diagnosticsText}>
Device target hint: use this accordion for get-text and exists assertions
Expand Down
2 changes: 1 addition & 1 deletion examples/test-app/src/theme.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { DarkTheme, DefaultTheme, type Theme } from '@react-navigation/native';
import { DarkTheme, DefaultTheme, type Theme } from 'expo-router/react-navigation';
import { useColorScheme, type ColorSchemeName } from 'react-native';

export interface AppColors {
Expand Down
1 change: 1 addition & 0 deletions src/core/interactors/android.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor {
'snapshot_capture',
async () =>
await snapshotAndroid(device, {
appBundleId: options?.appBundleId,
interactiveOnly: options?.interactiveOnly,
depth: options?.depth,
scope: options?.scope,
Expand Down
28 changes: 28 additions & 0 deletions src/platforms/android/__tests__/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,34 @@ test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible ov
);
});

test('parseUiHierarchy keeps React Native content under a transparent Expo tools overlay', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.TextView" text="Agent Device Tester" bounds="[24,80][280,140]" enabled="true" visible-to-user="true" drawing-order="1"/>
<node class="android.widget.Button" text="Gesture lab" bounds="[24,180][280,240]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
<node class="android.widget.ImageView" content-desc="Tools" bounds="[320,80][360,120]" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;

const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Agent Device Tester'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Gesture lab'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Tools'),
true,
);
});

test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
const xml =
"<hierarchy><node class='android.widget.TextView' hint-text='Spoofed' text='Actual' bounds='[10,20][110,60]'/></hierarchy>";
Expand Down
239 changes: 236 additions & 3 deletions src/platforms/android/__tests__/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,15 @@ const installedHelperProbe = {
stderr: '',
};

function snapshotAndroidWithHelper(helperAdb: AndroidAdbExecutor) {
function snapshotAndroidWithHelper(
helperAdb: AndroidAdbExecutor,
options: Omit<
NonNullable<Parameters<typeof snapshotAndroid>[1]>,
'helperAdb' | 'helperArtifact'
> = {},
) {
return snapshotAndroid(device, {
...options,
helperAdb,
helperArtifact,
});
Expand Down Expand Up @@ -316,10 +323,11 @@ test('screenshotAndroid throws when PNG payload is truncated', async () => {

function helperOutput(
xml: string,
options: { truncated?: boolean; nodeCount?: number } = {},
options: { truncated?: boolean; nodeCount?: number; windowCount?: number } = {},
): string {
const truncated = options.truncated ?? false;
const nodeCount = options.nodeCount ?? 1;
const windowCount = options.windowCount ?? 1;
return [
'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1',
'INSTRUMENTATION_STATUS: helperApiVersion=1',
Expand All @@ -338,14 +346,69 @@ function helperOutput(
'INSTRUMENTATION_RESULT: maxNodes=5000',
'INSTRUMENTATION_RESULT: rootPresent=true',
'INSTRUMENTATION_RESULT: captureMode=interactive-windows',
'INSTRUMENTATION_RESULT: windowCount=1',
`INSTRUMENTATION_RESULT: windowCount=${windowCount}`,
`INSTRUMENTATION_RESULT: nodeCount=${nodeCount}`,
`INSTRUMENTATION_RESULT: truncated=${truncated}`,
'INSTRUMENTATION_RESULT: elapsedMs=12',
'INSTRUMENTATION_CODE: 0',
].join('\n');
}

function androidSystemWindowOnlyXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node window-index="0" window-type="3" window-layer="30" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node content-desc="Back" class="android.widget.ImageButton" package="com.android.systemui" bounds="[0,792][96,844]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
' <node content-desc="Home" class="android.widget.ImageButton" package="com.android.systemui" bounds="[147,792][243,844]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}

function androidFabricAppXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node index="0" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true">',
' <node index="0" text="Fabric dashboard" resource-id="io.example.fabric:id/title" class="android.widget.TextView" package="io.example.fabric" bounds="[24,96][280,140]" enabled="true" />',
' <node index="1" text="Open details" class="android.widget.Button" package="io.example.fabric" bounds="[24,180][220,236]" clickable="true" enabled="true" focusable="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}

function androidContentPoorFabricAppWindowXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node index="0" class="androidx.compose.ui.platform.ComposeView" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" />',
' </node>',
' <node window-index="1" window-type="3" window-layer="30" window-active="false" window-focused="false" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
' <node content-desc="Battery" class="android.widget.ImageView" package="com.android.systemui" bounds="[340,4][370,20]" enabled="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}

function androidContentPoorExpoToolsOverlayXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node index="0" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
' <node text="7:52" resource-id="com.android.systemui:id/clock" class="android.widget.TextView" package="com.android.systemui" bounds="[12,4][54,20]" enabled="true" visible-to-user="true" />',
' <node content-desc="Battery 100 percent." resource-id="com.android.systemui:id/battery" class="android.widget.LinearLayout" package="com.android.systemui" bounds="[340,4][380,20]" enabled="true" visible-to-user="true" />',
' </node>',
' <node index="1" class="android.widget.FrameLayout" package="host.exp.exponent" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node index="0" class="androidx.compose.ui.platform.ComposeView" package="host.exp.exponent" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" />',
' <node index="1" text="Agent Device Tester" class="android.widget.TextView" package="host.exp.exponent" bounds="[0,0][0,0]" enabled="true" visible-to-user="false" />',
' <node index="1" text="Tools" class="android.widget.ImageView" package="host.exp.exponent" bounds="[20,760][64,804]" enabled="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}

function mockScreenshotEvents(events: string[]): void {
mockRunCmd.mockImplementation(async (_cmd, args) => {
if (args.includes('exec-out')) {
Expand Down Expand Up @@ -730,6 +793,176 @@ test('snapshotAndroid falls back to stock uiautomator when helper fails', async
assert.equal(mockRunCmd.mock.calls.length, 0);
});

test('snapshotAndroid falls back to stock uiautomator when helper returns only system windows', async () => {
const adbCalls: string[][] = [];
const helperXml = androidSystemWindowOnlyXml();
const stockXml = androidFabricAppXml();
const helperAdb: AndroidAdbExecutor = async (args) => {
adbCalls.push(args);
if (args.includes('--show-versioncode')) return installedHelperProbe;
if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop') {
return { exitCode: 0, stdout: '', stderr: '' };
}
if (args.includes('instrument')) {
return { exitCode: 0, stdout: helperOutput(helperXml, { nodeCount: 3 }), stderr: '' };
}
if (args.includes('exec-out')) {
return { exitCode: 0, stdout: stockXml, stderr: '' };
}
throw new Error(`unexpected helper adb args: ${args.join(' ')}`);
};

const result = await snapshotAndroidWithHelper(helperAdb);

assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned only non-application windows',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Open details'),
true,
);
assert.equal(
adbCalls.some(
(args) => args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper',
),
true,
);
assert.equal(
adbCalls.some((args) => args.includes('exec-out')),
true,
);
});

test('snapshotAndroid falls back to stock uiautomator when helper returns no nodes', async () => {
const helperXml = '<?xml version="1.0" encoding="UTF-8"?><hierarchy rotation="0"></hierarchy>';
const stockXml = androidFabricAppXml();
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 0 }),
stderr: '',
}),
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
});

const result = await snapshotAndroidWithHelper(helperAdb);

assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned no accessibility nodes',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
});

test('snapshotAndroid falls back to stock uiautomator when foreground app window lacks content', async () => {
const helperXml = androidContentPoorFabricAppWindowXml();
const stockXml = androidFabricAppXml();
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 4, windowCount: 2 }),
stderr: '',
}),
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
});

const result = await snapshotAndroidWithHelper(helperAdb, {
appBundleId: 'io.example.fabric',
});

assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned insufficient foreground app content',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Open details'),
true,
);
});

test('snapshotAndroid falls back to stock uiautomator when standalone helper sees only an app overlay', async () => {
const helperXml = androidContentPoorExpoToolsOverlayXml();
const stockXml = androidFabricAppXml();
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 4, windowCount: 2 }),
stderr: '',
}),
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
});

const result = await snapshotAndroidWithHelper(helperAdb);

assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned insufficient application window content',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Open details'),
true,
);
});

test('snapshotAndroid keeps helper output when application and system windows are both present', async () => {
let stockAttempted = false;
const helperXml = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node text="Fabric dashboard" class="android.widget.TextView" package="io.example.fabric" bounds="[24,96][260,140]" enabled="true" visible-to-user="true" />',
' <node text="Open details" class="android.widget.Button" package="io.example.fabric" bounds="[24,180][220,236]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
' </node>',
' <node window-index="1" window-type="3" window-layer="20" window-active="false" window-focused="false" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
' <node content-desc="Battery" class="android.widget.ImageView" package="com.android.systemui" bounds="[340,4][370,20]" enabled="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 4 }),
stderr: '',
}),
stock: async () => {
stockAttempted = true;
throw new Error('stock fallback should not run');
},
});

const result = await snapshotAndroidWithHelper(helperAdb, {
appBundleId: 'io.example.fabric',
});

assert.equal(result.androidSnapshot.backend, 'android-helper');
assert.equal(result.androidSnapshot.fallbackReason, undefined);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(stockAttempted, false);
});

test('snapshotAndroid emits fallback and stock capture diagnostics', async () => {
const stockXml =
'<?xml version="1.0" encoding="UTF-8"?><hierarchy><node text="stock" bounds="[0,0][10,10]" /></hierarchy>';
Expand Down
Loading
Loading