feat(extension): report editable element ids alongside the a11y snapshot

packages/extension/tests/a11y-editable-ids.test.ts

@@ -0,0 +1,52 @@
+import type { Protocol } from "devtools-protocol";
+import { describe, expect, it } from "vitest";
+import type { CDPSessionLike } from "../understudy/cdp.js";
+import { a11yForFrame } from "../understudy/a11y/snapshot/a11yTree.js";
+
+function axNode(
+  nodeId: string,
+  backendDOMNodeId: number,
+  role: string,
+  name: string,
+  extra: Partial<Protocol.Accessibility.AXNode> = {},
+): Protocol.Accessibility.AXNode {
+  return {
+    nodeId,
+    backendDOMNodeId,
+    ignored: false,
+    role: { type: "role", value: role },
+    name: { type: "computedString", value: name },
+    ...extra,
+  };
+}
+
+function editable(value: string): Protocol.Accessibility.AXProperty[] {
+  return [{ name: "editable", value: { type: "token", value } }];
+}
+
+describe("a11yForFrame editable ids", () => {
+  it("reports editable nodes by encoded id without changing the outline", async () => {
+    const nodes = [
+      axNode("1", 1, "RootWebArea", "Editor", { childIds: ["2", "3", "4", "5"] }),
+      axNode("2", 2, "textbox", "Title", { parentId: "1", properties: editable("plaintext") }),
+      // A contenteditable div: its role says nothing about being typeable.
+      axNode("3", 3, "generic", "Body", { parentId: "1", properties: editable("richtext") }),
+      axNode("4", 4, "button", "Save", { parentId: "1" }),
+      axNode("5", 5, "textbox", "Hidden", { parentId: "1", properties: editable("plaintext") }),
+    ];
+    const session = {
+      send: async (method: string) => (method === "Accessibility.getFullAXTree" ? { nodes } : {}),
+    } as unknown as CDPSessionLike;
+
+    const result = await a11yForFrame(session, undefined, {
+      tagNameMap: {},
+      scrollableMap: {},
+      encode: (backendNodeId) => `0-${backendNodeId}`,
+      isIgnoredBackendNode: (backendNodeId) => backendNodeId === 5,
+    });
+
+    expect(result.editableIds).toEqual(["0-2", "0-3"]);
+    expect(result.outline).toContain("button: Save");
+    expect(result.outline).not.toContain("editable");
+  });
+});

packages/extension/types/private/snapshot.ts

@@ -35,6 +35,11 @@ export type HybridSnapshot = {
   combinedXpathMap: Record<string, string>;
   /** EncodedId -> URL extracted from AX properties. */
   combinedUrlMap: Record<string, string>;
+  /**
+   * EncodedIds the browser reports as editable (inputs, and contenteditable
+   * nodes whose role says nothing about it). Not part of the outline.
+   */
+  combinedEditableIds?: string[];
   /** Per-frame payloads expose the original relative data for debugging. */
   perFrame?: PerFrameSnapshot[];
 };
@@ -44,6 +49,7 @@ export type PerFrameSnapshot = {
   outline: string;
   xpathMap: Record<string, string>;
   urlMap: Record<string, string>;
+  editableIds?: string[];
 };
 
 /**
@@ -66,6 +72,7 @@ export type FrameDomMaps = {
   xpathMap: Record<string, string>;
   scrollableMap: Record<string, boolean>;
   urlMap: Record<string, string>;
+  editableIds?: string[];
 };
 
 export type ResolvedLocation = {
@@ -120,6 +127,7 @@ export type A11yOptions = {
 export type AccessibilityTreeResult = {
   outline: string;
   urlMap: Record<string, string>;
+  editableIds?: string[];
   scopeApplied: boolean;
 };
 

packages/extension/understudy/a11y/snapshot/a11yTree.ts

@@ -94,11 +94,19 @@ export async function a11yForFrame(
     urlMap[enc] = url;
   }
 
+  const editableIds: string[] = [];
+  for (const n of filteredNodes) {
+    if (typeof n.backendDOMNodeId !== "number") continue;
+    const editable = n.properties?.find((property) => property.name === "editable")?.value?.value;
+    if (editable === "richtext" || editable === "plaintext")
+      editableIds.push(opts.encode(n.backendDOMNodeId));
+  }
+
   const decorated = decorateRoles(filteredNodes, opts);
   const { tree } = await buildHierarchicalTree(decorated, opts);
 
   const simplified = tree.map((n) => formatTreeLine(n)).join("\n");
-  return { outline: simplified.trimEnd(), urlMap, scopeApplied };
+  return { outline: simplified.trimEnd(), urlMap, scopeApplied, editableIds };
 }
 
 export function decorateRoles(

packages/extension/understudy/a11y/snapshot/capture.ts

@@ -224,22 +224,26 @@ export async function tryScopedSnapshot(
       sameSessionAsParent,
     );
 
-    const { outline, urlMap, scopeApplied } = await a11yForFrame(owningSess, targetFrameId, {
-      focusLocator: tailSelector
-        ? {
-            selector: tailSelector,
-            ...(focusNth === undefined ? {} : { nth: focusNth }),
-          }
-        : undefined,
-      isIgnoredBackendNode: makeIsIgnoredBackendNode(
-        targetFrameId,
-        ownerSessionIndexForFrame(page, targetFrameId, sessionToIndex),
-        exclusionIntervalsByFrame,
-      ),
-      tagNameMap,
-      scrollableMap,
-      encode: (backendNodeId) => `${page.getOrdinal(targetFrameId)}-${backendNodeId}`,
-    });
+    const { outline, urlMap, scopeApplied, editableIds } = await a11yForFrame(
+      owningSess,
+      targetFrameId,
+      {
+        focusLocator: tailSelector
+          ? {
+              selector: tailSelector,
+              ...(focusNth === undefined ? {} : { nth: focusNth }),
+            }
+          : undefined,
+        isIgnoredBackendNode: makeIsIgnoredBackendNode(
+          targetFrameId,
+          ownerSessionIndexForFrame(page, targetFrameId, sessionToIndex),
+          exclusionIntervalsByFrame,
+        ),
+        tagNameMap,
+        scrollableMap,
+        encode: (backendNodeId) => `${page.getOrdinal(targetFrameId)}-${backendNodeId}`,
+      },
+    );
 
     const scopedXpathMap: Record<string, string> = {};
     const isIgnoredBackendNode = makeIsIgnoredBackendNode(
@@ -275,12 +279,14 @@ export async function tryScopedSnapshot(
       combinedTree: wellFormedOutline,
       combinedXpathMap: scopedXpathMap,
       combinedUrlMap: scopedUrlMap,
+      combinedEditableIds: editableIds ?? [],
       perFrame: [
         {
           frameId: targetFrameId,
           outline: wellFormedOutline,
           xpathMap,
           urlMap,
+          editableIds: editableIds ?? [],
         },
       ],
     };
@@ -381,15 +387,15 @@ export async function collectPerFrameMaps(
       if (idx.scrollByBe.get(be)) scrollableMap[key] = true;
     }
 
-    const { outline, urlMap } = await a11yForFrame(sess, frameId, {
+    const { outline, urlMap, editableIds } = await a11yForFrame(sess, frameId, {
       isIgnoredBackendNode,
       tagNameMap,
       scrollableMap,
       encode: (backendNodeId) => `${page.getOrdinal(frameId)}-${backendNodeId}`,
     });
 
     perFrameOutlines.push({ frameId, outline });
-    perFrameMaps.set(frameId, { tagNameMap, xpathMap, scrollableMap, urlMap });
+    perFrameMaps.set(frameId, { tagNameMap, xpathMap, scrollableMap, urlMap, editableIds });
   }
 
   return { perFrameMaps, perFrameOutlines };
@@ -835,13 +841,15 @@ export function mergeFramesIntoSnapshot(
     combinedTree,
     combinedXpathMap,
     combinedUrlMap,
+    combinedEditableIds: [...perFrameMaps.values()].flatMap((maps) => maps.editableIds ?? []),
     perFrame: perFrameOutlines.map(({ frameId, outline }) => {
       const maps = perFrameMaps.get(frameId);
       return {
         frameId,
         outline: toWellFormed(outline),
         xpathMap: maps?.xpathMap ?? {},
         urlMap: maps?.urlMap ?? {},
+        editableIds: maps?.editableIds ?? [],
       };
     }),
   };