fix(desktop): expose browser control state to agents

Author: waleedlatif1

apps/desktop/src/main/browser-agent/page-functions.test.ts

@@ -625,6 +625,29 @@ describe('collectSnapshot', () => {
     expect(outlineOf(collectSnapshot())).toContain('gridcell "party-parrot"')
   })
 
+  it('reports native and ARIA control state in the snapshot', () => {
+    document.body.innerHTML = `
+      <input type="checkbox" aria-label="Email alerts" />
+      <input type="radio" aria-label="Weekly" checked />
+      <button aria-expanded="false" aria-pressed="true">Filters</button>
+      <div role="switch" aria-label="Dark mode" aria-checked="mixed"></div>
+      <div role="tab" aria-selected="true">Activity</div>
+      <textarea aria-label="Notes" readonly required></textarea>
+      <div role="textbox" aria-label="Summary" aria-readonly="true" aria-required="true"></div>
+    `
+    for (const element of document.body.children) visible(element as HTMLElement)
+
+    const outline = outlineOf(collectSnapshot())
+
+    expect(outline).toMatch(/checkbox "Email alerts" \[ref=\d+\] unchecked/)
+    expect(outline).toMatch(/radio "Weekly" \[ref=\d+\] checked/)
+    expect(outline).toMatch(/button "Filters" \[ref=\d+\] aria-expanded=false aria-pressed=true/)
+    expect(outline).toMatch(/switch "Dark mode" \[ref=\d+\] aria-checked=mixed/)
+    expect(outline).toMatch(/tab "Activity" \[ref=\d+\] aria-selected=true/)
+    expect(outline).toMatch(/textbox "Notes" \[ref=\d+\] readonly required/)
+    expect(outline).toMatch(/textbox "Summary" \[ref=\d+\] aria-readonly aria-required/)
+  })
+
   it('does not duplicate every descendant of an inherited pointer target', () => {
     document.body.innerHTML = `
       <div style="cursor: pointer">

apps/desktop/src/main/browser-agent/page-functions.ts

@@ -448,18 +448,42 @@ export function collectSnapshot(startingElementId = 0): unknown {
       // like any other. Redaction above is realm-safe and runs first, so
       // widening this cannot expose a credential field.
       const value = (el as HTMLInputElement).value
-      if (tag === 'INPUT' && (el as HTMLInputElement).type === 'file') {
+      const inputType = tag === 'INPUT' ? (el as HTMLInputElement).type : ''
+      if (inputType === 'file') {
         parts.push('upload-unsupported')
-      } else if (value && isSensitiveValueField(el)) parts.push('value-withheld')
-      else if (value) parts.push(`value=${quote(cut(String(value), 120))}`)
+      } else if (inputType !== 'checkbox' && inputType !== 'radio') {
+        if (value && isSensitiveValueField(el)) parts.push('value-withheld')
+        else if (value) parts.push(`value=${quote(cut(String(value), 120))}`)
+      }
     }
     if (tag === 'A') {
       const href = el.getAttribute('href')
       if (href) parts.push(`href=${quote(cut(href, 200))}`)
     }
     if ((el as HTMLInputElement).disabled === true) parts.push('disabled')
     if (el.getAttribute('aria-disabled') === 'true') parts.push('aria-disabled')
-    if ((el as HTMLInputElement).checked === true) parts.push('checked')
+    if (el.getAttribute('aria-readonly') === 'true') parts.push('aria-readonly')
+    if (el.getAttribute('aria-required') === 'true') parts.push('aria-required')
+    if (tag === 'INPUT') {
+      const input = el as HTMLInputElement
+      if (input.type === 'checkbox' || input.type === 'radio') {
+        parts.push(input.checked ? 'checked' : 'unchecked')
+      }
+      if (input.readOnly) parts.push('readonly')
+      if (input.required) parts.push('required')
+    } else if (tag === 'TEXTAREA') {
+      const textarea = el as HTMLTextAreaElement
+      if (textarea.readOnly) parts.push('readonly')
+      if (textarea.required) parts.push('required')
+    } else if (tag === 'SELECT' && (el as HTMLSelectElement).required) {
+      parts.push('required')
+    }
+    for (const attribute of ['aria-checked', 'aria-expanded', 'aria-pressed', 'aria-selected']) {
+      const value = el.getAttribute(attribute)
+      if (value === 'true' || value === 'false' || value === 'mixed') {
+        parts.push(`${attribute}=${value}`)
+      }
+    }
     const suffix = parts.length > 0 ? ` ${parts.join(' ')}` : ''
     const lineIndex = lines.length
     if (push(`${indent}- ${role} ${quote(name)} [ref=${id}]${suffix}`)) {