feat(desktop): add semantic browser controls

Author: waleedlatif1

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -501,18 +501,25 @@ describe('browser-agent screenshot capture', () => {
       return Promise.resolve(undefined)
     })
     const resized = {
+      getSize: vi.fn(() => ({ width: 1024, height: 512 })),
       toJPEG: vi.fn(() => Buffer.from('resized')),
     }
+    const cropped = {
+      getSize: vi.fn(() => ({ width: 400, height: 200 })),
+      resize: vi.fn(() => resized),
+      toJPEG: vi.fn(() => Buffer.from('cropped')),
+    }
     // Shared module-level mock: without this, a later fixture reads the
     // earlier test's decoded image.
     vi.mocked(nativeImage.createFromBuffer).mockReset()
     vi.mocked(nativeImage.createFromBuffer).mockReturnValue({
       isEmpty: vi.fn(() => imageSize === null),
       getSize: vi.fn(() => imageSize ?? { width: 0, height: 0 }),
+      crop: vi.fn(() => cropped),
       resize: vi.fn(() => resized),
       toJPEG: vi.fn(() => Buffer.alloc(0)),
     } as unknown as ReturnType<typeof nativeImage.createFromBuffer>)
-    return { contents, resized }
+    return { contents, resized, cropped }
   }
 
   function screenshotParams(contents: WebContents): Record<string, unknown> {
@@ -531,6 +538,23 @@ describe('browser-agent screenshot capture', () => {
     expect(screenshotParams(contents)).not.toHaveProperty('clip')
   })
 
+  it('crops the decoded image in memory without sending a CDP clip', async () => {
+    const { contents, cropped } = captureFixture({ width: 4096, height: 2048 })
+
+    const shot = await captureScreenshot(contents, { x: 100, y: 50, width: 200, height: 100 })
+
+    const image = vi.mocked(nativeImage.createFromBuffer).mock.results[0].value
+    expect(screenshotParams(contents)).not.toHaveProperty('clip')
+    expect(image.crop).toHaveBeenCalledWith({ x: 200, y: 100, width: 400, height: 200 })
+    expect(cropped.resize).not.toHaveBeenCalled()
+    expect(shot).toEqual({
+      dataUrl: `data:image/jpeg;base64,${Buffer.from('cropped').toString('base64')}`,
+      scale: 2,
+      viewport: { width: 2048, height: 1024 },
+      imageSize: { width: 400, height: 200 },
+    })
+  })
+
   /**
    * A 2048px CSS viewport bounded to 1024px is scale 0.5, and the capture
    * arrives at device resolution (4096px on a 2x display). The resize is what
@@ -597,6 +621,24 @@ describe('browser-agent screenshot capture', () => {
     expect(shot.imageSize).toEqual({ width: 1024, height: 512 })
   })
 
+  it('refuses element cropping without verified CSS viewport metrics', async () => {
+    const { contents } = captureFixture({ width: 1024, height: 512 })
+    vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => {
+      if (method === 'Page.getLayoutMetrics') {
+        return Promise.resolve({ layoutViewport: { clientWidth: 2048, clientHeight: 1024 } })
+      }
+      return Promise.resolve(undefined)
+    })
+
+    await expect(
+      captureScreenshot(contents, { x: 10, y: 10, width: 100, height: 50 })
+    ).rejects.toThrow(/CSS viewport/)
+    expect(contents.debugger.sendCommand).not.toHaveBeenCalledWith(
+      'Page.captureScreenshot',
+      expect.anything()
+    )
+  })
+
   it('accepts stable finite scroll offsets around the capture', async () => {
     const { contents } = captureFixture({ width: 1024, height: 512 })
     vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => {

apps/desktop/src/main/browser-agent/cdp.ts

@@ -404,6 +404,13 @@ export interface ScreenshotCapture {
   imageSize: ScreenshotSize | null
 }
 
+export interface ScreenshotClip {
+  x: number
+  y: number
+  width: number
+  height: number
+}
+
 function screenshotViewportMetrics(
   metrics: {
     cssLayoutViewport?: CdpViewport
@@ -466,7 +473,10 @@ function sameScreenshotViewport(
  * (cssX = imageX / scale) — including on a 2x display, where an unclipped
  * capture arrives at device resolution and this is what brings it back down.
  */
-export async function captureScreenshot(contents: WebContents): Promise<ScreenshotCapture> {
+export async function captureScreenshot(
+  contents: WebContents,
+  clip?: ScreenshotClip
+): Promise<ScreenshotCapture> {
   const metrics = await send<{
     cssLayoutViewport?: CdpViewport
     layoutViewport?: CdpViewport
@@ -478,6 +488,9 @@ export async function captureScreenshot(contents: WebContents): Promise<Screensh
   const cssWidth = metrics?.cssLayoutViewport?.clientWidth ?? 0
   const cssHeight = metrics?.cssLayoutViewport?.clientHeight ?? 0
   const cssViewport = cssWidth > 0 && cssHeight > 0 ? { width: cssWidth, height: cssHeight } : null
+  if (clip && !cssViewport) {
+    throw new Error('A CSS viewport is required for element screenshot cropping')
+  }
   const scale =
     width > 0 && height > 0 ? Math.min(1, MAX_SCREENSHOT_EDGE / Math.max(width, height)) : 1
 
@@ -499,8 +512,49 @@ export async function captureScreenshot(contents: WebContents): Promise<Screensh
   const image = nativeImage.createFromBuffer(Buffer.from(result.data, 'base64'))
   const size = image.isEmpty() ? { width: 0, height: 0 } : image.getSize()
   if (size.width === 0 || size.height === 0) {
+    if (clip) throw new Error('The screenshot could not be decoded for element cropping')
     return { dataUrl: captured, scale, viewport: cssViewport, imageSize: null }
   }
+  if (clip && cssViewport) {
+    const xScale = size.width / cssViewport.width
+    const yScale = size.height / cssViewport.height
+    const cropX = Math.max(0, Math.floor(clip.x * xScale))
+    const cropY = Math.max(0, Math.floor(clip.y * yScale))
+    const cropRight = Math.min(size.width, Math.ceil((clip.x + clip.width) * xScale))
+    const cropBottom = Math.min(size.height, Math.ceil((clip.y + clip.height) * yScale))
+    if (cropRight <= cropX || cropBottom <= cropY) {
+      throw new Error('The requested screenshot element is outside the current viewport')
+    }
+    const cropped = image.crop({
+      x: cropX,
+      y: cropY,
+      width: cropRight - cropX,
+      height: cropBottom - cropY,
+    })
+    const croppedSize = cropped.getSize()
+    if (croppedSize.width === 0 || croppedSize.height === 0) {
+      throw new Error('The requested screenshot element produced an empty crop')
+    }
+    const cropScale = Math.min(
+      1,
+      MAX_SCREENSHOT_EDGE / Math.max(croppedSize.width, croppedSize.height)
+    )
+    const output =
+      cropScale < 1
+        ? cropped.resize({
+            width: Math.round(croppedSize.width * cropScale),
+            height: Math.round(croppedSize.height * cropScale),
+            quality: 'good',
+          })
+        : cropped
+    const outputSize = output.getSize()
+    return {
+      dataUrl: `data:image/jpeg;base64,${output.toJPEG(SCREENSHOT_QUALITY).toString('base64')}`,
+      scale: outputSize.width / clip.width,
+      viewport: cssViewport,
+      imageSize: outputSize,
+    }
+  }
   if (size.width === targetWidth && size.height === targetHeight) {
     return { dataUrl: captured, scale, viewport: cssViewport, imageSize: size }
   }

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -187,6 +187,31 @@ describe('executeTool', () => {
     expect(second.result).toMatchObject({ tabs: [] })
   })
 
+  it('lists safe download metadata without opening a page', async () => {
+    const result = await driver.executeTool('chat-test', 'browser_list_downloads', {})
+
+    expect(result).toEqual({
+      ok: true,
+      result: { scopeId: 'chat-test', downloads: [] },
+    })
+    expect(session.peekTabsState().tabs).toEqual([])
+  })
+
+  it('reloads the active tab and waits for its load boundary', async () => {
+    await driver.executeTool('chat-test', 'browser_open_tab', {})
+    const contents = session.requireTab().view.webContents
+    vi.useFakeTimers()
+    try {
+      const reload = driver.executeTool('chat-test', 'browser_reload', {})
+      await vi.advanceTimersByTimeAsync(500)
+
+      await expect(reload).resolves.toMatchObject({ ok: true })
+      expect(contents.reload).toHaveBeenCalledOnce()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('keeps a takeover pending when the clock advances beyond twelve hours', async () => {
     await driver.executeTool('chat-test', 'browser_open_tab', {})
     vi.useFakeTimers()
@@ -3197,6 +3222,163 @@ describe('credential protection', () => {
     expect(result.error).toMatch(/same point/)
   })
 
+  it('finds only fresh ref-bearing snapshot lines with literal text matching', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      collectSnapshot: {
+        url: 'https://example.com/login',
+        title: 'Example',
+        outline:
+          '- button "Continue [ref\u200b=999]" [ref=4]\n- heading "Continue without a ref"\n- button "Other" [ref=5]',
+        truncated: false,
+        refIds: [4, 5],
+        refLineIndexes: { 4: 0, 5: 2 },
+        nextElementId: 6,
+      },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_find', { query: 'continue' })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: {
+        matches: [{ elementId: 4 }],
+        totalMatches: 1,
+        truncated: false,
+      },
+    })
+  })
+
+  it('does not click a checkable control already in the requested state', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      readCheckableElementState: {
+        checked: true,
+        disabled: false,
+        readOnly: false,
+        kind: 'input:checkbox',
+      },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_set_checked', {
+      elementId: 0,
+      checked: true,
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: { checked: true, changed: false, dispatched: false },
+    })
+    expect(cdpCalls(contents, 'Input.dispatchMouseEvent')).toHaveLength(0)
+  })
+
+  it('uses the trusted click path and verifies a changed checkable control', async () => {
+    const contents = await openPage()
+    let stateReads = 0
+    vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
+      if (isPageCall(expression, 'readCheckableElementState')) {
+        stateReads++
+        return Promise.resolve({
+          checked: stateReads > 1,
+          disabled: false,
+          readOnly: false,
+          kind: 'input:checkbox',
+        })
+      }
+      if (isPageCall(expression, 'clickElement')) {
+        return Promise.resolve({ dispatched: false, x: 24, y: 48, element: 'Checkbox' })
+      }
+      if (isPageCall(expression, 'readPageActionState')) {
+        return Promise.resolve({
+          url: 'https://example.com/login',
+          title: 'Example',
+          focus: 'body',
+          mutationRevision: 0,
+          dialogs: [],
+          scroll: [0],
+        })
+      }
+      if (isPageCall(expression, 'readActiveElementState')) return Promise.resolve({})
+      return Promise.resolve(undefined)
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_set_checked', {
+      elementId: 0,
+      checked: true,
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: { checked: true, changed: true, dispatched: true, trusted: true },
+    })
+    expect(cdpCalls(contents, 'Input.dispatchMouseEvent')).toHaveLength(3)
+  })
+
+  it('waits for URL and semantic element state together', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      readPageActionState: {
+        targetState: { present: true, rendered: true, disabled: false },
+      },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_wait_for', {
+      urlContains: '/login',
+      elementId: 0,
+      state: 'enabled',
+      timeoutMs: 1_000,
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: { found: true, matched: ['url', 'element'] },
+    })
+  })
+
+  it('crops an element screenshot without changing the live viewport', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      getElementScreenshotRect: {
+        x: 20,
+        y: 30,
+        width: 200,
+        height: 100,
+        element: 'button',
+        refRecovered: false,
+      },
+    })
+    const capture = vi.spyOn(cdp, 'captureScreenshot').mockResolvedValue({
+      dataUrl: 'data:image/jpeg;base64,c2lt',
+      scale: 1,
+      viewport: { width: 800, height: 600 },
+      imageSize: { width: 200, height: 100 },
+    })
+
+    try {
+      const result = await driver.executeTool('chat-test', 'browser_screenshot', { elementId: 0 })
+
+      expect(capture).toHaveBeenCalledWith(contents, { x: 20, y: 30, width: 200, height: 100 })
+      expect(result).toMatchObject({
+        ok: true,
+        result: { element: 'button', clip: { x: 20, y: 30, width: 200, height: 100 } },
+      })
+    } finally {
+      capture.mockRestore()
+    }
+  })
+
+  it('zooms by a standard step and invalidates existing element refs', async () => {
+    const contents = await openPage()
+
+    const zoomed = await driver.executeTool('chat-test', 'browser_zoom', { action: 'in' })
+    const staleRef = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 })
+
+    expect(zoomed).toMatchObject({ ok: true, result: { action: 'in' } })
+    expect(contents.setZoomFactor).toHaveBeenCalled()
+    expect(staleRef).toMatchObject({ ok: false })
+    expect(staleRef.error).toMatch(/Call browser_snapshot/)
+  })
+
   it('returns the screenshot scale for coordinate mapping', async () => {
     const contents = await openPage()
     mockScreenshotImage({ width: 1024, height: 512 })