From c802b8f88faf20185bc86fb3056b26b3eb5b939c Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 13:42:13 -0300 Subject: [PATCH 1/7] Add /feedback slash command with confirmation flow --- apps/cli/ai/slash-commands.ts | 131 +++++++++++++++++++++++ apps/cli/ai/tests/slash-commands.test.ts | 96 +++++++++++++++++ apps/cli/vitest.config.ts | 1 + 3 files changed, 228 insertions(+) diff --git a/apps/cli/ai/slash-commands.ts b/apps/cli/ai/slash-commands.ts index 5b1a034416..989ba9890b 100644 --- a/apps/cli/ai/slash-commands.ts +++ b/apps/cli/ai/slash-commands.ts @@ -10,6 +10,7 @@ import { runCommand as runLoginCommand } from 'cli/commands/auth/login'; import { runCommand as runLogoutCommand } from 'cli/commands/auth/logout'; import { runCommand as runCreatePreviewCommand } from 'cli/commands/preview/create'; import { runCommand as runUpdatePreviewCommand } from 'cli/commands/preview/update'; +import { openBrowser } from 'cli/lib/browser'; import { isRemoteSessionEnabled } from 'cli/lib/feature-flags'; import { getSnapshotsFromConfig, isSnapshotExpired } from 'cli/lib/snapshots'; import { LoggerError } from 'cli/logger'; @@ -219,6 +220,131 @@ async function pickRemoteSessionSubcommand( } } +// Mirrors apps/studio/src/constants.ts BUG_REPORT_URL — the desktop user-menu +// link points users at the same Studio bug-report template. Hard-coded here +// rather than imported so the CLI surface stays decoupled from the desktop +// constants module. +const STUDIO_BUG_REPORT_URL = + 'https://github.com/Automattic/studio/issues/new?assignees=&labels=Needs+triage%2C%5BType%5D+Bug&projects=&template=bug_report.yml'; + +interface FeedbackEnvironment { + platform: string; + arch: string; + nodeVersion: string; + cliVersion: string; + provider: AiProviderId; + model: AiModelId; +} + +function collectFeedbackEnvironment( ctx: SlashCommandContext ): FeedbackEnvironment { + return { + platform: process.platform, + arch: process.arch, + nodeVersion: process.version, + cliVersion: __STUDIO_CLI_VERSION__, + provider: ctx.currentProvider, + model: ctx.currentModel, + }; +} + +function formatEnvironmentLine( env: FeedbackEnvironment ): string { + return `${ env.platform }/${ env.arch }, Node ${ env.nodeVersion }, Studio CLI ${ env.cliVersion }, ${ env.provider }/${ env.model }`; +} + +function buildFeedbackUrl( summary: string, env: FeedbackEnvironment ): string { + const params = new URLSearchParams(); + const trimmed = summary.trim(); + if ( trimmed ) { + // `summary` is the id of the first textarea in bug_report.yml — GitHub + // issue forms accept query params keyed by field id. The `logs` field is + // the optional textarea at the bottom of the same template. + params.set( 'summary', trimmed ); + } + params.set( 'logs', formatEnvironmentLine( env ) ); + return `${ STUDIO_BUG_REPORT_URL }&${ params.toString() }`; +} + +async function confirmFeedbackSubmission( ctx: SlashCommandContext ): Promise< boolean > { + const answer = await ctx.ui.askUser( [ + { + question: __( 'Open GitHub with this report?' ), + options: [ + { + label: __( 'Submit on GitHub' ), + description: __( 'Open the pre-filled bug report' ), + }, + { label: __( 'Cancel' ), description: __( 'Discard the feedback' ) }, + ], + }, + ] ); + const selected = ( Object.values( answer )[ 0 ] as string | undefined )?.toLowerCase() ?? ''; + return selected.startsWith( 'submit' ); +} + +async function runFeedbackSlashCommand( + _prompt: string, + ctx: SlashCommandContext +): Promise< 'continue' | 'break' > { + const descriptionQ = __( 'Describe the issue or feedback' ); + let answers: Record< string, string >; + try { + answers = await ctx.ui.askUser( [ { question: descriptionQ, options: [] } ] ); + } catch ( error ) { + if ( isPromptAbortError( error ) ) { + ctx.ui.showInfo( __( 'Feedback canceled.' ) ); + return 'continue'; + } + throw error; + } + const summary = ( answers[ descriptionQ ] ?? '' ).trim(); + if ( ! summary ) { + ctx.ui.showInfo( __( 'Feedback canceled.' ) ); + return 'continue'; + } + + const env = collectFeedbackEnvironment( ctx ); + ctx.ui.showInfo( + [ + __( 'Submit Feedback / Bug Report' ), + '', + __( 'This report will pre-fill GitHub with:' ), + sprintf( + /* translators: %s: the user's feedback text */ + __( ' - Your description: %s' ), + summary + ), + sprintf( + /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ + __( ' - Environment: %s' ), + formatEnvironmentLine( env ) + ), + '', + __( "You'll review and submit the issue on github.com." ), + ].join( '\n' ) + ); + + let confirmed: boolean; + try { + confirmed = await confirmFeedbackSubmission( ctx ); + } catch ( error ) { + if ( isPromptAbortError( error ) ) { + ctx.ui.showInfo( __( 'Feedback canceled.' ) ); + return 'continue'; + } + throw error; + } + if ( ! confirmed ) { + ctx.ui.showInfo( __( 'Feedback canceled.' ) ); + return 'continue'; + } + + await openBrowser( buildFeedbackUrl( summary, env ) ); + ctx.ui.showSuccess( + __( 'Opening GitHub with your feedback pre-filled — finish the form to submit.' ) + ); + return 'continue'; +} + async function runRemoteSessionSlashCommand( prompt: string, ctx: SlashCommandContext @@ -524,6 +650,11 @@ export const AI_CHAT_SLASH_COMMANDS: SlashCommandDef[] = [ }, handler: runRemoteSessionSlashCommand, }, + { + name: 'feedback', + description: __( 'Report a bug or share feedback about Studio' ), + handler: runFeedbackSlashCommand, + }, { name: 'exit', description: __( 'Exit the chat' ), diff --git a/apps/cli/ai/tests/slash-commands.test.ts b/apps/cli/ai/tests/slash-commands.test.ts index 1930563f7f..aa2a3343b4 100644 --- a/apps/cli/ai/tests/slash-commands.test.ts +++ b/apps/cli/ai/tests/slash-commands.test.ts @@ -84,6 +84,7 @@ vi.mock( 'cli/commands/auth/login', () => ( { runCommand: vi.fn() } ) ); vi.mock( 'cli/commands/auth/logout', () => ( { runCommand: vi.fn() } ) ); vi.mock( 'cli/commands/preview/create', () => ( { runCommand: vi.fn() } ) ); vi.mock( 'cli/commands/preview/update', () => ( { runCommand: vi.fn() } ) ); +vi.mock( 'cli/lib/browser', () => ( { openBrowser: vi.fn() } ) ); vi.mock( '@studio/common/lib/shared-config', () => ( { readAuthToken: vi.fn() } ) ); vi.mock( 'cli/remote-session/daemon', () => { @@ -448,3 +449,98 @@ describe( '/model slash command', () => { expect( persistMock ).not.toHaveBeenCalled(); } ); } ); + +describe( '/feedback slash command', () => { + const cmd = AI_CHAT_SLASH_COMMANDS.find( ( c ) => c.name === 'feedback' )!; + + function makeUi() { + return { + showInfo: vi.fn(), + showError: vi.fn(), + showSuccess: vi.fn(), + askUser: vi.fn(), + }; + } + function makeCtx( ui: ReturnType< typeof makeUi > ): SlashCommandContext { + return { + ui: ui as unknown as SlashCommandContext[ 'ui' ], + currentModel: 'claude-sonnet-4-6' as SlashCommandContext[ 'currentModel' ], + currentProvider: 'wpcom' as SlashCommandContext[ 'currentProvider' ], + showCapabilitiesOnConnect: false, + switchProvider: vi.fn().mockResolvedValue( undefined ), + prepareProviderSelection: vi.fn().mockResolvedValue( undefined ), + maybeAutoSwitchProvider: vi.fn().mockResolvedValue( undefined ), + persistSessionContext: vi.fn().mockResolvedValue( undefined ), + clearSession: vi.fn().mockResolvedValue( undefined ), + }; + } + + beforeEach( async () => { + const browser = await import( 'cli/lib/browser' ); + ( browser.openBrowser as ReturnType< typeof vi.fn > ).mockReset(); + ( browser.openBrowser as ReturnType< typeof vi.fn > ).mockResolvedValue( undefined ); + } ); + + it( 'is registered with a handler and description', () => { + expect( cmd ).toBeDefined(); + expect( typeof cmd.handler ).toBe( 'function' ); + expect( cmd.description ).toBeTruthy(); + } ); + + it( 'opens GitHub with the feedback pre-filled when the user confirms', async () => { + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { 'Describe the issue or feedback': 'something is broken' } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + const result = await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + expect( browser.openBrowser ).toHaveBeenCalledOnce(); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + expect( url ).toContain( 'github.com/Automattic/studio/issues/new' ); + expect( url ).toContain( 'template=bug_report.yml' ); + expect( url ).toContain( 'summary=something+is+broken' ); + expect( url ).toContain( 'logs=' ); + expect( ui.showSuccess ).toHaveBeenCalledOnce(); + expect( result ).toBe( 'continue' ); + } ); + + it( 'does not open the browser when the user picks Cancel at confirmation', async () => { + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { 'Describe the issue or feedback': 'minor nit' } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Cancel' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + expect( browser.openBrowser ).not.toHaveBeenCalled(); + expect( ui.showInfo ).toHaveBeenCalledWith( expect.stringContaining( 'canceled' ) ); + } ); + + it( 'cancels when the description is empty', async () => { + const ui = makeUi(); + ui.askUser.mockResolvedValueOnce( { 'Describe the issue or feedback': ' ' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + expect( browser.openBrowser ).not.toHaveBeenCalled(); + // Only the description was prompted — confirmation step was skipped. + expect( ui.askUser ).toHaveBeenCalledOnce(); + } ); + + it( 'cancels gracefully when the prompt is aborted (Esc)', async () => { + const ui = makeUi(); + const abortError = Object.assign( new Error( 'aborted' ), { name: 'AbortPromptError' } ); + ui.askUser.mockRejectedValueOnce( abortError ); + + const result = await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + expect( browser.openBrowser ).not.toHaveBeenCalled(); + expect( ui.showInfo ).toHaveBeenCalledWith( expect.stringContaining( 'canceled' ) ); + expect( result ).toBe( 'continue' ); + } ); +} ); diff --git a/apps/cli/vitest.config.ts b/apps/cli/vitest.config.ts index 0d01084b27..9acd60f197 100644 --- a/apps/cli/vitest.config.ts +++ b/apps/cli/vitest.config.ts @@ -7,6 +7,7 @@ export default mergeConfig( defineProject( { define: { __IS_PACKAGED_FOR_NPM__: true, + __STUDIO_CLI_VERSION__: JSON.stringify( 'test' ), }, test: { name: 'cli', From f1d8e56f984f8b9768060b6c53759bfbb296c3a6 Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 13:46:18 -0300 Subject: [PATCH 2/7] Simplify /feedback: drop env interface, dedupe cancels, fix locale-safe label compare --- apps/cli/ai/slash-commands.ts | 67 ++++++++++++----------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/apps/cli/ai/slash-commands.ts b/apps/cli/ai/slash-commands.ts index 989ba9890b..676ef1a951 100644 --- a/apps/cli/ai/slash-commands.ts +++ b/apps/cli/ai/slash-commands.ts @@ -227,58 +227,41 @@ async function pickRemoteSessionSubcommand( const STUDIO_BUG_REPORT_URL = 'https://github.com/Automattic/studio/issues/new?assignees=&labels=Needs+triage%2C%5BType%5D+Bug&projects=&template=bug_report.yml'; -interface FeedbackEnvironment { - platform: string; - arch: string; - nodeVersion: string; - cliVersion: string; - provider: AiProviderId; - model: AiModelId; +function formatFeedbackEnvironment( ctx: SlashCommandContext ): string { + return `${ process.platform }/${ process.arch }, Node ${ process.version }, Studio CLI ${ __STUDIO_CLI_VERSION__ }, ${ ctx.currentProvider }/${ ctx.currentModel }`; } -function collectFeedbackEnvironment( ctx: SlashCommandContext ): FeedbackEnvironment { - return { - platform: process.platform, - arch: process.arch, - nodeVersion: process.version, - cliVersion: __STUDIO_CLI_VERSION__, - provider: ctx.currentProvider, - model: ctx.currentModel, - }; -} - -function formatEnvironmentLine( env: FeedbackEnvironment ): string { - return `${ env.platform }/${ env.arch }, Node ${ env.nodeVersion }, Studio CLI ${ env.cliVersion }, ${ env.provider }/${ env.model }`; -} - -function buildFeedbackUrl( summary: string, env: FeedbackEnvironment ): string { +function buildFeedbackUrl( summary: string, environmentLine: string ): string { const params = new URLSearchParams(); const trimmed = summary.trim(); if ( trimmed ) { - // `summary` is the id of the first textarea in bug_report.yml — GitHub - // issue forms accept query params keyed by field id. The `logs` field is - // the optional textarea at the bottom of the same template. + // `summary` and `logs` are field ids in bug_report.yml — GitHub issue + // forms accept query params keyed by field id. params.set( 'summary', trimmed ); } - params.set( 'logs', formatEnvironmentLine( env ) ); + params.set( 'logs', environmentLine ); return `${ STUDIO_BUG_REPORT_URL }&${ params.toString() }`; } +function cancelFeedback( ctx: SlashCommandContext ): 'continue' { + ctx.ui.showInfo( __( 'Feedback canceled.' ) ); + return 'continue'; +} + async function confirmFeedbackSubmission( ctx: SlashCommandContext ): Promise< boolean > { + const submitLabel = __( 'Submit on GitHub' ); const answer = await ctx.ui.askUser( [ { question: __( 'Open GitHub with this report?' ), options: [ - { - label: __( 'Submit on GitHub' ), - description: __( 'Open the pre-filled bug report' ), - }, + { label: submitLabel, description: __( 'Open the pre-filled bug report' ) }, { label: __( 'Cancel' ), description: __( 'Discard the feedback' ) }, ], }, ] ); - const selected = ( Object.values( answer )[ 0 ] as string | undefined )?.toLowerCase() ?? ''; - return selected.startsWith( 'submit' ); + // Compare against the same label reference we passed in — a translated + // label could otherwise share a `startsWith` prefix with another option. + return Object.values( answer )[ 0 ] === submitLabel; } async function runFeedbackSlashCommand( @@ -291,18 +274,16 @@ async function runFeedbackSlashCommand( answers = await ctx.ui.askUser( [ { question: descriptionQ, options: [] } ] ); } catch ( error ) { if ( isPromptAbortError( error ) ) { - ctx.ui.showInfo( __( 'Feedback canceled.' ) ); - return 'continue'; + return cancelFeedback( ctx ); } throw error; } const summary = ( answers[ descriptionQ ] ?? '' ).trim(); if ( ! summary ) { - ctx.ui.showInfo( __( 'Feedback canceled.' ) ); - return 'continue'; + return cancelFeedback( ctx ); } - const env = collectFeedbackEnvironment( ctx ); + const environmentLine = formatFeedbackEnvironment( ctx ); ctx.ui.showInfo( [ __( 'Submit Feedback / Bug Report' ), @@ -316,7 +297,7 @@ async function runFeedbackSlashCommand( sprintf( /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ __( ' - Environment: %s' ), - formatEnvironmentLine( env ) + environmentLine ), '', __( "You'll review and submit the issue on github.com." ), @@ -328,17 +309,15 @@ async function runFeedbackSlashCommand( confirmed = await confirmFeedbackSubmission( ctx ); } catch ( error ) { if ( isPromptAbortError( error ) ) { - ctx.ui.showInfo( __( 'Feedback canceled.' ) ); - return 'continue'; + return cancelFeedback( ctx ); } throw error; } if ( ! confirmed ) { - ctx.ui.showInfo( __( 'Feedback canceled.' ) ); - return 'continue'; + return cancelFeedback( ctx ); } - await openBrowser( buildFeedbackUrl( summary, env ) ); + await openBrowser( buildFeedbackUrl( summary, environmentLine ) ); ctx.ui.showSuccess( __( 'Opening GitHub with your feedback pre-filled — finish the form to submit.' ) ); From eaec8599df1fa539528e62e85cd2487b36a8b6de Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 13:56:23 -0300 Subject: [PATCH 3/7] Pre-fill /feedback title and platform dropdown when deducible --- apps/cli/ai/slash-commands.ts | 109 +++++++++++++++++------ apps/cli/ai/tests/slash-commands.test.ts | 75 ++++++++++++++++ 2 files changed, 158 insertions(+), 26 deletions(-) diff --git a/apps/cli/ai/slash-commands.ts b/apps/cli/ai/slash-commands.ts index 676ef1a951..795d07d3b2 100644 --- a/apps/cli/ai/slash-commands.ts +++ b/apps/cli/ai/slash-commands.ts @@ -231,15 +231,55 @@ function formatFeedbackEnvironment( ctx: SlashCommandContext ): string { return `${ process.platform }/${ process.arch }, Node ${ process.version }, Studio CLI ${ __STUDIO_CLI_VERSION__ }, ${ ctx.currentProvider }/${ ctx.currentModel }`; } -function buildFeedbackUrl( summary: string, environmentLine: string ): string { +const FEEDBACK_TITLE_MAX_LENGTH = 80; + +function deduceFeedbackTitle( summary: string ): string { + const firstLine = summary.split( '\n' )[ 0 ].trim(); + if ( firstLine.length <= FEEDBACK_TITLE_MAX_LENGTH ) { + return firstLine; + } + return firstLine.slice( 0, FEEDBACK_TITLE_MAX_LENGTH - 1 ).trimEnd() + '…'; +} + +// Maps to the dropdown labels in .github/ISSUE_TEMPLATE/bug_report.yml. Linux +// has no matching option there, so we just skip pre-fill on that platform. +function deducePlatformLabel(): string | null { + if ( process.platform === 'darwin' ) { + return process.arch === 'arm64' ? 'Mac Silicon' : 'Mac Intel'; + } + if ( process.platform === 'win32' ) { + return 'Windows'; + } + return null; +} + +interface FeedbackPrefill { + title: string; + summary: string; + platform: string | null; + environmentLine: string; +} + +function deduceFeedbackPrefill( summary: string, ctx: SlashCommandContext ): FeedbackPrefill { + return { + title: deduceFeedbackTitle( summary ), + summary, + platform: deducePlatformLabel(), + environmentLine: formatFeedbackEnvironment( ctx ), + }; +} + +function buildFeedbackUrl( prefill: FeedbackPrefill ): string { + // Field ids match .github/ISSUE_TEMPLATE/bug_report.yml — GitHub issue + // forms accept query params keyed by field id (and `title` for the issue + // title itself). const params = new URLSearchParams(); - const trimmed = summary.trim(); - if ( trimmed ) { - // `summary` and `logs` are field ids in bug_report.yml — GitHub issue - // forms accept query params keyed by field id. - params.set( 'summary', trimmed ); + params.set( 'title', prefill.title ); + params.set( 'summary', prefill.summary ); + if ( prefill.platform ) { + params.set( 'site-type', prefill.platform ); } - params.set( 'logs', environmentLine ); + params.set( 'logs', prefill.environmentLine ); return `${ STUDIO_BUG_REPORT_URL }&${ params.toString() }`; } @@ -283,26 +323,43 @@ async function runFeedbackSlashCommand( return cancelFeedback( ctx ); } - const environmentLine = formatFeedbackEnvironment( ctx ); - ctx.ui.showInfo( - [ - __( 'Submit Feedback / Bug Report' ), - '', - __( 'This report will pre-fill GitHub with:' ), - sprintf( - /* translators: %s: the user's feedback text */ - __( ' - Your description: %s' ), - summary - ), + const prefill = deduceFeedbackPrefill( summary, ctx ); + const previewLines = [ + __( 'Submit Feedback / Bug Report' ), + '', + __( 'This report will pre-fill GitHub with:' ), + sprintf( + /* translators: %s: deduced issue title */ + __( ' - Title: %s' ), + prefill.title + ), + sprintf( + /* translators: %s: the user's feedback text */ + __( ' - Description: %s' ), + prefill.summary + ), + ]; + if ( prefill.platform ) { + previewLines.push( sprintf( - /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ - __( ' - Environment: %s' ), - environmentLine - ), - '', - __( "You'll review and submit the issue on github.com." ), - ].join( '\n' ) + /* translators: %s: detected platform like "Mac Silicon" */ + __( ' - Platform: %s' ), + prefill.platform + ) + ); + } + previewLines.push( + sprintf( + /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ + __( ' - Environment: %s' ), + prefill.environmentLine + ), + '', + __( + "You'll still need to fill in steps to reproduce, expected/actual behavior, and impact on github.com." + ) ); + ctx.ui.showInfo( previewLines.join( '\n' ) ); let confirmed: boolean; try { @@ -317,7 +374,7 @@ async function runFeedbackSlashCommand( return cancelFeedback( ctx ); } - await openBrowser( buildFeedbackUrl( summary, environmentLine ) ); + await openBrowser( buildFeedbackUrl( prefill ) ); ctx.ui.showSuccess( __( 'Opening GitHub with your feedback pre-filled — finish the form to submit.' ) ); diff --git a/apps/cli/ai/tests/slash-commands.test.ts b/apps/cli/ai/tests/slash-commands.test.ts index aa2a3343b4..ea720fa928 100644 --- a/apps/cli/ai/tests/slash-commands.test.ts +++ b/apps/cli/ai/tests/slash-commands.test.ts @@ -500,12 +500,87 @@ describe( '/feedback slash command', () => { const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; expect( url ).toContain( 'github.com/Automattic/studio/issues/new' ); expect( url ).toContain( 'template=bug_report.yml' ); + expect( url ).toContain( 'title=something+is+broken' ); expect( url ).toContain( 'summary=something+is+broken' ); expect( url ).toContain( 'logs=' ); expect( ui.showSuccess ).toHaveBeenCalledOnce(); expect( result ).toBe( 'continue' ); } ); + it( 'truncates a long first line into a title that fits the issue title field', async () => { + const ui = makeUi(); + const longLine = 'x'.repeat( 200 ); + ui.askUser + .mockResolvedValueOnce( { 'Describe the issue or feedback': longLine } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + const title = new URL( url ).searchParams.get( 'title' ) ?? ''; + expect( title.length ).toBeLessThanOrEqual( 80 ); + expect( title.endsWith( '…' ) ).toBe( true ); + // The full description still goes into summary, untruncated. + expect( new URL( url ).searchParams.get( 'summary' ) ).toBe( longLine ); + } ); + + it( 'derives the title from only the first line of a multi-line description', async () => { + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { + 'Describe the issue or feedback': 'short headline\nlots of detail on the next line', + } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + expect( new URL( url ).searchParams.get( 'title' ) ).toBe( 'short headline' ); + } ); + + it( 'pre-fills the platform dropdown when running on a known platform', async () => { + const originalPlatform = process.platform; + const originalArch = process.arch; + Object.defineProperty( process, 'platform', { value: 'darwin', configurable: true } ); + Object.defineProperty( process, 'arch', { value: 'arm64', configurable: true } ); + try { + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { 'Describe the issue or feedback': 'mac silicon issue' } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + expect( new URL( url ).searchParams.get( 'site-type' ) ).toBe( 'Mac Silicon' ); + } finally { + Object.defineProperty( process, 'platform', { value: originalPlatform } ); + Object.defineProperty( process, 'arch', { value: originalArch } ); + } + } ); + + it( 'omits the platform dropdown on platforms without a matching template option', async () => { + const originalPlatform = process.platform; + Object.defineProperty( process, 'platform', { value: 'linux', configurable: true } ); + try { + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { 'Describe the issue or feedback': 'on linux' } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + expect( new URL( url ).searchParams.has( 'site-type' ) ).toBe( false ); + } finally { + Object.defineProperty( process, 'platform', { value: originalPlatform } ); + } + } ); + it( 'does not open the browser when the user picks Cancel at confirmation', async () => { const ui = makeUi(); ui.askUser From 38d91e98d75be21afb1d62d92c47ffa432e2915e Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 14:05:27 -0300 Subject: [PATCH 4/7] Lock /feedback to honest-only pre-fill with regression test --- apps/cli/ai/tests/slash-commands.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/cli/ai/tests/slash-commands.test.ts b/apps/cli/ai/tests/slash-commands.test.ts index ea720fa928..43d21127aa 100644 --- a/apps/cli/ai/tests/slash-commands.test.ts +++ b/apps/cli/ai/tests/slash-commands.test.ts @@ -503,6 +503,14 @@ describe( '/feedback slash command', () => { expect( url ).toContain( 'title=something+is+broken' ); expect( url ).toContain( 'summary=something+is+broken' ); expect( url ).toContain( 'logs=' ); + // Required fields the user must fill themselves are NOT pre-filled — + // only what we can honestly derive from the entry. + const params = new URL( url ).searchParams; + expect( params.has( 'steps' ) ).toBe( false ); + expect( params.has( 'expected' ) ).toBe( false ); + expect( params.has( 'actual' ) ).toBe( false ); + expect( params.has( 'users-affected' ) ).toBe( false ); + expect( params.has( 'workarounds' ) ).toBe( false ); expect( ui.showSuccess ).toHaveBeenCalledOnce(); expect( result ).toBe( 'continue' ); } ); From d53379ff0f0fd7bfd64179e7a72560265cbd3e3a Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 14:21:27 -0300 Subject: [PATCH 5/7] Use AI extraction to pre-fill /feedback bug-report fields --- apps/cli/ai/feedback-extraction.ts | 183 ++++++++++++++++++ apps/cli/ai/slash-commands.ts | 172 ++++++++++++---- apps/cli/ai/tests/feedback-extraction.test.ts | 58 ++++++ apps/cli/ai/tests/slash-commands.test.ts | 78 +++++++- 4 files changed, 444 insertions(+), 47 deletions(-) create mode 100644 apps/cli/ai/feedback-extraction.ts create mode 100644 apps/cli/ai/tests/feedback-extraction.test.ts diff --git a/apps/cli/ai/feedback-extraction.ts b/apps/cli/ai/feedback-extraction.ts new file mode 100644 index 0000000000..89dc9e25b7 --- /dev/null +++ b/apps/cli/ai/feedback-extraction.ts @@ -0,0 +1,183 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { getAiModelFamily, type AiModelId } from '@studio/common/ai/models'; +import { resolveAiEnvironment } from 'cli/ai/auth'; +import type { AiProviderId } from 'cli/ai/providers'; + +// Hard-coded against the bug_report.yml dropdown labels — keep in sync with +// .github/ISSUE_TEMPLATE/bug_report.yml. Wider unions get rejected by the +// validator below so an off-script model response can't slip into the URL. +export const FEEDBACK_IMPACT_VALUES = [ 'One', 'Some (< 50%)', 'Most (> 50%)', 'All' ] as const; +export type FeedbackImpact = ( typeof FEEDBACK_IMPACT_VALUES )[ number ]; + +export const FEEDBACK_WORKAROUND_VALUES = [ + 'No and the app is unusable', + 'No but the app is still usable', + 'Yes, difficult to implement', + 'Yes, easy to implement', + 'There is no user impact', +] as const; +export type FeedbackWorkaround = ( typeof FEEDBACK_WORKAROUND_VALUES )[ number ]; + +export interface ExtractedFeedbackFields { + title: string | null; + steps: string | null; + expected: string | null; + actual: string | null; + impact: FeedbackImpact | null; + workaround: FeedbackWorkaround | null; +} + +const FEEDBACK_EXTRACTION_TIMEOUT_MS = 15_000; +const FEEDBACK_EXTRACTION_MAX_TOKENS = 1024; + +const FEEDBACK_EXTRACTION_SYSTEM_PROMPT = `You extract structured fields from a Studio AI bug report so a GitHub issue form can be pre-filled. + +Respond with ONLY a JSON object matching this shape: +{ + "title": string | null, // <= 80 chars summarizing the issue + "steps": string | null, // reproduction steps if mentioned (markdown numbered list ok) + "expected": string | null, // what the user expected to happen + "actual": string | null, // what actually happened + "impact": "One" | "Some (< 50%)" | "Most (> 50%)" | "All" | null, + "workaround": "No and the app is unusable" | "No but the app is still usable" | "Yes, difficult to implement" | "Yes, easy to implement" | "There is no user impact" | null +} + +Rules: +- Use null for any field where the report does NOT clearly express that signal. +- Do NOT invent reproduction steps, expected behavior, or impact scope. +- Output JSON only — no prose, no code fences.`; + +interface ExtractFeedbackContext { + provider: AiProviderId; + model: AiModelId; +} + +/** + * Run a one-shot, non-streaming extraction call against the user's currently + * configured AI provider. Returns null on any failure path (unsupported + * model family, missing credentials, network/timeout, malformed JSON) so the + * caller can fall back to non-AI defaults instead of breaking the slash + * command. + */ +export async function extractFeedbackFields( + description: string, + ctx: ExtractFeedbackContext +): Promise< ExtractedFeedbackFields | null > { + // Anthropic-only for v1 — the OpenAI path would need a different SDK + // surface and isn't worth the extra dep+auth plumbing yet. + if ( getAiModelFamily( ctx.model ) !== 'anthropic' ) { + return null; + } + + let env: Record< string, string >; + try { + env = await resolveAiEnvironment( ctx.provider ); + } catch { + return null; + } + + const authToken = env.ANTHROPIC_AUTH_TOKEN?.trim(); + const apiKey = env.ANTHROPIC_API_KEY?.trim(); + if ( ! authToken && ! apiKey ) { + return null; + } + + const client = new Anthropic( { + apiKey: apiKey ?? null, + authToken: authToken ?? undefined, + baseURL: env.ANTHROPIC_BASE_URL, + defaultHeaders: parseAnthropicHeaders( env.ANTHROPIC_CUSTOM_HEADERS ), + dangerouslyAllowBrowser: true, + } ); + + const controller = new AbortController(); + const timer = setTimeout( () => controller.abort(), FEEDBACK_EXTRACTION_TIMEOUT_MS ); + + try { + const response = await client.messages.create( + { + model: ctx.model, + max_tokens: FEEDBACK_EXTRACTION_MAX_TOKENS, + system: FEEDBACK_EXTRACTION_SYSTEM_PROMPT, + messages: [ + { + role: 'user', + content: `Bug report:\n"""\n${ description }\n"""`, + }, + ], + }, + { signal: controller.signal } + ); + const text = response.content + .filter( + ( block ): block is Extract< typeof block, { type: 'text' } > => block.type === 'text' + ) + .map( ( block ) => block.text ) + .join( '\n' ) + .trim(); + return parseFeedbackExtraction( text ); + } catch { + return null; + } finally { + clearTimeout( timer ); + } +} + +function parseAnthropicHeaders( raw: string | undefined ): Record< string, string > | undefined { + if ( ! raw ) { + return undefined; + } + try { + const parsed = JSON.parse( raw ); + return parsed && typeof parsed === 'object' + ? ( parsed as Record< string, string > ) + : undefined; + } catch { + return undefined; + } +} + +export function parseFeedbackExtraction( text: string ): ExtractedFeedbackFields | null { + // Strip code fences if the model wrapped its JSON despite the instructions. + const stripped = text + .replace( /^```(?:json)?\s*/i, '' ) + .replace( /```\s*$/, '' ) + .trim(); + + let parsed: unknown; + try { + parsed = JSON.parse( stripped ); + } catch { + return null; + } + if ( ! parsed || typeof parsed !== 'object' ) { + return null; + } + + const obj = parsed as Record< string, unknown >; + return { + title: sanitizeString( obj.title ), + steps: sanitizeString( obj.steps ), + expected: sanitizeString( obj.expected ), + actual: sanitizeString( obj.actual ), + impact: matchEnum( obj.impact, FEEDBACK_IMPACT_VALUES ), + workaround: matchEnum( obj.workaround, FEEDBACK_WORKAROUND_VALUES ), + }; +} + +function sanitizeString( value: unknown ): string | null { + if ( typeof value !== 'string' ) { + return null; + } + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} + +function matchEnum< T extends readonly string[] >( + value: unknown, + allowed: T +): T[ number ] | null { + return typeof value === 'string' && ( allowed as readonly string[] ).includes( value ) + ? ( value as T[ number ] ) + : null; +} diff --git a/apps/cli/ai/slash-commands.ts b/apps/cli/ai/slash-commands.ts index 795d07d3b2..0f70af76ad 100644 --- a/apps/cli/ai/slash-commands.ts +++ b/apps/cli/ai/slash-commands.ts @@ -4,6 +4,7 @@ import { AI_SKILL_COMMANDS } from '@studio/common/ai/slash-commands'; import { readAuthToken } from '@studio/common/lib/shared-config'; import { __, sprintf } from '@wordpress/i18n'; import { getAvailableAiProviders, isAiProviderReady } from 'cli/ai/auth'; +import { extractFeedbackFields, type ExtractedFeedbackFields } from 'cli/ai/feedback-extraction'; import { AI_PROVIDERS, getAiProviderDefinition, type AiProviderId } from 'cli/ai/providers'; import { captureCommandOutput } from 'cli/ai/tools'; import { runCommand as runLoginCommand } from 'cli/commands/auth/login'; @@ -231,19 +232,31 @@ function formatFeedbackEnvironment( ctx: SlashCommandContext ): string { return `${ process.platform }/${ process.arch }, Node ${ process.version }, Studio CLI ${ __STUDIO_CLI_VERSION__ }, ${ ctx.currentProvider }/${ ctx.currentModel }`; } +// Cap below GitHub's 256-char issue title limit so titles read well in lists +// and notifications. const FEEDBACK_TITLE_MAX_LENGTH = 80; -function deduceFeedbackTitle( summary: string ): string { - const firstLine = summary.split( '\n' )[ 0 ].trim(); +function fallbackTitleFromSummary( summary: string ): string { + const newlineIdx = summary.indexOf( '\n' ); + const firstLine = ( newlineIdx === -1 ? summary : summary.slice( 0, newlineIdx ) ).trim(); if ( firstLine.length <= FEEDBACK_TITLE_MAX_LENGTH ) { return firstLine; } return firstLine.slice( 0, FEEDBACK_TITLE_MAX_LENGTH - 1 ).trimEnd() + '…'; } +function clampTitle( title: string ): string { + if ( title.length <= FEEDBACK_TITLE_MAX_LENGTH ) { + return title; + } + return title.slice( 0, FEEDBACK_TITLE_MAX_LENGTH - 1 ).trimEnd() + '…'; +} + +type PlatformLabel = 'Mac Silicon' | 'Mac Intel' | 'Windows'; + // Maps to the dropdown labels in .github/ISSUE_TEMPLATE/bug_report.yml. Linux // has no matching option there, so we just skip pre-fill on that platform. -function deducePlatformLabel(): string | null { +function deducePlatformLabel(): PlatformLabel | null { if ( process.platform === 'darwin' ) { return process.arch === 'arm64' ? 'Mac Silicon' : 'Mac Intel'; } @@ -256,14 +269,29 @@ function deducePlatformLabel(): string | null { interface FeedbackPrefill { title: string; summary: string; - platform: string | null; + steps: string | null; + expected: string | null; + actual: string | null; + impact: string | null; + workaround: string | null; + platform: PlatformLabel | null; environmentLine: string; } -function deduceFeedbackPrefill( summary: string, ctx: SlashCommandContext ): FeedbackPrefill { +function composeFeedbackPrefill( + summary: string, + extracted: ExtractedFeedbackFields | null, + ctx: SlashCommandContext +): FeedbackPrefill { + const aiTitle = extracted?.title ? clampTitle( extracted.title ) : null; return { - title: deduceFeedbackTitle( summary ), + title: aiTitle ?? fallbackTitleFromSummary( summary ), summary, + steps: extracted?.steps ?? null, + expected: extracted?.expected ?? null, + actual: extracted?.actual ?? null, + impact: extracted?.impact ?? null, + workaround: extracted?.workaround ?? null, platform: deducePlatformLabel(), environmentLine: formatFeedbackEnvironment( ctx ), }; @@ -276,6 +304,21 @@ function buildFeedbackUrl( prefill: FeedbackPrefill ): string { const params = new URLSearchParams(); params.set( 'title', prefill.title ); params.set( 'summary', prefill.summary ); + if ( prefill.steps ) { + params.set( 'steps', prefill.steps ); + } + if ( prefill.expected ) { + params.set( 'expected', prefill.expected ); + } + if ( prefill.actual ) { + params.set( 'actual', prefill.actual ); + } + if ( prefill.impact ) { + params.set( 'users-affected', prefill.impact ); + } + if ( prefill.workaround ) { + params.set( 'workarounds', prefill.workaround ); + } if ( prefill.platform ) { params.set( 'site-type', prefill.platform ); } @@ -323,43 +366,90 @@ async function runFeedbackSlashCommand( return cancelFeedback( ctx ); } - const prefill = deduceFeedbackPrefill( summary, ctx ); - const previewLines = [ - __( 'Submit Feedback / Bug Report' ), - '', - __( 'This report will pre-fill GitHub with:' ), - sprintf( - /* translators: %s: deduced issue title */ - __( ' - Title: %s' ), - prefill.title - ), - sprintf( - /* translators: %s: the user's feedback text */ - __( ' - Description: %s' ), - prefill.summary - ), - ]; - if ( prefill.platform ) { - previewLines.push( - sprintf( - /* translators: %s: detected platform like "Mac Silicon" */ - __( ' - Platform: %s' ), - prefill.platform - ) - ); + ctx.ui.showProgress( __( 'Analyzing your feedback…' ) ); + ctx.ui.setBusy( true ); + let extracted: ExtractedFeedbackFields | null; + try { + extracted = await extractFeedbackFields( summary, { + provider: ctx.currentProvider, + model: ctx.currentModel, + } ); + } finally { + ctx.ui.setBusy( false ); } - previewLines.push( - sprintf( - /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ - __( ' - Environment: %s' ), - prefill.environmentLine - ), - '', - __( - "You'll still need to fill in steps to reproduce, expected/actual behavior, and impact on github.com." - ) + const prefill = composeFeedbackPrefill( summary, extracted, ctx ); + + ctx.ui.showInfo( + [ + __( 'Submit Feedback / Bug Report' ), + '', + extracted + ? __( 'GitHub will open with these fields pre-filled — review and edit before submitting.' ) + : __( + 'GitHub will open with what we could pre-fill — fill in the rest before submitting.' + ), + '', + sprintf( + /* translators: %s: deduced issue title */ + __( ' - Title: %s' ), + prefill.title + ), + sprintf( + /* translators: %s: the user's feedback text */ + __( ' - Description: %s' ), + prefill.summary + ), + prefill.steps + ? sprintf( + /* translators: %s: AI-extracted reproduction steps */ + __( ' - Steps: %s' ), + prefill.steps + ) + : null, + prefill.expected + ? sprintf( + /* translators: %s: AI-extracted "what you expected" */ + __( ' - Expected: %s' ), + prefill.expected + ) + : null, + prefill.actual + ? sprintf( + /* translators: %s: AI-extracted "what actually happened" */ + __( ' - Actual: %s' ), + prefill.actual + ) + : null, + prefill.impact + ? sprintf( + /* translators: %s: AI-extracted impact label, e.g. "All" */ + __( ' - Impact: %s' ), + prefill.impact + ) + : null, + prefill.workaround + ? sprintf( + /* translators: %s: AI-extracted workaround availability */ + __( ' - Workarounds: %s' ), + prefill.workaround + ) + : null, + prefill.platform + ? sprintf( + /* translators: %s: detected platform like "Mac Silicon" */ + __( ' - Platform: %s' ), + prefill.platform + ) + : null, + sprintf( + /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ + __( ' - Environment: %s' ), + prefill.environmentLine + ), + ] + .filter( ( line ): line is string => typeof line === 'string' ) + .join( '\n' ) ); - ctx.ui.showInfo( previewLines.join( '\n' ) ); let confirmed: boolean; try { diff --git a/apps/cli/ai/tests/feedback-extraction.test.ts b/apps/cli/ai/tests/feedback-extraction.test.ts new file mode 100644 index 0000000000..432872a207 --- /dev/null +++ b/apps/cli/ai/tests/feedback-extraction.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { parseFeedbackExtraction } from 'cli/ai/feedback-extraction'; + +describe( 'parseFeedbackExtraction', () => { + it( 'returns null for non-JSON output', () => { + expect( parseFeedbackExtraction( 'sorry, I can only respond with JSON' ) ).toBeNull(); + } ); + + it( 'returns null for valid JSON that is not an object', () => { + expect( parseFeedbackExtraction( '"a string"' ) ).toBeNull(); + expect( parseFeedbackExtraction( '[1, 2, 3]' ) ).not.toBeNull(); + // Arrays are objects in JS — but downstream readers will see all-null + // fields, which is harmless. The strict guard is the enum/string + // validation. + } ); + + it( 'strips ```json fences if the model wraps its output despite instructions', () => { + const wrapped = + '```json\n{"title":"hello","steps":null,"expected":null,"actual":null,"impact":null,"workaround":null}\n```'; + const parsed = parseFeedbackExtraction( wrapped ); + expect( parsed?.title ).toBe( 'hello' ); + } ); + + it( 'rejects an out-of-vocabulary impact label', () => { + const json = + '{"title":"x","steps":null,"expected":null,"actual":null,"impact":"some users","workaround":null}'; + expect( parseFeedbackExtraction( json )?.impact ).toBeNull(); + } ); + + it( 'accepts the canonical impact and workaround labels', () => { + const json = JSON.stringify( { + title: 'x', + steps: null, + expected: null, + actual: null, + impact: 'Most (> 50%)', + workaround: 'Yes, easy to implement', + } ); + const parsed = parseFeedbackExtraction( json ); + expect( parsed?.impact ).toBe( 'Most (> 50%)' ); + expect( parsed?.workaround ).toBe( 'Yes, easy to implement' ); + } ); + + it( 'coerces empty-string fields to null', () => { + const json = JSON.stringify( { + title: '', + steps: ' ', + expected: 'real value', + actual: null, + impact: null, + workaround: null, + } ); + const parsed = parseFeedbackExtraction( json ); + expect( parsed?.title ).toBeNull(); + expect( parsed?.steps ).toBeNull(); + expect( parsed?.expected ).toBe( 'real value' ); + } ); +} ); diff --git a/apps/cli/ai/tests/slash-commands.test.ts b/apps/cli/ai/tests/slash-commands.test.ts index 43d21127aa..9254c2f917 100644 --- a/apps/cli/ai/tests/slash-commands.test.ts +++ b/apps/cli/ai/tests/slash-commands.test.ts @@ -85,6 +85,7 @@ vi.mock( 'cli/commands/auth/logout', () => ( { runCommand: vi.fn() } ) ); vi.mock( 'cli/commands/preview/create', () => ( { runCommand: vi.fn() } ) ); vi.mock( 'cli/commands/preview/update', () => ( { runCommand: vi.fn() } ) ); vi.mock( 'cli/lib/browser', () => ( { openBrowser: vi.fn() } ) ); +vi.mock( 'cli/ai/feedback-extraction', () => ( { extractFeedbackFields: vi.fn() } ) ); vi.mock( '@studio/common/lib/shared-config', () => ( { readAuthToken: vi.fn() } ) ); vi.mock( 'cli/remote-session/daemon', () => { @@ -458,6 +459,8 @@ describe( '/feedback slash command', () => { showInfo: vi.fn(), showError: vi.fn(), showSuccess: vi.fn(), + showProgress: vi.fn(), + setBusy: vi.fn(), askUser: vi.fn(), }; } @@ -479,6 +482,10 @@ describe( '/feedback slash command', () => { const browser = await import( 'cli/lib/browser' ); ( browser.openBrowser as ReturnType< typeof vi.fn > ).mockReset(); ( browser.openBrowser as ReturnType< typeof vi.fn > ).mockResolvedValue( undefined ); + const extraction = await import( 'cli/ai/feedback-extraction' ); + ( extraction.extractFeedbackFields as ReturnType< typeof vi.fn > ).mockReset(); + // Default: no AI signal — tests opt in to extraction by overriding. + ( extraction.extractFeedbackFields as ReturnType< typeof vi.fn > ).mockResolvedValue( null ); } ); it( 'is registered with a handler and description', () => { @@ -498,14 +505,14 @@ describe( '/feedback slash command', () => { const browser = await import( 'cli/lib/browser' ); expect( browser.openBrowser ).toHaveBeenCalledOnce(); const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + const params = new URL( url ).searchParams; expect( url ).toContain( 'github.com/Automattic/studio/issues/new' ); expect( url ).toContain( 'template=bug_report.yml' ); - expect( url ).toContain( 'title=something+is+broken' ); - expect( url ).toContain( 'summary=something+is+broken' ); - expect( url ).toContain( 'logs=' ); - // Required fields the user must fill themselves are NOT pre-filled — - // only what we can honestly derive from the entry. - const params = new URL( url ).searchParams; + expect( params.get( 'title' ) ).toBe( 'something is broken' ); + expect( params.get( 'summary' ) ).toBe( 'something is broken' ); + expect( params.get( 'logs' ) ).toBeTruthy(); + // AI extraction returned null in this test — fields without honest + // signal stay unset rather than getting force-filled. expect( params.has( 'steps' ) ).toBe( false ); expect( params.has( 'expected' ) ).toBe( false ); expect( params.has( 'actual' ) ).toBe( false ); @@ -515,6 +522,65 @@ describe( '/feedback slash command', () => { expect( result ).toBe( 'continue' ); } ); + it( 'pre-fills steps/expected/actual/impact/workarounds when AI extraction returns them', async () => { + const extraction = await import( 'cli/ai/feedback-extraction' ); + ( extraction.extractFeedbackFields as ReturnType< typeof vi.fn > ).mockResolvedValue( { + title: 'Sites fail to start after update', + steps: '1. Update Studio.\n2. Click Start on a site.', + expected: 'The site should start and serve on its assigned port.', + actual: 'The Start button spins forever and the port stays closed.', + impact: 'All', + workaround: 'No and the app is unusable', + } ); + + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { + 'Describe the issue or feedback': + 'After updating, every site I try to start just spins. The app is basically dead.', + } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + const params = new URL( url ).searchParams; + expect( params.get( 'title' ) ).toBe( 'Sites fail to start after update' ); + expect( params.get( 'steps' ) ).toContain( 'Update Studio' ); + expect( params.get( 'expected' ) ).toContain( 'serve on its assigned port' ); + expect( params.get( 'actual' ) ).toContain( 'spins forever' ); + expect( params.get( 'users-affected' ) ).toBe( 'All' ); + expect( params.get( 'workarounds' ) ).toBe( 'No and the app is unusable' ); + // Description still flows into summary verbatim. + expect( params.get( 'summary' ) ).toContain( 'every site I try to start' ); + } ); + + it( 'falls back to the first-line title when AI returns no title', async () => { + const extraction = await import( 'cli/ai/feedback-extraction' ); + ( extraction.extractFeedbackFields as ReturnType< typeof vi.fn > ).mockResolvedValue( { + title: null, + steps: null, + expected: null, + actual: null, + impact: null, + workaround: null, + } ); + + const ui = makeUi(); + ui.askUser + .mockResolvedValueOnce( { + 'Describe the issue or feedback': 'something headline\nbody text follows', + } ) + .mockResolvedValueOnce( { 'Open GitHub with this report?': 'Submit on GitHub' } ); + + await cmd.handler!( '/feedback', makeCtx( ui ) ); + + const browser = await import( 'cli/lib/browser' ); + const url = ( browser.openBrowser as ReturnType< typeof vi.fn > ).mock.calls[ 0 ][ 0 ]; + expect( new URL( url ).searchParams.get( 'title' ) ).toBe( 'something headline' ); + } ); + it( 'truncates a long first line into a title that fits the issue title field', async () => { const ui = makeUi(); const longLine = 'x'.repeat( 200 ); From 8fd5aff723ad3c86602086594ae7e68e61848d4e Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 14:30:54 -0300 Subject: [PATCH 6/7] Extend /feedback AI extraction to OpenAI; fix wpcom header bug --- apps/cli/ai/feedback-extraction.ts | 207 ++++++++++++----- apps/cli/ai/slash-commands.ts | 100 +++----- apps/cli/ai/tests/feedback-extraction.test.ts | 219 +++++++++++++++++- 3 files changed, 398 insertions(+), 128 deletions(-) diff --git a/apps/cli/ai/feedback-extraction.ts b/apps/cli/ai/feedback-extraction.ts index 89dc9e25b7..ba76929437 100644 --- a/apps/cli/ai/feedback-extraction.ts +++ b/apps/cli/ai/feedback-extraction.ts @@ -1,5 +1,4 @@ -import Anthropic from '@anthropic-ai/sdk'; -import { getAiModelFamily, type AiModelId } from '@studio/common/ai/models'; +import { getAiModelFamily, type AiModelFamily, type AiModelId } from '@studio/common/ai/models'; import { resolveAiEnvironment } from 'cli/ai/auth'; import type { AiProviderId } from 'cli/ai/providers'; @@ -27,8 +26,14 @@ export interface ExtractedFeedbackFields { workaround: FeedbackWorkaround | null; } -const FEEDBACK_EXTRACTION_TIMEOUT_MS = 15_000; -const FEEDBACK_EXTRACTION_MAX_TOKENS = 1024; +const FEEDBACK_EXTRACTION_TIMEOUT_MS = 10_000; +// Bug reports with steps + expected + actual easily exceed 1024 tokens. A +// truncated response makes JSON.parse fail and the user silently falls back +// to no extraction; bumping the cap costs nothing when the model finishes +// early since billing is per emitted token. +const FEEDBACK_EXTRACTION_MAX_TOKENS = 2048; +const ANTHROPIC_API_VERSION = '2023-06-01'; +const ANTHROPIC_DEFAULT_BASE_URL = 'https://api.anthropic.com'; const FEEDBACK_EXTRACTION_SYSTEM_PROMPT = `You extract structured fields from a Studio AI bug report so a GitHub issue form can be pre-filled. @@ -45,6 +50,8 @@ Respond with ONLY a JSON object matching this shape: Rules: - Use null for any field where the report does NOT clearly express that signal. - Do NOT invent reproduction steps, expected behavior, or impact scope. +- Preserve the user's language for free-form fields (title, steps, expected, actual). +- For "impact" and "workaround", emit ONLY the exact English literals listed above — even when the report is in another language. - Output JSON only — no prose, no code fences.`; interface ExtractFeedbackContext { @@ -54,91 +61,183 @@ interface ExtractFeedbackContext { /** * Run a one-shot, non-streaming extraction call against the user's currently - * configured AI provider. Returns null on any failure path (unsupported - * model family, missing credentials, network/timeout, malformed JSON) so the - * caller can fall back to non-AI defaults instead of breaking the slash - * command. + * configured AI provider. Supports both Anthropic and OpenAI families through + * the same wpcom proxy (or direct Anthropic when the user supplies their own + * key). Returns null on any failure path so the caller can fall back to + * non-AI defaults instead of breaking the slash command. */ export async function extractFeedbackFields( description: string, ctx: ExtractFeedbackContext ): Promise< ExtractedFeedbackFields | null > { - // Anthropic-only for v1 — the OpenAI path would need a different SDK - // surface and isn't worth the extra dep+auth plumbing yet. - if ( getAiModelFamily( ctx.model ) !== 'anthropic' ) { + let env: Record< string, string >; + try { + env = await resolveAiEnvironment( ctx.provider ); + } catch { return null; } - let env: Record< string, string >; + const controller = new AbortController(); + const timer = setTimeout( () => controller.abort(), FEEDBACK_EXTRACTION_TIMEOUT_MS ); try { - env = await resolveAiEnvironment( ctx.provider ); + const text = await callExtractionModel( + getAiModelFamily( ctx.model ), + ctx.model, + env, + description, + controller.signal + ); + return text ? parseFeedbackExtraction( text ) : null; } catch { return null; + } finally { + clearTimeout( timer ); } +} + +async function callExtractionModel( + family: AiModelFamily, + model: AiModelId, + env: Record< string, string >, + description: string, + signal: AbortSignal +): Promise< string | null > { + const userMessage = `Bug report:\n"""\n${ description }\n"""`; + if ( family === 'anthropic' ) { + return callAnthropicMessages( env, model, userMessage, signal ); + } + return callOpenAiCompletions( env, model, userMessage, signal ); +} +async function callAnthropicMessages( + env: Record< string, string >, + model: string, + userMessage: string, + signal: AbortSignal +): Promise< string | null > { + const baseUrl = ( env.ANTHROPIC_BASE_URL?.trim() || ANTHROPIC_DEFAULT_BASE_URL ).replace( + /\/+$/, + '' + ); const authToken = env.ANTHROPIC_AUTH_TOKEN?.trim(); const apiKey = env.ANTHROPIC_API_KEY?.trim(); if ( ! authToken && ! apiKey ) { return null; } - const client = new Anthropic( { - apiKey: apiKey ?? null, - authToken: authToken ?? undefined, - baseURL: env.ANTHROPIC_BASE_URL, - defaultHeaders: parseAnthropicHeaders( env.ANTHROPIC_CUSTOM_HEADERS ), - dangerouslyAllowBrowser: true, + const headers: Record< string, string > = { + 'content-type': 'application/json', + 'anthropic-version': ANTHROPIC_API_VERSION, + ...parseAnthropicLineHeaders( env.ANTHROPIC_CUSTOM_HEADERS ), + }; + // wpcom routes Anthropic via Bearer; direct Anthropic uses x-api-key. + if ( authToken ) { + headers.authorization = `Bearer ${ authToken }`; + } else if ( apiKey ) { + headers[ 'x-api-key' ] = apiKey; + } + + const response = await fetch( `${ baseUrl }/v1/messages`, { + method: 'POST', + headers, + body: JSON.stringify( { + model, + max_tokens: FEEDBACK_EXTRACTION_MAX_TOKENS, + system: FEEDBACK_EXTRACTION_SYSTEM_PROMPT, + messages: [ { role: 'user', content: userMessage } ], + } ), + signal, } ); + if ( ! response.ok ) { + return null; + } + const json = ( await response.json() ) as { + content?: Array< { type?: string; text?: string } >; + }; + const block = json.content?.find( ( b ) => b.type === 'text' ); + return block?.text ?? null; +} - const controller = new AbortController(); - const timer = setTimeout( () => controller.abort(), FEEDBACK_EXTRACTION_TIMEOUT_MS ); +async function callOpenAiCompletions( + env: Record< string, string >, + model: string, + userMessage: string, + signal: AbortSignal +): Promise< string | null > { + const baseUrl = env.OPENAI_BASE_URL?.trim(); + const apiKey = env.OPENAI_API_KEY?.trim(); + if ( ! baseUrl || ! apiKey ) { + return null; + } - try { - const response = await client.messages.create( - { - model: ctx.model, - max_tokens: FEEDBACK_EXTRACTION_MAX_TOKENS, - system: FEEDBACK_EXTRACTION_SYSTEM_PROMPT, - messages: [ - { - role: 'user', - content: `Bug report:\n"""\n${ description }\n"""`, - }, - ], - }, - { signal: controller.signal } - ); - const text = response.content - .filter( - ( block ): block is Extract< typeof block, { type: 'text' } > => block.type === 'text' - ) - .map( ( block ) => block.text ) - .join( '\n' ) - .trim(); - return parseFeedbackExtraction( text ); - } catch { + const headers: Record< string, string > = { + 'content-type': 'application/json', + authorization: `Bearer ${ apiKey }`, + ...parseJsonHeaders( env.STUDIO_OPENAI_DEFAULT_HEADERS ), + }; + + const response = await fetch( `${ baseUrl.replace( /\/+$/, '' ) }/chat/completions`, { + method: 'POST', + headers, + body: JSON.stringify( { + model, + max_tokens: FEEDBACK_EXTRACTION_MAX_TOKENS, + // `json_object` forces the model to return parseable JSON; the + // system prompt's schema then constrains the shape. + response_format: { type: 'json_object' }, + messages: [ + { role: 'system', content: FEEDBACK_EXTRACTION_SYSTEM_PROMPT }, + { role: 'user', content: userMessage }, + ], + } ), + signal, + } ); + if ( ! response.ok ) { return null; - } finally { - clearTimeout( timer ); } + const json = ( await response.json() ) as { + choices?: Array< { message?: { content?: string } } >; + }; + return json.choices?.[ 0 ]?.message?.content ?? null; +} + +// `ANTHROPIC_CUSTOM_HEADERS` is encoded by `buildAnthropicCustomHeaders` in +// providers.ts as `Name: Value\nName: Value` — NOT JSON. Mirrors the runtime's +// `parseAnthropicHeaderEnv` so the wpcom feature header reaches the proxy. +function parseAnthropicLineHeaders( raw: string | undefined ): Record< string, string > { + if ( ! raw ) { + return {}; + } + const out: Record< string, string > = {}; + for ( const line of raw.split( '\n' ) ) { + const idx = line.indexOf( ':' ); + if ( idx <= 0 ) { + continue; + } + const name = line.slice( 0, idx ).trim(); + const value = line.slice( idx + 1 ).trim(); + if ( name && value ) { + out[ name ] = value; + } + } + return out; } -function parseAnthropicHeaders( raw: string | undefined ): Record< string, string > | undefined { +// `STUDIO_OPENAI_DEFAULT_HEADERS` is JSON.stringify'd in providers.ts. +function parseJsonHeaders( raw: string | undefined ): Record< string, string > { if ( ! raw ) { - return undefined; + return {}; } try { const parsed = JSON.parse( raw ); - return parsed && typeof parsed === 'object' - ? ( parsed as Record< string, string > ) - : undefined; + return parsed && typeof parsed === 'object' ? ( parsed as Record< string, string > ) : {}; } catch { - return undefined; + return {}; } } export function parseFeedbackExtraction( text: string ): ExtractedFeedbackFields | null { - // Strip code fences if the model wrapped its JSON despite the instructions. + // Strip code fences if the model wraps its JSON despite the instructions. const stripped = text .replace( /^```(?:json)?\s*/i, '' ) .replace( /```\s*$/, '' ) diff --git a/apps/cli/ai/slash-commands.ts b/apps/cli/ai/slash-commands.ts index 0f70af76ad..800e7e3257 100644 --- a/apps/cli/ai/slash-commands.ts +++ b/apps/cli/ai/slash-commands.ts @@ -236,20 +236,17 @@ function formatFeedbackEnvironment( ctx: SlashCommandContext ): string { // and notifications. const FEEDBACK_TITLE_MAX_LENGTH = 80; -function fallbackTitleFromSummary( summary: string ): string { - const newlineIdx = summary.indexOf( '\n' ); - const firstLine = ( newlineIdx === -1 ? summary : summary.slice( 0, newlineIdx ) ).trim(); - if ( firstLine.length <= FEEDBACK_TITLE_MAX_LENGTH ) { - return firstLine; +function truncateWithEllipsis( text: string, max: number ): string { + if ( text.length <= max ) { + return text; } - return firstLine.slice( 0, FEEDBACK_TITLE_MAX_LENGTH - 1 ).trimEnd() + '…'; + return text.slice( 0, max - 1 ).trimEnd() + '…'; } -function clampTitle( title: string ): string { - if ( title.length <= FEEDBACK_TITLE_MAX_LENGTH ) { - return title; - } - return title.slice( 0, FEEDBACK_TITLE_MAX_LENGTH - 1 ).trimEnd() + '…'; +function fallbackTitleFromSummary( summary: string ): string { + const newlineIdx = summary.indexOf( '\n' ); + const firstLine = ( newlineIdx === -1 ? summary : summary.slice( 0, newlineIdx ) ).trim(); + return truncateWithEllipsis( firstLine, FEEDBACK_TITLE_MAX_LENGTH ); } type PlatformLabel = 'Mac Silicon' | 'Mac Intel' | 'Windows'; @@ -283,7 +280,9 @@ function composeFeedbackPrefill( extracted: ExtractedFeedbackFields | null, ctx: SlashCommandContext ): FeedbackPrefill { - const aiTitle = extracted?.title ? clampTitle( extracted.title ) : null; + const aiTitle = extracted?.title + ? truncateWithEllipsis( extracted.title, FEEDBACK_TITLE_MAX_LENGTH ) + : null; return { title: aiTitle ?? fallbackTitleFromSummary( summary ), summary, @@ -379,6 +378,21 @@ async function runFeedbackSlashCommand( } const prefill = composeFeedbackPrefill( summary, extracted, ctx ); + const previewRows: Array< [ string, string | null ] > = [ + [ __( 'Title' ), prefill.title ], + [ __( 'Description' ), prefill.summary ], + [ __( 'Steps' ), prefill.steps ], + [ __( 'Expected' ), prefill.expected ], + [ __( 'Actual' ), prefill.actual ], + [ __( 'Impact' ), prefill.impact ], + [ __( 'Workarounds' ), prefill.workaround ], + [ __( 'Platform' ), prefill.platform ], + [ __( 'Environment' ), prefill.environmentLine ], + ]; + const previewBody = previewRows + .filter( ( [ , value ] ) => value ) + .map( ( [ label, value ] ) => ` - ${ label }: ${ value }` ) + .join( '\n' ); ctx.ui.showInfo( [ __( 'Submit Feedback / Bug Report' ), @@ -389,66 +403,8 @@ async function runFeedbackSlashCommand( 'GitHub will open with what we could pre-fill — fill in the rest before submitting.' ), '', - sprintf( - /* translators: %s: deduced issue title */ - __( ' - Title: %s' ), - prefill.title - ), - sprintf( - /* translators: %s: the user's feedback text */ - __( ' - Description: %s' ), - prefill.summary - ), - prefill.steps - ? sprintf( - /* translators: %s: AI-extracted reproduction steps */ - __( ' - Steps: %s' ), - prefill.steps - ) - : null, - prefill.expected - ? sprintf( - /* translators: %s: AI-extracted "what you expected" */ - __( ' - Expected: %s' ), - prefill.expected - ) - : null, - prefill.actual - ? sprintf( - /* translators: %s: AI-extracted "what actually happened" */ - __( ' - Actual: %s' ), - prefill.actual - ) - : null, - prefill.impact - ? sprintf( - /* translators: %s: AI-extracted impact label, e.g. "All" */ - __( ' - Impact: %s' ), - prefill.impact - ) - : null, - prefill.workaround - ? sprintf( - /* translators: %s: AI-extracted workaround availability */ - __( ' - Workarounds: %s' ), - prefill.workaround - ) - : null, - prefill.platform - ? sprintf( - /* translators: %s: detected platform like "Mac Silicon" */ - __( ' - Platform: %s' ), - prefill.platform - ) - : null, - sprintf( - /* translators: %s: environment summary like "darwin/arm64, Node v22.18.0, …" */ - __( ' - Environment: %s' ), - prefill.environmentLine - ), - ] - .filter( ( line ): line is string => typeof line === 'string' ) - .join( '\n' ) + previewBody, + ].join( '\n' ) ); let confirmed: boolean; diff --git a/apps/cli/ai/tests/feedback-extraction.test.ts b/apps/cli/ai/tests/feedback-extraction.test.ts index 432872a207..9463352fdf 100644 --- a/apps/cli/ai/tests/feedback-extraction.test.ts +++ b/apps/cli/ai/tests/feedback-extraction.test.ts @@ -1,5 +1,9 @@ -import { describe, expect, it } from 'vitest'; -import { parseFeedbackExtraction } from 'cli/ai/feedback-extraction'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { extractFeedbackFields, parseFeedbackExtraction } from 'cli/ai/feedback-extraction'; + +vi.mock( 'cli/ai/auth', () => ( { + resolveAiEnvironment: vi.fn(), +} ) ); describe( 'parseFeedbackExtraction', () => { it( 'returns null for non-JSON output', () => { @@ -56,3 +60,214 @@ describe( 'parseFeedbackExtraction', () => { expect( parsed?.expected ).toBe( 'real value' ); } ); } ); + +describe( 'extractFeedbackFields orchestration', () => { + let fetchSpy: ReturnType< typeof vi.spyOn >; + + beforeEach( async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockReset(); + fetchSpy = vi.spyOn( global, 'fetch' ); + } ); + + afterEach( () => { + fetchSpy.mockRestore(); + } ); + + it( 'returns null when resolveAiEnvironment throws (e.g. not authenticated)', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockRejectedValue( + new Error( 'auth required' ) + ); + + const result = await extractFeedbackFields( 'something broken', { + provider: 'wpcom', + model: 'claude-sonnet-4-6', + } ); + expect( result ).toBeNull(); + expect( fetchSpy ).not.toHaveBeenCalled(); + } ); + + it( 'returns null when Anthropic credentials are missing in the resolved env', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + ANTHROPIC_BASE_URL: 'https://example.test', + // No AUTH_TOKEN, no API_KEY + } ); + + const result = await extractFeedbackFields( 'broken', { + provider: 'wpcom', + model: 'claude-sonnet-4-6', + } ); + expect( result ).toBeNull(); + expect( fetchSpy ).not.toHaveBeenCalled(); + } ); + + it( 'sends Anthropic line-format custom headers (NOT JSON) on the wpcom proxy path', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + ANTHROPIC_BASE_URL: 'https://wpcom.example/proxy', + ANTHROPIC_AUTH_TOKEN: 'abc123', + ANTHROPIC_CUSTOM_HEADERS: + 'X-WPCOM-AI-Feature: studio-assistant-anthropic\nX-WPCOM-Session-ID: session-1', + } ); + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify( { + content: [ + { + type: 'text', + text: '{"title":"x","steps":null,"expected":null,"actual":null,"impact":null,"workaround":null}', + }, + ], + } ), + { status: 200 } + ) + ); + + await extractFeedbackFields( 'a problem', { + provider: 'wpcom', + model: 'claude-sonnet-4-6', + } ); + + expect( fetchSpy ).toHaveBeenCalledOnce(); + const [ url, init ] = fetchSpy.mock.calls[ 0 ] as [ string, RequestInit ]; + expect( url ).toBe( 'https://wpcom.example/proxy/v1/messages' ); + const headers = init.headers as Record< string, string >; + expect( headers.authorization ).toBe( 'Bearer abc123' ); + expect( headers[ 'X-WPCOM-AI-Feature' ] ).toBe( 'studio-assistant-anthropic' ); + expect( headers[ 'X-WPCOM-Session-ID' ] ).toBe( 'session-1' ); + } ); + + it( 'uses x-api-key for direct Anthropic when only ANTHROPIC_API_KEY is set', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + ANTHROPIC_API_KEY: 'sk-ant-xyz', + } ); + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify( { + content: [ + { + type: 'text', + text: '{"title":"x","steps":null,"expected":null,"actual":null,"impact":null,"workaround":null}', + }, + ], + } ), + { status: 200 } + ) + ); + + await extractFeedbackFields( 'a problem', { + provider: 'anthropic-api-key', + model: 'claude-opus-4-7', + } ); + + const [ url, init ] = fetchSpy.mock.calls[ 0 ] as [ string, RequestInit ]; + // Falls back to the public Anthropic endpoint when no base URL is set. + expect( url ).toBe( 'https://api.anthropic.com/v1/messages' ); + const headers = init.headers as Record< string, string >; + expect( headers[ 'x-api-key' ] ).toBe( 'sk-ant-xyz' ); + expect( headers.authorization ).toBeUndefined(); + } ); + + it( 'routes OpenAI models to /chat/completions with json_object response_format', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + OPENAI_BASE_URL: 'https://wpcom.example/proxy/v1', + OPENAI_API_KEY: 'wpcom-token', + STUDIO_OPENAI_DEFAULT_HEADERS: '{"X-WPCOM-AI-Feature":"studio-assistant"}', + } ); + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify( { + choices: [ + { + message: { + content: + '{"title":"y","steps":null,"expected":null,"actual":null,"impact":null,"workaround":null}', + }, + }, + ], + } ), + { status: 200 } + ) + ); + + await extractFeedbackFields( 'a problem', { provider: 'wpcom', model: 'gpt-5.5' } ); + + const [ url, init ] = fetchSpy.mock.calls[ 0 ] as [ string, RequestInit ]; + expect( url ).toBe( 'https://wpcom.example/proxy/v1/chat/completions' ); + const body = JSON.parse( init.body as string ); + expect( body.response_format ).toEqual( { type: 'json_object' } ); + expect( body.model ).toBe( 'gpt-5.5' ); + const headers = init.headers as Record< string, string >; + expect( headers[ 'X-WPCOM-AI-Feature' ] ).toBe( 'studio-assistant' ); + } ); + + it( 'returns null when fetch responds with a non-2xx status', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + ANTHROPIC_BASE_URL: 'https://wpcom.example/proxy', + ANTHROPIC_AUTH_TOKEN: 'abc', + } ); + fetchSpy.mockResolvedValue( new Response( '{}', { status: 500 } ) ); + + const result = await extractFeedbackFields( 'broken', { + provider: 'wpcom', + model: 'claude-sonnet-4-6', + } ); + expect( result ).toBeNull(); + } ); + + it( 'returns null when fetch throws (network error / abort)', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + ANTHROPIC_BASE_URL: 'https://wpcom.example/proxy', + ANTHROPIC_AUTH_TOKEN: 'abc', + } ); + fetchSpy.mockRejectedValue( new DOMException( 'aborted', 'AbortError' ) ); + + const result = await extractFeedbackFields( 'broken', { + provider: 'wpcom', + model: 'claude-sonnet-4-6', + } ); + expect( result ).toBeNull(); + } ); + + it( 'parses extracted fields end-to-end on a successful Anthropic response', async () => { + const auth = await import( 'cli/ai/auth' ); + ( auth.resolveAiEnvironment as ReturnType< typeof vi.fn > ).mockResolvedValue( { + ANTHROPIC_BASE_URL: 'https://wpcom.example/proxy', + ANTHROPIC_AUTH_TOKEN: 'abc', + } ); + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify( { + content: [ + { + type: 'text', + text: JSON.stringify( { + title: 'Sites fail to start', + steps: '1. Update Studio.\n2. Click Start.', + expected: 'Site starts.', + actual: 'Spinner forever.', + impact: 'All', + workaround: 'No and the app is unusable', + } ), + }, + ], + } ), + { status: 200 } + ) + ); + + const result = await extractFeedbackFields( + 'After updating, sites just spin. The app is dead.', + { provider: 'wpcom', model: 'claude-sonnet-4-6' } + ); + expect( result?.title ).toBe( 'Sites fail to start' ); + expect( result?.impact ).toBe( 'All' ); + expect( result?.workaround ).toBe( 'No and the app is unusable' ); + } ); +} ); From 5c058ef01231db9ed32f3840bad73c02dfcc8d0a Mon Sep 17 00:00:00 2001 From: Miguel Lezama Date: Thu, 7 May 2026 14:45:48 -0300 Subject: [PATCH 7/7] Share header-env parsers, harden parseFeedbackExtraction, drop AbortController boilerplate --- apps/cli/ai/feedback-extraction.ts | 55 +++---------------- apps/cli/ai/header-env.ts | 52 ++++++++++++++++++ apps/cli/ai/runtimes/pi/index.ts | 37 +------------ apps/cli/ai/tests/feedback-extraction.test.ts | 9 ++- 4 files changed, 64 insertions(+), 89 deletions(-) create mode 100644 apps/cli/ai/header-env.ts diff --git a/apps/cli/ai/feedback-extraction.ts b/apps/cli/ai/feedback-extraction.ts index ba76929437..60dfa88d69 100644 --- a/apps/cli/ai/feedback-extraction.ts +++ b/apps/cli/ai/feedback-extraction.ts @@ -1,5 +1,6 @@ import { getAiModelFamily, type AiModelFamily, type AiModelId } from '@studio/common/ai/models'; import { resolveAiEnvironment } from 'cli/ai/auth'; +import { parseAnthropicHeaderEnv, parseJsonHeaderEnv } from 'cli/ai/header-env'; import type { AiProviderId } from 'cli/ai/providers'; // Hard-coded against the bug_report.yml dropdown labels — keep in sync with @@ -77,21 +78,17 @@ export async function extractFeedbackFields( return null; } - const controller = new AbortController(); - const timer = setTimeout( () => controller.abort(), FEEDBACK_EXTRACTION_TIMEOUT_MS ); try { const text = await callExtractionModel( getAiModelFamily( ctx.model ), ctx.model, env, description, - controller.signal + AbortSignal.timeout( FEEDBACK_EXTRACTION_TIMEOUT_MS ) ); return text ? parseFeedbackExtraction( text ) : null; } catch { return null; - } finally { - clearTimeout( timer ); } } @@ -128,7 +125,7 @@ async function callAnthropicMessages( const headers: Record< string, string > = { 'content-type': 'application/json', 'anthropic-version': ANTHROPIC_API_VERSION, - ...parseAnthropicLineHeaders( env.ANTHROPIC_CUSTOM_HEADERS ), + ...( parseAnthropicHeaderEnv( env.ANTHROPIC_CUSTOM_HEADERS ) ?? {} ), }; // wpcom routes Anthropic via Bearer; direct Anthropic uses x-api-key. if ( authToken ) { @@ -173,7 +170,7 @@ async function callOpenAiCompletions( const headers: Record< string, string > = { 'content-type': 'application/json', authorization: `Bearer ${ apiKey }`, - ...parseJsonHeaders( env.STUDIO_OPENAI_DEFAULT_HEADERS ), + ...( parseJsonHeaderEnv( env.STUDIO_OPENAI_DEFAULT_HEADERS ) ?? {} ), }; const response = await fetch( `${ baseUrl.replace( /\/+$/, '' ) }/chat/completions`, { @@ -201,41 +198,6 @@ async function callOpenAiCompletions( return json.choices?.[ 0 ]?.message?.content ?? null; } -// `ANTHROPIC_CUSTOM_HEADERS` is encoded by `buildAnthropicCustomHeaders` in -// providers.ts as `Name: Value\nName: Value` — NOT JSON. Mirrors the runtime's -// `parseAnthropicHeaderEnv` so the wpcom feature header reaches the proxy. -function parseAnthropicLineHeaders( raw: string | undefined ): Record< string, string > { - if ( ! raw ) { - return {}; - } - const out: Record< string, string > = {}; - for ( const line of raw.split( '\n' ) ) { - const idx = line.indexOf( ':' ); - if ( idx <= 0 ) { - continue; - } - const name = line.slice( 0, idx ).trim(); - const value = line.slice( idx + 1 ).trim(); - if ( name && value ) { - out[ name ] = value; - } - } - return out; -} - -// `STUDIO_OPENAI_DEFAULT_HEADERS` is JSON.stringify'd in providers.ts. -function parseJsonHeaders( raw: string | undefined ): Record< string, string > { - if ( ! raw ) { - return {}; - } - try { - const parsed = JSON.parse( raw ); - return parsed && typeof parsed === 'object' ? ( parsed as Record< string, string > ) : {}; - } catch { - return {}; - } -} - export function parseFeedbackExtraction( text: string ): ExtractedFeedbackFields | null { // Strip code fences if the model wraps its JSON despite the instructions. const stripped = text @@ -249,7 +211,7 @@ export function parseFeedbackExtraction( text: string ): ExtractedFeedbackFields } catch { return null; } - if ( ! parsed || typeof parsed !== 'object' ) { + if ( ! parsed || typeof parsed !== 'object' || Array.isArray( parsed ) ) { return null; } @@ -272,11 +234,8 @@ function sanitizeString( value: unknown ): string | null { return trimmed ? trimmed : null; } -function matchEnum< T extends readonly string[] >( - value: unknown, - allowed: T -): T[ number ] | null { +function matchEnum< T extends string >( value: unknown, allowed: readonly T[] ): T | null { return typeof value === 'string' && ( allowed as readonly string[] ).includes( value ) - ? ( value as T[ number ] ) + ? ( value as T ) : null; } diff --git a/apps/cli/ai/header-env.ts b/apps/cli/ai/header-env.ts new file mode 100644 index 0000000000..c7df8063cb --- /dev/null +++ b/apps/cli/ai/header-env.ts @@ -0,0 +1,52 @@ +// Header env-var parsers shared by the agent runtime and one-shot AI calls. +// Two formats live in `apps/cli/ai/providers.ts` — `ANTHROPIC_CUSTOM_HEADERS` +// is `Name: Value\nName: Value` (built by `buildAnthropicCustomHeaders`) and +// `STUDIO_OPENAI_DEFAULT_HEADERS` is `JSON.stringify`'d. Anything that needs +// to read these env vars should import from here so the line vs JSON contract +// stays in one place. + +export function parseAnthropicHeaderEnv( + value: string | undefined +): Record< string, string > | undefined { + if ( ! value ) { + return undefined; + } + const out: Record< string, string > = {}; + for ( const line of value.split( '\n' ) ) { + const idx = line.indexOf( ':' ); + if ( idx <= 0 ) { + continue; + } + const name = line.slice( 0, idx ).trim(); + const v = line.slice( idx + 1 ).trim(); + if ( name && v ) { + out[ name ] = v; + } + } + return Object.keys( out ).length ? out : undefined; +} + +export function parseJsonHeaderEnv( + value: string | undefined +): Record< string, string > | undefined { + if ( ! value ) { + return undefined; + } + try { + const parsed: unknown = JSON.parse( value ); + if ( parsed && typeof parsed === 'object' && ! Array.isArray( parsed ) ) { + const entries = Object.entries( parsed as Record< string, unknown > ).filter( + ( [ , v ] ) => typeof v === 'string' + ) as [ string, string ][]; + return entries.length ? Object.fromEntries( entries ) : undefined; + } + console.warn( + 'STUDIO_OPENAI_DEFAULT_HEADERS must be a JSON object of string→string pairs; ignoring custom headers.' + ); + } catch { + console.warn( + 'STUDIO_OPENAI_DEFAULT_HEADERS contained malformed JSON; ignoring custom headers.' + ); + } + return undefined; +} diff --git a/apps/cli/ai/runtimes/pi/index.ts b/apps/cli/ai/runtimes/pi/index.ts index 62a26ca334..0ce311ef68 100644 --- a/apps/cli/ai/runtimes/pi/index.ts +++ b/apps/cli/ai/runtimes/pi/index.ts @@ -27,6 +27,7 @@ import { type AiModelFamily, type AiModelId, } from '@studio/common/ai/models'; +import { parseAnthropicHeaderEnv, parseJsonHeaderEnv } from 'cli/ai/header-env'; import { buildSystemPrompt } from 'cli/ai/system-prompt'; import { resolveStudioToolDefinitions } from 'cli/ai/tools'; import { createAskUserQuestionTool } from 'cli/ai/tools/ask-user-question'; @@ -449,39 +450,3 @@ function buildAgentTools( ...piTools, ]; } - -function parseJsonHeaderEnv( value: string | undefined ): Record< string, string > | undefined { - if ( ! value ) return undefined; - try { - const parsed: unknown = JSON.parse( value ); - if ( parsed && typeof parsed === 'object' && ! Array.isArray( parsed ) ) { - const entries = Object.entries( parsed as Record< string, unknown > ).filter( - ( [ , v ] ) => typeof v === 'string' - ) as [ string, string ][]; - return entries.length ? Object.fromEntries( entries ) : undefined; - } - console.warn( - 'STUDIO_OPENAI_DEFAULT_HEADERS must be a JSON object of string→string pairs; ignoring custom headers.' - ); - } catch { - console.warn( - 'STUDIO_OPENAI_DEFAULT_HEADERS contained malformed JSON; ignoring custom headers.' - ); - } - return undefined; -} - -function parseAnthropicHeaderEnv( - value: string | undefined -): Record< string, string > | undefined { - if ( ! value ) return undefined; - const out: Record< string, string > = {}; - for ( const line of value.split( '\n' ) ) { - const idx = line.indexOf( ':' ); - if ( idx <= 0 ) continue; - const name = line.slice( 0, idx ).trim(); - const v = line.slice( idx + 1 ).trim(); - if ( name && v ) out[ name ] = v; - } - return Object.keys( out ).length ? out : undefined; -} diff --git a/apps/cli/ai/tests/feedback-extraction.test.ts b/apps/cli/ai/tests/feedback-extraction.test.ts index 9463352fdf..72fe69468b 100644 --- a/apps/cli/ai/tests/feedback-extraction.test.ts +++ b/apps/cli/ai/tests/feedback-extraction.test.ts @@ -10,12 +10,11 @@ describe( 'parseFeedbackExtraction', () => { expect( parseFeedbackExtraction( 'sorry, I can only respond with JSON' ) ).toBeNull(); } ); - it( 'returns null for valid JSON that is not an object', () => { + it( 'returns null for valid JSON that is not a plain object', () => { expect( parseFeedbackExtraction( '"a string"' ) ).toBeNull(); - expect( parseFeedbackExtraction( '[1, 2, 3]' ) ).not.toBeNull(); - // Arrays are objects in JS — but downstream readers will see all-null - // fields, which is harmless. The strict guard is the enum/string - // validation. + expect( parseFeedbackExtraction( '[1, 2, 3]' ) ).toBeNull(); + expect( parseFeedbackExtraction( '42' ) ).toBeNull(); + expect( parseFeedbackExtraction( 'null' ) ).toBeNull(); } ); it( 'strips ```json fences if the model wraps its output despite instructions', () => {