diff --git a/apps/web/src/app/api/openrouter/[...path]/route.test.ts b/apps/web/src/app/api/openrouter/[...path]/route.test.ts index c41f559eb3..93696bb624 100644 --- a/apps/web/src/app/api/openrouter/[...path]/route.test.ts +++ b/apps/web/src/app/api/openrouter/[...path]/route.test.ts @@ -134,6 +134,7 @@ const provider = { apiUrl: 'https://openrouter.ai/api/v1', apiKey: 'test-key', supportedChatApis: ['chat_completions', 'responses', 'messages'], + responseTransforms: null, transformRequest: jest.fn(), } satisfies Provider; @@ -291,7 +292,31 @@ describe('POST /api/openrouter/v1/chat/completions rules-engine actions', () => expect.anything(), expect.anything(), expect.anything(), - expect.objectContaining({ vercel_request_id: 'iad1::iad1::request-id' }) + expect.objectContaining({ vercel_request_id: 'iad1::iad1::request-id' }), + null + ); + }); + + it('passes provider response transforms to the response rewriter', async () => { + const responseTransforms = { thoughtContentMapping: 'extra_content.flags.thought' }; + mockedGetProvider.mockResolvedValue({ + kind: 'provider', + provider: { ...provider, responseTransforms }, + userByok: null, + bypassAccessCheck: false, + }); + const { POST } = await import('./route'); + + const response = await POST(makeRequest(makeBody()) as never); + + expect(response.status).toBe(200); + expect(mockedRewriteModelResponse).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + responseTransforms ); }); diff --git a/apps/web/src/app/api/openrouter/[...path]/route.ts b/apps/web/src/app/api/openrouter/[...path]/route.ts index 2441650c63..de892013dd 100644 --- a/apps/web/src/app/api/openrouter/[...path]/route.ts +++ b/apps/web/src/app/api/openrouter/[...path]/route.ts @@ -1039,6 +1039,7 @@ export async function POST(request: NextRequest): Promise { ); }); +describe('custom LLM thought content mapping configuration', () => { + const config = { + internal_id: 'upstream-model', + base_url: 'https://llm.example.com/v1', + }; + + it('accepts a dot-separated property path', () => { + expect( + CustomLlmApiConfigSchema.safeParse({ + ...config, + thought_content_mapping: 'extra_content.flags.thought', + }).success + ).toBe(true); + }); + + it.each(['', 'extra_content..thought', 'constructor.prototype.thought'])( + 'rejects unsafe property path %p', + thought_content_mapping => { + expect( + CustomLlmApiConfigSchema.safeParse({ ...config, thought_content_mapping }).success + ).toBe(false); + } + ); +}); + +describe('buildDirectProvider response transforms', () => { + it('exposes the thought content property path without other upstream configuration', () => { + const provider = buildDirectProvider('custom', ['chat_completions'], { + internal_id: 'upstream-model', + base_url: 'https://llm.example.com/v1', + api_key: 'test-key', + thought_content_mapping: 'extra_content.flags.thought', + }); + + expect(provider.responseTransforms).toEqual({ + thoughtContentMapping: 'extra_content.flags.thought', + }); + }); + + it('sets response transforms to null when no mapping is configured', () => { + const provider = buildDirectProvider('custom', ['chat_completions'], { + internal_id: 'upstream-model', + base_url: 'https://llm.example.com/v1', + api_key: 'test-key', + }); + + expect(provider.responseTransforms).toBeNull(); + }); +}); + describe('buildDirectProvider thought signature mapping', () => { it('maps assistant tool-call signatures and removes camel-case transport fields', async () => { const request = makeRequest(); diff --git a/apps/web/src/lib/ai-gateway/experiments/build-direct-provider.ts b/apps/web/src/lib/ai-gateway/experiments/build-direct-provider.ts index 0a73488fd3..74a8696b0f 100644 --- a/apps/web/src/lib/ai-gateway/experiments/build-direct-provider.ts +++ b/apps/web/src/lib/ai-gateway/experiments/build-direct-provider.ts @@ -202,6 +202,9 @@ export function buildDirectProvider( apiUrl: upstream.base_url, apiKey: upstream.api_key, supportedChatApis, + responseTransforms: upstream.thought_content_mapping + ? { thoughtContentMapping: upstream.thought_content_mapping } + : null, async transformRequest(context) { if (upstream.remove_from_body) { const body = context.request.body as Record; diff --git a/apps/web/src/lib/ai-gateway/providers/get-provider.ts b/apps/web/src/lib/ai-gateway/providers/get-provider.ts index 16f5e53fe3..5e814a16eb 100644 --- a/apps/web/src/lib/ai-gateway/providers/get-provider.ts +++ b/apps/web/src/lib/ai-gateway/providers/get-provider.ts @@ -80,6 +80,7 @@ async function checkDirectBYOK( apiUrl: directByok.base_url, apiKey: userByok[0].decryptedAPIKey, supportedChatApis: directByok.supported_chat_apis, + responseTransforms: null, async transformRequest(context) { context.request.body.model = directByokModel.id; directByok.transformRequest(context); diff --git a/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts b/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts index b151f2a5ae..559350b9ee 100644 --- a/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts +++ b/apps/web/src/lib/ai-gateway/providers/provider-definitions.ts @@ -9,6 +9,7 @@ export default { apiUrl: 'https://openrouter.ai/api/v1', apiKey: getEnvVariable('OPENROUTER_API_KEY'), supportedChatApis: ['chat_completions', 'messages', 'responses'], + responseTransforms: null, async transformRequest() {}, }, ALIBABA: { @@ -17,6 +18,7 @@ export default { apiKey: getEnvVariable('ALIBABA_API_KEY'), // Prompt caching is not supported on the responses API for Alibaba; enabling it is therefore dangerous. supportedChatApis: ['chat_completions' /*, 'responses'*/], + responseTransforms: null, async transformRequest(context) { context.request.body.enable_thinking = !isReasoningExplicitlyDisabled(context.request); }, @@ -27,6 +29,7 @@ export default { apiKey: getEnvVariable('BYTEDANCE_API_KEY'), // Prompt caching is not supported on the responses API for Bytedance; enabling it is therefore dangerous. supportedChatApis: ['chat_completions' /*, 'responses'*/], + responseTransforms: null, async transformRequest(context) { if (!isReasoningExplicitlyDisabled(context.request)) { context.request.body.thinking = { type: 'enabled' }; @@ -49,6 +52,7 @@ export default { apiUrl: 'https://api.withmartian.com/v1', apiKey: getEnvVariable('MARTIAN_API_KEY'), supportedChatApis: ['chat_completions', 'responses', 'messages'], + responseTransforms: null, async transformRequest(context) { delete context.request.body.provider; }, @@ -58,6 +62,7 @@ export default { apiUrl: 'https://api.mistral.ai/v1', apiKey: getEnvVariable('MISTRAL_API_KEY'), supportedChatApis: [], + responseTransforms: null, async transformRequest() {}, }, STREAMLAKE: { @@ -65,6 +70,7 @@ export default { apiUrl: 'https://vanchin.streamlake.ai/api/gateway/v1/endpoints', apiKey: getEnvVariable('STREAMLAKE_API_KEY'), supportedChatApis: ['chat_completions'], + responseTransforms: null, async transformRequest(context) { delete context.request.body.provider; }, @@ -74,6 +80,7 @@ export default { apiUrl: 'https://ai-gateway.vercel.sh/v1', apiKey: getEnvVariable('VERCEL_AI_GATEWAY_API_KEY'), supportedChatApis: ['chat_completions', 'messages', 'responses'], + responseTransforms: null, async transformRequest(context) { await applyVercelSettings(context.model, context.request, context.userByok); }, diff --git a/apps/web/src/lib/ai-gateway/providers/types.ts b/apps/web/src/lib/ai-gateway/providers/types.ts index 58138eade9..c0e950cf31 100644 --- a/apps/web/src/lib/ai-gateway/providers/types.ts +++ b/apps/web/src/lib/ai-gateway/providers/types.ts @@ -35,10 +35,15 @@ export type TransformRequestContext = { export type GatewayChatApiKind = GatewayRequest['kind']; +export type ProviderResponseTransforms = { + thoughtContentMapping: string | null; +}; + export type Provider = { id: ProviderId; apiUrl: string; apiKey: string; supportedChatApis: ReadonlyArray; + responseTransforms: ProviderResponseTransforms | null; transformRequest(context: TransformRequestContext): Promise; }; diff --git a/apps/web/src/lib/ai-gateway/providers/upstream-request.generation.test.ts b/apps/web/src/lib/ai-gateway/providers/upstream-request.generation.test.ts index 9cd3b48437..1f27b36433 100644 --- a/apps/web/src/lib/ai-gateway/providers/upstream-request.generation.test.ts +++ b/apps/web/src/lib/ai-gateway/providers/upstream-request.generation.test.ts @@ -20,6 +20,7 @@ const provider: Provider = { apiUrl: 'https://openrouter.example/api/v1', apiKey: 'test-api-key', supportedChatApis: [], + responseTransforms: null, transformRequest: async () => {}, }; diff --git a/apps/web/src/lib/rewriteModelResponse.test.ts b/apps/web/src/lib/rewriteModelResponse.test.ts index 64826224e1..7a424d51fc 100644 --- a/apps/web/src/lib/rewriteModelResponse.test.ts +++ b/apps/web/src/lib/rewriteModelResponse.test.ts @@ -116,8 +116,15 @@ function dataObjects(sse: string): unknown[] { .map(payload => JSON.parse(payload)); } +const rewriteChatCompletionsWithoutTransforms: typeof rewriteModelResponse_Messages = ( + response, + removeCost, + capture, + vercelRequestId +) => rewriteModelResponse_ChatCompletions(response, removeCost, capture, vercelRequestId, null); + const rewriters = [ - ['Chat Completions', rewriteModelResponse_ChatCompletions], + ['Chat Completions', rewriteChatCompletionsWithoutTransforms], ['Messages', rewriteModelResponse_Messages], ['Responses', rewriteModelResponse_Responses], ] as const; @@ -206,7 +213,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { }, }); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const json = await result.json(); expect(json.model).toBe('upstream-model'); @@ -229,7 +236,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { }, }); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const json = await result.json(); expect(json.usage.prompt_tokens_details.cached_tokens).toBe(0); @@ -242,7 +249,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { headers: { 'content-type': 'application/json' }, }); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); expect(result.status).toBe(502); expect(await result.text()).toBe('not-json{'); @@ -270,7 +277,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { ); try { - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const reader = result.body?.getReader(); expect(reader).toBeDefined(); await reader?.read(); @@ -295,7 +302,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { 'data: {"id":"gen-chat","model":"upstream-model","choices":[]}\n\n' ); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const sse = await readOutputStream(result); const events = dataObjects(sse) as Array<{ error?: { code: number; type: string } }>; @@ -311,7 +318,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { 'data: [DONE]\n\n' ); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const sse = await readOutputStream(result); const [chunk] = dataObjects(sse) as Array<{ model: string; @@ -324,6 +331,27 @@ describe('rewriteModelResponse_ChatCompletions', () => { expect(dataPayloads(sse)).toContain('[DONE]'); }); + test('moves marked delta content to reasoning content', async () => { + const upstream = sseResponse( + 'data: {"model":"upstream-model","choices":[{"index":0,"delta":{"content":"first thought","extra_content":{"flags":{"thought":true}}}},{"index":1,"delta":{"content":"answer","extra_content":{"flags":{"thought":false}}}},{"index":2,"delta":{"content":"more answer"}}]}\n\n' + + 'data: [DONE]\n\n' + ); + + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, { + thoughtContentMapping: 'extra_content.flags.thought', + }); + const [chunk] = dataObjects(await readOutputStream(result)) as Array<{ + choices: Array<{ delta: Record }>; + }>; + + expect(chunk.choices[0].delta).toEqual({ + reasoning_content: 'first thought', + extra_content: { flags: { thought: true } }, + }); + expect(chunk.choices[1].delta).toMatchObject({ content: 'answer' }); + expect(chunk.choices[2].delta).toMatchObject({ content: 'more answer' }); + }); + test('does not treat a null error field as terminal', async () => { const upstream = sseResponse( 'data: {"id":"gen-chat","error":null,"choices":[]}\n\n' + @@ -331,7 +359,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { 'data: [DONE]\n\n' ); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const sse = await readOutputStream(result); expect(sse).toContain('still streaming'); @@ -350,7 +378,8 @@ describe('rewriteModelResponse_ChatCompletions', () => { upstream, true, capture, - 'iad1::terminal-request' + 'iad1::terminal-request', + null ); const sse = await readOutputStream(result); @@ -375,7 +404,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { 'data: {"model":"upstream-model","usage":{"cost":1,"is_byok":true,"prompt_tokens":4,"completion_tokens":2,"total_tokens":6,"prompt_tokens_details":{}}}\n\n' ); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const sse = await readOutputStream(result); const [chunk] = dataObjects(sse) as Array<{ model: string; @@ -399,7 +428,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { ': openrouter heartbeat\n\n' + 'data: {"model":"upstream-model","choices":[]}\n\n' ); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); const sse = await readOutputStream(result); expect(sse).toContain(': KILO PROCESSING'); @@ -411,7 +440,7 @@ describe('rewriteModelResponse_ChatCompletions', () => { headers: { 'content-type': 'text/event-stream' }, }); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, null, null, null); expect(await readOutputStream(result)).toBe(''); }); @@ -699,7 +728,7 @@ describe.each([ [ 'Chat Completions', 'chat_completions', - rewriteModelResponse_ChatCompletions, + rewriteChatCompletionsWithoutTransforms, 'data: {"id":"gen-chat","choices":[]}\n\n', 'data: {"error":{"code":429,"message":"rate limited"}}\n\n', 'gen-chat', @@ -774,7 +803,8 @@ describe('rewriteModelResponse', () => { 'openai/gpt-5', 'openrouter', 'chat_completions', - makeLogging({ organization_id: KILO_ORGANIZATION_ID }) + makeLogging({ organization_id: KILO_ORGANIZATION_ID }), + null ); expect(result).not.toBeNull(); @@ -800,7 +830,8 @@ describe('rewriteModelResponse', () => { 'openai/gpt-5', 'openrouter', 'chat_completions', - makeLogging({ organization_id: '00000000-0000-0000-0000-000000000000' }) + makeLogging({ organization_id: '00000000-0000-0000-0000-000000000000' }), + null ); expect(await result.json()).toMatchObject({ @@ -821,7 +852,8 @@ describe('rewriteModelResponse', () => { 'google/gemma-4-26b-a4b-it:free', 'openrouter', 'chat_completions', - makeLogging() + makeLogging(), + null ); expect(result).not.toBeNull(); @@ -840,7 +872,8 @@ describe('rewriteModelResponse', () => { QWEN37_PLUS_MODEL_ID, 'openrouter', 'chat_completions', - makeLogging() + makeLogging(), + null ); // The upstream-reported cost does not reflect the custom pricing, so it @@ -858,7 +891,8 @@ describe('rewriteModelResponse', () => { 'openai/gpt-5', 'openrouter', 'chat_completions', - makeLogging({ organization_id: '00000000-0000-0000-0000-000000000000' }) + makeLogging({ organization_id: '00000000-0000-0000-0000-000000000000' }), + null ); expect(result).not.toBeNull(); @@ -976,7 +1010,7 @@ describe('request log capture', () => { headers: { 'content-type': 'text/event-stream' }, }); - const result = await rewriteModelResponse_ChatCompletions(upstream, true, capture, null); + const result = await rewriteModelResponse_ChatCompletions(upstream, true, capture, null, null); const reader = result.body?.getReader(); await reader?.cancel(); diff --git a/apps/web/src/lib/rewriteModelResponse.ts b/apps/web/src/lib/rewriteModelResponse.ts index ebf260680a..7c09cef0f3 100644 --- a/apps/web/src/lib/rewriteModelResponse.ts +++ b/apps/web/src/lib/rewriteModelResponse.ts @@ -3,7 +3,7 @@ import { isKiloExclusiveFreeModel } from '@/lib/ai-gateway/models'; import { getCustomPricing } from '@/lib/ai-gateway/custom-pricing'; import { detectToolCallArgumentErrors } from '@/lib/ai-gateway/api-request-log-errors'; import type { GatewayRequest } from '@/lib/ai-gateway/providers/openrouter/types'; -import type { ProviderId } from '@/lib/ai-gateway/providers/types'; +import type { ProviderId, ProviderResponseTransforms } from '@/lib/ai-gateway/providers/types'; import { getOutputHeaders } from '@/lib/ai-gateway/llm-proxy-helpers'; import type { ChatCompletionChunk, OpenRouterUsage } from '@/lib/ai-gateway/processUsage.types'; import { isDynamicallyOptedIntoRequestLogging } from '@/lib/ai-gateway/request-logging-opt-ins'; @@ -351,11 +351,40 @@ function rewriteUsage(usage: OpenRouterUsage, removeCost: boolean) { } } +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function getPropertyPath(target: unknown, path: string): unknown { + let current = target; + for (const segment of path.split('.')) { + if (!isRecord(current)) { + return undefined; + } + current = current[segment]; + } + return current; +} + +function rewriteThoughtContent(delta: unknown, path: string) { + if ( + !isRecord(delta) || + typeof delta.content !== 'string' || + getPropertyPath(delta, path) !== true + ) { + return; + } + + delta.reasoning_content = delta.content; + delete delta.content; +} + export async function rewriteModelResponse_ChatCompletions( response: Response, removeCost: boolean, capture: RequestLogCapture | null, - vercelRequestId: string | null + vercelRequestId: string | null, + responseTransforms: ProviderResponseTransforms | null ) { const headers = getOutputHeaders(response); @@ -430,12 +459,18 @@ export async function rewriteModelResponse_ChatCompletions( }); } - const delta = json.choices?.[0]?.delta; - if (delta) { + for (const choice of json.choices ?? []) { + const delta = choice.delta; + if (!delta) { + continue; + } // Some APIs set null here, which is not accepted by OpenCode - if (delta?.role === null) { + if (delta.role === null) { delete delta.role; } + if (responseTransforms?.thoughtContentMapping) { + rewriteThoughtContent(delta, responseTransforms.thoughtContentMapping); + } } if (!json.choices) { @@ -836,7 +871,8 @@ export async function rewriteModelResponse( model: string, providerId: ProviderId, kind: GatewayRequest['kind'], - logging: RequestLoggingParams + logging: RequestLoggingParams, + responseTransforms: ProviderResponseTransforms | null ): Promise { const capture = await createRequestLogCapture(response, model, providerId, logging); const requiresCostRemoval = @@ -850,7 +886,8 @@ export async function rewriteModelResponse( response, requiresCostRemoval, capture, - vercelRequestId + vercelRequestId, + responseTransforms ); } if (kind === 'responses') { diff --git a/packages/db/src/schema-types.ts b/packages/db/src/schema-types.ts index 751bf8f1b7..8474552727 100644 --- a/packages/db/src/schema-types.ts +++ b/packages/db/src/schema-types.ts @@ -1950,6 +1950,7 @@ export const CustomLlmApiConfigSchema = z.object({ remove_from_body: z.array(z.string()).optional(), compression: CustomLlmCompressionSchema.optional(), thought_signature_mapping: CustomLlmPropertyPathSchema.optional(), + thought_content_mapping: CustomLlmPropertyPathSchema.optional(), }); export type CustomLlmApiConfig = z.infer;