From ec38704ac9673ad326df1a963ee18ad5881ac563 Mon Sep 17 00:00:00 2001 From: Lukas Hirt Date: Mon, 27 Jul 2026 13:37:05 +0200 Subject: [PATCH] fix(web-app-ai-sensitive-data-scanner): render scan findings instead of raw json Signed-off-by: Lukas Hirt --- .../src/composables/useScanner.ts | 46 ++++- .../tests/unit/composables/useScanner.spec.ts | 173 ++++++++++++++++++ 2 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 packages/web-app-ai-sensitive-data-scanner/tests/unit/composables/useScanner.spec.ts diff --git a/packages/web-app-ai-sensitive-data-scanner/src/composables/useScanner.ts b/packages/web-app-ai-sensitive-data-scanner/src/composables/useScanner.ts index 451d4225f..414d13573 100644 --- a/packages/web-app-ai-sensitive-data-scanner/src/composables/useScanner.ts +++ b/packages/web-app-ai-sensitive-data-scanner/src/composables/useScanner.ts @@ -120,22 +120,58 @@ export function useScanner(llmConfig: LlmConfig | null, resources: Ref { + // Models are instructed to return raw JSON, but frequently wrap it in a markdown + // code fence (```json ... ``` or ``` ... ```) anyway. Strip that fence before parsing + // so we don't fall through to treating the whole fenced blob as narrative text. + const CODE_FENCE_RE = /^```(?:json)?\s*([\s\S]*?)\s*```$/i + + function extractJsonCandidate(rawContent: string): string { + const trimmed = rawContent.trim() + const fenceMatch = trimmed.match(CODE_FENCE_RE) + if (fenceMatch) { + return fenceMatch[1].trim() + } + return trimmed + } + + function tryParseFindings(candidate: string): ScanFinding[] | null { try { - const parsed = JSON.parse(rawContent) as { findings?: unknown[] } + const parsed = JSON.parse(candidate) as { findings?: unknown[] } if (Array.isArray(parsed.findings)) { - const findings = parsed.findings.filter( + return parsed.findings.filter( (f): f is ScanFinding => typeof f === 'object' && f !== null && typeof (f as Record).category === 'string' && typeof (f as Record).excerpt === 'string' ) - return { findings, narrative: '' } } } catch { - // not valid JSON — store raw response as narrative (plain-text model fallback) + // not valid JSON } + return null + } + + function parseLlmResponse(rawContent: string): Pick { + const candidate = extractJsonCandidate(rawContent) + + const directFindings = tryParseFindings(candidate) + if (directFindings) { + return { findings: directFindings, narrative: '' } + } + + // Fall back to extracting the substring between the first "{" and last "}" in case + // the model added leading/trailing prose around the JSON object. + const firstBrace = candidate.indexOf('{') + const lastBrace = candidate.lastIndexOf('}') + if (firstBrace !== -1 && lastBrace > firstBrace) { + const braceFindings = tryParseFindings(candidate.slice(firstBrace, lastBrace + 1)) + if (braceFindings) { + return { findings: braceFindings, narrative: '' } + } + } + + // Genuinely not JSON — store raw response as narrative (plain-text model fallback) return { findings: [], narrative: rawContent } } diff --git a/packages/web-app-ai-sensitive-data-scanner/tests/unit/composables/useScanner.spec.ts b/packages/web-app-ai-sensitive-data-scanner/tests/unit/composables/useScanner.spec.ts new file mode 100644 index 000000000..e207c9197 --- /dev/null +++ b/packages/web-app-ai-sensitive-data-scanner/tests/unit/composables/useScanner.spec.ts @@ -0,0 +1,173 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { ref } from 'vue' + +vi.mock('pdfjs-dist', () => ({ + getDocument: vi.fn() +})) +vi.mock('pdfjs-dist/build/pdf.worker.min.mjs', () => ({})) + +vi.mock('vue3-gettext', () => ({ + useGettext: () => ({ $gettext: (s: string) => s }) +})) + +vi.mock('@ownclouders/web-pkg', () => ({ + useClientService: vi.fn(), + useSpacesStore: vi.fn() +})) + +vi.mock('../../../src/composables/useLlm', () => ({ + useLlm: vi.fn() +})) + +import { useScanner } from '../../../src/composables/useScanner' +import type { ScanResource } from '../../../src/composables/useScanner' +import { useLlm } from '../../../src/composables/useLlm' +import { useClientService, useSpacesStore } from '@ownclouders/web-pkg' + +const BASE_CONFIG = { endpoint: window.location.origin + '/ai-llm-proxy/v1', model: 'test-model' } + +const TEXT_RESOURCE: ScanResource = { + id: 'f1', + name: 'report.txt', + extension: 'txt', + storageId: 'space-1', + path: '/report.txt' +} + +function setupUseLlmMock({ status = 'ready', callLlm = vi.fn() } = {}) { + vi.mocked(useLlm).mockReturnValue({ + config: ref(BASE_CONFIG as any), + status: ref(status as any), + callLlm + }) + return callLlm +} + +function setupClientServiceMock({ fileContents = 'Some document content.' } = {}) { + const getFileContents = vi.fn().mockResolvedValue({ response: { data: fileContents } }) + vi.mocked(useClientService).mockReturnValue({ + webdav: { getFileContents } + } as any) + vi.mocked(useSpacesStore).mockReturnValue({ + getSpace: vi.fn().mockReturnValue({ id: 'space-1' }) + } as any) + return { getFileContents } +} + +async function runScanForResource(resource: ScanResource) { + const instance = useScanner(BASE_CONFIG, ref([resource])) + await instance.runScan() + return instance +} + +describe('useScanner parseLlmResponse (via runScan)', () => { + beforeEach(() => { + vi.restoreAllMocks() + }) + + it('parses a plain (unfenced) JSON findings response', async () => { + const callLlm = setupUseLlmMock({ + callLlm: vi.fn().mockResolvedValue({ + choices: [ + { message: { content: '{"findings":[{"category":"pii","excerpt":"John Doe"}]}' } } + ] + }) + }) + setupClientServiceMock() + + const { scanResults } = await runScanForResource(TEXT_RESOURCE) + + expect(callLlm).toHaveBeenCalled() + expect(scanResults.value[0].state).toBe('done') + expect(scanResults.value[0].narrative).toBe('') + expect(scanResults.value[0].findings).toEqual([{ category: 'pii', excerpt: 'John Doe' }]) + }) + + it('strips a ```json markdown code fence before parsing, producing structured findings instead of raw-JSON narrative', async () => { + setupUseLlmMock({ + callLlm: vi.fn().mockResolvedValue({ + choices: [ + { + message: { + content: '```json\n{"findings":[{"category":"pii","excerpt":"John Doe"}]}\n```' + } + } + ] + }) + }) + setupClientServiceMock() + + const { scanResults } = await runScanForResource(TEXT_RESOURCE) + + expect(scanResults.value[0].state).toBe('done') + expect(scanResults.value[0].narrative).toBe('') + expect(scanResults.value[0].findings).toEqual([{ category: 'pii', excerpt: 'John Doe' }]) + }) + + it('strips a bare ``` fence (no "json" language tag) before parsing', async () => { + setupUseLlmMock({ + callLlm: vi.fn().mockResolvedValue({ + choices: [ + { + message: { + content: '```\n{"findings":[{"category":"credentials","excerpt":"API_KEY=[REDACTED]"}]}\n```' + } + } + ] + }) + }) + setupClientServiceMock() + + const { scanResults } = await runScanForResource(TEXT_RESOURCE) + + expect(scanResults.value[0].narrative).toBe('') + expect(scanResults.value[0].findings).toEqual([ + { category: 'credentials', excerpt: 'API_KEY=[REDACTED]' } + ]) + }) + + it('extracts JSON surrounded by leading/trailing prose', async () => { + setupUseLlmMock({ + callLlm: vi.fn().mockResolvedValue({ + choices: [ + { + message: { + content: + 'Sure, here is the analysis:\n{"findings":[{"category":"confidential","excerpt":"internal roadmap"}]}\nLet me know if you need more.' + } + } + ] + }) + }) + setupClientServiceMock() + + const { scanResults } = await runScanForResource(TEXT_RESOURCE) + + expect(scanResults.value[0].narrative).toBe('') + expect(scanResults.value[0].findings).toEqual([ + { category: 'confidential', excerpt: 'internal roadmap' } + ]) + }) + + it('falls back to narrative text for a genuinely non-JSON prose response', async () => { + setupUseLlmMock({ + callLlm: vi.fn().mockResolvedValue({ + choices: [ + { + message: { + content: 'The document contains a phone number and an email address.' + } + } + ] + }) + }) + setupClientServiceMock() + + const { scanResults } = await runScanForResource(TEXT_RESOURCE) + + expect(scanResults.value[0].findings).toEqual([]) + expect(scanResults.value[0].narrative).toBe( + 'The document contains a phone number and an email address.' + ) + }) +})