diff --git a/PanTS-Demo/package-lock.json b/PanTS-Demo/package-lock.json index a7caf084..c2b8c005 100644 --- a/PanTS-Demo/package-lock.json +++ b/PanTS-Demo/package-lock.json @@ -22,6 +22,7 @@ "esbuild": "^0.25.9", "jszip": "^3.10.1", "pako": "^3.0.1", + "pdfjs-dist": "^6.2.108", "react": "^19.1.1", "react-dom": "^19.1.1", "react-router-dom": "^7.8.1", @@ -32,7 +33,7 @@ "devDependencies": { "@eslint/js": "^9.33.0", "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^7.0.0", + "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/pako": "^2.0.4", @@ -2668,6 +2669,256 @@ "node": ">= 10" } }, + "node_modules/@napi-rs/canvas": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.6.tgz", + "integrity": "sha512-Ud4d0jLr9vSaakvfOTIjgP3hoY8kMIYgHC4mtE4L0bP3St9lf6TyCVsInvFIOEofzcUeMfDydHDy8Fs2WuWz+A==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "1.0.6", + "@napi-rs/canvas-darwin-arm64": "1.0.6", + "@napi-rs/canvas-darwin-x64": "1.0.6", + "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.6", + "@napi-rs/canvas-linux-arm64-gnu": "1.0.6", + "@napi-rs/canvas-linux-arm64-musl": "1.0.6", + "@napi-rs/canvas-linux-riscv64-gnu": "1.0.6", + "@napi-rs/canvas-linux-x64-gnu": "1.0.6", + "@napi-rs/canvas-linux-x64-musl": "1.0.6", + "@napi-rs/canvas-win32-arm64-msvc": "1.0.6", + "@napi-rs/canvas-win32-x64-msvc": "1.0.6" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.6.tgz", + "integrity": "sha512-CZUmZEyN/cmSN1OhRJ44vBE6xyb5ekBbsIF8dqNgizbnAiHSFNi8eAA4oyyIAe5KzJ72K16OKC2lAynZLhv32Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.6.tgz", + "integrity": "sha512-FGJsIngRfS9LDaX4t/lFthqn2Km6b61ONzjJFC6+yuabL1NgaK7x0fwrj8276OQFkjCvSpJZ7+CeZWgcy03F2w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.6.tgz", + "integrity": "sha512-wLDrtfX9V4bu6O55/WwGvPVCjfunL2bEm59lzIFk9ZQOt7oJrPzdx/ytZiQr6bMZ9XOFwvXujszYtpoCK7yUbg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.6.tgz", + "integrity": "sha512-8TllEP/9UPDVxrBN/tnbGBsxxwG8Eo2tgyW8wDy+Gi9rm9si2I6MsDv9MlfI3a7JOuzkBcurHYDHZxwLgzAIdA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.6.tgz", + "integrity": "sha512-YF15AKoGrgU7aoA03CR+cATCRPQcVt6L14Lb/a9hqK7pT7/DhIkqh4qyl+a2GLfqx++yIIJfBuZ3sHM3zlXM1Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.6.tgz", + "integrity": "sha512-zXeVWNFpj720HZln0bfAs+1/nP3Wy6Z5hUw60UJu+cvgips5jZtuX2+EjDHFk98q3BY35GaE1TQLwE9S244vPA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.6.tgz", + "integrity": "sha512-kvy1Ypq4ODEI2WRNGvs588qTMcWCjlEIuZRyW8JXpiBnMUirgxQukIDFVDQEcrLlHP5cqZ9cvvfOkBJ7koOktA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.6.tgz", + "integrity": "sha512-czHnfLgSCl48iluS6bNrytGYhsbSTUuIvMucAwO0mPEj1uhkHk6SUiRI0xoqv0PSFzpGFfSnQvFpm7vV+tDO+A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.6.tgz", + "integrity": "sha512-afsg6GyfUx8qZzjsywjimXUEWcSWETY5F80hmMb2FWVSXkRbor8IpftY2gXnasq+Y42Xug/76qTHwqvti7HRTw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.6.tgz", + "integrity": "sha512-byzha357piy0wgK6IZOQtU4plpt94f6CxHFS7LhJpWYrDWWT/AitQNxRMfCNNaQHZXUB3BwNhQSfH2RpiZ0WaQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.6.tgz", + "integrity": "sha512-gWLOgKGZz3hZhzg49q2PJIURwy2pCWh1zvNFXmLO1yineOPhEpoL3EV6/VIjOiKa1l2P6ieP12Eo385u1MLLVQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@niivue/niivue": { "version": "0.62.1", "resolved": "https://registry.npmjs.org/@niivue/niivue/-/niivue-0.62.1.tgz", @@ -3827,9 +4078,9 @@ } }, "node_modules/@testing-library/jest-dom": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-7.0.0.tgz", - "integrity": "sha512-HKAH9C6mBo5yBG6yRO5i43L2iisencAo5z+o5P/saHUoY+miC5ivXRxHBJcFyB5ypPNxHJdK3BoF/3O4DIptMg==", + "version": "6.9.1", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", + "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", "dev": true, "license": "MIT", "dependencies": { @@ -3841,12 +4092,9 @@ "redent": "^3.0.0" }, "engines": { - "node": ">=22", + "node": ">=14", "npm": ">=6", "yarn": ">=1" - }, - "peerDependencies": { - "@testing-library/dom": ">=10 <11" } }, "node_modules/@testing-library/jest-dom/node_modules/dom-accessibility-api": { @@ -9457,6 +9705,18 @@ "dev": true, "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "6.2.108", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.2.108.tgz", + "integrity": "sha512-YxFb+SQcodN2rnX9Tn3dHYlqfb7NjlzzfONPpJd+AKoKtUjEdevTfbC07d5TcczzOK6261auRkP/M8OBHs9vFQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=22.13.0 || >=24" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^1.0.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/PanTS-Demo/package.json b/PanTS-Demo/package.json index 8a39b6da..d465e2bd 100644 --- a/PanTS-Demo/package.json +++ b/PanTS-Demo/package.json @@ -27,6 +27,7 @@ "esbuild": "^0.25.9", "jszip": "^3.10.1", "pako": "^3.0.1", + "pdfjs-dist": "^6.2.108", "react": "^19.1.1", "react-dom": "^19.1.1", "react-router-dom": "^7.8.1", @@ -37,7 +38,7 @@ "devDependencies": { "@eslint/js": "^9.33.0", "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^7.0.0", + "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/pako": "^2.0.4", diff --git a/PanTS-Demo/src/components/AIAssistant/AISidebar.css b/PanTS-Demo/src/components/AIAssistant/AISidebar.css index 5d006d45..77e4db16 100644 --- a/PanTS-Demo/src/components/AIAssistant/AISidebar.css +++ b/PanTS-Demo/src/components/AIAssistant/AISidebar.css @@ -415,6 +415,44 @@ font-size: 11px; } +/* "Best for ..." line under each model name in the picker. */ +.ai-model-menu__desc { + overflow: hidden; + color: rgba(255, 255, 255, 0.42); + font-size: 10px; + line-height: 1.35; + text-overflow: ellipsis; + white-space: nowrap; +} + +.ai-model-menu__item:hover .ai-model-menu__desc, +.ai-model-menu__item[data-selected="true"] .ai-model-menu__desc { + color: rgba(255, 255, 255, 0.56); +} + +/* Notice shown in the menu when images are attached: the vision model takes + over for that message. Uses the sidebar's existing blue accent. */ +.ai-model-menu__note { + margin: 0 2px 5px; + padding: 7px 9px; + border: 1px solid rgba(110, 168, 254, 0.28); + border-radius: 8px; + background: rgba(110, 168, 254, 0.16); + color: #a8c8ff; + font-size: 10.5px; + line-height: 1.4; +} + +/* Picker button while the vision model is active for the pending message. */ +.ai-model-picker__button[data-vision="true"] { + border-color: rgba(110, 168, 254, 0.34); + color: #a8c8ff; +} + +.ai-model-picker__button[data-vision="true"] .ai-model-picker__dot { + opacity: 1; +} + @keyframes ai-model-pulse { 0%, 100% { diff --git a/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx b/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx index a5643fce..bc15be66 100644 --- a/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx +++ b/PanTS-Demo/src/components/AIAssistant/AISidebar.tsx @@ -1,7 +1,6 @@ import React, { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { useAuth } from "../../contexts/authContext"; -import { track } from "../../helpers/analytics"; import { API_BASE } from "../../helpers/constants"; import type { AIAction, @@ -27,7 +26,24 @@ const MODEL_STORAGE_KEY = "bodymaps-ai-model-v2"; // Reasoning models emit a chain-of-thought that can leak into the answer on // older Ollama; we avoid picking them as the initial default. -const REASONING_MODEL = /qwen3|deepseek-r1|-r1\b|:think|marco-o1|qwq/i; +const REASONING_MODEL = /qwen3(?!-vl)|deepseek-r1|-r1\b|:think|marco-o1|qwq/i; + +// Short "best for ..." line shown under each model in the picker, so someone +// who has never used local models knows which one to pick. Order matters: +// vision ("vl") must match before the generic qwen check. +function modelDescription(name: string): string { + const n = name.toLowerCase(); + if (n.includes("vl") || n.includes("vision")) { + return "For complex image and snapshot tasks"; + } + if (n.includes("llama")) { + return "Best all-around"; + } + if (n.includes("qwen")) { + return "Fastest for quick answers"; + } + return "General-purpose local model"; +} const SendIcon = () => ( { }); } +// Cap on extracted document text sent to the model — keeps the prompt inside +// the local model's context window (roughly 1.5k tokens of document). +const PDF_TEXT_LIMIT = 6000; + +// Extract the text layer of an attached PDF in the browser, so the model can +// actually read the document instead of only seeing its filename. Returns null +// for scanned/image-only PDFs (no text layer) and on any parse failure. +async function extractPdfText(file: File): Promise { + try { + const pdfjs = await import("pdfjs-dist"); + pdfjs.GlobalWorkerOptions.workerSrc = new URL( + "pdfjs-dist/build/pdf.worker.min.mjs", + import.meta.url + ).toString(); + const loadingTask = pdfjs.getDocument({ data: await file.arrayBuffer() }); + try { + const doc = await loadingTask.promise; + const maxPages = Math.min(doc.numPages, 12); + let text = ""; + for (let pageNum = 1; pageNum <= maxPages && text.length < PDF_TEXT_LIMIT; pageNum++) { + const page = await doc.getPage(pageNum); + const content = await page.getTextContent(); + const pageText = content.items + .map((item) => ("str" in item ? item.str : "")) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + if (pageText) text += pageText + "\n"; + } + const trimmed = text.trim(); + return trimmed ? trimmed.slice(0, PDF_TEXT_LIMIT) : null; + } finally { + // Release the worker-side parsed document — pdf.js pins it otherwise, + // so attaching several PDFs would grow tab memory until reload. + void loadingTask.destroy(); + } + } catch (error) { + console.warn("[BodyMaps AI pdf extract]", error); + return null; + } +} + // Minimal markdown: **bold** and line breaks. Kept intentionally small so the // assistant text stays clean and minimalist rather than heavily styled. function renderMessageText(content: string) { @@ -260,7 +318,10 @@ type StreamEvent = | { type: "actions"; actions?: AIAction[] } | { type: "final"; reply?: string; actions?: AIAction[]; source?: string; model?: string | null } | { type: "done" } - | { type: "error"; message?: string }; + | { type: "error"; message?: string } + // The agent decided it needs to SEE the CT views: the browser captures the + // four panes and re-sends this turn with the images attached (self-capture). + | { type: "need_capture" }; export default function AISidebar({ open, @@ -285,9 +346,12 @@ export default function AISidebar({ const [capturing, setCapturing] = useState(false); const [models, setModels] = useState([]); const [selectedModel, setSelectedModel] = useState(""); - // The assistant runs on the server and is metered per account, so it needs a - // signed-in user — the same rule the Upload page applies to inference. - const { isAuthenticated, promptAuth } = useAuth(); + // Backend's vision model — shown as the active model whenever images are + // attached, because the backend switches to it for those messages. + const [visionModel, setVisionModel] = useState(""); + // The assistant is open to everyone — no sign-in required. promptAuth is + // kept only to handle a 401 from an older backend that still gates it. + const { promptAuth } = useAuth(); const [modelState, setModelState] = useState("loading"); const [modelMenuOpen, setModelMenuOpen] = useState(false); @@ -314,6 +378,7 @@ export default function AISidebar({ const nextModels: AIModelInfo[] = Array.isArray(data.models) ? data.models : []; setModels(nextModels); + setVisionModel(String(data.vision_model || "")); if (!data.available || nextModels.length === 0) { setSelectedModel(""); @@ -512,6 +577,17 @@ export default function AISidebar({ } catch { next.push({ id: makeId("att"), name: file.name, kind: "file", source: "upload" }); } + } else if (file.type === "application/pdf" || /\.pdf$/i.test(file.name)) { + // Pull the PDF's text so the model can read the document and react to + // its content (and ask follow-ups), not just see a filename. + const textContent = await extractPdfText(file); + next.push({ + id: makeId("att"), + name: file.name, + kind: "file", + source: "upload", + textContent: textContent ?? undefined, + }); } else { next.push({ id: makeId("att"), name: file.name, kind: "file", source: "upload" }); } @@ -632,7 +708,7 @@ export default function AISidebar({ assistantId: string, payload: Record, signal?: AbortSignal - ): Promise => { + ): Promise<{ captureRequested: boolean }> => { const response = await fetch(`${API_BASE}/api/ai-command-stream`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -656,9 +732,13 @@ export default function AISidebar({ const decoder = new TextDecoder(); let buffer = ""; let actionsApplied = false; + let captureRequested = false; const handleEvent = (event: StreamEvent) => { switch (event.type) { + case "need_capture": + captureRequested = true; + break; case "status": updateMessage(assistantId, (m) => ({ ...m, status: event.text ?? "" })); break; @@ -733,7 +813,7 @@ export default function AISidebar({ /* ignore trailing partial */ } } - return true; + return { captureRequested }; }, [applyReturnedActions, updateMessage] ); @@ -774,19 +854,11 @@ export default function AISidebar({ const outgoingAttachments = attachments; if ((!text && outgoingAttachments.length === 0) || loading) return; - // Caught here as well as server-side: no point sending a request that can - // only come back 401, and the popup is the useful response either way. - if (!isAuthenticated) { - promptAuth(); - return; - } - const conversation = messages .filter((message) => message.role === "user" || message.role === "assistant") .slice(-12) .map((message) => ({ role: message.role, content: message.content })); - track("assistant_send_message"); setInput(""); setAttachments([]); if (textareaRef.current) textareaRef.current.style.height = "auto"; @@ -825,9 +897,19 @@ export default function AISidebar({ .filter((item) => item.kind === "file") .map((item) => item.name); - const composedMessage = fileNames.length - ? `${text}\n\n[Attached files: ${fileNames.join(", ")}]`.trim() - : text; + // Extracted document text rides inside the message so the model can + // read what was attached and respond to its actual content. + const documentExcerpts = outgoingAttachments + .filter((item) => item.kind === "file" && item.textContent) + .map((item) => `Content of attached document "${item.name}":\n${item.textContent}`); + + let composedMessage = text; + if (fileNames.length) { + composedMessage = `${composedMessage}\n\n[Attached files: ${fileNames.join(", ")}]`.trim(); + } + if (documentExcerpts.length) { + composedMessage = `${composedMessage}\n\n${documentExcerpts.join("\n\n")}`.trim(); + } // When screenshots are attached, include the color→organ legend so the // vision model can identify each colored region. @@ -847,6 +929,10 @@ export default function AISidebar({ model: selectedModel || null, images, mask_legend: maskLegend, + // Lets the backend's agent offer its capture_views tool; auto_captured + // marks the follow-up request so capture is requested at most once. + can_capture: !!captureViewport, + auto_captured: false, }; const controller = new AbortController(); @@ -855,7 +941,43 @@ export default function AISidebar({ e instanceof DOMException ? e.name === "AbortError" : (e as { name?: string })?.name === "AbortError"; try { - await streamResponse(assistantId, payload, controller.signal); + const first = await streamResponse(assistantId, payload, controller.signal); + // Self-capture: the agent asked to SEE the views. Capture the four + // panes right here in the browser and continue the same turn with the + // images attached (the backend switches to the vision model). + if (first.captureRequested && captureViewport && !controller.signal.aborted) { + updateMessage(assistantId, (m) => ({ ...m, status: "Capturing the CT views" })); + let shots: { name: string; dataUrl: string }[] = []; + try { + shots = await captureViewport(); + } catch (error) { + console.error("[BodyMaps AI self-capture]", error); + } + if (shots.length) { + // Transparency: show the shots the assistant took on the user's + // message, exactly as if they had clicked the camera themselves. + const shotAttachments: ChatAttachment[] = shots.map((shot) => ({ + id: makeId("shot"), + name: `${shot.name} view`, + kind: "image", + dataUrl: shot.dataUrl, + label: shot.name, + source: "screenshot", + })); + updateMessage(userId, (m) => ({ + ...m, + attachments: [...(m.attachments ?? []), ...shotAttachments], + })); + } + updateMessage(assistantId, (m) => ({ ...m, status: "Reading the views" })); + const followPayload: Record = { + ...payload, + images: shots.map((shot) => shot.dataUrl), + mask_legend: getMaskLegend ? getMaskLegend() : maskLegend, + auto_captured: true, + }; + await streamResponse(assistantId, followPayload, controller.signal); + } } catch (streamError) { if (isAbort(streamError)) { // User pressed Stop — keep whatever was streamed, no error. @@ -907,9 +1029,6 @@ export default function AISidebar({ attachments, loading, messages, - // Without these the guard closes over a stale auth state, and signing in - // mid-session would leave the composer still refusing to send. - isAuthenticated, promptAuth, caseId, sessionId, @@ -937,12 +1056,18 @@ export default function AISidebar({ } }; + // The instant images are attached, the picker reflects the vision model — + // that IS the model that will answer this message (backend switches too). + const hasImageAttachments = attachments.some((att) => att.kind === "image"); + const effectiveModel = + hasImageAttachments && visionModel ? visionModel : selectedModel; + const modelLabel = modelState === "loading" ? "Loading models" : modelState === "fallback" ? "Local fallback" - : selectedModel || models[0]?.name || "Model"; + : effectiveModel || models[0]?.name || "Model"; const canSend = !loading && (input.trim().length > 0 || attachments.length > 0); @@ -1148,10 +1273,11 @@ export default function AISidebar({ aria-hidden="true" tabIndex={-1} /> + {/* Attach stays usable while a reply is generating, so the next + message can be prepared without waiting. */}
{modelMenuOpen && (
-
Local model
+
Local models
+ {hasImageAttachments && visionModel && ( +
+ Images attached — {visionModel} will answer this message. +
+ )} {models.length > 0 ? ( models.map((model) => ( )) @@ -1209,10 +1345,16 @@ export default function AISidebar({