From 74ac5cda1867d6f5b7fb32b937cb5b18599913e1 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Thu, 13 Aug 2026 17:07:02 +0800 Subject: [PATCH 01/15] feat(storage): add Phase 5 SQLite contract preview --- .../modules/canvas/persistence-validation.ts | 45 ++ .../storage/backends/disk/space-nodes.test.ts | 4 +- .../backends/disk/space-record-validation.ts | 47 +- .../backends/disk/structured-store.test.ts | 77 +-- .../storage/backends/sqlite/contracts.test.ts | 142 +++++ .../storage/backends/sqlite/database.ts | 381 ++++++++++++ .../storage/backends/sqlite/fixtures/v1.sql | 109 ++++ .../backends/sqlite/integration.test.ts | 578 ++++++++++++++++++ .../storage/backends/sqlite/space-logs.ts | 269 ++++++++ .../storage/backends/sqlite/space-nodes.ts | 251 ++++++++ .../backends/sqlite/space-repository.ts | 236 +++++++ .../storage/backends/sqlite/space-tasks.ts | 202 ++++++ .../storage/backends/sqlite/space-write.ts | 214 +++++++ .../backends/sqlite/structured-store.ts | 68 +++ .../storage/backends/sqlite/test-support.ts | 172 ++++++ .../modules/storage/backends/sqlite/values.ts | 360 +++++++++++ .../ports/contracts/space-nodes.contract.ts | 63 +- .../ports/contracts/space-tasks.contract.ts | 309 ++++++++++ .../src/modules/storage/ports/structured.ts | 31 +- .../src/modules/storage/profile.test.ts | 11 +- apps/server/src/modules/storage/profile.ts | 26 +- docs/proposals/multi-backend-storage.md | 7 +- 22 files changed, 3454 insertions(+), 148 deletions(-) create mode 100644 apps/server/src/modules/canvas/persistence-validation.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/contracts.test.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/database.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql create mode 100644 apps/server/src/modules/storage/backends/sqlite/integration.test.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/space-logs.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/space-nodes.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/space-repository.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/space-tasks.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/space-write.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/structured-store.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/test-support.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/values.ts create mode 100644 apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts diff --git a/apps/server/src/modules/canvas/persistence-validation.ts b/apps/server/src/modules/canvas/persistence-validation.ts new file mode 100644 index 000000000..20b3613f4 --- /dev/null +++ b/apps/server/src/modules/canvas/persistence-validation.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Runtime validation shared by structured storage adapters. */ + +function finiteNumber(value: unknown): boolean { + return typeof value === 'number' && Number.isFinite(value); +} + +/** Return the first minimal CanvasFile shape violation, if any. */ +export function canvasFileShapeError( + value: unknown, + expectedCanvasId: string, +): string | null { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return 'must be an object'; + } + + const record = value as Record; + if (record['canvasId'] !== expectedCanvasId) { + return `canvasId must equal ${JSON.stringify(expectedCanvasId)}`; + } + if (record['title'] !== null && typeof record['title'] !== 'string') { + return 'title must be a string or null'; + } + if (!finiteNumber(record['version'])) + return 'version must be a finite number'; + if (!finiteNumber(record['createdAt'])) { + return 'createdAt must be a finite number'; + } + if (!finiteNumber(record['updatedAt'])) { + return 'updatedAt must be a finite number'; + } + + const state = record['state']; + if (typeof state !== 'object' || state === null || Array.isArray(state)) { + return 'state must be an object'; + } + const stateRecord = state as Record; + if (!Array.isArray(stateRecord['nodes'])) + return 'state.nodes must be an array'; + if (!Array.isArray(stateRecord['edges'])) + return 'state.edges must be an array'; + return null; +} diff --git a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts index 19cc23b7c..8c91fed63 100644 --- a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts +++ b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts @@ -54,8 +54,10 @@ describeSpaceNodesContract('Disk', async () => { if (!created.ok) throw new Error('Node contract Space already exists'); const store = new DiskStructuredStore(); + const space = store.space('node-space'); return { - repository: store.space('node-space').nodes, + repository: space.nodes, + space, missingRepository: store.space('missing-node-space').nodes, expectedCanvasId: 'node-space', cleanup: () => { diff --git a/apps/server/src/modules/storage/backends/disk/space-record-validation.ts b/apps/server/src/modules/storage/backends/disk/space-record-validation.ts index 1792458c4..e31bb697c 100644 --- a/apps/server/src/modules/storage/backends/disk/space-record-validation.ts +++ b/apps/server/src/modules/storage/backends/disk/space-record-validation.ts @@ -1,55 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -/** Runtime validation shared by strict Disk Space-record boundaries. */ +/** Runtime validation and strict reads for Disk Space-record boundaries. */ import { readJsonStrict } from '../../../../utils/fs.js'; +import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; import type { CanvasFile } from '../../../canvas/persistence-types.js'; -function finiteNumber(value: unknown): boolean { - return typeof value === 'number' && Number.isFinite(value); -} - -/** Return the first minimal {@link CanvasFile} shape violation, if any. */ -export function canvasFileShapeError( - value: unknown, - expectedCanvasId: string, -): string | null { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return 'must be an object'; - } - - const record = value as Record; - if (record['canvasId'] !== expectedCanvasId) { - return `canvasId must equal ${JSON.stringify(expectedCanvasId)}`; - } - if (record['title'] !== null && typeof record['title'] !== 'string') { - return 'title must be a string or null'; - } - if (!finiteNumber(record['version'])) { - return 'version must be a finite number'; - } - if (!finiteNumber(record['createdAt'])) { - return 'createdAt must be a finite number'; - } - if (!finiteNumber(record['updatedAt'])) { - return 'updatedAt must be a finite number'; - } - - const state = record['state']; - if (typeof state !== 'object' || state === null || Array.isArray(state)) { - return 'state must be an object'; - } - const stateRecord = state as Record; - if (!Array.isArray(stateRecord['nodes'])) { - return 'state.nodes must be an array'; - } - if (!Array.isArray(stateRecord['edges'])) { - return 'state.edges must be an array'; - } - return null; -} +export { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; /** * Strictly read and validate one indexed `space.json` path. diff --git a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts index 0f3dadd8d..8b20988d9 100644 --- a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts @@ -28,6 +28,7 @@ import { import { DiskStructuredStore } from './structured-store.js'; import { toSafeFilename } from '../../../../utils/naming.js'; import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; +import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; import type { CanvasFile } from '../../../canvas/persistence-types.js'; @@ -141,68 +142,6 @@ describe('Disk Space Tasks', () => { rmSync(root, { recursive: true, force: true }); }); - it('serializes Task and Run mutations across independent handles', async () => { - const first = store.space('canvas-task').tasks; - const second = store.space('canvas-task').tasks; - await Promise.all([ - first.create({ - taskId: 'task-a', - canvasId: 'canvas-task', - goal: 'Goal A', - defaultRootProfileId: 'profile-a', - anchorNodeId: 'node-a', - createdAt: 1, - }), - second.create({ - taskId: 'task-b', - canvasId: 'canvas-task', - goal: 'Goal B', - defaultRootProfileId: 'profile-b', - anchorNodeId: 'node-b', - createdAt: 2, - }), - ]); - await first.runs.create({ - runId: 'run-a', - taskId: 'task-a', - canvasIdSnapshot: 'canvas-task', - goalSnapshot: 'Goal A', - rootProfileIdSnapshot: 'profile-a', - status: 'pending', - createdAt: 3, - }); - const updated = await second.runs.update('run-a', { - rootNodeId: 'node-root', - rootThreadId: 'thread-root', - status: 'running', - startedAt: 4, - }); - - expect(updated.status).toBe('running'); - await expect(first.read()).resolves.toMatchObject({ - version: 1, - tasks: [ - expect.objectContaining({ taskId: 'task-a' }), - expect.objectContaining({ taskId: 'task-b' }), - ], - runs: [ - expect.objectContaining({ - runId: 'run-a', - rootNodeId: 'node-root', - rootThreadId: 'thread-root', - }), - ], - }); - }); - - it('returns an empty versioned snapshot when no Task store exists', async () => { - await expect(store.space('canvas-empty').tasks.read()).resolves.toEqual({ - version: 1, - tasks: [], - runs: [], - }); - }); - it('completes a running Run atomically and keeps its message immutable', async () => { const runs = store.space('canvas-task').tasks.runs; await expect( @@ -253,20 +192,8 @@ describe('Disk Space Tasks', () => { ).resolves.toEqual({ outcome: 'run_not_found' }); }); - it('rejects mutations for a missing Space', async () => { - await expect( - store.space('missing-canvas').tasks.create({ - taskId: 'task-missing', - canvasId: 'missing-canvas', - goal: 'Missing', - defaultRootProfileId: 'profile-a', - anchorNodeId: 'node-missing', - createdAt: 1, - }), - ).rejects.toThrow(/cannot write a missing Space/); - }); - it('fails fast on malformed and internally inconsistent Task stores', async () => { + mkdirSync(path.dirname(tasksPath('canvas-task')), { recursive: true }); writeFileSync(tasksPath('canvas-task'), '{"version":1,"tasks":{}}'); await expect(store.space('canvas-task').tasks.read()).rejects.toThrow( /Invalid Task store/, diff --git a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts new file mode 100644 index 000000000..b30803d4e --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SqliteStructuredStore } from './structured-store.js'; +import { + createSqliteTestFile, + installDeltaAbortTrigger, + openSqliteTestStore, + readSqliteDeltaLog, +} from './test-support.js'; +import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; +import { describeSpaceNodesContract } from '../../ports/contracts/space-nodes.contract.js'; +import { describeSpaceRepositoryContract } from '../../ports/contracts/space-repository.contract.js'; +import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; +import { describeSpaceWriteContract } from '../../ports/contracts/space-write.contract.js'; +import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; + +import type { NodeContent } from '../../../canvas/persistence-types.js'; + +function note(nodeId: string, label: string, content: string): NodeContent { + return { nodeId, type: 'note', label, content }; +} + +async function createOrdinarySpace( + store: SqliteStructuredStore, + canvasId: string, + title: string, +): Promise { + const created = await store.spaces().create({ canvasId, title }); + if (!created.ok) throw new Error(`Could not create test Space ${canvasId}`); +} + +describeStructuredStoreContract('SQLite', () => { + const file = createSqliteTestFile('huabu-sqlite-structured-contract-'); + return { + store: new SqliteStructuredStore(file.filename), + cleanup: file.remove, + }; +}); + +describeSpaceRepositoryContract('SQLite', async () => { + const harness = await openSqliteTestStore( + 'huabu-sqlite-space-repository-contract-', + ); + return { + repository: harness.store.spaces(), + read: (canvasId: string) => harness.store.space(canvasId).read(), + worldCanvasId: harness.world.canvasId, + attemptMutation: (canvasId: string) => + harness.store.space(canvasId).nodes.put({ + nodeId: 'contract-delete-fence-node', + record: note( + 'contract-delete-fence-node', + 'Deletion fence node', + 'body', + ), + }), + cleanup: harness.cleanup, + }; +}); + +describeSpaceNodesContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-nodes-contract-'); + const canvasId = 'sqlite-nodes-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Nodes Contract'); + const space = harness.store.space(canvasId); + return { + repository: space.nodes, + space, + missingRepository: harness.store.space('sqlite-nodes-missing').nodes, + expectedCanvasId: canvasId, + cleanup: harness.cleanup, + }; +}); + +describeSpaceWriteContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-write-contract-'); + const canvasId = 'sqlite-write-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Write Contract'); + const existingNode = note( + 'contract-existing-node', + 'Existing contract node', + 'before', + ); + const space = harness.store.space(canvasId); + const put = await space.nodes.put({ + nodeId: existingNode.nodeId, + record: existingNode, + }); + if (!put.ok) { + throw new Error(`Could not seed SQLite write contract: ${put.reason}`); + } + + return { + space, + concurrent: harness.store.space(canvasId), + missing: harness.store.space('sqlite-write-missing'), + existingNode, + newNode: note('contract-new-node', 'New contract node', 'after'), + readJournal: async () => readSqliteDeltaLog(harness.filename, canvasId), + failNextDeltaAppend: (error: Error) => + installDeltaAbortTrigger(harness.filename, error.message), + cleanup: harness.cleanup, + }; +}); + +describeSpaceLogsContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-logs-contract-'); + const canvasId = 'sqlite-logs-contract'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Logs Contract'); + const first = harness.store.space(canvasId); + const second = harness.store.space(canvasId); + return { + events: first.events, + changes: first.changes, + concurrent: { + events: second.events, + changes: second.changes, + }, + cleanup: harness.cleanup, + }; +}); + +describeSpaceTasksContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-tasks-contract-'); + const canvasId = 'sqlite-tasks-contract'; + const missingCanvasId = 'sqlite-tasks-missing'; + await createOrdinarySpace(harness.store, canvasId, 'SQLite Tasks Contract'); + return { + tasks: harness.store.space(canvasId).tasks, + concurrent: harness.store.space(canvasId).tasks, + canvasId, + missing: harness.store.space(missingCanvasId).tasks, + missingCanvasId, + beginDelete: async () => { + const result = await harness.store.spaces().beginDelete({ canvasId }); + if (!result.ok) throw new Error('Ordinary Space must be deletable'); + return result.session; + }, + cleanup: harness.cleanup, + }; +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/database.ts b/apps/server/src/modules/storage/backends/sqlite/database.ts new file mode 100644 index 000000000..d4459bbe4 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/database.ts @@ -0,0 +1,381 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { DatabaseSync } from 'node:sqlite'; + +import type { StorageHealth } from '../../ports/common.js'; + +export const SQLITE_SCHEMA_VERSION = 1; +export const SQLITE_WORLD_COLLISION_KEY = '.world'; + +const SCHEMA_V1 = ` + CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + title TEXT, + collision_key TEXT NOT NULL UNIQUE, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)) + ) STRICT; + + CREATE UNIQUE INDEX spaces_single_world + ON spaces(is_world) + WHERE is_world = 1; + + CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision INTEGER NOT NULL CHECK (revision > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + + CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; +`; + +export interface SqliteMigration { + readonly version: number; + readonly sql: string; +} + +export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = Object.freeze([ + Object.freeze({ version: 1, sql: SCHEMA_V1 }), +]); + +function readUserVersion(database: DatabaseSync): number { + const row = database.prepare('PRAGMA user_version').get(); + const version = row?.['user_version']; + if (typeof version !== 'number' || !Number.isSafeInteger(version)) { + throw new Error('SQLite returned an invalid PRAGMA user_version'); + } + return version; +} + +export function applySqliteMigrations( + database: DatabaseSync, + migrations: readonly SqliteMigration[] = SQLITE_MIGRATIONS, +): void { + for (let index = 0; index < migrations.length; index += 1) { + const expectedVersion = index + 1; + if (migrations[index]?.version !== expectedVersion) { + throw new Error( + `SQLite migrations must be contiguous from version 1; expected ${expectedVersion}`, + ); + } + } + const targetVersion = migrations.at(-1)?.version ?? 0; + const current = readUserVersion(database); + if (current > targetVersion) { + throw new Error( + `SQLite schema version ${current} is newer than supported version ${targetVersion}`, + ); + } + if (current === targetVersion) return; + + database.exec('BEGIN IMMEDIATE'); + try { + let version = readUserVersion(database); + for (const migration of migrations) { + if (migration.version <= version) continue; + if (migration.version !== version + 1) { + throw new Error( + `No SQLite migration path from schema version ${version} to ${targetVersion}`, + ); + } + database.exec(migration.sql); + database.exec(`PRAGMA user_version = ${migration.version}`); + version = migration.version; + } + if (version !== targetVersion) { + throw new Error( + `No SQLite migration path from schema version ${version} to ${targetVersion}`, + ); + } + database.exec('COMMIT'); + } catch (error) { + if (database.isTransaction) database.exec('ROLLBACK'); + throw error; + } +} + +function reserveWorldCollisionKey(database: DatabaseSync): void { + withImmediateTransaction(database, () => { + const world = database + .prepare('SELECT canvas_id, collision_key FROM spaces WHERE is_world = 1') + .get(); + if (world === undefined) return; + + const canvasId = world['canvas_id']; + const collisionKey = world['collision_key']; + if (typeof canvasId !== 'string' || typeof collisionKey !== 'string') { + throw new SyntaxError('SQLite World Space has malformed identity fields'); + } + if (collisionKey === SQLITE_WORLD_COLLISION_KEY) return; + + const conflict = database + .prepare( + `SELECT canvas_id + FROM spaces + WHERE collision_key = ? AND canvas_id <> ?`, + ) + .get(SQLITE_WORLD_COLLISION_KEY, canvasId); + if (conflict !== undefined) { + throw new Error( + `Cannot reserve SQLite World collision slot ${JSON.stringify( + SQLITE_WORLD_COLLISION_KEY, + )}: it is already occupied`, + ); + } + + const result = database + .prepare( + `UPDATE spaces + SET collision_key = ? + WHERE canvas_id = ? AND is_world = 1`, + ) + .run(SQLITE_WORLD_COLLISION_KEY, canvasId); + if (Number(result.changes) !== 1) { + throw new Error('Could not reserve the SQLite World collision slot'); + } + }); +} + +type DeleteAdmission = { + readonly resolve: (release: () => void) => void; + readonly reject: (error: Error) => void; +}; + +class SpaceDeleteGate { + #active = false; + #closed = false; + readonly #waiting: DeleteAdmission[] = []; + + get pending(): boolean { + return this.#active || this.#waiting.length > 0; + } + + get idle(): boolean { + return !this.#active && this.#waiting.length === 0; + } + + acquire(): Promise<() => void> { + if (this.#closed) { + return Promise.reject(new Error('SQLite store is closed')); + } + if (!this.#active) { + this.#active = true; + return Promise.resolve(this.#releaseFunction()); + } + return new Promise((resolve, reject) => { + this.#waiting.push({ resolve, reject }); + }); + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + const error = new Error('SQLite store is closed'); + for (const admission of this.#waiting.splice(0)) { + admission.reject(error); + } + } + + #releaseFunction(): () => void { + let released = false; + return () => { + if (released) return; + released = true; + this.#active = false; + if (this.#closed) return; + const next = this.#waiting.shift(); + if (!next) return; + this.#active = true; + next.resolve(this.#releaseFunction()); + }; + } +} + +/** One connection and all adapter-lifetime process-local state. */ +export class SqliteStoreContext { + readonly now: () => number; + + readonly #database: DatabaseSync; + readonly #deleteGates = new Map(); + readonly #nodeTombstones = new Set(); + #state: 'new' | 'open' | 'closed' = 'new'; + + constructor(filename: string, now: () => number) { + this.now = now; + this.#database = new DatabaseSync(filename, { open: false }); + } + + init(): void { + if (this.#state === 'open') return; + if (this.#state === 'closed') { + throw new Error('SQLite store is closed'); + } + + try { + this.#database.open(); + this.#database.exec('PRAGMA foreign_keys = ON'); + const foreignKeys = this.#database.prepare('PRAGMA foreign_keys').get()?.[ + 'foreign_keys' + ]; + if (foreignKeys !== 1) { + throw new Error('Could not enable SQLite foreign key enforcement'); + } + applySqliteMigrations(this.#database); + reserveWorldCollisionKey(this.#database); + this.#state = 'open'; + } catch (error) { + if (this.#database.isOpen) this.#database.close(); + this.#state = 'closed'; + throw error; + } + } + + health(kind: string): StorageHealth { + this.assertOpen(); + try { + const value = this.#database.prepare('SELECT 1 AS ok').get()?.['ok']; + return value === 1 + ? { ok: true, kind } + : { ok: false, kind, detail: 'SQLite liveness query returned no row' }; + } catch (error) { + return { + ok: false, + kind, + detail: error instanceof Error ? error.message : String(error), + }; + } + } + + close(): void { + if (this.#state === 'closed') return; + this.#state = 'closed'; + for (const gate of this.#deleteGates.values()) gate.close(); + this.#deleteGates.clear(); + if (this.#database.isOpen) this.#database.close(); + } + + database(): DatabaseSync { + this.assertOpen(); + return this.#database; + } + + assertOpen(): void { + if (this.#state !== 'open') { + throw new Error( + this.#state === 'closed' + ? 'SQLite store is closed' + : 'SQLite store is not initialized', + ); + } + } + + assertMutationAllowed(canvasId: string): void { + this.assertOpen(); + if (this.#deleteGates.get(canvasId)?.pending) { + throw new Error( + `Cannot mutate Space "${canvasId}" while deletion is pending`, + ); + } + } + + async acquireDelete(canvasId: string): Promise<() => void> { + this.assertOpen(); + let gate = this.#deleteGates.get(canvasId); + if (!gate) { + gate = new SpaceDeleteGate(); + this.#deleteGates.set(canvasId, gate); + } + const releaseGate = await gate.acquire(); + try { + this.assertOpen(); + } catch (error) { + releaseGate(); + throw error; + } + + let released = false; + return () => { + if (released) return; + released = true; + releaseGate(); + if (gate?.idle && this.#deleteGates.get(canvasId) === gate) { + this.#deleteGates.delete(canvasId); + } + }; + } + + isNodeTombstoned(canvasId: string, nodeId: string): boolean { + return this.#nodeTombstones.has(this.#nodeKey(canvasId, nodeId)); + } + + setNodeTombstone(canvasId: string, nodeId: string, present: boolean): void { + const key = this.#nodeKey(canvasId, nodeId); + if (present) this.#nodeTombstones.add(key); + else this.#nodeTombstones.delete(key); + } + + clearCanvasTombstones(canvasId: string): void { + const prefix = `${canvasId}\0`; + for (const key of this.#nodeTombstones) { + if (key.startsWith(prefix)) this.#nodeTombstones.delete(key); + } + } + + #nodeKey(canvasId: string, nodeId: string): string { + return `${canvasId}\0${nodeId}`; + } +} + +export function withImmediateTransaction( + database: DatabaseSync, + operation: () => T, +): T { + database.exec('BEGIN IMMEDIATE'); + try { + const result = operation(); + database.exec('COMMIT'); + return result; + } catch (error) { + if (database.isTransaction) database.exec('ROLLBACK'); + throw error; + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql new file mode 100644 index 000000000..c50d52a0e --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql @@ -0,0 +1,109 @@ +-- Immutable SQLite structured-store schema v1 fixture. +-- Add a new fixture for later schema versions; do not rewrite this history. + +PRAGMA foreign_keys = ON; +BEGIN IMMEDIATE; + +CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + title TEXT, + collision_key TEXT NOT NULL UNIQUE, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)) +) STRICT; + +CREATE UNIQUE INDEX spaces_single_world + ON spaces(is_world) + WHERE is_world = 1; + +CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision INTEGER NOT NULL CHECK (revision > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + +CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + +INSERT INTO spaces ( + canvas_id, title, collision_key, version, state_json, + created_at, updated_at, is_world +) VALUES ( + 'fixture-world', 'World', 'world', 0, + '{"nodes":[],"edges":[]}', 1, 1, 1 +); + +INSERT INTO spaces ( + canvas_id, title, collision_key, version, state_json, + created_at, updated_at, is_world +) VALUES ( + 'fixture-space', 'Fixture Space', 'fixture space', 3, + '{"nodes":[{"id":"fixture-node","type":"note"}],"edges":[]}', + 10, 13, 0 +); + +INSERT INTO nodes ( + canvas_id, node_id, record_json, revision, label_collision_key +) VALUES ( + 'fixture-space', 'fixture-node', + '{"nodeId":"fixture-node","type":"note","label":"Fixture Node","content":"fixture body"}', + 7, 'fixture node' +); + +INSERT INTO events (canvas_id, event_json) VALUES ( + 'fixture-space', + '{"payload":{"action":"node_selected","node":{"id":"fixture-node","type":"note","label":"Fixture Node"}},"ts":12}' +); + +INSERT INTO changes (canvas_id, thread_id, snapshot_json) VALUES ( + 'fixture-space', 'fixture-thread', '[]' +); + +INSERT INTO tasks (canvas_id, snapshot_json) VALUES ( + 'fixture-space', '{"version":1,"tasks":[],"runs":[]}' +); + +INSERT INTO delta_log (canvas_id, version, entry_json) VALUES ( + 'fixture-space', 3, + '{"version":3,"ts":13,"commands":[],"deltas":[],"originator":{"source":"system"}}' +); + +PRAGMA user_version = 1; +COMMIT; diff --git a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts new file mode 100644 index 000000000..43311516c --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts @@ -0,0 +1,578 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { readFileSync } from 'node:fs'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { extractCanvasChanges } from '@huabu/shared/canvas-engine'; + +import { applySqliteMigrations, SQLITE_SCHEMA_VERSION } from './database.js'; +import { SqliteStructuredStore } from './structured-store.js'; +import { + createSqliteTestFile, + installDeltaAbortTrigger, + openSqliteTestStore, + readSqliteDeltaLog, + withTestDatabase, +} from './test-support.js'; + +import type { + CanvasFile, + DeltaLogEntry, + NodeContent, +} from '../../../canvas/persistence-types.js'; +import type { TaskRecord } from '@huabu/shared'; +import type { CanvasNode } from '@huabu/shared/canvas-engine'; + +const cleanups: Array<() => Promise | void> = []; + +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) { + await cleanup(); + } +}); + +function trackedFile(prefix: string) { + const file = createSqliteTestFile(prefix); + cleanups.push(file.remove); + return file; +} + +function trackedStore(filename: string): SqliteStructuredStore { + const store = new SqliteStructuredStore(filename); + cleanups.push(() => store.close()); + return store; +} + +async function trackedOpenStore(prefix: string) { + const harness = await openSqliteTestStore(prefix); + cleanups.push(harness.cleanup); + return harness; +} + +function note(nodeId: string, label: string, content: string): NodeContent { + return { nodeId, type: 'note', label, content }; +} + +function nextRecord(current: CanvasFile): CanvasFile { + return { + ...current, + version: current.version + 1, + updatedAt: current.updatedAt + 1, + }; +} + +function delta(version: number, marker: string): DeltaLogEntry { + return { + version, + ts: version + 100, + commands: [{ marker }], + deltas: [{ marker }], + originator: { source: 'system' }, + }; +} + +async function createSpace( + store: SqliteStructuredStore, + canvasId: string, + title: string, +): Promise { + const result = await store.spaces().create({ canvasId, title }); + if (!result.ok) throw new Error(`Could not create test Space ${canvasId}`); + return result.record; +} + +describe('SqliteStructuredStore lifecycle and schema', () => { + it('rejects an empty database filename', () => { + expect(() => new SqliteStructuredStore('')).toThrow(/filename.*empty/i); + }); + + it('rejects before init and after close while lifecycle operations stay idempotent', async () => { + const file = trackedFile('huabu-sqlite-lifecycle-'); + const store = trackedStore(file.filename); + + await expect(store.health()).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/not initialized/); + await expect( + Promise.resolve().then(() => store.space('lifecycle-space').read()), + ).rejects.toThrow(/not initialized/); + + await expect(store.init()).resolves.toBeUndefined(); + await expect(store.init()).resolves.toBeUndefined(); + await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + + await expect(store.close()).resolves.toBeUndefined(); + await expect(store.close()).resolves.toBeUndefined(); + await expect(store.health()).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/closed/); + await expect( + Promise.resolve().then(() => store.space('lifecycle-space').read()), + ).rejects.toThrow(/closed/); + await expect(store.init()).rejects.toThrow(/closed/); + }); + + it('creates the complete STRICT v1 schema in a fresh database', async () => { + const file = trackedFile('huabu-sqlite-fresh-schema-'); + const store = trackedStore(file.filename); + await store.init(); + + withTestDatabase(file.filename, (database) => { + expect(database.prepare('PRAGMA user_version').get()).toEqual({ + user_version: SQLITE_SCHEMA_VERSION, + }); + const expectedTables = [ + 'changes', + 'delta_log', + 'events', + 'nodes', + 'spaces', + 'tasks', + ]; + const tableRows = database.prepare('PRAGMA table_list').all(); + const productionTables = tableRows.filter((row) => + expectedTables.includes(String(row['name'])), + ); + expect(productionTables.map((row) => row['name']).sort()).toEqual( + expectedTables, + ); + expect(productionTables.every((row) => row['strict'] === 1)).toBe(true); + expect( + database + .prepare('PRAGMA foreign_key_list(nodes)') + .all() + .map((row) => ({ + table: row['table'], + from: row['from'], + to: row['to'], + onDelete: row['on_delete'], + })), + ).toContainEqual({ + table: 'spaces', + from: 'canvas_id', + to: 'canvas_id', + onDelete: 'CASCADE', + }); + }); + }); + + it('opens the immutable v1 SQL fixture without rewriting its records', async () => { + const file = trackedFile('huabu-sqlite-v1-fixture-'); + const fixtureSql = readFileSync( + new URL('./fixtures/v1.sql', import.meta.url), + 'utf8', + ); + withTestDatabase(file.filename, (database) => database.exec(fixtureSql)); + + const store = trackedStore(file.filename); + await store.init(); + await expect(store.spaces().worldId()).resolves.toBe('fixture-world'); + await expect(store.spaces().list()).resolves.toEqual([ + { + canvasId: 'fixture-space', + title: 'Fixture Space', + nodeCount: 1, + createdAt: 10, + updatedAt: 13, + }, + ]); + const space = store.space('fixture-space'); + await expect(space.read()).resolves.toEqual({ + canvasId: 'fixture-space', + title: 'Fixture Space', + version: 3, + state: { + nodes: [{ id: 'fixture-node', type: 'note' }], + edges: [], + }, + createdAt: 10, + updatedAt: 13, + }); + await expect(space.nodes.read('fixture-node')).resolves.toEqual({ + record: note('fixture-node', 'Fixture Node', 'fixture body'), + revision: '7', + }); + await expect(space.events.read()).resolves.toEqual([ + { + payload: { + action: 'node_selected', + node: { id: 'fixture-node', type: 'note', label: 'Fixture Node' }, + }, + ts: 12, + }, + ]); + await expect(space.changes.read('fixture-thread')).resolves.toEqual([]); + await expect(space.tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + expect(readSqliteDeltaLog(file.filename, 'fixture-space')).toEqual([ + { + version: 3, + ts: 13, + commands: [], + deltas: [], + originator: { source: 'system' }, + }, + ]); + }); + + it('rejects a database whose user_version is from the future', async () => { + const file = trackedFile('huabu-sqlite-future-schema-'); + withTestDatabase(file.filename, (database) => { + database.exec(`PRAGMA user_version = ${SQLITE_SCHEMA_VERSION + 1}`); + }); + const store = trackedStore(file.filename); + + await expect(store.init()).rejects.toThrow(/newer than supported/); + await expect(store.health()).rejects.toThrow(/closed/); + }); + + it('rolls every migration step and user_version back when a later step fails', () => { + const file = trackedFile('huabu-sqlite-migration-rollback-'); + withTestDatabase(file.filename, (database) => { + expect(() => + applySqliteMigrations(database, [ + { + version: 1, + sql: 'CREATE TABLE migration_v1 (id INTEGER PRIMARY KEY) STRICT;', + }, + { + version: 2, + sql: ` + CREATE TABLE migration_v2 (id INTEGER PRIMARY KEY) STRICT; + INSERT INTO missing_migration_table (id) VALUES (1); + `, + }, + ]), + ).toThrow(/missing_migration_table|no such table/); + + expect(database.prepare('PRAGMA user_version').get()).toEqual({ + user_version: 0, + }); + expect( + database + .prepare( + `SELECT name + FROM sqlite_schema + WHERE type = 'table' AND name LIKE 'migration_%'`, + ) + .all(), + ).toEqual([]); + }); + }); +}); + +describe('SqliteStructuredStore persistence and transactions', () => { + it('persists Space and Node records across close and reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-reopen-'); + const canvasId = 'reopen-space'; + const created = await createSpace(harness.store, canvasId, 'Reopen Space'); + const record = note('reopen-node', 'Reopen Node', 'persisted body'); + const put = await harness.store.space(canvasId).nodes.put({ + nodeId: record.nodeId, + record, + }); + expect(put).toMatchObject({ ok: true, record }); + + await harness.store.close(); + const reopened = trackedStore(harness.filename); + await reopened.init(); + + await expect(reopened.spaces().worldId()).resolves.toBe( + harness.world.canvasId, + ); + await expect(reopened.space(canvasId).read()).resolves.toEqual(created); + await expect( + reopened.space(canvasId).nodes.read(record.nodeId), + ).resolves.toEqual(put.ok ? { record, revision: put.revision } : null); + }); + + it('rolls node, record, delta, and tombstone state back on a real trigger abort', async () => { + const harness = await trackedOpenStore('huabu-sqlite-trigger-rollback-'); + const canvasId = 'trigger-rollback-space'; + const baseline = await createSpace( + harness.store, + canvasId, + 'Trigger Rollback Space', + ); + const oldNode = note('old-node', 'Old Node', 'before'); + const newNode = note('new-node', 'New Node', 'after'); + const oldPut = await harness.store.space(canvasId).nodes.put({ + nodeId: oldNode.nodeId, + record: oldNode, + }); + if (!oldPut.ok) throw new Error('Could not seed rollback node'); + + const next: CanvasFile = { + ...nextRecord(baseline), + state: { + nodes: [{ id: newNode.nodeId, type: newNode.type }], + edges: [], + }, + }; + const restore = installDeltaAbortTrigger( + harness.filename, + 'forced delta abort', + ); + try { + await expect( + harness.store.space(canvasId).write({ + expectedVersion: baseline.version, + nextRecord: next, + nodeMutations: [ + { kind: 'delete', nodeId: oldNode.nodeId }, + { + kind: 'put', + nodeId: newNode.nodeId, + record: newNode, + authoritativeInsert: true, + }, + ], + delta: delta(next.version, 'trigger-abort'), + }), + ).rejects.toThrow('forced delta abort'); + } finally { + restore(); + } + + const space = harness.store.space(canvasId); + await expect(space.read()).resolves.toEqual(baseline); + await expect(space.nodes.read(oldNode.nodeId)).resolves.toEqual({ + record: oldPut.record, + revision: oldPut.revision, + }); + await expect(space.nodes.read(newNode.nodeId)).resolves.toBeNull(); + expect(readSqliteDeltaLog(harness.filename, canvasId)).toEqual([]); + await expect( + space.nodes.put({ + nodeId: oldNode.nodeId, + expectedRevision: oldPut.revision, + record: { ...oldNode, content: 'still writable' }, + }), + ).resolves.toMatchObject({ ok: true }); + }); + + it('rejects sparse JSON arrays without changing the exact persisted Node', async () => { + const harness = await trackedOpenStore('huabu-sqlite-sparse-json-'); + const canvasId = 'sparse-json-space'; + await createSpace(harness.store, canvasId, 'Sparse JSON Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('sparse-json-node', 'Sparse JSON Node', 'before'); + const baseline = await nodes.put({ nodeId: record.nodeId, record }); + if (!baseline.ok) throw new Error('Could not seed sparse JSON node'); + const sparse: unknown[] = []; + sparse[1] = 'present'; + expect(0 in sparse).toBe(false); + + await expect( + nodes.put({ + nodeId: record.nodeId, + expectedRevision: baseline.revision, + record: { ...record, metadata: sparse }, + }), + ).rejects.toThrow(/sparse array/i); + await expect(nodes.read(record.nodeId)).resolves.toEqual({ + record, + revision: baseline.revision, + }); + }); + + it('releases deletion admission when post-acquire Space setup throws', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-setup-'); + const canvasId = 'delete-setup-space'; + const record = await createSpace( + harness.store, + canvasId, + 'Delete Setup Space', + ); + const repository = harness.store.spaces(); + + const malformedAttempt = repository.beginDelete({ canvasId }); + withTestDatabase(harness.filename, (database) => { + database + .prepare('UPDATE spaces SET state_json = ? WHERE canvas_id = ?') + .run('[]', canvasId); + }); + await expect(malformedAttempt).rejects.toThrow(/Invalid Space/); + withTestDatabase(harness.filename, (database) => { + database + .prepare('UPDATE spaces SET state_json = ? WHERE canvas_id = ?') + .run(JSON.stringify(record.state), canvasId); + }); + + let secondResult: + | Awaited> + | undefined; + let secondError: unknown; + const secondSettled = repository.beginDelete({ canvasId }).then( + (result) => { + secondResult = result; + }, + (error: unknown) => { + secondError = error; + }, + ); + await new Promise((resolve) => setImmediate(resolve)); + + expect(secondError).toBeUndefined(); + expect(secondResult).toMatchObject({ ok: true }); + if (!secondResult?.ok) { + throw new Error('Deletion gate remained occupied after setup failure'); + } + await secondResult.session.abort(); + await secondSettled; + }); + + it('cascades every child record when a deletion session finishes', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-session-'); + const canvasId = 'delete-session-space'; + const baseline = await createSpace( + harness.store, + canvasId, + 'Delete Session Space', + ); + const record = note('deleted-node', 'Deleted Node', 'stale body'); + const handle = harness.store.space(canvasId); + await handle.nodes.put({ nodeId: record.nodeId, record }); + await handle.events.append([ + { + payload: { + action: 'node_selected', + node: { + id: record.nodeId, + type: 'note', + label: record.label ?? undefined, + }, + }, + ts: 2, + }, + ]); + const changeNode: CanvasNode = { + id: 'change-node', + type: 'note', + position: { x: 0, y: 0 }, + data: { label: 'Change Node', content: 'change body' }, + } as CanvasNode; + await handle.changes.append( + 'delete-thread', + extractCanvasChanges([{ type: 'INSERT_NODE', node: changeNode }]), + ); + const task: TaskRecord = { + taskId: 'delete-task', + canvasId, + goal: 'Delete this fixture', + defaultRootProfileId: 'profile-delete', + anchorNodeId: record.nodeId, + createdAt: 3, + }; + await handle.tasks.create(task); + const next = nextRecord(baseline); + await expect( + handle.write({ + expectedVersion: baseline.version, + nextRecord: next, + nodeMutations: [], + delta: delta(next.version, 'delete-session'), + }), + ).resolves.toEqual({ ok: true }); + + withTestDatabase(harness.filename, (database) => { + for (const table of [ + 'nodes', + 'events', + 'changes', + 'tasks', + 'delta_log', + ]) { + expect( + database + .prepare( + `SELECT count(*) AS count FROM ${table} WHERE canvas_id = ?`, + ) + .get(canvasId)?.['count'], + ).toBe(1); + } + }); + + const started = await harness.store.spaces().beginDelete({ canvasId }); + if (!started.ok) throw new Error('Ordinary Space must be deletable'); + await expect(handle.read()).resolves.toEqual(next); + await expect(handle.nodes.read(record.nodeId)).resolves.toMatchObject({ + record, + }); + await expect(started.session.finish()).resolves.toEqual({ + ok: true, + reason: 'deleted', + }); + + withTestDatabase(harness.filename, (database) => { + for (const table of [ + 'nodes', + 'events', + 'changes', + 'tasks', + 'delta_log', + ]) { + expect( + database + .prepare( + `SELECT count(*) AS count FROM ${table} WHERE canvas_id = ?`, + ) + .get(canvasId)?.['count'], + ).toBe(0); + } + }); + + await expect(handle.read()).resolves.toBeNull(); + }); + + it('does not create a tombstone when deleting an already absent node', async () => { + const harness = await trackedOpenStore('huabu-sqlite-absent-delete-'); + const canvasId = 'absent-delete-space'; + await createSpace(harness.store, canvasId, 'Absent Delete Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('not-yet-created', 'Not Yet Created', 'body'); + + await expect(nodes.delete(record.nodeId)).resolves.toBe('absent'); + await expect( + nodes.put({ nodeId: record.nodeId, record }), + ).resolves.toMatchObject({ ok: true, record }); + }); + + it('forgets a successful node deletion tombstone after close and reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-tombstone-reopen-'); + const canvasId = 'tombstone-reopen-space'; + await createSpace(harness.store, canvasId, 'Tombstone Reopen Space'); + const record = note('tombstoned-node', 'Tombstoned Node', 'before'); + const nodes = harness.store.space(canvasId).nodes; + await nodes.put({ nodeId: record.nodeId, record }); + + await expect(nodes.delete(record.nodeId)).resolves.toBe('deleted'); + await expect( + nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'late stale write' }, + }), + ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); + + await harness.store.close(); + const reopened = trackedStore(harness.filename); + await reopened.init(); + await expect( + reopened.space(canvasId).nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'after reopen' }, + }), + ).resolves.toMatchObject({ + ok: true, + record: { ...record, content: 'after reopen' }, + }); + }); +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts new file mode 100644 index 000000000..2ae4b2d1e --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { canvasEventInputSchema, canvasEventRecordSchema } from '@huabu/shared'; +import { + coalesceChanges, + type CanvasChangeRecord, +} from '@huabu/shared/canvas-engine'; + +import { withImmediateTransaction } from './database.js'; +import { parseJson, stringifyJson } from './values.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { CanvasEvent } from '../../../canvas/persistence-types.js'; +import type { + NewCanvasEvent, + SpaceChanges, + SpaceEvents, +} from '../../ports/structured.js'; +import type { z } from 'zod'; + +function firstIssue(error: z.ZodError): string { + const issue = error.issues[0]; + if (!issue) return 'unknown schema violation'; + const location = issue.path.length > 0 ? issue.path.join('.') : ''; + return `${location}: ${issue.message}`; +} + +function requireSpace(context: SqliteStoreContext, canvasId: string): void { + context.assertMutationAllowed(canvasId); + if ( + context + .database() + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] !== 1 + ) { + throw new Error( + `SQLite Space logs(${canvasId}) cannot mutate a missing Space`, + ); + } +} + +function decodeEvents(rows: readonly Record[]): CanvasEvent[] { + return rows.map((row, index) => { + const parsedJson = parseJson( + row['event_json'], + `Canvas event ${index + 1}`, + ); + const parsed = canvasEventRecordSchema.safeParse(parsedJson); + if (!parsed.success) { + throw new SyntaxError( + `Invalid persisted Canvas event ${index + 1}: ${firstIssue(parsed.error)}`, + ); + } + return parsedJson as CanvasEvent; + }); +} + +function decodeChanges( + value: unknown, + canvasId: string, + threadId: string, +): CanvasChangeRecord[] { + const parsed = parseJson( + value, + `changes for Space ${JSON.stringify(canvasId)} thread ${JSON.stringify(threadId)}`, + ); + if (!Array.isArray(parsed)) { + throw new SyntaxError( + `Persisted changes for Space ${canvasId} thread ${threadId} must be an array`, + ); + } + return coalesceChanges(parsed as CanvasChangeRecord[]); +} + +export interface SqliteSpaceLogs { + readonly events: SpaceEvents; + readonly changes: SpaceChanges; +} + +class SqliteSpaceLogCoordinator { + readonly #context: SqliteStoreContext; + readonly #canvasId: string; + + constructor(context: SqliteStoreContext, canvasId: string) { + this.#context = context; + this.#canvasId = canvasId; + } + + async readEvents(limit?: number): Promise { + const database = this.#context.database(); + if (limit !== undefined && !(limit > 0)) return []; + if (limit === undefined || !Number.isFinite(limit)) { + return decodeEvents( + database + .prepare( + `SELECT event_json + FROM events + WHERE canvas_id = ? + ORDER BY event_id ASC`, + ) + .all(this.#canvasId), + ); + } + const rows = database + .prepare( + `SELECT event_json + FROM events + WHERE canvas_id = ? + ORDER BY event_id DESC + LIMIT ?`, + ) + .all(this.#canvasId, Math.ceil(limit)) + .reverse(); + return decodeEvents(rows); + } + + async appendEvents(events: readonly NewCanvasEvent[]): Promise { + this.#context.assertOpen(); + if (events.length === 0) return; + const records: CanvasEvent[] = events.map((event, index) => { + const input = canvasEventInputSchema.safeParse(event); + if (!input.success) { + throw new TypeError( + `Invalid Canvas event append input at index ${index}: ${firstIssue(input.error)}`, + ); + } + const record = { + payload: event.payload, + ts: event.ts ?? this.#context.now(), + }; + const parsed = canvasEventRecordSchema.safeParse(record); + if (!parsed.success) { + throw new TypeError( + `Invalid Canvas event append record at index ${index}: ${firstIssue(parsed.error)}`, + ); + } + stringifyJson(record, `Canvas event append input ${index}`); + return record; + }); + + requireSpace(this.#context, this.#canvasId); + const database = this.#context.database(); + withImmediateTransaction(database, () => { + const insert = database.prepare( + 'INSERT INTO events (canvas_id, event_json) VALUES (?, ?)', + ); + for (const record of records) { + insert.run( + this.#canvasId, + stringifyJson(record, `Canvas event for ${this.#canvasId}`), + ); + } + }); + } + + async readChanges(threadIdInput: string): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + const row = this.#context + .database() + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + return row === undefined + ? [] + : decodeChanges(row['snapshot_json'], this.#canvasId, threadId); + } + + async appendChanges( + threadIdInput: string, + records: readonly CanvasChangeRecord[], + ): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + stringifyJson(records, `Changes for thread ${JSON.stringify(threadId)}`); + requireSpace(this.#context, this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const current = database + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + const existing = + current === undefined + ? [] + : decodeChanges(current['snapshot_json'], this.#canvasId, threadId); + const merged = coalesceChanges([...existing, ...records]); + database + .prepare( + `INSERT INTO changes (canvas_id, thread_id, snapshot_json) + VALUES (?, ?, ?) + ON CONFLICT(canvas_id, thread_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json`, + ) + .run( + this.#canvasId, + threadId, + stringifyJson(merged, `Changes for thread ${threadId}`), + ); + return merged; + }); + } + + async deleteChange( + threadIdInput: string, + changeId: string, + ): Promise { + const threadId = sanitizeId(threadIdInput, 'threadId'); + requireSpace(this.#context, this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const current = database + .prepare( + `SELECT snapshot_json + FROM changes + WHERE canvas_id = ? AND thread_id = ?`, + ) + .get(this.#canvasId, threadId); + if (current === undefined) return null; + const existing = decodeChanges( + current['snapshot_json'], + this.#canvasId, + threadId, + ); + const index = existing.findIndex((record) => record.id === changeId); + if (index < 0) return null; + const [removed] = existing.splice(index, 1); + database + .prepare( + `UPDATE changes + SET snapshot_json = ? + WHERE canvas_id = ? AND thread_id = ?`, + ) + .run( + stringifyJson(existing, `Changes for thread ${threadId}`), + this.#canvasId, + threadId, + ); + return removed ?? null; + }); + } +} + +export function createSqliteSpaceLogs( + context: SqliteStoreContext, + canvasId: string, +): SqliteSpaceLogs { + const coordinator = new SqliteSpaceLogCoordinator(context, canvasId); + return Object.freeze({ + events: Object.freeze({ + read: (limit?: number) => coordinator.readEvents(limit), + append: (events: readonly NewCanvasEvent[]) => + coordinator.appendEvents(events), + }), + changes: Object.freeze({ + read: (threadId: string) => coordinator.readChanges(threadId), + append: (threadId: string, records: readonly CanvasChangeRecord[]) => + coordinator.appendChanges(threadId, records), + delete: (threadId: string, changeId: string) => + coordinator.deleteChange(threadId, changeId), + }), + }); +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts new file mode 100644 index 000000000..25279b478 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { + allocateNodeIdentity, + decodeNodeRecord, + requirePositiveRevision, + stringifyJson, + validateNodeContent, +} from './values.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + NodeDeleteResult, + NodePutInput, + NodePutResult, + NodeSnapshot, + SpaceNodes, +} from '../../ports/structured.js'; +import type { DatabaseSync } from 'node:sqlite'; + +interface NodeRow { + readonly record: NodeSnapshot['record']; + readonly revision: number; + readonly collisionKey: string; +} + +function decodeNodeRow(value: unknown, nodeId: string): NodeRow { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError(`Malformed persisted Node ${JSON.stringify(nodeId)}`); + } + const row = value as Record; + const collisionKey = row['label_collision_key']; + if (typeof collisionKey !== 'string') { + throw new SyntaxError( + `Invalid collision key for Node ${JSON.stringify(nodeId)}`, + ); + } + return { + record: decodeNodeRecord(row['record_json'], nodeId), + revision: requirePositiveRevision(row['revision'], nodeId), + collisionKey, + }; +} + +function readNodeRow( + database: DatabaseSync, + canvasId: string, + nodeId: string, +): NodeRow | null { + const row = database + .prepare( + `SELECT record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id = ?`, + ) + .get(canvasId, nodeId); + return row === undefined ? null : decodeNodeRow(row, nodeId); +} + +function spaceExists(database: DatabaseSync, canvasId: string): boolean { + return ( + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] === 1 + ); +} + +function validatePut(input: NodePutInput): string { + const nodeId = sanitizeId(input.nodeId, 'nodeId'); + validateNodeContent(input.record, nodeId); + if ( + input.expectedRevision !== undefined && + input.expectedRevision !== null && + typeof input.expectedRevision !== 'string' + ) { + throw new TypeError('expectedRevision must be a string, null, or omitted'); + } + return nodeId; +} + +export interface SqliteNodePutOptions { + readonly tombstoned: boolean; + readonly bypassTombstone?: boolean; +} + +/** Apply one node put inside the caller's active transaction. */ +export function putSqliteNodeInTransaction( + database: DatabaseSync, + canvasId: string, + input: NodePutInput, + options: SqliteNodePutOptions, +): NodePutResult { + const nodeId = validatePut(input); + if (options.tombstoned && options.bypassTombstone !== true) { + return { ok: false, reason: 'write-suppressed' }; + } + if (!spaceExists(database, canvasId)) { + return { ok: false, reason: 'not-found' }; + } + + const current = readNodeRow(database, canvasId, nodeId); + const currentRevision = current === null ? null : String(current.revision); + if ( + input.expectedRevision !== undefined && + input.expectedRevision !== currentRevision + ) { + return { + ok: false, + reason: 'revision-conflict', + currentRevision, + }; + } + + const occupied = database + .prepare( + `SELECT label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id <> ?`, + ) + .all(canvasId, nodeId) + .map((row) => row['label_collision_key']) + .filter((value): value is string => typeof value === 'string'); + const allocation = allocateNodeIdentity( + input.record, + nodeId, + current?.collisionKey ?? null, + input.strictLabel === true ? [] : occupied, + ); + + if (input.strictLabel === true) { + const conflict = database + .prepare( + `SELECT node_id, record_json, label_collision_key + FROM nodes + WHERE canvas_id = ? + AND label_collision_key = ? + AND node_id <> ?`, + ) + .get(canvasId, allocation.desiredCollisionKey, nodeId); + if (conflict !== undefined) { + const conflictingNodeId = conflict['node_id']; + const collisionKey = conflict['label_collision_key']; + if (typeof conflictingNodeId !== 'string') { + throw new SyntaxError('Invalid conflicting SQLite Node id'); + } + const conflicting = decodeNodeRecord( + conflict['record_json'], + conflictingNodeId, + ); + return { + ok: false, + reason: 'label-conflict', + conflictingNodeId, + conflictingLabel: + typeof conflicting.label === 'string' + ? conflicting.label + : typeof collisionKey === 'string' + ? collisionKey + : conflictingNodeId, + }; + } + } + + const revision = (current?.revision ?? 0) + 1; + if (!Number.isSafeInteger(revision)) { + throw new Error(`Node ${JSON.stringify(nodeId)} revision overflow`); + } + database + .prepare( + `INSERT INTO nodes ( + canvas_id, node_id, record_json, revision, label_collision_key + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(canvas_id, node_id) DO UPDATE SET + record_json = excluded.record_json, + revision = excluded.revision, + label_collision_key = excluded.label_collision_key`, + ) + .run( + canvasId, + nodeId, + stringifyJson(allocation.record, `Node ${JSON.stringify(nodeId)} record`), + revision, + allocation.collisionKey, + ); + return { + ok: true, + record: allocation.record, + revision: String(revision), + }; +} + +export class SqliteSpaceNodes implements SpaceNodes { + readonly canvasId: string; + + readonly #context: SqliteStoreContext; + + constructor(context: SqliteStoreContext, canvasId: string) { + this.#context = context; + this.canvasId = canvasId; + } + + async read(nodeIdInput: string): Promise { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + const current = readNodeRow( + this.#context.database(), + this.canvasId, + nodeId, + ); + return current === null + ? null + : { record: current.record, revision: String(current.revision) }; + } + + async put(input: NodePutInput): Promise { + const nodeId = validatePut(input); + this.#context.assertMutationAllowed(this.canvasId); + if (this.#context.isNodeTombstoned(this.canvasId, nodeId)) { + return { ok: false, reason: 'write-suppressed' }; + } + const database = this.#context.database(); + return withImmediateTransaction(database, () => + putSqliteNodeInTransaction(database, this.canvasId, input, { + tombstoned: false, + }), + ); + } + + async delete(nodeIdInput: string): Promise { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + this.#context.assertMutationAllowed(this.canvasId); + const database = this.#context.database(); + const result = withImmediateTransaction(database, () => { + if (!spaceExists(database, this.canvasId)) + return 'missing-space' as const; + const deleted = Number( + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(this.canvasId, nodeId).changes, + ); + return deleted === 1 ? ('deleted' as const) : ('absent' as const); + }); + if (result === 'missing-space') return 'absent'; + if (result === 'deleted') { + this.#context.setNodeTombstone(this.canvasId, nodeId, true); + } + return result; + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts new file mode 100644 index 000000000..f4631ce14 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { + allocateSpaceIdentity, + collisionKeyForTitle, + decodeSpaceRow, + insertSpaceRow, + readSpaceRow, + SPACE_COLUMNS, +} from './values.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { CanvasFile } from '../../../canvas/persistence-types.js'; +import type { + SpaceBeginDeleteResult, + SpaceCreateInput, + SpaceCreateResult, + SpaceDeleteInput, + SpaceDeleteSession, + SpaceRenameInput, + SpaceRenameResult, + SpaceRepository, +} from '../../ports/structured.js'; +import type { CanvasSummary } from '@huabu/shared'; + +function validateTitle(title: unknown): asserts title is string | null { + if (title !== null && typeof title !== 'string') { + throw new TypeError('Space title must be a string or null'); + } +} + +export class SqliteSpaceRepository implements SpaceRepository { + readonly #context: SqliteStoreContext; + + constructor(context: SqliteStoreContext) { + this.#context = context; + } + + async list(): Promise { + const database = this.#context.database(); + return database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 0`) + .all() + .map((row) => { + const { record } = decodeSpaceRow(row); + return { + canvasId: record.canvasId, + title: record.title, + nodeCount: record.state.nodes.length, + createdAt: record.createdAt, + updatedAt: record.updatedAt, + }; + }); + } + + async worldId(): Promise { + const database = this.#context.database(); + const rows = database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 1`) + .all(); + if (rows.length !== 1) { + throw new Error( + rows.length === 0 + ? 'SQLite namespace has no World Space' + : 'SQLite namespace has multiple World Spaces', + ); + } + const world = decodeSpaceRow(rows[0]); + if (!world.isWorld) throw new Error('SQLite World Space is malformed'); + return sanitizeId(world.record.canvasId, 'world canvasId'); + } + + async create(input: SpaceCreateInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + validateTitle(input.title); + this.#context.assertMutationAllowed(canvasId); + const database = this.#context.database(); + + return withImmediateTransaction(database, () => { + if (readSpaceRow(database, canvasId) !== null) { + return { ok: false as const, reason: 'already-exists' as const }; + } + const occupied = database + .prepare('SELECT collision_key FROM spaces') + .all() + .map((row) => row['collision_key']) + .filter((value): value is string => typeof value === 'string'); + const identity = allocateSpaceIdentity(input.title, canvasId, occupied); + const timestamp = this.#context.now(); + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Space clock returned a non-finite value'); + } + const record: CanvasFile = { + canvasId, + title: identity.title, + version: 0, + state: { nodes: [], edges: [] }, + createdAt: timestamp, + updatedAt: timestamp, + }; + insertSpaceRow(database, record, identity.collisionKey); + return { ok: true as const, record }; + }); + } + + async beginDelete(input: SpaceDeleteInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + const beforeAdmission = readSpaceRow(this.#context.database(), canvasId); + if (beforeAdmission?.isWorld) { + return { ok: false, reason: 'world-forbidden' }; + } + + const release = await this.#context.acquireDelete(canvasId); + let sessionOwnsGate = false; + try { + const afterAdmission = readSpaceRow(this.#context.database(), canvasId); + if (afterAdmission?.isWorld) { + return { ok: false, reason: 'world-forbidden' }; + } + + let state: 'open' | 'finishing' | 'closed' = 'open'; + const close = (): void => { + if (state === 'closed') return; + state = 'closed'; + release(); + }; + const session: SpaceDeleteSession = Object.freeze({ + finish: async () => { + if (state !== 'open') { + throw new Error(`Space deletion session for ${canvasId} is closed`); + } + state = 'finishing'; + try { + this.#context.assertOpen(); + const database = this.#context.database(); + const result = withImmediateTransaction(database, () => { + const current = readSpaceRow(database, canvasId); + if (current?.isWorld) { + throw new Error(`Refusing to delete World Space ${canvasId}`); + } + if (current === null) { + return { + deleted: false, + }; + } + const deleted = Number( + database + .prepare('DELETE FROM spaces WHERE canvas_id = ?') + .run(canvasId).changes, + ); + return { deleted: deleted === 1 }; + }); + if (result.deleted) { + this.#context.clearCanvasTombstones(canvasId); + return { ok: true as const, reason: 'deleted' as const }; + } + return { ok: false as const, reason: 'not-found' as const }; + } finally { + close(); + } + }, + abort: async () => { + if (state === 'finishing') { + throw new Error( + `Space deletion session for ${canvasId} is already finishing`, + ); + } + if (state === 'closed') return; + try { + this.#context.assertOpen(); + } finally { + close(); + } + }, + }); + sessionOwnsGate = true; + return { ok: true, session }; + } finally { + if (!sessionOwnsGate) release(); + } + } + + async rename(input: SpaceRenameInput): Promise { + const canvasId = sanitizeId(input.canvasId, 'canvasId'); + validateTitle(input.title); + this.#context.assertMutationAllowed(canvasId); + const database = this.#context.database(); + + return withImmediateTransaction(database, () => { + const current = readSpaceRow(database, canvasId); + if (current === null) return { ok: false, reason: 'not-found' } as const; + if (current.isWorld) { + return { ok: false, reason: 'world-forbidden' } as const; + } + if (current.record.title === input.title) { + return { ok: true, record: current.record } as const; + } + + const collisionKey = collisionKeyForTitle(input.title, canvasId); + if (collisionKey !== current.collisionKey) { + const conflict = database + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE collision_key = ? AND canvas_id <> ?`, + ) + .get(collisionKey, canvasId); + if (conflict !== undefined) { + return { + ok: false, + reason: 'title-conflict', + conflictingTitle: decodeSpaceRow(conflict).record.title, + } as const; + } + } + + const result = database + .prepare( + `UPDATE spaces + SET title = ?, collision_key = ? + WHERE canvas_id = ?`, + ) + .run(input.title, collisionKey, canvasId); + if (Number(result.changes) !== 1) { + throw new Error(`Could not rename SQLite Space ${canvasId}`); + } + return { + ok: true, + record: { ...current.record, title: input.title }, + } as const; + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts new file mode 100644 index 000000000..9c6cd7c4a --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + taskRecordSchema, + taskRunRecordSchema, + taskStoreSnapshotSchema, + type TaskRecord, + type TaskRunRecord, + type TaskStoreSnapshot, +} from '@huabu/shared'; + +import { withImmediateTransaction } from './database.js'; +import { parseJson, stringifyJson } from './values.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + SpaceTaskRuns, + SpaceTasks, + TaskRunUpdate, +} from '../../ports/structured.js'; + +const EMPTY_TASKS: TaskStoreSnapshot = { + version: 1, + tasks: [], + runs: [], +}; + +function validateSnapshot(value: unknown, canvasId: string): TaskStoreSnapshot { + const parsed = taskStoreSnapshotSchema.safeParse(value); + if (!parsed.success) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: ${parsed.error.issues[0]?.message ?? 'schema violation'}`, + ); + } + const taskIds = new Set(); + for (const task of parsed.data.tasks) { + if (task.canvasId !== canvasId) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Task ${task.taskId} belongs to Canvas ${task.canvasId}`, + ); + } + if (taskIds.has(task.taskId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: duplicate Task ${task.taskId}`, + ); + } + taskIds.add(task.taskId); + } + const runIds = new Set(); + for (const run of parsed.data.runs) { + if (run.canvasIdSnapshot !== canvasId) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Run ${run.runId} belongs to Canvas ${run.canvasIdSnapshot}`, + ); + } + if (runIds.has(run.runId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: duplicate Run ${run.runId}`, + ); + } + if (!taskIds.has(run.taskId)) { + throw new SyntaxError( + `Invalid Task store for Canvas ${canvasId}: Run ${run.runId} references missing Task ${run.taskId}`, + ); + } + runIds.add(run.runId); + } + return parsed.data; +} + +function readSnapshot( + context: SqliteStoreContext, + canvasId: string, +): TaskStoreSnapshot { + const row = context + .database() + .prepare('SELECT snapshot_json FROM tasks WHERE canvas_id = ?') + .get(canvasId); + if (row === undefined) { + return { ...EMPTY_TASKS, tasks: [], runs: [] }; + } + return validateSnapshot( + parseJson(row['snapshot_json'], `Task store for Canvas ${canvasId}`), + canvasId, + ); +} + +export class SqliteSpaceTasks implements SpaceTasks { + readonly runs: SpaceTaskRuns; + + readonly #context: SqliteStoreContext; + readonly #canvasId: string; + + constructor(context: SqliteStoreContext, canvasId: string) { + this.#context = context; + this.#canvasId = canvasId; + this.runs = Object.freeze({ + create: (run: TaskRunRecord) => this.#createRun(run), + update: (runId: string, update: TaskRunUpdate) => + this.#updateRun(runId, update), + }); + } + + async read(): Promise { + this.#context.assertOpen(); + return readSnapshot(this.#context, this.#canvasId); + } + + async create(task: TaskRecord): Promise { + const parsed = taskRecordSchema.safeParse(task); + if (!parsed.success || parsed.data.canvasId !== this.#canvasId) { + throw new TypeError(`Invalid Task record for Canvas ${this.#canvasId}`); + } + this.#mutate((snapshot) => { + if ( + snapshot.tasks.some( + (candidate) => candidate.taskId === parsed.data.taskId, + ) + ) { + throw new Error(`Task ${parsed.data.taskId} already exists`); + } + snapshot.tasks.push(parsed.data); + }); + } + + async #createRun(run: TaskRunRecord): Promise { + const parsed = taskRunRecordSchema.safeParse(run); + if (!parsed.success || parsed.data.canvasIdSnapshot !== this.#canvasId) { + throw new TypeError(`Invalid Run record for Canvas ${this.#canvasId}`); + } + this.#mutate((snapshot) => { + if ( + snapshot.runs.some((candidate) => candidate.runId === parsed.data.runId) + ) { + throw new Error(`Run ${parsed.data.runId} already exists`); + } + if ( + !snapshot.tasks.some( + (candidate) => candidate.taskId === parsed.data.taskId, + ) + ) { + throw new Error(`Task ${parsed.data.taskId} does not exist`); + } + snapshot.runs.push(parsed.data); + }); + } + + async #updateRun( + runId: string, + update: TaskRunUpdate, + ): Promise { + return this.#mutate((snapshot) => { + const index = snapshot.runs.findIndex((run) => run.runId === runId); + if (index < 0) throw new Error(`Run ${runId} does not exist`); + const parsed = taskRunRecordSchema.safeParse({ + ...snapshot.runs[index], + ...update, + }); + if (!parsed.success) { + throw new TypeError(`Invalid update for Run ${runId}`); + } + snapshot.runs[index] = parsed.data; + return parsed.data; + }); + } + + #mutate(apply: (snapshot: TaskStoreSnapshot) => T): T { + this.#context.assertMutationAllowed(this.#canvasId); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if ( + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(this.#canvasId)?.['present'] !== 1 + ) { + throw new Error( + `Space Tasks(${this.#canvasId}) cannot write a missing Space`, + ); + } + const current = readSnapshot(this.#context, this.#canvasId); + const next: TaskStoreSnapshot = { + version: 1, + tasks: [...current.tasks], + runs: [...current.runs], + }; + const result = apply(next); + database + .prepare( + `INSERT INTO tasks (canvas_id, snapshot_json) + VALUES (?, ?) + ON CONFLICT(canvas_id) DO UPDATE SET + snapshot_json = excluded.snapshot_json`, + ) + .run( + this.#canvasId, + stringifyJson(next, `Task store for Canvas ${this.#canvasId}`), + ); + return result; + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-write.ts b/apps/server/src/modules/storage/backends/sqlite/space-write.ts new file mode 100644 index 000000000..50f96a5a5 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-write.ts @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { withImmediateTransaction } from './database.js'; +import { putSqliteNodeInTransaction } from './space-nodes.js'; +import { + allocateSpaceIdentity, + insertSpaceRow, + readSpaceRow, + stringifyJson, + updateSpaceRow, + validateCanvasFile, + validateNodeContent, +} from './values.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + NodePutResult, + SpaceHandle, + SpaceNodeMutation, + SpaceWriteInput, + SpaceWriteResult, +} from '../../ports/structured.js'; + +function mutationError( + mutation: SpaceNodeMutation, + result: NodePutResult, +): Error { + const prefix = `Space write failed for node ${JSON.stringify(mutation.nodeId)}`; + if (result.ok) return new Error(`${prefix}: unexpected success result`); + switch (result.reason) { + case 'not-found': + return new Error(`${prefix}: Space does not exist`); + case 'revision-conflict': + return new Error(`${prefix}: unexpected revision conflict`); + case 'label-conflict': + return new Error( + `${prefix}: label conflicts with node ${JSON.stringify(result.conflictingNodeId)}`, + ); + case 'duplicate-node': + return new Error(`${prefix}: duplicate persisted node`); + case 'write-suppressed': + return new Error(`${prefix}: write is suppressed after deletion`); + } +} + +function validateInput(canvasId: string, input: SpaceWriteInput): void { + if (!Number.isFinite(input.expectedVersion)) { + throw new TypeError('expectedVersion must be a finite number'); + } + validateCanvasFile(input.nextRecord, canvasId); + if (input.nextRecord.version !== input.expectedVersion + 1) { + throw new Error( + `SpaceWrite(${canvasId}) expected nextRecord.version ` + + `${input.expectedVersion + 1}, received ${input.nextRecord.version}`, + ); + } + if ( + input.allowCreate === true && + (input.nodeMutations.length > 0 || input.delta !== undefined) + ) { + throw new Error( + 'allowCreate is valid only for a record-only structural write', + ); + } + if ( + input.delta !== undefined && + input.delta.version !== input.nextRecord.version + ) { + throw new Error( + 'delta.version must equal the committed Space record version', + ); + } + if (input.delta !== undefined) { + stringifyJson(input.delta, `Space ${JSON.stringify(canvasId)} delta`); + } + for (const mutation of input.nodeMutations) { + sanitizeId(mutation.nodeId, 'nodeId'); + if (mutation.kind === 'put') { + validateNodeContent(mutation.record, mutation.nodeId); + } + } +} + +/** Bind the atomic SQLite record/node/delta write to one Space. */ +export function createSqliteSpaceWrite( + context: SqliteStoreContext, + canvasId: string, +): SpaceHandle['write'] { + return async function write( + input: SpaceWriteInput, + ): Promise { + context.assertMutationAllowed(canvasId); + validateInput(canvasId, input); + const database = context.database(); + + const completed = withImmediateTransaction(database, () => { + const current = readSpaceRow(database, canvasId); + if (current === null) { + if (!input.allowCreate) { + return { + result: { ok: false, reason: 'not-found' } as const, + tombstones: new Map(), + }; + } + if (input.expectedVersion !== 0) { + throw new Error( + `SpaceWrite(${canvasId}) can create only from version 0`, + ); + } + const occupied = database + .prepare('SELECT collision_key FROM spaces') + .all() + .map((row) => row['collision_key']) + .filter((value): value is string => typeof value === 'string'); + const identity = allocateSpaceIdentity( + input.nextRecord.title, + canvasId, + occupied, + ); + insertSpaceRow( + database, + { ...input.nextRecord, title: identity.title }, + identity.collisionKey, + ); + return { + result: { ok: true } as const, + tombstones: new Map(), + }; + } + + if (current.record.version !== input.expectedVersion) { + return { + result: { + ok: false, + reason: 'version-conflict', + actualVersion: current.record.version, + } as const, + tombstones: new Map(), + }; + } + if (input.nextRecord.createdAt !== current.record.createdAt) { + throw new Error(`SpaceWrite(${canvasId}) refusing to change createdAt`); + } + if (input.nextRecord.title !== current.record.title) { + throw new Error( + `SpaceWrite(${canvasId}) cannot change title; ` + + 'use SpaceRepository.rename first', + ); + } + + const tombstones = new Map(); + const tombstoned = (nodeId: string): boolean => + tombstones.get(nodeId) ?? context.isNodeTombstoned(canvasId, nodeId); + + for (const mutation of input.nodeMutations) { + if (mutation.kind === 'delete') { + const deleted = Number( + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(canvasId, mutation.nodeId).changes, + ); + if (deleted === 1) tombstones.set(mutation.nodeId, true); + continue; + } + + const result = putSqliteNodeInTransaction( + database, + canvasId, + { + nodeId: mutation.nodeId, + record: mutation.record, + strictLabel: mutation.strictLabel, + }, + { + tombstoned: tombstoned(mutation.nodeId), + bypassTombstone: mutation.authoritativeInsert === true, + }, + ); + if (!result.ok) throw mutationError(mutation, result); + if (mutation.authoritativeInsert === true) { + tombstones.set(mutation.nodeId, false); + } + } + + if ( + updateSpaceRow(database, input.nextRecord, input.expectedVersion) !== 1 + ) { + throw new Error(`SpaceWrite(${canvasId}) lost its version race`); + } + if (input.delta !== undefined) { + database + .prepare( + `INSERT INTO delta_log (canvas_id, version, entry_json) + VALUES (?, ?, ?)`, + ) + .run( + canvasId, + input.delta.version, + stringifyJson(input.delta, `Space ${canvasId} delta`), + ); + } + return { result: { ok: true } as const, tombstones }; + }); + + if (completed.result.ok) { + for (const [nodeId, present] of completed.tombstones) { + context.setNodeTombstone(canvasId, nodeId, present); + } + } + return completed.result; + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts new file mode 100644 index 000000000..18b67aa18 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SqliteStoreContext } from './database.js'; +import { createSqliteSpaceLogs } from './space-logs.js'; +import { SqliteSpaceNodes } from './space-nodes.js'; +import { SqliteSpaceRepository } from './space-repository.js'; +import { SqliteSpaceTasks } from './space-tasks.js'; +import { createSqliteSpaceWrite } from './space-write.js'; +import { readSpaceRow } from './values.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +import type { StorageHealth } from '../../ports/common.js'; +import type { + SpaceHandle, + SpaceRepository, + StructuredStore, +} from '../../ports/structured.js'; + +/** Production structured-store adapter backed by one node:sqlite connection. */ +export class SqliteStructuredStore implements StructuredStore { + readonly kind = 'sqlite' as const; + + readonly #context: SqliteStoreContext; + + constructor(filename: string, now: () => number = Date.now) { + if (typeof filename !== 'string') { + throw new TypeError('SQLite filename must be a string'); + } + if (filename.length === 0) { + throw new TypeError('SQLite filename must not be empty'); + } + this.#context = new SqliteStoreContext(filename, now); + } + + async init(): Promise { + this.#context.init(); + } + + async health(): Promise { + return this.#context.health(this.kind); + } + + async close(): Promise { + this.#context.close(); + } + + spaces(): SpaceRepository { + return Object.freeze(new SqliteSpaceRepository(this.#context)); + } + + space(canvasIdInput: string): SpaceHandle { + const canvasId = sanitizeId(canvasIdInput, 'canvasId'); + const { events, changes } = createSqliteSpaceLogs(this.#context, canvasId); + const nodes = Object.freeze(new SqliteSpaceNodes(this.#context, canvasId)); + const tasks = Object.freeze(new SqliteSpaceTasks(this.#context, canvasId)); + return Object.freeze({ + canvasId, + read: async () => + readSpaceRow(this.#context.database(), canvasId)?.record ?? null, + write: createSqliteSpaceWrite(this.#context, canvasId), + nodes, + changes, + tasks, + events, + }); + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/test-support.ts b/apps/server/src/modules/storage/backends/sqlite/test-support.ts new file mode 100644 index 000000000..e69b84111 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/test-support.ts @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { SQLITE_SCHEMA_VERSION } from './database.js'; +import { SqliteStructuredStore } from './structured-store.js'; +import { collisionKeyForTitle, insertSpaceRow, parseJson } from './values.js'; + +import type { + CanvasFile, + DeltaLogEntry, +} from '../../../canvas/persistence-types.js'; + +export const SQLITE_TEST_WORLD_ID = 'sqlite-test-world'; + +export interface SqliteTestFile { + readonly directory: string; + readonly filename: string; + readonly remove: () => void; +} + +export interface OpenSqliteTestStore extends SqliteTestFile { + readonly store: SqliteStructuredStore; + readonly world: CanvasFile; + readonly cleanup: () => Promise; +} + +export function createSqliteTestFile(prefix = 'huabu-sqlite-'): SqliteTestFile { + const directory = mkdtempSync(path.join(tmpdir(), prefix)); + const filename = path.join(directory, 'structured.sqlite'); + let removed = false; + return { + directory, + filename, + remove: () => { + if (removed) return; + removed = true; + rmSync(directory, { recursive: true, force: true }); + }, + }; +} + +/** Run a short test-only query through a connection independent of the store. */ +export function withTestDatabase( + filename: string, + operation: (database: DatabaseSync) => T, +): T { + const database = new DatabaseSync(filename); + try { + database.exec('PRAGMA foreign_keys = ON'); + return operation(database); + } finally { + database.close(); + } +} + +/** + * Seed World without reaching through the adapter under test. + * + * The store first creates the production schema. This helper then opens a + * separate node:sqlite connection and uses the production row encoder, so a + * contract cannot pass because World creation accidentally shares private + * adapter state with the operation being exercised. + */ +export function seedSqliteWorld( + filename: string, + canvasId = SQLITE_TEST_WORLD_ID, +): CanvasFile { + const record: CanvasFile = { + canvasId, + title: 'World', + version: 0, + state: { nodes: [], edges: [] }, + createdAt: 1, + updatedAt: 1, + }; + withTestDatabase(filename, (database) => { + const version = database.prepare('PRAGMA user_version').get()?.[ + 'user_version' + ]; + if (version !== SQLITE_SCHEMA_VERSION) { + throw new Error( + `Expected production SQLite schema v${SQLITE_SCHEMA_VERSION}, got ${String(version)}`, + ); + } + insertSpaceRow( + database, + record, + collisionKeyForTitle(record.title, record.canvasId), + true, + ); + }); + return record; +} + +export async function openSqliteTestStore( + prefix = 'huabu-sqlite-', + now?: () => number, +): Promise { + const file = createSqliteTestFile(prefix); + const store = new SqliteStructuredStore(file.filename, now); + try { + await store.init(); + const world = seedSqliteWorld(file.filename); + return { + ...file, + store, + world, + cleanup: async () => { + await store.close(); + file.remove(); + }, + }; + } catch (error) { + await store.close(); + file.remove(); + throw error; + } +} + +export function readSqliteDeltaLog( + filename: string, + canvasId: string, +): DeltaLogEntry[] { + return withTestDatabase(filename, (database) => + database + .prepare( + `SELECT entry_json + FROM delta_log + WHERE canvas_id = ? + ORDER BY version`, + ) + .all(canvasId) + .map( + (row, index) => + parseJson( + row['entry_json'], + `test delta row ${index} for ${canvasId}`, + ) as DeltaLogEntry, + ), + ); +} + +/** Install a real SQLite failure immediately before a delta row is inserted. */ +export function installDeltaAbortTrigger( + filename: string, + message: string, +): () => void { + const quotedMessage = message.split("'").join("''"); + withTestDatabase(filename, (database) => { + database.exec('DROP TRIGGER IF EXISTS test_abort_delta_insert'); + database.exec(` + CREATE TRIGGER test_abort_delta_insert + BEFORE INSERT ON delta_log + BEGIN + SELECT RAISE(ABORT, '${quotedMessage}'); + END + `); + }); + let restored = false; + return () => { + if (restored) return; + restored = true; + withTestDatabase(filename, (database) => { + database.exec('DROP TRIGGER IF EXISTS test_abort_delta_insert'); + }); + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/values.ts b/apps/server/src/modules/storage/backends/sqlite/values.ts new file mode 100644 index 000000000..b08adde64 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/values.ts @@ -0,0 +1,360 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { SQLITE_WORLD_COLLISION_KEY } from './database.js'; +import { + dedupeName, + normalizeForCompare, + toSafeFilename, +} from '../../../../utils/naming.js'; +import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; + +import type { + CanvasFile, + NodeContent, +} from '../../../canvas/persistence-types.js'; +import type { DatabaseSync } from 'node:sqlite'; + +type JsonPrimitive = null | boolean | number | string; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +function assertJsonValue( + value: unknown, + context: string, + seen: Set, +): asserts value is JsonValue { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' + ) { + return; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new TypeError(`${context} contains a non-finite number`); + } + return; + } + if (typeof value !== 'object') { + throw new TypeError(`${context} contains a non-JSON value`); + } + if (seen.has(value)) throw new TypeError(`${context} contains a cycle`); + seen.add(value); + try { + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + if (!Object.prototype.hasOwnProperty.call(value, index)) { + throw new TypeError(`${context} contains a sparse array`); + } + assertJsonValue(value[index], `${context}[${index}]`, seen); + } + return; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${context} contains a non-plain object`); + } + for (const [key, entry] of Object.entries(value)) { + assertJsonValue(entry, `${context}.${key}`, seen); + } + } finally { + seen.delete(value); + } +} + +export function stringifyJson(value: unknown, context: string): string { + assertJsonValue(value, context, new Set()); + const encoded = JSON.stringify(value); + if (encoded === undefined) { + throw new TypeError(`${context} is not representable as JSON`); + } + return encoded; +} + +export function parseJson(value: unknown, context: string): unknown { + if (typeof value !== 'string') { + throw new SyntaxError(`${context} is not stored as JSON text`); + } + try { + return JSON.parse(value) as unknown; + } catch (error) { + throw new SyntaxError( + `Invalid JSON in ${context}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function rowObject(value: unknown, context: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError(`Missing or malformed SQLite row for ${context}`); + } + return value as Record; +} + +function stringColumn( + row: Record, + column: string, + context: string, +): string { + const value = row[column]; + if (typeof value !== 'string') { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +function nullableStringColumn( + row: Record, + column: string, + context: string, +): string | null { + const value = row[column]; + if (value !== null && typeof value !== 'string') { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +function numberColumn( + row: Record, + column: string, + context: string, +): number { + const value = row[column]; + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new SyntaxError(`Invalid ${column} in ${context}`); + } + return value; +} + +export interface PersistedSpace { + readonly record: CanvasFile; + readonly collisionKey: string; + readonly isWorld: boolean; +} + +export function decodeSpaceRow(value: unknown): PersistedSpace { + const row = rowObject(value, 'Space'); + const canvasId = stringColumn(row, 'canvas_id', 'Space'); + const context = `Space ${JSON.stringify(canvasId)}`; + const record: CanvasFile = { + canvasId, + title: nullableStringColumn(row, 'title', context), + version: numberColumn(row, 'version', context), + state: parseJson( + row['state_json'], + `${context} state`, + ) as CanvasFile['state'], + createdAt: numberColumn(row, 'created_at', context), + updatedAt: numberColumn(row, 'updated_at', context), + }; + const shapeError = canvasFileShapeError(record, canvasId); + if (shapeError) throw new SyntaxError(`Invalid ${context}: ${shapeError}`); + const world = numberColumn(row, 'is_world', context); + if (world !== 0 && world !== 1) { + throw new SyntaxError(`Invalid is_world in ${context}`); + } + return { + record, + collisionKey: stringColumn(row, 'collision_key', context), + isWorld: world === 1, + }; +} + +export const SPACE_COLUMNS = + 'canvas_id, title, collision_key, version, state_json, created_at, updated_at, is_world'; + +export function readSpaceRow( + database: DatabaseSync, + canvasId: string, +): PersistedSpace | null { + const row = database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE canvas_id = ?`) + .get(canvasId); + return row === undefined ? null : decodeSpaceRow(row); +} + +export function validateCanvasFile(record: CanvasFile, canvasId: string): void { + const shapeError = canvasFileShapeError(record, canvasId); + if (shapeError) { + throw new TypeError(`Invalid Space record: ${shapeError}`); + } + stringifyJson(record.state, `Space ${JSON.stringify(canvasId)} state`); +} + +function allocatedSpaceTitle( + requested: string | null, + canvasId: string, + allocatedName: string, +): string | null { + if (requested === null) return null; + const base = toSafeFilename(requested, canvasId); + if (allocatedName === base) return requested; + const candidate = `${requested}${allocatedName.slice(base.length)}`; + return toSafeFilename(candidate, canvasId) === allocatedName + ? candidate + : allocatedName; +} + +export function allocateSpaceIdentity( + requestedTitle: string | null, + canvasId: string, + occupiedCollisionKeys: Iterable, +): { readonly title: string | null; readonly collisionKey: string } { + const base = toSafeFilename(requestedTitle, canvasId); + const allocated = dedupeName(base, occupiedCollisionKeys); + return { + title: allocatedSpaceTitle(requestedTitle, canvasId, allocated), + collisionKey: normalizeForCompare(allocated), + }; +} + +export function collisionKeyForTitle( + title: string | null, + canvasId: string, +): string { + return normalizeForCompare(toSafeFilename(title, canvasId)); +} + +export function insertSpaceRow( + database: DatabaseSync, + record: CanvasFile, + collisionKey: string, + isWorld = false, +): void { + validateCanvasFile(record, record.canvasId); + database + .prepare( + `INSERT INTO spaces ( + canvas_id, title, collision_key, version, state_json, + created_at, updated_at, is_world + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + record.canvasId, + record.title, + isWorld ? SQLITE_WORLD_COLLISION_KEY : collisionKey, + record.version, + stringifyJson(record.state, `Space ${record.canvasId} state`), + record.createdAt, + record.updatedAt, + isWorld ? 1 : 0, + ); +} + +export function updateSpaceRow( + database: DatabaseSync, + record: CanvasFile, + expectedVersion: number, +): number { + validateCanvasFile(record, record.canvasId); + const result = database + .prepare( + `UPDATE spaces + SET version = ?, state_json = ?, updated_at = ? + WHERE canvas_id = ? AND version = ?`, + ) + .run( + record.version, + stringifyJson(record.state, `Space ${record.canvasId} state`), + record.updatedAt, + record.canvasId, + expectedVersion, + ); + return Number(result.changes); +} + +export function validateNodeContent( + record: NodeContent, + expectedNodeId: string, +): void { + if (typeof record !== 'object' || record === null || Array.isArray(record)) { + throw new TypeError('Node record must be an object'); + } + if (record.nodeId !== expectedNodeId) { + throw new Error( + `Node id mismatch: argument=${JSON.stringify(expectedNodeId)} ` + + `record=${JSON.stringify(record.nodeId)}`, + ); + } + if (typeof record.type !== 'string') { + throw new TypeError('Node record type must be a string'); + } + if (record.label !== null && typeof record.label !== 'string') { + throw new TypeError('Node record label must be a string or null'); + } + if (typeof record.content !== 'string') { + throw new TypeError('Node record content must be a string'); + } + stringifyJson(record, `Node ${JSON.stringify(expectedNodeId)} record`); +} + +export function decodeNodeRecord( + value: unknown, + expectedNodeId: string, +): NodeContent { + const parsed = parseJson(value, `Node ${JSON.stringify(expectedNodeId)}`); + try { + validateNodeContent(parsed as NodeContent, expectedNodeId); + } catch (error) { + throw new SyntaxError( + `Invalid persisted Node ${JSON.stringify(expectedNodeId)}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + return parsed as NodeContent; +} + +export function requirePositiveRevision( + value: unknown, + nodeId: string, +): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + throw new SyntaxError( + `Invalid persisted revision for Node ${JSON.stringify(nodeId)}`, + ); + } + return value; +} + +export function allocateNodeIdentity( + record: NodeContent, + nodeId: string, + existingCollisionKey: string | null, + occupiedCollisionKeys: Iterable, +): { + readonly record: NodeContent; + readonly collisionKey: string; + readonly desiredCollisionKey: string; +} { + const trimmedLabel = + typeof record.label === 'string' && record.label.trim().length > 0 + ? record.label + : null; + if (trimmedLabel === null && existingCollisionKey !== null) { + return { + record, + collisionKey: existingCollisionKey, + desiredCollisionKey: existingCollisionKey, + }; + } + + const desired = toSafeFilename(trimmedLabel, nodeId); + const allocated = dedupeName(desired, occupiedCollisionKeys); + const suffix = + allocated.length > desired.length && allocated.startsWith(desired) + ? allocated.slice(desired.length) + : ''; + return { + record: + suffix && trimmedLabel + ? { ...record, label: `${trimmedLabel}${suffix}` } + : record, + collisionKey: normalizeForCompare(allocated), + desiredCollisionKey: normalizeForCompare(desired), + }; +} diff --git a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts index 3d70bbd8f..0b0056afc 100644 --- a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts @@ -6,11 +6,18 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { NodeContent } from '../../../canvas/persistence-types.js'; -import type { NodePutInput, SpaceNodes, NodeSnapshot } from '../structured.js'; +import type { + NodePutInput, + SpaceHandle, + SpaceNodes, + NodeSnapshot, +} from '../structured.js'; export interface SpaceNodesContractHarness { /** Repository for an existing Space, initially empty at contract-owned ids. */ readonly repository: SpaceNodes; + /** Handle that owns `repository`, used to exercise ordered reinsertion. */ + readonly space: SpaceHandle; /** Repository scoped to a Space whose structural record is absent. */ readonly missingRepository: SpaceNodes; readonly expectedCanvasId: string; @@ -336,8 +343,8 @@ export function describeSpaceNodesContract( await expect(repository.delete(nodeId)).resolves.toBe('absent'); }); - it('suppresses a late standalone put after deletion', async () => { - const { repository } = await open(); + it('suppresses standalone resurrection until an authoritative ordered insert succeeds', async () => { + const { repository, space } = await open(); const nodeId = 'contract-late-put'; const record = note(nodeId, 'Contract late put', 'before'); await putSuccessfully(repository, { nodeId, record }); @@ -350,6 +357,56 @@ export function describeSpaceNodesContract( }), ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); await expect(repository.read(nodeId)).resolves.toBeNull(); + + const current = await space.read(); + if (current === null) + throw new Error('Contract fixture Space is missing'); + const authoritative = { + ...record, + content: 'authoritative resurrection', + }; + await expect( + space.write({ + expectedVersion: current.version, + nextRecord: { + ...current, + version: current.version + 1, + state: { + ...current.state, + nodes: [ + ...current.state.nodes, + { id: nodeId, type: authoritative.type }, + ], + }, + updatedAt: current.updatedAt + 1, + }, + nodeMutations: [ + { + kind: 'put', + nodeId, + record: authoritative, + authoritativeInsert: true, + }, + ], + }), + ).resolves.toEqual({ ok: true }); + + const restored = await repository.read(nodeId); + expect(restored).toMatchObject({ record: authoritative }); + if (restored === null) { + throw new Error('Authoritatively reinserted node is missing'); + } + + await expect( + repository.put({ + nodeId, + expectedRevision: restored.revision, + record: { ...authoritative, content: 'later standalone update' }, + }), + ).resolves.toMatchObject({ + ok: true, + record: { content: 'later standalone update' }, + }); }); }); } diff --git a/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts new file mode 100644 index 000000000..dc6257a34 --- /dev/null +++ b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts @@ -0,0 +1,309 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** Reusable behavioral contract for {@link SpaceTasks} and its Runs. */ + +import { afterEach, describe, expect, it } from 'vitest'; + +import type { + SpaceDeleteSession, + SpaceTasks, + TaskRunUpdate, +} from '../structured.js'; +import type { TaskRecord, TaskRunRecord } from '@huabu/shared'; + +export interface SpaceTasksContractHarness { + /** Task ledger for an existing Space, initially empty. */ + readonly tasks: SpaceTasks; + /** A second retained handle for the same existing Space. */ + readonly concurrent: SpaceTasks; + readonly canvasId: string; + /** Task ledger scoped to a Space whose structural record is absent. */ + readonly missing: SpaceTasks; + readonly missingCanvasId: string; + /** Open a structured-deletion fence for `canvasId`. */ + readonly beginDelete: () => Promise; + readonly cleanup?: () => Promise | void; +} + +function task(canvasId: string, taskId: string, createdAt: number): TaskRecord { + return { + taskId, + canvasId, + goal: `Goal for ${taskId}`, + defaultRootProfileId: `profile-${taskId}`, + anchorNodeId: `anchor-${taskId}`, + createdAt, + }; +} + +function run( + canvasId: string, + taskId: string, + runId: string, + createdAt: number, +): TaskRunRecord { + return { + runId, + taskId, + canvasIdSnapshot: canvasId, + goalSnapshot: `Goal snapshot for ${taskId}`, + rootProfileIdSnapshot: `profile-${taskId}`, + status: 'pending', + createdAt, + }; +} + +export function describeSpaceTasksContract( + name: string, + createHarness: () => + | Promise + | SpaceTasksContractHarness, +): void { + describe(`SpaceTasks contract: ${name}`, () => { + let harness: SpaceTasksContractHarness | null = null; + + async function open(): Promise { + harness = await createHarness(); + return harness; + } + + afterEach(async () => { + await harness?.cleanup?.(); + harness = null; + }); + + it('reads an empty versioned snapshot', async () => { + const { tasks } = await open(); + + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + }); + + it('creates a Task and rejects a duplicate id without replacing it', async () => { + const { tasks, canvasId } = await open(); + const original = task(canvasId, 'task-duplicate', 1); + await tasks.create(original); + + await expect( + tasks.create({ ...original, goal: 'Replacement goal', createdAt: 2 }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [original], + runs: [], + }); + }); + + it('requires an existing Task before creating its Run', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-owner', 1); + const ownedRun = run(canvasId, owner.taskId, 'run-owned', 2); + + await expect(tasks.runs.create(ownedRun)).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [], + runs: [], + }); + + await tasks.create(owner); + await tasks.runs.create(ownedRun); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [ownedRun], + }); + }); + + it('rejects a duplicate Run id without replacing it', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-run-duplicate', 1); + const original = run(canvasId, owner.taskId, 'run-duplicate', 2); + await tasks.create(owner); + await tasks.runs.create(original); + + await expect( + tasks.runs.create({ + ...original, + status: 'running', + startedAt: 3, + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [original], + }); + }); + + it('updates an existing Run and rejects a missing Run id', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-update', 1); + const original = run(canvasId, owner.taskId, 'run-update', 2); + await tasks.create(owner); + await tasks.runs.create(original); + const update: TaskRunUpdate = { + rootNodeId: 'root-node', + rootThreadId: 'root-thread', + status: 'running', + startedAt: 3, + }; + + await expect(tasks.runs.update(original.runId, update)).resolves.toEqual({ + ...original, + ...update, + }); + await expect( + tasks.runs.update('run-missing', { status: 'running' }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [{ ...original, ...update }], + }); + }); + + it('rejects Task and Run records scoped to another Space', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-scope', 1); + + await expect( + tasks.create({ ...owner, canvasId: 'another-space' }), + ).rejects.toThrow(); + await tasks.create(owner); + await expect( + tasks.runs.create({ + ...run(canvasId, owner.taskId, 'run-scope', 2), + canvasIdSnapshot: 'another-space', + }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [], + }); + }); + + it('rejects malformed Task, Run, and Run-update input', async () => { + const { tasks, canvasId } = await open(); + const owner = task(canvasId, 'task-validation', 1); + const ownedRun = run(canvasId, owner.taskId, 'run-validation', 2); + + await expect(tasks.create({ ...owner, goal: '' })).rejects.toThrow(); + await tasks.create(owner); + await expect( + tasks.runs.create({ ...ownedRun, goalSnapshot: '' }), + ).rejects.toThrow(); + await tasks.runs.create(ownedRun); + await expect( + tasks.runs.update(ownedRun.runId, { startedAt: -1 }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [ownedRun], + }); + }); + + it('preserves concurrent mutations through two retained handles', async () => { + const { tasks, concurrent, canvasId } = await open(); + const taskA = task(canvasId, 'task-concurrent-a', 1); + const taskB = task(canvasId, 'task-concurrent-b', 2); + await Promise.all([tasks.create(taskA), concurrent.create(taskB)]); + + const runA = run(canvasId, taskA.taskId, 'run-concurrent-a', 3); + const runB = run(canvasId, taskB.taskId, 'run-concurrent-b', 4); + await Promise.all([ + tasks.runs.create(runA), + concurrent.runs.create(runB), + ]); + await Promise.all([ + concurrent.runs.update(runA.runId, { + status: 'running', + startedAt: 5, + }), + tasks.runs.update(runB.runId, { + status: 'running', + startedAt: 6, + }), + ]); + + const snapshot = await tasks.read(); + expect(snapshot.tasks.map((record) => record.taskId).sort()).toEqual([ + taskA.taskId, + taskB.taskId, + ]); + expect(snapshot.runs.map((record) => record.runId).sort()).toEqual([ + runA.runId, + runB.runId, + ]); + expect(snapshot.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + runId: runA.runId, + status: 'running', + startedAt: 5, + }), + expect.objectContaining({ + runId: runB.runId, + status: 'running', + startedAt: 6, + }), + ]), + ); + }); + + it('rejects every mutation for a missing Space', async () => { + const { missing, missingCanvasId } = await open(); + const owner = task(missingCanvasId, 'task-missing-space', 1); + const ownedRun = run( + missingCanvasId, + owner.taskId, + 'run-missing-space', + 2, + ); + + await expect(missing.create(owner)).rejects.toThrow(); + await expect(missing.runs.create(ownedRun)).rejects.toThrow(); + await expect( + missing.runs.update(ownedRun.runId, { status: 'running' }), + ).rejects.toThrow(); + }); + + it('rejects mutations while structured deletion is fenced', async () => { + const { tasks, canvasId, beginDelete } = await open(); + const owner = task(canvasId, 'task-delete-fence', 1); + const original = run(canvasId, owner.taskId, 'run-delete-fence', 2); + await tasks.create(owner); + await tasks.runs.create(original); + const before = await tasks.read(); + const session = await beginDelete(); + + try { + await expect( + tasks.create(task(canvasId, 'task-too-late', 3)), + ).rejects.toThrow(); + await expect( + tasks.runs.create(run(canvasId, owner.taskId, 'run-too-late', 4)), + ).rejects.toThrow(); + await expect( + tasks.runs.update(original.runId, { status: 'running' }), + ).rejects.toThrow(); + await expect(tasks.read()).resolves.toEqual(before); + } finally { + await session.abort(); + } + + await expect( + tasks.runs.update(original.runId, { + status: 'running', + startedAt: 5, + }), + ).resolves.toMatchObject({ status: 'running', startedAt: 5 }); + }); + }); +} diff --git a/apps/server/src/modules/storage/ports/structured.ts b/apps/server/src/modules/storage/ports/structured.ts index 60332eebd..bed6678b2 100644 --- a/apps/server/src/modules/storage/ports/structured.ts +++ b/apps/server/src/modules/storage/ports/structured.ts @@ -61,7 +61,7 @@ import type { CanvasChangeRecord } from '@huabu/shared/canvas-engine'; * that are configurable but unimplemented — belongs to `profile.ts`, which * owns rejecting them with an actionable message. */ -export type StructuredBackendKind = 'disk'; +export type StructuredBackendKind = 'disk' | 'sqlite'; /** A connection to a structured backend. Process-wide; handles are derived. */ export interface StructuredStore { @@ -325,12 +325,17 @@ export type SpaceNodeMutation = /** * Marks an executor-authoritative INSERT. * - * **Adapter-shaped**, like {@link NodePutResult}'s `write-suppressed`. - * It exists for a backend that suppresses writes to a recently deleted - * id, and lets such an adapter distinguish a real re-insertion from a - * late direct write that should stay suppressed. It is intentionally - * batch-only. An adapter whose deletes are immediately final — a SQL - * table with a unique key — can ignore it. + * After {@link SpaceNodes.delete} removes an existing id, standalone + * puts for that id must return `write-suppressed` within the same running + * {@link StructuredStore}. A successful ordered put carrying this flag + * is the portable signal that the id is intentionally being reinserted; + * it admits the write and clears that suppression for later standalone + * puts. It is intentionally batch-only so a late direct write cannot + * claim authority for itself. + * + * This is an in-memory connection-lifetime guarantee, not restart + * durability. Closing or recreating the StructuredStore may discard the + * deletion fence. */ readonly authoritativeInsert?: boolean; } @@ -529,19 +534,19 @@ export type NodeDeleteResult = 'deleted' | 'absent'; * {@link SpaceNodes.readMany}, {@link SpaceNodes.list}, and * {@link SpaceNodes.stream}. * - * Two mutation outcomes are **adapter-shaped** and optional: + * One mutation outcome is **adapter-shaped** and optional: * * - `duplicate-node`, for adapters that can observe conflicting physical * representations of one stable id. Such an adapter may return one readable * representative from `read` so a caller can construct the attempted * update, but it must refuse the `put` rather than overwrite an arbitrary * representation. - * - `write-suppressed`, for adapters that keep a deleted id fenced against - * late in-flight writes. See {@link SpaceNodeMutation}'s - * `authoritativeInsert`, which is how a batch re-insertion is distinguished - * from such a late write. * - * A SQL adapter with a unique key produces neither. + * `write-suppressed` is portable anti-resurrection behavior. After a + * successful delete of an existing node, standalone puts for that id are + * suppressed for the lifetime of the running {@link StructuredStore} until a + * successful ordered put marks the id as an `authoritativeInsert`. The fence + * need not survive closing or recreating the store. */ export interface SpaceNodes { /** diff --git a/apps/server/src/modules/storage/profile.test.ts b/apps/server/src/modules/storage/profile.test.ts index 18be15c91..c51255000 100644 --- a/apps/server/src/modules/storage/profile.test.ts +++ b/apps/server/src/modules/storage/profile.test.ts @@ -64,7 +64,16 @@ describe('validateStorageProfile', () => { structured: { kind: 'postgres' }, blobs: { kind: 'disk' }, }), - ).toThrow(/not implemented yet.*disk/s); + ).toThrow(/not implemented yet.*disk, sqlite/s); + }); + + it('rejects an available preview adapter that is not selectable', () => { + expect(() => + validateStorageProfile({ + structured: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, + }), + ).toThrow(/preview adapter.*not selectable yet.*Selectable: disk/s); }); it('rejects a known but unimplemented blob backend', () => { diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index f962e8717..75fadddef 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -28,11 +28,20 @@ export interface StorageProfile { blobs: { kind: BlobBackendKind }; } +/** Backends with an adapter implementation, selectable or otherwise. */ +const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ + 'disk', + 'sqlite', +]; + /** - * Backends that exist today. Naming one that is not written yet must fail - * loudly rather than half-work. + * Backends whose complete capability matrix is safe for production use. + * + * SQLite deliberately stays out while physical Disk reads, World bootstrap, + * Blob placement, import/export, and Workspace remounting still have one + * authority only in the Disk profile. */ -const IMPLEMENTED_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; +const SELECTABLE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; const IMPLEMENTED_BLOBS: readonly BlobBackendKind[] = ['disk']; const STRUCTURED_KINDS: readonly RequestedStructuredKind[] = [ @@ -98,10 +107,17 @@ export function parseStorageProfile( * warning. */ export function validateStorageProfile(profile: StorageProfile): void { - if (!IMPLEMENTED_STRUCTURED.includes(profile.structured.kind)) { + if (!AVAILABLE_STRUCTURED.includes(profile.structured.kind)) { throw new StorageProfileError( `Structured backend "${profile.structured.kind}" is not implemented yet. ` + - `Available: ${IMPLEMENTED_STRUCTURED.join(', ')}.`, + `Adapters available: ${AVAILABLE_STRUCTURED.join(', ')}.`, + ); + } + if (!SELECTABLE_STRUCTURED.includes(profile.structured.kind)) { + throw new StorageProfileError( + `Structured backend "${profile.structured.kind}" has a preview adapter ` + + `but is not selectable yet. Required application capabilities still ` + + `depend on Disk. Selectable: ${SELECTABLE_STRUCTURED.join(', ')}.`, ); } if (!IMPLEMENTED_BLOBS.includes(profile.blobs.kind)) { diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index c6d2371be..25304cc9b 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -88,7 +88,7 @@ built above these ports, but its form is intentionally unresolved here. | Topic | Status | Current position | | ------------------------------------------------------ | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | -| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Only Disk exists. | +| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk is selectable; SQLite has an isolated contract-preview adapter but is not selectable; Postgres has no adapter. | | Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations. Only Disk exists. | | Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | | Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | @@ -2512,11 +2512,6 @@ guarantees. ### 12.9 Later phases — provisional -5. Add one new adapter at a time — SQLite, then Postgres, then Azure Blob — - running the same contract suites, migration fixtures, failure injection, - and concurrency tests against each. An adapter may exist for isolated - testing before its backend profile is selectable; profile validation keeps - rejecting it until the required capability matrix is satisfied. 6. Migrate the currently synchronous Agenetes persistence ports without changing their persist-before-notify, sequence, and fencing semantics. 7. Refactor RFS and built-in file tools only after a logical file-view contract From 5983aca1ec44a160da11d9dd86491281df1d0621 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Fri, 14 Aug 2026 12:24:31 +0800 Subject: [PATCH 02/15] refactor(storage): split the SQLite values grab-bag by dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `values.ts` held three unrelated concerns behind a name that described none of them: JSON codecs, `spaces` row statements, and collision-key allocation. Split it where a dependency boundary already ran. `rows.ts` owns everything that touches a stored column or `DatabaseSync`. `identity.ts` owns the pure title and label allocation rules, and imports no SQLite at all — which is what makes the cut a boundary rather than a preference. Several consumers now depend on less: `space-logs`, `space-tasks` and `structured-store` need only `rows.ts`, and `space-nodes` takes a single symbol from `identity.ts` in place of a mixed five-symbol block. Co-Authored-By: Claude Opus 5 (1M context) --- .../storage/backends/sqlite/identity.ts | 92 +++++++++++++++++++ .../backends/sqlite/{values.ts => rows.ts} | 86 ++--------------- .../storage/backends/sqlite/space-logs.ts | 2 +- .../storage/backends/sqlite/space-nodes.ts | 4 +- .../backends/sqlite/space-repository.ts | 5 +- .../storage/backends/sqlite/space-tasks.ts | 2 +- .../storage/backends/sqlite/space-write.ts | 6 +- .../backends/sqlite/structured-store.ts | 2 +- .../storage/backends/sqlite/test-support.ts | 3 +- 9 files changed, 113 insertions(+), 89 deletions(-) create mode 100644 apps/server/src/modules/storage/backends/sqlite/identity.ts rename apps/server/src/modules/storage/backends/sqlite/{values.ts => rows.ts} (78%) diff --git a/apps/server/src/modules/storage/backends/sqlite/identity.ts b/apps/server/src/modules/storage/backends/sqlite/identity.ts new file mode 100644 index 000000000..3a84a7497 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/identity.ts @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Allocation of the names a Space or Node is filed under. + * + * The `collision_key` columns carry a UNIQUE constraint, so a title or label + * has to be de-duplicated before it reaches the database rather than after a + * failed insert. These rules are pure and share `utils/naming` with Disk, so + * both backends hand out the same ` (2)` suffixes for the same inputs — see + * `backends/disk/space-title.ts` for the directory-locator half. + */ + +import { + dedupeName, + normalizeForCompare, + toSafeFilename, +} from '../../../../utils/naming.js'; + +import type { NodeContent } from '../../../canvas/persistence-types.js'; + +function allocatedSpaceTitle( + requested: string | null, + canvasId: string, + allocatedName: string, +): string | null { + if (requested === null) return null; + const base = toSafeFilename(requested, canvasId); + if (allocatedName === base) return requested; + const candidate = `${requested}${allocatedName.slice(base.length)}`; + return toSafeFilename(candidate, canvasId) === allocatedName + ? candidate + : allocatedName; +} + +export function allocateSpaceIdentity( + requestedTitle: string | null, + canvasId: string, + occupiedCollisionKeys: Iterable, +): { readonly title: string | null; readonly collisionKey: string } { + const base = toSafeFilename(requestedTitle, canvasId); + const allocated = dedupeName(base, occupiedCollisionKeys); + return { + title: allocatedSpaceTitle(requestedTitle, canvasId, allocated), + collisionKey: normalizeForCompare(allocated), + }; +} + +export function collisionKeyForTitle( + title: string | null, + canvasId: string, +): string { + return normalizeForCompare(toSafeFilename(title, canvasId)); +} + +export function allocateNodeIdentity( + record: NodeContent, + nodeId: string, + existingCollisionKey: string | null, + occupiedCollisionKeys: Iterable, +): { + readonly record: NodeContent; + readonly collisionKey: string; + readonly desiredCollisionKey: string; +} { + const trimmedLabel = + typeof record.label === 'string' && record.label.trim().length > 0 + ? record.label + : null; + if (trimmedLabel === null && existingCollisionKey !== null) { + return { + record, + collisionKey: existingCollisionKey, + desiredCollisionKey: existingCollisionKey, + }; + } + + const desired = toSafeFilename(trimmedLabel, nodeId); + const allocated = dedupeName(desired, occupiedCollisionKeys); + const suffix = + allocated.length > desired.length && allocated.startsWith(desired) + ? allocated.slice(desired.length) + : ''; + return { + record: + suffix && trimmedLabel + ? { ...record, label: `${trimmedLabel}${suffix}` } + : record, + collisionKey: normalizeForCompare(allocated), + desiredCollisionKey: normalizeForCompare(desired), + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/values.ts b/apps/server/src/modules/storage/backends/sqlite/rows.ts similarity index 78% rename from apps/server/src/modules/storage/backends/sqlite/values.ts rename to apps/server/src/modules/storage/backends/sqlite/rows.ts index b08adde64..e949a350d 100644 --- a/apps/server/src/modules/storage/backends/sqlite/values.ts +++ b/apps/server/src/modules/storage/backends/sqlite/rows.ts @@ -1,12 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +/** + * Movement of persisted values between domain records and SQLite rows. + * + * Every column this backend stores is either JSON text or a scalar, so the + * codecs here are the single place that decides what a well-formed stored + * value looks like. Reads validate on the way out: a row that no longer + * matches the domain shape is a corruption report, not a silent default. + */ + import { SQLITE_WORLD_COLLISION_KEY } from './database.js'; -import { - dedupeName, - normalizeForCompare, - toSafeFilename, -} from '../../../../utils/naming.js'; import { canvasFileShapeError } from '../../../canvas/persistence-validation.js'; import type { @@ -185,40 +189,6 @@ export function validateCanvasFile(record: CanvasFile, canvasId: string): void { stringifyJson(record.state, `Space ${JSON.stringify(canvasId)} state`); } -function allocatedSpaceTitle( - requested: string | null, - canvasId: string, - allocatedName: string, -): string | null { - if (requested === null) return null; - const base = toSafeFilename(requested, canvasId); - if (allocatedName === base) return requested; - const candidate = `${requested}${allocatedName.slice(base.length)}`; - return toSafeFilename(candidate, canvasId) === allocatedName - ? candidate - : allocatedName; -} - -export function allocateSpaceIdentity( - requestedTitle: string | null, - canvasId: string, - occupiedCollisionKeys: Iterable, -): { readonly title: string | null; readonly collisionKey: string } { - const base = toSafeFilename(requestedTitle, canvasId); - const allocated = dedupeName(base, occupiedCollisionKeys); - return { - title: allocatedSpaceTitle(requestedTitle, canvasId, allocated), - collisionKey: normalizeForCompare(allocated), - }; -} - -export function collisionKeyForTitle( - title: string | null, - canvasId: string, -): string { - return normalizeForCompare(toSafeFilename(title, canvasId)); -} - export function insertSpaceRow( database: DatabaseSync, record: CanvasFile, @@ -320,41 +290,3 @@ export function requirePositiveRevision( } return value; } - -export function allocateNodeIdentity( - record: NodeContent, - nodeId: string, - existingCollisionKey: string | null, - occupiedCollisionKeys: Iterable, -): { - readonly record: NodeContent; - readonly collisionKey: string; - readonly desiredCollisionKey: string; -} { - const trimmedLabel = - typeof record.label === 'string' && record.label.trim().length > 0 - ? record.label - : null; - if (trimmedLabel === null && existingCollisionKey !== null) { - return { - record, - collisionKey: existingCollisionKey, - desiredCollisionKey: existingCollisionKey, - }; - } - - const desired = toSafeFilename(trimmedLabel, nodeId); - const allocated = dedupeName(desired, occupiedCollisionKeys); - const suffix = - allocated.length > desired.length && allocated.startsWith(desired) - ? allocated.slice(desired.length) - : ''; - return { - record: - suffix && trimmedLabel - ? { ...record, label: `${trimmedLabel}${suffix}` } - : record, - collisionKey: normalizeForCompare(allocated), - desiredCollisionKey: normalizeForCompare(desired), - }; -} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts index 2ae4b2d1e..5fe88a840 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts @@ -8,7 +8,7 @@ import { } from '@huabu/shared/canvas-engine'; import { withImmediateTransaction } from './database.js'; -import { parseJson, stringifyJson } from './values.js'; +import { parseJson, stringifyJson } from './rows.js'; import { sanitizeId } from '../../../../utils/fs.js'; import type { SqliteStoreContext } from './database.js'; diff --git a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts index 25279b478..e12afd180 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts @@ -2,13 +2,13 @@ // Licensed under the MIT license. import { withImmediateTransaction } from './database.js'; +import { allocateNodeIdentity } from './identity.js'; import { - allocateNodeIdentity, decodeNodeRecord, requirePositiveRevision, stringifyJson, validateNodeContent, -} from './values.js'; +} from './rows.js'; import { sanitizeId } from '../../../../utils/fs.js'; import type { SqliteStoreContext } from './database.js'; diff --git a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts index f4631ce14..e12705cb4 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts @@ -2,14 +2,13 @@ // Licensed under the MIT license. import { withImmediateTransaction } from './database.js'; +import { allocateSpaceIdentity, collisionKeyForTitle } from './identity.js'; import { - allocateSpaceIdentity, - collisionKeyForTitle, decodeSpaceRow, insertSpaceRow, readSpaceRow, SPACE_COLUMNS, -} from './values.js'; +} from './rows.js'; import { sanitizeId } from '../../../../utils/fs.js'; import type { SqliteStoreContext } from './database.js'; diff --git a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts index 9c6cd7c4a..dda9096f3 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts @@ -11,7 +11,7 @@ import { } from '@huabu/shared'; import { withImmediateTransaction } from './database.js'; -import { parseJson, stringifyJson } from './values.js'; +import { parseJson, stringifyJson } from './rows.js'; import type { SqliteStoreContext } from './database.js'; import type { diff --git a/apps/server/src/modules/storage/backends/sqlite/space-write.ts b/apps/server/src/modules/storage/backends/sqlite/space-write.ts index 50f96a5a5..ad914619b 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-write.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-write.ts @@ -2,16 +2,16 @@ // Licensed under the MIT license. import { withImmediateTransaction } from './database.js'; -import { putSqliteNodeInTransaction } from './space-nodes.js'; +import { allocateSpaceIdentity } from './identity.js'; import { - allocateSpaceIdentity, insertSpaceRow, readSpaceRow, stringifyJson, updateSpaceRow, validateCanvasFile, validateNodeContent, -} from './values.js'; +} from './rows.js'; +import { putSqliteNodeInTransaction } from './space-nodes.js'; import { sanitizeId } from '../../../../utils/fs.js'; import type { SqliteStoreContext } from './database.js'; diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts index 18b67aa18..f1d372b11 100644 --- a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -2,12 +2,12 @@ // Licensed under the MIT license. import { SqliteStoreContext } from './database.js'; +import { readSpaceRow } from './rows.js'; import { createSqliteSpaceLogs } from './space-logs.js'; import { SqliteSpaceNodes } from './space-nodes.js'; import { SqliteSpaceRepository } from './space-repository.js'; import { SqliteSpaceTasks } from './space-tasks.js'; import { createSqliteSpaceWrite } from './space-write.js'; -import { readSpaceRow } from './values.js'; import { sanitizeId } from '../../../../utils/fs.js'; import type { StorageHealth } from '../../ports/common.js'; diff --git a/apps/server/src/modules/storage/backends/sqlite/test-support.ts b/apps/server/src/modules/storage/backends/sqlite/test-support.ts index e69b84111..bddebc8a8 100644 --- a/apps/server/src/modules/storage/backends/sqlite/test-support.ts +++ b/apps/server/src/modules/storage/backends/sqlite/test-support.ts @@ -7,8 +7,9 @@ import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { SQLITE_SCHEMA_VERSION } from './database.js'; +import { collisionKeyForTitle } from './identity.js'; +import { insertSpaceRow, parseJson } from './rows.js'; import { SqliteStructuredStore } from './structured-store.js'; -import { collisionKeyForTitle, insertSpaceRow, parseJson } from './values.js'; import type { CanvasFile, From 251dd110c5d03d66db2567c069f1865c367194c7 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Mon, 17 Aug 2026 19:00:12 +0800 Subject: [PATCH 03/15] fix(storage): complete SQLite task-run parity --- .../backends/disk/structured-store.test.ts | 50 ------- .../storage/backends/sqlite/space-tasks.ts | 48 +++++++ .../ports/contracts/space-tasks.contract.ts | 132 ++++++++++++++++++ docs/architecture/canvas-storage.md | 6 +- docs/proposals/multi-backend-storage.md | 23 +-- 5 files changed, 196 insertions(+), 63 deletions(-) diff --git a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts index 8b20988d9..d410af0db 100644 --- a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts @@ -142,56 +142,6 @@ describe('Disk Space Tasks', () => { rmSync(root, { recursive: true, force: true }); }); - it('completes a running Run atomically and keeps its message immutable', async () => { - const runs = store.space('canvas-task').tasks.runs; - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 5, - message: 'PR merged', - }), - ).resolves.toMatchObject({ - outcome: 'completed', - run: { - status: 'completed', - completion: { completedAt: 5, message: 'PR merged' }, - }, - }); - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 6, - message: 'PR merged', - }), - ).resolves.toMatchObject({ - outcome: 'unchanged', - run: { completion: { completedAt: 5, message: 'PR merged' } }, - }); - await expect( - runs.complete('task-a', 'run-a', { - completedAt: 7, - message: 'Different result', - }), - ).resolves.toMatchObject({ outcome: 'completion_conflict' }); - - await runs.create({ - runId: 'run-pending', - taskId: 'task-b', - canvasIdSnapshot: 'canvas-task', - goalSnapshot: 'Goal B', - rootProfileIdSnapshot: 'profile-b', - status: 'pending', - createdAt: 8, - }); - await expect( - runs.complete('task-b', 'run-pending', { completedAt: 9 }), - ).resolves.toMatchObject({ outcome: 'run_not_running' }); - await expect( - runs.complete('missing-task', 'run-a', { completedAt: 9 }), - ).resolves.toEqual({ outcome: 'task_not_found' }); - await expect( - runs.complete('task-a', 'missing-run', { completedAt: 9 }), - ).resolves.toEqual({ outcome: 'run_not_found' }); - }); - it('fails fast on malformed and internally inconsistent Task stores', async () => { mkdirSync(path.dirname(tasksPath('canvas-task')), { recursive: true }); writeFileSync(tasksPath('canvas-task'), '{"version":1,"tasks":{}}'); diff --git a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts index dda9096f3..4d27e2d17 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts @@ -3,9 +3,11 @@ import { taskRecordSchema, + taskRunCompletionSchema, taskRunRecordSchema, taskStoreSnapshotSchema, type TaskRecord, + type TaskRunCompletion, type TaskRunRecord, type TaskStoreSnapshot, } from '@huabu/shared'; @@ -17,6 +19,7 @@ import type { SqliteStoreContext } from './database.js'; import type { SpaceTaskRuns, SpaceTasks, + TaskRunCompletionResult, TaskRunUpdate, } from '../../ports/structured.js'; @@ -99,6 +102,11 @@ export class SqliteSpaceTasks implements SpaceTasks { create: (run: TaskRunRecord) => this.#createRun(run), update: (runId: string, update: TaskRunUpdate) => this.#updateRun(runId, update), + complete: ( + taskId: string, + runId: string, + completion: TaskRunCompletion, + ) => this.#completeRun(taskId, runId, completion), }); } @@ -165,6 +173,46 @@ export class SqliteSpaceTasks implements SpaceTasks { }); } + async #completeRun( + taskId: string, + runId: string, + completion: TaskRunCompletion, + ): Promise { + const parsedCompletion = taskRunCompletionSchema.safeParse(completion); + if (!parsedCompletion.success) { + throw new TypeError(`Invalid completion for Run ${runId}`); + } + return this.#mutate((snapshot) => { + if (!snapshot.tasks.some((task) => task.taskId === taskId)) { + return { outcome: 'task_not_found' }; + } + const index = snapshot.runs.findIndex((run) => run.runId === runId); + if (index < 0 || snapshot.runs[index]?.taskId !== taskId) { + return { outcome: 'run_not_found' }; + } + const current = snapshot.runs[index]; + if (!current) return { outcome: 'run_not_found' }; + if (current.status === 'completed') { + return current.completion?.message === parsedCompletion.data.message + ? { outcome: 'unchanged', run: current } + : { outcome: 'completion_conflict', run: current }; + } + if (current.status !== 'running') { + return { outcome: 'run_not_running', run: current }; + } + const parsedRun = taskRunRecordSchema.safeParse({ + ...current, + status: 'completed', + completion: parsedCompletion.data, + }); + if (!parsedRun.success) { + throw new TypeError(`Invalid completion update for Run ${runId}`); + } + snapshot.runs[index] = parsedRun.data; + return { outcome: 'completed', run: parsedRun.data }; + }); + } + #mutate(apply: (snapshot: TaskStoreSnapshot) => T): T { this.#context.assertMutationAllowed(this.#canvasId); const database = this.#context.database(); diff --git a/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts index dc6257a34..a02aba86f 100644 --- a/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/space-tasks.contract.ts @@ -167,6 +167,123 @@ export function describeSpaceTasksContract( }); }); + it('completes only a running Run and keeps the first completion immutable', async () => { + const { tasks, concurrent, canvasId } = await open(); + const owner = task(canvasId, 'task-complete', 1); + const other = task(canvasId, 'task-complete-other', 2); + const original = run(canvasId, owner.taskId, 'run-complete', 3); + await tasks.create(owner); + await tasks.create(other); + await tasks.runs.create(original); + + await expect( + tasks.runs.complete(owner.taskId, original.runId, { completedAt: 4 }), + ).resolves.toMatchObject({ outcome: 'run_not_running', run: original }); + + await tasks.runs.update(original.runId, { + status: 'running', + startedAt: 5, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 6, + message: 'Done', + }), + ).resolves.toMatchObject({ + outcome: 'completed', + run: { + status: 'completed', + completion: { completedAt: 6, message: 'Done' }, + }, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 7, + message: 'Done', + }), + ).resolves.toMatchObject({ + outcome: 'unchanged', + run: { completion: { completedAt: 6, message: 'Done' } }, + }); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 8, + message: 'Different', + }), + ).resolves.toMatchObject({ outcome: 'completion_conflict' }); + await expect( + tasks.runs.complete('task-missing', original.runId, { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'task_not_found' }); + await expect( + tasks.runs.complete(other.taskId, original.runId, { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'run_not_found' }); + await expect( + tasks.runs.complete(owner.taskId, 'run-missing', { completedAt: 9 }), + ).resolves.toEqual({ outcome: 'run_not_found' }); + await expect(concurrent.read()).resolves.toEqual({ + version: 1, + tasks: [owner, other], + runs: [ + { + ...original, + status: 'completed', + startedAt: 5, + completion: { completedAt: 6, message: 'Done' }, + }, + ], + }); + }); + + it('serializes competing completions and persists exactly one winner', async () => { + const { tasks, concurrent, canvasId } = await open(); + const owner = task(canvasId, 'task-competing-completion', 1); + const original = run( + canvasId, + owner.taskId, + 'run-competing-completion', + 2, + ); + await tasks.create(owner); + await tasks.runs.create(original); + await tasks.runs.update(original.runId, { + status: 'running', + startedAt: 3, + }); + + const results = await Promise.all([ + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 4, + message: 'First candidate', + }), + concurrent.runs.complete(owner.taskId, original.runId, { + completedAt: 5, + message: 'Second candidate', + }), + ]); + expect(results.map((result) => result.outcome).sort()).toEqual([ + 'completed', + 'completion_conflict', + ]); + const completed = results.find( + (result) => result.outcome === 'completed', + ); + const conflict = results.find( + (result) => result.outcome === 'completion_conflict', + ); + if (completed?.outcome !== 'completed') { + throw new Error('Expected one completion winner'); + } + if (conflict?.outcome !== 'completion_conflict') { + throw new Error('Expected one completion conflict'); + } + expect(conflict.run).toEqual(completed.run); + await expect(concurrent.read()).resolves.toEqual({ + version: 1, + tasks: [owner], + runs: [completed.run], + }); + }); + it('rejects Task and Run records scoped to another Space', async () => { const { tasks, canvasId } = await open(); const owner = task(canvasId, 'task-scope', 1); @@ -202,6 +319,11 @@ export function describeSpaceTasksContract( await expect( tasks.runs.update(ownedRun.runId, { startedAt: -1 }), ).rejects.toThrow(); + await expect( + tasks.runs.complete(owner.taskId, ownedRun.runId, { + completedAt: -1, + }), + ).rejects.toThrow(); await expect(tasks.read()).resolves.toEqual({ version: 1, tasks: [owner], @@ -272,6 +394,11 @@ export function describeSpaceTasksContract( await expect( missing.runs.update(ownedRun.runId, { status: 'running' }), ).rejects.toThrow(); + await expect( + missing.runs.complete(owner.taskId, ownedRun.runId, { + completedAt: 3, + }), + ).rejects.toThrow(); }); it('rejects mutations while structured deletion is fenced', async () => { @@ -293,6 +420,11 @@ export function describeSpaceTasksContract( await expect( tasks.runs.update(original.runId, { status: 'running' }), ).rejects.toThrow(); + await expect( + tasks.runs.complete(owner.taskId, original.runId, { + completedAt: 5, + }), + ).rejects.toThrow(); await expect(tasks.read()).resolves.toEqual(before); } finally { await session.abort(); diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 2b5afc624..221ecb125 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -80,7 +80,7 @@ Key points: - The memory analyzer reads Space existence and at most 100 recent action events through one `SpaceHandle`. A missing Space skips the pass before reading memory files or calling the model; corrupt part data still fails the pass. Memory body/state files remain materialized workspace paths, while Agenetes-owned chat history is not part of the curator bundle. - **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** The canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. - Durable Agenetes workload records live in `.history/threads.json` (`agenetes-v2` schema, one record per thread; written by `FileThreadStore`). The host-local `namespace.storage.root` is never persisted: reads bind each record to the current Space namespace, so a Home synchronized across computers cannot redirect storage back to another machine's absolute path. -- Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. +- Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`/`runs.complete`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. - Legacy chat files are one-way migrated into `chat_v2/` at workspace activation and retired to `.bak`: the oldest pi-ai `Context` `chat/.json` via `migrate-chat-threads.ts` (hop 1), then the M5.6 `chat/.turns.jsonl` / `.active.json` via `migrate-chat-turns.ts` (hop 2). If hop 1 finds both formats after an interrupted launch, it completes a strict converted prefix atomically or preserves an existing tail when the full conversion is its prefix. Divergent logs are retained rather than guessed or overwritten; hop 2 skips the paired turn log while a valid same-thread legacy Context remains or its JSON cannot be read safely, so a later activation can retry both copies without blocking unrelated migrations. The obsolete `CanvasStore` chat methods and `chatPath()` helper were removed in Phase 2; `chatDir()` remains because change-review and agent-owned files still use that directory. ## 3. Storage composition and ownership @@ -103,7 +103,7 @@ Key points: | `index.ts` | Public exports only; application code imports here rather than reaching into an adapter. | | `canvas-store.ts`, `paths.ts`, `canvas-dirs.ts` | Deprecated forwarding shims with no logic, retained only for high-fanout compatibility imports. | -The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; a SQL adapter may use a native transaction. +The Disk structured adapter and compatibility facade resolve the same cached legacy object, so migration does not create two in-memory authorities. The SQLite adapter instead owns one explicit database filename and connection; retained handles stay bound to that connection, and its `init`, `health`, and `close` lifecycle is exercised only by direct tests. All portable repository methods are async. `SpaceRepository` owns membership reads, structured create/rename, and an exclusive `beginDelete()` session; composition holds that session across the existing blob-first delete saga and then calls `finish()` or `abort()`. Every Space-record write goes through `SpaceHandle.write`, which is the version-checked replacement with the node and delta batch attached; `SpaceHandle.read` reads only. `SpaceNodes` returns complete records plus revision tokens without exposing filenames. `write` preserves the old node mutations → Space record → optional delta order. When a normal in-process node → record → delta batch rejects, the adapter must restore that batch's prestate before returning the rejection. An explicit title rename remains the preceding ordered, best-effort boundary and is not rolled back with the batch. The port does not promise process-crash or power-loss recovery, a determinate result after an unknown remote outcome, multi-process serialization, idempotent retry, or publication. Disk meets the in-process restoration requirement with its existing before-image rollback; SQLite uses a native transaction. Canvas persistence DTOs and the write coordinator live under `modules/canvas/`; Workspace identity, durable membership, and Disk locators live under `modules/storage/`, while active-Workspace lifecycle and boot migrations remain under `modules/workspace/`; generic filesystem and Markdown codecs live under `utils/`. `canvasRoot()` validates the identifier and then verifies that the resolved Space directory remains a strict descendant of the active Workspace before any downstream Disk operation receives it. `module-boundaries.test.ts` enforces the storage dependency direction, prevents new consumers of the forwarding shims, and holds the neutrality guard: no production file outside `storage/` may import a Disk layout symbol or a legacy `CanvasStore` symbol. The check is import-level and symbol-level — a local variable that happens to be called `artifactPath` is not a violation, while importing `canvasRoot`, or reaching `getCanvasStore` through the barrel, is. Migrations are exempt because they rewrite frozen historical on-disk shapes; tests are exempt for the same reason they may name an adapter. @@ -133,7 +133,7 @@ The launch path deliberately has no compensation transaction. A launch failure l ### 3.3 Task Run completion -`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. The Disk adapter performs lookup, `running → completed`, and persistence under the existing per-Canvas Task mutation mutex, so HTTP and built-in-tool callers share one atomic transition rather than performing a read-then-update race. +`RunCompletionService.complete()` validates the shared request and delegates the guarded transition to `SpaceTaskRuns.complete()`. Both structured adapters perform lookup, `running → completed`, and persistence inside one Task-snapshot mutation boundary: Disk uses the per-Canvas Task mutex and atomic file replacement, while SQLite uses an immediate transaction. HTTP and built-in-tool callers therefore share one atomic transition rather than performing a read-then-update race. A completed Run stores immutable `completion.completedAt` and an optional trimmed caller-owned `completion.message`. The platform treats the message as untrusted text and does not interpret issue, pull-request, or URL semantics. A retry with the same normalized message is idempotent and preserves the original timestamp; a different message conflicts. A `pending` Run cannot complete, and Agent turn termination never implies Run completion. diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 25304cc9b..c45b78f41 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -144,8 +144,9 @@ external-note discovery watches `nodes/`, and export archives the entire Space directory. Therefore wrapping `CanvasStore` in a database adapter would not by itself make the application backend-neutral. -Canvas/Space persistence is currently Disk-only. SQLite, Postgres, and Azure -Blob adapters for this data do not yet exist. +Runtime Canvas/Space persistence remains Disk-only. An isolated SQLite +structured adapter exists for contract and integration tests, while Postgres +and Azure Blob adapters do not yet exist. ## 4. Goals @@ -175,9 +176,9 @@ Blob adapters for this data do not yet exist. their product semantics are defined. - Implementing online backend migration, replication, backup, or disaster recovery. -- Shipping any non-Disk adapter. The phases in §12 remove reasons why SQLite, - Postgres, and Azure _cannot_ be implemented; that is not the same as - implementing them. +- Making a non-Disk adapter runtime-selectable. Phase 5 proves an isolated + adapter against the contracts without registering it in composition or + changing product capabilities. ## 6. Settled backend split and implemented minimum contracts @@ -301,8 +302,9 @@ into place makes the failed write invisible instead of unremovable. ### 6.3 Composition -Configuration has two axes. The current shape carries only a backend kind per -axis, because no adapter yet needs more: +Configuration has two axes. The runtime-selectable profile carries only a +backend kind per axis. The isolated SQLite preview receives its explicit +database filename directly and is not constructed from this profile: ```ts interface StorageProfile { @@ -325,7 +327,8 @@ node-local DiskBlob implementation is unsafe in a multi-replica deployment unless the path is a deliberately shared and supported filesystem. SQLite on a network filesystem has different correctness and availability constraints from local SQLite. `validateStorageProfile()` is where such rules live; today it -rejects kinds that are named but not implemented, so an unsupported profile +rejects recognized kinds that are unavailable or deliberately unselectable, +including SQLite's preview-specific diagnostic, so an unsupported profile fails at startup with an actionable message rather than nondeterministically while serving data. @@ -684,7 +687,7 @@ exceptions: one names what it returns, the other opens a session. ```ts interface StructuredStore { - readonly kind: StructuredBackendKind; // 'disk' — implemented adapters only + readonly kind: StructuredBackendKind; // 'disk' | 'sqlite'; only Disk is selectable init(): Promise; health(): Promise; @@ -746,7 +749,7 @@ interface SpaceChanges { interface SpaceTasks { read(): Promise; // Tasks and Runs in one snapshot create(task: TaskRecord): Promise; - readonly runs: SpaceTaskRuns; // create(run), update(runId, patch) + readonly runs: SpaceTaskRuns; // create, update, and atomic complete } ``` From 3b96a40b3fed4de2dd870ac8e2fbeed87371af90 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 18 Aug 2026 13:04:28 +0800 Subject: [PATCH 04/15] docs(storage): refresh Phase 5 after boundary merge --- docs/proposals/multi-backend-storage.md | 53 +++++++++++++++---------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index c45b78f41..277770d22 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -167,7 +167,9 @@ and Azure Blob adapters do not yet exist. ## 5. Non-goals -- Selecting an ORM, SQL query builder, Postgres driver, or SQLite driver. +- Selecting a production ORM, SQL query builder, Postgres driver, or final + SQLite driver. The isolated Phase 5 preview uses built-in `node:sqlite` + without making that production choice. - Defining the final relational schema or migration framework. - Choosing a VFS, FUSE, materialization, cache, or write-back design. - Replacing RFS or the canonical `SpaceQuery` / `CanvasCommand` contracts in @@ -946,9 +948,10 @@ explicitly: ## 12. Migration plan -Phases 1–4 are implemented and specified below. Phase 5 onward keeps the -provisional character of the original outline: those entries record intended -order, not approved designs. +Phases 1–4.5 are implemented and merged. Phase 5 is implemented by this +isolated contract preview. Phase 6 onward keeps the provisional character of +the original outline: those entries record intended order, not approved +designs. The current on-disk format remains readable throughout port extraction. A database adapter must not require Disk consumers to simulate tables, and the @@ -1810,12 +1813,15 @@ justify. footing and was left alone as Phase-1 surface. Not changed, deliberately: `authoritativeInsert` and the `write-suppressed` -put outcome remain in the portable shapes. Both exist for Disk's in-memory -deletion fence, and neither has a portable meaning a SQL adapter would -produce. They are now documented as adapter-shaped, the way `duplicate-node` -already was, rather than renamed or pushed behind the adapter — the honest -resolution needs a second adapter to say what the shared abstraction is, and -inventing one now would be the same speculative move this trim is undoing. +put outcome remain in the portable shapes. At this phase boundary, both +existed for Disk's in-memory deletion fence and a second adapter was still +needed to establish their shared meaning. + +**Superseded by Phase 5:** the SQLite contract preview supplies that second +adapter and confirms the portable rule as a connection-lifetime +anti-resurrection fence: after deletion, standalone puts are suppressed until +an ordered authoritative insert commits (§12.6.2). The outcome is therefore +no longer merely adapter-shaped. The review also asked composition to move default-title allocation ("Untitled", "Untitled (1)", …) into `create`, which would have removed the @@ -1964,10 +1970,11 @@ bearing: change-review records and Tasks are not history, whatever Disk's arrives, the group comes back — and `events` is where it was before, so nothing else has to move. -### 12.5 Phase 4.5 — storage-owned layout moves inside the boundary — **implemented** +### 12.5 Phase 4.5 — storage-owned layout moves inside the boundary — **merged** -Phase 5 adds a second structured backend. Before it does, the layout knowledge -that belongs to the _Disk_ backend has to stop living outside `storage/`. +Phase 5 would introduce a second structured backend. Before that work, the +layout knowledge that belongs to the _Disk_ backend had to stop living outside +`storage/`. Otherwise every later backend inherits a module named `disk` as the ambient description of where Spaces are, and each one pays to migrate the same callers again. @@ -2050,11 +2057,11 @@ substrate-specific but fails the test for the same reason — it exists so Windows can rename a Space _directory_ safely, and under SQLite there is no such rename. -`naming.ts` is misfiled in a different way: pure string logic with no I/O, -already re-exported rather than owned. It passes the test trivially (a second -backend needs the identical rules) but has no business behind a `disk` -segment. Phase 5 extracts it to `utils/naming.ts` as a side effect of needing -it twice; that extraction belongs here, where it is the point. +`naming.ts` was misfiled in a different way: pure string logic with no I/O, +already re-exported rather than owned. It passed the test trivially (a second +backend needs the identical rules) but had no business behind a `disk` +segment. Phase 4.5 extracted it to `utils/naming.ts`, where the shared rule has +a backend-neutral owner. Because the residue that survives the test is three setting helpers and `getWorkspacePath()` itself — none of it filesystem-specific — the target is a @@ -2131,8 +2138,9 @@ boundary test; behavior parity is asserted by the existing Disk suites, which must pass unchanged — a diff that alters a Disk test's expectations is out of scope by definition. -Phase 5 rebases onto this and drops its `utils/naming.ts` extraction, its -`workspace/disk/naming.ts` shim, and the corresponding roadmap edits. +Phase 5 builds on this merged result and carries none of its former +`utils/naming.ts` extraction, `workspace/disk/naming.ts` shim, or parallel +roadmap edits. **Landed for the Workspace-to-storage substrate move.** `modules/workspace/` is flat and holds `paths.ts` plus `migrations/`; the Disk record layout, blob @@ -2728,18 +2736,19 @@ Before a new backend is production-ready: persistence ownership, namespace, sequence, and replay invariants. - [Agenetes-Agentlet Gateway Consolidation](./agenetes-agentlet-gateway-consolidation.md) — records removal of the old Agentlet SQLite session store; it must not be - confused with the proposed SQLite structured backend. + confused with the SQLite structured contract-preview backend. ## 17. Code entry points | File/dir | Responsibility | | -------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–4 tree (§§12.1–12.4), guarded by `module-boundaries.test.ts`. | +| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–5 tree (§§12.1–12.6), guarded by `module-boundaries.test.ts`. | | [`apps/server/src/modules/storage/ports/`](../../apps/server/src/modules/storage/ports/) | The two ports; reusable suites live in `ports/contracts/`. `blob.ts` is normative (§7.1); `structured.ts` owns the Space collection and the per-Space handle: record read/write, nodes, changes, Tasks, and history. | | [`apps/server/src/modules/storage/storage.ts`](../../apps/server/src/modules/storage/storage.ts) | Composition root: maps profiles to adapters, guards blob puts, and holds a lifecycle deletion session across the blob-first cleanup saga. | | [`.../storage/backends/disk/legacy/canvas-store-cache.ts`](../../apps/server/src/modules/storage/backends/disk/legacy/canvas-store-cache.ts) | Bounded LRU of legacy Disk Space objects. The single owner both the adapter and the facade resolve through, and the real limit of `space(id)` identity (§12.2.4). | | [`apps/server/src/modules/storage/profile.ts`](../../apps/server/src/modules/storage/profile.ts) | Two-axis backend selection from env, and the fail-fast validation hook for unsupported combinations. | | [`apps/server/src/modules/storage/backends/disk/`](../../apps/server/src/modules/storage/backends/disk/) | Every Disk implementation: blob/structured stores, the Space collection, and the per-Space record, node, log, and Task adapters, in-process batch restoration, and the legacy class under `legacy/`. | +| [`apps/server/src/modules/storage/backends/sqlite/`](../../apps/server/src/modules/storage/backends/sqlite/) | Isolated `node:sqlite` structured adapter, strict schema and migrations, transaction-backed writes, and real-file contract/integration tests; available for proof but not runtime-selectable. | | [`.../storage/compatibility/canvas.ts`](../../apps/server/src/modules/storage/compatibility/canvas.ts) | Residual Disk read surface plus direct-module lifecycle test fixtures; production structured mutations enumerated in §12.4 use the portable ports. | | [`apps/server/src/modules/agent/memory/analyzer.ts`](../../apps/server/src/modules/agent/memory/analyzer.ts) | P3 repository consumer for strict Space existence, bounded action events, and intent episodes; physical chat and memory files remain Disk-specific. | | [`apps/server/src/modules/canvas/write-coordinator.ts`](../../apps/server/src/modules/canvas/write-coordinator.ts) | Canvas mutation coordinator and per-Space write lock, held across asynchronous node read, revision CAS, and put. | From 5569913894111798a14110b228f62f2eb3d26ea6 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Fri, 4 Sep 2026 13:07:10 +0800 Subject: [PATCH 05/15] fix(storage): align SQLite preview with current contracts --- .../agent/conversation/prompt/debug-prompt.ts | 3 + .../src/modules/agent/memory/trigger.ts | 3 + .../src/modules/canvas/canvas.route.test.ts | 6 +- .../storage/backends/disk/space-nodes.test.ts | 2 +- .../backends/disk/space-repository.test.ts | 7 +- .../backends/disk/structured-store.test.ts | 26 +++ .../storage/backends/sqlite/contracts.test.ts | 60 ++++++- .../storage/backends/sqlite/database.ts | 167 +++--------------- .../storage/backends/sqlite/fixtures/v1.sql | 14 +- .../backends/sqlite/integration.test.ts | 93 +++++++++- .../modules/storage/backends/sqlite/rows.ts | 38 ++-- .../backends/sqlite/space-extension.ts | 58 ++++++ .../storage/backends/sqlite/space-nodes.ts | 104 +++++++---- .../backends/sqlite/space-repository.ts | 43 ++++- .../storage/backends/sqlite/space-write.ts | 65 ++----- .../backends/sqlite/structured-store.ts | 4 +- .../storage/backends/sqlite/test-support.ts | 28 +++ .../src/modules/storage/capabilities.test.ts | 13 +- .../ports/contracts/space-nodes.contract.ts | 81 ++------- .../contracts/structured-store.contract.ts | 1 + .../src/modules/storage/ports/structured.ts | 53 +++--- apps/server/src/modules/storage/profile.ts | 15 +- docs/proposals/multi-backend-storage.md | 108 +++++++++-- 23 files changed, 606 insertions(+), 386 deletions(-) create mode 100644 apps/server/src/modules/storage/backends/sqlite/space-extension.ts diff --git a/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts b/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts index b923b2098..c56496286 100644 --- a/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts +++ b/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts @@ -153,6 +153,9 @@ type SubstrateResolution = /** Where this module keeps one log per thread on a Disk substrate. */ function diskLogPath(substrate: SpaceSubstrate, threadId: string): string { + if (substrate.kind !== 'disk') { + throw new Error('Debug prompt logs require a Disk extension substrate'); + } return path.join( substrate.directory, `${sanitizeId(threadId, 'threadId')}.prompt.log`, diff --git a/apps/server/src/modules/agent/memory/trigger.ts b/apps/server/src/modules/agent/memory/trigger.ts index f2d4b1ced..28b8eb2ec 100644 --- a/apps/server/src/modules/agent/memory/trigger.ts +++ b/apps/server/src/modules/agent/memory/trigger.ts @@ -45,6 +45,9 @@ const MEMORY_NAMESPACE = 'huabu.memory'; * the substrate, never a port member. */ function diskStatePath(substrate: SpaceSubstrate): string { + if (substrate.kind !== 'disk') { + throw new Error('Memory state requires a Disk extension substrate'); + } return path.join(substrate.directory, 'state.json'); } diff --git a/apps/server/src/modules/canvas/canvas.route.test.ts b/apps/server/src/modules/canvas/canvas.route.test.ts index fbe276855..1cbbee60a 100644 --- a/apps/server/src/modules/canvas/canvas.route.test.ts +++ b/apps/server/src/modules/canvas/canvas.route.test.ts @@ -769,7 +769,9 @@ describe('Space export/import persistence', () => { createCanvas('c1', 'Private Export'); const promptStore = await space('c1').extension('huabu.prompt.log'); const memoryStore = await space('c1').extension('huabu.memory'); - if (!promptStore || !memoryStore) throw new Error('Expected Disk stores'); + if (promptStore?.kind !== 'disk' || memoryStore?.kind !== 'disk') { + throw new Error('Expected Disk stores'); + } writeFileSync( join(promptStore.directory, 'thread.prompt.log'), 'private system and user prompt', @@ -806,7 +808,7 @@ describe('Space export/import persistence', () => { const importedPrompt = await space(importedId).extension('huabu.prompt.log'); const importedMemory = await space(importedId).extension('huabu.memory'); - if (!importedPrompt || !importedMemory) { + if (importedPrompt?.kind !== 'disk' || importedMemory?.kind !== 'disk') { throw new Error('Expected imported Disk stores'); } expect( diff --git a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts index 8c91fed63..ebf7d6760 100644 --- a/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts +++ b/apps/server/src/modules/storage/backends/disk/space-nodes.test.ts @@ -57,9 +57,9 @@ describeSpaceNodesContract('Disk', async () => { const space = store.space('node-space'); return { repository: space.nodes, - space, missingRepository: store.space('missing-node-space').nodes, expectedCanvasId: 'node-space', + deletedNodePut: 'write-suppressed', cleanup: () => { vi.restoreAllMocks(); resetStorageCache(); diff --git a/apps/server/src/modules/storage/backends/disk/space-repository.test.ts b/apps/server/src/modules/storage/backends/disk/space-repository.test.ts index 250a61022..033cbb0b6 100644 --- a/apps/server/src/modules/storage/backends/disk/space-repository.test.ts +++ b/apps/server/src/modules/storage/backends/disk/space-repository.test.ts @@ -141,9 +141,12 @@ describeSpaceExtensionContract('Disk', () => { // An owner of a Disk namespace writes files into its directory; nothing // about the shape is storage's business, so the suite borrows the // simplest one an owner could pick. - write: (substrate, value) => - writeFileSync(path.join(substrate.directory, 'value'), value, 'utf8'), + write: (substrate, value) => { + if (substrate.kind !== 'disk') throw new Error('Expected Disk substrate'); + writeFileSync(path.join(substrate.directory, 'value'), value, 'utf8'); + }, read: (substrate) => { + if (substrate.kind !== 'disk') throw new Error('Expected Disk substrate'); const file = path.join(substrate.directory, 'value'); return existsSync(file) ? readFileSync(file, 'utf8') : null; }, diff --git a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts index d410af0db..ef0ad0772 100644 --- a/apps/server/src/modules/storage/backends/disk/structured-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/structured-store.test.ts @@ -112,6 +112,8 @@ describe('Disk Space extension workspace binding', () => { try { const substrate = await pending; expect(substrate?.kind).toBe('disk'); + if (substrate?.kind !== 'disk') + throw new Error('Expected Disk substrate'); expect(substrate?.directory.startsWith(`${firstRoot}${path.sep}`)).toBe( true, ); @@ -126,6 +128,30 @@ describe('Disk Space extension workspace binding', () => { }); }); +describeSpaceTasksContract('Disk', () => { + const root = freshWorkspace('huabu-task-contract-'); + seedSpace(root, 'canvas-task', 'Canvas Task'); + const store = new DiskStructuredStore(); + return { + tasks: store.space('canvas-task').tasks, + concurrent: store.space('canvas-task').tasks, + canvasId: 'canvas-task', + missing: store.space('missing-canvas').tasks, + missingCanvasId: 'missing-canvas', + beginDelete: async () => { + const result = await store.spaces().beginDelete({ + canvasId: 'canvas-task', + }); + if (!result.ok) throw new Error('Ordinary Space must be deletable'); + return result.session; + }, + cleanup: () => { + resetStorageCache(); + rmSync(root, { recursive: true, force: true }); + }, + }; +}); + describe('Disk Space Tasks', () => { let root = ''; let store: DiskStructuredStore; diff --git a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts index b30803d4e..0de7391bc 100644 --- a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts +++ b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts @@ -5,9 +5,11 @@ import { SqliteStructuredStore } from './structured-store.js'; import { createSqliteTestFile, installDeltaAbortTrigger, + openEmptySqliteTestStore, openSqliteTestStore, readSqliteDeltaLog, } from './test-support.js'; +import { describeSpaceExtensionContract } from '../../ports/contracts/space-extension.contract.js'; import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; import { describeSpaceNodesContract } from '../../ports/contracts/space-nodes.contract.js'; import { describeSpaceRepositoryContract } from '../../ports/contracts/space-repository.contract.js'; @@ -42,6 +44,9 @@ describeSpaceRepositoryContract('SQLite', async () => { const harness = await openSqliteTestStore( 'huabu-sqlite-space-repository-contract-', ); + const emptyStores: Array< + Awaited> + > = []; return { repository: harness.store.spaces(), read: (canvasId: string) => harness.store.space(canvasId).read(), @@ -55,7 +60,20 @@ describeSpaceRepositoryContract('SQLite', async () => { 'body', ), }), - cleanup: harness.cleanup, + openEmptyNamespace: async () => { + const empty = await openEmptySqliteTestStore( + 'huabu-sqlite-empty-namespace-contract-', + ); + emptyStores.push(empty); + return { + repository: empty.store.spaces(), + read: (canvasId: string) => empty.store.space(canvasId).read(), + }; + }, + cleanup: async () => { + for (const empty of emptyStores.splice(0)) await empty.cleanup(); + await harness.cleanup(); + }, }; }); @@ -66,9 +84,47 @@ describeSpaceNodesContract('SQLite', async () => { const space = harness.store.space(canvasId); return { repository: space.nodes, - space, missingRepository: harness.store.space('sqlite-nodes-missing').nodes, expectedCanvasId: canvasId, + deletedNodePut: 'allowed', + cleanup: harness.cleanup, + }; +}); + +describeSpaceExtensionContract('SQLite', async () => { + const harness = await openSqliteTestStore('huabu-sqlite-extension-contract-'); + const table = 'contract_extension_values'; + return { + repository: harness.store.spaces(), + space: (canvasId: string) => harness.store.space(canvasId), + write: (substrate, value: string) => { + if (substrate.kind !== 'sqlite') { + throw new Error('Expected a SQLite substrate'); + } + substrate.database.exec( + `CREATE TABLE IF NOT EXISTS ${table} ( + extension_id INTEGER PRIMARY KEY, + value TEXT NOT NULL, + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT`, + ); + substrate.database + .prepare( + `INSERT INTO ${table} (extension_id, value) VALUES (?, ?) + ON CONFLICT(extension_id) DO UPDATE SET value = excluded.value`, + ) + .run(substrate.extensionId, value); + }, + read: (substrate) => { + if (substrate.kind !== 'sqlite') { + throw new Error('Expected a SQLite substrate'); + } + const row = substrate.database + .prepare(`SELECT value FROM ${table} WHERE extension_id = ?`) + .get(substrate.extensionId); + return typeof row?.['value'] === 'string' ? row['value'] : null; + }, cleanup: harness.cleanup, }; }); diff --git a/apps/server/src/modules/storage/backends/sqlite/database.ts b/apps/server/src/modules/storage/backends/sqlite/database.ts index d4459bbe4..acd0e4afb 100644 --- a/apps/server/src/modules/storage/backends/sqlite/database.ts +++ b/apps/server/src/modules/storage/backends/sqlite/database.ts @@ -3,6 +3,11 @@ import { DatabaseSync } from 'node:sqlite'; +import { + assertSpaceMutationAllowed, + beginSpaceDeleteAdmission, +} from '../../space-lifecycle-admission.js'; + import type { StorageHealth } from '../../ports/common.js'; export const SQLITE_SCHEMA_VERSION = 1; @@ -28,7 +33,7 @@ const SCHEMA_V1 = ` canvas_id TEXT NOT NULL, node_id TEXT NOT NULL, record_json TEXT NOT NULL CHECK (json_valid(record_json)), - revision INTEGER NOT NULL CHECK (revision > 0), + revision TEXT NOT NULL CHECK (length(revision) > 0), label_collision_key TEXT NOT NULL, PRIMARY KEY (canvas_id, node_id), UNIQUE (canvas_id, label_collision_key), @@ -59,6 +64,14 @@ const SCHEMA_V1 = ` FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE ) STRICT; + CREATE TABLE space_extensions ( + extension_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + namespace TEXT NOT NULL, + UNIQUE (canvas_id, namespace), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + CREATE TABLE delta_log ( canvas_id TEXT NOT NULL, version INTEGER NOT NULL, @@ -133,114 +146,17 @@ export function applySqliteMigrations( } } -function reserveWorldCollisionKey(database: DatabaseSync): void { - withImmediateTransaction(database, () => { - const world = database - .prepare('SELECT canvas_id, collision_key FROM spaces WHERE is_world = 1') - .get(); - if (world === undefined) return; - - const canvasId = world['canvas_id']; - const collisionKey = world['collision_key']; - if (typeof canvasId !== 'string' || typeof collisionKey !== 'string') { - throw new SyntaxError('SQLite World Space has malformed identity fields'); - } - if (collisionKey === SQLITE_WORLD_COLLISION_KEY) return; - - const conflict = database - .prepare( - `SELECT canvas_id - FROM spaces - WHERE collision_key = ? AND canvas_id <> ?`, - ) - .get(SQLITE_WORLD_COLLISION_KEY, canvasId); - if (conflict !== undefined) { - throw new Error( - `Cannot reserve SQLite World collision slot ${JSON.stringify( - SQLITE_WORLD_COLLISION_KEY, - )}: it is already occupied`, - ); - } - - const result = database - .prepare( - `UPDATE spaces - SET collision_key = ? - WHERE canvas_id = ? AND is_world = 1`, - ) - .run(SQLITE_WORLD_COLLISION_KEY, canvasId); - if (Number(result.changes) !== 1) { - throw new Error('Could not reserve the SQLite World collision slot'); - } - }); -} - -type DeleteAdmission = { - readonly resolve: (release: () => void) => void; - readonly reject: (error: Error) => void; -}; - -class SpaceDeleteGate { - #active = false; - #closed = false; - readonly #waiting: DeleteAdmission[] = []; - - get pending(): boolean { - return this.#active || this.#waiting.length > 0; - } - - get idle(): boolean { - return !this.#active && this.#waiting.length === 0; - } - - acquire(): Promise<() => void> { - if (this.#closed) { - return Promise.reject(new Error('SQLite store is closed')); - } - if (!this.#active) { - this.#active = true; - return Promise.resolve(this.#releaseFunction()); - } - return new Promise((resolve, reject) => { - this.#waiting.push({ resolve, reject }); - }); - } - - close(): void { - if (this.#closed) return; - this.#closed = true; - const error = new Error('SQLite store is closed'); - for (const admission of this.#waiting.splice(0)) { - admission.reject(error); - } - } - - #releaseFunction(): () => void { - let released = false; - return () => { - if (released) return; - released = true; - this.#active = false; - if (this.#closed) return; - const next = this.#waiting.shift(); - if (!next) return; - this.#active = true; - next.resolve(this.#releaseFunction()); - }; - } -} - /** One connection and all adapter-lifetime process-local state. */ export class SqliteStoreContext { readonly now: () => number; readonly #database: DatabaseSync; - readonly #deleteGates = new Map(); - readonly #nodeTombstones = new Set(); + readonly #admissionScope: string; #state: 'new' | 'open' | 'closed' = 'new'; constructor(filename: string, now: () => number) { this.now = now; + this.#admissionScope = `sqlite:${filename}`; this.#database = new DatabaseSync(filename, { open: false }); } @@ -260,7 +176,6 @@ export class SqliteStoreContext { throw new Error('Could not enable SQLite foreign key enforcement'); } applySqliteMigrations(this.#database); - reserveWorldCollisionKey(this.#database); this.#state = 'open'; } catch (error) { if (this.#database.isOpen) this.#database.close(); @@ -288,8 +203,6 @@ export class SqliteStoreContext { close(): void { if (this.#state === 'closed') return; this.#state = 'closed'; - for (const gate of this.#deleteGates.values()) gate.close(); - this.#deleteGates.clear(); if (this.#database.isOpen) this.#database.close(); } @@ -310,58 +223,22 @@ export class SqliteStoreContext { assertMutationAllowed(canvasId: string): void { this.assertOpen(); - if (this.#deleteGates.get(canvasId)?.pending) { - throw new Error( - `Cannot mutate Space "${canvasId}" while deletion is pending`, - ); - } + assertSpaceMutationAllowed(this.#admissionScope, canvasId); } async acquireDelete(canvasId: string): Promise<() => void> { this.assertOpen(); - let gate = this.#deleteGates.get(canvasId); - if (!gate) { - gate = new SpaceDeleteGate(); - this.#deleteGates.set(canvasId, gate); - } - const releaseGate = await gate.acquire(); + const releaseGate = await beginSpaceDeleteAdmission( + this.#admissionScope, + canvasId, + ); try { this.assertOpen(); } catch (error) { releaseGate(); throw error; } - - let released = false; - return () => { - if (released) return; - released = true; - releaseGate(); - if (gate?.idle && this.#deleteGates.get(canvasId) === gate) { - this.#deleteGates.delete(canvasId); - } - }; - } - - isNodeTombstoned(canvasId: string, nodeId: string): boolean { - return this.#nodeTombstones.has(this.#nodeKey(canvasId, nodeId)); - } - - setNodeTombstone(canvasId: string, nodeId: string, present: boolean): void { - const key = this.#nodeKey(canvasId, nodeId); - if (present) this.#nodeTombstones.add(key); - else this.#nodeTombstones.delete(key); - } - - clearCanvasTombstones(canvasId: string): void { - const prefix = `${canvasId}\0`; - for (const key of this.#nodeTombstones) { - if (key.startsWith(prefix)) this.#nodeTombstones.delete(key); - } - } - - #nodeKey(canvasId: string, nodeId: string): string { - return `${canvasId}\0${nodeId}`; + return releaseGate; } } diff --git a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql index c50d52a0e..467674c0c 100644 --- a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql +++ b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql @@ -23,7 +23,7 @@ CREATE TABLE nodes ( canvas_id TEXT NOT NULL, node_id TEXT NOT NULL, record_json TEXT NOT NULL CHECK (json_valid(record_json)), - revision INTEGER NOT NULL CHECK (revision > 0), + revision TEXT NOT NULL CHECK (length(revision) > 0), label_collision_key TEXT NOT NULL, PRIMARY KEY (canvas_id, node_id), UNIQUE (canvas_id, label_collision_key), @@ -54,6 +54,14 @@ CREATE TABLE tasks ( FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE ) STRICT; +CREATE TABLE space_extensions ( + extension_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + namespace TEXT NOT NULL, + UNIQUE (canvas_id, namespace), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE +) STRICT; + CREATE TABLE delta_log ( canvas_id TEXT NOT NULL, version INTEGER NOT NULL, @@ -66,7 +74,7 @@ INSERT INTO spaces ( canvas_id, title, collision_key, version, state_json, created_at, updated_at, is_world ) VALUES ( - 'fixture-world', 'World', 'world', 0, + 'fixture-world', 'World', '.world', 0, '{"nodes":[],"edges":[]}', 1, 1, 1 ); @@ -84,7 +92,7 @@ INSERT INTO nodes ( ) VALUES ( 'fixture-space', 'fixture-node', '{"nodeId":"fixture-node","type":"note","label":"Fixture Node","content":"fixture body"}', - 7, 'fixture node' + 'fixture-revision', 'fixture node' ); INSERT INTO events (canvas_id, event_json) VALUES ( diff --git a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts index 43311516c..b1b2fff67 100644 --- a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts +++ b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts @@ -22,6 +22,7 @@ import type { DeltaLogEntry, NodeContent, } from '../../../canvas/persistence-types.js'; +import type { NodeSnapshot } from '../../ports/structured.js'; import type { TaskRecord } from '@huabu/shared'; import type { CanvasNode } from '@huabu/shared/canvas-engine'; @@ -99,6 +100,9 @@ describe('SqliteStructuredStore lifecycle and schema', () => { await expect( Promise.resolve().then(() => store.space('lifecycle-space').read()), ).rejects.toThrow(/not initialized/); + await expect( + store.space('lifecycle-space').nodes.readMany([]), + ).rejects.toThrow(/not initialized/); await expect(store.init()).resolves.toBeUndefined(); await expect(store.init()).resolves.toBeUndefined(); @@ -114,6 +118,9 @@ describe('SqliteStructuredStore lifecycle and schema', () => { await expect( Promise.resolve().then(() => store.space('lifecycle-space').read()), ).rejects.toThrow(/closed/); + await expect( + store.space('lifecycle-space').nodes.readMany([]), + ).rejects.toThrow(/closed/); await expect(store.init()).rejects.toThrow(/closed/); }); @@ -131,6 +138,7 @@ describe('SqliteStructuredStore lifecycle and schema', () => { 'delta_log', 'events', 'nodes', + 'space_extensions', 'spaces', 'tasks', ]; @@ -195,7 +203,7 @@ describe('SqliteStructuredStore lifecycle and schema', () => { }); await expect(space.nodes.read('fixture-node')).resolves.toEqual({ record: note('fixture-node', 'Fixture Node', 'fixture body'), - revision: '7', + revision: 'fixture-revision', }); await expect(space.events.read()).resolves.toEqual([ { @@ -294,7 +302,7 @@ describe('SqliteStructuredStore persistence and transactions', () => { ).resolves.toEqual(put.ok ? { record, revision: put.revision } : null); }); - it('rolls node, record, delta, and tombstone state back on a real trigger abort', async () => { + it('rolls node, record, and delta state back on a real trigger abort', async () => { const harness = await trackedOpenStore('huabu-sqlite-trigger-rollback-'); const canvasId = 'trigger-rollback-space'; const baseline = await createSpace( @@ -384,6 +392,56 @@ describe('SqliteStructuredStore persistence and transactions', () => { }); }); + it('recovers malformed stored Node content through every read shape', async () => { + const harness = await trackedOpenStore('huabu-sqlite-node-recovery-'); + const canvasId = 'node-recovery-space'; + await createSpace(harness.store, canvasId, 'Node Recovery Space'); + const nodes = harness.store.space(canvasId).nodes; + const record = note('recoverable-node', 'Recoverable Node', 'before'); + const baseline = await nodes.put({ nodeId: record.nodeId, record }); + if (!baseline.ok) throw new Error('Could not seed recoverable Node'); + + withTestDatabase(harness.filename, (database) => { + database + .prepare( + `UPDATE nodes + SET record_json = ? + WHERE canvas_id = ? AND node_id = ?`, + ) + .run('{"content":"recoverable body"}', canvasId, record.nodeId); + }); + + const recovered: NodeSnapshot = { + record: { + nodeId: record.nodeId, + type: 'note', + label: null, + content: 'recoverable body', + }, + revision: baseline.revision, + }; + await expect(nodes.read(record.nodeId)).resolves.toEqual(recovered); + await expect(nodes.readMany([record.nodeId])).resolves.toEqual( + new Map([[record.nodeId, recovered]]), + ); + await expect(nodes.list()).resolves.toEqual( + new Map([[record.nodeId, recovered]]), + ); + const delivered: NodeSnapshot[] = []; + await expect( + nodes.stream((snapshot) => delivered.push(snapshot)), + ).resolves.toEqual(new Map([[record.nodeId, recovered]])); + expect(delivered).toEqual([recovered]); + + await expect( + nodes.put({ + nodeId: record.nodeId, + expectedRevision: baseline.revision, + record: { ...record, content: 'repaired' }, + }), + ).resolves.toMatchObject({ ok: true }); + }); + it('releases deletion admission when post-acquire Space setup throws', async () => { const harness = await trackedOpenStore('huabu-sqlite-delete-setup-'); const canvasId = 'delete-setup-space'; @@ -533,7 +591,7 @@ describe('SqliteStructuredStore persistence and transactions', () => { await expect(handle.read()).resolves.toBeNull(); }); - it('does not create a tombstone when deleting an already absent node', async () => { + it('allows a first write after deleting an already absent node', async () => { const harness = await trackedOpenStore('huabu-sqlite-absent-delete-'); const canvasId = 'absent-delete-space'; await createSpace(harness.store, canvasId, 'Absent Delete Space'); @@ -546,25 +604,44 @@ describe('SqliteStructuredStore persistence and transactions', () => { ).resolves.toMatchObject({ ok: true, record }); }); - it('forgets a successful node deletion tombstone after close and reopen', async () => { - const harness = await trackedOpenStore('huabu-sqlite-tombstone-reopen-'); + it('allows immediate reuse of a deleted primary key across reopen', async () => { + const harness = await trackedOpenStore('huabu-sqlite-delete-reopen-'); const canvasId = 'tombstone-reopen-space'; await createSpace(harness.store, canvasId, 'Tombstone Reopen Space'); const record = note('tombstoned-node', 'Tombstoned Node', 'before'); const nodes = harness.store.space(canvasId).nodes; - await nodes.put({ nodeId: record.nodeId, record }); + const initial = await nodes.put({ nodeId: record.nodeId, record }); + if (!initial.ok) throw new Error('Could not create initial test Node'); await expect(nodes.delete(record.nodeId)).resolves.toBe('deleted'); + const recreated = await nodes.put({ + nodeId: record.nodeId, + record: { ...record, content: 'immediate replacement' }, + }); + if (!recreated.ok) throw new Error('Could not recreate test Node'); + expect(recreated.record).toEqual({ + ...record, + content: 'immediate replacement', + }); + expect(recreated.revision).not.toBe(initial.revision); await expect( nodes.put({ nodeId: record.nodeId, - record: { ...record, content: 'late stale write' }, + expectedRevision: initial.revision, + record: { ...record, content: 'stale replacement' }, }), - ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); + ).resolves.toEqual({ + ok: false, + reason: 'revision-conflict', + currentRevision: recreated.revision, + }); await harness.store.close(); const reopened = trackedStore(harness.filename); await reopened.init(); + await expect( + reopened.space(canvasId).nodes.delete(record.nodeId), + ).resolves.toBe('deleted'); await expect( reopened.space(canvasId).nodes.put({ nodeId: record.nodeId, diff --git a/apps/server/src/modules/storage/backends/sqlite/rows.ts b/apps/server/src/modules/storage/backends/sqlite/rows.ts index e949a350d..9cbe48f35 100644 --- a/apps/server/src/modules/storage/backends/sqlite/rows.ts +++ b/apps/server/src/modules/storage/backends/sqlite/rows.ts @@ -6,8 +6,9 @@ * * Every column this backend stores is either JSON text or a scalar, so the * codecs here are the single place that decides what a well-formed stored - * value looks like. Reads validate on the way out: a row that no longer - * matches the domain shape is a corruption report, not a silent default. + * value looks like. Space and log reads reject malformed domain values. Node + * reads preserve the port's repair path by recovering malformed JSON values + * into a valid record whose content still exposes the stored value. */ import { SQLITE_WORLD_COLLISION_KEY } from './database.js'; @@ -269,21 +270,30 @@ export function decodeNodeRecord( const parsed = parseJson(value, `Node ${JSON.stringify(expectedNodeId)}`); try { validateNodeContent(parsed as NodeContent, expectedNodeId); - } catch (error) { - throw new SyntaxError( - `Invalid persisted Node ${JSON.stringify(expectedNodeId)}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); + return parsed as NodeContent; + } catch { + // A valid JSON value can still have a damaged Node shape after an + // out-of-band database edit. Keep it reachable so a normal put can repair + // it, matching the lenient content rule used by the Disk adapter. + const fields = + typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + return { + ...fields, + nodeId: expectedNodeId, + type: typeof fields['type'] === 'string' ? fields['type'] : 'note', + label: typeof fields['label'] === 'string' ? fields['label'] : null, + content: + typeof fields['content'] === 'string' + ? fields['content'] + : stringifyJson(parsed, `Malformed Node ${expectedNodeId}`), + } as NodeContent; } - return parsed as NodeContent; } -export function requirePositiveRevision( - value: unknown, - nodeId: string, -): number { - if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { +export function requireRevision(value: unknown, nodeId: string): string { + if (typeof value !== 'string' || value.length === 0) { throw new SyntaxError( `Invalid persisted revision for Node ${JSON.stringify(nodeId)}`, ); diff --git a/apps/server/src/modules/storage/backends/sqlite/space-extension.ts b/apps/server/src/modules/storage/backends/sqlite/space-extension.ts new file mode 100644 index 000000000..51c868321 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/space-extension.ts @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** SQLite connection point for one extension namespace in one Space. */ + +import { withImmediateTransaction } from './database.js'; +import { assertValidNamespace } from '../../ports/namespace.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { SpaceHandle } from '../../ports/structured.js'; + +export function createSqliteSpaceExtension( + context: SqliteStoreContext, + canvasId: string, +): SpaceHandle['extension'] { + return async function extension(namespaceInput: string) { + const namespace = assertValidNamespace(namespaceInput); + context.assertMutationAllowed(canvasId); + const database = context.database(); + + return withImmediateTransaction(database, () => { + const exists = + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] === 1; + if (!exists) return null; + + database + .prepare( + `INSERT INTO space_extensions (canvas_id, namespace) + VALUES (?, ?) + ON CONFLICT(canvas_id, namespace) DO NOTHING`, + ) + .run(canvasId, namespace); + const extensionId = database + .prepare( + `SELECT extension_id + FROM space_extensions + WHERE canvas_id = ? AND namespace = ?`, + ) + .get(canvasId, namespace)?.['extension_id']; + if ( + typeof extensionId !== 'number' || + !Number.isSafeInteger(extensionId) || + extensionId <= 0 + ) { + throw new Error( + `Could not resolve SQLite extension ${JSON.stringify(namespace)}`, + ); + } + return Object.freeze({ + kind: 'sqlite' as const, + database, + extensionId, + }); + }); + }; +} diff --git a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts index e12afd180..acd17fd0e 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { randomUUID } from 'node:crypto'; + import { withImmediateTransaction } from './database.js'; import { allocateNodeIdentity } from './identity.js'; import { decodeNodeRecord, - requirePositiveRevision, + requireRevision, stringifyJson, validateNodeContent, } from './rows.js'; @@ -17,13 +19,14 @@ import type { NodePutInput, NodePutResult, NodeSnapshot, + NodeStreamOptions, SpaceNodes, } from '../../ports/structured.js'; import type { DatabaseSync } from 'node:sqlite'; interface NodeRow { readonly record: NodeSnapshot['record']; - readonly revision: number; + readonly revision: string; readonly collisionKey: string; } @@ -40,7 +43,7 @@ function decodeNodeRow(value: unknown, nodeId: string): NodeRow { } return { record: decodeNodeRecord(row['record_json'], nodeId), - revision: requirePositiveRevision(row['revision'], nodeId), + revision: requireRevision(row['revision'], nodeId), collisionKey, }; } @@ -81,28 +84,19 @@ function validatePut(input: NodePutInput): string { return nodeId; } -export interface SqliteNodePutOptions { - readonly tombstoned: boolean; - readonly bypassTombstone?: boolean; -} - /** Apply one node put inside the caller's active transaction. */ export function putSqliteNodeInTransaction( database: DatabaseSync, canvasId: string, input: NodePutInput, - options: SqliteNodePutOptions, ): NodePutResult { const nodeId = validatePut(input); - if (options.tombstoned && options.bypassTombstone !== true) { - return { ok: false, reason: 'write-suppressed' }; - } if (!spaceExists(database, canvasId)) { return { ok: false, reason: 'not-found' }; } const current = readNodeRow(database, canvasId, nodeId); - const currentRevision = current === null ? null : String(current.revision); + const currentRevision = current?.revision ?? null; if ( input.expectedRevision !== undefined && input.expectedRevision !== currentRevision @@ -164,10 +158,7 @@ export function putSqliteNodeInTransaction( } } - const revision = (current?.revision ?? 0) + 1; - if (!Number.isSafeInteger(revision)) { - throw new Error(`Node ${JSON.stringify(nodeId)} revision overflow`); - } + const revision = randomUUID(); database .prepare( `INSERT INTO nodes ( @@ -188,7 +179,7 @@ export function putSqliteNodeInTransaction( return { ok: true, record: allocation.record, - revision: String(revision), + revision, }; } @@ -211,20 +202,71 @@ export class SqliteSpaceNodes implements SpaceNodes { ); return current === null ? null - : { record: current.record, revision: String(current.revision) }; + : { record: current.record, revision: current.revision }; + } + + async readMany( + nodeIds: readonly string[], + ): Promise> { + const database = this.#context.database(); + const snapshots = new Map(); + for (const nodeIdInput of new Set(nodeIds)) { + const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + const row = readNodeRow(database, this.canvasId, nodeId); + if (row !== null) { + snapshots.set(nodeId, { + record: row.record, + revision: row.revision, + }); + } + } + return snapshots; + } + + async list(): Promise> { + const rows = this.#context + .database() + .prepare( + `SELECT node_id, record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ?`, + ) + .all(this.canvasId); + const snapshots = new Map(); + for (const value of rows) { + const nodeId = value['node_id']; + if (typeof nodeId !== 'string') { + throw new SyntaxError('Invalid node_id in persisted SQLite Node'); + } + const row = decodeNodeRow(value, nodeId); + snapshots.set(nodeId, { + record: row.record, + revision: row.revision, + }); + } + return snapshots; + } + + async stream( + onNode: (snapshot: NodeSnapshot) => void, + options?: NodeStreamOptions, + ): Promise> { + const snapshots = await this.list(); + const delivered = new Map(); + for (const [nodeId, snapshot] of snapshots) { + if (options?.signal?.aborted) break; + onNode(snapshot); + delivered.set(nodeId, snapshot); + } + return delivered; } async put(input: NodePutInput): Promise { - const nodeId = validatePut(input); + validatePut(input); this.#context.assertMutationAllowed(this.canvasId); - if (this.#context.isNodeTombstoned(this.canvasId, nodeId)) { - return { ok: false, reason: 'write-suppressed' }; - } const database = this.#context.database(); return withImmediateTransaction(database, () => - putSqliteNodeInTransaction(database, this.canvasId, input, { - tombstoned: false, - }), + putSqliteNodeInTransaction(database, this.canvasId, input), ); } @@ -232,9 +274,8 @@ export class SqliteSpaceNodes implements SpaceNodes { const nodeId = sanitizeId(nodeIdInput, 'nodeId'); this.#context.assertMutationAllowed(this.canvasId); const database = this.#context.database(); - const result = withImmediateTransaction(database, () => { - if (!spaceExists(database, this.canvasId)) - return 'missing-space' as const; + return withImmediateTransaction(database, () => { + if (!spaceExists(database, this.canvasId)) return 'absent' as const; const deleted = Number( database .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') @@ -242,10 +283,5 @@ export class SqliteSpaceNodes implements SpaceNodes { ); return deleted === 1 ? ('deleted' as const) : ('absent' as const); }); - if (result === 'missing-space') return 'absent'; - if (result === 'deleted') { - this.#context.setNodeTombstone(this.canvasId, nodeId, true); - } - return result; } } diff --git a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts index e12705cb4..898a52b42 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { randomUUID } from 'node:crypto'; + import { withImmediateTransaction } from './database.js'; import { allocateSpaceIdentity, collisionKeyForTitle } from './identity.js'; import { @@ -72,6 +74,43 @@ export class SqliteSpaceRepository implements SpaceRepository { return sanitizeId(world.record.canvasId, 'world canvasId'); } + async ensureWorld(): Promise { + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const existing = database + .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 1`) + .all(); + if (existing.length > 1) { + throw new Error('SQLite namespace has multiple World Spaces'); + } + if (existing.length === 1) { + const world = decodeSpaceRow(existing[0]); + if (!world.isWorld) throw new Error('SQLite World Space is malformed'); + return sanitizeId(world.record.canvasId, 'world canvasId'); + } + + const canvasId = randomUUID(); + const timestamp = this.#context.now(); + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Space clock returned a non-finite value'); + } + insertSpaceRow( + database, + { + canvasId, + title: 'World', + version: 0, + state: { nodes: [], edges: [] }, + createdAt: timestamp, + updatedAt: timestamp, + }, + '', + true, + ); + return canvasId; + }); + } + async create(input: SpaceCreateInput): Promise { const canvasId = sanitizeId(input.canvasId, 'canvasId'); validateTitle(input.title); @@ -152,10 +191,8 @@ export class SqliteSpaceRepository implements SpaceRepository { ); return { deleted: deleted === 1 }; }); - if (result.deleted) { - this.#context.clearCanvasTombstones(canvasId); + if (result.deleted) return { ok: true as const, reason: 'deleted' as const }; - } return { ok: false as const, reason: 'not-found' as const }; } finally { close(); diff --git a/apps/server/src/modules/storage/backends/sqlite/space-write.ts b/apps/server/src/modules/storage/backends/sqlite/space-write.ts index ad914619b..e97da4239 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-write.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-write.ts @@ -99,10 +99,7 @@ export function createSqliteSpaceWrite( const current = readSpaceRow(database, canvasId); if (current === null) { if (!input.allowCreate) { - return { - result: { ok: false, reason: 'not-found' } as const, - tombstones: new Map(), - }; + return { ok: false, reason: 'not-found' } as const; } if (input.expectedVersion !== 0) { throw new Error( @@ -124,21 +121,15 @@ export function createSqliteSpaceWrite( { ...input.nextRecord, title: identity.title }, identity.collisionKey, ); - return { - result: { ok: true } as const, - tombstones: new Map(), - }; + return { ok: true } as const; } if (current.record.version !== input.expectedVersion) { return { - result: { - ok: false, - reason: 'version-conflict', - actualVersion: current.record.version, - } as const, - tombstones: new Map(), - }; + ok: false, + reason: 'version-conflict', + actualVersion: current.record.version, + } as const; } if (input.nextRecord.createdAt !== current.record.createdAt) { throw new Error(`SpaceWrite(${canvasId}) refusing to change createdAt`); @@ -150,38 +141,20 @@ export function createSqliteSpaceWrite( ); } - const tombstones = new Map(); - const tombstoned = (nodeId: string): boolean => - tombstones.get(nodeId) ?? context.isNodeTombstoned(canvasId, nodeId); - for (const mutation of input.nodeMutations) { if (mutation.kind === 'delete') { - const deleted = Number( - database - .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') - .run(canvasId, mutation.nodeId).changes, - ); - if (deleted === 1) tombstones.set(mutation.nodeId, true); + database + .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') + .run(canvasId, mutation.nodeId); continue; } - const result = putSqliteNodeInTransaction( - database, - canvasId, - { - nodeId: mutation.nodeId, - record: mutation.record, - strictLabel: mutation.strictLabel, - }, - { - tombstoned: tombstoned(mutation.nodeId), - bypassTombstone: mutation.authoritativeInsert === true, - }, - ); + const result = putSqliteNodeInTransaction(database, canvasId, { + nodeId: mutation.nodeId, + record: mutation.record, + strictLabel: mutation.strictLabel, + }); if (!result.ok) throw mutationError(mutation, result); - if (mutation.authoritativeInsert === true) { - tombstones.set(mutation.nodeId, false); - } } if ( @@ -201,14 +174,8 @@ export function createSqliteSpaceWrite( stringifyJson(input.delta, `Space ${canvasId} delta`), ); } - return { result: { ok: true } as const, tombstones }; + return { ok: true } as const; }); - - if (completed.result.ok) { - for (const [nodeId, present] of completed.tombstones) { - context.setNodeTombstone(canvasId, nodeId, present); - } - } - return completed.result; + return completed; }; } diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts index f1d372b11..a48449880 100644 --- a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -3,6 +3,7 @@ import { SqliteStoreContext } from './database.js'; import { readSpaceRow } from './rows.js'; +import { createSqliteSpaceExtension } from './space-extension.js'; import { createSqliteSpaceLogs } from './space-logs.js'; import { SqliteSpaceNodes } from './space-nodes.js'; import { SqliteSpaceRepository } from './space-repository.js'; @@ -17,7 +18,7 @@ import type { StructuredStore, } from '../../ports/structured.js'; -/** Production structured-store adapter backed by one node:sqlite connection. */ +/** Isolated structured-store adapter backed by one node:sqlite connection. */ export class SqliteStructuredStore implements StructuredStore { readonly kind = 'sqlite' as const; @@ -63,6 +64,7 @@ export class SqliteStructuredStore implements StructuredStore { changes, tasks, events, + extension: createSqliteSpaceExtension(this.#context, canvasId), }); } } diff --git a/apps/server/src/modules/storage/backends/sqlite/test-support.ts b/apps/server/src/modules/storage/backends/sqlite/test-support.ts index bddebc8a8..4c5af7033 100644 --- a/apps/server/src/modules/storage/backends/sqlite/test-support.ts +++ b/apps/server/src/modules/storage/backends/sqlite/test-support.ts @@ -30,6 +30,11 @@ export interface OpenSqliteTestStore extends SqliteTestFile { readonly cleanup: () => Promise; } +export interface EmptySqliteTestStore extends SqliteTestFile { + readonly store: SqliteStructuredStore; + readonly cleanup: () => Promise; +} + export function createSqliteTestFile(prefix = 'huabu-sqlite-'): SqliteTestFile { const directory = mkdtempSync(path.join(tmpdir(), prefix)); const filename = path.join(directory, 'structured.sqlite'); @@ -123,6 +128,29 @@ export async function openSqliteTestStore( } } +export async function openEmptySqliteTestStore( + prefix = 'huabu-sqlite-empty-', + now?: () => number, +): Promise { + const file = createSqliteTestFile(prefix); + const store = new SqliteStructuredStore(file.filename, now); + try { + await store.init(); + return { + ...file, + store, + cleanup: async () => { + await store.close(); + file.remove(); + }, + }; + } catch (error) { + await store.close(); + file.remove(); + throw error; + } +} + export function readSqliteDeltaLog( filename: string, canvasId: string, diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index 527dd8b2c..127829078 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -58,7 +58,7 @@ describe('storage capability matrix', () => { expect(describeUnavailableCapabilities(DISK)).toEqual([]); }); - it('answers for a backend that has no adapter yet', () => { + it('answers for a backend whose adapter is not selectable yet', () => { const missing = unavailableCapabilities(TABLES); // Every entry is Disk-only today, so a structured backend that is not @@ -75,13 +75,12 @@ describe('storage capability matrix', () => { expect(hasStorageCapability(TABLES, 'something-portable')).toBe(true); }); - it('states a limitation without making it a misconfiguration', () => { - // A profile that merely offers fewer features must not fail validation — - // that is reserved for a backend that cannot serve at all. `sqlite` has - // no adapter yet, so it does fail; the distinction is which check - // rejects it. + it('reports capability gaps separately from profile selectability', () => { + // The matrix describes what SQLite lacks regardless of whether the + // preview can be selected. Validation rejects it at the separate + // production-readiness gate. expect(describeUnavailableCapabilities(TABLES).length).toBeGreaterThan(0); - expect(() => validateStorageProfile(TABLES)).toThrow(/not implemented/); + expect(() => validateStorageProfile(TABLES)).toThrow(/not selectable yet/); expect(() => validateStorageProfile(DISK)).not.toThrow(); }); diff --git a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts index 0b0056afc..f5649985a 100644 --- a/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/space-nodes.contract.ts @@ -6,21 +6,16 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { NodeContent } from '../../../canvas/persistence-types.js'; -import type { - NodePutInput, - SpaceHandle, - SpaceNodes, - NodeSnapshot, -} from '../structured.js'; +import type { NodePutInput, SpaceNodes, NodeSnapshot } from '../structured.js'; export interface SpaceNodesContractHarness { /** Repository for an existing Space, initially empty at contract-owned ids. */ readonly repository: SpaceNodes; - /** Handle that owns `repository`, used to exercise ordered reinsertion. */ - readonly space: SpaceHandle; /** Repository scoped to a Space whose structural record is absent. */ readonly missingRepository: SpaceNodes; readonly expectedCanvasId: string; + /** Whether this adapter fences a deleted id against late standalone puts. */ + readonly deletedNodePut: 'allowed' | 'write-suppressed'; readonly cleanup?: () => Promise | void; } @@ -343,70 +338,24 @@ export function describeSpaceNodesContract( await expect(repository.delete(nodeId)).resolves.toBe('absent'); }); - it('suppresses standalone resurrection until an authoritative ordered insert succeeds', async () => { - const { repository, space } = await open(); + it('reports the adapter-defined result for a standalone put after deletion', async () => { + const { repository, deletedNodePut } = await open(); const nodeId = 'contract-late-put'; const record = note(nodeId, 'Contract late put', 'before'); await putSuccessfully(repository, { nodeId, record }); await repository.delete(nodeId); - await expect( - repository.put({ - nodeId, - record: { ...record, content: 'late resurrection' }, - }), - ).resolves.toEqual({ ok: false, reason: 'write-suppressed' }); - await expect(repository.read(nodeId)).resolves.toBeNull(); - - const current = await space.read(); - if (current === null) - throw new Error('Contract fixture Space is missing'); - const authoritative = { - ...record, - content: 'authoritative resurrection', - }; - await expect( - space.write({ - expectedVersion: current.version, - nextRecord: { - ...current, - version: current.version + 1, - state: { - ...current.state, - nodes: [ - ...current.state.nodes, - { id: nodeId, type: authoritative.type }, - ], - }, - updatedAt: current.updatedAt + 1, - }, - nodeMutations: [ - { - kind: 'put', - nodeId, - record: authoritative, - authoritativeInsert: true, - }, - ], - }), - ).resolves.toEqual({ ok: true }); - - const restored = await repository.read(nodeId); - expect(restored).toMatchObject({ record: authoritative }); - if (restored === null) { - throw new Error('Authoritatively reinserted node is missing'); + const late = { ...record, content: 'late resurrection' }; + const result = await repository.put({ nodeId, record: late }); + if (deletedNodePut === 'write-suppressed') { + expect(result).toEqual({ ok: false, reason: 'write-suppressed' }); + await expect(repository.read(nodeId)).resolves.toBeNull(); + } else { + expect(result).toMatchObject({ ok: true, record: late }); + await expect(repository.read(nodeId)).resolves.toMatchObject({ + record: late, + }); } - - await expect( - repository.put({ - nodeId, - expectedRevision: restored.revision, - record: { ...authoritative, content: 'later standalone update' }, - }), - ).resolves.toMatchObject({ - ok: true, - record: { content: 'later standalone update' }, - }); }); }); } diff --git a/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts b/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts index 9ede9188f..42c08e6a9 100644 --- a/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts +++ b/apps/server/src/modules/storage/ports/contracts/structured-store.contract.ts @@ -67,6 +67,7 @@ export function describeStructuredStoreContract( for (const method of [ 'list', 'worldId', + 'ensureWorld', 'create', 'beginDelete', 'rename', diff --git a/apps/server/src/modules/storage/ports/structured.ts b/apps/server/src/modules/storage/ports/structured.ts index bed6678b2..ee69234a0 100644 --- a/apps/server/src/modules/storage/ports/structured.ts +++ b/apps/server/src/modules/storage/ports/structured.ts @@ -52,6 +52,7 @@ import type { TaskStoreSnapshot, } from '@huabu/shared'; import type { CanvasChangeRecord } from '@huabu/shared/canvas-engine'; +import type { DatabaseSync } from 'node:sqlite'; /** * Backends with a structured adapter today. @@ -299,14 +300,23 @@ export interface SpaceHandle { * * One member per backend that exists, like {@link StructuredBackendKind} and * for the same reason: a union that named `sqlite` today would advertise a - * substrate no adapter can supply. It grows with each adapter — a table prefix - * for SQLite, a schema for Postgres — and an owner switches on `kind`. + * substrate no adapter can supply. It grows with each adapter — a scoped + * connection and parent id for SQLite, a schema for Postgres — and an owner + * switches on `kind`. */ -export type SpaceSubstrate = { - readonly kind: 'disk'; - /** A directory reserved for this namespace, created and ready to write. */ - readonly directory: string; -}; +export type SpaceSubstrate = + | { + readonly kind: 'disk'; + /** A directory reserved for this namespace, created and ready to write. */ + readonly directory: string; + } + | { + readonly kind: 'sqlite'; + /** The adapter connection on which the owner creates its own tables. */ + readonly database: DatabaseSync; + /** Stable parent row for owner tables to reference with ON DELETE CASCADE. */ + readonly extensionId: number; + }; // ─── The ordered Space write ───────────────────────────────────────────────── @@ -325,17 +335,12 @@ export type SpaceNodeMutation = /** * Marks an executor-authoritative INSERT. * - * After {@link SpaceNodes.delete} removes an existing id, standalone - * puts for that id must return `write-suppressed` within the same running - * {@link StructuredStore}. A successful ordered put carrying this flag - * is the portable signal that the id is intentionally being reinserted; - * it admits the write and clears that suppression for later standalone - * puts. It is intentionally batch-only so a late direct write cannot - * claim authority for itself. - * - * This is an in-memory connection-lifetime guarantee, not restart - * durability. Closing or recreating the StructuredStore may discard the - * deletion fence. + * **Adapter-shaped**, like {@link NodePutResult}'s `write-suppressed`. + * It exists for a backend that suppresses writes to a recently deleted + * id, and lets such an adapter distinguish a real re-insertion from a + * late direct write that should stay suppressed. It is intentionally + * batch-only. An adapter whose deletes are immediately final — a SQL + * table with a unique key — can ignore it. */ readonly authoritativeInsert?: boolean; } @@ -534,19 +539,19 @@ export type NodeDeleteResult = 'deleted' | 'absent'; * {@link SpaceNodes.readMany}, {@link SpaceNodes.list}, and * {@link SpaceNodes.stream}. * - * One mutation outcome is **adapter-shaped** and optional: + * Two mutation outcomes are **adapter-shaped** and optional: * * - `duplicate-node`, for adapters that can observe conflicting physical * representations of one stable id. Such an adapter may return one readable * representative from `read` so a caller can construct the attempted * update, but it must refuse the `put` rather than overwrite an arbitrary * representation. + * - `write-suppressed`, for adapters that keep a deleted id fenced against + * late in-flight writes. See {@link SpaceNodeMutation}'s + * `authoritativeInsert`, which is how a batch re-insertion is distinguished + * from such a late write. * - * `write-suppressed` is portable anti-resurrection behavior. After a - * successful delete of an existing node, standalone puts for that id are - * suppressed for the lifetime of the running {@link StructuredStore} until a - * successful ordered put marks the id as an `authoritativeInsert`. The fence - * need not survive closing or recreating the store. + * A SQL adapter with a unique key produces neither. */ export interface SpaceNodes { /** diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index 75fadddef..e045c72cf 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -37,9 +37,9 @@ const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ /** * Backends whose complete capability matrix is safe for production use. * - * SQLite deliberately stays out while physical Disk reads, World bootstrap, - * Blob placement, import/export, and Workspace remounting still have one - * authority only in the Disk profile. + * SQLite deliberately stays out while product composition, Blob placement, + * Disk-only capabilities, and Workspace remounting still have one authority + * only in the Disk profile. */ const SELECTABLE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; const IMPLEMENTED_BLOBS: readonly BlobBackendKind[] = ['disk']; @@ -94,10 +94,11 @@ export function parseStorageProfile( /** * Reject profiles that cannot serve correctly, before any connection opens. * - * Today that means "named but not implemented". This is also where - * cross-axis rules belong as backends land — for example, Postgres paired - * with a node-local disk blob root is unsafe across replicas unless the - * path is a deliberately shared filesystem. + * Today that means either "named but not implemented" or "implemented only as + * an isolated preview". This is also where cross-axis rules belong as + * backends land — for example, Postgres paired with a node-local disk blob + * root is unsafe across replicas unless the path is a deliberately shared + * filesystem. * * A profile that merely offers *fewer features* is not rejected here. Those * are stated limitations rather than misconfigurations, and they are declared diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 277770d22..159e967cc 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -1,7 +1,7 @@ # Multi-Backend Storage -Status: Phases 1–4.5 and §§12.6–12.8 implemented -Last updated: 2026-08-24 +Status: Phases 1–5 implemented; SQLite remains a contract preview +Last updated: 2026-09-04 > **Scope and decision confidence.** This proposal records the two-port > `StructuredStore` / `BlobStore` split and their target backend families as @@ -48,8 +48,7 @@ Last updated: 2026-08-24 > review are recorded in place, including the CAS race ordering (§12.2.5), > log-family interface segregation (§12.2.6), and retained-handle Workspace > guards (§12.2.4). Remaining Disk-only read and physical capabilities still -> keep non-Disk profiles unselectable. No SQLite, Postgres, or Azure adapter -> exists. +> keep non-Disk profiles unselectable. > > Phase 4.5 moved storage-owned Disk layout behind the storage boundary in > PR #93. What remains between the portable contracts and a second structured @@ -59,6 +58,11 @@ Last updated: 2026-08-24 > **implemented**), and §12.8 (the dispositions and the product-level > harness, **implemented**). §12 is the authoritative plan; > the decision table in §2 marks what each step has actually settled. +> +> Phase 5 is specified in §12.9 and is **implemented by this branch** as an +> isolated SQLite structured-store preview. It exercises the portable +> contracts with real SQLite files but is deliberately absent from runtime +> composition; Postgres and Azure adapters do not exist. --- @@ -1817,11 +1821,12 @@ put outcome remain in the portable shapes. At this phase boundary, both existed for Disk's in-memory deletion fence and a second adapter was still needed to establish their shared meaning. -**Superseded by Phase 5:** the SQLite contract preview supplies that second -adapter and confirms the portable rule as a connection-lifetime -anti-resurrection fence: after deletion, standalone puts are suppressed until -an ordered authoritative insert commits (§12.6.2). The outcome is therefore -no longer merely adapter-shaped. +**Resolved by Phase 5:** the SQLite contract preview supplies that second +adapter and confirms that these outcomes are adapter-shaped. Disk keeps its +process-local anti-resurrection fence and uses `authoritativeInsert` to lift +it. SQLite deletion is final at transaction commit, permits immediate reuse of +the primary key, and issues a fresh opaque revision so a token from the +deleted row cannot win a later compare-and-swap (§12.9.2). The review also asked composition to move default-title allocation ("Untitled", "Untitled (1)", …) into `create`, which would have removed the @@ -2488,14 +2493,15 @@ temporary Workspace through the production lifecycle — prepared Workspace, opened connections, `ensureWorld()` — rather than swapping in a stub. A stub proves the application talks to an interface; only a real backend proves one serves the product, which is the half that decides whether a second adapter -works. `product-boundary.test.ts` runs the criterion against every profile in -`PRODUCT_STORAGE_PROFILES`, naming no directory, filename, or `space.json`; -Phase 5 adds one entry to that list and the same behaviours are covered for -SQLite. A guard reads the suite's own source and rejects a directory, a +works. `product-boundary.test.ts` runs the criterion against every selectable +profile in `PRODUCT_STORAGE_PROFILES`, naming no directory, filename, or +`space.json`. A guard reads the suite's own source and rejects a directory, a filename, or a `readFileSync` appearing in it, because the failure mode here is a helpful-looking assertion someone adds later. The records the suite reads back are built through the write engine, because a fixture that skips the -engine asserts nothing about what the product actually stores. +engine asserts nothing about what the product actually stores. The isolated +Phase 5 adapter runs the lower-level portable contracts; it does not enter +this product-profile harness until it becomes selectable. `closeStorage()` arrives with it, registered on graceful Server shutdown and used by the harness between profiles. On Disk it is close to a no-op — which @@ -2513,15 +2519,81 @@ bundle export, external-note claim), RFS's sidecar-to-record mapping (**B**, deferred until a second backend has a file plane at all), and the ACP session path that leaves with the Agenetes `Namespace` change. -Out of scope, unchanged: a SQLite adapter or schema, Disk→SQLite data -migration, SQLite profile registration, Postgres/Azure, the portable +Out of scope, unchanged: SQLite runtime composition and profile selection, +Disk→SQLite data migration, Postgres/Azure, the portable change-notification capability, RFS's backend-neutral path vocabulary, ACP session relocation, the rest of the Agenetes persistence migration, the portable export format, a writable general-purpose virtual filesystem or OS mount, protocol or UI changes, and stronger crash/distributed transaction guarantees. -### 12.9 Later phases — provisional +### 12.9 Phase 5 — SQLite contract preview — **implemented** + +Phase 5 adds one non-Disk structured adapter to test whether the boundary +survives a database implementation. It is an isolated implementation and test +target, not a product profile. The composition root does not construct or +export it, and `HUABU_STRUCTURED_BACKEND=sqlite` continues to fail during +profile validation with a preview-specific diagnostic. + +#### 12.9.1 Scope and lifecycle + +- The adapter uses built-in `node:sqlite`, owns one explicit database filename + and connection, and adds no package or native-addon dependency. +- Retained handles stay bound to that connection. `init`, `health`, and + `close` are real lifecycle operations; Workspace remounting and production + factory registration remain selectability work. +- The current portable surface is implemented: Space listing/lifecycle and + `ensureWorld`, record read/write, node read/readMany/list/stream and + mutations, events, changes, Tasks/Runs including atomic completion, and the + extension substrate. +- Postgres, Azure Blob, Disk-to-SQLite migration, RFS/file tools, external-note + watching, import/export, client/API changes, and product UI remain outside + this phase. + +#### 12.9.2 Schema and behavior + +Schema versioning uses `PRAGMA user_version`; migrations run transactionally, +reject databases from the future, and create `STRICT` tables with foreign keys +enabled. Version 1 stores Space records and World membership, complete node +JSON with opaque revision tokens, ordered events, coalesced changes, +Task/Run snapshots, extension namespaces, and the private delta journal. + +Every ordered Space write applies node mutations, record replacement, and the +optional delta insert in one immediate transaction. Same-baseline writers have +one winner. Space deletion uses the shared process-local admission coordinator +under a database-specific scope: reads remain available, mutations reject, +concurrent deletion sessions queue, and `finish()` removes all owned rows by +foreign-key cascade. No SQL transaction remains open across blob cleanup, and +no multi-process deletion fence is promised. + +SQLite does not emulate Disk's node tombstones. A committed delete immediately +frees the `(canvas_id, node_id)` key. Every successful put receives a new UUID +revision token, including a delete/recreate cycle, so a stale token from the +old row cannot match the replacement. `write-suppressed` and +`authoritativeInsert` remain valid adapter-specific parts of the common shape +for Disk rather than requirements every SQL adapter must reproduce. + +The extension substrate returns the shared connection plus a stable, +Space-owned namespace id. Owner tables can reference that id with +`ON DELETE CASCADE`, preserving namespace isolation and cleanup without +putting a generic key/value API in the storage port. + +#### 12.9.3 Proof + +The reusable structured contracts run against Disk and real temporary SQLite +files. They cover fresh World bootstrap, store lifecycle, Space deletion +admission, node CAS and read shapes, ordered transactional writes, event/change +ordering, Task/Run completion, and extension isolation and cleanup. SQLite +integration tests additionally cover strict schema creation, close/reopen +persistence, an immutable v1 fixture, future-version rejection, migration +rollback, SQL fault injection, foreign-key cascades, and revision safety across +delete/recreate. + +The preview changes only the storage implementation, contracts, focused type +narrowing for the new substrate union, and this documentation. Runtime +composition and product capability owners remain unchanged. + +### 12.10 Later phases — provisional 6. Migrate the currently synchronous Agenetes persistence ports without changing their persist-before-notify, sequence, and fencing semantics. @@ -2742,7 +2814,7 @@ Before a new backend is production-ready: | File/dir | Responsibility | | -------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–5 tree (§§12.1–12.6), guarded by `module-boundaries.test.ts`. | +| [`apps/server/src/modules/storage/`](../../apps/server/src/modules/storage/) | Ports, composition, adapters, compatibility, tests, and three forwarding shims — the canonical Phase-1–5 tree (§§12.1–12.9), guarded by `module-boundaries.test.ts`. | | [`apps/server/src/modules/storage/ports/`](../../apps/server/src/modules/storage/ports/) | The two ports; reusable suites live in `ports/contracts/`. `blob.ts` is normative (§7.1); `structured.ts` owns the Space collection and the per-Space handle: record read/write, nodes, changes, Tasks, and history. | | [`apps/server/src/modules/storage/storage.ts`](../../apps/server/src/modules/storage/storage.ts) | Composition root: maps profiles to adapters, guards blob puts, and holds a lifecycle deletion session across the blob-first cleanup saga. | | [`.../storage/backends/disk/legacy/canvas-store-cache.ts`](../../apps/server/src/modules/storage/backends/disk/legacy/canvas-store-cache.ts) | Bounded LRU of legacy Disk Space objects. The single owner both the adapter and the facade resolve through, and the real limit of `space(id)` identity (§12.2.4). | From 30972f41d56740881c1359f625f21a0036ab4e1f Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Fri, 4 Sep 2026 19:28:35 +0800 Subject: [PATCH 06/15] feat(storage): make SQLite a profile the app can run on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 5 shipped an isolated SQLite adapter that proved the ports survive a database. It could not be selected, so nothing proved the harder claim behind it: that a deployment can run with no Workspace folder and no Space directories at all. `HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` is now a real profile. Everything durable — Workspaces, Spaces, nodes, events, changes, Tasks, blob bytes, extension namespaces, and agent conversations — lives in one file under `/storage/sqlite/`. What that took, and what each piece is: - A Workspace becomes a row. `spaces` carries a `workspace_id`, one connection serves every Workspace, and activation re-points the namespace rather than reopening anything — the "backend selection scope" decision the proposal settled but nothing had implemented. Handles bind the Workspace they were resolved in and refuse afterwards, as the Disk adapters already do with a retained path. `remove()` is a forget, not a delete: Disk honours the port's wording for free because the folder outlives the registry entry, and a database has to say so with a column. - A SQLite `BlobStore`, so bytes stop needing a folder. `put` buffers and then replaces one row, which gets the contract's replacement atomicity for free; `materialize()` spools to the OS temp directory and unlinks on release, which is the behaviour `BlobLease`'s post-release rule exists to keep honest. `blobs` deliberately has no foreign key to `spaces` — deletion order is the composition saga's, and that saga must also sweep orphans for a record that is already gone. - The Server stops assuming a Workspace is a place. `getWorkspaceKey()` is the identity leases and admission gates actually needed; `getWorkspacePath()` still refuses, for the callers that genuinely want a directory. World identity moves onto the composition root, so `isWorldCanvasId` answers for whichever backend is configured instead of scanning a directory index, and the World portal rules take the live Space ids as an argument rather than reading them off disk. - An agent conversation follows its Space. Agenetes takes its three storage ports at mount, so the mounted stores dispatch per namespace: a namespace with a `storage.root` keeps the file stores that wrote what is already there, a Space in SQLite gets tables that cascade from its extension namespace, and an unnamed namespace stays in memory as Agenetes intends. Because those ports are synchronous and `extension()` is not, the composition root also exposes `sqliteTree` — the synchronous form of the same resolution, named for the backend that has it and `null` elsewhere, with its own single-consumer census beside `diskTree`'s. - Four more capability rows, because the honest answer to a filesystem-shaped feature is still absence: Workspace folder selection, RFS's file plane, the Workspace memory document, and user-authored skills. Each refuses at its own call site in the words the startup log used. The `builtin-file-tools` rationale is corrected — it claimed RFS as the fallback, and RFS turns out to need the same directory. Two adapter defects found while testing the preview against Disk: - SQLite rejected an `undefined` own property that Disk drops, because Disk persists through `JSON.stringify`. A record writable on one backend and not the other is §13's silent-divergence risk in its most ordinary form — an optional field spread onto a node. The encoder now follows `JSON.stringify`; cycles and non-finite numbers still reject. - `stream()` materialized the whole collection before delivering the first node, which is the opposite of the latency shape the port describes, and an aborted scan had already paid for every row. It reads off a cursor now. `readMany` went from one statement per id to one per chunk. `PRODUCT_STORAGE_PROFILES` gains `sqlite/sqlite`, so the whole product suite runs against it unchanged — that was the point of writing it that way — plus a new case that everything is still there after a restart. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Gm37mhSirkohJJcnLR4SWs --- .../agent/agenetes/conversation-stores.ts | 132 ++++++ .../src/modules/agent/agenetes/drivers.ts | 20 +- .../agent/agenetes/sqlite-stores.test.ts | 218 +++++++++ .../modules/agent/agenetes/sqlite-stores.ts | 427 ++++++++++++++++++ .../agent/conversation/prompt/debug-prompt.ts | 19 +- .../src/modules/agent/memory/analyzer.test.ts | 1 + .../src/modules/agent/memory/analyzer.ts | 12 +- apps/server/src/modules/agent/memory/read.ts | 9 +- .../src/modules/agent/memory/trigger.ts | 23 +- .../src/modules/agent/substrate-store.ts | 137 ++++++ .../modules/agent/tools/handlers/fs-write.ts | 27 +- .../agent/tools/world-target-read.test.ts | 1 + .../canvas/canvas-command-router.test.ts | 29 +- .../src/modules/canvas/canvas-executor.ts | 19 +- .../server/src/modules/canvas/canvas.route.ts | 6 +- .../modules/canvas/external-watcher.test.ts | 22 +- .../src/modules/canvas/external-watcher.ts | 18 +- .../src/modules/canvas/world-portal-policy.ts | 33 +- .../src/modules/canvas/world-portals.test.ts | 42 +- .../canvas/world-reference-resolver.test.ts | 1 + .../interactive-view.service.ts | 27 +- .../server/src/modules/remote_fs/rfs.route.ts | 17 + .../backends/disk/storage-recovery.test.ts | 1 + .../storage/backends/sqlite/blob-store.ts | 339 ++++++++++++++ .../storage/backends/sqlite/contracts.test.ts | 48 +- .../storage/backends/sqlite/database.ts | 191 ++++---- .../storage/backends/sqlite/fixtures/v1.sql | 56 ++- .../backends/sqlite/integration.test.ts | 338 +++++++++++++- .../modules/storage/backends/sqlite/rows.ts | 76 +++- .../modules/storage/backends/sqlite/schema.ts | 140 ++++++ .../backends/sqlite/space-extension.ts | 120 +++-- .../storage/backends/sqlite/space-logs.ts | 46 +- .../storage/backends/sqlite/space-nodes.ts | 131 ++++-- .../backends/sqlite/space-repository.ts | 112 ++++- .../storage/backends/sqlite/space-tasks.ts | 25 +- .../storage/backends/sqlite/space-write.ts | 38 +- .../backends/sqlite/structured-store.ts | 101 ++++- .../storage/backends/sqlite/test-support.ts | 73 +-- .../backends/sqlite/workspace-repository.ts | 222 +++++++++ .../src/modules/storage/capabilities.test.ts | 23 +- .../src/modules/storage/capabilities.ts | 43 +- .../compatibility/delete-canvas.test.ts | 1 + apps/server/src/modules/storage/index.ts | 20 +- .../modules/storage/module-boundaries.test.ts | 26 +- apps/server/src/modules/storage/ports/blob.ts | 9 +- .../modules/storage/product-boundary.test.ts | 37 ++ .../src/modules/storage/profile.test.ts | 31 +- apps/server/src/modules/storage/profile.ts | 50 +- apps/server/src/modules/storage/storage.ts | 268 +++++++++-- apps/server/src/modules/storage/testing.ts | 52 ++- .../src/modules/workspace.route.test.ts | 1 + apps/server/src/modules/workspace.route.ts | 37 +- apps/server/src/modules/workspace.ts | 72 ++- apps/server/src/modules/workspace/paths.ts | 43 +- .../src/modules/workspaces.route.test.ts | 8 + apps/server/src/modules/workspaces.route.ts | 53 ++- 56 files changed, 3589 insertions(+), 482 deletions(-) create mode 100644 apps/server/src/modules/agent/agenetes/conversation-stores.ts create mode 100644 apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts create mode 100644 apps/server/src/modules/agent/agenetes/sqlite-stores.ts create mode 100644 apps/server/src/modules/agent/substrate-store.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/blob-store.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/schema.ts create mode 100644 apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts diff --git a/apps/server/src/modules/agent/agenetes/conversation-stores.ts b/apps/server/src/modules/agent/agenetes/conversation-stores.ts new file mode 100644 index 000000000..2c235ed4d --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/conversation-stores.ts @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Which Agenetes conversation stores this deployment runs on. + * + * Agenetes takes its three storage ports at mount, once, while the storage + * profile is only known at runtime and the active Workspace can change under + * a running process. So the mounted stores are dispatchers: each call picks + * the implementation that suits the namespace it was handed. + * + * The choice is made per namespace rather than per process because that is + * where the answer actually lives. A namespace carries a `storage.root` when + * the Space it belongs to is a directory, and does not when it is rows — the + * same fact the Space facade reports, arriving here through Agenetes's own + * vocabulary. + * + * The in-memory fall-through is not a backend choice. It is what an *unnamed* + * namespace has always got: a conversation with no Space to belong to, which + * Agenetes explicitly treats as non-persistent. + */ + +import { + FileEventLogStore, + FileThreadStore, + FileTurnStore, + InMemoryEventLogStore, + InMemoryThreadStore, + InMemoryTurnStore, +} from '@agenetes/agenetes'; + +import { + conversationTables, + SqliteEventLogStore, + SqliteThreadStore, + SqliteTurnStore, +} from './sqlite-stores.js'; + +import type { + EventLogEntry, + EventLogRecord, + EventLogStore, + PersistedTurn, + ThreadRecord, + ThreadStore, + TurnStartLogEntry, + TurnStore, +} from '@agenetes/agenetes'; +import type { AgentSubmission, Namespace } from '@agenetes/protocol'; + +interface Backing { + readonly threads: ThreadStore; + readonly events: EventLogStore; + readonly turns: TurnStore; +} + +const file: Backing = { + threads: new FileThreadStore(), + events: new FileEventLogStore(), + turns: new FileTurnStore(), +}; + +const sqlite: Backing = { + threads: new SqliteThreadStore(), + events: new SqliteEventLogStore(), + turns: new SqliteTurnStore(), +}; + +/** + * Shared, so an unnamed namespace keeps one conversation for the life of the + * process instead of a fresh empty one per port. + */ +const memory: Backing = { + threads: new InMemoryThreadStore(), + events: new InMemoryEventLogStore(), + turns: new InMemoryTurnStore(), +}; + +/** The stores that own this namespace's durable conversation state. */ +function backingFor(namespace: Namespace): Backing { + // A directory to write into settles it: that is the Disk profile, and the + // file stores are what wrote whatever is already there. + if (namespace.storage?.root) return file; + if (namespace.name && conversationTables(namespace) !== null) return sqlite; + return memory; +} + +export const conversationThreadStore: ThreadStore = { + upsert: (namespace, threadId, record: ThreadRecord) => + backingFor(namespace).threads.upsert(namespace, threadId, record), + get: (namespace, threadId) => + backingFor(namespace).threads.get(namespace, threadId), + list: (namespace) => backingFor(namespace).threads.list(namespace), + delete: (namespace, threadId) => + backingFor(namespace).threads.delete(namespace, threadId), +}; + +export const conversationEventLogStore: EventLogStore = { + appendTurnStart: ( + namespace, + threadId, + request: AgentSubmission | null, + ): TurnStartLogEntry => + backingFor(namespace).events.appendTurnStart(namespace, threadId, request), + append: (namespace, threadId, event): EventLogEntry => + backingFor(namespace).events.append(namespace, threadId, event), + read: (namespace, threadId, sinceSeq) => + backingFor(namespace).events.read(namespace, threadId, sinceSeq), + readRecords: (namespace, threadId, sinceSeq) => + backingFor(namespace).events.readRecords(namespace, threadId, sinceSeq), + maxSeq: (namespace, threadId) => + backingFor(namespace).events.maxSeq(namespace, threadId), + replace: (namespace, threadId, records: readonly EventLogRecord[]) => + backingFor(namespace).events.replace(namespace, threadId, records), + delete: (namespace, threadId) => + backingFor(namespace).events.delete(namespace, threadId), +}; + +export const conversationTurnStore: TurnStore = { + append: (namespace, threadId, persisted: PersistedTurn) => + backingFor(namespace).turns.append(namespace, threadId, persisted), + list: (namespace, threadId) => + backingFor(namespace).turns.list(namespace, threadId), + count: (namespace, threadId) => + backingFor(namespace).turns.count(namespace, threadId), + fence: (namespace, threadId) => + backingFor(namespace).turns.fence(namespace, threadId), + replace: (namespace, threadId, persisted: readonly PersistedTurn[]) => + backingFor(namespace).turns.replace(namespace, threadId, persisted), + delete: (namespace, threadId) => + backingFor(namespace).turns.delete(namespace, threadId), +}; diff --git a/apps/server/src/modules/agent/agenetes/drivers.ts b/apps/server/src/modules/agent/agenetes/drivers.ts index e514dffcc..6292ae0fa 100644 --- a/apps/server/src/modules/agent/agenetes/drivers.ts +++ b/apps/server/src/modules/agent/agenetes/drivers.ts @@ -7,15 +7,15 @@ import { type AcpCreateSpec, type AcpTurnCtx, } from '@agenetes/acp-driver'; -import { - FileEventLogStore, - FileThreadStore, - FileTurnStore, - mountAgenetes, -} from '@agenetes/agenetes'; +import { mountAgenetes } from '@agenetes/agenetes'; import { getAgentTeamRegistry } from '@agenetes/agentlet-host'; import { piDriverFactory, type PiTurnCtx } from '@agenetes/pi-driver'; +import { + conversationEventLogStore, + conversationThreadStore, + conversationTurnStore, +} from './conversation-stores.js'; import { type AgentHandle } from './handle.js'; import { HISTORY_LOAD_SANITY_LIMIT } from './history-replay.js'; import { huabuPiDriverPorts } from './pi-driver.js'; @@ -62,9 +62,11 @@ export const agenetes: Agenetes = mountAgenetes({ [INTERNAL_DRIVER_KIND]: piDriverFactory({ ports: huabuPiDriverPorts }), [EXTERNAL_DRIVER_KIND]: externalDriver, }, - threadStore: new FileThreadStore(), - eventLogStore: new FileEventLogStore(), - turnStore: new FileTurnStore(), + // Dispatchers, not one backing: which store owns a conversation depends on + // where its Space lives, and that is a runtime fact (`conversation-stores`). + threadStore: conversationThreadStore, + eventLogStore: conversationEventLogStore, + turnStore: conversationTurnStore, // Corruption guard, not a context budget: replay restores whatever the // live handle would still be holding, and trimming that is the // conversation's problem, not recovery's. diff --git a/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts new file mode 100644 index 000000000..f8e593311 --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The Agenetes conversation stores against a real SQLite profile. + * + * The claim under test is the one a user would notice: a conversation held in + * a Space that has no directory survives a restart, and goes away with its + * Space. Everything is driven through the mounted profile rather than a stub, + * so a broken extension substrate or a missing cascade fails here. + */ + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + conversationEventLogStore, + conversationThreadStore, + conversationTurnStore, +} from './conversation-stores.js'; +import { deleteSpace } from '../../storage/index.js'; +import { + mountTestWorkspace, + type MountedTestStorage, +} from '../../storage/testing.js'; +import { canvasAcpNamespace } from '../../workspace/paths.js'; + +import type { StorageProfile } from '../../storage/profile.js'; +import type { ThreadRecord } from '@agenetes/agenetes'; +import type { AgentStateSnapshot, WorkloadSpec } from '@agenetes/protocol'; + +const SQLITE: StorageProfile = { + structured: { kind: 'sqlite' }, + blobs: { kind: 'sqlite' }, +}; + +const CANVAS_ID = 'canvas-conversation'; +const THREAD_ID = 'thread-1'; + +let mounted: MountedTestStorage | null = null; + +afterEach(async () => { + await mounted?.close(); + mounted = null; +}); + +/** Open the profile and create the Space the conversation belongs to. */ +async function openWithSpace(): Promise { + const opened = await mountTestWorkspace(SQLITE, 'huabu-agenetes-sqlite-'); + mounted = opened; + const created = await opened.storage.structured + .spaces() + .create({ canvasId: CANVAS_ID, title: 'Conversation Space' }); + if (!created.ok) throw new Error('Expected to create the Space'); + return opened; +} + +function threadRecord(threadId = THREAD_ID): ThreadRecord { + return { + driverSchemaVersion: 1, + spec: { + kind: 'internal', + threadId, + namespace: { name: CANVAS_ID }, + } as unknown as WorkloadSpec, + state: { status: 'idle' } as unknown as AgentStateSnapshot, + }; +} + +describe('Agenetes conversation stores on SQLite', () => { + it('keeps a Space with no directory out of the file stores', async () => { + await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + + // The absence of `storage.root` is the whole signal: it is what tells the + // dispatcher this Space is not a folder. + expect(namespace.storage).toBeUndefined(); + expect(namespace.name).toBe(CANVAS_ID); + }); + + it('round-trips threads, events, and folded turns', async () => { + await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + + conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + expect(conversationThreadStore.get(namespace, THREAD_ID)).toEqual( + threadRecord(), + ); + expect(conversationThreadStore.list(namespace)).toHaveLength(1); + + const start = conversationEventLogStore.appendTurnStart( + namespace, + THREAD_ID, + null, + ); + expect(start).toMatchObject({ seq: 1, kind: 'turn_start', request: null }); + const appended = conversationEventLogStore.append(namespace, THREAD_ID, { + type: 'text', + text: 'hello', + } as never); + expect(appended.seq).toBe(2); + expect(conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe(2); + + // `read` is the streamed frames only; `readRecords` includes the internal + // turn boundary. + expect(conversationEventLogStore.read(namespace, THREAD_ID)).toHaveLength( + 1, + ); + expect( + conversationEventLogStore.readRecords(namespace, THREAD_ID), + ).toHaveLength(2); + expect( + conversationEventLogStore.read(namespace, THREAD_ID, 2), + ).toHaveLength(0); + + conversationTurnStore.append(namespace, THREAD_ID, { + turn: { id: 'turn-1' } as never, + seqStart: 1, + seqEnd: 2, + }); + expect(conversationTurnStore.count(namespace, THREAD_ID)).toBe(1); + expect(conversationTurnStore.fence(namespace, THREAD_ID)).toBe(2); + expect(conversationTurnStore.list(namespace, THREAD_ID)).toEqual([ + { turn: { id: 'turn-1' }, seqStart: 1, seqEnd: 2 }, + ]); + }); + + it('isolates one Space from another', async () => { + const opened = await openWithSpace(); + const other = 'canvas-conversation-other'; + const created = await opened.storage.structured + .spaces() + .create({ canvasId: other, title: 'Other Space' }); + if (!created.ok) throw new Error('Expected to create the second Space'); + + conversationThreadStore.upsert( + canvasAcpNamespace(CANVAS_ID), + THREAD_ID, + threadRecord(), + ); + + expect( + conversationThreadStore.get(canvasAcpNamespace(other), THREAD_ID), + ).toBeUndefined(); + expect(conversationThreadStore.list(canvasAcpNamespace(other))).toEqual([]); + }); + + it('destroys a conversation with the Space that held it', async () => { + await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + conversationEventLogStore.append(namespace, THREAD_ID, { + type: 'text', + text: 'hello', + } as never); + conversationTurnStore.append(namespace, THREAD_ID, { + turn: { id: 'turn-1' } as never, + seqStart: 1, + seqEnd: 1, + }); + + await expect(deleteSpace(CANVAS_ID)).resolves.toEqual({ + ok: true, + reason: 'deleted', + }); + + // The Space is gone, so there is no substrate to answer from — which is + // the port's rule, and is also what the foreign-key cascade leaves behind. + expect(conversationThreadStore.get(namespace, THREAD_ID)).toBeUndefined(); + expect(conversationEventLogStore.maxSeq(namespace, THREAD_ID)).toBe(0); + expect(conversationTurnStore.count(namespace, THREAD_ID)).toBe(0); + }); + + it('survives a restart', async () => { + const opened = await openWithSpace(); + const namespace = canvasAcpNamespace(CANVAS_ID); + conversationThreadStore.upsert(namespace, THREAD_ID, threadRecord()); + conversationEventLogStore.appendTurnStart(namespace, THREAD_ID, null); + conversationEventLogStore.append(namespace, THREAD_ID, { + type: 'text', + text: 'hello', + } as never); + conversationTurnStore.append(namespace, THREAD_ID, { + turn: { id: 'turn-1' } as never, + seqStart: 1, + seqEnd: 2, + }); + + await opened.reopen(); + + // Same namespace, new connection: this is the whole reason these stores + // exist rather than the in-memory defaults. + const after = canvasAcpNamespace(CANVAS_ID); + expect(conversationThreadStore.get(after, THREAD_ID)).toEqual( + threadRecord(), + ); + expect(conversationEventLogStore.maxSeq(after, THREAD_ID)).toBe(2); + expect( + conversationEventLogStore.readRecords(after, THREAD_ID), + ).toHaveLength(2); + expect(conversationTurnStore.fence(after, THREAD_ID)).toBe(2); + }); + + it('reports an unnamed namespace as having no durable place', async () => { + await openWithSpace(); + const anonymous = canvasAcpNamespace(''); + + // Agenetes's own rule: a namespace with no name is non-persistent. It must + // not fall through to some other Space's tables. + expect(conversationThreadStore.list(anonymous)).toEqual([]); + conversationThreadStore.upsert(anonymous, THREAD_ID, threadRecord()); + expect(conversationThreadStore.get(anonymous, THREAD_ID)).toEqual( + threadRecord(), + ); + expect( + conversationThreadStore.get(canvasAcpNamespace(CANVAS_ID), THREAD_ID), + ).toBeUndefined(); + }); +}); diff --git a/apps/server/src/modules/agent/agenetes/sqlite-stores.ts b/apps/server/src/modules/agent/agenetes/sqlite-stores.ts new file mode 100644 index 000000000..95d58cf16 --- /dev/null +++ b/apps/server/src/modules/agent/agenetes/sqlite-stores.ts @@ -0,0 +1,427 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The Agenetes conversation stores, for a Space that lives in SQLite. + * + * Agenetes ships three narrow storage ports — the durable thread table, the + * Tier-1 event log, and the Tier-2 folded turn log — plus an in-memory and a + * file implementation of each. A host picks. On Disk we pick the file ones and + * they write under the Space's `.history/`. Where a Space is rows there is no + * such directory, and the honest choice is not "in memory": a conversation + * that vanishes on restart is a worse answer than the one this file gives. + * + * These are the storage proposal's §6.4.4 arrangement in practice. The port + * hands over a *place* — a connection plus a Space-owned parent row — and the + * owner brings its own tables and its own queries. Every table below hangs off + * `space_extensions` with `ON DELETE CASCADE`, so deleting a Space takes its + * conversations with it without storage knowing what a conversation is, and + * without this module knowing where a Space is stored. + * + * The ports are synchronous, which is why they reach for `sqliteTree` rather + * than the async `extension()`: `node:sqlite` is synchronous all the way down, + * so nothing is lost by saying so. + */ + +import { space } from '../../storage/index.js'; + +import type { SqliteSpaceSubstrate } from '../../storage/index.js'; +import type { + EventLogEntry, + EventLogRecord, + EventLogStore, + PersistedTurn, + ThreadRecord, + ThreadStore, + TurnStartLogEntry, + TurnStore, +} from '@agenetes/agenetes'; +import type { AgentSubmission, Namespace } from '@agenetes/protocol'; +import type { DatabaseSync } from 'node:sqlite'; + +/** + * One namespace for all three stores. + * + * They are one owner — the conversation — split into three ports for reasons + * that belong to Agenetes, not to storage. Giving each its own namespace would + * buy three parent rows and no isolation that matters. + */ +const CONVERSATION_NAMESPACE = 'agenetes.conversations'; + +const SCHEMA = ` + CREATE TABLE IF NOT EXISTS agenetes_threads ( + extension_id INTEGER NOT NULL, + thread_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + PRIMARY KEY (extension_id, thread_id), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; + + CREATE TABLE IF NOT EXISTS agenetes_events ( + extension_id INTEGER NOT NULL, + thread_id TEXT NOT NULL, + seq INTEGER NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + PRIMARY KEY (extension_id, thread_id, seq), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; + + CREATE TABLE IF NOT EXISTS agenetes_turns ( + extension_id INTEGER NOT NULL, + thread_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + seq_start INTEGER NOT NULL, + seq_end INTEGER NOT NULL, + turn_json TEXT NOT NULL CHECK (json_valid(turn_json)), + PRIMARY KEY (extension_id, thread_id, ordinal), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; +`; + +/** Namespaces whose tables have been created on this connection. */ +const prepared = new WeakSet(); + +/** + * The place this Space's conversations live, or `null` when there is none. + * + * `null` covers three ordinary situations that all mean the same thing to a + * caller: the profile is not SQLite, the namespace has no Space (an unnamed + * Agenetes namespace), or the Space has been deleted. + */ +export function conversationTables( + namespace: Namespace, +): SqliteSpaceSubstrate | null { + if (!namespace.name) return null; + const tree = space(namespace.name).sqliteTree; + if (!tree) return null; + const substrate = tree.extension(CONVERSATION_NAMESPACE); + if (!substrate) return null; + if (!prepared.has(substrate.database)) { + substrate.database.exec(SCHEMA); + prepared.add(substrate.database); + } + return substrate; +} + +function encode(value: unknown, what: string): string { + const encoded = JSON.stringify(value); + if (encoded === undefined) { + throw new TypeError(`${what} is not representable as JSON`); + } + return encoded; +} + +function decode(value: unknown, what: string): T { + if (typeof value !== 'string') { + throw new SyntaxError(`${what} is not stored as JSON text`); + } + return JSON.parse(value) as T; +} + +function requireSubstrate(namespace: Namespace): SqliteSpaceSubstrate { + const substrate = conversationTables(namespace); + if (!substrate) { + throw new Error( + `No SQLite conversation store for namespace ${JSON.stringify(namespace.name)}`, + ); + } + return substrate; +} + +export class SqliteThreadStore implements ThreadStore { + upsert(namespace: Namespace, threadId: string, record: ThreadRecord): void { + const { database, extensionId } = requireSubstrate(namespace); + database + .prepare( + `INSERT INTO agenetes_threads (extension_id, thread_id, record_json) + VALUES (?, ?, ?) + ON CONFLICT(extension_id, thread_id) DO UPDATE SET + record_json = excluded.record_json`, + ) + .run(extensionId, threadId, encode(record, `Thread ${threadId}`)); + } + + get(namespace: Namespace, threadId: string): ThreadRecord | undefined { + const substrate = conversationTables(namespace); + if (!substrate) return undefined; + const row = substrate.database + .prepare( + `SELECT record_json FROM agenetes_threads + WHERE extension_id = ? AND thread_id = ?`, + ) + .get(substrate.extensionId, threadId); + return row === undefined + ? undefined + : decode(row['record_json'], `Thread ${threadId}`); + } + + list(namespace: Namespace): ThreadRecord[] { + const substrate = conversationTables(namespace); + if (!substrate) return []; + return substrate.database + .prepare( + `SELECT thread_id, record_json FROM agenetes_threads + WHERE extension_id = ? + ORDER BY thread_id`, + ) + .all(substrate.extensionId) + .map((row) => + decode( + row['record_json'], + `Thread ${String(row['thread_id'])}`, + ), + ); + } + + delete(namespace: Namespace, threadId: string): void { + const substrate = conversationTables(namespace); + if (!substrate) return; + substrate.database + .prepare( + `DELETE FROM agenetes_threads + WHERE extension_id = ? AND thread_id = ?`, + ) + .run(substrate.extensionId, threadId); + } +} + +export class SqliteEventLogStore implements EventLogStore { + appendTurnStart( + namespace: Namespace, + threadId: string, + request: AgentSubmission | null, + ): TurnStartLogEntry { + const entry: TurnStartLogEntry = { + seq: this.maxSeq(namespace, threadId) + 1, + ts: Date.now(), + kind: 'turn_start' as const, + request, + }; + this.#insert(namespace, threadId, entry.seq, entry); + return entry; + } + + append( + namespace: Namespace, + threadId: string, + event: EventLogEntry['event'], + ): EventLogEntry { + const entry: EventLogEntry = { + seq: this.maxSeq(namespace, threadId) + 1, + ts: Date.now(), + event, + }; + this.#insert(namespace, threadId, entry.seq, entry); + return entry; + } + + read(namespace: Namespace, threadId: string, sinceSeq = 0): EventLogEntry[] { + return this.readRecords(namespace, threadId, sinceSeq).filter( + (record): record is EventLogEntry => !('kind' in record), + ); + } + + readRecords( + namespace: Namespace, + threadId: string, + sinceSeq = 0, + ): EventLogRecord[] { + const substrate = conversationTables(namespace); + if (!substrate) return []; + return substrate.database + .prepare( + `SELECT record_json FROM agenetes_events + WHERE extension_id = ? AND thread_id = ? AND seq > ? + ORDER BY seq`, + ) + .all(substrate.extensionId, threadId, sinceSeq) + .map((row) => + decode( + row['record_json'], + `Event log for thread ${threadId}`, + ), + ); + } + + maxSeq(namespace: Namespace, threadId: string): number { + const substrate = conversationTables(namespace); + if (!substrate) return 0; + const value = substrate.database + .prepare( + `SELECT COALESCE(MAX(seq), 0) AS max_seq FROM agenetes_events + WHERE extension_id = ? AND thread_id = ?`, + ) + .get(substrate.extensionId, threadId)?.['max_seq']; + return typeof value === 'number' ? value : 0; + } + + replace( + namespace: Namespace, + threadId: string, + records: readonly EventLogRecord[], + ): void { + const { database, extensionId } = requireSubstrate(namespace); + // One statement batch, not a transaction: `rehome()` calls this while the + // instance holds its own ordering, and an adapter that opened a nested + // transaction here would collide with a caller that already has one. + database + .prepare( + 'DELETE FROM agenetes_events WHERE extension_id = ? AND thread_id = ?', + ) + .run(extensionId, threadId); + const insert = database.prepare( + `INSERT INTO agenetes_events (extension_id, thread_id, seq, record_json) + VALUES (?, ?, ?, ?)`, + ); + for (const record of records) { + insert.run( + extensionId, + threadId, + record.seq, + encode(record, `Event log for thread ${threadId}`), + ); + } + } + + delete(namespace: Namespace, threadId: string): void { + const substrate = conversationTables(namespace); + if (!substrate) return; + substrate.database + .prepare( + 'DELETE FROM agenetes_events WHERE extension_id = ? AND thread_id = ?', + ) + .run(substrate.extensionId, threadId); + } + + #insert( + namespace: Namespace, + threadId: string, + seq: number, + record: EventLogRecord, + ): void { + const { database, extensionId } = requireSubstrate(namespace); + database + .prepare( + `INSERT INTO agenetes_events (extension_id, thread_id, seq, record_json) + VALUES (?, ?, ?, ?)`, + ) + .run( + extensionId, + threadId, + seq, + encode(record, `Event log for thread ${threadId}`), + ); + } +} + +export class SqliteTurnStore implements TurnStore { + append( + namespace: Namespace, + threadId: string, + persisted: PersistedTurn, + ): void { + const { database, extensionId } = requireSubstrate(namespace); + const ordinal = this.count(namespace, threadId) + 1; + database + .prepare( + `INSERT INTO agenetes_turns ( + extension_id, thread_id, ordinal, seq_start, seq_end, turn_json + ) VALUES (?, ?, ?, ?, ?, ?)`, + ) + .run( + extensionId, + threadId, + ordinal, + persisted.seqStart, + persisted.seqEnd, + encode(persisted.turn, `Turn for thread ${threadId}`), + ); + } + + list(namespace: Namespace, threadId: string): PersistedTurn[] { + const substrate = conversationTables(namespace); + if (!substrate) return []; + return substrate.database + .prepare( + `SELECT seq_start, seq_end, turn_json FROM agenetes_turns + WHERE extension_id = ? AND thread_id = ? + ORDER BY ordinal`, + ) + .all(substrate.extensionId, threadId) + .map((row) => ({ + turn: decode( + row['turn_json'], + `Turn for thread ${threadId}`, + ), + seqStart: Number(row['seq_start']), + seqEnd: Number(row['seq_end']), + })); + } + + count(namespace: Namespace, threadId: string): number { + const substrate = conversationTables(namespace); + if (!substrate) return 0; + const value = substrate.database + .prepare( + `SELECT COUNT(*) AS turns FROM agenetes_turns + WHERE extension_id = ? AND thread_id = ?`, + ) + .get(substrate.extensionId, threadId)?.['turns']; + return typeof value === 'number' ? value : 0; + } + + fence(namespace: Namespace, threadId: string): number { + const substrate = conversationTables(namespace); + if (!substrate) return 0; + const value = substrate.database + .prepare( + `SELECT seq_end FROM agenetes_turns + WHERE extension_id = ? AND thread_id = ? + ORDER BY ordinal DESC + LIMIT 1`, + ) + .get(substrate.extensionId, threadId)?.['seq_end']; + return typeof value === 'number' ? value : 0; + } + + replace( + namespace: Namespace, + threadId: string, + persisted: readonly PersistedTurn[], + ): void { + const { database, extensionId } = requireSubstrate(namespace); + database + .prepare( + 'DELETE FROM agenetes_turns WHERE extension_id = ? AND thread_id = ?', + ) + .run(extensionId, threadId); + const insert = database.prepare( + `INSERT INTO agenetes_turns ( + extension_id, thread_id, ordinal, seq_start, seq_end, turn_json + ) VALUES (?, ?, ?, ?, ?, ?)`, + ); + persisted.forEach((record, index) => { + insert.run( + extensionId, + threadId, + index + 1, + record.seqStart, + record.seqEnd, + encode(record.turn, `Turn for thread ${threadId}`), + ); + }); + } + + delete(namespace: Namespace, threadId: string): void { + const substrate = conversationTables(namespace); + if (!substrate) return; + substrate.database + .prepare( + 'DELETE FROM agenetes_turns WHERE extension_id = ? AND thread_id = ?', + ) + .run(substrate.extensionId, threadId); + } +} diff --git a/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts b/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts index c56496286..2e91df4d7 100644 --- a/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts +++ b/apps/server/src/modules/agent/conversation/prompt/debug-prompt.ts @@ -14,11 +14,8 @@ * Gated and fully wrapped in try/catch so it can never affect a request. */ -import { appendFileSync } from 'node:fs'; -import path from 'node:path'; - -import { sanitizeId } from '../../../../utils/fs.js'; import { space } from '../../../storage/index.js'; +import { appendSubstrateLog } from '../../substrate-store.js'; import type { SpaceSubstrate } from '../../../storage/index.js'; import type { Context } from '@earendil-works/pi-ai'; @@ -151,16 +148,8 @@ type SubstrateResolution = | { readonly ok: true; readonly substrate: SpaceSubstrate | null } | { readonly ok: false; readonly error: unknown }; -/** Where this module keeps one log per thread on a Disk substrate. */ -function diskLogPath(substrate: SpaceSubstrate, threadId: string): string { - if (substrate.kind !== 'disk') { - throw new Error('Debug prompt logs require a Disk extension substrate'); - } - return path.join( - substrate.directory, - `${sanitizeId(threadId, 'threadId')}.prompt.log`, - ); -} +/** The suffix one thread's log carries, whatever the substrate stores it in. */ +const LOG_SUFFIX = '.prompt.log'; /** * Append a readable dump of the assembled prompt for one turn. No-op @@ -215,7 +204,7 @@ export function dumpAssembledPrompt(params: DumpPromptParams): void { if (!resolved.ok) throw resolved.error; const { substrate } = resolved; if (!substrate) return; - appendFileSync(diskLogPath(substrate, params.threadId), block, 'utf-8'); + appendSubstrateLog(substrate, params.threadId, LOG_SUFFIX, block); }) .catch((err: unknown) => { params.logger.warn( diff --git a/apps/server/src/modules/agent/memory/analyzer.test.ts b/apps/server/src/modules/agent/memory/analyzer.test.ts index 6aaa95b7a..0c75db5a3 100644 --- a/apps/server/src/modules/agent/memory/analyzer.test.ts +++ b/apps/server/src/modules/agent/memory/analyzer.test.ts @@ -33,6 +33,7 @@ vi.mock('../../storage/index.js', () => ({ SPACE_MEMORY_BLOB_NAME: 'space.md', })); vi.mock('../../workspace/paths.js', () => ({ + hasWorkspaceSettingDirectory: () => true, workspaceMemoryPath: () => `${physicalState.root}/setting/user.md`, })); diff --git a/apps/server/src/modules/agent/memory/analyzer.ts b/apps/server/src/modules/agent/memory/analyzer.ts index e5da5d82f..c99f2d0a3 100644 --- a/apps/server/src/modules/agent/memory/analyzer.ts +++ b/apps/server/src/modules/agent/memory/analyzer.ts @@ -33,7 +33,10 @@ import { type CanvasFile, type SpaceHandle, } from '../../storage/index.js'; -import { workspaceMemoryPath } from '../../workspace/paths.js'; +import { + hasWorkspaceSettingDirectory, + workspaceMemoryPath, +} from '../../workspace/paths.js'; import { runAgent } from '../agent.service.js'; import { readCanvasMemory } from './read.js'; @@ -280,7 +283,12 @@ function readEventsDigest(events: readonly CanvasEvent[]): EventsDigest | null { async function readMemorySnapshot(canvasId: string): Promise { const parts: string[] = []; - const longTerm = readFileSafe(workspaceMemoryPath()); + // Empty rather than missing on a backend with no Workspace folder: the + // curator's prompt keeps its shape, and the tier it cannot write to simply + // reads as empty (`workspace-user-memory` capability). + const longTerm = hasWorkspaceSettingDirectory() + ? readFileSafe(workspaceMemoryPath()) + : ''; parts.push('## Long-term memory'); parts.push(longTerm.trim().length > 0 ? longTerm.trim() : '(empty)'); diff --git a/apps/server/src/modules/agent/memory/read.ts b/apps/server/src/modules/agent/memory/read.ts index 281d16292..638e664f4 100644 --- a/apps/server/src/modules/agent/memory/read.ts +++ b/apps/server/src/modules/agent/memory/read.ts @@ -16,7 +16,10 @@ import { existsSync, readFileSync } from 'node:fs'; import { space, SPACE_MEMORY_BLOB_NAME } from '../../storage/index.js'; -import { workspaceMemoryPath } from '../../workspace/paths.js'; +import { + hasWorkspaceSettingDirectory, + workspaceMemoryPath, +} from '../../workspace/paths.js'; /** * Read the user memory body. @@ -28,6 +31,10 @@ import { workspaceMemoryPath } from '../../workspace/paths.js'; * zero-information `(empty)` line. */ export function readWorkspaceMemory(): string | null { + // A backend with no Workspace folder has no user memory document. Absence, + // not failure: the preamble is optional context either way, and the + // limitation is stated up front as the `workspace-user-memory` capability. + if (!hasWorkspaceSettingDirectory()) return null; return readNonEmpty(workspaceMemoryPath()); } diff --git a/apps/server/src/modules/agent/memory/trigger.ts b/apps/server/src/modules/agent/memory/trigger.ts index 28b8eb2ec..08b870785 100644 --- a/apps/server/src/modules/agent/memory/trigger.ts +++ b/apps/server/src/modules/agent/memory/trigger.ts @@ -24,13 +24,12 @@ * one analysis pass, which is harmless. */ -import path from 'node:path'; - -import { atomicWriteJson, readJson } from '../../../utils/fs.js'; import { createKeyedMutex } from '../../../utils/keyed-mutex.js'; import { space } from '../../storage/index.js'; - -import type { SpaceSubstrate } from '../../storage/index.js'; +import { + readSubstrateDocument, + writeSubstrateDocument, +} from '../substrate-store.js'; /** This module's namespace on the substrate. */ const MEMORY_NAMESPACE = 'huabu.memory'; @@ -44,12 +43,7 @@ const MEMORY_NAMESPACE = 'huabu.memory'; * (§6.4.4). An owner that later wants the same shape extracts a helper *over* * the substrate, never a port member. */ -function diskStatePath(substrate: SpaceSubstrate): string { - if (substrate.kind !== 'disk') { - throw new Error('Memory state requires a Disk extension substrate'); - } - return path.join(substrate.directory, 'state.json'); -} +const STATE_DOCUMENT = 'state'; /** Op-count threshold that triggers a memory analysis pass. */ export const OP_THRESHOLD = 50; @@ -84,7 +78,10 @@ const EMPTY_STATE: MemoryState = { export async function readMemoryState(canvasId: string): Promise { const substrate = await space(canvasId).extension(MEMORY_NAMESPACE); if (!substrate) return { ...EMPTY_STATE }; - const raw = readJson>(diskStatePath(substrate)); + const raw = readSubstrateDocument>( + substrate, + STATE_DOCUMENT, + ); if (!raw || typeof raw !== 'object') return { ...EMPTY_STATE }; return { counter: typeof raw.counter === 'number' ? raw.counter : 0, @@ -114,7 +111,7 @@ export async function writeMemoryState( ): Promise { const substrate = await space(canvasId).extension(MEMORY_NAMESPACE); if (!substrate) return; - atomicWriteJson(diskStatePath(substrate), state); + writeSubstrateDocument(substrate, STATE_DOCUMENT, state); } /** diff --git a/apps/server/src/modules/agent/substrate-store.ts b/apps/server/src/modules/agent/substrate-store.ts new file mode 100644 index 000000000..0f94fc004 --- /dev/null +++ b/apps/server/src/modules/agent/substrate-store.ts @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Two shapes an extension namespace can be stored in, on either substrate. + * + * The storage port hands a namespace a *place* and nothing else — a directory + * on Disk, a connection plus a parent row on SQLite (proposal §6.4.4). That is + * deliberate: a key/value member on the port would have fixed one access shape + * for every owner forever. What the port's own commentary anticipates instead + * is a helper *over* the substrate, written by owners who happen to want the + * same shape. This is that helper, for the two shapes the agent module needs: + * + * - a whole JSON document, rewritten each time (memory bookkeeping); + * - an append-only text log per key (the debug prompt dump). + * + * Nothing here is a port. It is one owner's storage code, kept in one file + * because two owners wanted the same thing rather than because storage said + * they should. + */ + +import { appendFileSync, mkdirSync } from 'node:fs'; +import path from 'node:path'; + +import { atomicWriteJson, readJson, sanitizeId } from '../../utils/fs.js'; + +import type { SpaceSubstrate } from '../storage/index.js'; +import type { DatabaseSync } from 'node:sqlite'; + +/** Tables created on demand, once per connection. */ +const prepared = new WeakSet(); + +const SCHEMA = ` + CREATE TABLE IF NOT EXISTS extension_documents ( + extension_id INTEGER NOT NULL, + name TEXT NOT NULL, + body TEXT NOT NULL, + PRIMARY KEY (extension_id, name), + FOREIGN KEY (extension_id) REFERENCES space_extensions(extension_id) + ON DELETE CASCADE + ) STRICT; +`; + +function ensureTables(database: DatabaseSync): void { + if (prepared.has(database)) return; + database.exec(SCHEMA); + prepared.add(database); +} + +/** + * Read one JSON document from a namespace, or `null` when it is not there. + * + * Absence and damage are the same answer on purpose: both callers treat a + * missing document as "start from nothing", and a bookkeeping file a user can + * corrupt by hand must not be able to fail a request. + */ +export function readSubstrateDocument( + substrate: SpaceSubstrate, + name: string, +): T | null { + const safe = sanitizeId(name, 'document name'); + if (substrate.kind === 'disk') { + return readJson(path.join(substrate.directory, `${safe}.json`)); + } + ensureTables(substrate.database); + const row = substrate.database + .prepare( + `SELECT body FROM extension_documents + WHERE extension_id = ? AND name = ?`, + ) + .get(substrate.extensionId, safe); + if (row === undefined || typeof row['body'] !== 'string') return null; + try { + return JSON.parse(row['body']) as T; + } catch { + return null; + } +} + +/** Replace one JSON document in a namespace. */ +export function writeSubstrateDocument( + substrate: SpaceSubstrate, + name: string, + value: unknown, +): void { + const safe = sanitizeId(name, 'document name'); + if (substrate.kind === 'disk') { + atomicWriteJson(path.join(substrate.directory, `${safe}.json`), value); + return; + } + ensureTables(substrate.database); + const body = JSON.stringify(value); + if (body === undefined) { + throw new TypeError(`Document ${safe} is not representable as JSON`); + } + substrate.database + .prepare( + `INSERT INTO extension_documents (extension_id, name, body) + VALUES (?, ?, ?) + ON CONFLICT(extension_id, name) DO UPDATE SET body = excluded.body`, + ) + .run(substrate.extensionId, safe, body); +} + +/** + * Append to one text log in a namespace. + * + * On Disk this is a real file, which is the point of the debug log: a + * developer tails it. Elsewhere it is a row that grows, which keeps the same + * feature working without pretending there is a file to tail. + */ +export function appendSubstrateLog( + substrate: SpaceSubstrate, + name: string, + suffix: string, + block: string, +): void { + const safe = sanitizeId(name, 'log name'); + if (substrate.kind === 'disk') { + mkdirSync(substrate.directory, { recursive: true }); + appendFileSync( + path.join(substrate.directory, `${safe}${suffix}`), + block, + 'utf8', + ); + return; + } + ensureTables(substrate.database); + substrate.database + .prepare( + `INSERT INTO extension_documents (extension_id, name, body) + VALUES (?, ?, ?) + ON CONFLICT(extension_id, name) DO UPDATE SET + body = extension_documents.body || excluded.body`, + ) + .run(substrate.extensionId, `${safe}${suffix}`, block); +} diff --git a/apps/server/src/modules/agent/tools/handlers/fs-write.ts b/apps/server/src/modules/agent/tools/handlers/fs-write.ts index ab2de9a28..b3a5974fb 100644 --- a/apps/server/src/modules/agent/tools/handlers/fs-write.ts +++ b/apps/server/src/modules/agent/tools/handlers/fs-write.ts @@ -32,8 +32,16 @@ import { existsSync } from 'node:fs'; import path from 'node:path'; import { normalizeRel } from './fs-sandbox.js'; -import { space, SPACE_MEMORY_BLOB_NAME } from '../../../storage/index.js'; -import { settingDir, userSkillsDir } from '../../../workspace/paths.js'; +import { + space, + SPACE_MEMORY_BLOB_NAME, + unavailableCapabilityMessage, +} from '../../../storage/index.js'; +import { + hasWorkspaceSettingDirectory, + settingDir, + userSkillsDir, +} from '../../../workspace/paths.js'; import { resolveLongTermPath, resolveUserSkillPath, @@ -110,6 +118,15 @@ function resolveTarget( const rel = normalizeRel(args.path); if (rel === 'memory/user.md') { + // Refused in the words the profile declared, rather than crashing on a + // path the backend cannot build. A Space's own memory body still works — + // it is a blob, not a Workspace file. + if (!hasWorkspaceSettingDirectory()) { + return { + path: rel, + error: unavailableCapabilityMessage('workspace-user-memory'), + }; + } return { tier: 'workspace', document: fileDocument(resolveLongTermPath(), settingDir()), @@ -154,6 +171,12 @@ function resolveTarget( error: `fs_write only accepts skill paths of the form "skills//SKILL.md"`, }; } + if (!hasWorkspaceSettingDirectory()) { + return { + path: rel, + error: unavailableCapabilityMessage('workspace-user-skills'), + }; + } const skillId = segs[1]; try { const absPath = resolveUserSkillPath(skillId); diff --git a/apps/server/src/modules/agent/tools/world-target-read.test.ts b/apps/server/src/modules/agent/tools/world-target-read.test.ts index bd7a6eff7..12a02ce9b 100644 --- a/apps/server/src/modules/agent/tools/world-target-read.test.ts +++ b/apps/server/src/modules/agent/tools/world-target-read.test.ts @@ -11,6 +11,7 @@ const workspaceState = vi.hoisted(() => ({ path: '', leases: 0 })); vi.mock('../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, acquireWorkspaceOperationLease: () => { const workspacePath = workspaceState.path; workspaceState.leases += 1; diff --git a/apps/server/src/modules/canvas/canvas-command-router.test.ts b/apps/server/src/modules/canvas/canvas-command-router.test.ts index f9b6b1072..c2972dab5 100644 --- a/apps/server/src/modules/canvas/canvas-command-router.test.ts +++ b/apps/server/src/modules/canvas/canvas-command-router.test.ts @@ -39,6 +39,16 @@ interface TestStoredNode { data?: Record; } +/** + * The live Spaces the World's rules are checked against. + * + * The rules are pure: they need to know which Portal targets still exist, + * and a test says so directly rather than standing up a catalogue. + */ +function liveSpaceIds(): ReadonlySet { + return new Set(['canvas-a', 'canvas-b']); +} + function writeCanvas( directory: string, canvasId: string, @@ -681,6 +691,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', brokenTopology, convertedDescendant, + liveSpaceIds(), ), ).toThrow('A node reference cannot change node type'); @@ -804,6 +815,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', directlyLocked, directlyLocked, + liveSpaceIds(), ), ).not.toThrow(); @@ -833,6 +845,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', portalLocked, portalLocked, + liveSpaceIds(), ), ).not.toThrow(); @@ -892,6 +905,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', styled ?? [], copiedTarget, + liveSpaceIds(), ), ).toThrow('contains unsupported source-owned data'); @@ -933,7 +947,12 @@ describe.skip('legacy World Portal pin command routing', () => { if (!previous) throw new Error('Missing World state'); const canonical = structuredClone(previous); expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, canonical), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + canonical, + liveSpaceIds(), + ), ).not.toThrow(); const resized = structuredClone(previous) as Array<{ @@ -944,7 +963,12 @@ describe.skip('legacy World Portal pin command routing', () => { if (!portal?.style) throw new Error('Missing Portal'); portal.style.width = (portal.style.width ?? 0) + 100; expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, resized), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + resized, + liveSpaceIds(), + ), ).toThrow(WorldPortalMutationError); const withoutNodeRef = ( @@ -955,6 +979,7 @@ describe.skip('legacy World Portal pin command routing', () => { 'canvas-world', previous, withoutNodeRef, + liveSpaceIds(), ), ).toThrow('Node references must be removed with SET_PORTAL_NODE_PINS'); }); diff --git a/apps/server/src/modules/canvas/canvas-executor.ts b/apps/server/src/modules/canvas/canvas-executor.ts index 5202ddcd7..3ce60b4f8 100644 --- a/apps/server/src/modules/canvas/canvas-executor.ts +++ b/apps/server/src/modules/canvas/canvas-executor.ts @@ -61,9 +61,11 @@ import { importForeignNodeSources } from './import-node-src.js'; import { assertWorldPortalMutationsAllowed, assertWorldPortalResultAllowed, + readLiveSpaceIds, } from './world-portal-policy.js'; import { getLogger } from '../../utils/logger.js'; import { + isWorldCanvasId, space, withCanvasMutex, type BlobScope, @@ -74,6 +76,9 @@ import { type SpaceNodeMutation, } from '../storage/index.js'; +/** Reused for every Space that cannot hold a Portal, which is all but one. */ +const EMPTY_CANVAS_IDS: ReadonlySet = new Set(); + const log = getLogger('canvas.executor'); function insertedNodeIds(deltas: readonly Delta[]): Set { @@ -731,11 +736,18 @@ export async function executeOnServerAlreadyLocked( ); const prestateEdges = (canvas.state.edges ?? []) as CanvasEdge[]; + // Only the World's rules consult it, and only the World can hold Portals, + // so an ordinary Space never pays for the catalogue read. + const liveCanvasIds = isWorldCanvasId(canvasId) + ? await readLiveSpaceIds() + : EMPTY_CANVAS_IDS; + assertWorldPortalMutationsAllowed( canvasId, commands, prestateNodes, originator.source, + liveCanvasIds, ); if (originator.source === 'agent') { @@ -834,7 +846,12 @@ export async function executeOnServerAlreadyLocked( const sharedOut = applySharedPostEffectsFromWriteResult(writeResult); const finalNodes = writeResult.nodes; const finalEdges = sharedOut.edges; - assertWorldPortalResultAllowed(canvasId, prestateNodes, finalNodes); + assertWorldPortalResultAllowed( + canvasId, + prestateNodes, + finalNodes, + liveCanvasIds, + ); const deltas = diffCanvasState( { nodes: prestateNodes, edges: prestateEdges }, diff --git a/apps/server/src/modules/canvas/canvas.route.ts b/apps/server/src/modules/canvas/canvas.route.ts index 337cf76d1..f81e72856 100644 --- a/apps/server/src/modules/canvas/canvas.route.ts +++ b/apps/server/src/modules/canvas/canvas.route.ts @@ -41,6 +41,7 @@ import { } from './space-preview-scene.js'; import { assertWorldPortalTopologyAllowed, + readLiveSpaceIds, WorldPortalMutationError, } from './world-portal-policy.js'; import { reconcileWorldPortals } from './world-portals.js'; @@ -52,11 +53,11 @@ import { MAX_UPLOAD_BYTES } from '../../upload-limits.js'; import { ARTIFACT_URL_REGEX } from '../artifact/utils.js'; import { getPreprocessDispatcher, getProfile } from '../preprocessing/index.js'; import { stripOfficeparserPreamble } from '../preprocessing/loaders/office-strip.js'; -import { isWorldCanvasId } from '../storage/canvas-dirs.js'; import { space, createSpace, deleteSpace, + isWorldCanvasId, stageSpaceImport, unavailableCapabilityMessage, getStructuredStore, @@ -1184,6 +1185,9 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { canvasId, (existing?.state.nodes ?? []) as NodeLike[], incomingState.nodes ?? [], + isWorldCanvasId(canvasId) + ? await readLiveSpaceIds() + : new Set(), ); } catch (error) { if (error instanceof WorldPortalMutationError) { diff --git a/apps/server/src/modules/canvas/external-watcher.test.ts b/apps/server/src/modules/canvas/external-watcher.test.ts index 8203416e8..5c919b1f1 100644 --- a/apps/server/src/modules/canvas/external-watcher.test.ts +++ b/apps/server/src/modules/canvas/external-watcher.test.ts @@ -72,6 +72,26 @@ const spaceHandle = vi.hoisted(() => ({ read: vi.fn(async () => ({ state: { nodes: [] } })), })); +/** + * The Space facade the watcher actually consults. + * + * `diskTree` is resolved per call from the same directory index the real one + * uses, so the "renamed outside the server" case still moves the watched path + * — and a Space with no directory is `null`, the way a non-Disk backend + * reports it. + */ +const spaceFacade = vi.hoisted(() => (canvasId: string) => ({ + ...spaceHandle, + diskTree: (() => { + const entry = canvasDirs + .list() + .find((candidate) => candidate.id === canvasId); + return entry + ? { canvasId, directory: () => `/ws/${entry.filename}` } + : null; + })(), +})); + // The facade is stubbed for the Space handle, but the directory-handle // helpers must stay the real ones: these cases drive // `withSpaceDirHandlesReleased` and assert the watcher released its handles, @@ -80,7 +100,7 @@ const spaceHandle = vi.hoisted(() => ({ vi.mock('../storage/index.js', async () => { const handles = await import('../storage/backends/disk/space-dir-handles.js'); return { - space: () => spaceHandle, + space: spaceFacade, registerSpaceDirHandleOwner: handles.registerSpaceDirHandleOwner, withSpaceDirHandlesReleased: handles.withSpaceDirHandlesReleased, }; diff --git a/apps/server/src/modules/canvas/external-watcher.ts b/apps/server/src/modules/canvas/external-watcher.ts index c20430521..bd40c8eb5 100644 --- a/apps/server/src/modules/canvas/external-watcher.ts +++ b/apps/server/src/modules/canvas/external-watcher.ts @@ -32,10 +32,8 @@ import path from 'node:path'; import { getLogger } from '../../utils/logger.js'; import { parseFrontmatter } from '../../utils/markdown-frontmatter.js'; -import { listAllCanvasDirEntries } from '../storage/canvas-dirs.js'; -import { space } from '../storage/index.js'; -import { registerSpaceDirHandleOwner } from '../storage/index.js'; -import { getWorkspacePath, isWorkspaceConfigured } from '../workspace.js'; +import { registerSpaceDirHandleOwner, space } from '../storage/index.js'; +import { isWorkspaceConfigured } from '../workspace.js'; import type { CanvasFile } from '../storage/index.js'; import type { ExternalNoteEvent, ExternalNoteItem } from '@huabu/shared'; @@ -112,11 +110,13 @@ function isSessionCurrent(session: ActiveSpaceWatch, stamp?: string): boolean { function nodesPathFor(canvasId: string): string | null { if (!isWorkspaceConfigured()) return null; - const entry = listAllCanvasDirEntries().find( - (candidate) => candidate.id === canvasId, - ); - if (!entry) return null; - return path.join(getWorkspacePath(), entry.filename, 'nodes'); + // `null` when the Space has no directory to watch, which covers both an + // unknown id and a backend that keeps Spaces in tables. Watching for + // documents that arrived without going through the application is the + // declared `external-note-discovery` capability, and this is where its + // absence becomes "there is nothing to watch". + const directory = space(canvasId).diskTree?.directory(); + return directory === undefined ? null : path.join(directory, 'nodes'); } function noteIdsFromCanvas(canvas: CanvasFile | null): Set { diff --git a/apps/server/src/modules/canvas/world-portal-policy.ts b/apps/server/src/modules/canvas/world-portal-policy.ts index cd74e7d74..f1336b1c2 100644 --- a/apps/server/src/modules/canvas/world-portal-policy.ts +++ b/apps/server/src/modules/canvas/world-portal-policy.ts @@ -3,10 +3,7 @@ import { fitPortals, getDescendantIds } from '@huabu/shared/canvas-engine'; -import { - isWorldCanvasId, - listCanvasDirEntries, -} from '../storage/canvas-dirs.js'; +import { getStructuredStore, isWorldCanvasId } from '../storage/index.js'; import type { CanvasCommand } from '@huabu/shared'; import type { NestableNode } from '@huabu/shared/canvas-engine'; @@ -24,6 +21,22 @@ function storedNodes(nodes: readonly unknown[]): StoredNode[] { ); } +/** + * Every ordinary Space in the active Workspace, by id. + * + * The World's Portals point at Spaces, so the rules below need to know which + * of those targets still exist. Read through the catalogue rather than a + * directory listing: the answer is the same on every backend, and the World is + * the one place in the product that asks it. + * + * The checks that consume this set take it as an argument, so the rules + * themselves stay pure and testable without a live backend. + */ +export async function readLiveSpaceIds(): Promise> { + const summaries = await getStructuredStore().spaces().list(); + return new Set(summaries.map((summary) => summary.canvasId)); +} + export class WorldPortalMutationError extends Error { constructor(message: string) { super(message); @@ -130,6 +143,7 @@ export function assertWorldPortalTopologyAllowed( canvasId: string, previousNodesInput: readonly unknown[], nextNodesInput: readonly unknown[], + liveCanvasIds: ReadonlySet, ): void { const previousNodes = storedNodes(previousNodesInput); const nextNodes = storedNodes(nextNodesInput); @@ -308,9 +322,6 @@ export function assertWorldPortalTopologyAllowed( } } - const liveCanvasIds = new Set( - listCanvasDirEntries().map((entry) => entry.id), - ); for (const previous of previousNodes) { const previousNodeRef = nodeRefTarget(previous); if (previousNodeRef) { @@ -372,12 +383,10 @@ export function assertWorldPortalResultAllowed( canvasId: string, previousNodesInput: readonly unknown[], nextNodesInput: readonly unknown[], + liveCanvasIds: ReadonlySet, ): void { if (!isWorldCanvasId(canvasId)) return; - const liveCanvasIds = new Set( - listCanvasDirEntries().map((entry) => entry.id), - ); const nextById = new Map( storedNodes(nextNodesInput).map((node) => [node.id, node]), ); @@ -418,6 +427,7 @@ export function assertWorldPortalMutationsAllowed( commands: readonly CanvasCommand[], nodes: readonly StoredNode[], source: 'ui' | 'agent' | 'system', + liveCanvasIds: ReadonlySet, ): void { if (source === 'system') return; @@ -439,9 +449,6 @@ export function assertWorldPortalMutationsAllowed( if (!isWorldCanvasId(canvasId)) return; - const liveCanvasIds = new Set( - listCanvasDirEntries().map((entry) => entry.id), - ); const byId = new Map(nodes.map((node) => [node.id, node])); for (const command of commands) { diff --git a/apps/server/src/modules/canvas/world-portals.test.ts b/apps/server/src/modules/canvas/world-portals.test.ts index 3af3c2bc1..16df35aff 100644 --- a/apps/server/src/modules/canvas/world-portals.test.ts +++ b/apps/server/src/modules/canvas/world-portals.test.ts @@ -13,6 +13,7 @@ const workspaceState = vi.hoisted(() => ({ path: '' })); vi.mock('../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, })); import { executeOnServer } from './canvas-executor.js'; @@ -52,6 +53,19 @@ function writeCanvas( ); } +/** + * The live Spaces the World's rules are checked against. + * + * Passed in rather than read from a backend, because the rules are pure: what + * they need to know is which Portal targets still exist, and a test says so + * directly instead of standing up a catalogue to be asked. + */ +function liveSpaceIds( + ids: readonly string[] = ['canvas-a', 'canvas-b'], +): ReadonlySet { + return new Set(ids); +} + function portals(): Array<{ id: string; position: { x: number; y: number }; @@ -237,7 +251,12 @@ describe('World Space preview reconciliation', () => { if (!previous) throw new Error('Missing World topology'); expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, []), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + [], + liveSpaceIds(), + ), ).toThrow(WorldPortalMutationError); const moved = structuredClone(previous) as Array<{ @@ -248,7 +267,12 @@ describe('World Space preview reconciliation', () => { if (!portal) throw new Error('Missing Space preview'); portal.position = { x: 999, y: 999 }; expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, moved), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + moved, + liveSpaceIds(), + ), ).not.toThrow(); expect(() => @@ -263,6 +287,7 @@ describe('World Space preview reconciliation', () => { data: { targetCanvasId: 'canvas-b' }, }, ], + liveSpaceIds(), ), ).toThrow(WorldPortalMutationError); }); @@ -308,8 +333,15 @@ describe('World Space preview reconciliation', () => { }); refreshCanvasDirIndex(); + // `canvas-a` is gone, so its Portal is broken and the subtree under it may + // be removed. expect(() => - assertWorldPortalTopologyAllowed('canvas-world', previous, []), + assertWorldPortalTopologyAllowed( + 'canvas-world', + previous, + [], + liveSpaceIds(['canvas-b']), + ), ).not.toThrow(); }); @@ -354,6 +386,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', canonical, structuredClone(canonical), + liveSpaceIds(), ), ).not.toThrow(); @@ -375,6 +408,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', canonical, apparentlyFitted, + liveSpaceIds(), ), ).toThrow('Frame reference size is managed by its contents'); }); @@ -417,6 +451,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', canonical, apparentlyFitted, + liveSpaceIds(), ), ).toThrow('Frame reference size is managed by its contents'); }); @@ -460,6 +495,7 @@ describe('World Space preview reconciliation', () => { 'canvas-world', cyclic, structuredClone(cyclic), + liveSpaceIds(), ), ).toThrow('World reference hierarchy is cyclic'); }); diff --git a/apps/server/src/modules/canvas/world-reference-resolver.test.ts b/apps/server/src/modules/canvas/world-reference-resolver.test.ts index 277f0b86e..df79eb6a1 100644 --- a/apps/server/src/modules/canvas/world-reference-resolver.test.ts +++ b/apps/server/src/modules/canvas/world-reference-resolver.test.ts @@ -11,6 +11,7 @@ const workspaceState = vi.hoisted(() => ({ path: '', leases: 0 })); vi.mock('../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, acquireWorkspaceOperationLease: () => { const workspacePath = workspaceState.path; workspaceState.leases += 1; diff --git a/apps/server/src/modules/interactive-view/interactive-view.service.ts b/apps/server/src/modules/interactive-view/interactive-view.service.ts index 917f64e82..491693976 100644 --- a/apps/server/src/modules/interactive-view/interactive-view.service.ts +++ b/apps/server/src/modules/interactive-view/interactive-view.service.ts @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { existsSync } from 'node:fs'; -import path from 'node:path'; - import { createId, interactiveViewDefinitionV1Schema, @@ -27,7 +24,6 @@ import { agentThreadService, type ExternalAgentThreadTarget, } from '../agent/agent-thread.service.js'; -import { safeResolve } from '../agent/tools/handlers/fs-sandbox.js'; import { executeOnServer, type InteractiveViewConflict, @@ -88,17 +84,21 @@ async function resolveOwnerThread( } } -function stagedRendererPath( +/** + * Whether a `upload/` renderer has actually been staged. + * + * Asked of the Space's uploads scope rather than of a directory: the scratch + * an upload lands in is a blob area on every backend, and it is the same place + * on Disk that this used to resolve by hand. + */ +async function stagedRendererExists( canvasId: string, rendererArtifact: string, -): string | null { +): Promise { const match = STAGED_RENDERER_ARTIFACT_RE.exec(rendererArtifact); const filename = match?.[1]; - if (!filename) return null; - const uploadRoot = safeResolve(canvasId, '.upload'); - const candidate = path.resolve(uploadRoot, filename); - if (!candidate.startsWith(uploadRoot + path.sep)) return null; - return candidate; + if (!filename) return false; + return (await space(canvasId).uploads.head(filename)) !== null; } function validateDefinition(definition: InteractiveViewDefinitionV1): void { @@ -358,11 +358,8 @@ export class InteractiveViewService { `Owner thread ${request.ownerThreadId} is not an external Agent thread in this Canvas`, ); } - const stagedPath = request.rendererArtifact.startsWith('upload/') - ? stagedRendererPath(canvasId, request.rendererArtifact) - : null; const rendererExists = request.rendererArtifact.startsWith('upload/') - ? stagedPath !== null && existsSync(stagedPath) + ? await stagedRendererExists(canvasId, request.rendererArtifact) : Boolean(await space(canvasId).artifacts.head(request.rendererArtifact)); if (!rendererExists) { throw new InteractiveViewServiceError( diff --git a/apps/server/src/modules/remote_fs/rfs.route.ts b/apps/server/src/modules/remote_fs/rfs.route.ts index d72d38465..a8727a65b 100644 --- a/apps/server/src/modules/remote_fs/rfs.route.ts +++ b/apps/server/src/modules/remote_fs/rfs.route.ts @@ -112,6 +112,11 @@ import { InteractiveViewServiceError, interactiveViewService, } from '../interactive-view/interactive-view.service.js'; +import { + hasStorageCapability, + parseStorageProfile, + unavailableCapabilityMessage, +} from '../storage/index.js'; import { RunCompletionError, runCompletionService, @@ -255,6 +260,18 @@ function logReachbackEvent( // ── Route plugin ── const rfsRoutes: FastifyPluginAsync = async (app) => { + // RFS is the Space *as files*. A backend that keeps Spaces in tables has no + // tree to project, and the honest answer is the declared refusal rather + // than a partial projection assembled from records — see the + // `space-file-plane` capability. One hook, because every route below + // resolves a real path sooner or later. + app.addHook('onRequest', async (_request, reply) => { + if (hasStorageCapability(parseStorageProfile(), 'space-file-plane')) return; + return reply + .code(409) + .send(rfsError(unavailableCapabilityMessage('space-file-plane'))); + }); + // Consume every request body as raw bytes within this plugin: uploads are // arbitrary binary, and the `agent` endpoint accepts either a JSON body or a // raw text prompt. Handlers interpret the Buffer per Content-Type. diff --git a/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts b/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts index 47858cbd2..1022b4b21 100644 --- a/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts +++ b/apps/server/src/modules/storage/backends/disk/storage-recovery.test.ts @@ -23,6 +23,7 @@ const workspaceState = vi.hoisted(() => ({ path: '' })); vi.mock('../../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, })); import { refreshCanvasDirIndex } from './canvas-dirs.js'; diff --git a/apps/server/src/modules/storage/backends/sqlite/blob-store.ts b/apps/server/src/modules/storage/backends/sqlite/blob-store.ts new file mode 100644 index 000000000..c6717c31b --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/blob-store.ts @@ -0,0 +1,339 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * SQLite implementation of the blob port. + * + * Bytes live in the same database file as the records, in one row per blob. + * That is what lets a SQL deployment need no folder at all: uploads, + * artifacts, the guide document and the agent's memory body stop being files + * without becoming a second service to run. + * + * The price is stated rather than hidden. A row is read and written whole, so + * this backend is sized for the documents and images a Space actually holds, + * not for arbitrarily large media, and a database holding blobs grows to the + * size of everything ever uploaded. `materialize()` therefore spools to the + * OS temp directory — the port's own escape hatch for consumers that need a + * real path — and unlinks on release, which is exactly the "temp copy" + * behaviour `BlobLease` was written to keep honest. + * + * Atomicity comes free where Disk had to work for it: a `put` buffers its body + * and then replaces the row in one statement, so a reader mid-write sees the + * previous blob and a failed body leaves the previous blob in place. + */ + +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +import { + BlobNameError, + createBlobLease, + normalizeBlobName, + SPACE_GUIDE_BLOB_NAMES, +} from '../../ports/blob.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + BlobInfo, + BlobLease, + BlobRange, + BlobRead, + BlobScope, + BlobStore, + SpaceBlobs, +} from '../../ports/blob.js'; +import type { StorageHealth } from '../../ports/common.js'; + +type SpaceBlobArea = keyof SpaceBlobs; + +/** + * Names an area owns, or `null` when it owns whatever is put in it. + * + * The distinction is the port's, not this backend's: `guide` is bounded by a + * fixed member list because on Disk it shares the Space root with records that + * are not blobs. A table has no such neighbours, but the boundary is a + * contract term — a name outside the set must be refused on every backend, or + * a caller could write one where only one adapter accepts it. + */ +function areaMembers(area: SpaceBlobArea): readonly string[] | null { + return area === 'guide' ? SPACE_GUIDE_BLOB_NAMES : null; +} + +async function collect(body: Readable | Buffer): Promise { + if (Buffer.isBuffer(body)) return body; + const chunks: Buffer[] = []; + for await (const chunk of body) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string)); + } + return Buffer.concat(chunks); +} + +function decodeBytes(value: unknown, name: string): Buffer { + if (value instanceof Uint8Array) return Buffer.from(value); + if (typeof value === 'string') return Buffer.from(value, 'utf8'); + throw new SyntaxError(`Persisted blob ${JSON.stringify(name)} is not bytes`); +} + +function decodeInfo(value: unknown): BlobInfo { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError('Malformed persisted SQLite blob row'); + } + const row = value as Record; + const name = row['name']; + const size = row['size']; + const updatedAt = row['updated_at']; + if (typeof name !== 'string') { + throw new SyntaxError('Invalid name in persisted SQLite blob'); + } + if (typeof size !== 'number' || !Number.isFinite(size)) { + throw new SyntaxError(`Invalid size for persisted blob ${name}`); + } + if (typeof updatedAt !== 'number' || !Number.isFinite(updatedAt)) { + throw new SyntaxError(`Invalid updated_at for persisted blob ${name}`); + } + return { name, size, updatedAt }; +} + +class SqliteBlobScope implements BlobScope { + readonly #context: SqliteStoreContext; + readonly #workspaceId: string; + readonly #canvasId: string; + readonly #area: SpaceBlobArea; + + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + area: SpaceBlobArea, + ) { + this.#context = context; + this.#workspaceId = workspaceId; + this.#canvasId = canvasId; + this.#area = area; + } + + /** Re-check the binding, exactly as a Disk scope re-checks its path. */ + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite blob scope for Space "${this.#canvasId}"`, + ); + } + + /** Refuse a name this area does not own, before it reaches the database. */ + #assertMember(name: string): string { + const safe = normalizeBlobName(name); + const members = areaMembers(this.#area); + if (members && !members.includes(safe)) { + throw new BlobNameError( + `"${safe}" is not a member of the ${this.#area} area. ` + + `It holds: ${members.join(', ')}.`, + ); + } + return safe; + } + + #key(name: string): [string, string, string, string] { + return [this.#workspace(), this.#canvasId, this.#area, name]; + } + + async put(name: string, body: Readable | Buffer): Promise { + const safe = this.#assertMember(name); + // Collect before touching the row: a body that fails mid-stream must + // leave the previous blob exactly as it was, and a reader must never see + // a prefix of the replacement. + const bytes = await collect(body); + const updatedAt = this.#context.now(); + this.#context + .database() + .prepare( + `INSERT INTO blobs ( + workspace_id, canvas_id, area, name, bytes, size, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(workspace_id, canvas_id, area, name) DO UPDATE SET + bytes = excluded.bytes, + size = excluded.size, + updated_at = excluded.updated_at`, + ) + .run(...this.#key(safe), bytes, bytes.byteLength, updatedAt); + return { name: safe, size: bytes.byteLength, updatedAt }; + } + + async head(name: string): Promise { + const safe = this.#assertMember(name); + const row = this.#context + .database() + .prepare( + `SELECT name, size, updated_at + FROM blobs + WHERE workspace_id = ? AND canvas_id = ? AND area = ? AND name = ?`, + ) + .get(...this.#key(safe)); + return row === undefined ? null : decodeInfo(row); + } + + async open(name: string, range?: BlobRange): Promise { + const safe = this.#assertMember(name); + const found = await this.#load(safe); + if (found === null) return null; + const { info, bytes } = found; + // `info.size` stays the whole blob; the range only bounds the body, and + // an over-long end is clamped the way a filesystem read stream clamps it. + const start = Math.max(0, range?.start ?? 0); + const end = + range?.end === undefined + ? bytes.byteLength - 1 + : Math.min(range.end, bytes.byteLength - 1); + const slice = + end < start ? Buffer.alloc(0) : bytes.subarray(start, end + 1); + return { info, body: Readable.from([slice]) }; + } + + async read(name: string): Promise { + const safe = this.#assertMember(name); + return (await this.#load(safe))?.bytes ?? null; + } + + async hasMany(names: readonly string[]): Promise> { + this.#workspace(); + const requested = new Set(names.map(normalizeBlobName)); + const members = areaMembers(this.#area); + const wanted = [...requested].filter( + (candidate) => !members || members.includes(candidate), + ); + if (wanted.length === 0) return new Set(); + + const placeholders = wanted.map(() => '?').join(', '); + const rows = this.#context + .database() + .prepare( + `SELECT name + FROM blobs + WHERE workspace_id = ? AND canvas_id = ? AND area = ? + AND name IN (${placeholders})`, + ) + .all(this.#workspace(), this.#canvasId, this.#area, ...wanted); + return new Set( + rows.map((row) => { + const name = (row as Record)['name']; + if (typeof name !== 'string') { + throw new SyntaxError('Invalid name in persisted SQLite blob'); + } + return name; + }), + ); + } + + async list(): Promise { + const members = areaMembers(this.#area); + const rows = this.#context + .database() + .prepare( + `SELECT name, size, updated_at + FROM blobs + WHERE workspace_id = ? AND canvas_id = ? AND area = ? + ORDER BY name`, + ) + .all(this.#workspace(), this.#canvasId, this.#area) + .map(decodeInfo); + return members ? rows.filter((info) => members.includes(info.name)) : rows; + } + + async materialize(name: string): Promise { + const safe = this.#assertMember(name); + const found = await this.#load(safe); + if (found === null) return null; + // No permanent path exists, so one is spooled for the life of the lease. + // The directory is unique per lease, so the blob keeps its own name for + // consumers that infer a type from the extension. + const directory = await mkdtemp(path.join(tmpdir(), 'huabu-blob-')); + const file = path.join(directory, safe); + try { + await writeFile(file, found.bytes); + } catch (error) { + await rm(directory, { recursive: true, force: true }).catch(() => {}); + throw error; + } + return createBlobLease(file, async () => { + await rm(directory, { recursive: true, force: true }).catch(() => {}); + }); + } + + async deleteAll(): Promise { + const members = areaMembers(this.#area); + const database = this.#context.database(); + if (!members) { + database + .prepare( + `DELETE FROM blobs + WHERE workspace_id = ? AND canvas_id = ? AND area = ?`, + ) + .run(this.#workspace(), this.#canvasId, this.#area); + return; + } + const placeholders = members.map(() => '?').join(', '); + database + .prepare( + `DELETE FROM blobs + WHERE workspace_id = ? AND canvas_id = ? AND area = ? + AND name IN (${placeholders})`, + ) + .run(this.#workspace(), this.#canvasId, this.#area, ...members); + } + + async #load(name: string): Promise<{ info: BlobInfo; bytes: Buffer } | null> { + const row = this.#context + .database() + .prepare( + `SELECT name, size, updated_at, bytes + FROM blobs + WHERE workspace_id = ? AND canvas_id = ? AND area = ? AND name = ?`, + ) + .get(...this.#key(name)); + if (row === undefined) return null; + const info = decodeInfo(row); + return { + info, + bytes: decodeBytes((row as Record)['bytes'], info.name), + }; + } +} + +export class SqliteBlobStore implements BlobStore { + readonly kind = 'sqlite' as const; + + readonly #context: SqliteStoreContext; + readonly #ownsContext: boolean; + + constructor(context: SqliteStoreContext, ownsContext = false) { + this.#context = context; + this.#ownsContext = ownsContext; + } + + async init(): Promise { + if (this.#ownsContext) this.#context.init(); + else this.#context.assertOpen(); + } + + async health(): Promise { + return this.#context.health(this.kind); + } + + async close(): Promise { + if (this.#ownsContext) this.#context.close(); + } + + space(canvasId: string): SpaceBlobs { + const workspaceId = this.#context.workspaceId(); + const scope = (area: SpaceBlobArea): BlobScope => + new SqliteBlobScope(this.#context, workspaceId, canvasId, area); + return { + artifacts: scope('artifacts'), + guide: scope('guide'), + memory: scope('memory'), + uploads: scope('uploads'), + }; + } +} diff --git a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts index 0de7391bc..459746d4e 100644 --- a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts +++ b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { SqliteStructuredStore } from './structured-store.js'; +import { SqliteBlobStore } from './blob-store.js'; +import { SqliteStoreContext } from './database.js'; import { createSqliteTestFile, installDeltaAbortTrigger, @@ -9,6 +10,8 @@ import { openSqliteTestStore, readSqliteDeltaLog, } from './test-support.js'; +import { SqliteWorkspaceRepository } from './workspace-repository.js'; +import { describeBlobStoreContract } from '../../ports/contracts/blob-store.contract.js'; import { describeSpaceExtensionContract } from '../../ports/contracts/space-extension.contract.js'; import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; import { describeSpaceNodesContract } from '../../ports/contracts/space-nodes.contract.js'; @@ -16,7 +19,9 @@ import { describeSpaceRepositoryContract } from '../../ports/contracts/space-rep import { describeSpaceTasksContract } from '../../ports/contracts/space-tasks.contract.js'; import { describeSpaceWriteContract } from '../../ports/contracts/space-write.contract.js'; import { describeStructuredStoreContract } from '../../ports/contracts/structured-store.contract.js'; +import { describeWorkspaceRepositoryContract } from '../../ports/contracts/workspace-repository.contract.js'; +import type { SqliteStructuredStore } from './structured-store.js'; import type { NodeContent } from '../../../canvas/persistence-types.js'; function note(nodeId: string, label: string, content: string): NodeContent { @@ -32,12 +37,15 @@ async function createOrdinarySpace( if (!created.ok) throw new Error(`Could not create test Space ${canvasId}`); } -describeStructuredStoreContract('SQLite', () => { - const file = createSqliteTestFile('huabu-sqlite-structured-contract-'); - return { - store: new SqliteStructuredStore(file.filename), - cleanup: file.remove, - }; +describeStructuredStoreContract('SQLite', async () => { + // Through the same lifecycle a Server uses: open the connection, then select + // a Workspace. A handle resolved before one is active has no namespace to + // address, which is the SQL twin of the Disk adapter refusing before a + // workspace path is committed. + const harness = await openEmptySqliteTestStore( + 'huabu-sqlite-structured-contract-', + ); + return { store: harness.store, cleanup: harness.cleanup }; }); describeSpaceRepositoryContract('SQLite', async () => { @@ -196,3 +204,29 @@ describeSpaceTasksContract('SQLite', async () => { cleanup: harness.cleanup, }; }); + +describeBlobStoreContract('SqliteBlobStore', async () => { + const harness = await openEmptySqliteTestStore('huabu-sqlite-blob-contract-'); + return { + // The blob store shares the structured store's connection, because both + // ports are one database file. + store: new SqliteBlobStore(harness.context), + canvasId: 'sqlite-blob-contract-space', + cleanup: harness.cleanup, + }; +}); + +describeWorkspaceRepositoryContract('SQLite', async () => { + const file = createSqliteTestFile('huabu-sqlite-workspace-contract-'); + const context = new SqliteStoreContext(file.filename); + context.init(); + const repository = new SqliteWorkspaceRepository(context); + return { + repository, + create: (name: string) => repository.create(name), + cleanup: () => { + context.close(); + file.remove(); + }, + }; +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/database.ts b/apps/server/src/modules/storage/backends/sqlite/database.ts index acd0e4afb..fc0850751 100644 --- a/apps/server/src/modules/storage/backends/sqlite/database.ts +++ b/apps/server/src/modules/storage/backends/sqlite/database.ts @@ -1,8 +1,28 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +/** + * The one SQLite connection a process holds, and the state that lives as long + * as it does. + * + * Both storage axes share this object when the profile selects SQLite for + * either of them. That is not a convenience: the structured records and the + * blob bytes are in one database file, so two connections would be two + * writers to the same file, and SQLite's answer to that is a lock error rather + * than a queue. One connection also makes the ordered Space write a real + * transaction across everything it touches. + * + * The active Workspace is held here for the same reason the Disk adapters hold + * the active workspace path: it is the namespace every query is scoped to. + * Switching Workspaces re-points this field and reopens nothing — the settled + * "Backend selection scope" decision in proposal §2. + */ + +import { mkdirSync } from 'node:fs'; +import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; +import { SQLITE_MIGRATIONS, type SqliteMigration } from './schema.js'; import { assertSpaceMutationAllowed, beginSpaceDeleteAdmission, @@ -10,85 +30,20 @@ import { import type { StorageHealth } from '../../ports/common.js'; -export const SQLITE_SCHEMA_VERSION = 1; -export const SQLITE_WORLD_COLLISION_KEY = '.world'; - -const SCHEMA_V1 = ` - CREATE TABLE spaces ( - canvas_id TEXT PRIMARY KEY, - title TEXT, - collision_key TEXT NOT NULL UNIQUE, - version INTEGER NOT NULL, - state_json TEXT NOT NULL CHECK (json_valid(state_json)), - created_at REAL NOT NULL, - updated_at REAL NOT NULL, - is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)) - ) STRICT; - - CREATE UNIQUE INDEX spaces_single_world - ON spaces(is_world) - WHERE is_world = 1; - - CREATE TABLE nodes ( - canvas_id TEXT NOT NULL, - node_id TEXT NOT NULL, - record_json TEXT NOT NULL CHECK (json_valid(record_json)), - revision TEXT NOT NULL CHECK (length(revision) > 0), - label_collision_key TEXT NOT NULL, - PRIMARY KEY (canvas_id, node_id), - UNIQUE (canvas_id, label_collision_key), - FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE - ) STRICT; - - CREATE TABLE events ( - event_id INTEGER PRIMARY KEY AUTOINCREMENT, - canvas_id TEXT NOT NULL, - event_json TEXT NOT NULL CHECK (json_valid(event_json)), - FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE - ) STRICT; - - CREATE INDEX events_by_canvas_order - ON events(canvas_id, event_id); - - CREATE TABLE changes ( - canvas_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), - PRIMARY KEY (canvas_id, thread_id), - FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE - ) STRICT; - - CREATE TABLE tasks ( - canvas_id TEXT PRIMARY KEY, - snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), - FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE - ) STRICT; +export { SQLITE_MIGRATIONS, SQLITE_SCHEMA_VERSION } from './schema.js'; +export type { SqliteMigration } from './schema.js'; - CREATE TABLE space_extensions ( - extension_id INTEGER PRIMARY KEY AUTOINCREMENT, - canvas_id TEXT NOT NULL, - namespace TEXT NOT NULL, - UNIQUE (canvas_id, namespace), - FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE - ) STRICT; - - CREATE TABLE delta_log ( - canvas_id TEXT NOT NULL, - version INTEGER NOT NULL, - entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), - PRIMARY KEY (canvas_id, version), - FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE - ) STRICT; -`; - -export interface SqliteMigration { - readonly version: number; - readonly sql: string; -} +/** + * The collision key the hidden World Space is filed under. + * + * Unreachable from any user title: `toSafeFilename` strips leading dots, so + * no requested name normalizes to it and the World slot cannot be taken by + * an ordinary Space. + */ +export const SQLITE_WORLD_COLLISION_KEY = '.world'; -export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = Object.freeze([ - Object.freeze({ version: 1, sql: SCHEMA_V1 }), -]); +/** Milliseconds a statement waits for a lock before reporting SQLITE_BUSY. */ +const BUSY_TIMEOUT_MS = 5_000; function readUserVersion(database: DatabaseSync): number { const row = database.prepare('PRAGMA user_version').get(); @@ -146,20 +101,35 @@ export function applySqliteMigrations( } } +/** Raised when a handle outlives the Workspace it was resolved in. */ +export class SqliteWorkspaceScopeError extends Error { + override name = 'SqliteWorkspaceScopeError'; +} + /** One connection and all adapter-lifetime process-local state. */ export class SqliteStoreContext { readonly now: () => number; readonly #database: DatabaseSync; + readonly #filename: string; readonly #admissionScope: string; #state: 'new' | 'open' | 'closed' = 'new'; + #workspaceId: string | null = null; - constructor(filename: string, now: () => number) { + constructor(filename: string, now: () => number = Date.now) { + if (typeof filename !== 'string' || filename.length === 0) { + throw new TypeError('SQLite filename must be a non-empty string'); + } this.now = now; + this.#filename = filename; this.#admissionScope = `sqlite:${filename}`; this.#database = new DatabaseSync(filename, { open: false }); } + get filename(): string { + return this.#filename; + } + init(): void { if (this.#state === 'open') return; if (this.#state === 'closed') { @@ -167,7 +137,27 @@ export class SqliteStoreContext { } try { + // A database file names a directory that may not exist yet — the whole + // point of this profile is that the operator never had to create one. + // In-memory and URI filenames name no directory at all. + const directory = path.dirname(this.#filename); + if ( + !this.#filename.startsWith(':') && + !this.#filename.startsWith('file:') + ) { + mkdirSync(directory, { recursive: true }); + } this.#database.open(); + // Write-ahead logging so a reader is never blocked by the writer, and a + // bounded wait so a second connection (an external tool, a stale + // process) reports a busy database instead of failing instantly. + this.#database.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`); + this.#database.exec('PRAGMA journal_mode = WAL'); + // NORMAL is the documented pairing for WAL: durable across a process + // crash, and only a machine-level crash can lose the most recent + // commits — which is the same guarantee the Disk adapter's atomic + // renames give, stated rather than assumed. + this.#database.exec('PRAGMA synchronous = NORMAL'); this.#database.exec('PRAGMA foreign_keys = ON'); const foreignKeys = this.#database.prepare('PRAGMA foreign_keys').get()?.[ 'foreign_keys' @@ -221,6 +211,51 @@ export class SqliteStoreContext { } } + // ─── The active Workspace ──────────────────────────────────────────────── + + /** Point every subsequent query at one Workspace. Reopens nothing. */ + useWorkspace(workspaceId: string | null): void { + this.#workspaceId = workspaceId; + } + + /** The active Workspace id, or `null` when none has been selected. */ + activeWorkspaceId(): string | null { + return this.#workspaceId; + } + + /** The active Workspace id, or a refusal when none has been selected. */ + workspaceId(): string { + this.assertOpen(); + if (this.#workspaceId === null) { + throw new SqliteWorkspaceScopeError( + 'No Workspace is active on the SQLite backend. Activate one before ' + + 'reading or writing Spaces.', + ); + } + return this.#workspaceId; + } + + /** + * The Workspace a retained handle was resolved in, or a refusal. + * + * A handle keeps the id it was built with and re-checks it here, so a + * Workspace switch makes the stale handle reject rather than silently + * addressing rows in the newly active namespace. That is the same rule the + * Disk adapters apply to a retained workspace path. + */ + assertBoundWorkspace(boundWorkspaceId: string, what: string): string { + const active = this.workspaceId(); + if (active !== boundWorkspaceId) { + throw new SqliteWorkspaceScopeError( + `${what} belongs to an inactive Workspace. Resolve a fresh handle ` + + 'after Workspace activation.', + ); + } + return active; + } + + // ─── Space lifecycle admission ─────────────────────────────────────────── + assertMutationAllowed(canvasId: string): void { this.assertOpen(); assertSpaceMutationAllowed(this.#admissionScope, canvasId); diff --git a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql index 467674c0c..ea1f6f30c 100644 --- a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql +++ b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql @@ -1,22 +1,38 @@ --- Immutable SQLite structured-store schema v1 fixture. --- Add a new fixture for later schema versions; do not rewrite this history. +-- Immutable SQLite storage schema v1 fixture. +-- +-- Hand-written to match `schema.ts`'s version 1 exactly, and never rewritten +-- once a version ships: the point of the fixture is to prove that opening an +-- existing database migrates and reads it rather than reshaping it. A later +-- schema version gets its own fixture beside this one. PRAGMA foreign_keys = ON; BEGIN IMMEDIATE; +CREATE TABLE workspaces ( + workspace_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at REAL NOT NULL, + last_opened_at REAL NOT NULL, + forgotten_at REAL +) STRICT; + CREATE TABLE spaces ( canvas_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, title TEXT, - collision_key TEXT NOT NULL UNIQUE, + collision_key TEXT NOT NULL, version INTEGER NOT NULL, state_json TEXT NOT NULL CHECK (json_valid(state_json)), created_at REAL NOT NULL, updated_at REAL NOT NULL, - is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)) + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)), + UNIQUE (workspace_id, collision_key), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) + ON DELETE CASCADE ) STRICT; CREATE UNIQUE INDEX spaces_single_world - ON spaces(is_world) + ON spaces(workspace_id) WHERE is_world = 1; CREATE TABLE nodes ( @@ -70,19 +86,34 @@ CREATE TABLE delta_log ( FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE ) STRICT; +CREATE TABLE blobs ( + workspace_id TEXT NOT NULL, + canvas_id TEXT NOT NULL, + area TEXT NOT NULL, + name TEXT NOT NULL, + bytes BLOB NOT NULL, + size INTEGER NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (workspace_id, canvas_id, area, name) +) STRICT; + +INSERT INTO workspaces ( + workspace_id, name, created_at, last_opened_at, forgotten_at +) VALUES ('fixture-workspace', 'Fixture Workspace', 1, 1, NULL); + INSERT INTO spaces ( - canvas_id, title, collision_key, version, state_json, + canvas_id, workspace_id, title, collision_key, version, state_json, created_at, updated_at, is_world ) VALUES ( - 'fixture-world', 'World', '.world', 0, + 'fixture-world', 'fixture-workspace', 'World', '.world', 0, '{"nodes":[],"edges":[]}', 1, 1, 1 ); INSERT INTO spaces ( - canvas_id, title, collision_key, version, state_json, + canvas_id, workspace_id, title, collision_key, version, state_json, created_at, updated_at, is_world ) VALUES ( - 'fixture-space', 'Fixture Space', 'fixture space', 3, + 'fixture-space', 'fixture-workspace', 'Fixture Space', 'fixture space', 3, '{"nodes":[{"id":"fixture-node","type":"note"}],"edges":[]}', 10, 13, 0 ); @@ -113,5 +144,12 @@ INSERT INTO delta_log (canvas_id, version, entry_json) VALUES ( '{"version":3,"ts":13,"commands":[],"deltas":[],"originator":{"source":"system"}}' ); +INSERT INTO blobs ( + workspace_id, canvas_id, area, name, bytes, size, updated_at +) VALUES ( + 'fixture-workspace', 'fixture-space', 'artifacts', 'fixture.txt', + CAST('fixture bytes' AS BLOB), 13, 14 +); + PRAGMA user_version = 1; COMMIT; diff --git a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts index b1b2fff67..706b8930c 100644 --- a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts +++ b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts @@ -1,13 +1,19 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import { extractCanvasChanges } from '@huabu/shared/canvas-engine'; -import { applySqliteMigrations, SQLITE_SCHEMA_VERSION } from './database.js'; +import { SqliteBlobStore } from './blob-store.js'; +import { + applySqliteMigrations, + SqliteStoreContext, + SQLITE_SCHEMA_VERSION, +} from './database.js'; import { SqliteStructuredStore } from './structured-store.js'; import { createSqliteTestFile, @@ -16,6 +22,7 @@ import { readSqliteDeltaLog, withTestDatabase, } from './test-support.js'; +import { SqliteWorkspaceRepository } from './workspace-repository.js'; import type { CanvasFile, @@ -46,6 +53,32 @@ function trackedStore(filename: string): SqliteStructuredStore { return store; } +/** A tracked context, so a test can drive the shared connection directly. */ +function trackedContext(filename: string): SqliteStoreContext { + const context = new SqliteStoreContext(filename); + cleanups.push(() => context.close()); + return context; +} + +/** + * Open a store on an existing file and activate a Workspace on it. + * + * Reopening is the interesting half of persistence, and every Space query is + * Workspace-scoped, so a reopened store has to select one before it can read + * anything — exactly as a restarted Server does. + */ +async function reopenWithWorkspace( + filename: string, +): Promise { + const context = trackedContext(filename); + context.init(); + const workspaces = new SqliteWorkspaceRepository(context); + const [first] = await workspaces.list(); + if (!first) throw new Error('Reopened SQLite database holds no Workspace'); + context.useWorkspace(first.workspaceId); + return new SqliteStructuredStore(context); +} + async function trackedOpenStore(prefix: string) { const harness = await openSqliteTestStore(prefix); cleanups.push(harness.cleanup); @@ -101,7 +134,9 @@ describe('SqliteStructuredStore lifecycle and schema', () => { Promise.resolve().then(() => store.space('lifecycle-space').read()), ).rejects.toThrow(/not initialized/); await expect( - store.space('lifecycle-space').nodes.readMany([]), + Promise.resolve().then(() => + store.space('lifecycle-space').nodes.readMany([]), + ), ).rejects.toThrow(/not initialized/); await expect(store.init()).resolves.toBeUndefined(); @@ -109,6 +144,12 @@ describe('SqliteStructuredStore lifecycle and schema', () => { await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); await expect(store.health()).resolves.toEqual({ ok: true, kind: 'sqlite' }); + // Open is not the same as ready: a Space query needs a Workspace, and an + // open store with none says so rather than answering for an arbitrary one. + await expect( + Promise.resolve().then(() => store.spaces().list()), + ).rejects.toThrow(/No Workspace is active/); + await expect(store.close()).resolves.toBeUndefined(); await expect(store.close()).resolves.toBeUndefined(); await expect(store.health()).rejects.toThrow(/closed/); @@ -119,7 +160,9 @@ describe('SqliteStructuredStore lifecycle and schema', () => { Promise.resolve().then(() => store.space('lifecycle-space').read()), ).rejects.toThrow(/closed/); await expect( - store.space('lifecycle-space').nodes.readMany([]), + Promise.resolve().then(() => + store.space('lifecycle-space').nodes.readMany([]), + ), ).rejects.toThrow(/closed/); await expect(store.init()).rejects.toThrow(/closed/); }); @@ -134,6 +177,7 @@ describe('SqliteStructuredStore lifecycle and schema', () => { user_version: SQLITE_SCHEMA_VERSION, }); const expectedTables = [ + 'blobs', 'changes', 'delta_log', 'events', @@ -141,6 +185,7 @@ describe('SqliteStructuredStore lifecycle and schema', () => { 'space_extensions', 'spaces', 'tasks', + 'workspaces', ]; const tableRows = database.prepare('PRAGMA table_list').all(); const productionTables = tableRows.filter((row) => @@ -166,6 +211,28 @@ describe('SqliteStructuredStore lifecycle and schema', () => { to: 'canvas_id', onDelete: 'CASCADE', }); + expect( + database + .prepare('PRAGMA foreign_key_list(spaces)') + .all() + .map((row) => ({ + table: row['table'], + from: row['from'], + to: row['to'], + onDelete: row['on_delete'], + })), + ).toContainEqual({ + table: 'workspaces', + from: 'workspace_id', + to: 'workspace_id', + onDelete: 'CASCADE', + }); + // Blob rows deliberately do not reference `spaces`: the deletion saga + // sweeps them before the record goes, and must also be able to sweep + // orphans for a record that is already missing. + expect(database.prepare('PRAGMA foreign_key_list(blobs)').all()).toEqual( + [], + ); }); }); @@ -177,8 +244,7 @@ describe('SqliteStructuredStore lifecycle and schema', () => { ); withTestDatabase(file.filename, (database) => database.exec(fixtureSql)); - const store = trackedStore(file.filename); - await store.init(); + const store = await reopenWithWorkspace(file.filename); await expect(store.spaces().worldId()).resolves.toBe('fixture-world'); await expect(store.spaces().list()).resolves.toEqual([ { @@ -229,6 +295,13 @@ describe('SqliteStructuredStore lifecycle and schema', () => { originator: { source: 'system' }, }, ]); + const blobs = new SqliteBlobStore( + // The same connection the structured store just read through. + (store as unknown as { context: SqliteStoreContext }).context, + ); + await expect( + blobs.space('fixture-space').artifacts.read('fixture.txt'), + ).resolves.toEqual(Buffer.from('fixture bytes')); }); it('rejects a database whose user_version is from the future', async () => { @@ -289,9 +362,8 @@ describe('SqliteStructuredStore persistence and transactions', () => { }); expect(put).toMatchObject({ ok: true, record }); - await harness.store.close(); - const reopened = trackedStore(harness.filename); - await reopened.init(); + harness.closeConnection(); + const reopened = await reopenWithWorkspace(harness.filename); await expect(reopened.spaces().worldId()).resolves.toBe( harness.world.canvasId, @@ -636,9 +708,8 @@ describe('SqliteStructuredStore persistence and transactions', () => { currentRevision: recreated.revision, }); - await harness.store.close(); - const reopened = trackedStore(harness.filename); - await reopened.init(); + harness.closeConnection(); + const reopened = await reopenWithWorkspace(harness.filename); await expect( reopened.space(canvasId).nodes.delete(record.nodeId), ).resolves.toBe('deleted'); @@ -653,3 +724,246 @@ describe('SqliteStructuredStore persistence and transactions', () => { }); }); }); + +describe('SqliteStructuredStore durability and encoding', () => { + it('opens in WAL with a bounded busy wait and foreign keys enforced', async () => { + const harness = await trackedOpenStore('huabu-sqlite-pragmas-'); + + withTestDatabase(harness.filename, (database) => { + // Read on a *second* connection: `journal_mode` is a property of the + // database file, so this proves the mode was actually persisted rather + // than set on the adapter's own handle and forgotten. + expect(database.prepare('PRAGMA journal_mode').get()).toEqual({ + journal_mode: 'wal', + }); + }); + const database = harness.context.database(); + expect(database.prepare('PRAGMA foreign_keys').get()).toEqual({ + foreign_keys: 1, + }); + expect( + Number(database.prepare('PRAGMA busy_timeout').get()?.['timeout']), + ).toBeGreaterThan(0); + }); + + it('accepts an undefined field the way JSON.stringify does', async () => { + const harness = await trackedOpenStore('huabu-sqlite-undefined-'); + const canvasId = 'undefined-field-space'; + const base = await createSpace(harness.store, canvasId, 'Undefined Space'); + const handle = harness.store.space(canvasId); + + // Disk persists through `JSON.stringify`, which drops an undefined own + // property. A record it accepts must not become a rejected write here — + // that divergence is invisible until a caller happens to spread an + // optional field onto a node. + const next = { + ...base, + version: 1, + state: { + nodes: [ + { + id: 'node-undefined', + type: 'note', + position: { x: 0, y: 0 }, + data: { kept: 'yes', dropped: undefined }, + }, + ], + edges: [], + }, + } as unknown as CanvasFile; + await expect( + handle.write({ expectedVersion: 0, nextRecord: next, nodeMutations: [] }), + ).resolves.toEqual({ ok: true }); + const stored = await handle.read(); + expect( + (stored?.state.nodes[0] as { data: Record }).data, + ).toEqual({ kept: 'yes' }); + + // What is genuinely unrepresentable still rejects. + const cyclic: Record = { id: 'node-cyclic' }; + cyclic['self'] = cyclic; + await expect( + handle.write({ + expectedVersion: 1, + nextRecord: { + ...base, + version: 2, + state: { nodes: [cyclic], edges: [] }, + } as unknown as CanvasFile, + nodeMutations: [], + }), + ).rejects.toThrow(/cycle/); + await expect( + handle.write({ + expectedVersion: 1, + nextRecord: { + ...base, + version: 2, + state: { nodes: [{ id: 'n', size: Number.NaN }], edges: [] }, + } as unknown as CanvasFile, + nodeMutations: [], + }), + ).rejects.toThrow(/non-finite/); + }); + + it('delivers streamed nodes before the scan finishes and stops on abort', async () => { + const harness = await trackedOpenStore('huabu-sqlite-stream-'); + const canvasId = 'stream-space'; + await createSpace(harness.store, canvasId, 'Stream Space'); + const nodes = harness.store.space(canvasId).nodes; + for (let index = 0; index < 6; index += 1) { + const put = await nodes.put({ + nodeId: `node-${index}`, + record: note(`node-${index}`, `Node ${index}`, `body ${index}`), + }); + if (!put.ok) throw new Error('Could not seed a stream node'); + } + + const signal = { aborted: false }; + const seen: NodeSnapshot[] = []; + const delivered = await nodes.stream( + (snapshot) => { + seen.push(snapshot); + if (seen.length === 2) signal.aborted = true; + }, + { signal }, + ); + + // An aborted scan stops reading rather than materializing the whole + // collection first and discarding it, so the map it settles with is the + // partial one the port describes. + expect(seen).toHaveLength(2); + expect(delivered.size).toBe(2); + await expect(nodes.list()).resolves.toHaveProperty('size', 6); + + const complete: string[] = []; + const all = await nodes.stream((snapshot) => + complete.push(snapshot.record.nodeId), + ); + expect(complete).toHaveLength(6); + expect(all.size).toBe(6); + }); + + it('reads a batch of nodes in one pass, including duplicates and absences', async () => { + const harness = await trackedOpenStore('huabu-sqlite-readmany-'); + const canvasId = 'readmany-space'; + await createSpace(harness.store, canvasId, 'ReadMany Space'); + const nodes = harness.store.space(canvasId).nodes; + for (const nodeId of ['a', 'b', 'c']) { + await nodes.put({ + nodeId, + record: note(nodeId, `Node ${nodeId}`, nodeId), + }); + } + + const selection = await nodes.readMany(['a', 'a', 'missing', 'c']); + expect([...selection.keys()].sort()).toEqual(['a', 'c']); + expect(selection.get('a')).toEqual(await nodes.read('a')); + }); + + it('scopes every Space operation to the active Workspace', async () => { + const harness = await trackedOpenStore('huabu-sqlite-workspaces-'); + const first = harness.workspaceId; + await createSpace(harness.store, 'workspace-a-space', 'Space A'); + + const workspaces = new SqliteWorkspaceRepository(harness.context); + const second = await workspaces.create('Second Workspace'); + const retained = harness.store.space('workspace-a-space'); + + harness.context.useWorkspace(second.workspaceId); + // A Space in another Workspace is not visible, and a handle resolved + // before the switch refuses rather than answering for the new namespace. + await expect(harness.store.spaces().list()).resolves.toEqual([]); + await expect( + harness.store.space('workspace-a-space').read(), + ).resolves.toBeNull(); + await expect(retained.read()).rejects.toThrow(/inactive Workspace/); + + // Same title, different Workspace: no collision, no suffix. + const created = await harness.store + .spaces() + .create({ canvasId: 'workspace-b-space', title: 'Space A' }); + expect(created).toMatchObject({ ok: true, record: { title: 'Space A' } }); + + harness.context.useWorkspace(first); + await expect(harness.store.spaces().list()).resolves.toHaveLength(1); + }); + + it('keeps a forgotten Workspace out of listings without destroying it', async () => { + const harness = await trackedOpenStore('huabu-sqlite-forget-'); + const workspaces = new SqliteWorkspaceRepository(harness.context); + await createSpace(harness.store, 'forgotten-space', 'Forgotten Space'); + + await expect(workspaces.remove(harness.workspaceId)).resolves.toBe(true); + await expect(workspaces.list()).resolves.toEqual([]); + await expect(workspaces.get(harness.workspaceId)).resolves.toBeNull(); + await expect(workspaces.remove(harness.workspaceId)).resolves.toBe(false); + + // "Forget" is not "delete": the port's wording is deliberate, and on a + // backend with no folder left behind the rows have to be what honours it. + expect( + withTestDatabase(harness.filename, (database) => + database + .prepare('SELECT canvas_id FROM spaces WHERE canvas_id = ?') + .all('forgotten-space'), + ), + ).toHaveLength(1); + }); +}); + +describe('SqliteBlobStore', () => { + async function openBlobs(prefix: string) { + const harness = await trackedOpenStore(prefix); + const store = new SqliteBlobStore(harness.context); + await store.init(); + return { harness, store }; + } + + it('keeps bytes exactly, including binary that is not text', async () => { + const { store } = await openBlobs('huabu-sqlite-blob-bytes-'); + const bytes = Buffer.from([0, 1, 2, 250, 251, 252, 0, 255]); + + const scope = store.space('blob-space').artifacts; + const info = await scope.put('binary.bin', bytes); + expect(info.size).toBe(bytes.byteLength); + expect(await scope.read('binary.bin')).toEqual(bytes); + }); + + it('spools a lease to a real path and removes it on release', async () => { + const { store } = await openBlobs('huabu-sqlite-blob-lease-'); + const scope = store.space('blob-space').artifacts; + await scope.put('leased.png', Buffer.from('pretend png')); + + const lease = await scope.materialize('leased.png'); + if (!lease) throw new Error('Expected a lease'); + const leasedPath = lease.path; + // The blob keeps its own name, so a consumer that infers a type from the + // extension still works. + expect(path.basename(leasedPath)).toBe('leased.png'); + expect(readFileSync(leasedPath)).toEqual(Buffer.from('pretend png')); + + await lease.release(); + // A temp copy, not the storage: it must not survive the lease. + expect(existsSync(leasedPath)).toBe(false); + expect(await scope.read('leased.png')).toEqual(Buffer.from('pretend png')); + }); + + it('separates the bytes of one Workspace from another', async () => { + const { harness, store } = await openBlobs('huabu-sqlite-blob-workspace-'); + const first = store.space('shared-canvas-id').artifacts; + await first.put('same-name.bin', Buffer.from('first workspace')); + + const workspaces = new SqliteWorkspaceRepository(harness.context); + const second = await workspaces.create('Second Workspace'); + harness.context.useWorkspace(second.workspaceId); + + const other = store.space('shared-canvas-id').artifacts; + expect(await other.head('same-name.bin')).toBeNull(); + await other.put('same-name.bin', Buffer.from('second workspace')); + + harness.context.useWorkspace(harness.workspaceId); + expect( + await store.space('shared-canvas-id').artifacts.read('same-name.bin'), + ).toEqual(Buffer.from('first workspace')); + }); +}); diff --git a/apps/server/src/modules/storage/backends/sqlite/rows.ts b/apps/server/src/modules/storage/backends/sqlite/rows.ts index 9cbe48f35..94faf3b96 100644 --- a/apps/server/src/modules/storage/backends/sqlite/rows.ts +++ b/apps/server/src/modules/storage/backends/sqlite/rows.ts @@ -9,6 +9,17 @@ * value looks like. Space and log reads reject malformed domain values. Node * reads preserve the port's repair path by recovering malformed JSON values * into a valid record whose content still exposes the stored value. + * + * The encoder's job is to refuse what SQLite could not faithfully return — + * cycles, non-finite numbers, values `JSON.stringify` would silently reshape + * into something else. It deliberately does **not** refuse what + * `JSON.stringify` already handles by rule, because Disk persists through + * that same function: a record it accepts must not become a rejected write + * here. `undefined` is the case that matters in practice — an optional field + * spread onto a node makes an own property whose value is `undefined`, and + * Disk drops it. See §13's "silent divergence" risk: a portable contract that + * only holds where the adapters already agree certifies both sides of a + * disagreement. */ import { SQLITE_WORLD_COLLISION_KEY } from './database.js'; @@ -41,6 +52,10 @@ function assertJsonValue( } return; } + // `JSON.stringify` drops an `undefined` object property and encodes an + // `undefined` array element as null, so Disk already accepts both. Matching + // that rule keeps one record from being writable on one backend only. + if (value === undefined) return; if (typeof value !== 'object') { throw new TypeError(`${context} contains a non-JSON value`); } @@ -137,6 +152,7 @@ function numberColumn( export interface PersistedSpace { readonly record: CanvasFile; + readonly workspaceId: string; readonly collisionKey: string; readonly isWorld: boolean; } @@ -164,24 +180,68 @@ export function decodeSpaceRow(value: unknown): PersistedSpace { } return { record, + workspaceId: stringColumn(row, 'workspace_id', context), collisionKey: stringColumn(row, 'collision_key', context), isWorld: world === 1, }; } export const SPACE_COLUMNS = - 'canvas_id, title, collision_key, version, state_json, created_at, updated_at, is_world'; + 'canvas_id, workspace_id, title, collision_key, version, state_json, ' + + 'created_at, updated_at, is_world'; +/** + * Read one Space, scoped to the Workspace that owns it. + * + * The Workspace predicate is not an optimization. `canvas_id` is unique across + * the whole database, so without it a handle resolved in one Workspace would + * answer for a Space in another — which is exactly the confusion the Disk + * adapters prevent by binding to a workspace path. + */ export function readSpaceRow( database: DatabaseSync, + workspaceId: string, canvasId: string, ): PersistedSpace | null { const row = database - .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE canvas_id = ?`) - .get(canvasId); + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND canvas_id = ?`, + ) + .get(workspaceId, canvasId); return row === undefined ? null : decodeSpaceRow(row); } +/** Whether the named Space exists in this Workspace. */ +export function spaceRowExists( + database: DatabaseSync, + workspaceId: string, + canvasId: string, +): boolean { + return ( + database + .prepare( + `SELECT 1 AS present + FROM spaces + WHERE workspace_id = ? AND canvas_id = ?`, + ) + .get(workspaceId, canvasId)?.['present'] === 1 + ); +} + +/** Collision keys already taken in one Workspace, for name allocation. */ +export function occupiedCollisionKeys( + database: DatabaseSync, + workspaceId: string, +): string[] { + return database + .prepare('SELECT collision_key FROM spaces WHERE workspace_id = ?') + .all(workspaceId) + .map((row) => row['collision_key']) + .filter((value): value is string => typeof value === 'string'); +} + export function validateCanvasFile(record: CanvasFile, canvasId: string): void { const shapeError = canvasFileShapeError(record, canvasId); if (shapeError) { @@ -192,6 +252,7 @@ export function validateCanvasFile(record: CanvasFile, canvasId: string): void { export function insertSpaceRow( database: DatabaseSync, + workspaceId: string, record: CanvasFile, collisionKey: string, isWorld = false, @@ -200,12 +261,13 @@ export function insertSpaceRow( database .prepare( `INSERT INTO spaces ( - canvas_id, title, collision_key, version, state_json, + canvas_id, workspace_id, title, collision_key, version, state_json, created_at, updated_at, is_world - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .run( record.canvasId, + workspaceId, record.title, isWorld ? SQLITE_WORLD_COLLISION_KEY : collisionKey, record.version, @@ -218,6 +280,7 @@ export function insertSpaceRow( export function updateSpaceRow( database: DatabaseSync, + workspaceId: string, record: CanvasFile, expectedVersion: number, ): number { @@ -226,12 +289,13 @@ export function updateSpaceRow( .prepare( `UPDATE spaces SET version = ?, state_json = ?, updated_at = ? - WHERE canvas_id = ? AND version = ?`, + WHERE workspace_id = ? AND canvas_id = ? AND version = ?`, ) .run( record.version, stringifyJson(record.state, `Space ${record.canvasId} state`), record.updatedAt, + workspaceId, record.canvasId, expectedVersion, ); diff --git a/apps/server/src/modules/storage/backends/sqlite/schema.ts b/apps/server/src/modules/storage/backends/sqlite/schema.ts new file mode 100644 index 000000000..8f9d00629 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/schema.ts @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * The SQLite schema, and the rule for changing it. + * + * One file per version, applied in order, never edited once released. The + * migration runner in `database.ts` enforces that shape; this file is only the + * SQL. Everything is `STRICT` so a column's declared type is a real + * constraint, and every child row reaches its owner through a foreign key so + * deleting a Space or a Workspace cannot leave the rest behind. + * + * Two collections sit at the top: Workspaces, which are the namespaces a + * deployment holds, and Spaces, which belong to exactly one of them. That is + * the whole reason a SQL profile needs no Workspace directory — a Workspace is + * a row, not a folder, and switching to another one re-scopes queries through + * the same connection rather than reopening anything (proposal §2, "Backend + * selection scope"). + * + * Blobs deliberately do **not** reference `spaces`. The two ports are + * configured independently and their lifecycles are joined only by the + * deletion saga in `storage.ts`, which sweeps every blob area *before* the + * structured record goes. A foreign key here would quietly move that ordering + * decision into the schema, and would refuse the orphan sweep the saga + * performs when a record has already gone missing. + */ + +/** + * Version 1 — Workspaces, Spaces, and everything a Space owns. + * + * `collision_key` is the de-duplicated, case-folded name a Space or node is + * filed under. It exists because titles and labels collide and the product + * resolves that with " (2)" suffixes; the UNIQUE constraints are what make + * the allocation in `identity.ts` authoritative rather than advisory. + */ +const SCHEMA_V1 = ` + CREATE TABLE workspaces ( + workspace_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + created_at REAL NOT NULL, + last_opened_at REAL NOT NULL, + -- Membership is forgettable without being destructive: the port's + -- remove() drops a Workspace from the listing and keeps everything it + -- owns, the way forgetting a Disk Workspace leaves its folder on disk. + forgotten_at REAL + ) STRICT; + + CREATE TABLE spaces ( + canvas_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + title TEXT, + collision_key TEXT NOT NULL, + version INTEGER NOT NULL, + state_json TEXT NOT NULL CHECK (json_valid(state_json)), + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + is_world INTEGER NOT NULL DEFAULT 0 CHECK (is_world IN (0, 1)), + UNIQUE (workspace_id, collision_key), + FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) + ON DELETE CASCADE + ) STRICT; + + CREATE UNIQUE INDEX spaces_single_world + ON spaces(workspace_id) + WHERE is_world = 1; + + CREATE TABLE nodes ( + canvas_id TEXT NOT NULL, + node_id TEXT NOT NULL, + record_json TEXT NOT NULL CHECK (json_valid(record_json)), + revision TEXT NOT NULL CHECK (length(revision) > 0), + label_collision_key TEXT NOT NULL, + PRIMARY KEY (canvas_id, node_id), + UNIQUE (canvas_id, label_collision_key), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + event_json TEXT NOT NULL CHECK (json_valid(event_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE INDEX events_by_canvas_order + ON events(canvas_id, event_id); + + CREATE TABLE changes ( + canvas_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + PRIMARY KEY (canvas_id, thread_id), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE tasks ( + canvas_id TEXT PRIMARY KEY, + snapshot_json TEXT NOT NULL CHECK (json_valid(snapshot_json)), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE space_extensions ( + extension_id INTEGER PRIMARY KEY AUTOINCREMENT, + canvas_id TEXT NOT NULL, + namespace TEXT NOT NULL, + UNIQUE (canvas_id, namespace), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE delta_log ( + canvas_id TEXT NOT NULL, + version INTEGER NOT NULL, + entry_json TEXT NOT NULL CHECK (json_valid(entry_json)), + PRIMARY KEY (canvas_id, version), + FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE + ) STRICT; + + CREATE TABLE blobs ( + workspace_id TEXT NOT NULL, + canvas_id TEXT NOT NULL, + area TEXT NOT NULL, + name TEXT NOT NULL, + bytes BLOB NOT NULL, + size INTEGER NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (workspace_id, canvas_id, area, name) + ) STRICT; +`; + +export interface SqliteMigration { + readonly version: number; + readonly sql: string; +} + +export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = Object.freeze([ + Object.freeze({ version: 1, sql: SCHEMA_V1 }), +]); + +export const SQLITE_SCHEMA_VERSION = + SQLITE_MIGRATIONS[SQLITE_MIGRATIONS.length - 1]?.version ?? 0; diff --git a/apps/server/src/modules/storage/backends/sqlite/space-extension.ts b/apps/server/src/modules/storage/backends/sqlite/space-extension.ts index 51c868321..6f66b34d9 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-extension.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-extension.ts @@ -1,58 +1,94 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -/** SQLite connection point for one extension namespace in one Space. */ +/** + * SQLite connection point for one extension namespace in one Space. + * + * The port's `extension()` is async because a backend may have to go and open + * something. SQLite does not: `node:sqlite` is synchronous, so the whole + * operation is a row read and possibly one insert. That matters beyond + * tidiness — an owner whose own interface is synchronous (the Agenetes + * conversation stores are the live example) cannot await, and would otherwise + * have to keep its own cache warmed by an unrelated code path. So the work + * lives in a synchronous function and the port member wraps it. + */ import { withImmediateTransaction } from './database.js'; +import { spaceRowExists } from './rows.js'; import { assertValidNamespace } from '../../ports/namespace.js'; import type { SqliteStoreContext } from './database.js'; -import type { SpaceHandle } from '../../ports/structured.js'; +import type { SpaceHandle, SpaceSubstrate } from '../../ports/structured.js'; + +/** The SQLite arm of {@link SpaceSubstrate}, for callers that narrowed already. */ +export type SqliteSpaceSubstrate = Extract; + +/** + * Resolve — creating if absent — the namespace's connection point. + * + * `null` when the Space does not exist, which is the port's rule: refusing a + * substrate for a Space that is gone is what stops an owner from resurrecting + * one through an ad-hoc write. + */ +export function resolveSqliteSpaceExtension( + context: SqliteStoreContext, + boundWorkspaceId: string, + canvasId: string, + namespaceInput: string, +): SqliteSpaceSubstrate | null { + const namespace = assertValidNamespace(namespaceInput); + const workspaceId = context.assertBoundWorkspace( + boundWorkspaceId, + `SQLite Space extension(${canvasId})`, + ); + context.assertMutationAllowed(canvasId); + const database = context.database(); + + return withImmediateTransaction(database, () => { + if (!spaceRowExists(database, workspaceId, canvasId)) return null; + + database + .prepare( + `INSERT INTO space_extensions (canvas_id, namespace) + VALUES (?, ?) + ON CONFLICT(canvas_id, namespace) DO NOTHING`, + ) + .run(canvasId, namespace); + const extensionId = database + .prepare( + `SELECT extension_id + FROM space_extensions + WHERE canvas_id = ? AND namespace = ?`, + ) + .get(canvasId, namespace)?.['extension_id']; + if ( + typeof extensionId !== 'number' || + !Number.isSafeInteger(extensionId) || + extensionId <= 0 + ) { + throw new Error( + `Could not resolve SQLite extension ${JSON.stringify(namespace)}`, + ); + } + return Object.freeze({ + kind: 'sqlite' as const, + database, + extensionId, + }); + }); +} export function createSqliteSpaceExtension( context: SqliteStoreContext, + boundWorkspaceId: string, canvasId: string, ): SpaceHandle['extension'] { return async function extension(namespaceInput: string) { - const namespace = assertValidNamespace(namespaceInput); - context.assertMutationAllowed(canvasId); - const database = context.database(); - - return withImmediateTransaction(database, () => { - const exists = - database - .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') - .get(canvasId)?.['present'] === 1; - if (!exists) return null; - - database - .prepare( - `INSERT INTO space_extensions (canvas_id, namespace) - VALUES (?, ?) - ON CONFLICT(canvas_id, namespace) DO NOTHING`, - ) - .run(canvasId, namespace); - const extensionId = database - .prepare( - `SELECT extension_id - FROM space_extensions - WHERE canvas_id = ? AND namespace = ?`, - ) - .get(canvasId, namespace)?.['extension_id']; - if ( - typeof extensionId !== 'number' || - !Number.isSafeInteger(extensionId) || - extensionId <= 0 - ) { - throw new Error( - `Could not resolve SQLite extension ${JSON.stringify(namespace)}`, - ); - } - return Object.freeze({ - kind: 'sqlite' as const, - database, - extensionId, - }); - }); + return resolveSqliteSpaceExtension( + context, + boundWorkspaceId, + canvasId, + namespaceInput, + ); }; } diff --git a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts index 5fe88a840..a948d9920 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-logs.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-logs.ts @@ -8,7 +8,7 @@ import { } from '@huabu/shared/canvas-engine'; import { withImmediateTransaction } from './database.js'; -import { parseJson, stringifyJson } from './rows.js'; +import { parseJson, spaceRowExists, stringifyJson } from './rows.js'; import { sanitizeId } from '../../../../utils/fs.js'; import type { SqliteStoreContext } from './database.js'; @@ -27,14 +27,13 @@ function firstIssue(error: z.ZodError): string { return `${location}: ${issue.message}`; } -function requireSpace(context: SqliteStoreContext, canvasId: string): void { +function requireSpace( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, +): void { context.assertMutationAllowed(canvasId); - if ( - context - .database() - .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') - .get(canvasId)?.['present'] !== 1 - ) { + if (!spaceRowExists(context.database(), workspaceId, canvasId)) { throw new Error( `SQLite Space logs(${canvasId}) cannot mutate a missing Space`, ); @@ -81,14 +80,28 @@ export interface SqliteSpaceLogs { class SqliteSpaceLogCoordinator { readonly #context: SqliteStoreContext; + readonly #workspaceId: string; readonly #canvasId: string; - constructor(context: SqliteStoreContext, canvasId: string) { + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + ) { this.#context = context; + this.#workspaceId = workspaceId; this.#canvasId = canvasId; } + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite Space logs(${this.#canvasId})`, + ); + } + async readEvents(limit?: number): Promise { + this.#workspace(); const database = this.#context.database(); if (limit !== undefined && !(limit > 0)) return []; if (limit === undefined || !Number.isFinite(limit)) { @@ -118,6 +131,7 @@ class SqliteSpaceLogCoordinator { async appendEvents(events: readonly NewCanvasEvent[]): Promise { this.#context.assertOpen(); + const workspaceId = this.#workspace(); if (events.length === 0) return; const records: CanvasEvent[] = events.map((event, index) => { const input = canvasEventInputSchema.safeParse(event); @@ -140,7 +154,7 @@ class SqliteSpaceLogCoordinator { return record; }); - requireSpace(this.#context, this.#canvasId); + requireSpace(this.#context, workspaceId, this.#canvasId); const database = this.#context.database(); withImmediateTransaction(database, () => { const insert = database.prepare( @@ -157,6 +171,7 @@ class SqliteSpaceLogCoordinator { async readChanges(threadIdInput: string): Promise { const threadId = sanitizeId(threadIdInput, 'threadId'); + this.#workspace(); const row = this.#context .database() .prepare( @@ -176,7 +191,7 @@ class SqliteSpaceLogCoordinator { ): Promise { const threadId = sanitizeId(threadIdInput, 'threadId'); stringifyJson(records, `Changes for thread ${JSON.stringify(threadId)}`); - requireSpace(this.#context, this.#canvasId); + requireSpace(this.#context, this.#workspace(), this.#canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => { const current = database @@ -212,7 +227,7 @@ class SqliteSpaceLogCoordinator { changeId: string, ): Promise { const threadId = sanitizeId(threadIdInput, 'threadId'); - requireSpace(this.#context, this.#canvasId); + requireSpace(this.#context, this.#workspace(), this.#canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => { const current = database @@ -249,9 +264,14 @@ class SqliteSpaceLogCoordinator { export function createSqliteSpaceLogs( context: SqliteStoreContext, + workspaceId: string, canvasId: string, ): SqliteSpaceLogs { - const coordinator = new SqliteSpaceLogCoordinator(context, canvasId); + const coordinator = new SqliteSpaceLogCoordinator( + context, + workspaceId, + canvasId, + ); return Object.freeze({ events: Object.freeze({ read: (limit?: number) => coordinator.readEvents(limit), diff --git a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts index acd17fd0e..48ac2cc63 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-nodes.ts @@ -8,6 +8,7 @@ import { allocateNodeIdentity } from './identity.js'; import { decodeNodeRecord, requireRevision, + spaceRowExists, stringifyJson, validateNodeContent, } from './rows.js'; @@ -63,12 +64,30 @@ function readNodeRow( return row === undefined ? null : decodeNodeRow(row, nodeId); } -function spaceExists(database: DatabaseSync, canvasId: string): boolean { - return ( - database - .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') - .get(canvasId)?.['present'] === 1 - ); +/** + * Ids per `readMany` statement. + * + * Comfortably under SQLite's default 999-parameter ceiling with room for the + * `canvas_id` bind, so a caller never has to know the limit exists. + */ +const READ_MANY_CHUNK = 500; + +/** Decode one scanned row into the id the port keys collections by. */ +function decodeIdentifiedNodeRow(value: unknown): [string, NodeSnapshot] { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError('Malformed persisted SQLite Node row'); + } + const nodeId = (value as Record)['node_id']; + if (typeof nodeId !== 'string') { + throw new SyntaxError('Invalid node_id in persisted SQLite Node'); + } + const row = decodeNodeRow(value, nodeId); + return [nodeId, { record: row.record, revision: row.revision }]; +} + +function collectNodeRow(value: unknown, into: Map): void { + const [nodeId, snapshot] = decodeIdentifiedNodeRow(value); + into.set(nodeId, snapshot); } function validatePut(input: NodePutInput): string { @@ -87,11 +106,12 @@ function validatePut(input: NodePutInput): string { /** Apply one node put inside the caller's active transaction. */ export function putSqliteNodeInTransaction( database: DatabaseSync, + workspaceId: string, canvasId: string, input: NodePutInput, ): NodePutResult { const nodeId = validatePut(input); - if (!spaceExists(database, canvasId)) { + if (!spaceRowExists(database, workspaceId, canvasId)) { return { ok: false, reason: 'not-found' }; } @@ -187,14 +207,28 @@ export class SqliteSpaceNodes implements SpaceNodes { readonly canvasId: string; readonly #context: SqliteStoreContext; + readonly #workspaceId: string; - constructor(context: SqliteStoreContext, canvasId: string) { + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + ) { this.#context = context; + this.#workspaceId = workspaceId; this.canvasId = canvasId; } + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite Space nodes(${this.canvasId})`, + ); + } + async read(nodeIdInput: string): Promise { const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + this.#workspace(); const current = readNodeRow( this.#context.database(), this.canvasId, @@ -208,42 +242,39 @@ export class SqliteSpaceNodes implements SpaceNodes { async readMany( nodeIds: readonly string[], ): Promise> { + const wanted = [...new Set(nodeIds)].map((nodeId) => + sanitizeId(nodeId, 'nodeId'), + ); + // Before the empty-batch shortcut: asking a closed store for nothing is + // still asking a closed store. + this.#workspace(); const database = this.#context.database(); const snapshots = new Map(); - for (const nodeIdInput of new Set(nodeIds)) { - const nodeId = sanitizeId(nodeIdInput, 'nodeId'); - const row = readNodeRow(database, this.canvasId, nodeId); - if (row !== null) { - snapshots.set(nodeId, { - record: row.record, - revision: row.revision, - }); - } + if (wanted.length === 0) return snapshots; + + // One statement per batch rather than one per id: a neighbourhood read + // asks for tens of nodes, and the port exists so that cost stays + // proportional to the request. SQLite caps a statement at + // SQLITE_MAX_VARIABLE_NUMBER parameters, so the batch is chunked rather + // than assumed to fit. + for (let start = 0; start < wanted.length; start += READ_MANY_CHUNK) { + const chunk = wanted.slice(start, start + READ_MANY_CHUNK); + const placeholders = chunk.map(() => '?').join(', '); + const rows = database + .prepare( + `SELECT node_id, record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ? AND node_id IN (${placeholders})`, + ) + .all(this.canvasId, ...chunk); + for (const value of rows) collectNodeRow(value, snapshots); } return snapshots; } async list(): Promise> { - const rows = this.#context - .database() - .prepare( - `SELECT node_id, record_json, revision, label_collision_key - FROM nodes - WHERE canvas_id = ?`, - ) - .all(this.canvasId); const snapshots = new Map(); - for (const value of rows) { - const nodeId = value['node_id']; - if (typeof nodeId !== 'string') { - throw new SyntaxError('Invalid node_id in persisted SQLite Node'); - } - const row = decodeNodeRow(value, nodeId); - snapshots.set(nodeId, { - record: row.record, - revision: row.revision, - }); - } + for (const value of this.#scan()) collectNodeRow(value, snapshots); return snapshots; } @@ -251,31 +282,51 @@ export class SqliteSpaceNodes implements SpaceNodes { onNode: (snapshot: NodeSnapshot) => void, options?: NodeStreamOptions, ): Promise> { - const snapshots = await this.list(); const delivered = new Map(); - for (const [nodeId, snapshot] of snapshots) { + // Decoded row by row off a live cursor, so a reader that renders partial + // results sees the first node without waiting for the last, and an + // aborted scan stops reading rather than discarding rows it already + // materialized. + for (const value of this.#scan()) { if (options?.signal?.aborted) break; + const [nodeId, snapshot] = decodeIdentifiedNodeRow(value); onNode(snapshot); delivered.set(nodeId, snapshot); } return delivered; } + #scan(): Iterable { + this.#workspace(); + return this.#context + .database() + .prepare( + `SELECT node_id, record_json, revision, label_collision_key + FROM nodes + WHERE canvas_id = ?`, + ) + .iterate(this.canvasId); + } + async put(input: NodePutInput): Promise { validatePut(input); + const workspaceId = this.#workspace(); this.#context.assertMutationAllowed(this.canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => - putSqliteNodeInTransaction(database, this.canvasId, input), + putSqliteNodeInTransaction(database, workspaceId, this.canvasId, input), ); } async delete(nodeIdInput: string): Promise { const nodeId = sanitizeId(nodeIdInput, 'nodeId'); + const workspaceId = this.#workspace(); this.#context.assertMutationAllowed(this.canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => { - if (!spaceExists(database, this.canvasId)) return 'absent' as const; + if (!spaceRowExists(database, workspaceId, this.canvasId)) { + return 'absent' as const; + } const deleted = Number( database .prepare('DELETE FROM nodes WHERE canvas_id = ? AND node_id = ?') diff --git a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts index 898a52b42..019d0136d 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-repository.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-repository.ts @@ -8,6 +8,7 @@ import { allocateSpaceIdentity, collisionKeyForTitle } from './identity.js'; import { decodeSpaceRow, insertSpaceRow, + occupiedCollisionKeys, readSpaceRow, SPACE_COLUMNS, } from './rows.js'; @@ -26,6 +27,24 @@ import type { SpaceRepository, } from '../../ports/structured.js'; import type { CanvasSummary } from '@huabu/shared'; +import type { DatabaseSync } from 'node:sqlite'; + +/** + * Whether this Space id is taken anywhere in the database. + * + * Space ids are the primary key across every Workspace, so creation has to ask + * globally even though everything else is scoped. + */ +function spaceRowExistsAnywhere( + database: DatabaseSync, + canvasId: string, +): boolean { + return ( + database + .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') + .get(canvasId)?.['present'] === 1 + ); +} function validateTitle(title: unknown): asserts title is string | null { if (title !== null && typeof title !== 'string') { @@ -35,16 +54,35 @@ function validateTitle(title: unknown): asserts title is string | null { export class SqliteSpaceRepository implements SpaceRepository { readonly #context: SqliteStoreContext; + readonly #workspaceId: string; constructor(context: SqliteStoreContext) { this.#context = context; + // Bound at construction, like the Disk repository binds the workspace + // path: one repository instance spans a caller's read and its follow-up + // write, and a Workspace switch in between must reject rather than + // silently retarget the write. + this.#workspaceId = context.workspaceId(); + } + + /** The Workspace every query below is scoped to, re-checked per call. */ + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + 'SQLite Space repository', + ); } async list(): Promise { + const workspaceId = this.#workspace(); const database = this.#context.database(); return database - .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 0`) - .all() + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND is_world = 0`, + ) + .all(workspaceId) .map((row) => { const { record } = decodeSpaceRow(row); return { @@ -58,10 +96,15 @@ export class SqliteSpaceRepository implements SpaceRepository { } async worldId(): Promise { + const workspaceId = this.#workspace(); const database = this.#context.database(); const rows = database - .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 1`) - .all(); + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND is_world = 1`, + ) + .all(workspaceId); if (rows.length !== 1) { throw new Error( rows.length === 0 @@ -75,11 +118,16 @@ export class SqliteSpaceRepository implements SpaceRepository { } async ensureWorld(): Promise { + const workspaceId = this.#workspace(); const database = this.#context.database(); return withImmediateTransaction(database, () => { const existing = database - .prepare(`SELECT ${SPACE_COLUMNS} FROM spaces WHERE is_world = 1`) - .all(); + .prepare( + `SELECT ${SPACE_COLUMNS} + FROM spaces + WHERE workspace_id = ? AND is_world = 1`, + ) + .all(workspaceId); if (existing.length > 1) { throw new Error('SQLite namespace has multiple World Spaces'); } @@ -96,6 +144,7 @@ export class SqliteSpaceRepository implements SpaceRepository { } insertSpaceRow( database, + workspaceId, { canvasId, title: 'World', @@ -114,19 +163,22 @@ export class SqliteSpaceRepository implements SpaceRepository { async create(input: SpaceCreateInput): Promise { const canvasId = sanitizeId(input.canvasId, 'canvasId'); validateTitle(input.title); + const workspaceId = this.#workspace(); this.#context.assertMutationAllowed(canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => { - if (readSpaceRow(database, canvasId) !== null) { + // Existence is checked across every Workspace, not just the active one: + // `canvas_id` is the primary key, so an id already used elsewhere is + // taken here too, and reporting it as free would fail on INSERT. + if (spaceRowExistsAnywhere(database, canvasId)) { return { ok: false as const, reason: 'already-exists' as const }; } - const occupied = database - .prepare('SELECT collision_key FROM spaces') - .all() - .map((row) => row['collision_key']) - .filter((value): value is string => typeof value === 'string'); - const identity = allocateSpaceIdentity(input.title, canvasId, occupied); + const identity = allocateSpaceIdentity( + input.title, + canvasId, + occupiedCollisionKeys(database, workspaceId), + ); const timestamp = this.#context.now(); if (!Number.isFinite(timestamp)) { throw new TypeError('SQLite Space clock returned a non-finite value'); @@ -139,14 +191,19 @@ export class SqliteSpaceRepository implements SpaceRepository { createdAt: timestamp, updatedAt: timestamp, }; - insertSpaceRow(database, record, identity.collisionKey); + insertSpaceRow(database, workspaceId, record, identity.collisionKey); return { ok: true as const, record }; }); } async beginDelete(input: SpaceDeleteInput): Promise { const canvasId = sanitizeId(input.canvasId, 'canvasId'); - const beforeAdmission = readSpaceRow(this.#context.database(), canvasId); + const workspaceId = this.#workspace(); + const beforeAdmission = readSpaceRow( + this.#context.database(), + workspaceId, + canvasId, + ); if (beforeAdmission?.isWorld) { return { ok: false, reason: 'world-forbidden' }; } @@ -154,7 +211,11 @@ export class SqliteSpaceRepository implements SpaceRepository { const release = await this.#context.acquireDelete(canvasId); let sessionOwnsGate = false; try { - const afterAdmission = readSpaceRow(this.#context.database(), canvasId); + const afterAdmission = readSpaceRow( + this.#context.database(), + workspaceId, + canvasId, + ); if (afterAdmission?.isWorld) { return { ok: false, reason: 'world-forbidden' }; } @@ -175,7 +236,7 @@ export class SqliteSpaceRepository implements SpaceRepository { this.#context.assertOpen(); const database = this.#context.database(); const result = withImmediateTransaction(database, () => { - const current = readSpaceRow(database, canvasId); + const current = readSpaceRow(database, workspaceId, canvasId); if (current?.isWorld) { throw new Error(`Refusing to delete World Space ${canvasId}`); } @@ -186,8 +247,10 @@ export class SqliteSpaceRepository implements SpaceRepository { } const deleted = Number( database - .prepare('DELETE FROM spaces WHERE canvas_id = ?') - .run(canvasId).changes, + .prepare( + 'DELETE FROM spaces WHERE workspace_id = ? AND canvas_id = ?', + ) + .run(workspaceId, canvasId).changes, ); return { deleted: deleted === 1 }; }); @@ -222,11 +285,12 @@ export class SqliteSpaceRepository implements SpaceRepository { async rename(input: SpaceRenameInput): Promise { const canvasId = sanitizeId(input.canvasId, 'canvasId'); validateTitle(input.title); + const workspaceId = this.#workspace(); this.#context.assertMutationAllowed(canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => { - const current = readSpaceRow(database, canvasId); + const current = readSpaceRow(database, workspaceId, canvasId); if (current === null) return { ok: false, reason: 'not-found' } as const; if (current.isWorld) { return { ok: false, reason: 'world-forbidden' } as const; @@ -241,9 +305,9 @@ export class SqliteSpaceRepository implements SpaceRepository { .prepare( `SELECT ${SPACE_COLUMNS} FROM spaces - WHERE collision_key = ? AND canvas_id <> ?`, + WHERE workspace_id = ? AND collision_key = ? AND canvas_id <> ?`, ) - .get(collisionKey, canvasId); + .get(workspaceId, collisionKey, canvasId); if (conflict !== undefined) { return { ok: false, @@ -257,9 +321,9 @@ export class SqliteSpaceRepository implements SpaceRepository { .prepare( `UPDATE spaces SET title = ?, collision_key = ? - WHERE canvas_id = ?`, + WHERE workspace_id = ? AND canvas_id = ?`, ) - .run(input.title, collisionKey, canvasId); + .run(input.title, collisionKey, workspaceId, canvasId); if (Number(result.changes) !== 1) { throw new Error(`Could not rename SQLite Space ${canvasId}`); } diff --git a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts index 4d27e2d17..57f82cba2 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-tasks.ts @@ -13,7 +13,7 @@ import { } from '@huabu/shared'; import { withImmediateTransaction } from './database.js'; -import { parseJson, stringifyJson } from './rows.js'; +import { parseJson, spaceRowExists, stringifyJson } from './rows.js'; import type { SqliteStoreContext } from './database.js'; import type { @@ -93,10 +93,16 @@ export class SqliteSpaceTasks implements SpaceTasks { readonly runs: SpaceTaskRuns; readonly #context: SqliteStoreContext; + readonly #workspaceId: string; readonly #canvasId: string; - constructor(context: SqliteStoreContext, canvasId: string) { + constructor( + context: SqliteStoreContext, + workspaceId: string, + canvasId: string, + ) { this.#context = context; + this.#workspaceId = workspaceId; this.#canvasId = canvasId; this.runs = Object.freeze({ create: (run: TaskRunRecord) => this.#createRun(run), @@ -110,8 +116,16 @@ export class SqliteSpaceTasks implements SpaceTasks { }); } + #workspace(): string { + return this.#context.assertBoundWorkspace( + this.#workspaceId, + `SQLite Space Tasks(${this.#canvasId})`, + ); + } + async read(): Promise { this.#context.assertOpen(); + this.#workspace(); return readSnapshot(this.#context, this.#canvasId); } @@ -214,14 +228,11 @@ export class SqliteSpaceTasks implements SpaceTasks { } #mutate(apply: (snapshot: TaskStoreSnapshot) => T): T { + const workspaceId = this.#workspace(); this.#context.assertMutationAllowed(this.#canvasId); const database = this.#context.database(); return withImmediateTransaction(database, () => { - if ( - database - .prepare('SELECT 1 AS present FROM spaces WHERE canvas_id = ?') - .get(this.#canvasId)?.['present'] !== 1 - ) { + if (!spaceRowExists(database, workspaceId, this.#canvasId)) { throw new Error( `Space Tasks(${this.#canvasId}) cannot write a missing Space`, ); diff --git a/apps/server/src/modules/storage/backends/sqlite/space-write.ts b/apps/server/src/modules/storage/backends/sqlite/space-write.ts index e97da4239..774fb381c 100644 --- a/apps/server/src/modules/storage/backends/sqlite/space-write.ts +++ b/apps/server/src/modules/storage/backends/sqlite/space-write.ts @@ -5,6 +5,7 @@ import { withImmediateTransaction } from './database.js'; import { allocateSpaceIdentity } from './identity.js'; import { insertSpaceRow, + occupiedCollisionKeys, readSpaceRow, stringifyJson, updateSpaceRow, @@ -86,17 +87,22 @@ function validateInput(canvasId: string, input: SpaceWriteInput): void { /** Bind the atomic SQLite record/node/delta write to one Space. */ export function createSqliteSpaceWrite( context: SqliteStoreContext, + boundWorkspaceId: string, canvasId: string, ): SpaceHandle['write'] { return async function write( input: SpaceWriteInput, ): Promise { + const workspaceId = context.assertBoundWorkspace( + boundWorkspaceId, + `SpaceWrite(${canvasId})`, + ); context.assertMutationAllowed(canvasId); validateInput(canvasId, input); const database = context.database(); const completed = withImmediateTransaction(database, () => { - const current = readSpaceRow(database, canvasId); + const current = readSpaceRow(database, workspaceId, canvasId); if (current === null) { if (!input.allowCreate) { return { ok: false, reason: 'not-found' } as const; @@ -106,18 +112,14 @@ export function createSqliteSpaceWrite( `SpaceWrite(${canvasId}) can create only from version 0`, ); } - const occupied = database - .prepare('SELECT collision_key FROM spaces') - .all() - .map((row) => row['collision_key']) - .filter((value): value is string => typeof value === 'string'); const identity = allocateSpaceIdentity( input.nextRecord.title, canvasId, - occupied, + occupiedCollisionKeys(database, workspaceId), ); insertSpaceRow( database, + workspaceId, { ...input.nextRecord, title: identity.title }, identity.collisionKey, ); @@ -149,16 +151,26 @@ export function createSqliteSpaceWrite( continue; } - const result = putSqliteNodeInTransaction(database, canvasId, { - nodeId: mutation.nodeId, - record: mutation.record, - strictLabel: mutation.strictLabel, - }); + const result = putSqliteNodeInTransaction( + database, + workspaceId, + canvasId, + { + nodeId: mutation.nodeId, + record: mutation.record, + strictLabel: mutation.strictLabel, + }, + ); if (!result.ok) throw mutationError(mutation, result); } if ( - updateSpaceRow(database, input.nextRecord, input.expectedVersion) !== 1 + updateSpaceRow( + database, + workspaceId, + input.nextRecord, + input.expectedVersion, + ) !== 1 ) { throw new Error(`SpaceWrite(${canvasId}) lost its version race`); } diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts index a48449880..b3cfe5c9d 100644 --- a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -3,7 +3,10 @@ import { SqliteStoreContext } from './database.js'; import { readSpaceRow } from './rows.js'; -import { createSqliteSpaceExtension } from './space-extension.js'; +import { + createSqliteSpaceExtension, + resolveSqliteSpaceExtension, +} from './space-extension.js'; import { createSqliteSpaceLogs } from './space-logs.js'; import { SqliteSpaceNodes } from './space-nodes.js'; import { SqliteSpaceRepository } from './space-repository.js'; @@ -11,6 +14,7 @@ import { SqliteSpaceTasks } from './space-tasks.js'; import { createSqliteSpaceWrite } from './space-write.js'; import { sanitizeId } from '../../../../utils/fs.js'; +import type { SqliteSpaceSubstrate } from './space-extension.js'; import type { StorageHealth } from '../../ports/common.js'; import type { SpaceHandle, @@ -18,24 +22,48 @@ import type { StructuredStore, } from '../../ports/structured.js'; -/** Isolated structured-store adapter backed by one node:sqlite connection. */ +/** + * Structured-store adapter over one `node:sqlite` connection. + * + * The connection may be shared with the SQLite blob store — one database file + * cannot have two writers — so this class does not assume it owns the + * lifecycle. Constructed with a filename it opens and closes its own + * connection; constructed with an existing context it borrows one, and + * `init`/`close` become the shared owner's business. + */ export class SqliteStructuredStore implements StructuredStore { readonly kind = 'sqlite' as const; readonly #context: SqliteStoreContext; + readonly #ownsContext: boolean; - constructor(filename: string, now: () => number = Date.now) { - if (typeof filename !== 'string') { + constructor( + source: string | SqliteStoreContext, + now: () => number = Date.now, + ) { + if (source instanceof SqliteStoreContext) { + this.#context = source; + this.#ownsContext = false; + return; + } + if (typeof source !== 'string') { throw new TypeError('SQLite filename must be a string'); } - if (filename.length === 0) { + if (source.length === 0) { throw new TypeError('SQLite filename must not be empty'); } - this.#context = new SqliteStoreContext(filename, now); + this.#context = new SqliteStoreContext(source, now); + this.#ownsContext = true; + } + + /** The shared connection, for composition that wires a second port on it. */ + get context(): SqliteStoreContext { + return this.#context; } async init(): Promise { - this.#context.init(); + if (this.#ownsContext) this.#context.init(); + else this.#context.assertOpen(); } async health(): Promise { @@ -43,28 +71,71 @@ export class SqliteStructuredStore implements StructuredStore { } async close(): Promise { - this.#context.close(); + if (this.#ownsContext) this.#context.close(); } spaces(): SpaceRepository { return Object.freeze(new SqliteSpaceRepository(this.#context)); } + /** + * The synchronous form of `space(canvasId).extension(namespace)`. + * + * Off the port on purpose: it is a SQLite capability, and the composition + * root hands it to owners the same way it hands out `diskTree` — named for + * the backend that has it, absent everywhere else. + */ + extensionSync( + canvasIdInput: string, + namespace: string, + ): SqliteSpaceSubstrate | null { + const canvasId = sanitizeId(canvasIdInput, 'canvasId'); + return resolveSqliteSpaceExtension( + this.#context, + this.#context.workspaceId(), + canvasId, + namespace, + ); + } + space(canvasIdInput: string): SpaceHandle { const canvasId = sanitizeId(canvasIdInput, 'canvasId'); - const { events, changes } = createSqliteSpaceLogs(this.#context, canvasId); - const nodes = Object.freeze(new SqliteSpaceNodes(this.#context, canvasId)); - const tasks = Object.freeze(new SqliteSpaceTasks(this.#context, canvasId)); + // Bound once, here, so every part of this handle answers for the same + // Workspace and a switch invalidates all of them together. + const workspaceId = this.#context.workspaceId(); + const { events, changes } = createSqliteSpaceLogs( + this.#context, + workspaceId, + canvasId, + ); + const nodes = Object.freeze( + new SqliteSpaceNodes(this.#context, workspaceId, canvasId), + ); + const tasks = Object.freeze( + new SqliteSpaceTasks(this.#context, workspaceId, canvasId), + ); return Object.freeze({ canvasId, - read: async () => - readSpaceRow(this.#context.database(), canvasId)?.record ?? null, - write: createSqliteSpaceWrite(this.#context, canvasId), + read: async () => { + this.#context.assertBoundWorkspace( + workspaceId, + `SQLite Space(${canvasId})`, + ); + return ( + readSpaceRow(this.#context.database(), workspaceId, canvasId) + ?.record ?? null + ); + }, + write: createSqliteSpaceWrite(this.#context, workspaceId, canvasId), nodes, changes, tasks, events, - extension: createSqliteSpaceExtension(this.#context, canvasId), + extension: createSqliteSpaceExtension( + this.#context, + workspaceId, + canvasId, + ), }); } } diff --git a/apps/server/src/modules/storage/backends/sqlite/test-support.ts b/apps/server/src/modules/storage/backends/sqlite/test-support.ts index 4c5af7033..50155664c 100644 --- a/apps/server/src/modules/storage/backends/sqlite/test-support.ts +++ b/apps/server/src/modules/storage/backends/sqlite/test-support.ts @@ -6,10 +6,11 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; -import { SQLITE_SCHEMA_VERSION } from './database.js'; +import { SqliteStoreContext, SQLITE_SCHEMA_VERSION } from './database.js'; import { collisionKeyForTitle } from './identity.js'; import { insertSpaceRow, parseJson } from './rows.js'; import { SqliteStructuredStore } from './structured-store.js'; +import { SqliteWorkspaceRepository } from './workspace-repository.js'; import type { CanvasFile, @@ -17,6 +18,7 @@ import type { } from '../../../canvas/persistence-types.js'; export const SQLITE_TEST_WORLD_ID = 'sqlite-test-world'; +export const SQLITE_TEST_WORKSPACE_NAME = 'Test Workspace'; export interface SqliteTestFile { readonly directory: string; @@ -24,15 +26,17 @@ export interface SqliteTestFile { readonly remove: () => void; } -export interface OpenSqliteTestStore extends SqliteTestFile { +export interface EmptySqliteTestStore extends SqliteTestFile { readonly store: SqliteStructuredStore; - readonly world: CanvasFile; + readonly context: SqliteStoreContext; + readonly workspaceId: string; + /** Drop the connection but keep the file, so a test can reopen it. */ + readonly closeConnection: () => void; readonly cleanup: () => Promise; } -export interface EmptySqliteTestStore extends SqliteTestFile { - readonly store: SqliteStructuredStore; - readonly cleanup: () => Promise; +export interface OpenSqliteTestStore extends EmptySqliteTestStore { + readonly world: CanvasFile; } export function createSqliteTestFile(prefix = 'huabu-sqlite-'): SqliteTestFile { @@ -74,6 +78,7 @@ export function withTestDatabase( */ export function seedSqliteWorld( filename: string, + workspaceId: string, canvasId = SQLITE_TEST_WORLD_ID, ): CanvasFile { const record: CanvasFile = { @@ -95,6 +100,7 @@ export function seedSqliteWorld( } insertSpaceRow( database, + workspaceId, record, collisionKeyForTitle(record.title, record.canvasId), true, @@ -103,50 +109,55 @@ export function seedSqliteWorld( return record; } -export async function openSqliteTestStore( - prefix = 'huabu-sqlite-', +/** + * Open a store on a fresh file with one activated Workspace. + * + * Every Space query is Workspace-scoped, so a store with no active Workspace + * refuses — the same way a Disk adapter refuses before a workspace path is + * committed. Tests get one activated Workspace so they can address Spaces + * without repeating the lifecycle. + */ +export async function openEmptySqliteTestStore( + prefix = 'huabu-sqlite-empty-', now?: () => number, -): Promise { +): Promise { const file = createSqliteTestFile(prefix); - const store = new SqliteStructuredStore(file.filename, now); + const context = new SqliteStoreContext(file.filename, now); + const store = new SqliteStructuredStore(context); try { - await store.init(); - const world = seedSqliteWorld(file.filename); + context.init(); + const workspace = await new SqliteWorkspaceRepository(context).create( + SQLITE_TEST_WORKSPACE_NAME, + ); + context.useWorkspace(workspace.workspaceId); return { ...file, store, - world, + context, + workspaceId: workspace.workspaceId, + closeConnection: () => context.close(), cleanup: async () => { - await store.close(); + context.close(); file.remove(); }, }; } catch (error) { - await store.close(); + context.close(); file.remove(); throw error; } } -export async function openEmptySqliteTestStore( - prefix = 'huabu-sqlite-empty-', +export async function openSqliteTestStore( + prefix = 'huabu-sqlite-', now?: () => number, -): Promise { - const file = createSqliteTestFile(prefix); - const store = new SqliteStructuredStore(file.filename, now); +): Promise { + const opened = await openEmptySqliteTestStore(prefix, now); try { - await store.init(); - return { - ...file, - store, - cleanup: async () => { - await store.close(); - file.remove(); - }, - }; + const world = seedSqliteWorld(opened.filename, opened.workspaceId); + return { ...opened, world }; } catch (error) { - await store.close(); - file.remove(); + await opened.cleanup(); throw error; } } diff --git a/apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts b/apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts new file mode 100644 index 000000000..546850701 --- /dev/null +++ b/apps/server/src/modules/storage/backends/sqlite/workspace-repository.ts @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * SQLite implementation of the Workspace storage port. + * + * A Workspace here is a row, not a folder. That is the whole difference + * between this adapter and the Disk one, and it is why selecting a SQL profile + * asks the operator for no directory: the port never promised a location, only + * an identity and a name (`ports/workspace.ts`), and the Disk repository's + * path index is a materialization fact that lives beside it rather than in it. + * + * `remove()` is a **forget**, not a delete. The port's wording is deliberate — + * "forget one member without deleting any Workspace-owned data" — and on Disk + * that is easy to honour because the folder outlives the registry entry. A + * database has no such second copy, so forgetting is recorded as a timestamp + * and the rows stay: a listing skips them, and nothing a user authored is + * destroyed by an operation whose name does not say "delete". + */ + +import { randomUUID } from 'node:crypto'; + +import { withImmediateTransaction } from './database.js'; + +import type { SqliteStoreContext } from './database.js'; +import type { + WorkspaceHandle, + WorkspaceRepository, +} from '../../ports/workspace.js'; +import type { DatabaseSync } from 'node:sqlite'; + +const WORKSPACE_COLUMNS = 'workspace_id, name, created_at, last_opened_at'; + +function decodeWorkspaceRow(value: unknown): WorkspaceHandle { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new SyntaxError('Malformed persisted SQLite Workspace row'); + } + const row = value as Record; + const workspaceId = row['workspace_id']; + const name = row['name']; + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + throw new SyntaxError('Invalid workspace_id in persisted SQLite Workspace'); + } + if (typeof name !== 'string') { + throw new SyntaxError('Invalid name in persisted SQLite Workspace'); + } + return { workspaceId, name }; +} + +function requireName(name: unknown): string { + if (typeof name !== 'string') { + throw new TypeError('Workspace name must be a string'); + } + const trimmed = name.trim(); + if (trimmed.length === 0) { + throw new TypeError('Workspace name must not be empty'); + } + return trimmed; +} + +function readWorkspaceRow( + database: DatabaseSync, + workspaceId: string, +): WorkspaceHandle | null { + const row = database + .prepare( + `SELECT ${WORKSPACE_COLUMNS} + FROM workspaces + WHERE workspace_id = ? AND forgotten_at IS NULL`, + ) + .get(workspaceId); + return row === undefined ? null : decodeWorkspaceRow(row); +} + +function insertWorkspaceRow( + database: DatabaseSync, + workspaceId: string, + name: string, + timestamp: number, +): void { + if (!Number.isFinite(timestamp)) { + throw new TypeError('SQLite Workspace clock returned a non-finite value'); + } + database + .prepare( + `INSERT INTO workspaces ( + workspace_id, name, created_at, last_opened_at, forgotten_at + ) VALUES (?, ?, ?, ?, NULL)`, + ) + .run(workspaceId, name, timestamp, timestamp); +} + +function markOpenedIn( + database: DatabaseSync, + workspaceId: string, + timestamp: number, +): void { + database + .prepare('UPDATE workspaces SET last_opened_at = ? WHERE workspace_id = ?') + .run(timestamp, workspaceId); +} + +export class SqliteWorkspaceRepository implements WorkspaceRepository { + readonly #context: SqliteStoreContext; + + constructor(context: SqliteStoreContext) { + this.#context = context; + } + + async get(workspaceId: string): Promise { + if (typeof workspaceId !== 'string' || workspaceId.length === 0) { + return null; + } + return readWorkspaceRow(this.#context.database(), workspaceId); + } + + async list(): Promise { + // Most recently opened first, matching the Disk registry's ordering, so a + // client rendering the picker gets the same list on either backend. + return this.#context + .database() + .prepare( + `SELECT ${WORKSPACE_COLUMNS} + FROM workspaces + WHERE forgotten_at IS NULL + ORDER BY last_opened_at DESC, created_at DESC`, + ) + .all() + .map(decodeWorkspaceRow); + } + + async rename( + workspaceId: string, + name: string, + ): Promise { + const trimmed = requireName(name); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if (readWorkspaceRow(database, workspaceId) === null) return null; + database + .prepare( + `UPDATE workspaces + SET name = ? + WHERE workspace_id = ? AND forgotten_at IS NULL`, + ) + .run(trimmed, workspaceId); + return readWorkspaceRow(database, workspaceId); + }); + } + + async remove(workspaceId: string): Promise { + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + if (readWorkspaceRow(database, workspaceId) === null) return false; + database + .prepare( + 'UPDATE workspaces SET forgotten_at = ? WHERE workspace_id = ?', + ) + .run(this.#context.now(), workspaceId); + return true; + }); + } + + // ─── Beyond the port ───────────────────────────────────────────────────── + // + // Creating a Workspace and recording that one was opened are lifecycle + // operations the port deliberately leaves out: on Disk they are "adopt this + // directory", which is a materialization fact. They are named for what this + // backend actually does instead of being bent into the shared shape. + + /** Register a new Workspace and return its identity. */ + async create(name: string): Promise { + const trimmed = requireName(name); + const workspaceId = randomUUID(); + insertWorkspaceRow( + this.#context.database(), + workspaceId, + trimmed, + this.#context.now(), + ); + return { workspaceId, name: trimmed }; + } + + /** + * The Workspace a fresh deployment starts in. + * + * A database nobody has opened before holds no Workspace, and a Server with + * no Workspace has nothing to show. The Disk profile answers this by asking + * the user for a folder; a SQL profile has nothing to ask for, so it starts + * one. Idempotent, and narrower than "create if absent": it mints a + * Workspace only when the database holds none at all, so forgetting the last + * one does not silently mint a second. + */ + async ensureDefault(name: string): Promise { + const trimmed = requireName(name); + const database = this.#context.database(); + return withImmediateTransaction(database, () => { + const existing = database + .prepare( + `SELECT ${WORKSPACE_COLUMNS} + FROM workspaces + WHERE forgotten_at IS NULL + ORDER BY last_opened_at DESC, created_at DESC + LIMIT 1`, + ) + .get(); + if (existing !== undefined) { + const workspace = decodeWorkspaceRow(existing); + markOpenedIn(database, workspace.workspaceId, this.#context.now()); + return workspace; + } + const workspaceId = randomUUID(); + insertWorkspaceRow(database, workspaceId, trimmed, this.#context.now()); + return { workspaceId, name: trimmed }; + }); + } + + /** Record that a Workspace was activated, for recency ordering. */ + markOpened(workspaceId: string): void { + markOpenedIn(this.#context.database(), workspaceId, this.#context.now()); + } +} diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index 127829078..1805571c1 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -27,16 +27,10 @@ const DISK: StorageProfile = { blobs: { kind: 'disk' }, }; -/** - * A profile naming a structured backend that has no adapter. - * - * The matrix has to answer for one before it exists — that is the point of - * declaring rather than discovering — so this stands in for the first backend - * that keeps Spaces in tables. - */ +/** The profile that keeps Spaces in tables and bytes in rows. */ const TABLES: StorageProfile = { structured: { kind: 'sqlite' }, - blobs: { kind: 'disk' }, + blobs: { kind: 'sqlite' }, }; describe('storage capability matrix', () => { @@ -58,7 +52,7 @@ describe('storage capability matrix', () => { expect(describeUnavailableCapabilities(DISK)).toEqual([]); }); - it('answers for a backend whose adapter is not selectable yet', () => { + it('answers for the backend that keeps Spaces in tables', () => { const missing = unavailableCapabilities(TABLES); // Every entry is Disk-only today, so a structured backend that is not @@ -75,12 +69,13 @@ describe('storage capability matrix', () => { expect(hasStorageCapability(TABLES, 'something-portable')).toBe(true); }); - it('reports capability gaps separately from profile selectability', () => { - // The matrix describes what SQLite lacks regardless of whether the - // preview can be selected. Validation rejects it at the separate - // production-readiness gate. + it('reports capability gaps without making them a misconfiguration', () => { + // The two gates are separate on purpose. A profile that offers fewer + // features is a stated limitation and must still start; only a profile + // that cannot serve at all is rejected. Conflating them would refuse a + // legitimate deployment. expect(describeUnavailableCapabilities(TABLES).length).toBeGreaterThan(0); - expect(() => validateStorageProfile(TABLES)).toThrow(/not selectable yet/); + expect(() => validateStorageProfile(TABLES)).not.toThrow(); expect(() => validateStorageProfile(DISK)).not.toThrow(); }); diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 68c412685..8f7966265 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -82,8 +82,19 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ backends: ['disk'], rationale: 'They sandbox on the Space directory. Off Disk the first-party agent ' + - 'reaches a Space over RFS/HTTP, which is what external agents already ' + - 'use.', + 'reads and writes nodes through the Canvas tools instead, which is ' + + 'the portable surface it already prefers for structured edits.', + }, + { + id: 'space-file-plane', + summary: 'Reach a Space as files over RFS, the plane external agents mount', + backends: ['disk'], + rationale: + 'RFS projects the Space directory over HTTP — the same tree, reachable ' + + 'from another machine. It is listed apart from the built-in file tools ' + + 'because it is what those tools were said to fall back to: a Space ' + + 'with no file plane has neither, and an external agent bound to a ' + + 'Space on this backend reaches it through the Canvas API.', }, { id: 'external-note-discovery', @@ -95,6 +106,34 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ 'someone writes to the store out of band, and inventing one would buy ' + 'nothing.', }, + { + id: 'workspace-directory', + summary: 'Choose, create, or reveal a Workspace folder on this machine', + backends: ['disk'], + rationale: + 'A Workspace is a folder the user picks. Where Workspaces are rows, ' + + 'there is nothing to browse to: the Server opens its own on first ' + + 'start and Workspaces are managed by name instead of by path.', + }, + { + id: 'workspace-user-memory', + summary: 'The cross-Space user memory document (setting/user.md)', + backends: ['disk'], + rationale: + 'A user-editable file at the Workspace root, deliberately outside any ' + + 'Space so it applies to all of them. Every blob scope this port has is ' + + "scoped to a Space, so there is nowhere it belongs yet; a Space's own " + + 'memory body is unaffected.', + }, + { + id: 'workspace-user-skills', + summary: 'User-authored skills under the Workspace setting/skills folder', + backends: ['disk'], + rationale: + 'Skills are read as files a user can edit and drop in by hand, which ' + + 'is the same arrival path external notes rely on. Bundled and Agent ' + + 'Team skills are unaffected.', + }, { id: 'space-directory-handle-coordination', summary: 'Windows: rename or delete a Space while a watcher holds it open', diff --git a/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts b/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts index 8467909f2..d434dc49e 100644 --- a/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts +++ b/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts @@ -42,6 +42,7 @@ const workspaceState = vi.hoisted(() => ({ path: '', leaseCount: 0 })); vi.mock('../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, acquireWorkspaceOperationLease: () => { const workspacePath = workspaceState.path; workspaceState.leaseCount += 1; diff --git a/apps/server/src/modules/storage/index.ts b/apps/server/src/modules/storage/index.ts index 0a12f64c6..9f7f5a63b 100644 --- a/apps/server/src/modules/storage/index.ts +++ b/apps/server/src/modules/storage/index.ts @@ -24,11 +24,18 @@ export { } from './compatibility/canvas.js'; export type { RenameResult, RenameSelfResult } from './compatibility/canvas.js'; +/** + * World identity, answered for whichever backend is configured. + * + * These used to come straight from the Disk directory index. They are on the + * composition root now because the World is a Space like any other and every + * backend has one — the index is just how Disk finds it. + */ export { getWorldCanvasId, isWorldCanvasId, requireWorldCanvasId, -} from './backends/disk/canvas-dirs.js'; +} from './storage.js'; /** * Materialization-tier capabilities, re-exported so consumers that need a @@ -71,6 +78,7 @@ export type { // ─── Storage ports and composition ───────────────────────────────────────── export { + activateWorkspace, adoptWorkspaceDirectory, closeStorage, composeStorage, @@ -83,14 +91,22 @@ export { getWorkspaceRepository, hasWorkspaceRegistry, initStorage, + materializesWorkspaces, setStorageForTesting, space, + sqliteDatabasePath, stageSpaceImport, storageHealth, workspaceAtDirectory, workspaceDirectory, } from './storage.js'; -export type { Space, SpaceDeleteOutcome, Storage } from './storage.js'; +export type { + Space, + SpaceDeleteOutcome, + SqliteSpaceTree, + Storage, +} from './storage.js'; +export type { SqliteSpaceSubstrate } from './backends/sqlite/space-extension.js'; export type { DiskSpaceTree } from './backends/disk/space-tree.js'; export type { DiskSpaceImport } from './backends/disk/space-import.js'; export { diff --git a/apps/server/src/modules/storage/module-boundaries.test.ts b/apps/server/src/modules/storage/module-boundaries.test.ts index aa9c3ead2..37ee0e7cb 100644 --- a/apps/server/src/modules/storage/module-boundaries.test.ts +++ b/apps/server/src/modules/storage/module-boundaries.test.ts @@ -300,6 +300,10 @@ describe('workspace module names no backend', () => { */ describe('Disk Space tree capability', () => { const EXPECTED_CONSUMERS = [ + // A — external-note discovery. The watcher asks whether this Space has a + // directory to watch at all; `null` is the whole of its behaviour off + // Disk. + 'modules/canvas/external-watcher.ts', // A — the built-in file tools' sandbox root. 'modules/agent/tools/handlers/fs-sandbox.ts', // A — bundle export. @@ -316,6 +320,25 @@ describe('Disk Space tree capability', () => { 'modules/workspace/paths.ts', ].sort(); + /** + * `sqliteTree` is the same kind of thing as `diskTree` and gets the same + * fence. It is narrower on purpose: the *only* reason it exists rather than + * the port's async `extension()` is that Agenetes's storage ports are + * synchronous, so exactly one owner should ever appear here. + */ + const EXPECTED_SQLITE_CONSUMERS = [ + 'modules/agent/agenetes/sqlite-stores.ts', + ].sort(); + + it('keeps the exact synchronous SQLite substrate census', () => { + const consumers = sourceFiles + .filter((file) => !file.startsWith('modules/storage/')) + .filter((file) => !file.endsWith('.test.ts')) + .filter((file) => /\bsqliteTree\b/.test(read(file))); + + expect(consumers.sort()).toEqual(EXPECTED_SQLITE_CONSUMERS); + }); + it('keeps the exact production consumer census', () => { // Matched as a bare word, not as `.diskTree`: destructuring the member // off a handle (`const { diskTree } = space(id)`) or reaching it by @@ -606,10 +629,7 @@ describe('root forwarding shims', () => { 'storage/canvas-dirs.js': [ 'modules/agent/tools/world-target-read.test.ts', 'modules/canvas/canvas-command-router.test.ts', - 'modules/canvas/canvas.route.ts', 'modules/canvas/external-watcher.test.ts', - 'modules/canvas/external-watcher.ts', - 'modules/canvas/world-portal-policy.ts', 'modules/canvas/world-portals.test.ts', 'modules/canvas/world-reference-resolver.test.ts', 'modules/workspace.ts', diff --git a/apps/server/src/modules/storage/ports/blob.ts b/apps/server/src/modules/storage/ports/blob.ts index 3f324dd75..b8ba498b0 100644 --- a/apps/server/src/modules/storage/ports/blob.ts +++ b/apps/server/src/modules/storage/ports/blob.ts @@ -22,7 +22,14 @@ import type { StorageHealth } from './common.js'; import type { Readable } from 'node:stream'; -export type BlobBackendKind = 'disk' | 'azure'; +/** + * Backends with a blob adapter today. + * + * Like {@link StructuredBackendKind}, this names only what exists. The wider + * vocabulary a profile may *request* — including `azure`, which is a settled + * direction with no adapter — belongs to `profile.ts`. + */ +export type BlobBackendKind = 'disk' | 'sqlite'; /** * Every area of one Space that holds bytes. diff --git a/apps/server/src/modules/storage/product-boundary.test.ts b/apps/server/src/modules/storage/product-boundary.test.ts index eea943b4b..7ef7cf767 100644 --- a/apps/server/src/modules/storage/product-boundary.test.ts +++ b/apps/server/src/modules/storage/product-boundary.test.ts @@ -347,6 +347,43 @@ forEachProductProfile((profile: StorageProfile, label: string) => { await expect(m.storage.structured.spaces().list()).resolves.toEqual([]); }); + it('serves the same Space after a restart', async () => { + const canvasId = 'space-product-restart'; + const m = await seedSpace(canvasId); + const before = m.storage.space(canvasId); + await before.artifacts.put('kept.bin', Buffer.from('durable bytes')); + await before.events.append([ + { payload: { action: 'node_created', nodes: [] }, ts: 7 }, + ]); + const record = await before.read(); + const nodes = await before.nodes.list(); + const worldId = await m.storage.structured.spaces().worldId(); + + // The restart is the point. Everything above is in whatever the backend + // calls durable; nothing about this case says which. + const storage = await m.reopen(); + + await expect(storage.structured.spaces().worldId()).resolves.toBe( + worldId, + ); + const after = storage.space(canvasId); + await expect(after.read()).resolves.toEqual(record); + // Revisions are opaque tokens, so the records are compared rather than + // the snapshots: a backend may mint a new token for the same content. + expect( + [...(await after.nodes.list())].map(([id, snapshot]) => [ + id, + snapshot.record, + ]), + ).toEqual([...nodes].map(([id, snapshot]) => [id, snapshot.record])); + expect(await after.artifacts.read('kept.bin')).toEqual( + Buffer.from('durable bytes'), + ); + await expect(after.events.read()).resolves.toEqual([ + { payload: { action: 'node_created', nodes: [] }, ts: 7 }, + ]); + }); + it('refuses to delete the World', async () => { const m = await open(); const spaces = m.storage.structured.spaces(); diff --git a/apps/server/src/modules/storage/profile.test.ts b/apps/server/src/modules/storage/profile.test.ts index c51255000..99dbf118c 100644 --- a/apps/server/src/modules/storage/profile.test.ts +++ b/apps/server/src/modules/storage/profile.test.ts @@ -35,7 +35,7 @@ describe('parseStorageProfile', () => { it('names the supported set when a kind is unknown', () => { expect(() => parseStorageProfile({ HUABU_BLOB_BACKEND: 's3' })).toThrow( - /HUABU_BLOB_BACKEND="s3".*disk, azure/s, + /HUABU_BLOB_BACKEND="s3".*disk, sqlite, azure/s, ); }); @@ -67,13 +67,34 @@ describe('validateStorageProfile', () => { ).toThrow(/not implemented yet.*disk, sqlite/s); }); - it('rejects an available preview adapter that is not selectable', () => { + it('accepts the sqlite + sqlite profile', () => { + expect(() => + validateStorageProfile({ + structured: { kind: 'sqlite' }, + blobs: { kind: 'sqlite' }, + }), + ).not.toThrow(); + }); + + // Fewer features is a stated limitation, not a misconfiguration: a + // selectable profile may lose capabilities as long as the matrix declares + // them. Only an unimplemented or incoherent pairing fails here. + it('accepts sqlite records beside disk blobs', () => { expect(() => validateStorageProfile({ structured: { kind: 'sqlite' }, blobs: { kind: 'disk' }, }), - ).toThrow(/preview adapter.*not selectable yet.*Selectable: disk/s); + ).not.toThrow(); + }); + + it('rejects sqlite blobs without the sqlite structured database', () => { + expect(() => + validateStorageProfile({ + structured: { kind: 'disk' }, + blobs: { kind: 'sqlite' }, + }), + ).toThrow(/requires HUABU_STRUCTURED_BACKEND=sqlite/); }); it('rejects a known but unimplemented blob backend', () => { @@ -82,7 +103,7 @@ describe('validateStorageProfile', () => { structured: { kind: 'disk' }, blobs: { kind: 'azure' }, }), - ).toThrow(/not implemented yet.*disk/s); + ).toThrow(/not implemented yet.*disk, sqlite/s); }); }); @@ -102,7 +123,7 @@ describe('requiresExplicitInit', () => { it.each([ { structured: { kind: 'postgres' }, blobs: { kind: 'disk' } }, - { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, + { structured: { kind: 'sqlite' }, blobs: { kind: 'sqlite' } }, { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, ] as const)('requires an awaited init for %j', (profile) => { expect(requiresExplicitInit(profile)).toBe(true); diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index e045c72cf..4662c4c49 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -10,8 +10,6 @@ * deployment, so profiles are validated before any connection is opened. */ -import type { BlobBackendKind } from './ports/blob.js'; - /** * Structured backend families a profile may name. * @@ -23,9 +21,17 @@ import type { BlobBackendKind } from './ports/blob.js'; */ export type RequestedStructuredKind = 'disk' | 'sqlite' | 'postgres'; +/** + * Blob backend families a profile may name. + * + * Wider than the port's {@link BlobBackendKind} for the same reason + * {@link RequestedStructuredKind} is wider than the structured one. + */ +export type RequestedBlobKind = 'disk' | 'sqlite' | 'azure'; + export interface StorageProfile { structured: { kind: RequestedStructuredKind }; - blobs: { kind: BlobBackendKind }; + blobs: { kind: RequestedBlobKind }; } /** Backends with an adapter implementation, selectable or otherwise. */ @@ -35,21 +41,26 @@ const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ ]; /** - * Backends whose complete capability matrix is safe for production use. + * Backends whose capability matrix is complete enough to select. * - * SQLite deliberately stays out while product composition, Blob placement, - * Disk-only capabilities, and Workspace remounting still have one authority - * only in the Disk profile. + * "Complete enough" is not "identical to Disk". A selectable profile may offer + * fewer features, as long as every one it does not offer is declared in + * `capabilities.ts` and refused where a user would reach for it. What + * disqualifies a backend is an *undeclared* gap — a feature that would fail + * with a stack trace rather than a sentence. */ -const SELECTABLE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; -const IMPLEMENTED_BLOBS: readonly BlobBackendKind[] = ['disk']; +const SELECTABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ + 'disk', + 'sqlite', +]; +const AVAILABLE_BLOBS: readonly RequestedBlobKind[] = ['disk', 'sqlite']; const STRUCTURED_KINDS: readonly RequestedStructuredKind[] = [ 'disk', 'sqlite', 'postgres', ]; -const BLOB_KINDS: readonly string[] = ['disk', 'azure']; +const BLOB_KINDS: readonly RequestedBlobKind[] = ['disk', 'sqlite', 'azure']; export class StorageProfileError extends Error { override name = 'StorageProfileError'; @@ -86,7 +97,7 @@ export function parseStorageProfile( 'HUABU_BLOB_BACKEND', env['HUABU_BLOB_BACKEND'], BLOB_KINDS, - ) as BlobBackendKind, + ) as RequestedBlobKind, }, }; } @@ -121,10 +132,21 @@ export function validateStorageProfile(profile: StorageProfile): void { `depend on Disk. Selectable: ${SELECTABLE_STRUCTURED.join(', ')}.`, ); } - if (!IMPLEMENTED_BLOBS.includes(profile.blobs.kind)) { + if (!AVAILABLE_BLOBS.includes(profile.blobs.kind)) { throw new StorageProfileError( `Blob backend "${profile.blobs.kind}" is not implemented yet. ` + - `Available: ${IMPLEMENTED_BLOBS.join(', ')}.`, + `Available: ${AVAILABLE_BLOBS.join(', ')}.`, + ); + } + // The first real cross-axis rule. SQLite blobs are rows in the structured + // database, so they have nowhere to live unless that database exists — the + // two axes stay independent in the port design, but this particular pairing + // is a single file, and saying so here beats failing at the first upload. + if (profile.blobs.kind === 'sqlite' && profile.structured.kind !== 'sqlite') { + throw new StorageProfileError( + `Blob backend "sqlite" stores bytes in the SQLite structured database, ` + + `so it requires HUABU_STRUCTURED_BACKEND=sqlite (got ` + + `"${profile.structured.kind}").`, ); } } @@ -140,7 +162,7 @@ export function validateStorageProfile(profile: StorageProfile): void { * means adding an adapter forces a decision about it. */ const LAZY_SAFE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; -const LAZY_SAFE_BLOBS: readonly BlobBackendKind[] = ['disk']; +const LAZY_SAFE_BLOBS: readonly RequestedBlobKind[] = ['disk']; /** Whether this profile may only be built through an awaited `initStorage()`. */ export function requiresExplicitInit(profile: StorageProfile): boolean { diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index dc8ede82f..f73dacea8 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -28,9 +28,11 @@ import path from 'node:path'; import { getDataDir } from '../../data-dir.js'; import { acquireWorkspaceOperationLease, - getWorkspacePath, + commitWorkspaceIdentity, + getWorkspaceKey, } from '../workspace.js'; import { DiskBlobStore } from './backends/disk/blob-store.js'; +import { getWorldCanvasId as diskWorldCanvasId } from './backends/disk/canvas-dirs.js'; import { stageDiskSpaceImport } from './backends/disk/space-import.js'; import { diskSpaceTree } from './backends/disk/space-tree.js'; import { DiskStructuredStore } from './backends/disk/structured-store.js'; @@ -38,6 +40,10 @@ import { DiskWorkspaceRepository, workspaceRegistryPath, } from './backends/disk/workspace-repository.js'; +import { SqliteBlobStore } from './backends/sqlite/blob-store.js'; +import { SqliteStoreContext } from './backends/sqlite/database.js'; +import { SqliteStructuredStore } from './backends/sqlite/structured-store.js'; +import { SqliteWorkspaceRepository } from './backends/sqlite/workspace-repository.js'; import { spaceBlobAreas } from './ports/blob.js'; import { parseStorageProfile, @@ -50,6 +56,7 @@ import { withSpacePutAdmission } from './space-lifecycle-admission.js'; import type { DiskSpaceImport } from './backends/disk/space-import.js'; import type { DiskSpaceTree } from './backends/disk/space-tree.js'; +import type { SqliteSpaceSubstrate } from './backends/sqlite/space-extension.js'; import type { BlobInfo, BlobLease, @@ -83,12 +90,20 @@ export type SpaceDeleteOutcome = | SpaceDeleteFinishResult | { readonly ok: false; readonly reason: 'world-forbidden' }; -function activeWorkspacePath(): string { - return path.resolve(getWorkspacePath()); +/** + * The active Workspace as an identity to compare, not a location. + * + * The blob put saga has to prove that the Workspace has not changed under an + * awaited operation. On Disk that comparison was the resolved path; a + * Workspace that is a row has no path, so the key is what both backends can + * answer with. + */ +function activeWorkspaceKey(): string { + return getWorkspaceKey(); } -function assertActiveWorkspace(workspacePath: string, canvasId: string): void { - if (activeWorkspacePath() !== workspacePath) { +function assertActiveWorkspace(workspaceKey: string, canvasId: string): void { + if (activeWorkspaceKey() !== workspaceKey) { throw new Error( `Blob scope for Space "${canvasId}" belongs to an inactive workspace. ` + `Resolve a fresh scope after workspace activation.`, @@ -96,6 +111,19 @@ function assertActiveWorkspace(workspacePath: string, canvasId: string): void { } } +/** + * Where the SQLite profile keeps everything it has. + * + * One file, beside the Disk backend's own registry in the data directory, so + * an operator can find both in the same place. `HUABU_SQLITE_PATH` overrides + * it for deployments that keep their database elsewhere. + */ +export function sqliteDatabasePath(dataDir: string = getDataDir()): string { + const configured = process.env['HUABU_SQLITE_PATH']?.trim(); + if (configured) return configured; + return path.join(dataDir, 'storage', 'sqlite', 'huabu.sqlite'); +} + /** * Release a rejected streaming body that storage never fully consumed. * @@ -156,6 +184,29 @@ export interface Space extends SpaceHandle, SpaceBlobs { * this module's internal topology. */ readonly diskTree: DiskSpaceTree | null; + /** + * SQLite's connection point for an extension namespace, without awaiting. + * `null` on every other backend. + * + * The same shape as {@link diskTree} and there for the same reason: a + * capability one backend has, named for it and typed by its absence. What + * makes it worth its own member rather than the port's `extension()` is + * that it is *synchronous*. An owner whose own interface is synchronous — + * the Agenetes conversation stores — can resolve its place at the moment it + * needs it instead of keeping a cache primed from somewhere else. + */ + readonly sqliteTree: SqliteSpaceTree | null; +} + +/** What a namespace can ask of the SQLite backend for one Space. */ +export interface SqliteSpaceTree { + /** + * This namespace's connection point, created on demand. + * + * `null` when the Space does not exist — the same refusal the port's + * `extension()` makes, and for the same reason. + */ + extension(namespace: string): SqliteSpaceSubstrate | null; } function composeSpace(storage: Storage, canvasId: string): Space { @@ -180,6 +231,16 @@ function composeSpace(storage: Storage, canvasId: string): Space { storage.profile.structured.kind === 'disk' ? diskSpaceTree(canvasId) : null, + sqliteTree: + storage.structured instanceof SqliteStructuredStore + ? { + extension: (namespace: string) => + (storage.structured as SqliteStructuredStore).extensionSync( + canvasId, + namespace, + ), + } + : null, }; } @@ -187,6 +248,8 @@ function buildBlobStore(profile: StorageProfile): BlobStore { switch (profile.blobs.kind) { case 'disk': return new DiskBlobStore(); + case 'sqlite': + return new SqliteBlobStore(sqliteConnection()); default: // Unreachable: validateStorageProfile rejects unimplemented kinds. throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); @@ -197,6 +260,8 @@ function buildStructuredStore(profile: StorageProfile): StructuredStore { switch (profile.structured.kind) { case 'disk': return new DiskStructuredStore(); + case 'sqlite': + return new SqliteStructuredStore(sqliteConnection()); default: throw new Error( `Unsupported structured backend: ${profile.structured.kind}`, @@ -245,9 +310,27 @@ export function createStorage(profile: StorageProfile): Storage { // ─── Process-wide holder ──────────────────────────────────────────────────── let current: Storage | null = null; -let workspaces: DiskWorkspaceRepository | null = null; +let workspaces: WorkspaceRepository | null = null; +let sqlite: SqliteStoreContext | null = null; +let activeWorldCanvasId: string | null = null; let spaceCreateTail: Promise = Promise.resolve(); +/** + * The one SQLite connection this process holds, opened on first need. + * + * Opening it is synchronous, which is why the on-demand path stays legal for + * this profile: there is no `await` to skip. Both storage axes and the + * Workspace repository borrow it, because they are one database file and a + * second connection would be a second writer. + */ +function sqliteConnection(): SqliteStoreContext { + if (sqlite) return sqlite; + const context = new SqliteStoreContext(sqliteDatabasePath()); + context.init(); + sqlite = context; + return context; +} + /** * The Workspace repository for the configured structured backend. * @@ -267,14 +350,51 @@ let spaceCreateTail: Promise = Promise.resolve(); * wired during awaited startup rather than through the on-demand path. */ export function getWorkspaceRepository(): WorkspaceRepository { - return materializedWorkspaces(); + if (workspaces) return workspaces; + const profile = activeProfile(); + workspaces = + profile.structured.kind === 'sqlite' + ? new SqliteWorkspaceRepository(sqliteConnection()) + : new DiskWorkspaceRepository(workspaceRegistryPath(getDataDir())); + return workspaces; } -/** Whether the Disk Workspace membership registry already exists on disk. */ +/** + * Whether the Disk Workspace membership registry already exists on disk. + * + * `false` off Disk, where there is no such registry to import into: the one + * caller is the deprecated desktop-store import, which is a Disk migration. + */ export function hasWorkspaceRegistry(): boolean { + if (!materializesWorkspaces()) return false; return materializedWorkspaces().hasDurableRegistry(); } +/** + * Whether the configured backend gives a Workspace a real directory. + * + * The one question the rest of the Server should ask before reaching for a + * Workspace path: everything that follows from "no" — no folder picker, no + * bundle import, no user skills directory — is a stated capability rather + * than a runtime surprise. + */ +export function materializesWorkspaces(): boolean { + return activeProfile().structured.kind === 'disk'; +} + +/** + * The profile in force, preferring the one storage was actually opened with. + * + * The environment answers before startup — managed mode adopts its Workspace + * while `app.ts` is still evaluating — but once `initStorage` has run, the + * profile it was handed is the truth. A test that mounts an explicit profile + * would otherwise get a Workspace repository for whatever the environment + * happened to say. + */ +function activeProfile(): StorageProfile { + return current?.profile ?? parseStorageProfile(); +} + /** * The Workspace repository, narrowed to a backend that materializes * Workspaces as real directories. @@ -287,18 +407,17 @@ export function hasWorkspaceRegistry(): boolean { * refuses outright rather than handing back a path that does not exist. */ function materializedWorkspaces(): DiskWorkspaceRepository { - if (workspaces) return workspaces; - - const profile = parseStorageProfile(); - if (profile.structured.kind !== 'disk') { + const repository = getWorkspaceRepository(); + if (!(repository instanceof DiskWorkspaceRepository)) { + const profile = parseStorageProfile(); throw new StorageProfileError( `The "${profile.structured.kind}" structured backend does not materialize ` + - `Workspaces as directories. Implement a locator for it before using ` + - `directory-shaped Workspace activation.`, + `Workspaces as directories, so there is no folder to adopt, reveal, or ` + + `resolve. Select the disk structured backend for directory-shaped ` + + `Workspace activation.`, ); } - workspaces = new DiskWorkspaceRepository(workspaceRegistryPath(getDataDir())); - return workspaces; + return repository; } /** @@ -318,8 +437,16 @@ export function workspaceAtDirectory( return materializedWorkspaces().at(workspacePath); } -/** The directory backing a registered Workspace, or null if it is not one. */ +/** + * The directory backing a registered Workspace, or `null` if there is none. + * + * `null` covers both "not a registered Workspace" and "this backend does not + * put Workspaces in folders". Callers already handle the first, and treating + * the second the same way is what lets a listing render on either backend + * instead of failing whole. + */ export function workspaceDirectory(workspaceId: string): string | null { + if (!materializesWorkspaces()) return null; return materializedWorkspaces().directoryOf(workspaceId); } @@ -372,12 +499,90 @@ function ensure(): Storage { export async function initStorage( profile: StorageProfile = parseStorageProfile(), ): Promise { + // Rebuild the Workspace repository against this profile: a repository + // memoized from the environment before an explicit profile was chosen would + // answer for the wrong backend. + workspaces = null; const storage = createStorage(profile); await Promise.all([storage.structured.init(), storage.blobs.init()]); current = storage; + await ensureActiveWorkspace(profile); return storage; } +/** + * Make sure a Workspace is active, for a backend that can decide by itself. + * + * On Disk the Workspace is a folder the user chooses, so the Server waits. + * Where a Workspace is a row there is nothing to choose and nothing to ask + * for: the first start creates one and activates it, and the app is usable + * without a setup step. A Workspace already activated — by managed mode, or + * by a previous call — is left alone. + */ +async function ensureActiveWorkspace(profile: StorageProfile): Promise { + if (profile.structured.kind !== 'sqlite') return; + const repository = getWorkspaceRepository(); + if (!(repository instanceof SqliteWorkspaceRepository)) return; + // The question is whether *this connection* is pointed at a Workspace, not + // whether the process remembers one. A handle left over from a previous + // profile is a name without a namespace behind it. + if (sqliteConnection().activeWorkspaceId() !== null) return; + const workspace = await repository.ensureDefault(DEFAULT_WORKSPACE_NAME); + await activateWorkspace(workspace); +} + +/** The name a SQL deployment's first Workspace is given. */ +const DEFAULT_WORKSPACE_NAME = 'Workspace'; + +/** + * Select one Workspace as the process's active namespace. + * + * Two things have to agree: the Server's own active-Workspace state and the + * namespace the backend scopes its queries to. Doing both here keeps them + * from drifting — a connection still pointed at the previous Workspace would + * answer confidently with the wrong Spaces. + */ +export async function activateWorkspace( + workspace: WorkspaceHandle, +): Promise { + if (sqlite) sqlite.useWorkspace(workspace.workspaceId); + activeWorldCanvasId = null; + commitWorkspaceIdentity(workspace); + if (workspaces instanceof SqliteWorkspaceRepository) { + workspaces.markOpened(workspace.workspaceId); + } + // A Workspace with no World has no Portal target and no home view. On Disk + // the World is written by workspace preparation; here the same step belongs + // to activation, because activation is the whole of "open a Workspace". + activeWorldCanvasId = await ensure().structured.spaces().ensureWorld(); +} + +/** + * The hidden World Space of the active Workspace, or `null` before one is + * opened. + * + * Disk answers from its directory index, which re-scans, so a Workspace edited + * from outside the app stays correct. Elsewhere the id is remembered from + * activation: it is minted once per Workspace and never changes, and reading + * it is synchronous in call sites that cannot await. + */ +export function getWorldCanvasId(): string | null { + return materializesWorkspaces() ? diskWorldCanvasId() : activeWorldCanvasId; +} + +export function requireWorldCanvasId(): string { + const canvasId = getWorldCanvasId(); + if (!canvasId) { + throw new Error('Configured workspace has no World canvas'); + } + return canvasId; +} + +export function isWorldCanvasId(canvasId: string): boolean { + const world = getWorldCanvasId(); + return world !== null && world === canvasId; +} + export function getStorage(): Storage { return ensure(); } @@ -396,10 +601,17 @@ export function getStorage(): Storage { */ export async function closeStorage(): Promise { const storage = current; + const connection = sqlite; current = null; workspaces = null; - if (!storage) return; - await Promise.all([storage.structured.close(), storage.blobs.close()]); + sqlite = null; + activeWorldCanvasId = null; + if (storage) { + await Promise.all([storage.structured.close(), storage.blobs.close()]); + } + // The shared connection outlives either store, so closing it is this + // module's job rather than whichever adapter happens to hold it. + connection?.close(); } export function getBlobStore(): BlobStore { @@ -495,7 +707,7 @@ function guardedBlobScope( canvasId: string, delegate: BlobScope, ): BlobScope { - const workspacePath = activeWorkspacePath(); + const workspaceKey = activeWorkspaceKey(); async function requireSpace(): Promise { const record = await storage.structured.space(canvasId).read(); @@ -507,16 +719,12 @@ function guardedBlobScope( return { async put(name: string, body: Readable | Buffer): Promise { try { - return await withSpacePutAdmission( - workspacePath, - canvasId, - async () => { - assertActiveWorkspace(workspacePath, canvasId); - await requireSpace(); - assertActiveWorkspace(workspacePath, canvasId); - return delegate.put(name, body); - }, - ); + return await withSpacePutAdmission(workspaceKey, canvasId, async () => { + assertActiveWorkspace(workspaceKey, canvasId); + await requireSpace(); + assertActiveWorkspace(workspaceKey, canvasId); + return delegate.put(name, body); + }); } catch (error) { drainRejectedBody(body); throw error; diff --git a/apps/server/src/modules/storage/testing.ts b/apps/server/src/modules/storage/testing.ts index a52183145..aeeeae85c 100644 --- a/apps/server/src/modules/storage/testing.ts +++ b/apps/server/src/modules/storage/testing.ts @@ -36,6 +36,7 @@ import type { Storage } from './storage.js'; */ export const PRODUCT_STORAGE_PROFILES: readonly StorageProfile[] = [ { structured: { kind: 'disk' }, blobs: { kind: 'disk' } }, + { structured: { kind: 'sqlite' }, blobs: { kind: 'sqlite' } }, ]; /** Readable name for a profile, for test titles. */ @@ -46,8 +47,24 @@ export function describeProfile(profile: StorageProfile): string { export interface MountedTestStorage { readonly profile: StorageProfile; readonly storage: Storage; - /** The temporary Workspace. Only the harness itself should name paths. */ + /** + * The temporary directory this mount owns. + * + * For a Disk profile it is the Workspace itself; for a profile that keeps + * Workspaces in a database it is only where the harness put that database. + * Either way it is the harness's own business — a case that reads it has + * stopped being evidence of anything portable. + */ readonly workspacePath: string; + /** + * Close the connections and open them again on the same durable state. + * + * What a restart actually is, for a suite that needs to prove something + * survives one. Returns the fresh {@link Storage}; the mount's own + * `storage` field still refers to the closed one, so a caller uses the + * value this returns. + */ + reopen(): Promise; close(): Promise; } @@ -67,11 +84,22 @@ export async function mountTestWorkspace( // A profile label reads as `disk/disk`, which is not a directory name. const safePrefix = prefix.replace(/[^a-zA-Z0-9._-]/g, '-'); const workspacePath = mkdtempSync(path.join(tmpdir(), safePrefix)); - // Prepares and commits the Workspace, exactly as a synchronous activation - // does. Workspace selection precedes storage here for the same reason it - // does at boot: the backend is process-wide and the Workspace is the - // namespace selected inside it. - setWorkspacePath(workspacePath); + const previousSqlitePath = process.env['HUABU_SQLITE_PATH']; + + if (profile.structured.kind === 'disk') { + // Prepares and commits the Workspace, exactly as a synchronous activation + // does. Workspace selection precedes storage here for the same reason it + // does at boot: the backend is process-wide and the Workspace is the + // namespace selected inside it. + setWorkspacePath(workspacePath); + } else { + // Nothing to pick. The Workspace is a row the backend creates on first + // start, and `initStorage` activates it — which is exactly the behaviour + // that lets this profile run with no folder at all. The temp directory + // only gives this mount its own database file so parallel suites do not + // share one. + process.env['HUABU_SQLITE_PATH'] = path.join(workspacePath, 'huabu.sqlite'); + } const storage = await initStorage(profile); // A namespace nobody has opened before has no World, and a Workspace @@ -82,8 +110,20 @@ export async function mountTestWorkspace( profile, storage, workspacePath, + async reopen(): Promise { + await closeStorage(); + if (profile.structured.kind === 'disk') setWorkspacePath(workspacePath); + const reopened = await initStorage(profile); + await reopened.structured.spaces().ensureWorld(); + return reopened; + }, async close(): Promise { await closeStorage(); + if (previousSqlitePath === undefined) { + delete process.env['HUABU_SQLITE_PATH']; + } else { + process.env['HUABU_SQLITE_PATH'] = previousSqlitePath; + } rmSync(workspacePath, { recursive: true, force: true }); }, }; diff --git a/apps/server/src/modules/workspace.route.test.ts b/apps/server/src/modules/workspace.route.test.ts index 7973ca1fa..8a8ec8bb1 100644 --- a/apps/server/src/modules/workspace.route.test.ts +++ b/apps/server/src/modules/workspace.route.test.ts @@ -23,6 +23,7 @@ vi.mock('./workspace.js', async (importOriginal) => { name: workspaceState.name, } : null, + getWorkspaceDirectory: () => workspaceState.path, getWorkspacePath: () => workspaceState.path, isManagedMode: () => workspaceState.managed, isWorkspaceConfigured: () => workspaceState.configured, diff --git a/apps/server/src/modules/workspace.route.ts b/apps/server/src/modules/workspace.route.ts index e89c4fd50..126dc5bf4 100644 --- a/apps/server/src/modules/workspace.route.ts +++ b/apps/server/src/modules/workspace.route.ts @@ -9,15 +9,20 @@ import path from 'node:path'; import { validatePathSchema, workspacePathSchema } from '@huabu/shared'; import { resetPreprocessDispatcher } from './preprocessing/index.js'; -import { getStructuredStore, resetStorageCache } from './storage/index.js'; +import { + getStructuredStore, + materializesWorkspaces, + resetStorageCache, + unavailableCapabilityMessage, +} from './storage/index.js'; import { activateWorkspacePath, WorkspaceActivationInProgressError, WorkspaceActivationTimeoutError, } from './workspace-activation.js'; import { + getWorkspaceDirectory, getWorkspaceHandle, - getWorkspacePath, isManagedMode, } from './workspace.js'; @@ -161,15 +166,21 @@ async function buildWorkspaceState(): Promise { configured, workspaceId: workspace?.workspaceId ?? null, // Free-mode active absolute path. Never exposed in managed mode. - path: workspace && !managed ? getWorkspacePath() : null, + // Null in managed mode, and null wherever a Workspace has no folder at + // all. The client already renders a Workspace with no path. + path: workspace && !managed ? getWorkspaceDirectory() : null, // Persisted display label. Safe to send in either mode. name: workspace?.name ?? null, worldCanvasId: configured ? await getStructuredStore().spaces().worldId() : null, capabilities: { - canChangeWorkspace: !managed, - nativePicker: !managed && canShowNativePicker(), + // Switching Workspaces means picking a folder in this API. A backend + // that keeps Workspaces as rows has one already open and no folder to + // offer, so the client stops showing a picker it could not honour. + canChangeWorkspace: !managed && materializesWorkspaces(), + nativePicker: + !managed && materializesWorkspaces() && canShowNativePicker(), }, }; } @@ -194,6 +205,14 @@ const workspaceRoutes: FastifyPluginAsync = async (app) => { if (isManagedMode()) { return sendError(reply, 403, 'Workspace is locked'); } + if (!materializesWorkspaces()) { + return sendError( + reply, + 409, + unavailableCapabilityMessage('workspace-directory'), + 'STORAGE_CAPABILITY_UNAVAILABLE', + ); + } if (!isLocalhost(request.ip)) { return sendError( reply, @@ -253,6 +272,14 @@ const workspaceRoutes: FastifyPluginAsync = async (app) => { 'Forbidden: workspace settings can only be changed from localhost', ); } + if (!materializesWorkspaces()) { + return sendError( + reply, + 409, + unavailableCapabilityMessage('workspace-directory'), + 'STORAGE_CAPABILITY_UNAVAILABLE', + ); + } const parsed = workspacePathSchema.safeParse(request.body); if (!parsed.success) { return sendError( diff --git a/apps/server/src/modules/workspace.ts b/apps/server/src/modules/workspace.ts index 6d898fda5..c50d4f96f 100644 --- a/apps/server/src/modules/workspace.ts +++ b/apps/server/src/modules/workspace.ts @@ -41,7 +41,10 @@ import path from 'node:path'; import { resetExternalNoteSessions } from './canvas/external-watcher.js'; import { refreshCanvasDirIndex } from './storage/canvas-dirs.js'; -import { adoptWorkspaceDirectory } from './storage/index.js'; +import { + adoptWorkspaceDirectory, + materializesWorkspaces, +} from './storage/index.js'; import { prepareWorkspaceOnDisk } from './workspace-prepare.js'; import { invalidateUserSkill } from '../prompt/index.js'; @@ -130,6 +133,17 @@ export function initWorkspaceFromEnv(): void { `${ENV_KEY} must be an absolute path, got: ${JSON.stringify(fromEnv)}`, ); } + if (!materializesWorkspaces()) { + // `HUABU_WORKSPACE` names a folder, and this backend has none. Refusing + // here rather than half-way through preparation, because the operator's + // next move is a configuration change either way — and because a SQL + // profile is already "locked at startup" without being told a path. + throw new Error( + `${ENV_KEY} names a Workspace folder, which the configured structured ` + + 'backend does not use. Unset it (the backend opens its own ' + + 'Workspace), or select the disk structured backend.', + ); + } const resolvedPath = path.resolve(fromEnv); _managed = true; prepareWorkspaceOnDisk(resolvedPath); @@ -155,6 +169,34 @@ export function getWorkspacePath(): string { return _workspacePath; } +/** + * The active Workspace's directory, or `null` when the backend has none. + * + * The honest form of {@link getWorkspacePath} for code that can cope with a + * Workspace that is a row rather than a folder. Anything that genuinely needs + * a directory should keep calling {@link getWorkspacePath} and let it refuse. + */ +export function getWorkspaceDirectory(): string | null { + return _workspacePath; +} + +/** + * A stable process-local key for the active Workspace. + * + * Leases, admission gates, and scope bindings need to say "the same Workspace + * as before" without needing it to be a place. On Disk that is still the + * resolved path, so nothing about the existing behaviour changes; elsewhere it + * is the Workspace identity. + */ +export function getWorkspaceKey(): string { + if (_workspacePath) return _workspacePath; + if (_workspaceHandle) return `workspace:${_workspaceHandle.workspaceId}`; + throw new Error( + 'Workspace has not been configured. Activate a workspace first ' + + `(PUT /api/workspace) or set ${ENV_KEY} in the environment.`, + ); +} + /** The active immutable Workspace identity, or null before configuration. */ export function getWorkspaceHandle(): WorkspaceHandle | null { return _workspaceHandle; @@ -168,7 +210,7 @@ export function getWorkspaceHandle(): WorkspaceHandle | null { * same path remains allowed. */ export function acquireWorkspaceOperationLease(): WorkspaceOperationLease { - const workspacePath = getWorkspacePath(); + const workspacePath = getWorkspaceKey(); if ( _activatingWorkspacePath !== null && @@ -302,6 +344,32 @@ function commitResolvedWorkspacePath(resolvedPath: string): void { resetExternalNoteSessions(); } +/** + * Activate a Workspace that has no directory. + * + * The counterpart to {@link commitWorkspacePath} for a backend where a + * Workspace is a row: same in-process effects — identity, cache invalidation, + * watcher reset — with nothing to resolve on the filesystem. Kept separate + * rather than making the path optional, so no caller can commit "a Workspace + * somewhere" by accident. + */ +export function commitWorkspaceIdentity(workspace: WorkspaceHandle): void { + assertNoWorkspaceActivationInProgress(); + const key = `workspace:${workspace.workspaceId}`; + if ( + _workspaceOperationLeaseCount > 0 && + _leasedWorkspacePath !== null && + _leasedWorkspacePath !== key + ) { + throw new WorkspaceOperationInProgressError(); + } + _workspaceHandle = workspace; + _workspacePath = null; + refreshCanvasDirIndex(); + invalidateUserSkill(); + resetExternalNoteSessions(); +} + /** Refresh metadata for the active Workspace without switching namespaces. */ export function updateActiveWorkspaceHandle( workspace: WorkspaceHandle, diff --git a/apps/server/src/modules/workspace/paths.ts b/apps/server/src/modules/workspace/paths.ts index 9fd1a22e5..960ca2212 100644 --- a/apps/server/src/modules/workspace/paths.ts +++ b/apps/server/src/modules/workspace/paths.ts @@ -38,7 +38,7 @@ import path from 'node:path'; -import { space } from '../storage/index.js'; +import { materializesWorkspaces, space } from '../storage/index.js'; import { getWorkspacePath } from '../workspace.js'; import type { Namespace } from '@agenetes/protocol'; @@ -60,14 +60,19 @@ const LEGACY_HISTORY_DIR_NAME = '.history'; * these paths exist only where the backend has a tree. */ function spaceRoot(canvasId: string): string { - const tree = space(canvasId).diskTree; - if (!tree) { + const directory = optionalSpaceRoot(canvasId); + if (!directory) { throw new Error( `Per-Space files for "${canvasId}" need a Space directory, which the ` + 'active structured backend does not provide.', ); } - return tree.directory(); + return directory; +} + +/** The Space's directory, or `null` where the backend has no tree. */ +function optionalSpaceRoot(canvasId: string): string | null { + return space(canvasId).diskTree?.directory() ?? null; } function legacyHistoryDir(canvasId: string): string { @@ -85,6 +90,20 @@ export function workspaceMemoryPath(): string { return path.join(settingDir(), 'user.md'); } +/** + * Whether the Workspace-level `setting/` tier exists on this backend at all. + * + * `setting/` is a folder the user edits by hand — the memory document and the + * skills they author. A Workspace that is a row has nowhere to put it, and + * that is declared as the `workspace-user-memory` and `workspace-user-skills` + * capabilities rather than emulated. Callers that merely *read* the tier ask + * here and degrade to absence; callers that write refuse with the declared + * message. + */ +export function hasWorkspaceSettingDirectory(): boolean { + return materializesWorkspaces(); +} + // ─── Workspace-level setting / user skills ───────────────────────────────── /** @@ -130,8 +149,16 @@ export function acpSessionsPath(canvasId: string): string { * empty-canvasId no-op). See docs/proposals/layered-architecture.md §7 M5.0. */ export function canvasAcpNamespace(canvasId: string): Namespace { - return { - name: canvasId, - storage: canvasId ? { root: legacyHistoryDir(canvasId) } : undefined, - }; + if (!canvasId) return { name: canvasId }; + // `storage.root` is a *directory*, so it is present exactly when the Space + // has one. Omitting it is not a degraded namespace: it is how the + // conversation stores learn that this Space keeps its threads somewhere + // other than a folder (`agent/agenetes/conversation-stores.ts`). + const root = optionalSpaceRoot(canvasId); + return root === null + ? { name: canvasId } + : { + name: canvasId, + storage: { root: path.join(root, LEGACY_HISTORY_DIR_NAME) }, + }; } diff --git a/apps/server/src/modules/workspaces.route.test.ts b/apps/server/src/modules/workspaces.route.test.ts index f1af8821b..f399ca455 100644 --- a/apps/server/src/modules/workspaces.route.test.ts +++ b/apps/server/src/modules/workspaces.route.test.ts @@ -44,6 +44,7 @@ const testState = vi.hoisted(() => ({ const storageMocks = vi.hoisted(() => ({ resetStorageCache: vi.fn(), + activateWorkspace: vi.fn(async () => {}), })); const activationMocks = vi.hoisted(() => ({ @@ -141,9 +142,15 @@ const locatorMocks = vi.hoisted(() => ({ })); vi.mock('./storage/index.js', () => ({ + activateWorkspace: storageMocks.activateWorkspace, getWorkspaceRepository: () => repository, hasWorkspaceRegistry: () => testState.registryInitialized, + // These routes are the directory-shaped Workspace API, so the profile under + // test is the one that has directories. The non-materializing branches are + // covered where they are the point. + materializesWorkspaces: () => true, resetStorageCache: storageMocks.resetStorageCache, + unavailableCapabilityMessage: (id: string) => `capability ${id}`, adoptWorkspaceDirectory: locatorMocks.adoptWorkspaceDirectory, ensureWorkspaceManifestOnDisk: locatorMocks.ensureWorkspaceManifestOnDisk, workspaceAtDirectory: locatorMocks.workspaceAtDirectory, @@ -156,6 +163,7 @@ vi.mock('./workspace.js', () => ({ testState.active = locatorMocks.adoptWorkspaceDirectory(workspacePath); testState.activePath = workspacePath; }, + getWorkspaceDirectory: () => testState.activePath, getWorkspaceHandle: () => testState.active, getWorkspacePath: () => { if (!testState.activePath) throw new Error('No active Workspace path'); diff --git a/apps/server/src/modules/workspaces.route.ts b/apps/server/src/modules/workspaces.route.ts index 4adcba1bd..03fbb622b 100644 --- a/apps/server/src/modules/workspaces.route.ts +++ b/apps/server/src/modules/workspaces.route.ts @@ -10,11 +10,14 @@ import { workspaceCreateSchema, workspaceRenameSchema } from '@huabu/shared'; import { migrateLegacyDesktopWorkspaceStore } from './legacy-desktop-workspace-store.js'; import { resetPreprocessDispatcher } from './preprocessing/index.js'; import { + activateWorkspace, adoptWorkspaceDirectory, ensureWorkspaceManifestOnDisk, getWorkspaceRepository, hasWorkspaceRegistry, + materializesWorkspaces, resetStorageCache, + unavailableCapabilityMessage, workspaceAtDirectory, workspaceDirectory, workspaceIdentityOnDisk, @@ -27,6 +30,7 @@ import { } from './workspace-activation.js'; import { commitWorkspacePath, + getWorkspaceDirectory, getWorkspaceHandle, getWorkspacePath, isManagedMode, @@ -84,10 +88,16 @@ function rejectReadOnlyMutation( function descriptor(workspace: WorkspaceHandle): WorkspaceDescriptor { const workspacePath = workspaceDirectory(workspace.workspaceId); const activeHandle = getWorkspaceHandle(); + const activeDirectory = getWorkspaceDirectory(); + // Identity decides which Workspace is active. On Disk the location has to + // agree as well, because a registered id can be pointed at a folder the + // process is not the one serving; where there is no folder, there is + // nothing else to agree. const active = activeHandle?.workspaceId === workspace.workspaceId && - workspacePath !== null && - path.resolve(getWorkspacePath()) === path.resolve(workspacePath); + (workspacePath === null || activeDirectory === null + ? !materializesWorkspaces() + : path.resolve(activeDirectory) === path.resolve(workspacePath)); return { workspaceId: workspace.workspaceId, name: workspace.name, @@ -186,7 +196,14 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { * behind a preparation fork each. */ function importLegacyDesktopStore(): void { - if (legacyDesktopStoreImported || isManagedMode() || hasWorkspaceRegistry()) + if ( + legacyDesktopStoreImported || + isManagedMode() || + // The deprecated store remembers folders, so there is nothing to import + // where a Workspace is not one. + !materializesWorkspaces() || + hasWorkspaceRegistry() + ) return; const filePath = process.env.HUABU_LEGACY_WORKSPACE_STORE?.trim(); if (!filePath) return; @@ -209,6 +226,17 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { app.post<{ Body: WorkspaceCreateRequest }>('/', async (request, reply) => { const rejected = rejectReadOnlyMutation(request, reply); if (rejected) return rejected; + if (!materializesWorkspaces()) { + // Creating a Workspace here means adopting a folder. Where Workspaces + // are rows the Server opens its own, and adding more of them is a + // by-name operation this API does not have yet. + return sendError( + reply, + 409, + unavailableCapabilityMessage('workspace-directory'), + 'STORAGE_CAPABILITY_UNAVAILABLE', + ); + } const parsed = workspaceCreateSchema.safeParse(request.body); if (!parsed.success) { @@ -284,10 +312,23 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { if (typeof parsedId !== 'string') return parsedId; const workspace = await getWorkspaceRepository().get(parsedId); - const workspacePath = workspaceDirectory(parsedId); - if (!workspace || !workspacePath) { - return sendError(reply, 404, 'Workspace not found'); + if (!workspace) return sendError(reply, 404, 'Workspace not found'); + + // A Workspace that is a row needs no preparation: activation is + // re-scoping the connection, which is why this profile can switch + // Workspaces without a folder to prepare or a child process to fork. + if (!materializesWorkspaces()) { + try { + await activateWorkspace(workspace); + resetPreprocessDispatcher(); + return reply.send(descriptor(getWorkspaceHandle() ?? workspace)); + } catch (error) { + return sendPreparationError(reply, error); + } } + + const workspacePath = workspaceDirectory(parsedId); + if (!workspacePath) return sendError(reply, 404, 'Workspace not found'); try { await activateWorkspacePath(workspacePath); resetStorageCache(); From bf505369a9afcff4b2a89683caa261054678a117 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Fri, 4 Sep 2026 19:28:35 +0800 Subject: [PATCH 07/15] docs(storage): record what the SQLite profile serves and what it does not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proposal's §12.9 described a preview that could not be selected, and the architecture doc had one Disk layout. Both now describe a second profile an operator can actually choose, which means the interesting half is the list of things it does not do. §12.9 is rewritten around that: the file it keeps everything in, the schema and the two decisions inside it that are not obvious (blobs carry no foreign key to `spaces`; encoding follows `JSON.stringify` because Disk does), the one synchronous exception the extension substrate needed and why it has a one-consumer census, and a table of every capability the profile gives up with the reason it is not emulated. Two further limits are named without being capability rows, because nothing refuses them: blob bytes are read and written whole, and one process holds one connection. The architecture doc gains the SQLite layout beside the Disk one — the tables, what an operator needs to know about the shared connection and the row-shaped Workspace, and the same list of what is unavailable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Gm37mhSirkohJJcnLR4SWs --- docs/architecture/canvas-storage.md | 44 ++++- docs/proposals/multi-backend-storage.md | 225 +++++++++++++++++------- 2 files changed, 199 insertions(+), 70 deletions(-) diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 221ecb125..3a0187248 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -1,6 +1,6 @@ # Canvas Storage Architecture -> Last updated: 2026-08-24 +> Last updated: 2026-09-04 ## 1. Overview @@ -59,8 +59,8 @@ Key points: - `space(canvasId)` is the one entry point to a Space. It is a composition-layer facade, not a port type: `StructuredStore` and `BlobStore` never import each other, and they are joined only where the cross-store rules already live — the blob-put precondition and the blob-first delete saga. It composes from its receiver, so substituting one axis on a `Storage` object yields Spaces built on the substitute. - A capability only one backend has hangs off that same handle, named for the backend and typed by its absence rather than stubbed to throw: `diskTree` is the Disk Space directory and is `null` on every other backend. It is not a port and does not live in `ports/`. `module-boundaries.test.ts` holds its exact production consumer census — a list that may shrink and must not grow — and asserts the barrel exposes nothing that reads as a portable path API. - A Space's bytes are reached the same way as its records: `BlobStore.space(id)` returns one member per user-visible area — `artifacts`, `guide`, `memory`, `uploads` — so the Disk paths a user sees are unchanged and retention can diverge later without moving bytes. The `guide` area is bounded by its member names rather than by a directory, because its area is the Space root: a directory scope there would let `list()` claim `space.json` and `deleteAll()` remove the Space. Rename and per-key delete remain unsupported. -- `space(canvasId).extension(namespace)` hands an owner an isolated place to keep its own per-Space state — a reserved directory on Disk — and nothing else. Storage validates the namespace, creates it on demand, and destroys it with the Space, which is the one operation an owner cannot perform itself; it guarantees nothing about the contents, and cannot, because it never sees them. `extension()` returns `null` for a Space that is gone, which is where the per-owner `existsSync` resurrection guards went. Memory-worker bookkeeping and the debug prompt log are its first two owners; ACP session state is assigned here but moves with the Agenetes `Namespace` change. -- Features that are _about_ a filesystem are declared, not emulated. `capabilities.ts` lists bundle export and import, reveal-in-file-manager, the built-in file tools, external-note discovery, and Windows directory-handle coordination as Disk-only; startup logs the ones the selected profile does not offer and each refusal reuses that same wording. An unavailable feature is a stated limitation and startup continues, while a profile naming an unimplemented backend stays a misconfiguration that fails fast. +- `space(canvasId).extension(namespace)` hands an owner an isolated place to keep its own per-Space state — a reserved directory on Disk, a shared connection plus a Space-owned parent row on SQLite — and nothing else. Storage validates the namespace, creates it on demand, and destroys it with the Space, which is the one operation an owner cannot perform itself; it guarantees nothing about the contents, and cannot, because it never sees them. `extension()` returns `null` for a Space that is gone, which is where the per-owner `existsSync` resurrection guards went. Memory-worker bookkeeping, the debug prompt log, and the Agenetes conversation stores are its owners; ACP session state is assigned here but moves with the Agenetes `Namespace` change. Because Agenetes's storage ports are synchronous and `extension()` is not, the composition root also exposes `sqliteTree` — the synchronous form of the same resolution, named for the backend that has it and `null` elsewhere, with its own single-consumer census beside `diskTree`'s. +- Features that are _about_ a filesystem are declared, not emulated. `capabilities.ts` lists Workspace folder selection, bundle export and import, reveal-in-file-manager, the built-in file tools, RFS's file plane, external-note discovery, the Workspace memory document, user-authored skills, and Windows directory-handle coordination as Disk-only; startup logs the ones the selected profile does not offer and each refusal reuses that same wording. An unavailable feature is a stated limitation and startup continues, while a profile naming an unimplemented backend stays a misconfiguration that fails fast. Fewer features is therefore not a reason to make a backend unselectable — an _undeclared_ gap is. - `closeStorage()` closes both connections on graceful Server shutdown and forgets the holder. On Disk it releases nothing a process exit would not, and it exists for the backend that will hold a pool. - `SpaceRepository.ensureWorld()` is the backend-neutral World bootstrap: it returns the established World or mints exactly one version-0 World when the namespace holds none. An _established_ World that is missing or malformed stays the integrity error `worldId()` reports, because regenerating identity there would orphan every reference to it. Disk delegates to the same idempotent primitive Workspace preparation calls, so one file keeps one writer. - An ordinary Space **directory name** is derived from its title via `toSafeFilename(title)`, not from `canvasId`. The stable `canvasId` only lives inside `space.json`; the World is the reserved `.world` exception. @@ -74,15 +74,49 @@ Key points: - Canonical World preview identity is server-owned: non-system commands cannot create, repoint, or delete managed previews. Users may move and resize them. Ordinary Spaces may create and delete their own `spacePreview` nodes through normal UI commands. - Legacy `canvasRef`, `frameRef`, `nodeRef`, `SET_PORTAL_NODE_PINS`, and `GET /api/canvas/:worldCanvasId/references` remain compatibility surfaces for stored World data but are no longer created or exposed by the redesigned World UI. The current model is specified in [space-preview.md](./space-preview.md). - Node filenames are `safe(label).md`; the node's stable id lives in the `id:` frontmatter field. -- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Artifacts are one of four blob areas a Space has, resolved as `space(canvasId).artifacts`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Only the Disk blob and structured backends are implemented and selectable today. +- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Artifacts are one of four blob areas a Space has, resolved as `space(canvasId).artifacts`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Disk and SQLite are both implemented and selectable; `blobs=sqlite` requires `structured=sqlite`, because those bytes are rows in that same database. - Remote PDF preprocessing writes the already-fetched source bytes into the Space BlobStore as `artifact-.pdf` before structured persistence and replaces the node's remote `src` with that key. As with other artifact imports, this blob write precedes the node write operation; a later structured persistence failure may therefore leave an unreferenced blob until Space deletion, while a blob-write failure degrades to retaining the remote URL. - Events are append-only JSONL (`events.jsonl`); each line is `{ ts: number, payload: RecentAction }`. - The memory analyzer reads Space existence and at most 100 recent action events through one `SpaceHandle`. A missing Space skips the pass before reading memory files or calling the model; corrupt part data still fails the pass. Memory body/state files remain materialized workspace paths, while Agenetes-owned chat history is not part of the curator bundle. -- **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** The canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. +- **Chat history is Chat-V2, owned by Agenetes L2 — not `CanvasStore`.** Which store owns it depends on where the Space lives: a namespace carrying a `storage.root` (Disk) uses the file stores described below, a Space in SQLite uses the `agenetes_*` tables, and an unnamed namespace stays in memory as Agenetes intends. On Disk the canonical per-thread conversation is a two-tier append-only log under `chat_v2/`: Tier-1 `.events.jsonl` (`AgentStreamEvent` deltas a running turn appends, written by `FileEventLogStore`) and Tier-2 `.turns.jsonl` (folded `AgentTurn`s, written by `FileTurnStore` — the only tier `history()` reads back). These files sit under the canvas `.history/` only because it is the Agenetes namespace `storage.root` (`canvasAcpNamespace(canvasId)`); `CanvasStore` never touches them. Do **not** confuse `chat_v2/.events.jsonl` (agent stream events) with the sibling `events.jsonl` (canvas action log) — same suffix, unrelated content. - Durable Agenetes workload records live in `.history/threads.json` (`agenetes-v2` schema, one record per thread; written by `FileThreadStore`). The host-local `namespace.storage.root` is never persisted: reads bind each record to the current Space namespace, so a Home synchronized across computers cannot redirect storage back to another machine's absolute path. - Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`/`runs.complete`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. - Legacy chat files are one-way migrated into `chat_v2/` at workspace activation and retired to `.bak`: the oldest pi-ai `Context` `chat/.json` via `migrate-chat-threads.ts` (hop 1), then the M5.6 `chat/.turns.jsonl` / `.active.json` via `migrate-chat-turns.ts` (hop 2). If hop 1 finds both formats after an interrupted launch, it completes a strict converted prefix atomically or preserves an existing tail when the full conversion is its prefix. Divergent logs are retained rather than guessed or overwritten; hop 2 skips the paired turn log while a valid same-thread legacy Context remains or its JSON cannot be read safely, so a later activation can retry both copies without blocking unrelated migrations. The obsolete `CanvasStore` chat methods and `chatPath()` helper were removed in Phase 2; `chatDir()` remains because change-review and agent-owned files still use that directory. +## 2b. SQLite layout — the profile with no folders + +`HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` selects the second implemented profile. It needs **no Workspace folder and no Space directories**: every durable thing is a row in one file. + +``` +/ + storage/sqlite/ + huabu.sqlite # everything below; override with HUABU_SQLITE_PATH + huabu.sqlite-wal # WAL sidecars, managed by SQLite + huabu.sqlite-shm +``` + +| Table | Holds | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `workspaces` | Workspace identity and display name; `forgotten_at` is how `remove()` forgets a member without destroying what it owns | +| `spaces` | One row per Space, scoped by `workspace_id`; `is_world` marks the hidden World, and a partial unique index keeps it to one per Workspace | +| `nodes` | Complete node JSON plus its opaque revision token and de-duplicated label key | +| `events` | The Canvas action log, ordered by an autoincrement id | +| `changes` | One coalesced change-review snapshot per `(Space, thread)` | +| `tasks` | The versioned Task/Run snapshot | +| `delta_log` | The executor's private journal, keyed by committed Space version | +| `space_extensions` | One row per extension namespace — the parent an owner's own tables cascade from | +| `blobs` | Artifact, guide, memory, and upload bytes, keyed by `(workspace, canvas, area, name)` | + +Owner-created tables hanging off `space_extensions`: `extension_documents` (memory bookkeeping and the debug prompt log) and `agenetes_threads` / `agenetes_events` / `agenetes_turns` (the conversation stores). Storage never reads them; deleting a Space removes them by cascade. + +Notes an operator needs: + +- **A Workspace is a row.** There is no folder to pick, so the Server creates and activates one on first start and reports `path: null` with `canChangeWorkspace: false`; the client shows no picker. Switching Workspaces re-scopes the one connection and reopens nothing. +- **The connection is shared.** The structured store, the blob store, and the Workspace repository use one `node:sqlite` connection, opened in WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys enforced. One process, one connection: nothing here promises a multi-process fence. +- **Blob bytes are rows**, read and written whole. The profile is sized for the documents and images a Space holds, not arbitrarily large media, and the database grows to the size of everything ever uploaded. `materialize()` spools to the OS temp directory and unlinks on release. +- **`blobs` has no foreign key to `spaces`.** Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and that saga must also be able to sweep orphans for a record that is already missing. +- **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, user-authored skills under `setting/skills/`, and Windows directory-handle coordination. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. + ## 3. Storage composition and ownership `apps/server/src/modules/storage/` has three layers plus its composition root: diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 159e967cc..12434bcc6 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -1,6 +1,6 @@ # Multi-Backend Storage -Status: Phases 1–5 implemented; SQLite remains a contract preview +Status: Phases 1–5 implemented; SQLite is a selectable profile Last updated: 2026-09-04 > **Scope and decision confidence.** This proposal records the two-port @@ -59,10 +59,14 @@ Last updated: 2026-09-04 > harness, **implemented**). §12 is the authoritative plan; > the decision table in §2 marks what each step has actually settled. > -> Phase 5 is specified in §12.9 and is **implemented by this branch** as an -> isolated SQLite structured-store preview. It exercises the portable -> contracts with real SQLite files but is deliberately absent from runtime -> composition; Postgres and Azure adapters do not exist. +> Phase 5 is specified in §12.9 and is **implemented by this branch**. +> `HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` is a real +> profile: Workspaces, Spaces, nodes, logs, Tasks, blob bytes, and agent +> conversations all live in one database file under +> `/storage/sqlite/`, and the deployment needs no Workspace folder +> and no Space directories. What it does **not** serve is enumerated in +> §12.9.4 and declared in `storage/capabilities.ts`, which is the list an +> operator sees at startup. Postgres and Azure adapters still do not exist. --- @@ -89,24 +93,24 @@ built above these ports, but its form is intentionally unresolved here. ## 2. Decision status -| Topic | Status | Current position | -| ------------------------------------------------------ | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | -| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk is selectable; SQLite has an isolated contract-preview adapter but is not selectable; Postgres has no adapter. | -| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations. Only Disk exists. | -| Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | -| Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | -| Concrete interface shape and async migration | **Accepted** (P4) | Blob and portable structured repositories are async. `StructuredStore` exposes catalogue/lifecycle and scoped Space handles; the structured mutations enumerated in §12.4 use those ports. Disk-only physical capabilities remain explicit blockers for selecting another profile. | -| Exact structured repositories and aggregate boundaries | **Accepted minimum** (P4) | Catalogue, lifecycle, Space CAS, nodes, four Canvas-log families, Tasks, and the ordered writer have reusable contracts. A rejected in-process node → record → optional-delta batch restores prestate; explicit title rename remains an earlier best-effort boundary. Crash recovery, unknown remote outcomes, idempotency, publication, and multi-process serialization are not promised. | -| Node Markdown ownership | **Accepted** (P4) | Authored node content remains with structured node records because it participates in revision CAS, search, and node mutation. Opaque and large bytes remain in BlobStore. | -| Blob key, staging, deletion, and GC semantics | Proposed / open | Names are the existing `` keys; `deleteAll()` covers Space destruction. Staging, reference counting, and GC remain undesigned. Per-key deletion stays out of the public port, but the absence of any cleanup path is what makes atomic replace mandatory (§6.2). | -| Space-handle identity and caching | **Corrected** (P1) | `space(id)` returning a stable handle is bounded by the LRU behind it, not guaranteed. In-memory tombstones and the filename index are therefore adapter-local caches, never durable state (§12.1.1, §12.2.4). | -| Reaching one Space | **Accepted** (§12.6) | One `space(canvasId)` facade on the composition root joins both ports; the two ports keep their independence and are joined only where the cross-store rules already live. A capability only one backend has hangs off the same handle, named for that backend and typed by its absence — `diskTree`, `null` elsewhere (§6.4.1). | -| Residual per-Space files | **Accepted** (§12.8) | Four dispositions, not one: Disk-only and declared, portable and re-implemented, structured record, or blob (§6.4.2). Every current consumer is assigned in §6.4.3 and the ones that pay for themselves on Disk are built; what a second backend must pay for is named, not deferred silently. | -| Backend selection scope | **Accepted** | Backend selection and its connection/pool are process-global. Workspaces are namespaces inside the configured backend; activating another Workspace re-scopes repository/handle operations without dropping or reconnecting the backend. A SQL profile serves every Workspace through one live connection/pool. | -| Logical filesystem view | Open | A possible `SpaceFileView` above both stores; name and contract are not accepted yet. | -| Real agent workspace | Open | Materialized directory, OS mount, protocol-only access, or a combination remain under evaluation. | -| Agent-authored filesystem write-back | Open | Read-only projection, explicit checkout/commit, and live bidirectional sync are alternatives, not decisions. | +| Topic | Status | Current position | +| ------------------------------------------------------ | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | +| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk is selectable; SQLite has an isolated contract-preview adapter but is not selectable; Postgres has no adapter. | +| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations. Only Disk exists. | +| Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | +| Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | +| Concrete interface shape and async migration | **Accepted** (P4) | Blob and portable structured repositories are async. `StructuredStore` exposes catalogue/lifecycle and scoped Space handles; the structured mutations enumerated in §12.4 use those ports. Disk-only physical capabilities remain explicit blockers for selecting another profile. | +| Exact structured repositories and aggregate boundaries | **Accepted minimum** (P4) | Catalogue, lifecycle, Space CAS, nodes, four Canvas-log families, Tasks, and the ordered writer have reusable contracts. A rejected in-process node → record → optional-delta batch restores prestate; explicit title rename remains an earlier best-effort boundary. Crash recovery, unknown remote outcomes, idempotency, publication, and multi-process serialization are not promised. | +| Node Markdown ownership | **Accepted** (P4) | Authored node content remains with structured node records because it participates in revision CAS, search, and node mutation. Opaque and large bytes remain in BlobStore. | +| Blob key, staging, deletion, and GC semantics | Proposed / open | Names are the existing `` keys; `deleteAll()` covers Space destruction. Staging, reference counting, and GC remain undesigned. Per-key deletion stays out of the public port, but the absence of any cleanup path is what makes atomic replace mandatory (§6.2). | +| Space-handle identity and caching | **Corrected** (P1) | `space(id)` returning a stable handle is bounded by the LRU behind it, not guaranteed. In-memory tombstones and the filename index are therefore adapter-local caches, never durable state (§12.1.1, §12.2.4). | +| Reaching one Space | **Accepted** (§12.6) | One `space(canvasId)` facade on the composition root joins both ports; the two ports keep their independence and are joined only where the cross-store rules already live. A capability only one backend has hangs off the same handle, named for that backend and typed by its absence — `diskTree`, `null` elsewhere (§6.4.1). | +| Residual per-Space files | **Accepted** (§12.8) | Four dispositions, not one: Disk-only and declared, portable and re-implemented, structured record, or blob (§6.4.2). Every current consumer is assigned in §6.4.3 and the ones that pay for themselves on Disk are built; what a second backend must pay for is named, not deferred silently. | +| Backend selection scope | **Accepted** (§12.9) | Backend selection and its connection/pool are process-global. Workspaces are namespaces inside the configured backend; activating another Workspace re-scopes repository/handle operations without dropping or reconnecting the backend. Implemented for SQLite: one connection, a `workspace_id` on every Space, and a retained handle that refuses after a switch rather than answering for the new namespace. | +| Logical filesystem view | Open | A possible `SpaceFileView` above both stores; name and contract are not accepted yet. | +| Real agent workspace | Open | Materialized directory, OS mount, protocol-only access, or a combination remain under evaluation. | +| Agent-authored filesystem write-back | Open | Read-only projection, explicit checkout/commit, and live bidirectional sync are alternatives, not decisions. | ## 3. Current system @@ -182,9 +186,11 @@ and Azure Blob adapters do not yet exist. their product semantics are defined. - Implementing online backend migration, replication, backup, or disaster recovery. -- Making a non-Disk adapter runtime-selectable. Phase 5 proves an isolated - adapter against the contracts without registering it in composition or - changing product capabilities. +- Making **every** feature portable. Phase 5 makes a non-Disk adapter + selectable, which is a different claim: a selectable profile may serve fewer + features, as long as each missing one is declared in `capabilities.ts` and + refused in those words where a user reaches for it (§12.9.4). Emulating a + filesystem so that a file-shaped feature _nearly_ works stays out of scope. ## 6. Settled backend split and implemented minimum contracts @@ -2527,36 +2533,56 @@ portable export format, a writable general-purpose virtual filesystem or OS mount, protocol or UI changes, and stronger crash/distributed transaction guarantees. -### 12.9 Phase 5 — SQLite contract preview — **implemented** +### 12.9 Phase 5 — SQLite as a selectable profile — **implemented** -Phase 5 adds one non-Disk structured adapter to test whether the boundary -survives a database implementation. It is an isolated implementation and test -target, not a product profile. The composition root does not construct or -export it, and `HUABU_STRUCTURED_BACKEND=sqlite` continues to fail during -profile validation with a preview-specific diagnostic. +Phase 5 adds a second structured backend and a second blob backend, and turns +them on. The question it answers is not "does the boundary compile against a +database" — §12.8's harness already asked that — but the harder one behind it: +can a deployment run with **no Workspace folder and no Space directories at +all**, and can it say plainly what it gives up by doing so. + +`HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` is the profile. +Everything durable — Workspaces, Spaces, nodes, events, changes, Tasks, blob +bytes, extension namespaces, and agent conversations — lives in one file at +`/storage/sqlite/huabu.sqlite` (override with `HUABU_SQLITE_PATH`), +beside the Disk backend's own registry at `/storage/disk/`. Postgres +and Azure Blob adapters still do not exist. #### 12.9.1 Scope and lifecycle -- The adapter uses built-in `node:sqlite`, owns one explicit database filename - and connection, and adds no package or native-addon dependency. -- Retained handles stay bound to that connection. `init`, `health`, and - `close` are real lifecycle operations; Workspace remounting and production - factory registration remain selectability work. -- The current portable surface is implemented: Space listing/lifecycle and - `ensureWorld`, record read/write, node read/readMany/list/stream and - mutations, events, changes, Tasks/Runs including atomic completion, and the - extension substrate. -- Postgres, Azure Blob, Disk-to-SQLite migration, RFS/file tools, external-note - watching, import/export, client/API changes, and product UI remain outside - this phase. +- Built-in `node:sqlite`. No package, no native addon. That is not a + production driver decision (§5); it is what let this phase be about the + boundary rather than about dependencies. +- One connection per process, shared by the structured store, the blob store, + and the Workspace repository — because they are one file, and two writers to + one SQLite file is a lock error rather than a queue. The connection opens in + WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys + enforced. +- Opening it is _synchronous_, so the composition root can hand out a + Workspace repository before `initStorage()` has been awaited — which managed + mode needs, and which is the reason the profile does not have to weaken the + `requiresExplicitInit` guard for anyone else. +- A Workspace is a **row**. There is no folder to pick, so the first start + creates one and activates it; `activateWorkspace` re-points the connection's + namespace and reopens nothing. Handles bind the Workspace they were resolved + in and refuse afterwards, exactly as the Disk adapters refuse a retained + workspace path. #### 12.9.2 Schema and behavior Schema versioning uses `PRAGMA user_version`; migrations run transactionally, reject databases from the future, and create `STRICT` tables with foreign keys -enabled. Version 1 stores Space records and World membership, complete node -JSON with opaque revision tokens, ordered events, coalesced changes, -Task/Run snapshots, extension namespaces, and the private delta journal. +enabled. Version 1 holds Workspaces, Space records and World membership, +complete node JSON with opaque revision tokens, ordered events, coalesced +changes, Task/Run snapshots, extension namespaces, the private delta journal, +and blob bytes. + +Blobs deliberately carry **no** foreign key to `spaces`. The two ports are +configured independently and their lifecycles are joined only by the deletion +saga in `storage.ts`, which sweeps every blob area _before_ the structured +record goes; a foreign key would move that ordering decision into the schema +and would refuse the orphan sweep the saga performs when a record is already +missing. Every ordered Space write applies node mutations, record replacement, and the optional delta insert in one immediate transaction. Same-baseline writers have @@ -2573,25 +2599,94 @@ old row cannot match the replacement. `write-suppressed` and `authoritativeInsert` remain valid adapter-specific parts of the common shape for Disk rather than requirements every SQL adapter must reproduce. -The extension substrate returns the shared connection plus a stable, -Space-owned namespace id. Owner tables can reference that id with -`ON DELETE CASCADE`, preserving namespace isolation and cleanup without -putting a generic key/value API in the storage port. - -#### 12.9.3 Proof - -The reusable structured contracts run against Disk and real temporary SQLite -files. They cover fresh World bootstrap, store lifecycle, Space deletion -admission, node CAS and read shapes, ordered transactional writes, event/change -ordering, Task/Run completion, and extension isolation and cleanup. SQLite -integration tests additionally cover strict schema creation, close/reopen +Value encoding follows `JSON.stringify`, because Disk persists through that +same function. An `undefined` own property is dropped rather than rejected; +what is genuinely unrepresentable — a cycle, a non-finite number, a non-plain +object — still rejects. The alternative was a record Disk accepts and SQLite +refuses, which is §13's silent-divergence risk in its most ordinary form: an +optional field spread onto a node. + +`remove()` on the Workspace repository is a **forget**, not a delete. The +port's wording is "forget one member without deleting any Workspace-owned +data", which Disk honours for free because the folder outlives the registry +entry. A database has no second copy, so forgetting is a timestamp and the +rows stay. + +#### 12.9.3 The extension substrate, and one synchronous exception + +The substrate is §6.4.4 as written: the port hands a namespace a connection +plus a Space-owned parent row, owner tables reference it with +`ON DELETE CASCADE`, and storage never sees what is in them. Three owners use +it — memory bookkeeping, the debug prompt log, and the Agenetes conversation +stores. + +The conversation stores forced one addition. Agenetes's three storage ports +(thread table, Tier-1 event log, Tier-2 folded turns) are **synchronous**, and +the port's `extension()` is async. Rather than have that owner keep a cache +warmed from an unrelated code path, the composition root exposes +`Space.sqliteTree` — a synchronous resolver, named for the backend that has +it and `null` everywhere else, exactly like `Space.diskTree`. It has its own +census in `module-boundaries.test.ts` with exactly one production consumer, +because the only justification for it is that one owner's synchronous +interface; a second consumer would mean the reason had drifted. + +The payoff is the thing a user would notice: an agent conversation in a Space +that has no directory survives a restart, and is removed with its Space by the +same cascade as everything else. + +#### 12.9.4 What this profile does not serve + +Six capabilities are Disk-only, declared in `storage/capabilities.ts`, logged +at startup, and refused at their own call sites in the same words: + +| Capability | What is lost | Why it is not emulated | +| --------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | +| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | +| `reveal-space-folder` | "Show me this in Finder" | Without a folder there is nothing to show. | +| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | They sandbox on the Space directory. The first-party agent edits nodes through the Canvas tools instead. | +| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | +| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | +| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | Every blob scope is Space-scoped, so a Workspace-level document has nowhere to live yet. A Space's _own_ memory body is unaffected. | +| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | +| `space-directory-handle-coordination` | Windows rename-while-watched | No directory, no problem. | + +Two further limits are not capability rows because nothing refuses them, they +are simply properties of the backend: + +- **Blob size.** Bytes are a row read and written whole, so the profile is + sized for the documents and images a Space holds, not for arbitrarily large + media, and the database grows to the size of everything ever uploaded. + `materialize()` spools to the OS temp directory and unlinks on release — + which is what `BlobLease`'s post-release rule was written to keep honest. +- **Multi-process access.** One process, one connection. WAL and + `busy_timeout` make a second reader survivable, and nothing here promises a + multi-process deletion fence or a distributed transaction. + +#### 12.9.5 Proof + +The reusable contracts — structured store, Space repository, nodes, ordered +write, logs, Tasks, extension substrate, **blob store**, and **Workspace +repository** — run against Disk and against real temporary SQLite files. + +`PRODUCT_STORAGE_PROFILES` gains `sqlite/sqlite`, so the §12.8 product-boundary +suite runs unchanged against it: World bootstrap, Space creation, ordered +writes through every node read shape, version conflict, bytes in every area, +the cross-store put guard, extension isolation and cleanup, the log families, +the Task ledger, deletion, World protection, and — added here — that all of it +is still there after a restart. That suite names no directory and no filename; +`module-boundaries.test.ts` enforces that mechanically. + +SQLite integration tests additionally cover strict schema creation, WAL and +foreign-key pragmas read back on a second connection, close/reopen persistence, an immutable v1 fixture, future-version rejection, migration -rollback, SQL fault injection, foreign-key cascades, and revision safety across -delete/recreate. - -The preview changes only the storage implementation, contracts, focused type -narrowing for the new substrate union, and this documentation. Runtime -composition and product capability owners remain unchanged. +rollback, SQL fault injection, foreign-key cascades, revision safety across +delete/recreate, `JSON.stringify` encoding parity, incremental streaming and +early abort, batched `readMany`, Workspace scoping and handle invalidation +across a switch, forget-without-delete, and blob byte fidelity, lease +lifetime, and Workspace isolation. The Agenetes conversation stores have their +own suite against a mounted profile, covering round-trip, isolation, restart, +and destruction with the Space. ### 12.10 Later phases — provisional From 00cf98600585273c4ce176ca406d808cbf35d7ea Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 10:02:50 +0800 Subject: [PATCH 08/15] refactor(storage): keep blob bytes on a file system, whatever holds the records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A structured backend was allowed to hold bytes: `HUABU_BLOB_BACKEND=sqlite` put every artifact, upload, guide document and memory body in a `blobs` table, and a cross-axis rule then required `structured=sqlite` to go with it. That is the wrong shape. Bytes are files — a local directory now, an object store later — and a deployment should be able to pair SQL records with ordinary files without either axis knowing about the other. So the SQLite blob adapter and its table are gone, `BlobBackendKind` is `'disk'`, `RequestedBlobKind` is `'disk' | 'azure'`, and the cross-axis rule has nothing left to enforce. `sqlite` records beside `disk` bytes is now an ordinary profile rather than a rejected one, and it is what `PRODUCT_STORAGE_PROFILES` runs the §12.8 suite against. That needs the Disk blob adapter to work where a Space has no folder. It takes its Space root as an argument instead of resolving the Disk record layout: with Disk records the root is `canvasRoot()` and every existing Workspace is addressed byte-for-byte as before; without them, composition supplies `/storage/blobs///` (`HUABU_BLOB_ROOT`) and the adapter writes the same area layout underneath it. Scope binding moves from the workspace *path* to `getWorkspaceKey()`, which both kinds of Workspace can answer. Because no structured delete will ever remove that directory, the delete saga removes it after sweeping the areas. Multiple Workspaces on SQLite were half-supported: the schema, the repository and activation all handled them, but `POST /api/workspaces` refused, because creating a Workspace meant adopting a folder. It now creates one from a name where there is no folder to adopt — `createNamedWorkspace` on the composition root, the counterpart to `adoptWorkspaceDirectory` — so a deployment can hold more than the one the Server opens for itself. `workspaceCreateSchema` makes `path` optional to say so; the Server decides which form its backend requires. Also drops the now-dead `SELECTABLE_STRUCTURED` gate (identical to `AVAILABLE_STRUCTURED` since SQLite became selectable), `LAZY_SAFE_BLOBS` (a file system has no connection to open), and two layout resolvers with no callers left. Verified against a running Server on `structured=sqlite blobs=disk`: create a Space, upload and fetch an artifact, create and activate a second Workspace by name, confirm the two byte roots are separate, and delete a Space and see its directory go with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../agent/agenetes/sqlite-stores.test.ts | 2 +- .../disk/blob-store.rename-retry.test.ts | 1 + .../storage/backends/disk/blob-store.test.ts | 1 + .../storage/backends/disk/blob-store.ts | 79 ++-- .../modules/storage/backends/disk/layout.ts | 16 +- .../storage/backends/sqlite/blob-store.ts | 339 ------------------ .../storage/backends/sqlite/contracts.test.ts | 13 - .../storage/backends/sqlite/fixtures/v1.sql | 18 - .../backends/sqlite/integration.test.ts | 75 +--- .../modules/storage/backends/sqlite/schema.ts | 21 +- .../backends/sqlite/structured-store.ts | 2 +- .../src/modules/storage/capabilities.test.ts | 4 +- .../modules/storage/detached-blobs.test.ts | 121 +++++++ apps/server/src/modules/storage/index.ts | 1 + .../modules/storage/module-boundaries.test.ts | 1 + apps/server/src/modules/storage/ports/blob.ts | 7 +- .../src/modules/storage/profile.test.ts | 32 +- apps/server/src/modules/storage/profile.ts | 72 ++-- apps/server/src/modules/storage/storage.ts | 109 +++++- apps/server/src/modules/storage/testing.ts | 32 +- .../src/modules/workspaces.route.test.ts | 69 +++- apps/server/src/modules/workspaces.route.ts | 38 +- docs/architecture/canvas-storage.md | 23 +- docs/proposals/multi-backend-storage.md | 163 +++++---- packages/shared/src/types/api/workspace.ts | 10 +- 25 files changed, 545 insertions(+), 704 deletions(-) delete mode 100644 apps/server/src/modules/storage/backends/sqlite/blob-store.ts create mode 100644 apps/server/src/modules/storage/detached-blobs.test.ts diff --git a/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts index f8e593311..f3999e8ae 100644 --- a/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts +++ b/apps/server/src/modules/agent/agenetes/sqlite-stores.test.ts @@ -30,7 +30,7 @@ import type { AgentStateSnapshot, WorkloadSpec } from '@agenetes/protocol'; const SQLITE: StorageProfile = { structured: { kind: 'sqlite' }, - blobs: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, }; const CANVAS_ID = 'canvas-conversation'; diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts b/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts index 142e0806f..7329decde 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts @@ -23,6 +23,7 @@ vi.mock('node:fs/promises', async (importOriginal) => { vi.mock('../../../workspace.js', () => ({ getWorkspacePath: () => testState.workspacePath, + getWorkspaceKey: () => testState.workspacePath, })); function errno(code: string): NodeJS.ErrnoException { diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.test.ts b/apps/server/src/modules/storage/backends/disk/blob-store.test.ts index ba7317708..e4718bd8a 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.test.ts @@ -18,6 +18,7 @@ const workspaceState = vi.hoisted(() => ({ path: '' })); vi.mock('../../../workspace.js', () => ({ getWorkspacePath: () => workspaceState.path, + getWorkspaceKey: () => workspaceState.path, })); import { DiskBlobStore } from './blob-store.js'; diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.ts b/apps/server/src/modules/storage/backends/disk/blob-store.ts index 9404f22a0..3c752a053 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.ts @@ -4,13 +4,20 @@ /** * Disk implementation of the blob port. * - * Maps each area of a Space to a directory under its Space folder, preserving + * Maps each area of a Space to a directory under that Space's root, preserving * the layout the workspace format has always used: one file per blob, named by * the URL key, no manifest indirection. * + * Where the Space's root *is* depends on the structured backend, which is why + * it is injected rather than resolved here. When Disk also keeps the records, + * the areas sit inside the Space folder the user can see — unchanged from + * every Workspace that already exists. When the records live in a database + * there is no such folder, so composition hands over a server-owned directory + * instead; the layout beneath it is identical either way. + * * Each scope is bound to the workspace active when it is created. A fresh - * scope follows a free-mode workspace switch; a retained scope rejects the - * next operation instead of silently redirecting it into the new workspace. + * scope follows a workspace switch; a retained scope rejects the next + * operation instead of silently redirecting it into the new workspace. */ import { randomUUID } from 'node:crypto'; @@ -27,13 +34,13 @@ import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { - artifactsDir, + ARTIFACTS_DIR_NAME, canvasRoot, - spaceMemoryDir, - spaceUploadDir, + MEMORY_DIR_NAME, + UPLOAD_DIR_NAME, } from './layout.js'; import { renameOverWithRetry } from '../../../../utils/fs.js'; -import { getWorkspacePath } from '../../../workspace.js'; +import { getWorkspaceKey } from '../../../workspace.js'; import { BlobNameError, createBlobLease, @@ -69,6 +76,15 @@ function isTempEntry(entry: string): boolean { /** The areas of a Space, as this adapter places them. */ type SpaceBlobArea = keyof SpaceBlobs; +/** + * Where this Space keeps its bytes. + * + * The Disk structured backend's own {@link canvasRoot} is the default, so a + * Workspace that already exists is addressed exactly as before. A profile + * whose records live elsewhere supplies its own. + */ +export type SpaceBlobRoot = (canvasId: string) => string; + /** * Where one area's bytes sit, and which names it owns there. * @@ -81,21 +97,18 @@ interface ScopePlacement { readonly members: readonly string[] | null; } -function scopePlacement(area: SpaceBlobArea, canvasId: string): ScopePlacement { +function scopePlacement(area: SpaceBlobArea, root: string): ScopePlacement { switch (area) { case 'artifacts': - return { directory: artifactsDir(canvasId), members: null }; + return { directory: path.join(root, ARTIFACTS_DIR_NAME), members: null }; case 'memory': - return { directory: spaceMemoryDir(canvasId), members: null }; + return { directory: path.join(root, MEMORY_DIR_NAME), members: null }; case 'uploads': - return { directory: spaceUploadDir(canvasId), members: null }; + return { directory: path.join(root, UPLOAD_DIR_NAME), members: null }; case 'guide': - // The Space root, which also holds `space.json` and every node - // directory — so this area is the guide names, not the folder. - return { - directory: canvasRoot(canvasId), - members: SPACE_GUIDE_BLOB_NAMES, - }; + // The Space root itself, which on Disk also holds `space.json` and every + // node directory — so this area is the guide names, not the folder. + return { directory: root, members: SPACE_GUIDE_BLOB_NAMES }; } } @@ -112,17 +125,21 @@ function isMissing(err: unknown): boolean { class DiskBlobScope implements BlobScope { readonly #area: SpaceBlobArea; readonly #canvasId: string; - readonly #workspacePath: string; + readonly #root: SpaceBlobRoot; + readonly #workspaceKey: string; - constructor(area: SpaceBlobArea, canvasId: string) { + constructor(area: SpaceBlobArea, canvasId: string, root: SpaceBlobRoot) { this.#area = area; this.#canvasId = canvasId; - this.#workspacePath = path.resolve(getWorkspacePath()); + this.#root = root; + // The Workspace as an identity rather than a location: a Workspace that is + // a row has no path to compare, and the binding means the same thing + // either way. + this.#workspaceKey = getWorkspaceKey(); } #placement(): ScopePlacement { - const active = path.resolve(getWorkspacePath()); - if (active !== this.#workspacePath) { + if (getWorkspaceKey() !== this.#workspaceKey) { throw new Error( `DiskBlobScope(${this.#canvasId}) belongs to an inactive workspace. ` + `Resolve a fresh scope after workspace activation.`, @@ -131,7 +148,7 @@ class DiskBlobScope implements BlobScope { // Resolve once per operation, before its first await. Every later path in // that operation is derived from this absolute directory, so a workspace // switch cannot combine a temp in A with a destination in B. - return scopePlacement(this.#area, this.#canvasId); + return scopePlacement(this.#area, this.#root(this.#canvasId)); } /** Names this scope owns in `dir`, given what is actually there. */ @@ -312,6 +329,12 @@ class DiskBlobScope implements BlobScope { export class DiskBlobStore implements BlobStore { readonly kind = 'disk' as const; + readonly #root: SpaceBlobRoot; + + constructor(root: SpaceBlobRoot = canvasRoot) { + this.#root = root; + } + async init(): Promise { // Area directories are created on first write; nothing to prepare. } @@ -323,11 +346,13 @@ export class DiskBlobStore implements BlobStore { async close(): Promise {} space(canvasId: string): SpaceBlobs { + const scope = (area: SpaceBlobArea): BlobScope => + new DiskBlobScope(area, canvasId, this.#root); return { - artifacts: new DiskBlobScope('artifacts', canvasId), - guide: new DiskBlobScope('guide', canvasId), - memory: new DiskBlobScope('memory', canvasId), - uploads: new DiskBlobScope('uploads', canvasId), + artifacts: scope('artifacts'), + guide: scope('guide'), + memory: scope('memory'), + uploads: scope('uploads'), }; } } diff --git a/apps/server/src/modules/storage/backends/disk/layout.ts b/apps/server/src/modules/storage/backends/disk/layout.ts index 9201c4505..a6ea321a7 100644 --- a/apps/server/src/modules/storage/backends/disk/layout.ts +++ b/apps/server/src/modules/storage/backends/disk/layout.ts @@ -83,23 +83,17 @@ export function artifactsDir(canvasId: string): string { /** * Hidden directory holding the agent's private memory document. * - * Named here rather than in the workspace module because it is now a blob - * scope's placement — where Disk puts the bytes of one user-visible area — - * and every other such placement already lives beside this one. + * Named here rather than in the workspace module because it is a blob scope's + * placement — what Disk calls the folder holding one user-visible area — and + * every other such placement already lives beside this one. The blob adapter + * joins these names onto whichever Space root it was given, so they are + * constants rather than resolvers. */ export const MEMORY_DIR_NAME = '.memory'; -export function spaceMemoryDir(canvasId: string): string { - return path.join(canvasRoot(canvasId), MEMORY_DIR_NAME); -} - /** Hidden scratch an upload lands in before anything claims it. */ export const UPLOAD_DIR_NAME = '.upload'; -export function spaceUploadDir(canvasId: string): string { - return path.join(canvasRoot(canvasId), UPLOAD_DIR_NAME); -} - export function artifactPath(canvasId: string, filename: string): string { const base = path.basename(filename); if (!base || base === '.' || base === '..') { diff --git a/apps/server/src/modules/storage/backends/sqlite/blob-store.ts b/apps/server/src/modules/storage/backends/sqlite/blob-store.ts deleted file mode 100644 index c6717c31b..000000000 --- a/apps/server/src/modules/storage/backends/sqlite/blob-store.ts +++ /dev/null @@ -1,339 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -/** - * SQLite implementation of the blob port. - * - * Bytes live in the same database file as the records, in one row per blob. - * That is what lets a SQL deployment need no folder at all: uploads, - * artifacts, the guide document and the agent's memory body stop being files - * without becoming a second service to run. - * - * The price is stated rather than hidden. A row is read and written whole, so - * this backend is sized for the documents and images a Space actually holds, - * not for arbitrarily large media, and a database holding blobs grows to the - * size of everything ever uploaded. `materialize()` therefore spools to the - * OS temp directory — the port's own escape hatch for consumers that need a - * real path — and unlinks on release, which is exactly the "temp copy" - * behaviour `BlobLease` was written to keep honest. - * - * Atomicity comes free where Disk had to work for it: a `put` buffers its body - * and then replaces the row in one statement, so a reader mid-write sees the - * previous blob and a failed body leaves the previous blob in place. - */ - -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { Readable } from 'node:stream'; - -import { - BlobNameError, - createBlobLease, - normalizeBlobName, - SPACE_GUIDE_BLOB_NAMES, -} from '../../ports/blob.js'; - -import type { SqliteStoreContext } from './database.js'; -import type { - BlobInfo, - BlobLease, - BlobRange, - BlobRead, - BlobScope, - BlobStore, - SpaceBlobs, -} from '../../ports/blob.js'; -import type { StorageHealth } from '../../ports/common.js'; - -type SpaceBlobArea = keyof SpaceBlobs; - -/** - * Names an area owns, or `null` when it owns whatever is put in it. - * - * The distinction is the port's, not this backend's: `guide` is bounded by a - * fixed member list because on Disk it shares the Space root with records that - * are not blobs. A table has no such neighbours, but the boundary is a - * contract term — a name outside the set must be refused on every backend, or - * a caller could write one where only one adapter accepts it. - */ -function areaMembers(area: SpaceBlobArea): readonly string[] | null { - return area === 'guide' ? SPACE_GUIDE_BLOB_NAMES : null; -} - -async function collect(body: Readable | Buffer): Promise { - if (Buffer.isBuffer(body)) return body; - const chunks: Buffer[] = []; - for await (const chunk of body) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string)); - } - return Buffer.concat(chunks); -} - -function decodeBytes(value: unknown, name: string): Buffer { - if (value instanceof Uint8Array) return Buffer.from(value); - if (typeof value === 'string') return Buffer.from(value, 'utf8'); - throw new SyntaxError(`Persisted blob ${JSON.stringify(name)} is not bytes`); -} - -function decodeInfo(value: unknown): BlobInfo { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - throw new SyntaxError('Malformed persisted SQLite blob row'); - } - const row = value as Record; - const name = row['name']; - const size = row['size']; - const updatedAt = row['updated_at']; - if (typeof name !== 'string') { - throw new SyntaxError('Invalid name in persisted SQLite blob'); - } - if (typeof size !== 'number' || !Number.isFinite(size)) { - throw new SyntaxError(`Invalid size for persisted blob ${name}`); - } - if (typeof updatedAt !== 'number' || !Number.isFinite(updatedAt)) { - throw new SyntaxError(`Invalid updated_at for persisted blob ${name}`); - } - return { name, size, updatedAt }; -} - -class SqliteBlobScope implements BlobScope { - readonly #context: SqliteStoreContext; - readonly #workspaceId: string; - readonly #canvasId: string; - readonly #area: SpaceBlobArea; - - constructor( - context: SqliteStoreContext, - workspaceId: string, - canvasId: string, - area: SpaceBlobArea, - ) { - this.#context = context; - this.#workspaceId = workspaceId; - this.#canvasId = canvasId; - this.#area = area; - } - - /** Re-check the binding, exactly as a Disk scope re-checks its path. */ - #workspace(): string { - return this.#context.assertBoundWorkspace( - this.#workspaceId, - `SQLite blob scope for Space "${this.#canvasId}"`, - ); - } - - /** Refuse a name this area does not own, before it reaches the database. */ - #assertMember(name: string): string { - const safe = normalizeBlobName(name); - const members = areaMembers(this.#area); - if (members && !members.includes(safe)) { - throw new BlobNameError( - `"${safe}" is not a member of the ${this.#area} area. ` + - `It holds: ${members.join(', ')}.`, - ); - } - return safe; - } - - #key(name: string): [string, string, string, string] { - return [this.#workspace(), this.#canvasId, this.#area, name]; - } - - async put(name: string, body: Readable | Buffer): Promise { - const safe = this.#assertMember(name); - // Collect before touching the row: a body that fails mid-stream must - // leave the previous blob exactly as it was, and a reader must never see - // a prefix of the replacement. - const bytes = await collect(body); - const updatedAt = this.#context.now(); - this.#context - .database() - .prepare( - `INSERT INTO blobs ( - workspace_id, canvas_id, area, name, bytes, size, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(workspace_id, canvas_id, area, name) DO UPDATE SET - bytes = excluded.bytes, - size = excluded.size, - updated_at = excluded.updated_at`, - ) - .run(...this.#key(safe), bytes, bytes.byteLength, updatedAt); - return { name: safe, size: bytes.byteLength, updatedAt }; - } - - async head(name: string): Promise { - const safe = this.#assertMember(name); - const row = this.#context - .database() - .prepare( - `SELECT name, size, updated_at - FROM blobs - WHERE workspace_id = ? AND canvas_id = ? AND area = ? AND name = ?`, - ) - .get(...this.#key(safe)); - return row === undefined ? null : decodeInfo(row); - } - - async open(name: string, range?: BlobRange): Promise { - const safe = this.#assertMember(name); - const found = await this.#load(safe); - if (found === null) return null; - const { info, bytes } = found; - // `info.size` stays the whole blob; the range only bounds the body, and - // an over-long end is clamped the way a filesystem read stream clamps it. - const start = Math.max(0, range?.start ?? 0); - const end = - range?.end === undefined - ? bytes.byteLength - 1 - : Math.min(range.end, bytes.byteLength - 1); - const slice = - end < start ? Buffer.alloc(0) : bytes.subarray(start, end + 1); - return { info, body: Readable.from([slice]) }; - } - - async read(name: string): Promise { - const safe = this.#assertMember(name); - return (await this.#load(safe))?.bytes ?? null; - } - - async hasMany(names: readonly string[]): Promise> { - this.#workspace(); - const requested = new Set(names.map(normalizeBlobName)); - const members = areaMembers(this.#area); - const wanted = [...requested].filter( - (candidate) => !members || members.includes(candidate), - ); - if (wanted.length === 0) return new Set(); - - const placeholders = wanted.map(() => '?').join(', '); - const rows = this.#context - .database() - .prepare( - `SELECT name - FROM blobs - WHERE workspace_id = ? AND canvas_id = ? AND area = ? - AND name IN (${placeholders})`, - ) - .all(this.#workspace(), this.#canvasId, this.#area, ...wanted); - return new Set( - rows.map((row) => { - const name = (row as Record)['name']; - if (typeof name !== 'string') { - throw new SyntaxError('Invalid name in persisted SQLite blob'); - } - return name; - }), - ); - } - - async list(): Promise { - const members = areaMembers(this.#area); - const rows = this.#context - .database() - .prepare( - `SELECT name, size, updated_at - FROM blobs - WHERE workspace_id = ? AND canvas_id = ? AND area = ? - ORDER BY name`, - ) - .all(this.#workspace(), this.#canvasId, this.#area) - .map(decodeInfo); - return members ? rows.filter((info) => members.includes(info.name)) : rows; - } - - async materialize(name: string): Promise { - const safe = this.#assertMember(name); - const found = await this.#load(safe); - if (found === null) return null; - // No permanent path exists, so one is spooled for the life of the lease. - // The directory is unique per lease, so the blob keeps its own name for - // consumers that infer a type from the extension. - const directory = await mkdtemp(path.join(tmpdir(), 'huabu-blob-')); - const file = path.join(directory, safe); - try { - await writeFile(file, found.bytes); - } catch (error) { - await rm(directory, { recursive: true, force: true }).catch(() => {}); - throw error; - } - return createBlobLease(file, async () => { - await rm(directory, { recursive: true, force: true }).catch(() => {}); - }); - } - - async deleteAll(): Promise { - const members = areaMembers(this.#area); - const database = this.#context.database(); - if (!members) { - database - .prepare( - `DELETE FROM blobs - WHERE workspace_id = ? AND canvas_id = ? AND area = ?`, - ) - .run(this.#workspace(), this.#canvasId, this.#area); - return; - } - const placeholders = members.map(() => '?').join(', '); - database - .prepare( - `DELETE FROM blobs - WHERE workspace_id = ? AND canvas_id = ? AND area = ? - AND name IN (${placeholders})`, - ) - .run(this.#workspace(), this.#canvasId, this.#area, ...members); - } - - async #load(name: string): Promise<{ info: BlobInfo; bytes: Buffer } | null> { - const row = this.#context - .database() - .prepare( - `SELECT name, size, updated_at, bytes - FROM blobs - WHERE workspace_id = ? AND canvas_id = ? AND area = ? AND name = ?`, - ) - .get(...this.#key(name)); - if (row === undefined) return null; - const info = decodeInfo(row); - return { - info, - bytes: decodeBytes((row as Record)['bytes'], info.name), - }; - } -} - -export class SqliteBlobStore implements BlobStore { - readonly kind = 'sqlite' as const; - - readonly #context: SqliteStoreContext; - readonly #ownsContext: boolean; - - constructor(context: SqliteStoreContext, ownsContext = false) { - this.#context = context; - this.#ownsContext = ownsContext; - } - - async init(): Promise { - if (this.#ownsContext) this.#context.init(); - else this.#context.assertOpen(); - } - - async health(): Promise { - return this.#context.health(this.kind); - } - - async close(): Promise { - if (this.#ownsContext) this.#context.close(); - } - - space(canvasId: string): SpaceBlobs { - const workspaceId = this.#context.workspaceId(); - const scope = (area: SpaceBlobArea): BlobScope => - new SqliteBlobScope(this.#context, workspaceId, canvasId, area); - return { - artifacts: scope('artifacts'), - guide: scope('guide'), - memory: scope('memory'), - uploads: scope('uploads'), - }; - } -} diff --git a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts index 459746d4e..a35d2ecc0 100644 --- a/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts +++ b/apps/server/src/modules/storage/backends/sqlite/contracts.test.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { SqliteBlobStore } from './blob-store.js'; import { SqliteStoreContext } from './database.js'; import { createSqliteTestFile, @@ -11,7 +10,6 @@ import { readSqliteDeltaLog, } from './test-support.js'; import { SqliteWorkspaceRepository } from './workspace-repository.js'; -import { describeBlobStoreContract } from '../../ports/contracts/blob-store.contract.js'; import { describeSpaceExtensionContract } from '../../ports/contracts/space-extension.contract.js'; import { describeSpaceLogsContract } from '../../ports/contracts/space-logs.contract.js'; import { describeSpaceNodesContract } from '../../ports/contracts/space-nodes.contract.js'; @@ -205,17 +203,6 @@ describeSpaceTasksContract('SQLite', async () => { }; }); -describeBlobStoreContract('SqliteBlobStore', async () => { - const harness = await openEmptySqliteTestStore('huabu-sqlite-blob-contract-'); - return { - // The blob store shares the structured store's connection, because both - // ports are one database file. - store: new SqliteBlobStore(harness.context), - canvasId: 'sqlite-blob-contract-space', - cleanup: harness.cleanup, - }; -}); - describeWorkspaceRepositoryContract('SQLite', async () => { const file = createSqliteTestFile('huabu-sqlite-workspace-contract-'); const context = new SqliteStoreContext(file.filename); diff --git a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql index ea1f6f30c..adcaa7ec1 100644 --- a/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql +++ b/apps/server/src/modules/storage/backends/sqlite/fixtures/v1.sql @@ -86,17 +86,6 @@ CREATE TABLE delta_log ( FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE ) STRICT; -CREATE TABLE blobs ( - workspace_id TEXT NOT NULL, - canvas_id TEXT NOT NULL, - area TEXT NOT NULL, - name TEXT NOT NULL, - bytes BLOB NOT NULL, - size INTEGER NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (workspace_id, canvas_id, area, name) -) STRICT; - INSERT INTO workspaces ( workspace_id, name, created_at, last_opened_at, forgotten_at ) VALUES ('fixture-workspace', 'Fixture Workspace', 1, 1, NULL); @@ -144,12 +133,5 @@ INSERT INTO delta_log (canvas_id, version, entry_json) VALUES ( '{"version":3,"ts":13,"commands":[],"deltas":[],"originator":{"source":"system"}}' ); -INSERT INTO blobs ( - workspace_id, canvas_id, area, name, bytes, size, updated_at -) VALUES ( - 'fixture-workspace', 'fixture-space', 'artifacts', 'fixture.txt', - CAST('fixture bytes' AS BLOB), 13, 14 -); - PRAGMA user_version = 1; COMMIT; diff --git a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts index 706b8930c..6a13e9a62 100644 --- a/apps/server/src/modules/storage/backends/sqlite/integration.test.ts +++ b/apps/server/src/modules/storage/backends/sqlite/integration.test.ts @@ -1,14 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { existsSync, readFileSync } from 'node:fs'; -import path from 'node:path'; +import { readFileSync } from 'node:fs'; import { afterEach, describe, expect, it } from 'vitest'; import { extractCanvasChanges } from '@huabu/shared/canvas-engine'; -import { SqliteBlobStore } from './blob-store.js'; import { applySqliteMigrations, SqliteStoreContext, @@ -177,7 +175,6 @@ describe('SqliteStructuredStore lifecycle and schema', () => { user_version: SQLITE_SCHEMA_VERSION, }); const expectedTables = [ - 'blobs', 'changes', 'delta_log', 'events', @@ -227,12 +224,6 @@ describe('SqliteStructuredStore lifecycle and schema', () => { to: 'workspace_id', onDelete: 'CASCADE', }); - // Blob rows deliberately do not reference `spaces`: the deletion saga - // sweeps them before the record goes, and must also be able to sweep - // orphans for a record that is already missing. - expect(database.prepare('PRAGMA foreign_key_list(blobs)').all()).toEqual( - [], - ); }); }); @@ -295,13 +286,6 @@ describe('SqliteStructuredStore lifecycle and schema', () => { originator: { source: 'system' }, }, ]); - const blobs = new SqliteBlobStore( - // The same connection the structured store just read through. - (store as unknown as { context: SqliteStoreContext }).context, - ); - await expect( - blobs.space('fixture-space').artifacts.read('fixture.txt'), - ).resolves.toEqual(Buffer.from('fixture bytes')); }); it('rejects a database whose user_version is from the future', async () => { @@ -910,60 +894,3 @@ describe('SqliteStructuredStore durability and encoding', () => { ).toHaveLength(1); }); }); - -describe('SqliteBlobStore', () => { - async function openBlobs(prefix: string) { - const harness = await trackedOpenStore(prefix); - const store = new SqliteBlobStore(harness.context); - await store.init(); - return { harness, store }; - } - - it('keeps bytes exactly, including binary that is not text', async () => { - const { store } = await openBlobs('huabu-sqlite-blob-bytes-'); - const bytes = Buffer.from([0, 1, 2, 250, 251, 252, 0, 255]); - - const scope = store.space('blob-space').artifacts; - const info = await scope.put('binary.bin', bytes); - expect(info.size).toBe(bytes.byteLength); - expect(await scope.read('binary.bin')).toEqual(bytes); - }); - - it('spools a lease to a real path and removes it on release', async () => { - const { store } = await openBlobs('huabu-sqlite-blob-lease-'); - const scope = store.space('blob-space').artifacts; - await scope.put('leased.png', Buffer.from('pretend png')); - - const lease = await scope.materialize('leased.png'); - if (!lease) throw new Error('Expected a lease'); - const leasedPath = lease.path; - // The blob keeps its own name, so a consumer that infers a type from the - // extension still works. - expect(path.basename(leasedPath)).toBe('leased.png'); - expect(readFileSync(leasedPath)).toEqual(Buffer.from('pretend png')); - - await lease.release(); - // A temp copy, not the storage: it must not survive the lease. - expect(existsSync(leasedPath)).toBe(false); - expect(await scope.read('leased.png')).toEqual(Buffer.from('pretend png')); - }); - - it('separates the bytes of one Workspace from another', async () => { - const { harness, store } = await openBlobs('huabu-sqlite-blob-workspace-'); - const first = store.space('shared-canvas-id').artifacts; - await first.put('same-name.bin', Buffer.from('first workspace')); - - const workspaces = new SqliteWorkspaceRepository(harness.context); - const second = await workspaces.create('Second Workspace'); - harness.context.useWorkspace(second.workspaceId); - - const other = store.space('shared-canvas-id').artifacts; - expect(await other.head('same-name.bin')).toBeNull(); - await other.put('same-name.bin', Buffer.from('second workspace')); - - harness.context.useWorkspace(harness.workspaceId); - expect( - await store.space('shared-canvas-id').artifacts.read('same-name.bin'), - ).toEqual(Buffer.from('first workspace')); - }); -}); diff --git a/apps/server/src/modules/storage/backends/sqlite/schema.ts b/apps/server/src/modules/storage/backends/sqlite/schema.ts index 8f9d00629..d8b41fbb3 100644 --- a/apps/server/src/modules/storage/backends/sqlite/schema.ts +++ b/apps/server/src/modules/storage/backends/sqlite/schema.ts @@ -17,12 +17,10 @@ * the same connection rather than reopening anything (proposal §2, "Backend * selection scope"). * - * Blobs deliberately do **not** reference `spaces`. The two ports are - * configured independently and their lifecycles are joined only by the - * deletion saga in `storage.ts`, which sweeps every blob area *before* the - * structured record goes. A foreign key here would quietly move that ordering - * decision into the schema, and would refuse the orphan sweep the saga - * performs when a record has already gone missing. + * Nothing here holds bytes. Blobs are files on whichever file system the blob + * axis names, so a Space's uploads, artifacts, guide and memory body are never + * rows in this database; the two lifecycles are joined only by the deletion + * saga in `storage.ts`, which sweeps the byte areas before the record goes. */ /** @@ -114,17 +112,6 @@ const SCHEMA_V1 = ` PRIMARY KEY (canvas_id, version), FOREIGN KEY (canvas_id) REFERENCES spaces(canvas_id) ON DELETE CASCADE ) STRICT; - - CREATE TABLE blobs ( - workspace_id TEXT NOT NULL, - canvas_id TEXT NOT NULL, - area TEXT NOT NULL, - name TEXT NOT NULL, - bytes BLOB NOT NULL, - size INTEGER NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (workspace_id, canvas_id, area, name) - ) STRICT; `; export interface SqliteMigration { diff --git a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts index b3cfe5c9d..267facb5e 100644 --- a/apps/server/src/modules/storage/backends/sqlite/structured-store.ts +++ b/apps/server/src/modules/storage/backends/sqlite/structured-store.ts @@ -25,7 +25,7 @@ import type { /** * Structured-store adapter over one `node:sqlite` connection. * - * The connection may be shared with the SQLite blob store — one database file + * The connection is shared with the Workspace repository — one database file * cannot have two writers — so this class does not assume it owns the * lifecycle. Constructed with a filename it opens and closes its own * connection; constructed with an existing context it borrows one, and diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index 1805571c1..670a3d466 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -27,10 +27,10 @@ const DISK: StorageProfile = { blobs: { kind: 'disk' }, }; -/** The profile that keeps Spaces in tables and bytes in rows. */ +/** The profile that keeps Spaces in tables and their bytes in files. */ const TABLES: StorageProfile = { structured: { kind: 'sqlite' }, - blobs: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, }; describe('storage capability matrix', () => { diff --git a/apps/server/src/modules/storage/detached-blobs.test.ts b/apps/server/src/modules/storage/detached-blobs.test.ts new file mode 100644 index 000000000..601feb685 --- /dev/null +++ b/apps/server/src/modules/storage/detached-blobs.test.ts @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Where a Space's bytes go when its records are rows. + * + * The portable behaviour — put, read, sweep on delete — is already proven for + * every profile by `product-boundary.test.ts`, and naming a directory there + * would stop it being evidence of anything portable. What is left is the part + * that *is* about placement, and it belongs here: bytes are files on every + * profile, so a backend with no Space folder still needs one, and it has to be + * scoped to the Workspace that owns the Space and removed with it. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { ARTIFACTS_DIR_NAME } from './backends/disk/layout.js'; +import { + activateWorkspace, + createNamedWorkspace, + createSpace, + deleteSpace, + detachedBlobRoot, + space, +} from './storage.js'; +import { mountTestWorkspace, type MountedTestStorage } from './testing.js'; +import { getWorkspaceHandle } from '../workspace.js'; + +import type { StorageProfile } from './profile.js'; + +/** Records in SQLite, bytes on the file system — the hybrid this covers. */ +const HYBRID: StorageProfile = { + structured: { kind: 'sqlite' }, + blobs: { kind: 'disk' }, +}; + +const CANVAS_ID = 'canvas-detached-blobs'; + +let mounted: MountedTestStorage | null = null; + +afterEach(async () => { + await mounted?.close(); + mounted = null; +}); + +async function mount(): Promise { + mounted = await mountTestWorkspace(HYBRID, 'huabu-detached-blobs-'); + return mounted; +} + +/** The directory this profile puts one Space's artifacts in. */ +function artifactsDirectory(canvasId: string): string { + const workspace = getWorkspaceHandle(); + if (!workspace) throw new Error('Expected an active Workspace'); + return path.join( + detachedBlobRoot(), + workspace.workspaceId, + canvasId, + ARTIFACTS_DIR_NAME, + ); +} + +describe('Space bytes on a backend with no Space folder', () => { + it('writes real files under the Workspace-scoped byte root', async () => { + await mount(); + await createSpace(CANVAS_ID, 'Detached'); + + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('real bytes')); + + const file = path.join(artifactsDirectory(CANVAS_ID), 'art.bin'); + expect(readFileSync(file)).toEqual(Buffer.from('real bytes')); + }); + + it("files each Workspace's bytes under its own root", async () => { + await mount(); + const firstWorkspace = getWorkspaceHandle(); + if (!firstWorkspace) throw new Error('Expected an active Workspace'); + await createSpace(CANVAS_ID, 'First'); + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('first')); + const first = artifactsDirectory(CANVAS_ID); + + const second = await createNamedWorkspace('Second Workspace'); + await activateWorkspace(second); + const otherCanvasId = 'canvas-detached-blobs-second'; + await createSpace(otherCanvasId, 'Second'); + await space(otherCanvasId).artifacts.put('art.bin', Buffer.from('second')); + + // Two Workspaces served by one connection and one blob root, and neither + // can reach into the other's bytes: the Workspace segment is what keeps + // them apart, the same way a Workspace folder does on Disk. + const secondDirectory = artifactsDirectory(otherCanvasId); + expect(path.dirname(path.dirname(first))).not.toBe( + path.dirname(path.dirname(secondDirectory)), + ); + expect(readFileSync(path.join(first, 'art.bin'))).toEqual( + Buffer.from('first'), + ); + + await activateWorkspace(firstWorkspace); + expect(await space(CANVAS_ID).artifacts.read('art.bin')).toEqual( + Buffer.from('first'), + ); + }); + + it('leaves no directory behind when the Space is deleted', async () => { + await mount(); + await createSpace(CANVAS_ID, 'Detached'); + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('bytes')); + const spaceRoot = path.dirname(artifactsDirectory(CANVAS_ID)); + expect(existsSync(spaceRoot)).toBe(true); + + await expect(deleteSpace(CANVAS_ID)).resolves.toMatchObject({ ok: true }); + + // Sweeping the areas is the blob port's contract; removing the directory + // composition put them under is this module's, and nothing else would. + expect(existsSync(spaceRoot)).toBe(false); + }); +}); diff --git a/apps/server/src/modules/storage/index.ts b/apps/server/src/modules/storage/index.ts index 9f7f5a63b..f6ff8897b 100644 --- a/apps/server/src/modules/storage/index.ts +++ b/apps/server/src/modules/storage/index.ts @@ -82,6 +82,7 @@ export { adoptWorkspaceDirectory, closeStorage, composeStorage, + createNamedWorkspace, createSpace, createStorage, deleteSpace, diff --git a/apps/server/src/modules/storage/module-boundaries.test.ts b/apps/server/src/modules/storage/module-boundaries.test.ts index 37ee0e7cb..ce15f31d9 100644 --- a/apps/server/src/modules/storage/module-boundaries.test.ts +++ b/apps/server/src/modules/storage/module-boundaries.test.ts @@ -87,6 +87,7 @@ describe('storage module tree', () => { 'canvas-dirs.ts', 'capabilities.test.ts', 'capabilities.ts', + 'detached-blobs.test.ts', 'index.ts', 'module-boundaries.test.ts', 'paths.ts', diff --git a/apps/server/src/modules/storage/ports/blob.ts b/apps/server/src/modules/storage/ports/blob.ts index b8ba498b0..54bdabf5d 100644 --- a/apps/server/src/modules/storage/ports/blob.ts +++ b/apps/server/src/modules/storage/ports/blob.ts @@ -28,8 +28,13 @@ import type { Readable } from 'node:stream'; * Like {@link StructuredBackendKind}, this names only what exists. The wider * vocabulary a profile may *request* — including `azure`, which is a settled * direction with no adapter — belongs to `profile.ts`. + * + * Every member of that wider vocabulary is a **file system**: a local + * directory now, an object store later. Bytes are not records, and a + * structured backend is never asked to hold them — which is what lets a + * deployment pair SQL records with ordinary files (proposal §6.2). */ -export type BlobBackendKind = 'disk' | 'sqlite'; +export type BlobBackendKind = 'disk'; /** * Every area of one Space that holds bytes. diff --git a/apps/server/src/modules/storage/profile.test.ts b/apps/server/src/modules/storage/profile.test.ts index 99dbf118c..bd388d8a1 100644 --- a/apps/server/src/modules/storage/profile.test.ts +++ b/apps/server/src/modules/storage/profile.test.ts @@ -35,7 +35,7 @@ describe('parseStorageProfile', () => { it('names the supported set when a kind is unknown', () => { expect(() => parseStorageProfile({ HUABU_BLOB_BACKEND: 's3' })).toThrow( - /HUABU_BLOB_BACKEND="s3".*disk, sqlite, azure/s, + /HUABU_BLOB_BACKEND="s3".*disk, azure/s, ); }); @@ -67,18 +67,10 @@ describe('validateStorageProfile', () => { ).toThrow(/not implemented yet.*disk, sqlite/s); }); - it('accepts the sqlite + sqlite profile', () => { - expect(() => - validateStorageProfile({ - structured: { kind: 'sqlite' }, - blobs: { kind: 'sqlite' }, - }), - ).not.toThrow(); - }); - - // Fewer features is a stated limitation, not a misconfiguration: a - // selectable profile may lose capabilities as long as the matrix declares - // them. Only an unimplemented or incoherent pairing fails here. + // Fewer features is a stated limitation, not a misconfiguration: a profile + // may lose capabilities as long as the matrix declares them. Only an + // unimplemented backend fails here — the axes share nothing, so every + // pairing of implemented backends is a valid deployment. it('accepts sqlite records beside disk blobs', () => { expect(() => validateStorageProfile({ @@ -88,22 +80,13 @@ describe('validateStorageProfile', () => { ).not.toThrow(); }); - it('rejects sqlite blobs without the sqlite structured database', () => { - expect(() => - validateStorageProfile({ - structured: { kind: 'disk' }, - blobs: { kind: 'sqlite' }, - }), - ).toThrow(/requires HUABU_STRUCTURED_BACKEND=sqlite/); - }); - it('rejects a known but unimplemented blob backend', () => { expect(() => validateStorageProfile({ structured: { kind: 'disk' }, blobs: { kind: 'azure' }, }), - ).toThrow(/not implemented yet.*disk, sqlite/s); + ).toThrow(/not implemented yet.*disk/s); }); }); @@ -123,8 +106,7 @@ describe('requiresExplicitInit', () => { it.each([ { structured: { kind: 'postgres' }, blobs: { kind: 'disk' } }, - { structured: { kind: 'sqlite' }, blobs: { kind: 'sqlite' } }, - { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, + { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, ] as const)('requires an awaited init for %j', (profile) => { expect(requiresExplicitInit(profile)).toBe(true); }); diff --git a/apps/server/src/modules/storage/profile.ts b/apps/server/src/modules/storage/profile.ts index 4662c4c49..4eca6a547 100644 --- a/apps/server/src/modules/storage/profile.ts +++ b/apps/server/src/modules/storage/profile.ts @@ -6,8 +6,10 @@ * * Structured and blob storage are independent configuration axes — the * settled direction of docs/proposals/multi-backend-storage.md §6.3. A - * profile names one backend on each axis; not every pairing is a valid - * deployment, so profiles are validated before any connection is opened. + * profile names one backend on each axis and every pairing of implemented + * backends is a valid deployment, because the axes share nothing: records go + * to the structured backend, bytes go to a file system. `sqlite` records with + * `disk` bytes is an ordinary profile, not a special case. */ /** @@ -26,41 +28,40 @@ export type RequestedStructuredKind = 'disk' | 'sqlite' | 'postgres'; * * Wider than the port's {@link BlobBackendKind} for the same reason * {@link RequestedStructuredKind} is wider than the structured one. + * + * Every member is a file system. Bytes are files wherever they live — a local + * directory today, an object store later — and never rows in the structured + * database, so the two axes stay genuinely independent and a deployment may + * pair SQL records with ordinary files. */ -export type RequestedBlobKind = 'disk' | 'sqlite' | 'azure'; +export type RequestedBlobKind = 'disk' | 'azure'; export interface StorageProfile { structured: { kind: RequestedStructuredKind }; blobs: { kind: RequestedBlobKind }; } -/** Backends with an adapter implementation, selectable or otherwise. */ -const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ - 'disk', - 'sqlite', -]; - /** - * Backends whose capability matrix is complete enough to select. + * Backends with an adapter, and therefore selectable. * - * "Complete enough" is not "identical to Disk". A selectable profile may offer - * fewer features, as long as every one it does not offer is declared in - * `capabilities.ts` and refused where a user would reach for it. What - * disqualifies a backend is an *undeclared* gap — a feature that would fail - * with a stack trace rather than a sentence. + * Selectable is not "identical to Disk". A profile may offer fewer features, + * as long as every one it does not offer is declared in `capabilities.ts` and + * refused where a user would reach for it. What disqualifies a backend is an + * *undeclared* gap — a feature that would fail with a stack trace rather than + * a sentence. */ -const SELECTABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ +const AVAILABLE_STRUCTURED: readonly RequestedStructuredKind[] = [ 'disk', 'sqlite', ]; -const AVAILABLE_BLOBS: readonly RequestedBlobKind[] = ['disk', 'sqlite']; +const AVAILABLE_BLOBS: readonly RequestedBlobKind[] = ['disk']; const STRUCTURED_KINDS: readonly RequestedStructuredKind[] = [ 'disk', 'sqlite', 'postgres', ]; -const BLOB_KINDS: readonly RequestedBlobKind[] = ['disk', 'sqlite', 'azure']; +const BLOB_KINDS: readonly RequestedBlobKind[] = ['disk', 'azure']; export class StorageProfileError extends Error { override name = 'StorageProfileError'; @@ -125,49 +126,30 @@ export function validateStorageProfile(profile: StorageProfile): void { `Adapters available: ${AVAILABLE_STRUCTURED.join(', ')}.`, ); } - if (!SELECTABLE_STRUCTURED.includes(profile.structured.kind)) { - throw new StorageProfileError( - `Structured backend "${profile.structured.kind}" has a preview adapter ` + - `but is not selectable yet. Required application capabilities still ` + - `depend on Disk. Selectable: ${SELECTABLE_STRUCTURED.join(', ')}.`, - ); - } if (!AVAILABLE_BLOBS.includes(profile.blobs.kind)) { throw new StorageProfileError( `Blob backend "${profile.blobs.kind}" is not implemented yet. ` + `Available: ${AVAILABLE_BLOBS.join(', ')}.`, ); } - // The first real cross-axis rule. SQLite blobs are rows in the structured - // database, so they have nowhere to live unless that database exists — the - // two axes stay independent in the port design, but this particular pairing - // is a single file, and saying so here beats failing at the first upload. - if (profile.blobs.kind === 'sqlite' && profile.structured.kind !== 'sqlite') { - throw new StorageProfileError( - `Blob backend "sqlite" stores bytes in the SQLite structured database, ` + - `so it requires HUABU_STRUCTURED_BACKEND=sqlite (got ` + - `"${profile.structured.kind}").`, - ); - } } /** - * Backends whose `init()` has nothing to open, so building them on demand is - * safe. + * Structured backends whose `init()` has nothing to open, so building them on + * demand is safe. * * The lazy accessor in `storage.ts` is synchronous and therefore cannot - * `await init()`. That is harmless for backends which have no connection to - * establish, and silently wrong for any that do — they would be handed to + * `await init()`. That is harmless for a backend which has no connection to + * establish, and silently wrong for any that does — it would be handed to * callers unopened. Keeping the list here, next to the other backend facts, * means adding an adapter forces a decision about it. + * + * Only the structured axis appears: every blob backend is a file system, and + * a file system has no connection to open. */ const LAZY_SAFE_STRUCTURED: readonly RequestedStructuredKind[] = ['disk']; -const LAZY_SAFE_BLOBS: readonly RequestedBlobKind[] = ['disk']; /** Whether this profile may only be built through an awaited `initStorage()`. */ export function requiresExplicitInit(profile: StorageProfile): boolean { - return ( - !LAZY_SAFE_STRUCTURED.includes(profile.structured.kind) || - !LAZY_SAFE_BLOBS.includes(profile.blobs.kind) - ); + return !LAZY_SAFE_STRUCTURED.includes(profile.structured.kind); } diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index f73dacea8..db1332e84 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -23,12 +23,15 @@ * through it. */ +import { rm } from 'node:fs/promises'; import path from 'node:path'; import { getDataDir } from '../../data-dir.js'; +import { sanitizeId } from '../../utils/fs.js'; import { acquireWorkspaceOperationLease, commitWorkspaceIdentity, + getWorkspaceHandle, getWorkspaceKey, } from '../workspace.js'; import { DiskBlobStore } from './backends/disk/blob-store.js'; @@ -40,7 +43,6 @@ import { DiskWorkspaceRepository, workspaceRegistryPath, } from './backends/disk/workspace-repository.js'; -import { SqliteBlobStore } from './backends/sqlite/blob-store.js'; import { SqliteStoreContext } from './backends/sqlite/database.js'; import { SqliteStructuredStore } from './backends/sqlite/structured-store.js'; import { SqliteWorkspaceRepository } from './backends/sqlite/workspace-repository.js'; @@ -112,11 +114,12 @@ function assertActiveWorkspace(workspaceKey: string, canvasId: string): void { } /** - * Where the SQLite profile keeps everything it has. + * Where the SQLite profile keeps its records. * * One file, beside the Disk backend's own registry in the data directory, so * an operator can find both in the same place. `HUABU_SQLITE_PATH` overrides - * it for deployments that keep their database elsewhere. + * it for deployments that keep their database elsewhere. A Space's *bytes* are + * not in it — see {@link detachedBlobRoot}. */ export function sqliteDatabasePath(dataDir: string = getDataDir()): string { const configured = process.env['HUABU_SQLITE_PATH']?.trim(); @@ -124,6 +127,41 @@ export function sqliteDatabasePath(dataDir: string = getDataDir()): string { return path.join(dataDir, 'storage', 'sqlite', 'huabu.sqlite'); } +/** + * Where a Space keeps its bytes when the structured backend has no folder for + * it. + * + * Blobs are always files (`ports/blob.ts`), so a profile whose records are + * rows still needs somewhere on a file system to put uploads, artifacts, the + * guide document and the memory body. The Server owns that directory rather + * than the user: it is not a Workspace folder, nothing in it is a Space + * record, and none of the Disk-only features that need a real Space tree + * become available because it exists. + * + * Scoped by Workspace first, because a Space belongs to exactly one and its + * bytes should travel and be removed with it. `HUABU_BLOB_ROOT` overrides the + * base for deployments that keep bytes on a different volume. + */ +export function detachedBlobRoot(dataDir: string = getDataDir()): string { + const configured = process.env['HUABU_BLOB_ROOT']?.trim(); + return configured ? configured : path.join(dataDir, 'storage', 'blobs'); +} + +function detachedSpaceRoot(canvasId: string): string { + const workspace = getWorkspaceHandle(); + if (!workspace) { + throw new Error( + `Blob scope for Space "${canvasId}" needs an active Workspace. ` + + 'Activate one before reading or writing bytes.', + ); + } + return path.join( + detachedBlobRoot(), + sanitizeId(workspace.workspaceId, 'workspaceId'), + sanitizeId(canvasId, 'canvasId'), + ); +} + /** * Release a rejected streaming body that storage never fully consumed. * @@ -244,16 +282,23 @@ function composeSpace(storage: Storage, canvasId: string): Space { }; } +/** + * The blob connection for this profile. + * + * One adapter, two placements. Where Disk also keeps the records, a Space's + * bytes stay inside the Space folder — byte-for-byte the layout every existing + * Workspace has. Where the records are rows, the same adapter writes the same + * layout under a Server-owned root instead, which is what makes a hybrid + * profile (SQL records, ordinary files) an ordinary deployment. + */ function buildBlobStore(profile: StorageProfile): BlobStore { - switch (profile.blobs.kind) { - case 'disk': - return new DiskBlobStore(); - case 'sqlite': - return new SqliteBlobStore(sqliteConnection()); - default: - // Unreachable: validateStorageProfile rejects unimplemented kinds. - throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); + if (profile.blobs.kind !== 'disk') { + // Unreachable: validateStorageProfile rejects unimplemented kinds. + throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); } + return profile.structured.kind === 'disk' + ? new DiskBlobStore() + : new DiskBlobStore(detachedSpaceRoot); } function buildStructuredStore(profile: StorageProfile): StructuredStore { @@ -319,9 +364,9 @@ let spaceCreateTail: Promise = Promise.resolve(); * The one SQLite connection this process holds, opened on first need. * * Opening it is synchronous, which is why the on-demand path stays legal for - * this profile: there is no `await` to skip. Both storage axes and the - * Workspace repository borrow it, because they are one database file and a - * second connection would be a second writer. + * this profile: there is no `await` to skip. The structured store and the + * Workspace repository both borrow it, because they are one database file and + * a second connection would be a second writer. */ function sqliteConnection(): SqliteStoreContext { if (sqlite) return sqlite; @@ -430,6 +475,32 @@ export function adoptWorkspaceDirectory( return materializedWorkspaces().adopt(workspacePath); } +/** + * Create a Workspace that has no directory. + * + * The counterpart to {@link adoptWorkspaceDirectory} for a backend where a + * Workspace is a row: nothing to adopt, so a name is the whole of it. It is + * not a port member for the same reason locating a Workspace is not — Disk + * could only serve it by inventing a folder the user never picked, and the + * point of the port is that it says nothing about where a Workspace is. + * + * A deployment that keeps Workspaces in a database needs this to hold more + * than the one the Server opens for itself, which is the whole of multi- + * Workspace support there: every other operation — list, activate, rename, + * forget — is already on the port. + */ +export function createNamedWorkspace(name: string): Promise { + const repository = getWorkspaceRepository(); + if (!(repository instanceof SqliteWorkspaceRepository)) { + throw new StorageProfileError( + `The "${activeProfile().structured.kind}" structured backend keeps ` + + 'Workspaces as directories, so a Workspace is created by adopting a ' + + 'folder rather than by name.', + ); + } + return repository.create(name); +} + /** The registered Workspace materialized at a directory, if there is one. */ export function workspaceAtDirectory( workspacePath: string, @@ -680,6 +751,16 @@ export async function deleteSpace( area.deleteAll(), ), ); + // Where the record is a row, nothing else will ever remove the + // directory those areas sat in. Sweeping the areas is the port's + // contract; removing what composition placed them under is this + // module's, and it is what stops a deleted Space leaving a husk behind. + if (storage.profile.structured.kind !== 'disk') { + await rm(detachedSpaceRoot(canvasId), { + recursive: true, + force: true, + }); + } return await started.session.finish(); } catch (error) { await started.session.abort(); diff --git a/apps/server/src/modules/storage/testing.ts b/apps/server/src/modules/storage/testing.ts index aeeeae85c..658f2a96b 100644 --- a/apps/server/src/modules/storage/testing.ts +++ b/apps/server/src/modules/storage/testing.ts @@ -36,7 +36,7 @@ import type { Storage } from './storage.js'; */ export const PRODUCT_STORAGE_PROFILES: readonly StorageProfile[] = [ { structured: { kind: 'disk' }, blobs: { kind: 'disk' } }, - { structured: { kind: 'sqlite' }, blobs: { kind: 'sqlite' } }, + { structured: { kind: 'sqlite' }, blobs: { kind: 'disk' } }, ]; /** Readable name for a profile, for test titles. */ @@ -44,6 +44,11 @@ export function describeProfile(profile: StorageProfile): string { return `${profile.structured.kind}/${profile.blobs.kind}`; } +function restoreEnv(key: string, previous: string | undefined): void { + if (previous === undefined) delete process.env[key]; + else process.env[key] = previous; +} + export interface MountedTestStorage { readonly profile: StorageProfile; readonly storage: Storage; @@ -51,9 +56,9 @@ export interface MountedTestStorage { * The temporary directory this mount owns. * * For a Disk profile it is the Workspace itself; for a profile that keeps - * Workspaces in a database it is only where the harness put that database. - * Either way it is the harness's own business — a case that reads it has - * stopped being evidence of anything portable. + * Workspaces in a database it is only where the harness put that database + * and the Space byte root. Either way it is the harness's own business — a + * case that reads it has stopped being evidence of anything portable. */ readonly workspacePath: string; /** @@ -85,6 +90,7 @@ export async function mountTestWorkspace( const safePrefix = prefix.replace(/[^a-zA-Z0-9._-]/g, '-'); const workspacePath = mkdtempSync(path.join(tmpdir(), safePrefix)); const previousSqlitePath = process.env['HUABU_SQLITE_PATH']; + const previousBlobRoot = process.env['HUABU_BLOB_ROOT']; if (profile.structured.kind === 'disk') { // Prepares and commits the Workspace, exactly as a synchronous activation @@ -93,12 +99,13 @@ export async function mountTestWorkspace( // namespace selected inside it. setWorkspacePath(workspacePath); } else { - // Nothing to pick. The Workspace is a row the backend creates on first - // start, and `initStorage` activates it — which is exactly the behaviour - // that lets this profile run with no folder at all. The temp directory - // only gives this mount its own database file so parallel suites do not - // share one. + // No Workspace folder to pick. The Workspace is a row the backend creates + // on first start, and `initStorage` activates it — which is exactly the + // behaviour that lets this profile run without one. The temp directory + // only gives this mount its own database file and its own byte root, so + // parallel suites do not share either. process.env['HUABU_SQLITE_PATH'] = path.join(workspacePath, 'huabu.sqlite'); + process.env['HUABU_BLOB_ROOT'] = path.join(workspacePath, 'blobs'); } const storage = await initStorage(profile); @@ -119,11 +126,8 @@ export async function mountTestWorkspace( }, async close(): Promise { await closeStorage(); - if (previousSqlitePath === undefined) { - delete process.env['HUABU_SQLITE_PATH']; - } else { - process.env['HUABU_SQLITE_PATH'] = previousSqlitePath; - } + restoreEnv('HUABU_SQLITE_PATH', previousSqlitePath); + restoreEnv('HUABU_BLOB_ROOT', previousBlobRoot); rmSync(workspacePath, { recursive: true, force: true }); }, }; diff --git a/apps/server/src/modules/workspaces.route.test.ts b/apps/server/src/modules/workspaces.route.test.ts index f399ca455..6d73384fc 100644 --- a/apps/server/src/modules/workspaces.route.test.ts +++ b/apps/server/src/modules/workspaces.route.test.ts @@ -35,6 +35,8 @@ function handleOf({ workspaceId, name }: TestMember): TestHandle { const testState = vi.hoisted(() => ({ managed: false, + /** Whether the configured structured backend files Workspaces as folders. */ + materializes: true, active: null as TestHandle | null, activePath: null as string | null, members: [] as TestMember[], @@ -45,6 +47,10 @@ const testState = vi.hoisted(() => ({ const storageMocks = vi.hoisted(() => ({ resetStorageCache: vi.fn(), activateWorkspace: vi.fn(async () => {}), + createNamedWorkspace: vi.fn(async (name: string) => ({ + workspaceId: NEW_ID, + name, + })), })); const activationMocks = vi.hoisted(() => ({ @@ -145,10 +151,11 @@ vi.mock('./storage/index.js', () => ({ activateWorkspace: storageMocks.activateWorkspace, getWorkspaceRepository: () => repository, hasWorkspaceRegistry: () => testState.registryInitialized, - // These routes are the directory-shaped Workspace API, so the profile under - // test is the one that has directories. The non-materializing branches are - // covered where they are the point. - materializesWorkspaces: () => true, + // These routes are mostly the directory-shaped Workspace API, so the default + // profile under test is the one that has directories; a case that is about + // the other kind flips `materializes`. + materializesWorkspaces: () => testState.materializes, + createNamedWorkspace: storageMocks.createNamedWorkspace, resetStorageCache: storageMocks.resetStorageCache, unavailableCapabilityMessage: (id: string) => `capability ${id}`, adoptWorkspaceDirectory: locatorMocks.adoptWorkspaceDirectory, @@ -204,6 +211,7 @@ async function buildApp() { beforeEach(() => { testState.managed = false; + testState.materializes = true; testState.registryInitialized = true; testState.members = [ { @@ -550,3 +558,56 @@ describe('plural Workspace management routes', () => { } }); }); + +/** + * A deployment whose Workspaces are rows still holds more than one. + * + * Everything else the collection needs — list, activate, rename, forget — is + * already on the port and backend-neutral. Creation is the one operation the + * folder API could not express, because there is no folder to name. + */ +describe('Workspace collection on a backend with no folders', () => { + beforeEach(() => { + testState.materializes = false; + }); + + it('creates a Workspace from a name alone', async () => { + const app = await buildApp(); + try { + const response = await app.inject({ + method: 'POST', + url: '/workspaces', + payload: { name: 'Second' }, + }); + + expect(response.statusCode).toBe(201); + expect(storageMocks.createNamedWorkspace).toHaveBeenCalledWith('Second'); + expect(response.json()).toEqual({ + workspaceId: NEW_ID, + name: 'Second', + // No folder to report, and not the active one. + path: null, + active: false, + }); + } finally { + await app.close(); + } + }); + + it('asks for the name it can actually use', async () => { + const app = await buildApp(); + try { + const response = await app.inject({ + method: 'POST', + url: '/workspaces', + payload: { path: '/tmp/somewhere' }, + }); + + expect(response.statusCode).toBe(400); + expect(response.json().message).toMatch(/name is required/i); + expect(storageMocks.createNamedWorkspace).not.toHaveBeenCalled(); + } finally { + await app.close(); + } + }); +}); diff --git a/apps/server/src/modules/workspaces.route.ts b/apps/server/src/modules/workspaces.route.ts index 03fbb622b..77b78dc84 100644 --- a/apps/server/src/modules/workspaces.route.ts +++ b/apps/server/src/modules/workspaces.route.ts @@ -12,12 +12,12 @@ import { resetPreprocessDispatcher } from './preprocessing/index.js'; import { activateWorkspace, adoptWorkspaceDirectory, + createNamedWorkspace, ensureWorkspaceManifestOnDisk, getWorkspaceRepository, hasWorkspaceRegistry, materializesWorkspaces, resetStorageCache, - unavailableCapabilityMessage, workspaceAtDirectory, workspaceDirectory, workspaceIdentityOnDisk, @@ -226,17 +226,6 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { app.post<{ Body: WorkspaceCreateRequest }>('/', async (request, reply) => { const rejected = rejectReadOnlyMutation(request, reply); if (rejected) return rejected; - if (!materializesWorkspaces()) { - // Creating a Workspace here means adopting a folder. Where Workspaces - // are rows the Server opens its own, and adding more of them is a - // by-name operation this API does not have yet. - return sendError( - reply, - 409, - unavailableCapabilityMessage('workspace-directory'), - 'STORAGE_CAPABILITY_UNAVAILABLE', - ); - } const parsed = workspaceCreateSchema.safeParse(request.body); if (!parsed.success) { @@ -247,8 +236,31 @@ const workspacesRoutes: FastifyPluginAsync = async (app) => { ); } + // A Workspace that is a row is created by name: there is no folder to + // adopt, prepare, or fork a child process for. The deployment still holds + // as many Workspaces as it likes — this is the only one of the collection + // operations the folder API could not already express. + if (!materializesWorkspaces()) { + const name = parsed.data.name; + if (!name) { + return sendError(reply, 400, 'Workspace name is required'); + } + try { + return reply + .status(201) + .send(descriptor(await createNamedWorkspace(name))); + } catch (error) { + return sendPreparationError(reply, error); + } + } + + const requestedPath = parsed.data.path; + if (!requestedPath) { + return sendError(reply, 400, 'Workspace path is required'); + } + try { - const workspacePath = resolveWorkspacePath(parsed.data.path); + const workspacePath = resolveWorkspacePath(requestedPath); const repository = getWorkspaceRepository(); const existing = workspaceAtDirectory(workspacePath); if (existing) { diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 3a0187248..c79169f71 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -74,7 +74,7 @@ Key points: - Canonical World preview identity is server-owned: non-system commands cannot create, repoint, or delete managed previews. Users may move and resize them. Ordinary Spaces may create and delete their own `spacePreview` nodes through normal UI commands. - Legacy `canvasRef`, `frameRef`, `nodeRef`, `SET_PORTAL_NODE_PINS`, and `GET /api/canvas/:worldCanvasId/references` remain compatibility surfaces for stored World data but are no longer created or exposed by the redesigned World UI. The current model is specified in [space-preview.md](./space-preview.md). - Node filenames are `safe(label).md`; the node's stable id lives in the `id:` frontmatter field. -- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Artifacts are one of four blob areas a Space has, resolved as `space(canvasId).artifacts`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Disk and SQLite are both implemented and selectable; `blobs=sqlite` requires `structured=sqlite`, because those bytes are rows in that same database. +- The Disk `BlobStore` maps each Space scope to `.artifacts/`, with blobs named `` and no manifest file — the filename is the URL key. Artifacts are one of four blob areas a Space has, resolved as `space(canvasId).artifacts`: `put()` requires an existing Space record, while reads and `deleteAll()` remain available for recovery after a record goes missing. `CanvasStore` owns no artifact methods. Blobs are always files: `disk` is the only implemented backend and `azure` the settled next one, so no structured backend is ever asked to hold bytes and any structured backend pairs with any blob backend. - Remote PDF preprocessing writes the already-fetched source bytes into the Space BlobStore as `artifact-.pdf` before structured persistence and replaces the node's remote `src` with that key. As with other artifact imports, this blob write precedes the node write operation; a later structured persistence failure may therefore leave an unreferenced blob until Space deletion, while a blob-write failure degrades to retaining the remote URL. - Events are append-only JSONL (`events.jsonl`); each line is `{ ts: number, payload: RecentAction }`. - The memory analyzer reads Space existence and at most 100 recent action events through one `SpaceHandle`. A missing Space skips the pass before reading memory files or calling the model; corrupt part data still fails the pass. Memory body/state files remain materialized workspace paths, while Agenetes-owned chat history is not part of the curator bundle. @@ -83,18 +83,27 @@ Key points: - Canonical Task and Run records live in `.history/tasks.json`, owned by Huabu Server through the async `SpaceTasks` ledger (`read`, `create`, and `runs.create`/`runs.update`/`runs.complete`). The Disk adapter validates the versioned snapshot and referential integrity on every read, rejects duplicate identifiers and Runs whose Task is absent, serializes read-modify-write operations with an independent per-Canvas process-local mutex, and atomically replaces the file. This mutex is intentionally separate from the Canvas topology write coordinator, so Task metadata does not participate in `space.json` version CAS. - Legacy chat files are one-way migrated into `chat_v2/` at workspace activation and retired to `.bak`: the oldest pi-ai `Context` `chat/.json` via `migrate-chat-threads.ts` (hop 1), then the M5.6 `chat/.turns.jsonl` / `.active.json` via `migrate-chat-turns.ts` (hop 2). If hop 1 finds both formats after an interrupted launch, it completes a strict converted prefix atomically or preserves an existing tail when the full conversion is its prefix. Divergent logs are retained rather than guessed or overwritten; hop 2 skips the paired turn log while a valid same-thread legacy Context remains or its JSON cannot be read safely, so a later activation can retry both copies without blocking unrelated migrations. The obsolete `CanvasStore` chat methods and `chatPath()` helper were removed in Phase 2; `chatDir()` remains because change-review and agent-owned files still use that directory. -## 2b. SQLite layout — the profile with no folders +## 2b. SQLite layout — records in a database, bytes in files -`HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` selects the second implemented profile. It needs **no Workspace folder and no Space directories**: every durable thing is a row in one file. +`HUABU_STRUCTURED_BACKEND=sqlite` selects the second implemented structured backend. The blob axis stays `disk`, because bytes are always files. It needs **no Workspace folder and no Space directories**: every record is a row, and the only directories are the ones a Space's bytes sit in. ``` / storage/sqlite/ - huabu.sqlite # everything below; override with HUABU_SQLITE_PATH + huabu.sqlite # every record; override with HUABU_SQLITE_PATH huabu.sqlite-wal # WAL sidecars, managed by SQLite huabu.sqlite-shm + storage/blobs/ # override with HUABU_BLOB_ROOT + / + / + skill.md # blob area `guide` + .artifacts/ # blob area `artifacts` + .memory/space.md # blob area `memory` + .upload/ # blob area `uploads` ``` +The byte root is Server-owned, not a Workspace folder: nothing in it is a Space record, and the Disk-only capabilities below stay unavailable because they need a real Space tree, not merely a directory. The layout beneath `/` is byte-for-byte the one the Disk profile uses inside a Space folder, because it is the same adapter — composition only tells it where the Space's root is. + | Table | Holds | | ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | | `workspaces` | Workspace identity and display name; `forgotten_at` is how `remove()` forgets a member without destroying what it owns | @@ -105,16 +114,14 @@ Key points: | `tasks` | The versioned Task/Run snapshot | | `delta_log` | The executor's private journal, keyed by committed Space version | | `space_extensions` | One row per extension namespace — the parent an owner's own tables cascade from | -| `blobs` | Artifact, guide, memory, and upload bytes, keyed by `(workspace, canvas, area, name)` | Owner-created tables hanging off `space_extensions`: `extension_documents` (memory bookkeeping and the debug prompt log) and `agenetes_threads` / `agenetes_events` / `agenetes_turns` (the conversation stores). Storage never reads them; deleting a Space removes them by cascade. Notes an operator needs: - **A Workspace is a row.** There is no folder to pick, so the Server creates and activates one on first start and reports `path: null` with `canChangeWorkspace: false`; the client shows no picker. Switching Workspaces re-scopes the one connection and reopens nothing. -- **The connection is shared.** The structured store, the blob store, and the Workspace repository use one `node:sqlite` connection, opened in WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys enforced. One process, one connection: nothing here promises a multi-process fence. -- **Blob bytes are rows**, read and written whole. The profile is sized for the documents and images a Space holds, not arbitrarily large media, and the database grows to the size of everything ever uploaded. `materialize()` spools to the OS temp directory and unlinks on release. -- **`blobs` has no foreign key to `spaces`.** Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and that saga must also be able to sweep orphans for a record that is already missing. +- **The connection is shared.** The structured store and the Workspace repository use one `node:sqlite` connection, opened in WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys enforced. One process, one connection: nothing here promises a multi-process fence. +- **Bytes are outside the database.** The blob axis is a file system on every profile, so a Space's uploads, artifacts, guide and memory body are ordinary files under the byte root and the database stays the size of its records. Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and, where the record is a row, composition also removes the `//` directory it placed those areas under, because nothing else would. - **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, user-authored skills under `setting/skills/`, and Windows directory-handle coordination. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. ## 3. Storage composition and ownership diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 12434bcc6..39a5ab10b 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -60,13 +60,14 @@ Last updated: 2026-09-04 > the decision table in §2 marks what each step has actually settled. > > Phase 5 is specified in §12.9 and is **implemented by this branch**. -> `HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` is a real -> profile: Workspaces, Spaces, nodes, logs, Tasks, blob bytes, and agent -> conversations all live in one database file under -> `/storage/sqlite/`, and the deployment needs no Workspace folder -> and no Space directories. What it does **not** serve is enumerated in -> §12.9.4 and declared in `storage/capabilities.ts`, which is the list an -> operator sees at startup. Postgres and Azure adapters still do not exist. +> `HUABU_STRUCTURED_BACKEND=sqlite` is a real profile: Workspaces, Spaces, +> nodes, logs, Tasks, and agent conversations are rows in one database file +> under `/storage/sqlite/`, and the deployment needs no Workspace +> folder and no Space directories. The blob axis stays `disk`, because bytes +> are always files — SQL records beside ordinary files is the profile, not a +> compromise within it. What it does **not** serve is enumerated in §12.9.4 +> and declared in `storage/capabilities.ts`, which is the list an operator +> sees at startup. Postgres and Azure adapters still do not exist. --- @@ -96,8 +97,8 @@ built above these ports, but its form is intentionally unresolved here. | Topic | Status | Current position | | ------------------------------------------------------ | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Separate authoritative structured and blob ports | **Accepted** (P1, merged) | Storage is composed from `StructuredStore` and `BlobStore`; there is no single backend interface that mixes both concerns. | -| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk is selectable; SQLite has an isolated contract-preview adapter but is not selectable; Postgres has no adapter. | -| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations. Only Disk exists. | +| Structured backend family | **Settled direction** | Support Disk, SQLite, and Postgres implementations. Disk and SQLite are selectable; Postgres has no adapter. | +| Blob backend family | **Settled direction** | Support Disk and Azure Blob implementations — both file systems. Only Disk exists. A structured backend never holds bytes, so the two axes share nothing and any implemented pairing is a valid deployment. | | Independent composition | **Accepted** (P1, merged) | `StorageProfile` has two env-parsed axes; `validateStorageProfile` fails fast on unimplemented kinds and is the extension point for combination rules. The lazy `getStorage()` path now rejects profiles whose adapters require awaited initialization (§12.1.1). | | Blob port contract | **Accepted** (P1, merged) | Connection → scope, stream-oriented, no permanent absolute path in the common contract; `materialize()` returns a bounded lease for the one consumer needing a file. Replacement atomicity and post-release lease semantics are contract terms, not adapter accidents (§6.2, §12.1.1). | | Concrete interface shape and async migration | **Accepted** (P4) | Blob and portable structured repositories are async. `StructuredStore` exposes catalogue/lifecycle and scoped Space handles; the structured mutations enumerated in §12.4 use those ports. Disk-only physical capabilities remain explicit blockers for selecting another profile. | @@ -152,9 +153,8 @@ external-note discovery watches `nodes/`, and export archives the entire Space directory. Therefore wrapping `CanvasStore` in a database adapter would not by itself make the application backend-neutral. -Runtime Canvas/Space persistence remains Disk-only. An isolated SQLite -structured adapter exists for contract and integration tests, while Postgres -and Azure Blob adapters do not yet exist. +Runtime Canvas/Space persistence is Disk by default and SQLite by selection +(§12.9). Postgres and Azure Blob adapters do not yet exist. ## 4. Goals @@ -315,8 +315,9 @@ into place makes the failed write invisible instead of unremovable. ### 6.3 Composition Configuration has two axes. The runtime-selectable profile carries only a -backend kind per axis. The isolated SQLite preview receives its explicit -database filename directly and is not constructed from this profile: +backend kind per axis; where an adapter needs a location, composition resolves +it (`HUABU_SQLITE_PATH`, `HUABU_BLOB_ROOT`) rather than the profile carrying +it: ```ts interface StorageProfile { @@ -334,15 +335,16 @@ connection/pool. Credential references, config storage, and deployment-level backend migration remain open — a Postgres DSN or Azure container reference will extend these members. -Some combinations require capability validation. For example, Postgres plus a -node-local DiskBlob implementation is unsafe in a multi-replica deployment -unless the path is a deliberately shared and supported filesystem. SQLite on a -network filesystem has different correctness and availability constraints from -local SQLite. `validateStorageProfile()` is where such rules live; today it -rejects recognized kinds that are unavailable or deliberately unselectable, -including SQLite's preview-specific diagnostic, so an unsupported profile -fails at startup with an actionable message rather than nondeterministically -while serving data. +The axes share nothing — records go to the structured backend, bytes go to a +file system — so every pairing of implemented backends is a valid deployment +today. Future cross-axis rules are still possible on _deployment_ grounds +rather than storage ones: Postgres plus a node-local DiskBlob root is unsafe +across replicas unless the path is a deliberately shared filesystem, and +SQLite on a network filesystem has different correctness and availability +constraints from local SQLite. `validateStorageProfile()` is where such rules +would live; today it only rejects recognized kinds that have no adapter, so an +unsupported profile fails at startup with an actionable message rather than +nondeterministically while serving data. ### 6.4 One Space handle, four dispositions — revised direction @@ -1819,8 +1821,8 @@ justify. kind now names only kinds that exist. The wider vocabulary a profile may _request_ moved to `profile.ts` as `RequestedStructuredKind`, which is what preserves the actionable "not implemented yet" error for a configured - `sqlite` or `postgres`. `BlobBackendKind` still carries `azure` on the same - footing and was left alone as Phase-1 surface. + `sqlite` or `postgres`. Phase 5 narrowed `BlobBackendKind` the same way, to + `'disk'`, leaving `azure` in `RequestedBlobKind`. Not changed, deliberately: `authoritativeInsert` and the `write-suppressed` put outcome remain in the portable shapes. At this phase boundary, both @@ -2535,28 +2537,39 @@ guarantees. ### 12.9 Phase 5 — SQLite as a selectable profile — **implemented** -Phase 5 adds a second structured backend and a second blob backend, and turns -them on. The question it answers is not "does the boundary compile against a -database" — §12.8's harness already asked that — but the harder one behind it: -can a deployment run with **no Workspace folder and no Space directories at -all**, and can it say plainly what it gives up by doing so. +Phase 5 adds a second structured backend and turns it on. The question it +answers is not "does the boundary compile against a database" — §12.8's +harness already asked that — but the harder one behind it: can a deployment +run with **no Workspace folder and no Space directories at all**, and can it +say plainly what it gives up by doing so. -`HUABU_STRUCTURED_BACKEND=sqlite HUABU_BLOB_BACKEND=sqlite` is the profile. -Everything durable — Workspaces, Spaces, nodes, events, changes, Tasks, blob -bytes, extension namespaces, and agent conversations — lives in one file at +`HUABU_STRUCTURED_BACKEND=sqlite` is the profile. Every record — Workspaces, +Spaces, nodes, events, changes, Tasks, extension namespaces, and agent +conversations — is a row in one file at `/storage/sqlite/huabu.sqlite` (override with `HUABU_SQLITE_PATH`), -beside the Disk backend's own registry at `/storage/disk/`. Postgres -and Azure Blob adapters still do not exist. +beside the Disk backend's own registry at `/storage/disk/`. + +The blob axis stays `disk`, and there is no SQLite blob adapter. **Bytes are +always a file system** — a local directory now, Azure Blob later — so no +structured backend is asked to hold them and the two axes genuinely share +nothing. A Space's bytes therefore need a directory even where its record does +not: composition supplies `/storage/blobs///` +(override the base with `HUABU_BLOB_ROOT`) and hands it to the same Disk blob +adapter, which writes the same area layout it writes inside a Space folder. +That directory is Server-owned and holds nothing but bytes; it is not a +Workspace folder and it is not a Space tree, so none of §12.9.4's Disk-only +capabilities become available because it exists. Postgres and Azure Blob +adapters still do not exist. #### 12.9.1 Scope and lifecycle - Built-in `node:sqlite`. No package, no native addon. That is not a production driver decision (§5); it is what let this phase be about the boundary rather than about dependencies. -- One connection per process, shared by the structured store, the blob store, - and the Workspace repository — because they are one file, and two writers to - one SQLite file is a lock error rather than a queue. The connection opens in - WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys +- One connection per process, shared by the structured store and the Workspace + repository — because they are one file, and two writers to one SQLite file + is a lock error rather than a queue. The connection opens in WAL with + `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys enforced. - Opening it is _synchronous_, so the composition root can hand out a Workspace repository before `initStorage()` has been awaited — which managed @@ -2574,15 +2587,15 @@ Schema versioning uses `PRAGMA user_version`; migrations run transactionally, reject databases from the future, and create `STRICT` tables with foreign keys enabled. Version 1 holds Workspaces, Space records and World membership, complete node JSON with opaque revision tokens, ordered events, coalesced -changes, Task/Run snapshots, extension namespaces, the private delta journal, -and blob bytes. +changes, Task/Run snapshots, extension namespaces, and the private delta +journal. No table holds bytes. -Blobs deliberately carry **no** foreign key to `spaces`. The two ports are -configured independently and their lifecycles are joined only by the deletion -saga in `storage.ts`, which sweeps every blob area _before_ the structured -record goes; a foreign key would move that ordering decision into the schema -and would refuse the orphan sweep the saga performs when a record is already -missing. +The two ports are configured independently and their lifecycles are joined +only by the deletion saga in `storage.ts`, which sweeps every blob area +_before_ the structured record goes and can therefore also sweep orphans for a +record that is already missing. Where the record is a row, that saga then +removes the `//` directory composition placed those +areas under, because no structured delete ever will. Every ordered Space write applies node mutations, record replacement, and the optional delta insert in one immediate transaction. Same-baseline writers have @@ -2639,26 +2652,21 @@ same cascade as everything else. Six capabilities are Disk-only, declared in `storage/capabilities.ts`, logged at startup, and refused at their own call sites in the same words: -| Capability | What is lost | Why it is not emulated | -| --------------------------------------------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | -| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | -| `reveal-space-folder` | "Show me this in Finder" | Without a folder there is nothing to show. | -| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | They sandbox on the Space directory. The first-party agent edits nodes through the Canvas tools instead. | -| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | -| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | -| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | Every blob scope is Space-scoped, so a Workspace-level document has nowhere to live yet. A Space's _own_ memory body is unaffected. | -| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | -| `space-directory-handle-coordination` | Windows rename-while-watched | No directory, no problem. | - -Two further limits are not capability rows because nothing refuses them, they -are simply properties of the backend: - -- **Blob size.** Bytes are a row read and written whole, so the profile is - sized for the documents and images a Space holds, not for arbitrarily large - media, and the database grows to the size of everything ever uploaded. - `materialize()` spools to the OS temp directory and unlinks on release — - which is what `BlobLease`'s post-release rule was written to keep honest. +| Capability | What is lost | Why it is not emulated | +| --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | +| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | +| `reveal-space-folder` | "Show me this in Finder" | Without a folder there is nothing to show. | +| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | They sandbox on the Space directory. The first-party agent edits nodes through the Canvas tools instead. | +| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | +| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | +| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | It is a file the user edits at the root of a Workspace they chose, and there is no such folder. A Space's _own_ memory body is a blob and is unaffected. | +| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | +| `space-directory-handle-coordination` | Windows rename-while-watched | No directory, no problem. | + +One further limit is not a capability row because nothing refuses it; it is +simply a property of the backend: + - **Multi-process access.** One process, one connection. WAL and `busy_timeout` make a second reader survivable, and nothing here promises a multi-process deletion fence or a distributed transaction. @@ -2666,16 +2674,20 @@ are simply properties of the backend: #### 12.9.5 Proof The reusable contracts — structured store, Space repository, nodes, ordered -write, logs, Tasks, extension substrate, **blob store**, and **Workspace -repository** — run against Disk and against real temporary SQLite files. +write, logs, Tasks, extension substrate, and **Workspace repository** — run +against Disk and against real temporary SQLite files. The blob contract runs +once, against the one blob adapter there is. -`PRODUCT_STORAGE_PROFILES` gains `sqlite/sqlite`, so the §12.8 product-boundary +`PRODUCT_STORAGE_PROFILES` gains `sqlite/disk`, so the §12.8 product-boundary suite runs unchanged against it: World bootstrap, Space creation, ordered writes through every node read shape, version conflict, bytes in every area, the cross-store put guard, extension isolation and cleanup, the log families, the Task ledger, deletion, World protection, and — added here — that all of it is still there after a restart. That suite names no directory and no filename; -`module-boundaries.test.ts` enforces that mechanically. +`module-boundaries.test.ts` enforces that mechanically. What _is_ about +placement has its own small suite instead (`detached-blobs.test.ts`): bytes +land as real files under the Workspace-scoped root, one Workspace's root is +not another's, and deleting a Space leaves no directory behind. SQLite integration tests additionally cover strict schema creation, WAL and foreign-key pragmas read back on a second connection, close/reopen @@ -2683,10 +2695,9 @@ persistence, an immutable v1 fixture, future-version rejection, migration rollback, SQL fault injection, foreign-key cascades, revision safety across delete/recreate, `JSON.stringify` encoding parity, incremental streaming and early abort, batched `readMany`, Workspace scoping and handle invalidation -across a switch, forget-without-delete, and blob byte fidelity, lease -lifetime, and Workspace isolation. The Agenetes conversation stores have their -own suite against a mounted profile, covering round-trip, isolation, restart, -and destruction with the Space. +across a switch, and forget-without-delete. The Agenetes conversation stores +have their own suite against a mounted profile, covering round-trip, +isolation, restart, and destruction with the Space. ### 12.10 Later phases — provisional @@ -2915,7 +2926,7 @@ Before a new backend is production-ready: | [`.../storage/backends/disk/legacy/canvas-store-cache.ts`](../../apps/server/src/modules/storage/backends/disk/legacy/canvas-store-cache.ts) | Bounded LRU of legacy Disk Space objects. The single owner both the adapter and the facade resolve through, and the real limit of `space(id)` identity (§12.2.4). | | [`apps/server/src/modules/storage/profile.ts`](../../apps/server/src/modules/storage/profile.ts) | Two-axis backend selection from env, and the fail-fast validation hook for unsupported combinations. | | [`apps/server/src/modules/storage/backends/disk/`](../../apps/server/src/modules/storage/backends/disk/) | Every Disk implementation: blob/structured stores, the Space collection, and the per-Space record, node, log, and Task adapters, in-process batch restoration, and the legacy class under `legacy/`. | -| [`apps/server/src/modules/storage/backends/sqlite/`](../../apps/server/src/modules/storage/backends/sqlite/) | Isolated `node:sqlite` structured adapter, strict schema and migrations, transaction-backed writes, and real-file contract/integration tests; available for proof but not runtime-selectable. | +| [`apps/server/src/modules/storage/backends/sqlite/`](../../apps/server/src/modules/storage/backends/sqlite/) | Selectable `node:sqlite` structured adapter: strict schema and migrations, Workspace-scoped Spaces, transaction-backed writes, and real-file contract/integration tests. Records only — bytes stay on the blob axis. | | [`.../storage/compatibility/canvas.ts`](../../apps/server/src/modules/storage/compatibility/canvas.ts) | Residual Disk read surface plus direct-module lifecycle test fixtures; production structured mutations enumerated in §12.4 use the portable ports. | | [`apps/server/src/modules/agent/memory/analyzer.ts`](../../apps/server/src/modules/agent/memory/analyzer.ts) | P3 repository consumer for strict Space existence, bounded action events, and intent episodes; physical chat and memory files remain Disk-specific. | | [`apps/server/src/modules/canvas/write-coordinator.ts`](../../apps/server/src/modules/canvas/write-coordinator.ts) | Canvas mutation coordinator and per-Space write lock, held across asynchronous node read, revision CAS, and put. | diff --git a/packages/shared/src/types/api/workspace.ts b/packages/shared/src/types/api/workspace.ts index efeb439a4..9ffc9dc5e 100644 --- a/packages/shared/src/types/api/workspace.ts +++ b/packages/shared/src/types/api/workspace.ts @@ -55,8 +55,16 @@ export const workspaceDescriptorSchema = z.object({ export type WorkspaceDescriptor = z.infer; /** Body for `POST /api/workspaces`. */ +/** + * Body for `POST /api/workspaces`. + * + * `path` is the folder to adopt, and is how a Workspace is created where a + * Workspace *is* a folder. Where it is a row in a database there is nothing to + * adopt, so `name` alone creates one. Which form is required is the Server's + * answer, because the configured backend is the only thing that knows. + */ export const workspaceCreateSchema = z.object({ - path: z.string().min(1, 'Workspace path is required'), + path: z.string().min(1, 'Workspace path is required').optional(), name: z.string().trim().min(1, 'Workspace name is required').optional(), }); export type WorkspaceCreateRequest = z.infer; From d92bc4ed12d33d17b84bdc6e769a6a85567de557 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 10:28:54 +0800 Subject: [PATCH 09/15] refactor(storage): give each backend one owner for its area of the data dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Space byte root landed at `/storage/blobs/`, which names a concept where its neighbour `storage/sqlite/` names an implementation. It is the Disk blob adapter's directory, so it belongs under `storage/disk/` with the rest of that backend's state. That puts two adapters in one directory — the structured store's `workspaces.json` Workspace registry and the blob store's Space byte roots — so they get separate subtrees and one file decides both: storage/disk/ workspaces.json Disk structured store blobs//… Disk blob store (HUABU_BLOB_ROOT moves this alone) Keeping the registry out of `blobs/` is not tidiness. The blob store deletes whole directories — an area on `deleteAll()`, a Space's root when its record goes — and the registry is not its to delete. `backends/disk/data-dir.ts` is the one place that can make that true, and `detached-blobs.test.ts` asserts it as the path fact it is: the registry is inside the Disk area, outside the blob root, and unmoved by `HUABU_BLOB_ROOT`. Applied to the other backend too, so the rule is uniform rather than a one-off: `sqliteDatabasePath` moves from the composition root into `backends/sqlite/database.ts`, and `workspaceRegistryPath` from the Disk workspace repository into its new neighbour. `storage.ts` now builds no backend path at all — it supplies the active Workspace id and asks. A module-boundary census pins that: only those two files may name `'storage', 'disk'` or `'storage', 'sqlite'`. Also drops `sqliteDatabasePath` from the storage barrel, which nothing outside the module imported. Verified on a running Server at `structured=sqlite blobs=disk`: bytes land in `storage/disk/blobs///.artifacts/`, and deleting the Space removes that tree while the database is untouched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../modules/desktop-workspace-upgrade.test.ts | 2 +- .../modules/storage/backends/disk/data-dir.ts | 82 +++++++++++++++++++ .../disk/workspace-repository.test.ts | 6 +- .../backends/disk/workspace-repository.ts | 7 +- .../storage/backends/sqlite/database.ts | 30 +++++-- .../modules/storage/detached-blobs.test.ts | 59 ++++++++++++- apps/server/src/modules/storage/index.ts | 1 - .../modules/storage/module-boundaries.test.ts | 20 +++++ apps/server/src/modules/storage/storage.ts | 55 ++++--------- docs/architecture/canvas-storage.md | 24 ++++-- docs/proposals/multi-backend-storage.md | 25 ++++-- 11 files changed, 230 insertions(+), 81 deletions(-) create mode 100644 apps/server/src/modules/storage/backends/disk/data-dir.ts diff --git a/apps/server/src/modules/desktop-workspace-upgrade.test.ts b/apps/server/src/modules/desktop-workspace-upgrade.test.ts index 24765eda8..41a44c515 100644 --- a/apps/server/src/modules/desktop-workspace-upgrade.test.ts +++ b/apps/server/src/modules/desktop-workspace-upgrade.test.ts @@ -39,7 +39,7 @@ import path from 'node:path'; import fastify from 'fastify'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { workspaceRegistryPath } from './storage/backends/disk/workspace-repository.js'; +import { workspaceRegistryPath } from './storage/backends/disk/data-dir.js'; import { resetStorageCache } from './storage/index.js'; import { setWorkspacePath } from './workspace.js'; import workspaceRoutes from './workspace.route.js'; diff --git a/apps/server/src/modules/storage/backends/disk/data-dir.ts b/apps/server/src/modules/storage/backends/disk/data-dir.ts new file mode 100644 index 000000000..f250f8827 --- /dev/null +++ b/apps/server/src/modules/storage/backends/disk/data-dir.ts @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Where the Disk backend puts state that belongs to no Workspace folder. + * + * `layout.ts` answers "where inside a Workspace does a Space go". This file + * answers the other half: the Disk adapters also keep things in the Server's + * own data directory — a registry of Workspaces that is *about* folders rather + * than in one, and Space bytes for a deployment whose records live in a + * database and therefore has no folder at all. + * + * Both are under `/storage/disk/`, which names the backend the way + * `storage/sqlite/` names the other one. Two adapters share that directory, so + * each gets a subtree of its own and neither may grow into the other's: + * + * storage/disk/ + * workspaces.json the *structured* store's membership registry + * blobs//… the *blob* store's Space byte roots + * + * That separation is not cosmetic. The blob store deletes whole directories — + * an area on `deleteAll()`, a Space's root when its record goes — and the + * registry is not its to delete. Keeping the registry out of `blobs/` is what + * makes "sweep this Space's bytes" unable to reach it, and putting both paths + * in one file is what keeps that true when either moves. + * + * Nothing outside `storage/` may depend on these names (§12.5.2). The SQLite + * backend answers the same question for itself in `backends/sqlite/database.ts`. + */ + +import path from 'node:path'; + +import { getDataDir } from '../../../../data-dir.js'; +import { sanitizeId } from '../../../../utils/fs.js'; + +/** The Disk backend's own area in the Server data directory. */ +export function diskDataDir(dataDir: string = getDataDir()): string { + return path.join(dataDir, 'storage', 'disk'); +} + +export const WORKSPACE_REGISTRY_FILENAME = 'workspaces.json'; + +/** Structured store: the `workspaceId -> workspacePath` discovery index. */ +export function workspaceRegistryPath(dataDir: string = getDataDir()): string { + return path.join(diskDataDir(dataDir), WORKSPACE_REGISTRY_FILENAME); +} + +/** + * Blob store: the root its Space byte directories sit under. + * + * Only reached when the structured backend gives a Space no folder of its own; + * where Disk keeps the records too, a Space's bytes stay inside the Space + * folder the user can see and this path is never built. + * + * `HUABU_BLOB_ROOT` replaces it wholesale, for a deployment that keeps bytes + * on another volume. It moves the bytes and nothing else — the registry above + * is the structured store's and stays where it is. + */ +export function diskBlobRoot(dataDir: string = getDataDir()): string { + const configured = process.env['HUABU_BLOB_ROOT']?.trim(); + return configured ? configured : path.join(diskDataDir(dataDir), 'blobs'); +} + +/** + * Blob store: where one Space's areas go, Workspace-scoped. + * + * A Space belongs to exactly one Workspace, so its bytes are filed under that + * Workspace and removed with it. The directory holds bytes and nothing else: + * it is not a Workspace folder and not a Space tree, which is why none of the + * Disk-only capabilities become available because it exists. + */ +export function diskSpaceBlobRoot( + workspaceId: string, + canvasId: string, + dataDir: string = getDataDir(), +): string { + return path.join( + diskBlobRoot(dataDir), + sanitizeId(workspaceId, 'workspaceId'), + sanitizeId(canvasId, 'canvasId'), + ); +} diff --git a/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts b/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts index 7fae730b9..f1260aaea 100644 --- a/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts +++ b/apps/server/src/modules/storage/backends/disk/workspace-repository.test.ts @@ -13,10 +13,10 @@ import { import { tmpdir } from 'node:os'; import path from 'node:path'; +import { workspaceRegistryPath as registryPath } from './data-dir.js'; import { DiskWorkspaceRepository, WORKSPACE_MANIFEST_FILENAME, - WORKSPACE_REGISTRY_FILENAME, } from './workspace-repository.js'; import { describeWorkspaceRepositoryContract } from '../../ports/contracts/workspace-repository.contract.js'; import { adoptWorkspaceDirectory } from '../../storage.js'; @@ -34,10 +34,6 @@ describe('DiskWorkspaceRepository', () => { return path.join(root, WORKSPACE_MANIFEST_FILENAME); } - function registryPath(dataDir: string): string { - return path.join(dataDir, 'storage', 'disk', WORKSPACE_REGISTRY_FILENAME); - } - afterAll(() => { for (const root of roots) { rmSync(root, { recursive: true, force: true }); diff --git a/apps/server/src/modules/storage/backends/disk/workspace-repository.ts b/apps/server/src/modules/storage/backends/disk/workspace-repository.ts index 90f65fa9e..745f0e17c 100644 --- a/apps/server/src/modules/storage/backends/disk/workspace-repository.ts +++ b/apps/server/src/modules/storage/backends/disk/workspace-repository.ts @@ -12,6 +12,7 @@ * * The Server data directory holds a separate discovery index containing * `workspaceId -> workspacePath` plus the last time that Workspace was opened. + * Where that file sits is `data-dir.ts`'s to say, not this adapter's. * Array order has no meaning: listings sort by the explicit timestamp, and * adopting/activating a Workspace updates its timestamp in place. That * deliberate duplication is the minimum needed to recognize an externally @@ -48,7 +49,6 @@ import type { } from '../../ports/workspace.js'; export const WORKSPACE_MANIFEST_FILENAME = '.workspace.json'; -export const WORKSPACE_REGISTRY_FILENAME = 'workspaces.json'; const WORKSPACE_MANIFEST_SCHEMA_VERSION = 1; const WORKSPACE_REGISTRY_SCHEMA_VERSION = 1; @@ -85,11 +85,6 @@ type WorkspaceRegistryEntry = z.infer< typeof workspaceRegistrySchema >['workspaces'][number]; -/** Where the Disk backend keeps its discovery index inside the data dir. */ -export function workspaceRegistryPath(dataDir: string): string { - return path.join(dataDir, 'storage', 'disk', WORKSPACE_REGISTRY_FILENAME); -} - function manifestPath(workspacePath: string): string { return path.join(workspacePath, WORKSPACE_MANIFEST_FILENAME); } diff --git a/apps/server/src/modules/storage/backends/sqlite/database.ts b/apps/server/src/modules/storage/backends/sqlite/database.ts index fc0850751..5740be4b4 100644 --- a/apps/server/src/modules/storage/backends/sqlite/database.ts +++ b/apps/server/src/modules/storage/backends/sqlite/database.ts @@ -5,12 +5,11 @@ * The one SQLite connection a process holds, and the state that lives as long * as it does. * - * Both storage axes share this object when the profile selects SQLite for - * either of them. That is not a convenience: the structured records and the - * blob bytes are in one database file, so two connections would be two - * writers to the same file, and SQLite's answer to that is a lock error rather - * than a queue. One connection also makes the ordered Space write a real - * transaction across everything it touches. + * The structured store and the Workspace repository share this object. That is + * not a convenience: they are one database file, so two connections would be + * two writers to it, and SQLite's answer to that is a lock error rather than a + * queue. One connection also makes the ordered Space write a real transaction + * across everything it touches. * * The active Workspace is held here for the same reason the Disk adapters hold * the active workspace path: it is the namespace every query is scoped to. @@ -23,6 +22,7 @@ import path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { SQLITE_MIGRATIONS, type SqliteMigration } from './schema.js'; +import { getDataDir } from '../../../../data-dir.js'; import { assertSpaceMutationAllowed, beginSpaceDeleteAdmission, @@ -45,6 +45,24 @@ export const SQLITE_WORLD_COLLISION_KEY = '.world'; /** Milliseconds a statement waits for a lock before reporting SQLITE_BUSY. */ const BUSY_TIMEOUT_MS = 5_000; +/** + * Where this backend keeps its records in the Server data directory. + * + * One file, under a directory named for the backend the way + * `storage/disk/` names the other one, so an operator finds both in the same + * place. `HUABU_SQLITE_PATH` replaces it for a deployment that keeps its + * database elsewhere. + * + * Records only. A Space's *bytes* are the blob axis's business wherever this + * backend is selected, and this backend never learns where they went — see + * `backends/disk/data-dir.ts`. + */ +export function sqliteDatabasePath(dataDir: string = getDataDir()): string { + const configured = process.env['HUABU_SQLITE_PATH']?.trim(); + if (configured) return configured; + return path.join(dataDir, 'storage', 'sqlite', 'huabu.sqlite'); +} + function readUserVersion(database: DatabaseSync): number { const row = database.prepare('PRAGMA user_version').get(); const version = row?.['user_version']; diff --git a/apps/server/src/modules/storage/detached-blobs.test.ts b/apps/server/src/modules/storage/detached-blobs.test.ts index 601feb685..3a4474e7b 100644 --- a/apps/server/src/modules/storage/detached-blobs.test.ts +++ b/apps/server/src/modules/storage/detached-blobs.test.ts @@ -17,13 +17,18 @@ import path from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; +import { + diskBlobRoot, + diskDataDir, + diskSpaceBlobRoot, + workspaceRegistryPath, +} from './backends/disk/data-dir.js'; import { ARTIFACTS_DIR_NAME } from './backends/disk/layout.js'; import { activateWorkspace, createNamedWorkspace, createSpace, deleteSpace, - detachedBlobRoot, space, } from './storage.js'; import { mountTestWorkspace, type MountedTestStorage } from './testing.js'; @@ -56,13 +61,15 @@ function artifactsDirectory(canvasId: string): string { const workspace = getWorkspaceHandle(); if (!workspace) throw new Error('Expected an active Workspace'); return path.join( - detachedBlobRoot(), - workspace.workspaceId, - canvasId, + diskSpaceBlobRoot(workspace.workspaceId, canvasId), ARTIFACTS_DIR_NAME, ); } +function isInside(parent: string, child: string): boolean { + return path.resolve(child).startsWith(`${path.resolve(parent)}${path.sep}`); +} + describe('Space bytes on a backend with no Space folder', () => { it('writes real files under the Workspace-scoped byte root', async () => { await mount(); @@ -119,3 +126,47 @@ describe('Space bytes on a backend with no Space folder', () => { expect(existsSync(spaceRoot)).toBe(false); }); }); + +/** + * The two Disk adapters share `storage/disk/`, so the line between them is a + * path fact and is tested as one. + * + * They are never both in use — the registry belongs to the Disk *structured* + * store and the byte roots appear only when some other backend holds the + * records — but one data directory can see both across a backend switch. The + * blob store deletes whole directories; the registry is not its to delete. + */ +describe('the Disk backend area in the data directory', () => { + const DATA_DIR = '/var/lib/huabu'; + + it('gives the registry and the byte roots separate subtrees', () => { + const registry = workspaceRegistryPath(DATA_DIR); + const blobs = diskBlobRoot(DATA_DIR); + + expect(isInside(diskDataDir(DATA_DIR), registry)).toBe(true); + expect(isInside(diskDataDir(DATA_DIR), blobs)).toBe(true); + // The one that matters: no sweep of a Space's bytes, an area, or the whole + // blob root can reach the structured store's registry. + expect(isInside(blobs, registry)).toBe(false); + expect(isInside(blobs, diskSpaceBlobRoot('ws', 'canvas', DATA_DIR))).toBe( + true, + ); + }); + + it('moves only the bytes when HUABU_BLOB_ROOT is set', () => { + const previous = process.env['HUABU_BLOB_ROOT']; + process.env['HUABU_BLOB_ROOT'] = '/mnt/bulk/huabu-bytes'; + try { + expect(diskSpaceBlobRoot('ws', 'canvas', DATA_DIR)).toBe( + path.join('/mnt/bulk/huabu-bytes', 'ws', 'canvas'), + ); + // The registry is the structured store's and does not follow. + expect(workspaceRegistryPath(DATA_DIR)).toBe( + path.join(diskDataDir(DATA_DIR), 'workspaces.json'), + ); + } finally { + if (previous === undefined) delete process.env['HUABU_BLOB_ROOT']; + else process.env['HUABU_BLOB_ROOT'] = previous; + } + }); +}); diff --git a/apps/server/src/modules/storage/index.ts b/apps/server/src/modules/storage/index.ts index f6ff8897b..d7b6df724 100644 --- a/apps/server/src/modules/storage/index.ts +++ b/apps/server/src/modules/storage/index.ts @@ -95,7 +95,6 @@ export { materializesWorkspaces, setStorageForTesting, space, - sqliteDatabasePath, stageSpaceImport, storageHealth, workspaceAtDirectory, diff --git a/apps/server/src/modules/storage/module-boundaries.test.ts b/apps/server/src/modules/storage/module-boundaries.test.ts index ce15f31d9..043dd0c84 100644 --- a/apps/server/src/modules/storage/module-boundaries.test.ts +++ b/apps/server/src/modules/storage/module-boundaries.test.ts @@ -175,6 +175,26 @@ describe('storage dependency direction', () => { expect(violations).toEqual([]); }); + /** + * Each backend owns its own area of the Server data directory. + * + * `storage/disk/` and `storage/sqlite/` are backend-shaped names, so the + * only files allowed to build them are those backends'. The composition root + * asks; it does not know. Two adapters share `storage/disk/` — the + * structured store's Workspace registry and the blob store's Space byte + * roots — and one file deciding both is what keeps them from overlapping. + */ + it('lets each backend own its area of the data directory', () => { + const owners = sourceFiles + .filter((f) => !f.endsWith('.test.ts')) + .filter((f) => /'storage',\s*'(disk|sqlite)'/.test(read(f))); + + expect(owners.sort()).toEqual([ + 'modules/storage/backends/disk/data-dir.ts', + 'modules/storage/backends/sqlite/database.ts', + ]); + }); + it('selects a backend only in the composition root', () => { const importers = storageFiles // Tests construct adapters directly — that is how an adapter gets diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index db1332e84..0f148de1b 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -24,10 +24,7 @@ */ import { rm } from 'node:fs/promises'; -import path from 'node:path'; -import { getDataDir } from '../../data-dir.js'; -import { sanitizeId } from '../../utils/fs.js'; import { acquireWorkspaceOperationLease, commitWorkspaceIdentity, @@ -36,14 +33,18 @@ import { } from '../workspace.js'; import { DiskBlobStore } from './backends/disk/blob-store.js'; import { getWorldCanvasId as diskWorldCanvasId } from './backends/disk/canvas-dirs.js'; +import { + diskSpaceBlobRoot, + workspaceRegistryPath, +} from './backends/disk/data-dir.js'; import { stageDiskSpaceImport } from './backends/disk/space-import.js'; import { diskSpaceTree } from './backends/disk/space-tree.js'; import { DiskStructuredStore } from './backends/disk/structured-store.js'; +import { DiskWorkspaceRepository } from './backends/disk/workspace-repository.js'; import { - DiskWorkspaceRepository, - workspaceRegistryPath, -} from './backends/disk/workspace-repository.js'; -import { SqliteStoreContext } from './backends/sqlite/database.js'; + SqliteStoreContext, + sqliteDatabasePath, +} from './backends/sqlite/database.js'; import { SqliteStructuredStore } from './backends/sqlite/structured-store.js'; import { SqliteWorkspaceRepository } from './backends/sqlite/workspace-repository.js'; import { spaceBlobAreas } from './ports/blob.js'; @@ -113,40 +114,16 @@ function assertActiveWorkspace(workspaceKey: string, canvasId: string): void { } } -/** - * Where the SQLite profile keeps its records. - * - * One file, beside the Disk backend's own registry in the data directory, so - * an operator can find both in the same place. `HUABU_SQLITE_PATH` overrides - * it for deployments that keep their database elsewhere. A Space's *bytes* are - * not in it — see {@link detachedBlobRoot}. - */ -export function sqliteDatabasePath(dataDir: string = getDataDir()): string { - const configured = process.env['HUABU_SQLITE_PATH']?.trim(); - if (configured) return configured; - return path.join(dataDir, 'storage', 'sqlite', 'huabu.sqlite'); -} - /** * Where a Space keeps its bytes when the structured backend has no folder for * it. * * Blobs are always files (`ports/blob.ts`), so a profile whose records are - * rows still needs somewhere on a file system to put uploads, artifacts, the - * guide document and the memory body. The Server owns that directory rather - * than the user: it is not a Workspace folder, nothing in it is a Space - * record, and none of the Disk-only features that need a real Space tree - * become available because it exists. - * - * Scoped by Workspace first, because a Space belongs to exactly one and its - * bytes should travel and be removed with it. `HUABU_BLOB_ROOT` overrides the - * base for deployments that keep bytes on a different volume. + * rows still needs somewhere on a file system for uploads, artifacts, the + * guide document and the memory body. *Which* directory is the Disk blob + * adapter's own business — this only supplies the Workspace the Space belongs + * to, which is the one part of the answer the adapter cannot know. */ -export function detachedBlobRoot(dataDir: string = getDataDir()): string { - const configured = process.env['HUABU_BLOB_ROOT']?.trim(); - return configured ? configured : path.join(dataDir, 'storage', 'blobs'); -} - function detachedSpaceRoot(canvasId: string): string { const workspace = getWorkspaceHandle(); if (!workspace) { @@ -155,11 +132,7 @@ function detachedSpaceRoot(canvasId: string): string { 'Activate one before reading or writing bytes.', ); } - return path.join( - detachedBlobRoot(), - sanitizeId(workspace.workspaceId, 'workspaceId'), - sanitizeId(canvasId, 'canvasId'), - ); + return diskSpaceBlobRoot(workspace.workspaceId, canvasId); } /** @@ -400,7 +373,7 @@ export function getWorkspaceRepository(): WorkspaceRepository { workspaces = profile.structured.kind === 'sqlite' ? new SqliteWorkspaceRepository(sqliteConnection()) - : new DiskWorkspaceRepository(workspaceRegistryPath(getDataDir())); + : new DiskWorkspaceRepository(workspaceRegistryPath()); return workspaces; } diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index c79169f71..959c73c22 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -88,20 +88,26 @@ Key points: `HUABU_STRUCTURED_BACKEND=sqlite` selects the second implemented structured backend. The blob axis stays `disk`, because bytes are always files. It needs **no Workspace folder and no Space directories**: every record is a row, and the only directories are the ones a Space's bytes sit in. ``` -/ - storage/sqlite/ +/storage/ + sqlite/ # the SQLite backend's area huabu.sqlite # every record; override with HUABU_SQLITE_PATH huabu.sqlite-wal # WAL sidecars, managed by SQLite huabu.sqlite-shm - storage/blobs/ # override with HUABU_BLOB_ROOT - / - / - skill.md # blob area `guide` - .artifacts/ # blob area `artifacts` - .memory/space.md # blob area `memory` - .upload/ # blob area `uploads` + disk/ # the Disk backend's area + workspaces.json # Disk *structured* store: Workspace registry (unused here) + blobs/ # Disk *blob* store; override with HUABU_BLOB_ROOT + / + / + skill.md # blob area `guide` + .artifacts/ # blob area `artifacts` + .memory/space.md # blob area `memory` + .upload/ # blob area `uploads` ``` +Each directory under `storage/` is named for the backend that owns it, and one file per backend decides its layout: `backends/disk/data-dir.ts` and `backends/sqlite/database.ts`. The composition root asks them and builds no path of its own (`module-boundaries.test.ts` enforces that). + +`storage/disk/` has two owners, so they get separate subtrees. `workspaces.json` is the Disk _structured_ store's Workspace registry — present only on a Disk-structured deployment, and never inside `blobs/`, because the blob store deletes whole directories and the registry is not its to delete. `blobs/` is the Disk _blob_ store's, reached only when the structured backend gives a Space no folder; `HUABU_BLOB_ROOT` moves that subtree alone. + The byte root is Server-owned, not a Workspace folder: nothing in it is a Space record, and the Disk-only capabilities below stay unavailable because they need a real Space tree, not merely a directory. The layout beneath `/` is byte-for-byte the one the Disk profile uses inside a Space folder, because it is the same adapter — composition only tells it where the Space's root is. | Table | Holds | diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 39a5ab10b..323075de1 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -2546,21 +2546,29 @@ say plainly what it gives up by doing so. `HUABU_STRUCTURED_BACKEND=sqlite` is the profile. Every record — Workspaces, Spaces, nodes, events, changes, Tasks, extension namespaces, and agent conversations — is a row in one file at -`/storage/sqlite/huabu.sqlite` (override with `HUABU_SQLITE_PATH`), -beside the Disk backend's own registry at `/storage/disk/`. +`/storage/sqlite/huabu.sqlite` (override with `HUABU_SQLITE_PATH`). The blob axis stays `disk`, and there is no SQLite blob adapter. **Bytes are always a file system** — a local directory now, Azure Blob later — so no structured backend is asked to hold them and the two axes genuinely share nothing. A Space's bytes therefore need a directory even where its record does -not: composition supplies `/storage/blobs///` -(override the base with `HUABU_BLOB_ROOT`) and hands it to the same Disk blob -adapter, which writes the same area layout it writes inside a Space folder. -That directory is Server-owned and holds nothing but bytes; it is not a -Workspace folder and it is not a Space tree, so none of §12.9.4's Disk-only +not: the Disk blob adapter writes them to +`/storage/disk/blobs///` (override the base +with `HUABU_BLOB_ROOT`), in the same area layout it writes inside a Space +folder. That directory is Server-owned and holds nothing but bytes; it is not +a Workspace folder and it is not a Space tree, so none of §12.9.4's Disk-only capabilities become available because it exists. Postgres and Azure Blob adapters still do not exist. +Each directory under `/storage/` is named for the backend that owns +it, and each backend decides its own layout in one file — +`backends/disk/data-dir.ts` and `backends/sqlite/database.ts`. The composition +root asks and builds no path of its own, which `module-boundaries.test.ts` +enforces. `storage/disk/` has two owners and therefore two subtrees: the +structured store's `workspaces.json` Workspace registry, and the blob store's +`blobs/`. Keeping the registry outside `blobs/` is not tidiness — the blob +store deletes whole directories, and the registry is not its to delete. + #### 12.9.1 Scope and lifecycle - Built-in `node:sqlite`. No package, no native addon. That is not a @@ -2687,7 +2695,8 @@ is still there after a restart. That suite names no directory and no filename; `module-boundaries.test.ts` enforces that mechanically. What _is_ about placement has its own small suite instead (`detached-blobs.test.ts`): bytes land as real files under the Workspace-scoped root, one Workspace's root is -not another's, and deleting a Space leaves no directory behind. +not another's, deleting a Space leaves no directory behind, and the Workspace +registry sits outside the blob root so no byte sweep can reach it. SQLite integration tests additionally cover strict schema creation, WAL and foreign-key pragmas read back on a second connection, close/reopen From 52e4b210f8705418ab726bb6adb1f1b6d0038e49 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 10:42:50 +0800 Subject: [PATCH 10/15] docs(storage): say which layer decides where a Space's bytes go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `blobs=disk` produced two different directories depending on the *structured* backend — inside the user's Space folder with Disk records, under `storage/disk/blobs/` with rows — and nothing said why that is one rule rather than two meanings for one config value. It is one rule: **a Space's bytes live with the Space.** The Space has two possible homes, so the rule has two outcomes. `blobs=disk` names a medium — bytes are local files — and the place is composition's to choose, which is the layer allowed to know both axes. The code now says so where the choice is made. `buildBlobStore` passes the root explicitly in both branches instead of leaning on a default, and `DiskBlobStore`'s root argument is required: a default there is the cross-axis decision made silently by whichever caller forgot to pass one. Why the Disk-records outcome is not merely legacy compatibility, recorded because it is easy to "fix" and break four things: `space-bundle-export` is the Space folder archived, `space-bundle-import` unzips into it, `reveal-space-folder` shows it, and `builtin-file-tools` sandbox on it. Each needs that folder to be complete. Relocating artifacts to a Server-owned root would hollow out all four while every one still reported as available. That is also a real cross-axis constraint waiting to bite. A blob backend that cannot co-locate — an object store — would put bytes outside the Space folder even on Disk records, and those four rows would then have to be keyed on the profile rather than on the structured kind. `capabilities.ts` says that where its matrix is defined, in place of the comment claiming no such second matrix could exist. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../disk/blob-store.rename-retry.test.ts | 5 ++- .../storage/backends/disk/blob-store.test.ts | 11 ++++-- .../storage/backends/disk/blob-store.ts | 22 +++++------ .../src/modules/storage/capabilities.ts | 15 +++++++- .../compatibility/delete-canvas.test.ts | 12 ++++-- apps/server/src/modules/storage/storage.ts | 38 ++++++++++++++----- docs/architecture/canvas-storage.md | 4 +- docs/proposals/multi-backend-storage.md | 15 +++++++- 8 files changed, 88 insertions(+), 34 deletions(-) diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts b/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts index 7329decde..75219305f 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.rename-retry.test.ts @@ -8,6 +8,7 @@ import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; import { DiskBlobStore } from './blob-store.js'; +import { canvasRoot } from './layout.js'; import type * as NodeFsPromises from 'node:fs/promises'; @@ -39,7 +40,9 @@ describe('DiskBlobStore retry cleanup', () => { testState.renameAsync.mockRejectedValue(error); try { - const scope = new DiskBlobStore().space('canvas-under-test').artifacts; + const scope = new DiskBlobStore(canvasRoot).space( + 'canvas-under-test', + ).artifacts; await expect(scope.put('blocked.bin', Buffer.from('bytes'))).rejects.toBe( error, diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.test.ts b/apps/server/src/modules/storage/backends/disk/blob-store.test.ts index e4718bd8a..8cf608260 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.test.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.test.ts @@ -22,13 +22,14 @@ vi.mock('../../../workspace.js', () => ({ })); import { DiskBlobStore } from './blob-store.js'; +import { canvasRoot } from './layout.js'; import { describeBlobStoreContract } from '../../ports/contracts/blob-store.contract.js'; describeBlobStoreContract('DiskBlobStore', () => { const root = mkdtempSync(path.join(tmpdir(), 'huabu-blob-')); workspaceState.path = root; return { - store: new DiskBlobStore(), + store: new DiskBlobStore(canvasRoot), canvasId: 'canvas-under-test', cleanup: () => rmSync(root, { recursive: true, force: true }), }; @@ -55,7 +56,7 @@ describe('DiskBlobStore temp file hygiene', () => { }); it('cleans up after both successful and failed writes', async () => { - const scope = new DiskBlobStore().space(canvasId).artifacts; + const scope = new DiskBlobStore(canvasRoot).space(canvasId).artifacts; await scope.put('kept.bin', Buffer.from('fine')); await scope.put('streamed.bin', Readable.from([Buffer.from('also fine')])); @@ -78,7 +79,9 @@ describe('DiskBlobStore temp file hygiene', () => { }); it('cleans up siblings from concurrent writers to one key', async () => { - const scope = new DiskBlobStore().space('concurrent-canvas').artifacts; + const scope = new DiskBlobStore(canvasRoot).space( + 'concurrent-canvas', + ).artifacts; await Promise.all( Array.from({ length: 8 }, (_, i) => @@ -93,7 +96,7 @@ describe('DiskBlobStore temp file hygiene', () => { it('binds in-flight paths to their original workspace and rejects a held scope after activation', async () => { const otherRoot = mkdtempSync(path.join(tmpdir(), 'huabu-blob-switched-')); - const scope = new DiskBlobStore().space(canvasId).artifacts; + const scope = new DiskBlobStore(canvasRoot).space(canvasId).artifacts; let signalStarted = (): void => {}; const started = new Promise((resolve) => { signalStarted = resolve; diff --git a/apps/server/src/modules/storage/backends/disk/blob-store.ts b/apps/server/src/modules/storage/backends/disk/blob-store.ts index 3c752a053..30f72d469 100644 --- a/apps/server/src/modules/storage/backends/disk/blob-store.ts +++ b/apps/server/src/modules/storage/backends/disk/blob-store.ts @@ -8,12 +8,11 @@ * the layout the workspace format has always used: one file per blob, named by * the URL key, no manifest indirection. * - * Where the Space's root *is* depends on the structured backend, which is why - * it is injected rather than resolved here. When Disk also keeps the records, - * the areas sit inside the Space folder the user can see — unchanged from - * every Workspace that already exists. When the records live in a database - * there is no such folder, so composition hands over a server-owned directory - * instead; the layout beneath it is identical either way. + * The root is a constructor argument, because this adapter does not know where + * a Space is. `blobs=disk` names a *medium* — bytes are local files — and + * composition names the place (`storage.ts::buildBlobStore`, which carries the + * rule and the reason). Everything below the root is identical whichever place + * that turns out to be. * * Each scope is bound to the workspace active when it is created. A fresh * scope follows a workspace switch; a retained scope rejects the next @@ -35,7 +34,6 @@ import { pipeline } from 'node:stream/promises'; import { ARTIFACTS_DIR_NAME, - canvasRoot, MEMORY_DIR_NAME, UPLOAD_DIR_NAME, } from './layout.js'; @@ -79,9 +77,11 @@ type SpaceBlobArea = keyof SpaceBlobs; /** * Where this Space keeps its bytes. * - * The Disk structured backend's own {@link canvasRoot} is the default, so a - * Workspace that already exists is addressed exactly as before. A profile - * whose records live elsewhere supplies its own. + * Supplied, never defaulted. Which directory a Space's bytes belong in is a + * fact about the whole deployment — it depends on whether the *structured* + * backend gives that Space a directory of its own — and this adapter is not + * the layer that knows. A default here would be that cross-axis decision made + * silently, by whichever caller forgot to pass one. */ export type SpaceBlobRoot = (canvasId: string) => string; @@ -331,7 +331,7 @@ export class DiskBlobStore implements BlobStore { readonly #root: SpaceBlobRoot; - constructor(root: SpaceBlobRoot = canvasRoot) { + constructor(root: SpaceBlobRoot) { this.#root = root; } diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 8f7966265..0985d4821 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -31,8 +31,19 @@ import type { StorageProfile } from './profile.js'; * A product feature whose availability depends on the structured backend. * * Keyed by structured kind alone: every entry here needs a Space to be a real - * directory, which is a structured-backend property. A feature that turned on - * the blob backend instead would be a second matrix, and there are none. + * directory, which is a structured-backend property. + * + * Four of them need slightly more than that — they need the Space's *bytes* to + * be in that directory too. Bundle export archives the folder, bundle import + * unzips into it, reveal-in-file-manager shows it, and the built-in file tools + * sandbox on it; a Space whose artifacts had been relocated elsewhere would + * export as an incomplete bundle rather than fail. Today that is free: the one + * blob backend is a file system, and composition places a Space's bytes inside + * its directory whenever it has one (`storage.ts::buildBlobStore`). A blob + * backend that *cannot* co-locate — an object store — would break the + * implication, and those four rows would then have to be keyed on the profile + * rather than on the structured kind. That is the second matrix this comment + * used to say did not exist; it does not exist yet. */ export interface StorageCapability { /** Stable id, for a diagnostic an operator can search for. */ diff --git a/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts b/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts index d434dc49e..d7239dd3d 100644 --- a/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts +++ b/apps/server/src/modules/storage/compatibility/delete-canvas.test.ts @@ -16,7 +16,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { executeOnServer } from '../../canvas/canvas-executor.js'; import { DiskBlobStore } from '../backends/disk/blob-store.js'; import { refreshCanvasDirIndex } from '../backends/disk/canvas-dirs.js'; -import { artifactPath, canvasJsonPath } from '../backends/disk/layout.js'; +import { + artifactPath, + canvasJsonPath, + canvasRoot, +} from '../backends/disk/layout.js'; import { resetStorageCache } from '../backends/disk/legacy/canvas-store-cache.js'; import { DiskStructuredStore } from '../backends/disk/structured-store.js'; import { getCanvasStore } from '../index.js'; @@ -73,7 +77,7 @@ function wrapAreas( /** How many sweeps one Space deletion must perform. */ const SPACE_AREA_COUNT = spaceBlobAreas( - new DiskBlobStore().space('probe'), + new DiskBlobStore(canvasRoot).space('probe'), ).length; function writeCanvas(directory: string, canvasId: string, title: string): void { @@ -106,7 +110,7 @@ class OrderRecordingBlobStore implements BlobStore { readonly kind = 'disk' as const; readonly recordPresentAtSweep: boolean[] = []; - private readonly inner = new DiskBlobStore(); + private readonly inner = new DiskBlobStore(canvasRoot); init(): Promise { return this.inner.init(); @@ -155,7 +159,7 @@ class ControllableBlobStore implements BlobStore { readonly deleteStarted = deferred(); readonly #putsReleased = deferred(); readonly #deletesReleased = deferred(); - readonly #inner = new DiskBlobStore(); + readonly #inner = new DiskBlobStore(canvasRoot); blockPuts = false; blockDeletes = false; diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index 0f148de1b..7f8b198ad 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -37,6 +37,7 @@ import { diskSpaceBlobRoot, workspaceRegistryPath, } from './backends/disk/data-dir.js'; +import { canvasRoot } from './backends/disk/layout.js'; import { stageDiskSpaceImport } from './backends/disk/space-import.js'; import { diskSpaceTree } from './backends/disk/space-tree.js'; import { DiskStructuredStore } from './backends/disk/structured-store.js'; @@ -256,22 +257,39 @@ function composeSpace(storage: Storage, canvasId: string): Space { } /** - * The blob connection for this profile. - * - * One adapter, two placements. Where Disk also keeps the records, a Space's - * bytes stay inside the Space folder — byte-for-byte the layout every existing - * Workspace has. Where the records are rows, the same adapter writes the same - * layout under a Server-owned root instead, which is what makes a hybrid - * profile (SQL records, ordinary files) an ordinary deployment. + * The blob connection for this profile, and where it puts a Space's bytes. + * + * `blobs=disk` names a *medium* — bytes are local files — so there is one + * adapter. The place is composition's to choose, and the rule is one sentence: + * **a Space's bytes live with the Space.** + * + * Where the structured backend files a Space as a directory, that directory is + * where the Space *is*, so the bytes go inside it. That is not only + * backward-compatibility with every Workspace that already exists: a Space + * folder being self-contained is what several declared capabilities are made + * of. `.huabu.zip` export is that folder archived, reveal-in-file-manager + * shows it, RFS projects it, and the built-in file tools sandbox on it. + * Relocating artifacts to a Server-owned root would quietly hollow out all + * four while every one of them still reported as available. + * + * Where a Space is a row it has no directory to be inside, so the adapter gets + * a root of its own under the Disk backend's data-directory area. + * + * One rule, two outcomes, because a Space has two possible homes — not two + * meanings for `blobs=disk`. The corollary is a genuine cross-axis constraint + * for the day a blob backend cannot co-locate: an object store would put bytes + * outside the Space folder even on Disk records, and the four capabilities + * above would then depend on both axes rather than the structured one alone + * (see `capabilities.ts`). */ function buildBlobStore(profile: StorageProfile): BlobStore { if (profile.blobs.kind !== 'disk') { // Unreachable: validateStorageProfile rejects unimplemented kinds. throw new Error(`Unsupported blob backend: ${profile.blobs.kind}`); } - return profile.structured.kind === 'disk' - ? new DiskBlobStore() - : new DiskBlobStore(detachedSpaceRoot); + return new DiskBlobStore( + profile.structured.kind === 'disk' ? canvasRoot : detachedSpaceRoot, + ); } function buildStructuredStore(profile: StorageProfile): StructuredStore { diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 959c73c22..943ee9bd1 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -108,7 +108,9 @@ Each directory under `storage/` is named for the backend that owns it, and one f `storage/disk/` has two owners, so they get separate subtrees. `workspaces.json` is the Disk _structured_ store's Workspace registry — present only on a Disk-structured deployment, and never inside `blobs/`, because the blob store deletes whole directories and the registry is not its to delete. `blobs/` is the Disk _blob_ store's, reached only when the structured backend gives a Space no folder; `HUABU_BLOB_ROOT` moves that subtree alone. -The byte root is Server-owned, not a Workspace folder: nothing in it is a Space record, and the Disk-only capabilities below stay unavailable because they need a real Space tree, not merely a directory. The layout beneath `/` is byte-for-byte the one the Disk profile uses inside a Space folder, because it is the same adapter — composition only tells it where the Space's root is. +The byte root is Server-owned, not a Workspace folder: nothing in it is a Space record, and the Disk-only capabilities below stay unavailable because they need a real Space tree, not merely a directory. + +The layout beneath `/` is byte-for-byte the one the Disk profile uses inside a Space folder, because it is the same adapter. `blobs=disk` names a _medium_ — bytes are local files — and composition names the place, under one rule: **a Space's bytes live with the Space.** On a Disk-structured profile the Space _is_ a folder, so its bytes stay inside it, which is what keeps a Space folder self-contained for bundle export, reveal-in-file-manager, RFS, and the file tools. On a database-structured profile the Space has no folder to be inside, so the adapter gets the root above. One rule, two outcomes — the Space has two possible homes. | Table | Holds | | ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 323075de1..e5c6a3648 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -2555,7 +2555,20 @@ nothing. A Space's bytes therefore need a directory even where its record does not: the Disk blob adapter writes them to `/storage/disk/blobs///` (override the base with `HUABU_BLOB_ROOT`), in the same area layout it writes inside a Space -folder. That directory is Server-owned and holds nothing but bytes; it is not +folder. + +`blobs=disk` names a medium, not a directory. Where the bytes go is +composition's, under one rule — **a Space's bytes live with the Space** — and +the Space has two possible homes. On Disk records it is a folder, so the bytes +stay inside it; that is not merely backward compatibility, it is what +`space-bundle-export`, `space-bundle-import`, `reveal-space-folder` and +`builtin-file-tools` are made of, since each of them is the Space folder being +complete. On database records there is no folder to be inside. The corollary +is a cross-axis constraint that does not bite yet: a blob backend that cannot +co-locate — an object store — would put bytes outside the Space folder even on +Disk records, and those four capabilities would then depend on both axes +rather than the structured one alone. `capabilities.ts` records that where the +matrix is defined. That directory is Server-owned and holds nothing but bytes; it is not a Workspace folder and it is not a Space tree, so none of §12.9.4's Disk-only capabilities become available because it exists. Postgres and Azure Blob adapters still do not exist. From 8dc8a0f643b423c20850dc8b5ddb979c8ae32084 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 11:20:16 +0800 Subject: [PATCH 11/15] docs(storage): key the capability matrix on the structured backend, out loud MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hybrid profile gives a Space a real directory full of real files, so the matrix now has to answer a question it did not have before: does a file system for the bytes hand any Disk-only feature back? It does not, and the mechanism is already right — every refusal keys on `Space.diskTree` being `null`, which is a structured-backend fact, and stays `null` however many byte directories exist. Each of these features needs the Space's *record and node documents* to be files, and those are rows. The blob axis carries opaque bytes, which is not what any of them are about. What was wrong is what the matrix said. Five rationales argued from the absence of a directory — "Without a folder there is nothing to show", "No directory, no problem" — and those sentences are printed at startup and reused verbatim in the refusal a user sees. On this profile they are simply false. They now argue from what is actually missing: - reveal-space-folder: what a user means by "this" is the Space, and its byte directory is not the Space. - builtin-file-tools / space-file-plane: the documents at stake are the node sidecars under `nodes/`; the byte areas hold artifacts, not those. - external-note-discovery: `nodes/` is the tier a dropped note would arrive in, and no byte area is somewhere a user would drop one. - workspace-user-memory: the blob port has no Workspace-level scope, so the document has none to live in — not "there is nowhere on disk". - space-directory-handle-coordination: nothing renames a directory keyed by id, and nothing watches it, so there are no handles to arbitrate. - workspace-directory: the per-Workspace directory under the blob root is Server-owned byte storage, not a Workspace anyone could pick. `detached-blobs.test.ts` pins the fact all ten gates rest on: bytes are on the file system and `diskTree` is still `null`. `capabilities.test.ts` already ran against `sqlite/disk`, so it was answering this question; it now says so. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../src/modules/storage/capabilities.test.ts | 6 +++ .../src/modules/storage/capabilities.ts | 50 ++++++++++++------- .../modules/storage/detached-blobs.test.ts | 14 ++++++ docs/architecture/canvas-storage.md | 2 + docs/proposals/multi-backend-storage.md | 35 +++++++------ 5 files changed, 75 insertions(+), 32 deletions(-) diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index 670a3d466..3df349fbf 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -57,6 +57,12 @@ describe('storage capability matrix', () => { // Every entry is Disk-only today, so a structured backend that is not // Disk loses all of them. The assertion is the shape, not the count. + // + // `TABLES` pairs SQLite records with Disk *bytes*, which is the profile + // this deployment actually runs, so this is also the answer to "does a + // real file system for bytes give any of these back". It does not: every + // entry needs the Space's record and node documents to be files, and + // those are rows whatever holds the bytes. expect(missing).toEqual(STORAGE_CAPABILITIES); expect(hasStorageCapability(TABLES, 'reveal-space-folder')).toBe(false); expect(hasStorageCapability(DISK, 'reveal-space-folder')).toBe(true); diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 0985d4821..6c3beb67d 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -84,25 +84,33 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ summary: 'Reveal a Space in the OS file manager', backends: ['disk'], rationale: - 'The feature is "show me this in Finder". Without a folder there is ' + - 'nothing to show.', + 'The feature is "show me this in Finder", and what a user means by ' + + '"this" is the Space: its record and its node documents. Those are ' + + 'rows. The one directory such a Space has holds its opaque bytes and ' + + 'is Server-owned, so revealing it would open something that is not the ' + + 'thing that was asked for.', }, { id: 'builtin-file-tools', summary: 'Built-in agent file tools (read, write, glob, grep)', backends: ['disk'], rationale: - 'They sandbox on the Space directory. Off Disk the first-party agent ' + - 'reads and writes nodes through the Canvas tools instead, which is ' + - 'the portable surface it already prefers for structured edits.', + 'They sandbox on the Space directory and the documents they exist to ' + + 'edit are the node sidecars under `nodes/`, which are rows here. A ' + + "Space's byte areas are files on every profile, but they hold " + + 'artifacts and uploads, not the documents an agent reads and writes. ' + + 'Off Disk the first-party agent goes through the Canvas tools instead, ' + + 'which is the portable surface it already prefers for structured edits.', }, { id: 'space-file-plane', summary: 'Reach a Space as files over RFS, the plane external agents mount', backends: ['disk'], rationale: - 'RFS projects the Space directory over HTTP — the same tree, reachable ' + - 'from another machine. It is listed apart from the built-in file tools ' + + 'RFS projects the Space directory over HTTP — the record and the node ' + + 'sidecars, reachable from another machine. Those are rows here, and a ' + + 'projection of the byte areas alone would be a different plane wearing ' + + "this one's name. It is listed apart from the built-in file tools " + 'because it is what those tools were said to fall back to: a Space ' + 'with no file plane has neither, and an external agent bound to a ' + 'Space on this backend reaches it through the Canvas API.', @@ -112,19 +120,21 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ summary: 'Adopt Markdown files dropped into a Space from outside the app', backends: ['disk'], rationale: - 'It watches for documents that arrived without going through the ' + - 'application. A database backend has no such arrival path unless ' + - 'someone writes to the store out of band, and inventing one would buy ' + - 'nothing.', + 'It watches `nodes/` for documents that arrived without going through ' + + 'the application. That tier is rows here, and no byte area is a place ' + + 'a user would drop a note into: they are hidden, Server-owned, and ' + + 'hold artifacts. Inventing an arrival path would buy nothing.', }, { id: 'workspace-directory', summary: 'Choose, create, or reveal a Workspace folder on this machine', backends: ['disk'], rationale: - 'A Workspace is a folder the user picks. Where Workspaces are rows, ' + + 'A Workspace is a folder the user picks. Where Workspaces are rows ' + 'there is nothing to browse to: the Server opens its own on first ' + - 'start and Workspaces are managed by name instead of by path.', + 'start and Workspaces are created and managed by name instead of by ' + + 'path. The per-Workspace directory under the blob root is Server-owned ' + + 'storage for bytes, not a Workspace a user could choose or move.', }, { id: 'workspace-user-memory', @@ -132,9 +142,10 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ backends: ['disk'], rationale: 'A user-editable file at the Workspace root, deliberately outside any ' + - 'Space so it applies to all of them. Every blob scope this port has is ' + - "scoped to a Space, so there is nowhere it belongs yet; a Space's own " + - 'memory body is unaffected.', + 'Space so it applies to all of them. The blob port has no ' + + 'Workspace-level scope — every area it vends belongs to a Space — so ' + + 'the document has no scope to live in, whatever directories happen to ' + + "exist. A Space's own memory body is unaffected; it is a blob.", }, { id: 'workspace-user-skills', @@ -150,8 +161,11 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ summary: 'Windows: rename or delete a Space while a watcher holds it open', backends: ['disk'], rationale: - 'Exists so a directory rename can succeed against a live `fs.watch` ' + - 'handle. No directory, no problem.', + 'Exists so renaming a Space *directory* can succeed against a live ' + + '`fs.watch` handle. A Space that is a row is never filed under its ' + + 'title, so nothing renames its byte directory, and with external-note ' + + 'discovery unavailable nothing watches it either. No rename and no ' + + 'watcher, so there are no handles to arbitrate.', }, ]; diff --git a/apps/server/src/modules/storage/detached-blobs.test.ts b/apps/server/src/modules/storage/detached-blobs.test.ts index 3a4474e7b..9c81e6cd1 100644 --- a/apps/server/src/modules/storage/detached-blobs.test.ts +++ b/apps/server/src/modules/storage/detached-blobs.test.ts @@ -112,6 +112,20 @@ describe('Space bytes on a backend with no Space folder', () => { ); }); + it('is not a Space tree, so no Disk-only capability turns on', async () => { + await mount(); + await createSpace(CANVAS_ID, 'Detached'); + await space(CANVAS_ID).artifacts.put('art.bin', Buffer.from('bytes')); + + // The byte directory exists and holds real files, and the Space still has + // no `diskTree`. That single `null` is what every Disk-only feature keys + // on — bundle export, reveal-in-file-manager, the built-in file tools, + // RFS, external-note claim — so it is the fact worth pinning: a file + // system for bytes is not a Space directory and grants none of them. + expect(existsSync(artifactsDirectory(CANVAS_ID))).toBe(true); + expect(space(CANVAS_ID).diskTree).toBeNull(); + }); + it('leaves no directory behind when the Space is deleted', async () => { await mount(); await createSpace(CANVAS_ID, 'Detached'); diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 943ee9bd1..13d3c88c0 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -132,6 +132,8 @@ Notes an operator needs: - **Bytes are outside the database.** The blob axis is a file system on every profile, so a Space's uploads, artifacts, guide and memory body are ordinary files under the byte root and the database stays the size of its records. Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and, where the record is a row, composition also removes the `//` directory it placed those areas under, because nothing else would. - **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, user-authored skills under `setting/skills/`, and Windows directory-handle coordination. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. + These are keyed on the **structured** backend, not the blob one, and that survives the hybrid profile: every entry needs the Space's record and node documents to exist as files, so a real file system for its bytes gives none of them back. Each refusal keys on `Space.diskTree` being `null`, which stays `null` however many byte directories exist. + ## 3. Storage composition and ownership `apps/server/src/modules/storage/` has three layers plus its composition root: diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index e5c6a3648..30e8e9316 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -2670,20 +2670,27 @@ same cascade as everything else. #### 12.9.4 What this profile does not serve -Six capabilities are Disk-only, declared in `storage/capabilities.ts`, logged -at startup, and refused at their own call sites in the same words: - -| Capability | What is lost | Why it is not emulated | -| --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | -| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | -| `reveal-space-folder` | "Show me this in Finder" | Without a folder there is nothing to show. | -| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | They sandbox on the Space directory. The first-party agent edits nodes through the Canvas tools instead. | -| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | -| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | -| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | It is a file the user edits at the root of a Workspace they chose, and there is no such folder. A Space's _own_ memory body is a blob and is unaffected. | -| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | -| `space-directory-handle-coordination` | Windows rename-while-watched | No directory, no problem. | +These capabilities are Disk-only, declared in `storage/capabilities.ts`, +logged at startup, and refused at their own call sites in the same words. + +They are keyed on the **structured** backend, and the hybrid profile is what +makes that worth stating: every one needs the Space's record and node +documents to exist as files, and a real file system for the Space's _bytes_ +gives none of them back. Each refusal keys on `Space.diskTree` being `null`, +which is a structured-backend fact — the byte directory is not a Space tree, +and `detached-blobs.test.ts` pins that. + +| Capability | What is lost | Why it is not emulated | +| --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | +| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | +| `reveal-space-folder` | "Show me this in Finder" | What a user means by "this" is the Space — its record and node documents — and those are rows. Its byte directory is not the Space. | +| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | The documents they edit are the node sidecars under `nodes/`, which are rows. The first-party agent uses the Canvas tools instead. | +| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | +| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | +| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | A file the user edits at the root of a Workspace they chose. The blob port has no Workspace-level scope, so it has none to live in. A Space's _own_ memory body is a blob and is unaffected. | +| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | +| `space-directory-handle-coordination` | Windows rename-while-watched | Nothing renames a byte directory keyed by id, and with note discovery off nothing watches it. No handles to arbitrate. | One further limit is not a capability row because nothing refuses it; it is simply a property of the backend: From 243700863c7febd5dfb1b5606250d909af40c64d Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 11:36:03 +0800 Subject: [PATCH 12/15] refactor(storage): key the capability matrix on the profile, and gate on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A capability was keyed on the structured backend alone. That was defensible while the blob axis was one co-locating file system, and it stops being defensible the moment a second blob backend exists: `disk` records with an object store would keep every "needs a Space directory" row available, and `space-bundle-export` would archive a Space folder its artifacts had never been written to — reporting success while producing an incomplete bundle. `StorageCapability` now names both axes, and **omitting an axis means every backend on it serves the feature**. That default is the design, not a shortcut: a row that does not touch a Space's bytes must not need editing when a blob backend lands, and the rows that do are exactly the ones that should force a decision then. Four rows name the blob axis — bundle export, bundle import, the built-in file tools, and RFS — because each needs the bytes in the folder it archives, unzips into, or resolves a path inside. `reveal-space-folder` does not: the folder still holds the record and the node documents, so showing it is still showing the Space. Defects this found and fixes: - **Gates re-derived the requirement.** Bundle export, import, reveal, the file tools, and the external-note claim all inferred "is this available" from `diskTree` being non-null. That is a second copy of the rule, and it is how a row could grow a blob-axis requirement its own call site never learned about. Every refusal now asks `storageServes(id)`, and `diskTree` is left to supply the path it was always for. - **RFS asked the wrong profile.** It called `hasStorageCapability(parseStorageProfile(), …)`, reading the environment rather than the profile storage was opened with — a different answer as soon as a test or an embedder mounts an explicit one. `storageServes` binds the active profile, and `hasStorageCapability` leaves the barrel so no call site can pick a profile again. - **One row could never be refused.** `space-directory-handle-coordination` had no consumer anywhere: a Space with no directory registers no handle owner, so nothing ever asks. It was printed at boot as something the operator had lost, when the profile simply does not have the problem. Removed, and recorded as a backend property instead. `module-boundaries.test.ts` now fails if any declared row has no refusal outside `storage/`. - **The startup line named one axis.** It read "unavailable on the 'sqlite' structured backend" for a row that may be unavailable for either. It names the profile now. `capabilities.test.ts` asserts the pairing that motivates all of this — `disk`/`azure`, which no adapter serves and `validateStorageProfile` refuses, which is exactly why the matrix has to answer it correctly first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../agent/tools/handlers/fs-sandbox.ts | 13 +- .../modules/agent/tools/handlers/fs-write.ts | 11 +- .../server/src/modules/canvas/canvas.route.ts | 18 ++- .../src/modules/canvas/external.route.ts | 10 +- .../server/src/modules/remote_fs/rfs.route.ts | 5 +- .../src/modules/storage/capabilities.test.ts | 54 ++++++- .../src/modules/storage/capabilities.ts | 144 +++++++++++------- apps/server/src/modules/storage/index.ts | 7 +- .../modules/storage/module-boundaries.test.ts | 25 +++ apps/server/src/modules/storage/storage.ts | 18 +++ apps/server/src/modules/workspace.route.ts | 5 +- docs/architecture/canvas-storage.md | 4 +- docs/proposals/multi-backend-storage.md | 43 ++++-- 13 files changed, 268 insertions(+), 89 deletions(-) diff --git a/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts b/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts index 076739824..c1cc6438f 100644 --- a/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts +++ b/apps/server/src/modules/agent/tools/handlers/fs-sandbox.ts @@ -30,7 +30,11 @@ import { readFileSync, readdirSync, statSync, type Dirent } from 'node:fs'; import path from 'node:path'; import { parseFrontmatter } from '../../../../utils/markdown-frontmatter.js'; -import { space, unavailableCapabilityMessage } from '../../../storage/index.js'; +import { + space, + storageServes, + unavailableCapabilityMessage, +} from '../../../storage/index.js'; // ─── Always-skipped directory names ───────────────────────────────────────── @@ -147,7 +151,12 @@ export function safeResolve(canvasId: string, rel: string): string { // capability-matrix entry, so an operator learns this when they select a // profile rather than when an agent calls a tool. Refusing here is the // backstop behind that declaration, phrased the same way. - const tree = space(canvasId).diskTree; + // The matrix decides; `diskTree` supplies the root. These tools address the + // byte areas through `upload/` and `artifacts/` aliases as well as `nodes/`, + // so the requirement spans both axes and must not be re-derived here. + const tree = storageServes('builtin-file-tools') + ? space(canvasId).diskTree + : null; if (!tree) throw new Error(unavailableCapabilityMessage('builtin-file-tools')); const root = tree.directory(); diff --git a/apps/server/src/modules/agent/tools/handlers/fs-write.ts b/apps/server/src/modules/agent/tools/handlers/fs-write.ts index b3a5974fb..dbe2a5346 100644 --- a/apps/server/src/modules/agent/tools/handlers/fs-write.ts +++ b/apps/server/src/modules/agent/tools/handlers/fs-write.ts @@ -35,13 +35,10 @@ import { normalizeRel } from './fs-sandbox.js'; import { space, SPACE_MEMORY_BLOB_NAME, + storageServes, unavailableCapabilityMessage, } from '../../../storage/index.js'; -import { - hasWorkspaceSettingDirectory, - settingDir, - userSkillsDir, -} from '../../../workspace/paths.js'; +import { settingDir, userSkillsDir } from '../../../workspace/paths.js'; import { resolveLongTermPath, resolveUserSkillPath, @@ -121,7 +118,7 @@ function resolveTarget( // Refused in the words the profile declared, rather than crashing on a // path the backend cannot build. A Space's own memory body still works — // it is a blob, not a Workspace file. - if (!hasWorkspaceSettingDirectory()) { + if (!storageServes('workspace-user-memory')) { return { path: rel, error: unavailableCapabilityMessage('workspace-user-memory'), @@ -171,7 +168,7 @@ function resolveTarget( error: `fs_write only accepts skill paths of the form "skills//SKILL.md"`, }; } - if (!hasWorkspaceSettingDirectory()) { + if (!storageServes('workspace-user-skills')) { return { path: rel, error: unavailableCapabilityMessage('workspace-user-skills'), diff --git a/apps/server/src/modules/canvas/canvas.route.ts b/apps/server/src/modules/canvas/canvas.route.ts index f81e72856..c678f117e 100644 --- a/apps/server/src/modules/canvas/canvas.route.ts +++ b/apps/server/src/modules/canvas/canvas.route.ts @@ -59,6 +59,7 @@ import { deleteSpace, isWorldCanvasId, stageSpaceImport, + storageServes, unavailableCapabilityMessage, getStructuredStore, type CanvasFile, @@ -1621,7 +1622,10 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { // missing are different problems with different remedies, so they get // different answers — the first repeats the matrix sentence the operator // read when they chose the profile. - const tree = handle.diskTree; + // The matrix decides, and `diskTree` only supplies the path. Asking it + // directly would re-derive the requirement, and this one already spans + // both axes: a bundle needs the Space's bytes in the folder it archives. + const tree = storageServes('reveal-space-folder') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ message: unavailableCapabilityMessage('reveal-space-folder'), @@ -1667,7 +1671,10 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { // a portable export generated from records plus reachable blob references // is a separate later design. Refuse in the matrix's own words, and keep // that distinct from a Space whose directory has gone missing. - const tree = handle.diskTree; + // The matrix decides, and `diskTree` only supplies the path. Asking it + // directly would re-derive the requirement, and this one already spans + // both axes: a bundle needs the Space's bytes in the folder it archives. + const tree = storageServes('space-bundle-export') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ message: unavailableCapabilityMessage('space-bundle-export'), @@ -1739,7 +1746,12 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { // staging location, the title-derived directory, the record filename, // and the index entry are all layout. This route owns the `.huabu.zip` // format and nothing else (proposal §12.6.2). - const staged = stageSpaceImport(targetCanvasId); + // Same rule as export: the matrix decides, and staging only supplies + // the place. Import needs the bytes to land in the folder too, so the + // requirement spans both axes and re-deriving it here would miss that. + const staged = storageServes('space-bundle-import') + ? stageSpaceImport(targetCanvasId) + : null; if (!staged) { return reply.code(400).send({ message: unavailableCapabilityMessage('space-bundle-import'), diff --git a/apps/server/src/modules/canvas/external.route.ts b/apps/server/src/modules/canvas/external.route.ts index 600c4144c..a2a7e97bb 100644 --- a/apps/server/src/modules/canvas/external.route.ts +++ b/apps/server/src/modules/canvas/external.route.ts @@ -17,7 +17,11 @@ import { takeExternalNote, } from './external-watcher.js'; import { parseFrontmatter } from '../../utils/markdown-frontmatter.js'; -import { space, unavailableCapabilityMessage } from '../storage/index.js'; +import { + space, + storageServes, + unavailableCapabilityMessage, +} from '../storage/index.js'; import type { FastifyPluginAsync } from 'fastify'; @@ -98,7 +102,9 @@ const externalRoutes: FastifyPluginAsync = async (fastify): Promise => { // Disk-only, declared as `external-note-discovery` in the capability // matrix: it adopts documents that arrived without going through the // application, and no database backend has such an arrival path. - const tree = space(canvasId).diskTree; + const tree = storageServes('external-note-discovery') + ? space(canvasId).diskTree + : null; if (!tree) { return reply.code(400).send({ message: unavailableCapabilityMessage('external-note-discovery'), diff --git a/apps/server/src/modules/remote_fs/rfs.route.ts b/apps/server/src/modules/remote_fs/rfs.route.ts index a8727a65b..a399d3498 100644 --- a/apps/server/src/modules/remote_fs/rfs.route.ts +++ b/apps/server/src/modules/remote_fs/rfs.route.ts @@ -113,8 +113,7 @@ import { interactiveViewService, } from '../interactive-view/interactive-view.service.js'; import { - hasStorageCapability, - parseStorageProfile, + storageServes, unavailableCapabilityMessage, } from '../storage/index.js'; import { @@ -266,7 +265,7 @@ const rfsRoutes: FastifyPluginAsync = async (app) => { // `space-file-plane` capability. One hook, because every route below // resolves a real path sooner or later. app.addHook('onRequest', async (_request, reply) => { - if (hasStorageCapability(parseStorageProfile(), 'space-file-plane')) return; + if (storageServes('space-file-plane')) return; return reply .code(409) .send(rfsError(unavailableCapabilityMessage('space-file-plane'))); diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index 3df349fbf..e14f92131 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -39,9 +39,14 @@ describe('storage capability matrix', () => { expect(new Set(ids).size).toBe(ids.length); for (const capability of STORAGE_CAPABILITIES) { - expect(capability.backends.length).toBeGreaterThan(0); // A capability nothing can serve is not a limitation, it is a removed - // feature; a capability every backend serves does not belong here. + // feature; one that names no axis at all is served everywhere and does + // not belong on an exception list. + const axes = [capability.structured, capability.blobs].filter( + (axis) => axis !== undefined, + ); + expect(axes.length).toBeGreaterThan(0); + for (const axis of axes) expect(axis.length).toBeGreaterThan(0); expect(capability.summary).not.toHaveLength(0); expect(capability.rationale).not.toHaveLength(0); } @@ -68,6 +73,48 @@ describe('storage capability matrix', () => { expect(hasStorageCapability(DISK, 'reveal-space-folder')).toBe(true); }); + /** + * The reason the matrix is keyed on the profile rather than on one axis. + * + * `disk`/`azure` has no adapter and `validateStorageProfile` would refuse + * it, which is exactly why it is the right shape to assert against: the + * matrix must already answer correctly for the pairing before anyone can + * select it. Records are files here and Spaces are real directories — a + * structured-only matrix would call the bundle exportable, and it would + * archive a Space folder whose artifacts had never been written to it. + */ + it('takes the bundle with a blob backend that cannot co-locate', () => { + const OFFSITE_BYTES: StorageProfile = { + structured: { kind: 'disk' }, + blobs: { kind: 'azure' }, + }; + + expect(hasStorageCapability(OFFSITE_BYTES, 'space-bundle-export')).toBe( + false, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'space-bundle-import')).toBe( + false, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'builtin-file-tools')).toBe( + false, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'space-file-plane')).toBe(false); + + // What survives: the Space folder still holds the record and the node + // documents, so showing it to a user is still showing them the Space, and + // a note dropped into `nodes/` still arrives. Those rows name no blob + // axis, which is how they say they do not care where the bytes went. + expect(hasStorageCapability(OFFSITE_BYTES, 'reveal-space-folder')).toBe( + true, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'external-note-discovery')).toBe( + true, + ); + expect(hasStorageCapability(OFFSITE_BYTES, 'workspace-directory')).toBe( + true, + ); + }); + it('treats an unknown id as available rather than guessing', () => { // The matrix is an exception list. A feature nobody wrote down is // portable by construction, and inventing a refusal for it would make @@ -93,7 +140,8 @@ describe('storage capability matrix', () => { expect(line).toBeDefined(); // The id to search for, what is lost, and why it cannot be emulated. expect(line).toContain(capability.summary); - expect(line).toContain('sqlite'); + // The whole profile, because a row may be unavailable for either axis. + expect(line).toContain('sqlite/disk'); } }); }); diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 6c3beb67d..6cc2a3083 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -22,67 +22,85 @@ * This is a *declaration*, not an enforcement point. Each listed feature also * refuses at its own call site, because a matrix nobody consults at runtime is * documentation. What the matrix adds is the up-front answer. + * + * Two rules keep those call sites honest: + * + * - **A refusal asks the matrix.** `storageServes(id)` on the composition + * root, never a re-derivation of the requirement such as "is there a + * `diskTree`". A gate that re-derives is a second copy of the rule, and it + * is how a row could grow a blob-axis requirement its own call site never + * learned about. + * - **A degradation does not.** Code that renders absence rather than + * refusing — the memory preamble reading as empty — asks the concrete + * predicate, because it is not making the profile's promise, only reading + * what is there. + * + * Every row must therefore be refusable. A property that nothing can ask about + * is not a capability: it is a fact about a backend, and it belongs in that + * backend's own commentary. Windows directory-handle coordination was listed + * here and removed for exactly that reason — a Space with no directory + * registers no handle owner, so nothing ever asks and nothing is lost. */ +import type { BlobBackendKind } from './ports/blob.js'; import type { StructuredBackendKind } from './ports/structured.js'; import type { StorageProfile } from './profile.js'; /** - * A product feature whose availability depends on the structured backend. + * A product feature some storage profiles cannot serve. * - * Keyed by structured kind alone: every entry here needs a Space to be a real - * directory, which is a structured-backend property. + * Keyed on the **profile**, not on one axis. Most entries need a Space or a + * Workspace to be a real directory, which is a structured-backend property — + * but several need more than that: they need the Space's *bytes* to be in that + * directory too, and that is the blob backend's business. A matrix that asked + * only the structured axis would call a bundle exportable on a profile that + * archives a Space folder its artifacts had never been written to. * - * Four of them need slightly more than that — they need the Space's *bytes* to - * be in that directory too. Bundle export archives the folder, bundle import - * unzips into it, reveal-in-file-manager shows it, and the built-in file tools - * sandbox on it; a Space whose artifacts had been relocated elsewhere would - * export as an incomplete bundle rather than fail. Today that is free: the one - * blob backend is a file system, and composition places a Space's bytes inside - * its directory whenever it has one (`storage.ts::buildBlobStore`). A blob - * backend that *cannot* co-locate — an object store — would break the - * implication, and those four rows would then have to be keyed on the profile - * rather than on the structured kind. That is the second matrix this comment - * used to say did not exist; it does not exist yet. + * Each axis is a list of the backends that serve the feature, and **omitting + * an axis means every backend on it serves the feature**. That default is the + * point rather than a shortcut: a feature that does not touch a Space's bytes + * must not need editing when a blob backend is added, and the features that + * do are exactly the ones that should force a decision then. The same holds + * in reverse for a future feature that depends only on the blob axis. */ export interface StorageCapability { /** Stable id, for a diagnostic an operator can search for. */ readonly id: string; /** What a user loses, in their vocabulary rather than the port's. */ readonly summary: string; - /** Structured backends that serve it. */ - readonly backends: readonly StructuredBackendKind[]; + /** Structured backends that serve it; omitted means all of them do. */ + readonly structured?: readonly StructuredBackendKind[]; + /** Blob backends that serve it; omitted means all of them do. */ + readonly blobs?: readonly BlobBackendKind[]; /** Why it cannot be served elsewhere, and what remains instead. */ readonly rationale: string; } -/** - * Every feature that is not available on every backend. - * - * Deliberately not "every feature" — a matrix that listed the portable ones - * too would need updating whenever anything was built, and would go stale - * silently. What must stay accurate is the exception list. - */ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'space-bundle-export', summary: 'Export a Space as a .huabu.zip bundle', - backends: ['disk'], + structured: ['disk'], + blobs: ['disk'], rationale: - 'The bundle is a Disk projection — the Space directory, archived. A ' + + 'The bundle is a Disk projection — the Space directory, archived — so ' + + 'it needs both halves of that directory: the records and the bytes. A ' + 'portable export generated from records plus reachable blob references ' + 'is a separate design.', }, { id: 'space-bundle-import', summary: 'Import a Space from a .huabu.zip bundle', - backends: ['disk'], - rationale: 'Pairs with export; unzips into place.', + structured: ['disk'], + blobs: ['disk'], + rationale: + 'Pairs with export; unzips into place, which is only the whole Space ' + + 'where the whole Space is in that place.', }, { id: 'reveal-space-folder', summary: 'Reveal a Space in the OS file manager', - backends: ['disk'], + structured: ['disk'], rationale: 'The feature is "show me this in Finder", and what a user means by ' + '"this" is the Space: its record and its node documents. Those are ' + @@ -93,7 +111,8 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'builtin-file-tools', summary: 'Built-in agent file tools (read, write, glob, grep)', - backends: ['disk'], + structured: ['disk'], + blobs: ['disk'], rationale: 'They sandbox on the Space directory and the documents they exist to ' + 'edit are the node sidecars under `nodes/`, which are rows here. A ' + @@ -105,7 +124,8 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'space-file-plane', summary: 'Reach a Space as files over RFS, the plane external agents mount', - backends: ['disk'], + structured: ['disk'], + blobs: ['disk'], rationale: 'RFS projects the Space directory over HTTP — the record and the node ' + 'sidecars, reachable from another machine. Those are rows here, and a ' + @@ -118,7 +138,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'external-note-discovery', summary: 'Adopt Markdown files dropped into a Space from outside the app', - backends: ['disk'], + structured: ['disk'], rationale: 'It watches `nodes/` for documents that arrived without going through ' + 'the application. That tier is rows here, and no byte area is a place ' + @@ -128,7 +148,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'workspace-directory', summary: 'Choose, create, or reveal a Workspace folder on this machine', - backends: ['disk'], + structured: ['disk'], rationale: 'A Workspace is a folder the user picks. Where Workspaces are rows ' + 'there is nothing to browse to: the Server opens its own on first ' + @@ -139,7 +159,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'workspace-user-memory', summary: 'The cross-Space user memory document (setting/user.md)', - backends: ['disk'], + structured: ['disk'], rationale: 'A user-editable file at the Workspace root, deliberately outside any ' + 'Space so it applies to all of them. The blob port has no ' + @@ -150,47 +170,60 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'workspace-user-skills', summary: 'User-authored skills under the Workspace setting/skills folder', - backends: ['disk'], + structured: ['disk'], rationale: 'Skills are read as files a user can edit and drop in by hand, which ' + 'is the same arrival path external notes rely on. Bundled and Agent ' + 'Team skills are unaffected.', }, - { - id: 'space-directory-handle-coordination', - summary: 'Windows: rename or delete a Space while a watcher holds it open', - backends: ['disk'], - rationale: - 'Exists so renaming a Space *directory* can succeed against a live ' + - '`fs.watch` handle. A Space that is a row is never filed under its ' + - 'title, so nothing renames its byte directory, and with external-note ' + - 'discovery unavailable nothing watches it either. No rename and no ' + - 'watcher, so there are no handles to arbitrate.', - }, ]; +/** + * Whether one profile serves one capability. + * + * An axis the capability does not name is an axis it does not depend on, so + * every backend there passes. A profile may request a backend that has no + * adapter — `validateStorageProfile` is what rejects those — and such a kind + * appears in no list, which is the right answer: an unwritten backend serves + * nothing. + */ +function serves( + capability: StorageCapability, + profile: StorageProfile, +): boolean { + const onAxis = ( + serving: readonly string[] | undefined, + configured: string, + ): boolean => serving === undefined || serving.includes(configured); + return ( + onAxis(capability.structured, profile.structured.kind) && + onAxis(capability.blobs, profile.blobs.kind) + ); +} + /** Capabilities this profile cannot serve. */ export function unavailableCapabilities( profile: StorageProfile, ): readonly StorageCapability[] { return STORAGE_CAPABILITIES.filter( - (capability) => - !(capability.backends as readonly string[]).includes( - profile.structured.kind, - ), + (capability) => !serves(capability, profile), ); } -/** Whether this profile serves `id`. Unknown ids are available by omission. */ +/** + * Whether this profile serves `id`. Unknown ids are available by omission. + * + * Application code should not reach this directly — the only profile worth + * asking about is the one storage was opened with, and the composition root's + * `storageServes(id)` is bound to it. This form exists for the matrix's own + * tests, which need to ask about profiles the process is not running. + */ export function hasStorageCapability( profile: StorageProfile, id: string, ): boolean { const capability = STORAGE_CAPABILITIES.find((entry) => entry.id === id); - if (!capability) return true; - return (capability.backends as readonly string[]).includes( - profile.structured.kind, - ); + return capability === undefined || serves(capability, profile); } /** @@ -223,9 +256,10 @@ export function unavailableCapabilityMessage(id: string): string { export function describeUnavailableCapabilities( profile: StorageProfile, ): readonly string[] { + const label = `${profile.structured.kind}/${profile.blobs.kind}`; return unavailableCapabilities(profile).map( (capability) => `${capability.id}: ${capability.summary} — unavailable on the ` + - `"${profile.structured.kind}" structured backend. ${capability.rationale}`, + `"${label}" storage profile. ${capability.rationale}`, ); } diff --git a/apps/server/src/modules/storage/index.ts b/apps/server/src/modules/storage/index.ts index d7b6df724..78a123395 100644 --- a/apps/server/src/modules/storage/index.ts +++ b/apps/server/src/modules/storage/index.ts @@ -116,11 +116,16 @@ export { } from './profile.js'; export { describeUnavailableCapabilities, - hasStorageCapability, STORAGE_CAPABILITIES, unavailableCapabilities, unavailableCapabilityMessage, } from './capabilities.js'; +/** + * Capability questions are asked of the profile in force, never of one the + * caller assembled — so the bound accessor is what leaves the module and + * `hasStorageCapability` stays inside it. + */ +export { storageServes } from './storage.js'; export type { StorageCapability } from './capabilities.js'; export type { StorageProfile } from './profile.js'; export { diff --git a/apps/server/src/modules/storage/module-boundaries.test.ts b/apps/server/src/modules/storage/module-boundaries.test.ts index 043dd0c84..5e027b421 100644 --- a/apps/server/src/modules/storage/module-boundaries.test.ts +++ b/apps/server/src/modules/storage/module-boundaries.test.ts @@ -21,6 +21,8 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { STORAGE_CAPABILITIES } from './capabilities.js'; + const HERE = path.dirname(fileURLToPath(import.meta.url)); const STORAGE_DIR = HERE; const SRC_DIR = path.resolve(HERE, '../..'); @@ -175,6 +177,29 @@ describe('storage dependency direction', () => { expect(violations).toEqual([]); }); + /** + * A declared capability is refused somewhere, or it is not a capability. + * + * `capabilities.ts` promises that every row also refuses at its own call + * site, "because a matrix nobody consults at runtime is documentation". This + * is that promise, checked. It catches the two ways it rots: a row added for + * an operator's benefit that no feature ever asks about, and a refusal + * deleted while its row stays behind, still printed at boot. + */ + it('refuses every capability it declares, outside the storage module', () => { + const consumers = sourceFiles + .filter((f) => !f.startsWith('modules/storage/')) + .filter((f) => !f.endsWith('.test.ts')) + .map((f) => read(f)); + + const unenforced = STORAGE_CAPABILITIES.filter( + (capability) => + !consumers.some((source) => source.includes(`'${capability.id}'`)), + ).map((capability) => capability.id); + + expect(unenforced).toEqual([]); + }); + /** * Each backend owns its own area of the Server data directory. * diff --git a/apps/server/src/modules/storage/storage.ts b/apps/server/src/modules/storage/storage.ts index 7f8b198ad..7d4e5918f 100644 --- a/apps/server/src/modules/storage/storage.ts +++ b/apps/server/src/modules/storage/storage.ts @@ -48,6 +48,7 @@ import { } from './backends/sqlite/database.js'; import { SqliteStructuredStore } from './backends/sqlite/structured-store.js'; import { SqliteWorkspaceRepository } from './backends/sqlite/workspace-repository.js'; +import { hasStorageCapability } from './capabilities.js'; import { spaceBlobAreas } from './ports/blob.js'; import { parseStorageProfile, @@ -395,6 +396,23 @@ export function getWorkspaceRepository(): WorkspaceRepository { return workspaces; } +/** + * Whether the profile in force serves `id`. + * + * The one form application code should use. `hasStorageCapability` takes a + * profile, and the only profile worth asking about is the one storage was + * actually opened with — a call site that parses the environment instead gets + * a different answer the moment a test or an embedder mounts an explicit + * profile. Binding it here removes the choice. + * + * Ask this in a refusal. Code that degrades to absence instead of refusing + * should keep asking the concrete predicate it depends on; see + * `capabilities.ts`. + */ +export function storageServes(id: string): boolean { + return hasStorageCapability(activeProfile(), id); +} + /** * Whether the Disk Workspace membership registry already exists on disk. * diff --git a/apps/server/src/modules/workspace.route.ts b/apps/server/src/modules/workspace.route.ts index 126dc5bf4..173f05e6e 100644 --- a/apps/server/src/modules/workspace.route.ts +++ b/apps/server/src/modules/workspace.route.ts @@ -13,6 +13,7 @@ import { getStructuredStore, materializesWorkspaces, resetStorageCache, + storageServes, unavailableCapabilityMessage, } from './storage/index.js'; import { @@ -205,7 +206,7 @@ const workspaceRoutes: FastifyPluginAsync = async (app) => { if (isManagedMode()) { return sendError(reply, 403, 'Workspace is locked'); } - if (!materializesWorkspaces()) { + if (!storageServes('workspace-directory')) { return sendError( reply, 409, @@ -272,7 +273,7 @@ const workspaceRoutes: FastifyPluginAsync = async (app) => { 'Forbidden: workspace settings can only be changed from localhost', ); } - if (!materializesWorkspaces()) { + if (!storageServes('workspace-directory')) { return sendError( reply, 409, diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 13d3c88c0..54c719f56 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -130,9 +130,9 @@ Notes an operator needs: - **A Workspace is a row.** There is no folder to pick, so the Server creates and activates one on first start and reports `path: null` with `canChangeWorkspace: false`; the client shows no picker. Switching Workspaces re-scopes the one connection and reopens nothing. - **The connection is shared.** The structured store and the Workspace repository use one `node:sqlite` connection, opened in WAL with `synchronous = NORMAL`, a bounded `busy_timeout`, and foreign keys enforced. One process, one connection: nothing here promises a multi-process fence. - **Bytes are outside the database.** The blob axis is a file system on every profile, so a Space's uploads, artifacts, guide and memory body are ordinary files under the byte root and the database stays the size of its records. Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and, where the record is a row, composition also removes the `//` directory it placed those areas under, because nothing else would. -- **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, user-authored skills under `setting/skills/`, and Windows directory-handle coordination. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. +- **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, and user-authored skills under `setting/skills/`. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. - These are keyed on the **structured** backend, not the blob one, and that survives the hybrid profile: every entry needs the Space's record and node documents to exist as files, so a real file system for its bytes gives none of them back. Each refusal keys on `Space.diskTree` being `null`, which stays `null` however many byte directories exist. + These are keyed on the **profile**, both axes. Most need a Space or Workspace to be a real directory (a structured-backend property); bundle export, bundle import, the file tools, and RFS additionally need the Space's _bytes_ to be in that directory, which is the blob backend's. Omitting an axis in `capabilities.ts` means every backend on it serves the feature, so a blob-agnostic row never needs editing when a blob backend is added. On the hybrid profile the answer is unchanged: a file system for the bytes hands nothing back, because every row still needs the record and node documents to be files. Refusals ask `storageServes(id)`; `module-boundaries.test.ts` checks that every declared row is refused somewhere. ## 3. Storage composition and ownership diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 30e8e9316..5df1d0b98 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -2673,12 +2673,23 @@ same cascade as everything else. These capabilities are Disk-only, declared in `storage/capabilities.ts`, logged at startup, and refused at their own call sites in the same words. -They are keyed on the **structured** backend, and the hybrid profile is what -makes that worth stating: every one needs the Space's record and node -documents to exist as files, and a real file system for the Space's _bytes_ -gives none of them back. Each refusal keys on `Space.diskTree` being `null`, -which is a structured-backend fact — the byte directory is not a Space tree, -and `detached-blobs.test.ts` pins that. +They are keyed on the **profile**, not on one axis. Most need a Space or a +Workspace to be a real directory, which is a structured-backend property; four +need more than that — they need the Space's _bytes_ to be in that directory +too, and that is the blob backend's. A structured-only matrix would call a +bundle exportable on a profile that archives a Space folder its artifacts had +never been written to. + +Each axis lists the backends that serve the feature, and **omitting an axis +means every backend on it does**. That default is the point: a feature that +does not touch a Space's bytes must not need editing when a blob backend is +added, and the ones that do are exactly the ones that should force a decision +then. + +On the hybrid profile the answer is unchanged — a real file system for the +bytes hands nothing back, because every row still needs the record and node +documents to be files. `detached-blobs.test.ts` pins the fact underneath that: +bytes are files and `Space.diskTree` is still `null`. | Capability | What is lost | Why it is not emulated | | --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -2690,15 +2701,29 @@ and `detached-blobs.test.ts` pins that. | `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | | `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | A file the user edits at the root of a Workspace they chose. The blob port has no Workspace-level scope, so it has none to live in. A Space's _own_ memory body is a blob and is unaffected. | | `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | -| `space-directory-handle-coordination` | Windows rename-while-watched | Nothing renames a byte directory keyed by id, and with note discovery off nothing watches it. No handles to arbitrate. | -One further limit is not a capability row because nothing refuses it; it is -simply a property of the backend: +Two further limits are not capability rows, because nothing refuses them and +a row nothing can refuse is a fact about a backend rather than a capability: + +- **Windows directory-handle coordination.** It exists so renaming a Space + directory can succeed against a live `fs.watch` handle. A Space that is a + row is never filed under its title and nothing watches it, so no handle is + ever registered and nothing asks. It was listed as a capability and removed: + reading "unavailable" told an operator they had lost something, when the + profile simply does not have the problem. - **Multi-process access.** One process, one connection. WAL and `busy_timeout` make a second reader survivable, and nothing here promises a multi-process deletion fence or a distributed transaction. +Two rules keep the call sites honest, and `module-boundaries.test.ts` checks +the first: a **refusal asks the matrix** through `storageServes(id)` on the +composition root, never a re-derivation such as "is there a `diskTree`", +because a re-derivation is a second copy of the rule that never learns when a +row grows a second axis. A **degradation does not** — code that renders +absence instead of refusing asks the concrete predicate, because it is not +making the profile's promise. + #### 12.9.5 Proof The reusable contracts — structured store, Space repository, nodes, ordered From 118e3db63127e38422d639f31ca5f41695ff0580 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 11:53:47 +0800 Subject: [PATCH 13/15] refactor(storage): state a capability's requirement instead of two lists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `structured: ['disk']` beside `blobs: ['disk']` does not say how the two combine. Read one way it is "these backends serve it"; read another it is a set of supported profiles. The answer was `and` across axes and `or` within one, and nothing in the shape said so — the evaluator was the only place to find out. The two lists become one clause each of a `StorageRequirement`: requires: { structured: ['disk'], blobs: ['disk'] } which reads as the conjunction it is. Every clause present must hold; any listed backend satisfies its own clause; an absent clause requires nothing, so every backend on that axis passes. The type doc gives all three rules and the `{ structured: ['disk', 'postgres'] }` shape a second structured backend would take, so the `or` is not something a reader has to infer from a single-element list. `capabilities.test.ts` asserts the semantics rather than describing them: the hybrid profile meets `space-bundle-export`'s blob clause and fails its structured one, `disk`/`azure` is the mirror image, and both are refused — half a requirement is not a requirement met. `reveal-space-folder` names no blob clause and survives `disk`/`azure`, which is the absent-clause rule doing its job. No behaviour change; the evaluator computed this already. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../src/modules/storage/capabilities.test.ts | 38 +++++++-- .../src/modules/storage/capabilities.ts | 79 ++++++++++++------- docs/architecture/canvas-storage.md | 2 +- docs/proposals/multi-backend-storage.md | 11 ++- 4 files changed, 92 insertions(+), 38 deletions(-) diff --git a/apps/server/src/modules/storage/capabilities.test.ts b/apps/server/src/modules/storage/capabilities.test.ts index e14f92131..c7f0330f9 100644 --- a/apps/server/src/modules/storage/capabilities.test.ts +++ b/apps/server/src/modules/storage/capabilities.test.ts @@ -42,11 +42,14 @@ describe('storage capability matrix', () => { // A capability nothing can serve is not a limitation, it is a removed // feature; one that names no axis at all is served everywhere and does // not belong on an exception list. - const axes = [capability.structured, capability.blobs].filter( - (axis) => axis !== undefined, - ); - expect(axes.length).toBeGreaterThan(0); - for (const axis of axes) expect(axis.length).toBeGreaterThan(0); + const clauses = [ + capability.requires.structured, + capability.requires.blobs, + ].filter((clause) => clause !== undefined); + // A requirement with no clauses is met by every profile. + expect(clauses.length).toBeGreaterThan(0); + // A clause no backend satisfies is a removed feature, not a limitation. + for (const clause of clauses) expect(clause.length).toBeGreaterThan(0); expect(capability.summary).not.toHaveLength(0); expect(capability.rationale).not.toHaveLength(0); } @@ -115,6 +118,31 @@ describe('storage capability matrix', () => { ); }); + it('requires every clause, and any backend within one', () => { + // `and` across axes: the hybrid profile satisfies the blob clause of + // `space-bundle-export` and fails its structured one, and half a + // requirement is not a requirement met. + expect(hasStorageCapability(TABLES, 'space-bundle-export')).toBe(false); + // ...and the mirror image, which is the disk/azure case above. + expect( + hasStorageCapability( + { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, + 'space-bundle-export', + ), + ).toBe(false); + + // `or` within a clause: the one row that names a blob backend is met by + // that backend, and an absent clause is met by anything — which is what + // lets `reveal-space-folder` survive a blob backend it never named. + expect(hasStorageCapability(DISK, 'space-bundle-export')).toBe(true); + expect( + hasStorageCapability( + { structured: { kind: 'disk' }, blobs: { kind: 'azure' } }, + 'reveal-space-folder', + ), + ).toBe(true); + }); + it('treats an unknown id as available rather than guessing', () => { // The matrix is an exception list. A feature nobody wrote down is // portable by construction, and inventing a refusal for it would make diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 6cc2a3083..9a2b7b895 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -56,32 +56,53 @@ import type { StorageProfile } from './profile.js'; * only the structured axis would call a bundle exportable on a profile that * archives a Space folder its artifacts had never been written to. * - * Each axis is a list of the backends that serve the feature, and **omitting - * an axis means every backend on it serves the feature**. That default is the - * point rather than a shortcut: a feature that does not touch a Space's bytes - * must not need editing when a blob backend is added, and the features that - * do are exactly the ones that should force a decision then. The same holds - * in reverse for a future feature that depends only on the blob axis. + * Each row therefore states a {@link StorageRequirement} rather than a list of + * backends: every axis it names must hold, and an axis it does not name is one + * it does not depend on. */ export interface StorageCapability { /** Stable id, for a diagnostic an operator can search for. */ readonly id: string; /** What a user loses, in their vocabulary rather than the port's. */ readonly summary: string; - /** Structured backends that serve it; omitted means all of them do. */ - readonly structured?: readonly StructuredBackendKind[]; - /** Blob backends that serve it; omitted means all of them do. */ - readonly blobs?: readonly BlobBackendKind[]; + /** What a deployment must be for this feature to work. */ + readonly requires: StorageRequirement; /** Why it cannot be served elsewhere, and what remains instead. */ readonly rationale: string; } +/** + * The condition a profile has to meet, one clause per storage axis. + * + * **Every clause present must hold** — the axes are an `and`, because a + * feature that needs both a Space directory and the Space's bytes inside it + * needs both, not either. **Within a clause the backends are an `or`**: the + * configured backend has to be one of them. + * + * So `{ structured: ['disk'], blobs: ['disk'] }` reads "the structured backend + * must be Disk *and* the blob backend must be Disk", and a future + * `{ structured: ['disk', 'postgres'] }` would read "the structured backend + * must be Disk *or* Postgres, and the blob backend may be anything". + * + * **An absent clause is not a requirement**, so every backend on that axis + * passes. That default is the design rather than a shortcut: a feature that + * does not touch a Space's bytes must not need editing when a blob backend is + * added, and the features that do are exactly the ones that should be forced + * to decide then. A requirement with no clauses at all requires nothing, which + * is not a limitation — `capabilities.test.ts` rejects one. + */ +export interface StorageRequirement { + /** Structured backends that satisfy it; absent means any does. */ + readonly structured?: readonly StructuredBackendKind[]; + /** Blob backends that satisfy it; absent means any does. */ + readonly blobs?: readonly BlobBackendKind[]; +} + export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'space-bundle-export', summary: 'Export a Space as a .huabu.zip bundle', - structured: ['disk'], - blobs: ['disk'], + requires: { structured: ['disk'], blobs: ['disk'] }, rationale: 'The bundle is a Disk projection — the Space directory, archived — so ' + 'it needs both halves of that directory: the records and the bytes. A ' + @@ -91,8 +112,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'space-bundle-import', summary: 'Import a Space from a .huabu.zip bundle', - structured: ['disk'], - blobs: ['disk'], + requires: { structured: ['disk'], blobs: ['disk'] }, rationale: 'Pairs with export; unzips into place, which is only the whole Space ' + 'where the whole Space is in that place.', @@ -100,7 +120,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'reveal-space-folder', summary: 'Reveal a Space in the OS file manager', - structured: ['disk'], + requires: { structured: ['disk'] }, rationale: 'The feature is "show me this in Finder", and what a user means by ' + '"this" is the Space: its record and its node documents. Those are ' + @@ -111,8 +131,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'builtin-file-tools', summary: 'Built-in agent file tools (read, write, glob, grep)', - structured: ['disk'], - blobs: ['disk'], + requires: { structured: ['disk'], blobs: ['disk'] }, rationale: 'They sandbox on the Space directory and the documents they exist to ' + 'edit are the node sidecars under `nodes/`, which are rows here. A ' + @@ -124,8 +143,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'space-file-plane', summary: 'Reach a Space as files over RFS, the plane external agents mount', - structured: ['disk'], - blobs: ['disk'], + requires: { structured: ['disk'], blobs: ['disk'] }, rationale: 'RFS projects the Space directory over HTTP — the record and the node ' + 'sidecars, reachable from another machine. Those are rows here, and a ' + @@ -138,7 +156,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'external-note-discovery', summary: 'Adopt Markdown files dropped into a Space from outside the app', - structured: ['disk'], + requires: { structured: ['disk'] }, rationale: 'It watches `nodes/` for documents that arrived without going through ' + 'the application. That tier is rows here, and no byte area is a place ' + @@ -148,7 +166,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'workspace-directory', summary: 'Choose, create, or reveal a Workspace folder on this machine', - structured: ['disk'], + requires: { structured: ['disk'] }, rationale: 'A Workspace is a folder the user picks. Where Workspaces are rows ' + 'there is nothing to browse to: the Server opens its own on first ' + @@ -159,7 +177,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'workspace-user-memory', summary: 'The cross-Space user memory document (setting/user.md)', - structured: ['disk'], + requires: { structured: ['disk'] }, rationale: 'A user-editable file at the Workspace root, deliberately outside any ' + 'Space so it applies to all of them. The blob port has no ' + @@ -170,7 +188,7 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ { id: 'workspace-user-skills', summary: 'User-authored skills under the Workspace setting/skills folder', - structured: ['disk'], + requires: { structured: ['disk'] }, rationale: 'Skills are read as files a user can edit and drop in by hand, which ' + 'is the same arrival path external notes rely on. Bundled and Agent ' + @@ -191,13 +209,18 @@ function serves( capability: StorageCapability, profile: StorageProfile, ): boolean { - const onAxis = ( - serving: readonly string[] | undefined, + /** One clause: absent requires nothing, present is met by any member. */ + const satisfied = ( + allowed: readonly string[] | undefined, configured: string, - ): boolean => serving === undefined || serving.includes(configured); + ): boolean => allowed === undefined || allowed.includes(configured); + + const { structured, blobs } = capability.requires; + // Every clause, not any: a feature needing a Space directory *and* the + // Space's bytes inside it is not served by half of that. return ( - onAxis(capability.structured, profile.structured.kind) && - onAxis(capability.blobs, profile.blobs.kind) + satisfied(structured, profile.structured.kind) && + satisfied(blobs, profile.blobs.kind) ); } diff --git a/docs/architecture/canvas-storage.md b/docs/architecture/canvas-storage.md index 54c719f56..39b35ce23 100644 --- a/docs/architecture/canvas-storage.md +++ b/docs/architecture/canvas-storage.md @@ -132,7 +132,7 @@ Notes an operator needs: - **Bytes are outside the database.** The blob axis is a file system on every profile, so a Space's uploads, artifacts, guide and memory body are ordinary files under the byte root and the database stays the size of its records. Deletion order is the composition layer's saga — sweep every blob area, then drop the record — and, where the record is a row, composition also removes the `//` directory it placed those areas under, because nothing else would. - **What this profile does not serve** is declared in `capabilities.ts`, logged at startup, and refused in the same words at each call site: choosing/creating/revealing a Workspace folder, `.huabu.zip` export and import, reveal-in-file-manager, the built-in agent file tools, RFS's file plane, external-note discovery, the Workspace `setting/user.md` memory document, and user-authored skills under `setting/skills/`. A Space's _own_ memory body is unaffected — it is a blob. Bundled and Agent Team skills are unaffected. - These are keyed on the **profile**, both axes. Most need a Space or Workspace to be a real directory (a structured-backend property); bundle export, bundle import, the file tools, and RFS additionally need the Space's _bytes_ to be in that directory, which is the blob backend's. Omitting an axis in `capabilities.ts` means every backend on it serves the feature, so a blob-agnostic row never needs editing when a blob backend is added. On the hybrid profile the answer is unchanged: a file system for the bytes hands nothing back, because every row still needs the record and node documents to be files. Refusals ask `storageServes(id)`; `module-boundaries.test.ts` checks that every declared row is refused somewhere. + These are keyed on the **profile**, both axes. Most need a Space or Workspace to be a real directory (a structured-backend property); bundle export, bundle import, the file tools, and RFS additionally need the Space's _bytes_ to be in that directory, which is the blob backend's. Each row states a requirement with one clause per axis: every clause present must hold (`and` across axes), any listed backend satisfies its own clause (`or` within one), and an absent clause requires nothing — so a blob-agnostic row never needs editing when a blob backend is added. On the hybrid profile the answer is unchanged: a file system for the bytes hands nothing back, because every row still needs the record and node documents to be files. Refusals ask `storageServes(id)`; `module-boundaries.test.ts` checks that every declared row is refused somewhere. ## 3. Storage composition and ownership diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index 5df1d0b98..c36748822 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -2680,11 +2680,14 @@ too, and that is the blob backend's. A structured-only matrix would call a bundle exportable on a profile that archives a Space folder its artifacts had never been written to. -Each axis lists the backends that serve the feature, and **omitting an axis -means every backend on it does**. That default is the point: a feature that +Each row states a requirement with one clause per axis. **Every clause present +must hold** (the axes are an `and`) and **within a clause the backends are an +`or`**, so `{ structured: ['disk'], blobs: ['disk'] }` reads "Disk records +_and_ Disk bytes". **An absent clause is not a requirement**, so any backend on +that axis passes — which is the point rather than a shortcut: a feature that does not touch a Space's bytes must not need editing when a blob backend is -added, and the ones that do are exactly the ones that should force a decision -then. +added, and the ones that do are exactly the ones that should be forced to +decide then. On the hybrid profile the answer is unchanged — a real file system for the bytes hands nothing back, because every row still needs the record and node From dee49199f69e4a60d1af0deb095952cf55d65d22 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 8 Sep 2026 11:57:01 +0800 Subject: [PATCH 14/15] fix(storage): say what reveal actually opens, and drop a mis-pasted comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects reading the reveal gate. The comment above it had picked up a sentence about bundles needing their bytes, from the templated edit that moved these gates onto the matrix. Reveal has no blob clause, so the comment argued for a requirement the row does not have. The line under it still said "a backend without a folder has nothing to show", which stopped being true when the hybrid profile gave a Space a byte directory. The capability's summary was also wrong, and it is the sentence an operator reads at boot: it claimed "Reveal a Space in the OS file manager" while the route opens `/nodes/`. That distinction is the whole reason the row has no blob clause — the folder is the node *documents*, so where the bytes went does not enter into it. Both now say what the feature is: open the folder of node documents so a user can settle a duplicate-markdown collision by hand. Off Disk a node is a row, so there is no folder of documents to open and the collision cannot arise — label uniqueness is a constraint rather than a filename. The byte areas are files, but hidden, Server-owned, and full of artifacts rather than documents. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017Howod9Qp6y2sc4jPJY7Zw --- .../server/src/modules/canvas/canvas.route.ts | 25 ++++++++++--------- .../src/modules/storage/capabilities.ts | 15 ++++++----- docs/proposals/multi-backend-storage.md | 20 +++++++-------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/apps/server/src/modules/canvas/canvas.route.ts b/apps/server/src/modules/canvas/canvas.route.ts index c678f117e..b836ba8f5 100644 --- a/apps/server/src/modules/canvas/canvas.route.ts +++ b/apps/server/src/modules/canvas/canvas.route.ts @@ -1616,15 +1616,16 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { if (!(await handle.read())) { return reply.code(404).send({ message: 'Canvas not found' }); } - // Disk-only, declared as `reveal-space-folder`: the feature *is* "show me - // this in Finder", so a backend without a folder has nothing to show. + // Declared as `reveal-space-folder`: what this opens is the `nodes/` + // folder, and off Disk a node is a row, so there is no folder of node + // documents to open and no hand-editable collision to resolve in one. + // The matrix decides and `diskTree` only supplies the path — asking + // `diskTree` directly would re-derive the requirement here. + // // A profile that cannot serve the feature and a Space whose folder is // missing are different problems with different remedies, so they get // different answers — the first repeats the matrix sentence the operator // read when they chose the profile. - // The matrix decides, and `diskTree` only supplies the path. Asking it - // directly would re-derive the requirement, and this one already spans - // both axes: a bundle needs the Space's bytes in the folder it archives. const tree = storageServes('reveal-space-folder') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ @@ -1667,13 +1668,13 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { return reply.code(404).send({ message: 'Canvas not found' }); } - // Disk-only, declared as `space-bundle-export` in the capability matrix; - // a portable export generated from records plus reachable blob references - // is a separate later design. Refuse in the matrix's own words, and keep - // that distinct from a Space whose directory has gone missing. - // The matrix decides, and `diskTree` only supplies the path. Asking it - // directly would re-derive the requirement, and this one already spans - // both axes: a bundle needs the Space's bytes in the folder it archives. + // Declared as `space-bundle-export`; a portable export generated from + // records plus reachable blob references is a separate later design. The + // matrix decides and `diskTree` only supplies the path: this requirement + // spans both axes — the bundle is the Space folder archived, so it needs + // the bytes in it — and asking `diskTree` would re-derive only half. + // Refuse in the matrix's own words, and keep that distinct from a Space + // whose directory has gone missing. const tree = storageServes('space-bundle-export') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ diff --git a/apps/server/src/modules/storage/capabilities.ts b/apps/server/src/modules/storage/capabilities.ts index 9a2b7b895..989c9f965 100644 --- a/apps/server/src/modules/storage/capabilities.ts +++ b/apps/server/src/modules/storage/capabilities.ts @@ -119,14 +119,17 @@ export const STORAGE_CAPABILITIES: readonly StorageCapability[] = [ }, { id: 'reveal-space-folder', - summary: 'Reveal a Space in the OS file manager', + summary: "Open a Space's nodes folder in the OS file manager", requires: { structured: ['disk'] }, rationale: - 'The feature is "show me this in Finder", and what a user means by ' + - '"this" is the Space: its record and its node documents. Those are ' + - 'rows. The one directory such a Space has holds its opaque bytes and ' + - 'is Server-owned, so revealing it would open something that is not the ' + - 'thing that was asked for.', + 'It opens the folder of node documents so a user can settle a ' + + 'duplicate-markdown collision by hand. Off Disk a node is a row: there ' + + 'is no folder of documents to open, and the collision it exists to ' + + 'settle cannot arise, because label uniqueness is a constraint rather ' + + "than a filename. The Space's byte areas are files, but they are " + + 'hidden, Server-owned, and hold artifacts rather than documents — ' + + 'opening those would answer a question nobody asked. No blob clause: ' + + 'the folder is still the documents wherever the bytes went.', }, { id: 'builtin-file-tools', diff --git a/docs/proposals/multi-backend-storage.md b/docs/proposals/multi-backend-storage.md index c36748822..3c8288232 100644 --- a/docs/proposals/multi-backend-storage.md +++ b/docs/proposals/multi-backend-storage.md @@ -2694,16 +2694,16 @@ bytes hands nothing back, because every row still needs the record and node documents to be files. `detached-blobs.test.ts` pins the fact underneath that: bytes are files and `Space.diskTree` is still `null`. -| Capability | What is lost | Why it is not emulated | -| --------------------------------------------- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | -| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | -| `reveal-space-folder` | "Show me this in Finder" | What a user means by "this" is the Space — its record and node documents — and those are rows. Its byte directory is not the Space. | -| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | The documents they edit are the node sidecars under `nodes/`, which are rows. The first-party agent uses the Canvas tools instead. | -| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | -| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | -| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | A file the user edits at the root of a Workspace they chose. The blob port has no Workspace-level scope, so it has none to live in. A Space's _own_ memory body is a blob and is unaffected. | -| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | +| Capability | What is lost | Why it is not emulated | +| --------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workspace-directory` | Choosing, creating, or revealing a Workspace folder | A Workspace is a row. The Server opens its own on first start; the client is told `canChangeWorkspace: false` and shows no picker. | +| `space-bundle-export` / `space-bundle-import` | `.huabu.zip` round-trip | The bundle _is_ the Space directory, archived. A portable export built from records plus reachable blob references is a separate design. | +| `reveal-space-folder` | Open a Space's `nodes/` folder in the OS file manager | It exists so a user can settle a duplicate-markdown collision by hand. A node is a row here: no folder of documents to open, and no such collision to settle. | +| `builtin-file-tools` | The agent's `read`/`write`/`glob`/`grep` tools | The documents they edit are the node sidecars under `nodes/`, which are rows. The first-party agent uses the Canvas tools instead. | +| `space-file-plane` | RFS, the HTTP file plane external agents mount | Listed apart from the tools above because it is what they were previously said to fall back to. A Space with no file plane has neither. | +| `external-note-discovery` | Adopting Markdown dropped into a Space from outside | It watches for documents that arrived without going through the application. A database has no such arrival path. | +| `workspace-user-memory` | `setting/user.md`, the cross-Space memory document | A file the user edits at the root of a Workspace they chose. The blob port has no Workspace-level scope, so it has none to live in. A Space's _own_ memory body is a blob and is unaffected. | +| `workspace-user-skills` | `setting/skills//SKILL.md` | Same arrival path as external notes. Bundled and Agent Team skills are unaffected. | Two further limits are not capability rows, because nothing refuses them and a row nothing can refuse is a fact about a backend rather than a capability: From a505545a89438c4b2a33178a82e9110d055615b8 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Thu, 10 Sep 2026 12:28:13 +0800 Subject: [PATCH 15/15] fix(storage): surface unsupported Space import and export errors --- .../src/modules/canvas/canvas.route.test.ts | 63 +++++++++ .../server/src/modules/canvas/canvas.route.ts | 13 +- apps/web/src/api/canvas.test.ts | 63 +++++++++ apps/web/src/api/canvas.ts | 10 +- .../components/Panels/Header/CanvasMenu.tsx | 12 +- apps/web/src/hooks/useCanvasActions.test.tsx | 131 ++++++++++++++++++ apps/web/src/hooks/useCanvasActions.ts | 16 ++- apps/web/src/i18n/resources/en/common.json | 3 + apps/web/src/i18n/resources/zh-CN/common.json | 3 + apps/web/src/pages/CanvasListPage.test.tsx | 30 ++++ apps/web/src/pages/CanvasListPage.tsx | 8 +- packages/shared/src/types/api/canvas.ts | 2 + 12 files changed, 339 insertions(+), 15 deletions(-) create mode 100644 apps/web/src/api/canvas.test.ts create mode 100644 apps/web/src/hooks/useCanvasActions.test.tsx diff --git a/apps/server/src/modules/canvas/canvas.route.test.ts b/apps/server/src/modules/canvas/canvas.route.test.ts index 1cbbee60a..1cd744e90 100644 --- a/apps/server/src/modules/canvas/canvas.route.test.ts +++ b/apps/server/src/modules/canvas/canvas.route.test.ts @@ -711,6 +711,69 @@ function useTablesProfile(): () => void { } describe('Disk-only capability refusals', () => { + it('preflights Disk export without sending an archive, then still downloads it', async () => { + createCanvas('c1', 'Disk Space'); + const app = await buildApp(); + try { + const checked = await app.inject({ + method: 'GET', + url: '/canvas/c1/export?check=true', + }); + expect(checked.statusCode).toBe(204); + expect(checked.body).toBe(''); + expect(checked.headers['content-disposition']).toBeUndefined(); + const download = await app.inject({ + method: 'GET', + url: '/canvas/c1/export', + }); + expect(download.statusCode).toBe(200); + expect(download.headers['content-type']).toBe('application/zip'); + expect(download.rawPayload.subarray(0, 2).toString()).toBe('PK'); + const missing = await app.inject({ + method: 'GET', + url: '/canvas/missing/export?check=true', + }); + expect(missing.statusCode).toBe(404); + } finally { + await app.close(); + } + }); + + it('refuses an unsupported export during preflight using the same policy as download', async () => { + createCanvas('c1', 'Tables Space'); + const restore = useTablesProfile(); + const app = await buildApp(); + try { + const checked = await app.inject({ + method: 'GET', + url: '/canvas/c1/export?check=true', + }); + expect(checked.statusCode).toBe(400); + expect(checked.json()).toEqual({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: unavailableCapabilityMessage('space-bundle-export'), + }); + const body = multipartBody( + 'space.zip', + 'application/zip', + Buffer.from('zip'), + ); + const imported = await app.inject({ + method: 'POST', + url: '/canvas/import', + ...body, + }); + expect(imported.statusCode).toBe(400); + expect(imported.json()).toEqual({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: unavailableCapabilityMessage('space-bundle-import'), + }); + } finally { + await app.close(); + restore(); + } + }); + it('refuses in the same words the profile declared at startup', async () => { createCanvas('c1', 'Tables Space'); const restore = useTablesProfile(); diff --git a/apps/server/src/modules/canvas/canvas.route.ts b/apps/server/src/modules/canvas/canvas.route.ts index b836ba8f5..d4cae905c 100644 --- a/apps/server/src/modules/canvas/canvas.route.ts +++ b/apps/server/src/modules/canvas/canvas.route.ts @@ -1647,11 +1647,12 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { fastify.get<{ Params: { canvasId: string }; Querystring: ExportCanvasQuery; - // Success path streams a zip archive (Readable). Failure path is the + // Success streams a ZIP archive or returns 204 after an eligibility check. + // Failure is the // canonical ApiErrorBody — declared here so the 400/404 branches // type-check via the same `reply.send(...)` machinery the JSON // routes use. - Reply: ApiResult; + Reply: ApiResult; }>('/:canvasId/export', async function (request, reply) { const { canvasId } = request.params; const parsedQuery = exportCanvasQuerySchema.safeParse(request.query); @@ -1678,6 +1679,7 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { const tree = storageServes('space-bundle-export') ? handle.diskTree : null; if (!tree) { return reply.code(400).send({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', message: unavailableCapabilityMessage('space-bundle-export'), }); } @@ -1686,6 +1688,12 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { return reply.code(404).send({ message: 'Canvas directory not found' }); } + // The browser checks eligibility before following the native download link. + // Keep the checks above shared so preflight uses the same storage policy. + if (parsedQuery.data.check === 'true') { + return reply.code(204).send(undefined); + } + const manifest = { version: '2', exportedAt: new Date().toISOString(), @@ -1755,6 +1763,7 @@ const canvasRoutes: FastifyPluginAsync = async (fastify) => { : null; if (!staged) { return reply.code(400).send({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', message: unavailableCapabilityMessage('space-bundle-import'), }); } diff --git a/apps/web/src/api/canvas.test.ts b/apps/web/src/api/canvas.test.ts new file mode 100644 index 000000000..6cb34ecf3 --- /dev/null +++ b/apps/web/src/api/canvas.test.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { exportCanvas } from './canvas'; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe('Space export download', () => { + it.each([400, 404, 500])( + 'does not navigate or download on HTTP %s', + async (status) => { + const click = vi + .spyOn(HTMLAnchorElement.prototype, 'click') + .mockImplementation(() => {}); + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + message: 'Export is unavailable', + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + }), + { status }, + ), + ), + ); + + await expect(exportCanvas('space-1')).rejects.toMatchObject({ + status, + message: 'Export is unavailable', + }); + expect(click).not.toHaveBeenCalled(); + expect(document.querySelector('a')).toBeNull(); + }, + ); + + it('preflights eligibility then preserves the native streamed Disk download', async () => { + const fetch = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + vi.stubGlobal('fetch', fetch); + let download: string | undefined; + vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(function ( + this: HTMLAnchorElement, + ) { + download = this.getAttribute('href') ?? undefined; + }); + + await exportCanvas('space-1'); + + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch.mock.calls[0][0]).toMatch( + /\/canvas\/space-1\/export\?check=true$/, + ); + expect(download).toMatch(/\/canvas\/space-1\/export$/); + expect(document.querySelector('a')).toBeNull(); + }); +}); diff --git a/apps/web/src/api/canvas.ts b/apps/web/src/api/canvas.ts index 471942b7b..db1c0d83d 100644 --- a/apps/web/src/api/canvas.ts +++ b/apps/web/src/api/canvas.ts @@ -294,9 +294,9 @@ export async function getNodeContent( } /** - * Download the canvas as a self-contained `.huabu.json` export bundle. + * Download the canvas as a self-contained `.huabu.zip` export bundle. * - * Performs a lightweight existence check via getCanvas to catch errors early, + * Preflights export eligibility to surface refusals inside the application, * then triggers a native browser download via a temporary `` link * so the full response body never needs to live in JS memory. * @@ -304,11 +304,7 @@ export async function getNodeContent( * `Content-Disposition` header. */ export async function exportCanvas(canvasId: string): Promise { - // Lightweight pre-check: verify canvas exists without running the export. - const canvas = await getCanvas(canvasId); - if (!canvas) { - throw new Error('Canvas not found'); - } + await apiFetch(`${routes.canvasExport(canvasId)}?check=true`); const url = apiUrl(routes.canvasExport(canvasId)); const a = document.createElement('a'); diff --git a/apps/web/src/components/Panels/Header/CanvasMenu.tsx b/apps/web/src/components/Panels/Header/CanvasMenu.tsx index 505988d29..5bea70317 100644 --- a/apps/web/src/components/Panels/Header/CanvasMenu.tsx +++ b/apps/web/src/components/Panels/Header/CanvasMenu.tsx @@ -6,6 +6,7 @@ import { ChevronDown } from 'lucide-react'; import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import { ApiError } from '../../../api/_client'; import { exportCanvas } from '../../../api/canvas.ts'; import useCanvasStore from '../../../store/canvasStore.ts'; import { useWorkspaceStore } from '../../../store/workspaceStore.ts'; @@ -82,9 +83,14 @@ export const CanvasMenu: React.FC = ({ onOpenShortcuts }) => { await exportCanvas(canvasId); toast(t('canvasList.exportStarted'), { tone: 'success' }); } catch (err) { - toast(err instanceof Error ? err.message : t('canvasList.exportFailed'), { - tone: 'danger', - }); + toast( + err instanceof ApiError && err.code === 'STORAGE_CAPABILITY_UNAVAILABLE' + ? t('canvasList.exportUnavailable') + : err instanceof Error + ? err.message + : t('canvasList.exportFailed'), + { tone: 'danger' }, + ); } }, [canvasId, t]); diff --git a/apps/web/src/hooks/useCanvasActions.test.tsx b/apps/web/src/hooks/useCanvasActions.test.tsx new file mode 100644 index 000000000..9d2090d37 --- /dev/null +++ b/apps/web/src/hooks/useCanvasActions.test.tsx @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { createMemoryRouter, RouterProvider } from 'react-router-dom'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useCanvasActions } from './useCanvasActions'; +import { ToastContainer } from '../components/Common/Toast'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})); +vi.mock('./useInputMode', () => ({ useEffectiveInputMode: () => 'mouse' })); + +( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; +let container: HTMLDivElement; +let root: Root; +function ImportControl() { + const { onFileChange, isImporting } = useCanvasActions(); + return ( + <> + void onFileChange(e)} + disabled={isImporting} + /> + + + ); +} +beforeEach(() => { + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => { + document + .querySelectorAll('[aria-label="actions.dismiss"]') + .forEach((button) => button.click()); + }); + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +async function selectArchive() { + const input = container.querySelector('input'); + if (!input) throw new Error('Import input was not rendered'); + Object.defineProperty(input, 'files', { + value: [new File(['zip'], 'space.huabu.zip')], + configurable: true, + }); + await act(async () => { + input.dispatchEvent(new Event('change', { bubbles: true })); + }); + return input; +} +async function renderImport() { + const router = createMemoryRouter( + [ + { path: '/spaces', element: }, + { path: '/canvas/:id', element:
Imported Space
}, + ], + { initialEntries: ['/spaces'] }, + ); + await act(async () => root.render()); + return router; +} +describe('Space import feedback', () => { + it('shows a dismissible storage refusal, stays in the app, and allows retry', async () => { + const fetch = vi.fn(); + fetch.mockImplementation( + async () => + new Response( + JSON.stringify({ + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: 'Technical storage detail', + }), + { status: 400 }, + ), + ); + vi.stubGlobal('fetch', fetch); + const router = await renderImport(); + const input = await selectArchive(); + expect(document.querySelector('[role="status"]')?.textContent).toContain( + 'canvasList.importUnavailable', + ); + expect( + document.querySelector('[aria-label="actions.dismiss"]'), + ).not.toBeNull(); + expect(router.state.location.pathname).toBe('/spaces'); + expect(input.disabled).toBe(false); + expect(input.value).toBe(''); + await selectArchive(); + expect(fetch).toHaveBeenCalledTimes(2); + }); + it('shows other server failures instead of swallowing them', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ message: 'Invalid archive' }), { + status: 400, + }), + ), + ); + await renderImport(); + await selectArchive(); + expect(document.querySelector('[role="status"]')?.textContent).toContain( + 'Invalid archive', + ); + }); + it('opens a successfully imported Disk Space', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ canvasId: 'imported-space' }), { + status: 200, + }), + ), + ); + const router = await renderImport(); + await selectArchive(); + expect(router.state.location.pathname).toBe('/canvas/imported-space'); + expect(document.querySelector('[role="status"]')).toBeNull(); + }); +}); diff --git a/apps/web/src/hooks/useCanvasActions.ts b/apps/web/src/hooks/useCanvasActions.ts index 4e9748940..723ac7c84 100644 --- a/apps/web/src/hooks/useCanvasActions.ts +++ b/apps/web/src/hooks/useCanvasActions.ts @@ -2,10 +2,13 @@ // Licensed under the MIT license. import { useCallback, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; import { useNavigate } from 'react-router-dom'; import { useEffectiveInputMode } from './useInputMode'; +import { ApiError } from '../api/_client'; import { createCanvas, importCanvas } from '../api/canvas'; +import { toast } from '../components/Common/Toast'; /** * Shared "create / import canvas" actions. @@ -29,6 +32,7 @@ export interface UseCanvasActionsResult { export function useCanvasActions(): UseCanvasActionsResult { const navigate = useNavigate(); + const { t } = useTranslation(); const inputMode = useEffectiveInputMode(); const fileInputRef = useRef(null); const [isCreating, setIsCreating] = useState(false); @@ -70,12 +74,20 @@ export function useCanvasActions(): UseCanvasActionsResult { const result = await importCanvas(file); navigate(`/canvas/${result.canvasId}`); } catch (err) { - console.error('Failed to import canvas:', err); + toast( + err instanceof ApiError && + err.code === 'STORAGE_CAPABILITY_UNAVAILABLE' + ? t('canvasList.importUnavailable') + : err instanceof Error + ? err.message + : t('canvasList.importFailed'), + { tone: 'danger' }, + ); } finally { setIsImporting(false); } }, - [navigate], + [navigate, t], ); return { diff --git a/apps/web/src/i18n/resources/en/common.json b/apps/web/src/i18n/resources/en/common.json index 5da0d71c2..0a4e93ba0 100644 --- a/apps/web/src/i18n/resources/en/common.json +++ b/apps/web/src/i18n/resources/en/common.json @@ -356,6 +356,9 @@ "nodeCount_other": "{{count}} nodes", "updated": "Updated {{date}}", "exportStarted": "Export started", + "exportUnavailable": "Space export is not available with the current storage setup.", + "importUnavailable": "Space import is not available with the current storage setup.", + "importFailed": "Import failed", "exportFailed": "Export failed", "exporting": "Exporting…", "exportCanvas": "Export Space", diff --git a/apps/web/src/i18n/resources/zh-CN/common.json b/apps/web/src/i18n/resources/zh-CN/common.json index ebd51473c..38ba74bfb 100644 --- a/apps/web/src/i18n/resources/zh-CN/common.json +++ b/apps/web/src/i18n/resources/zh-CN/common.json @@ -356,6 +356,9 @@ "nodeCount_other": "{{count}} 个节点", "updated": "更新于 {{date}}", "exportStarted": "已开始导出", + "exportUnavailable": "当前存储配置不支持导出空间。", + "importUnavailable": "当前存储配置不支持导入空间。", + "importFailed": "导入失败", "exportFailed": "导出失败", "exporting": "正在导出…", "exportCanvas": "导出 Space", diff --git a/apps/web/src/pages/CanvasListPage.test.tsx b/apps/web/src/pages/CanvasListPage.test.tsx index b5c4cf763..af9707f90 100644 --- a/apps/web/src/pages/CanvasListPage.test.tsx +++ b/apps/web/src/pages/CanvasListPage.test.tsx @@ -7,6 +7,9 @@ import { createMemoryRouter, RouterProvider } from 'react-router-dom'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import CanvasListPage from './CanvasListPage'; +import { ApiError } from '../api/_client'; +import { exportCanvas } from '../api/canvas'; +import { toast } from '../components/Common/Toast'; import type { ReactNode } from 'react'; @@ -35,6 +38,8 @@ vi.mock('../api/canvas', () => ({ }), })); +vi.mock('../components/Common/Toast', () => ({ toast: vi.fn() })); + vi.mock('../components/Common/Modal', () => ({ Modal: () => null, })); @@ -129,6 +134,31 @@ async function renderPage() { } describe('CanvasListPage navigation', () => { + it('shows the export refusal without leaving the Spaces list or reporting success', async () => { + vi.mocked(exportCanvas).mockRejectedValueOnce( + new ApiError( + 400, + { + code: 'STORAGE_CAPABILITY_UNAVAILABLE', + message: 'Technical storage detail', + }, + 'Export failed', + ), + ); + const { router } = await renderPage(); + const button = container.querySelector( + 'button[aria-label="canvasList.exportCanvas"]', + ); + if (!button) throw new Error('Export control was not rendered'); + await act(async () => button.click()); + expect(router.state.location.pathname).toBe('/spaces'); + expect(toast).toHaveBeenCalledExactlyOnceWith( + 'canvasList.exportUnavailable', + { tone: 'danger' }, + ); + expect(button.disabled).toBe(false); + }); + it('renders each Space card as a link and uses in-tab routing for a plain click', async () => { const { link, router } = await renderPage(); diff --git a/apps/web/src/pages/CanvasListPage.tsx b/apps/web/src/pages/CanvasListPage.tsx index 86fa70978..46ea3bfe4 100644 --- a/apps/web/src/pages/CanvasListPage.tsx +++ b/apps/web/src/pages/CanvasListPage.tsx @@ -6,6 +6,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { Trans, useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; +import { ApiError } from '../api/_client'; import { listCanvases, exportCanvas, deleteCanvasById } from '../api/canvas'; import { Button } from '../components/Common/Button'; import { EmptyState } from '../components/Common/EmptyState'; @@ -84,7 +85,12 @@ export default function CanvasListPage() { toast(t('canvasList.exportStarted'), { tone: 'success' }); } catch (error) { toast( - error instanceof Error ? error.message : t('canvasList.exportFailed'), + error instanceof ApiError && + error.code === 'STORAGE_CAPABILITY_UNAVAILABLE' + ? t('canvasList.exportUnavailable') + : error instanceof Error + ? error.message + : t('canvasList.exportFailed'), { tone: 'danger', }, diff --git a/packages/shared/src/types/api/canvas.ts b/packages/shared/src/types/api/canvas.ts index 095663137..93fff48c7 100644 --- a/packages/shared/src/types/api/canvas.ts +++ b/packages/shared/src/types/api/canvas.ts @@ -327,6 +327,8 @@ export interface UpdateCanvasStateResult { * omitted, mirroring the pre-schema behaviour. */ export const exportCanvasQuerySchema = z.object({ + /** Validate export eligibility without building or downloading the archive. */ + check: z.enum(['true', 'false']).optional(), includeHistory: z.enum(['true', 'false']).optional(), }); export type ExportCanvasQuery = z.infer;