diff --git a/apps/docs/openapi-v2-files-audit.json b/apps/docs/openapi-v2-files-audit.json index 2ae1f1de435..494974f0965 100644 --- a/apps/docs/openapi-v2-files-audit.json +++ b/apps/docs/openapi-v2-files-audit.json @@ -3167,7 +3167,7 @@ "uploadedByEmail": { "type": "string", "format": "email", - "pattern": "^(?!\\.)(?!.*\\.\\.)([A-Za-z0-9_'+\\-\\.]*)[A-Za-z0-9_+-]@([A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$", + "pattern": "^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$", "description": "Current email address of the uploader.", "examples": ["jane@example.com"] }, @@ -4029,7 +4029,7 @@ "uploadedByEmail": { "type": "string", "format": "email", - "pattern": "^(?!\\.)(?!.*\\.\\.)([A-Za-z0-9_'+\\-\\.]*)[A-Za-z0-9_+-]@([A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$", + "pattern": "^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$", "description": "Current email address of the uploader.", "examples": ["jane@example.com"] }, diff --git a/apps/realtime/src/handlers/connection.test.ts b/apps/realtime/src/handlers/connection.test.ts new file mode 100644 index 00000000000..26018de95f0 --- /dev/null +++ b/apps/realtime/src/handlers/connection.test.ts @@ -0,0 +1,73 @@ +/** + * @vitest-environment node + */ +import { createServer, type Server as HttpServer } from 'node:http' +import type { AddressInfo } from 'node:net' +import { Server } from 'socket.io' +import { io as connect, type Socket } from 'socket.io-client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { setupConnectionHandlers, waitForConnectionCleanup } from '@/handlers/connection' +import type { AuthenticatedSocket } from '@/middleware/auth' +import { MemoryRoomManager } from '@/rooms' + +vi.mock('@/handlers/file-doc', () => ({ cleanupFileDocForSocket: vi.fn() })) +vi.mock('@/handlers/subblocks', () => ({ cleanupPendingSubblocksForSocket: vi.fn() })) +vi.mock('@/handlers/variables', () => ({ cleanupPendingVariablesForSocket: vi.fn() })) + +describe('server shutdown connection drain', () => { + let httpServer: HttpServer + let io: Server + let manager: MemoryRoomManager + let client: Socket + + beforeEach(async () => { + httpServer = createServer() + io = new Server(httpServer, { transports: ['websocket'] }) + manager = new MemoryRoomManager(io) + await manager.initialize() + io.on('connection', (socket) => setupConnectionHandlers(socket as AuthenticatedSocket, manager)) + await new Promise((resolve) => httpServer.listen(0, '127.0.0.1', resolve)) + const port = (httpServer.address() as AddressInfo).port + client = connect(`http://127.0.0.1:${port}`, { transports: ['websocket'], autoConnect: false }) + const connected = new Promise((resolve) => client.once('connect', resolve)) + client.connect() + await connected + }) + + afterEach(async () => { + client.disconnect() + await io.close() + await waitForConnectionCleanup() + await manager.shutdown() + vi.restoreAllMocks() + }) + + it('keeps automatic reconnection active after transport shutdown', async () => { + const disconnected = new Promise((resolve) => client.once('disconnect', resolve)) + await io.close() + expect(await disconnected).toBe('transport close') + expect(client.active).toBe(true) + await waitForConnectionCleanup() + }) + + it('waits for asynchronous presence cleanup before releasing its dependencies', async () => { + let finishRemoval: (() => void) | undefined + vi.spyOn(manager, 'removeSocketFromAllRooms').mockImplementation( + () => + new Promise((resolve) => { + finishRemoval = () => resolve([]) + }) + ) + await io.close() + let drained = false + const drain = waitForConnectionCleanup().then(() => { + drained = true + }) + await Promise.resolve() + expect(drained).toBe(false) + expect(finishRemoval).toBeDefined() + finishRemoval?.() + await drain + expect(drained).toBe(true) + }) +}) diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index 33d90b5bfb0..fcbdaade40f 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -16,6 +16,13 @@ const logger = createLogger('ConnectionHandlers') */ const PRESENCE_BEARING_TYPES = new Set([ROOM_TYPES.WORKFLOW, ROOM_TYPES.TABLE]) +const pendingDisconnects = new Set>() + +/** Keep Redis available until disconnect listeners finish removing presence. */ +export async function waitForConnectionCleanup(): Promise { + await Promise.all(pendingDisconnects) +} + export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { socket.on('error', (error) => { logger.error(`Socket ${socket.id} error:`, error) @@ -28,7 +35,7 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager // `disconnecting` (not `disconnect`): here `socket.rooms` is still populated and // authoritative, so presence is cleaned up even if the Redis room-set key was // evicted or TTL-expired (which would leave the manager's stored rooms empty). - socket.on('disconnecting', async (reason) => { + const handleDisconnect = async (reason: string) => { try { // Snapshot the live Socket.IO room membership SYNCHRONOUSLY, before any // await: Socket.IO clears `socket.rooms` via leaveAll() as soon as the @@ -91,5 +98,11 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager } catch (error) { logger.error(`Error handling disconnect for socket ${socket.id}:`, error) } + } + + socket.on('disconnecting', (reason) => { + const cleanup = handleDisconnect(reason) + pendingDisconnects.add(cleanup) + void cleanup.finally(() => pendingDisconnects.delete(cleanup)) }) } diff --git a/apps/realtime/src/handlers/file-doc-app.ts b/apps/realtime/src/handlers/file-doc-app.ts index 87e515954be..072d94f20d6 100644 --- a/apps/realtime/src/handlers/file-doc-app.ts +++ b/apps/realtime/src/handlers/file-doc-app.ts @@ -18,10 +18,8 @@ function postToApp(path: string, payload: unknown, timeoutMs: number): Promise +} + +interface TestRedisClient { + isOpen: boolean + connect(): Promise + quit(): Promise + on(): TestRedisClient + duplicate(): TestRedisClient + xAdd(key: string, id: string, fields: Record): Promise + xRange( + key: string, + start: string, + end: string, + options?: { COUNT?: number } + ): Promise + xRevRange( + key: string, + start: string, + end: string, + options?: { COUNT?: number } + ): Promise + xLen(key: string): Promise + xTrim(key: string, strategy: string, minId: string): Promise + xRead( + streams: { key: string; id: string }[], + options?: { BLOCK?: number; COUNT?: number } + ): Promise<{ name: string; messages: TestStreamEntry[] }[] | null> + set(key: string, value: string, options?: { NX?: boolean }): Promise + get(key: string): Promise + del(keys: string | string[]): Promise + eval( + script: string, + options: { keys: string[]; arguments: string[] } + ): Promise + expire(): Promise +} + /** * One shared in-memory Redis backing per test, so several {@link FileDocStore} instances (modelling * several ECS tasks) all talk to the "same Redis". A minimal fake of just the stream/lock ops the * store uses. */ interface Backing { - streams: Map }[]> + streams: Map kv: Map + dedupe: Map seq: number + /** Override generated IDs to model a recreated stream restarting its same-millisecond sequence. */ + nextIds?: string[] /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ failXAdd: number /** Set to fail every xRead the way node-redis does once a client has been closed. */ @@ -26,18 +70,34 @@ interface Backing { idleReads: number /** `connect()` calls, so a test can prove a closed reader is re-opened rather than abandoned. */ connects: number + /** Largest stream range response requested, proving replay is paginated. */ + maxRangeCount: number + /** Optional deterministic compaction hook invoked before each range page is read. */ + onRange?: (call: number, key: string, start: string) => void + rangeCalls: number + /** Largest multiplexed XREAD request and COUNT observed. */ + maxReadStreams: number + maxReadCount: number + onSnapshot?: () => Promise + failSnapshotTrim?: boolean + onLength?: () => Promise } const state = vi.hoisted(() => ({ backing: null as Backing | null })) -const seqOf = (id: string) => Number(id.split('-')[0]) +function compareStreamIds(left: string, right: string): bigint { + const [leftMs, leftSequence] = left.split('-').map(BigInt) + const [rightMs, rightSequence] = right.split('-').map(BigInt) + return leftMs === rightMs ? leftSequence - rightSequence : leftMs - rightMs +} -function makeClient(): any { +function makeClient(): TestRedisClient { const b = () => { if (!state.backing) throw new Error('backing not initialized') return state.backing } - const client: any = { + const nextId = () => b().nextIds?.shift() ?? `${++b().seq}-0` + const client: TestRedisClient = { isOpen: true, connect: async () => { client.isOpen = true @@ -51,32 +111,58 @@ function makeClient(): any { b().failXAdd-- throw new Error('transient xAdd failure') } - const id = `${++b().seq}-0` + const id = nextId() const arr = b().streams.get(key) ?? [] arr.push({ id, message: { ...fields } }) b().streams.set(key, arr) return id }, - xRange: async (key: string) => (b().streams.get(key) ?? []).map((e) => ({ ...e })), - xLen: async (key: string) => (b().streams.get(key) ?? []).length, + xRange: async (key: string, start: string, end: string, options?: { COUNT?: number }) => { + b().rangeCalls++ + b().onRange?.(b().rangeCalls, key, start) + const startId = start.startsWith('(') ? start.slice(1) : start + const entries = (b().streams.get(key) ?? []).filter( + (entry) => + (start === '-' || compareStreamIds(entry.id, startId) > 0n) && + (end === '+' || compareStreamIds(entry.id, end) <= 0n) + ) + const count = options?.COUNT ?? entries.length + b().maxRangeCount = Math.max(b().maxRangeCount, count) + return entries.slice(0, count).map((entry) => ({ ...entry })) + }, + xRevRange: async (key: string, _start: string, _end: string, options?: { COUNT?: number }) => + [...(b().streams.get(key) ?? [])] + .reverse() + .slice(0, options?.COUNT) + .map((entry) => ({ ...entry })), + xLen: async (key: string) => { + await b().onLength?.() + return (b().streams.get(key) ?? []).length + }, xTrim: async (key: string, _strategy: string, minid: string) => { const arr = b().streams.get(key) ?? [] b().streams.set( key, - arr.filter((e) => seqOf(e.id) >= seqOf(minid)) + arr.filter((e) => compareStreamIds(e.id, minid) >= 0n) ) }, - xRead: async (streams: { key: string; id: string }[]) => { + xRead: async ( + streams: { key: string; id: string }[], + options?: { BLOCK?: number; COUNT?: number } + ) => { b().reads++ + b().maxReadStreams = Math.max(b().maxReadStreams, streams.length) + b().maxReadCount = Math.max(b().maxReadCount, options?.COUNT ?? 0) if (b().readerClosed) { b().failedReadTimes.push(Date.now()) client.isOpen = false throw new Error('The client is closed') } - const res: { name: string; messages: { id: string; message: Record }[] }[] = - [] + const res: { name: string; messages: TestStreamEntry[] }[] = [] for (const { key, id } of streams) { - const after = (b().streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + const after = (b().streams.get(key) ?? []) + .filter((e) => compareStreamIds(e.id, id) > 0n) + .slice(0, options?.COUNT) if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) } if (res.length) { @@ -92,22 +178,129 @@ function makeClient(): any { b().kv.set(key, val) return 'OK' }, - del: async (key: string) => { - b().kv.delete(key) - return 1 + get: async (key: string) => b().kv.get(key) ?? null, + del: async (keys: string | string[]) => { + const targets = Array.isArray(keys) ? keys : [keys] + for (const key of targets) { + b().kv.delete(key) + b().streams.delete(key) + b().dedupe.delete(key) + } + return targets.length }, eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { const [key] = opts.keys + if (script.startsWith('for _, key in ipairs(KEYS)')) return 1 + if (script.includes("redis.call('exists', KEYS[1])") && !b().streams.has(key)) { + return script.includes('zscore') ? -1 : false + } + if (script.includes('return ARGV[1]')) { + const generation = b().kv.get(opts.keys[1]) + if (generation !== undefined) return generation + if (!b().streams.get(key)?.length) return false + b().kv.set(opts.keys[1], opts.arguments[0]) + return opts.arguments[0] + } + if (script.includes("redis.call('del', KEYS[1], KEYS[4], KEYS[5])")) { + const [, generationKey, versionKey, dedupeKey, agentKey, invalidationKey] = opts.keys + const [version, , marker] = opts.arguments + const current = b().kv.get(versionKey) + const invalidated = b().kv.get(invalidationKey) + if (invalidated && Number(invalidated) >= Number(version)) return null + if (current && Number(current) > Number(version)) return null + if (current === version && b().kv.get(generationKey) === marker) return null + const generation = b().kv.get(generationKey) ?? '' + b().kv.set(generationKey, marker) + b().kv.set(versionKey, version) + b().kv.set(invalidationKey, version) + b().streams.delete(key) + b().dedupe.delete(dedupeKey) + b().kv.delete(agentKey) + return generation + } + if (script.includes('zscore')) { + const [, dedupeKey, generationKey] = opts.keys + const [member, field, value, capacityText, , expectedGeneration] = opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return -1 + const members = b().dedupe.get(dedupeKey) ?? [] + if (members.includes(member)) return 0 + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value } }) + b().streams.set(key, arr) + members.push(member) + const capacity = Number(capacityText) + if (members.length > capacity) members.splice(0, members.length - capacity) + b().dedupe.set(dedupeKey, members) + return 1 + } // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both // append (the second sees a non-empty stream). if (script.includes('xlen')) { - const [field, value] = opts.arguments + const [, generationKey, versionKey] = opts.keys + const [field, value, generation, , generationField, version] = opts.arguments + if (Number(b().kv.get(versionKey) ?? 0) > Number(version)) return 0 const arr = b().streams.get(key) ?? [] if (arr.length > 0) return 0 - const id = `${++b().seq}-0` - arr.push({ id, message: { [field]: value } }) + b().kv.set(generationKey, generation) + if (version !== '0') b().kv.set(versionKey, version) + const id = nextId() + arr.push({ id, message: { [field]: value, [generationField]: generation } }) + b().streams.set(key, arr) + return 1 + } + if (script.includes('ARGV[5], ARGV[4]')) { + const [, generationKey] = opts.keys + const [field, value, marker, expectedGeneration, generationField, upTo, compactionField] = + opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return false + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ + id, + message: { + [field]: value, + [marker]: '1', + [generationField]: expectedGeneration, + ...(compactionField ? { [compactionField]: '1' } : {}), + }, + }) + b().streams.set(key, arr) + if (script.includes("redis.call('xtrim'")) { + if (b().failSnapshotTrim) throw new Error('snapshot trim failed') + b().streams.set( + key, + arr.filter((entry) => compareStreamIds(entry.id, upTo) >= 0n) + ) + } + await b().onSnapshot?.() + return id + } + if (script.includes("ARGV[3] ~= ''")) { + const [, generationKey] = opts.keys + const generation = b().kv.get(generationKey) + const expectedGeneration = opts.arguments[3] + if ((generation ?? '') !== expectedGeneration) return false + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } + const [field, value, marker] = opts.arguments + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value, ...(marker ? { [marker]: '1' } : {}) } }) b().streams.set(key, arr) + return id + } + if (script.includes('tonumber(c)')) { + const [value, , expectedGeneration] = opts.arguments + const generation = b().kv.get(opts.keys[1]) + if ((generation ?? '') !== expectedGeneration) return 0 + const current = b().kv.get(key) + if (current === undefined || Number(current) < Number(value)) b().kv.set(key, value) return 1 } // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. @@ -128,33 +321,37 @@ vi.mock('redis', () => ({ createClient: () => makeClient() })) import { FileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN } from '@/handlers/file-doc-store' const REDIS_URL = 'redis://fake' +const NAME = 'workspace-file-doc:file-1' -interface StoreRoomInternals { +interface StoreRoomTestAccess { + doc: Y.Doc lastId: string - pendingDeltas: Map - realEdited: boolean publishes: number + uncompactedDeltaBytes: number + lastDeltaBytes: number compactRetryAfter: number - doc: Y.Doc + compacting: boolean seededObserved: boolean + realEdited: boolean } -interface FileDocStoreInternals { - rooms: Map - applyEntry(room: StoreRoomInternals, id: string, message: Record): void - appendUpdate(name: string, update: Uint8Array, agent?: boolean): Promise - write: { xTrim: (...args: unknown[]) => Promise } - maybeCompact(name: string, force?: boolean): Promise +interface StoreTestAccess { + localInvalidations: Map + rooms: Map + maybeCompact(name: string): Promise + appendUpdate(name: string, update: Uint8Array): Promise + applyEntry( + name: string, + room: StoreRoomTestAccess, + id: string, + message: Record + ): void } -/** Reaches the private state these tests assert on, without `any`. */ -function internals(store: object): FileDocStoreInternals { - return store as unknown as FileDocStoreInternals +function storeInternals(store: FileDocStore): StoreTestAccess { + return store as unknown as StoreTestAccess } -const COMPACT_THRESHOLD_ENTRIES = 400 -const NAME = 'workspace-file-doc:file-1' - function docWithText(text: string): Y.Doc { const doc = new Y.Doc() doc.getText('body').insert(0, text) @@ -169,7 +366,28 @@ function updateFor(text: string): Uint8Array { return update } +function seedFor(text: string): Uint8Array { + const doc = docWithText(text) + const config = doc.getMap(FILE_DOC_SEED.configMap) + config.set(FILE_DOC_SEED.flag, true) + config.set(FILE_DOC_SEED.docIdKey, `doc-${text}`) + try { + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } +} + let stores: FileDocStore[] = [] + +/** An existing stream from a relay predating generation markers; modern seeds use seedIfEmpty. */ +function seedLegacyStream(update = updateFor('')): void { + const backing = state.backing! + backing.streams.set(`filedoc:stream:${NAME}`, [ + { id: `${++backing.seq}-0`, message: { u: Buffer.from(update).toString('base64') } }, + ]) +} + async function newStore(): Promise { const store = new FileDocStore(REDIS_URL) await store.init() @@ -182,6 +400,7 @@ describe('FileDocStore', () => { state.backing = { streams: new Map(), kv: new Map(), + dedupe: new Map(), seq: 0, failXAdd: 0, readerClosed: false, @@ -189,6 +408,10 @@ describe('FileDocStore', () => { failedReadTimes: [], idleReads: 0, connects: 0, + maxRangeCount: 0, + rangeCalls: 0, + maxReadStreams: 0, + maxReadCount: 0, } stores = [] }) @@ -289,7 +512,7 @@ describe('FileDocStore', () => { const token = await a.shouldSeed(NAME) expect(token).toBeTruthy() // A seeds and releases its lock. - a.publish(NAME, updateFor('hello')) + await a.seedIfEmpty(NAME, seedFor('hello')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) await a.releaseSeedLock(NAME, token as string) // A different task must NOT seed again — the lock is free but the stream is non-empty. @@ -297,9 +520,31 @@ describe('FileDocStore', () => { expect(await b.shouldSeed(NAME)).toBeNull() }) + it('fences stale publishers after invalidation and lets the next authoritative seed start fresh', async () => { + const store = await newStore() + const original = seedFor('old generation') + await store.seedIfEmpty(NAME, original) + await store.invalidateDocument(NAME, 10) + + await expect(store.getStreamState(NAME)).resolves.toBeNull() + await expect(store.publishAndWait(NAME, updateFor('stale write'))).rejects.toThrow( + 'replaced by a newer durable version' + ) + await expect( + store.publishClientUpdateAndWait(NAME, 'stale-update', updateFor('stale acknowledged write')) + ).rejects.toThrow('replaced by a newer durable version') + + const fresh = seedFor('fresh generation') + await expect(store.seedIfEmpty(NAME, fresh, 11)).resolves.toBe(true) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('fresh generation') + recovered.destroy() + }) + it('getStreamState reconstructs the shared document from the stream', async () => { const a = await newStore() - a.publish(NAME, updateFor('shared content')) + await a.seedIfEmpty(NAME, seedFor('shared content')) let state: Uint8Array | null = null await vi.waitFor(async () => { state = await a.getStreamState(NAME) @@ -311,9 +556,403 @@ describe('FileDocStore', () => { doc.destroy() }) + it('lets a headless replica append against the generation of its shared base', async () => { + const seeded = await newStore() + await seeded.seedIfEmpty(NAME, seedFor('shared'), 20) + const headless = await newStore() + const generation = await headless.getDocumentGeneration(NAME) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await headless.getStreamState(NAME, generation))!) + const before = Y.encodeStateVector(doc) + doc.getText('body').insert(6, ' edit') + await headless.publishAndWait(NAME, Y.encodeStateAsUpdate(doc, before), generation) + const replay = new Y.Doc() + Y.applyUpdate(replay, (await seeded.getStreamState(NAME))!) + expect(replay.getText('body').toString()).toBe('shared edit') + doc.destroy() + replay.destroy() + }) + + it('keeps a newer seeded generation when an older invalidation arrives', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('newest'), 20) + const generation = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 10)).resolves.toEqual({ status: 'stale' }) + expect(await store.getDocumentGeneration(NAME)).toBe(generation) + expect(await store.getSyncedVersion(NAME)).toBe(20) + await expect(store.getStreamState(NAME)).resolves.not.toBeNull() + }) + + it('rejects old seeds and version callbacks after an invalidation', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('old'), 10) + const generation = await store.getDocumentGeneration(NAME) + await store.invalidateDocument(NAME, 20) + await expect(store.seedIfEmpty(NAME, seedFor('late stale seed'), 10)).resolves.toBe(false) + await store.setSyncedVersion(NAME, 30, generation) + expect(await store.getSyncedVersion(NAME)).toBe(20) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('does not repeat an invalidation after the same durable version is reseeded', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('old'), 10) + await expect(store.invalidateDocument(NAME, 20)).resolves.toMatchObject({ status: 'applied' }) + await expect(store.seedIfEmpty(NAME, seedFor('replacement'), 20)).resolves.toBe(true) + const generation = await store.getDocumentGeneration(NAME) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await store.getStreamState(NAME))!) + const before = Y.encodeStateVector(doc) + doc.getText('body').insert(11, ' accepted') + await store.publishClientUpdateAndWait( + NAME, + 'accepted-edit', + Y.encodeStateAsUpdate(doc, before), + generation + ) + + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + expect(await store.getDocumentGeneration(NAME)).toBe(generation) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('replacement accepted') + expect(state.backing!.dedupe.get(`filedoc:updates:${NAME}`)).toHaveLength(1) + doc.destroy() + recovered.destroy() + }) + + it('applies the first invalidation even when its durable version was already seeded', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('same content, changed eligibility'), 20) + const docId = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'applied', docId }) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('returns the removed generation and qualifies consecutive unsupported replacements', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('old'), 10) + const oldId = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ + status: 'applied', + docId: oldId, + }) + await expect(store.invalidateDocument(NAME, 30)).resolves.toEqual({ status: 'applied' }) + await store.seedIfEmpty(NAME, seedFor('replacement'), 30) + const replacementId = await store.getDocumentGeneration(NAME) + expect(replacementId).not.toBe(oldId) + await expect(store.invalidateDocument(NAME, 30)).resolves.toEqual({ status: 'stale' }) + await expect(store.invalidateDocument(NAME, 40)).resolves.toEqual({ + status: 'applied', + docId: replacementId, + }) + }) + + it('does not resurrect a tracked stream with a dependency-only update after Redis loses it', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('base'), 10) + const generation = await store.getDocumentGeneration(NAME) + state.backing!.streams.delete(`filedoc:stream:${NAME}`) + state.backing!.kv.delete(`filedoc:generation:${NAME}`) + await expect(store.publishAndWait(NAME, updateFor('stale'), generation)).rejects.toThrow( + 'replaced' + ) + await expect( + store.publishClientUpdateAndWait(NAME, 'lost-stream-update', updateFor('stale'), generation) + ).rejects.toThrow('replaced') + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('rejects appends and duplicate acknowledgements when only the stream is lost', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('base'), 10) + const generation = await store.getDocumentGeneration(NAME) + const delta = updateFor('edit') + await store.publishClientUpdateAndWait(NAME, 'accepted-update', delta, generation) + state.backing!.streams.delete(`filedoc:stream:${NAME}`) + + await expect(store.publishAndWait(NAME, delta, generation)).rejects.toThrow('replaced') + await expect( + store.publishClientUpdateAndWait(NAME, 'new-update', delta, generation) + ).rejects.toThrow('replaced') + await expect( + store.publishClientUpdateAndWait(NAME, 'accepted-update', delta, generation) + ).rejects.toThrow('replaced') + expect(state.backing!.streams.has(`filedoc:stream:${NAME}`)).toBe(false) + }) + + it('adopts the identity of a pre-upgrade stream before acknowledging its edits', async () => { + const store = await newStore() + const seed = new Y.Doc() + seed.getMap('config').set('initialContentLoaded', true) + seed.getMap('config').set('docId', 'legacy-document') + seed.getText('body').insert(0, 'legacy') + seedLegacyStream(Y.encodeStateAsUpdate(seed)) + const attached = new Y.Doc() + await store.attachRoom(NAME, attached) + expect(await store.getDocumentGeneration(NAME)).toBe('legacy-document') + const before = Y.encodeStateVector(seed) + seed.getText('body').insert(6, ' edit') + await expect( + store.publishClientUpdateAndWait( + NAME, + 'legacy-edit', + Y.encodeStateAsUpdate(seed, before), + 'legacy-document' + ) + ).resolves.toBeUndefined() + store.detachRoom(NAME) + seed.destroy() + attached.destroy() + }) + + it('rejects a shared replay if the document generation changes between pages', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('old generation'), 10) + state.backing!.onRange = () => { + state.backing!.kv.set(`filedoc:generation:${NAME}`, 'new generation') + } + await expect(store.getStreamState(NAME)).rejects.toThrow('replaced') + }) + + it.each([true, false])( + 'validates a modern snapshot following a legacy seed (same identity: %s)', + async (sameIdentity) => { + const store = await newStore() + const seed = new Y.Doc() + seed.getMap('config').set('initialContentLoaded', true) + seed.getMap('config').set('docId', 'legacy-document') + seed.getText('body').insert(0, 'legacy') + seedLegacyStream(Y.encodeStateAsUpdate(seed)) + const backing = state.backing! + backing.kv.set( + `filedoc:generation:${NAME}`, + sameIdentity ? 'legacy-document' : 'different-document' + ) + backing.streams.get(`filedoc:stream:${NAME}`)!.push({ + id: `${++backing.seq}-0`, + message: { + u: Buffer.from(Y.encodeStateAsUpdate(seed)).toString('base64'), + s: '1', + g: sameIdentity ? 'legacy-document' : 'different-document', + }, + }) + const attached = new Y.Doc() + if (sameIdentity) { + await store.attachRoom(NAME, attached) + const before = Y.encodeStateVector(seed) + seed.getText('body').insert(6, ' peer') + await store.publishClientUpdateAndWait( + NAME, + 'peer-edit', + Y.encodeStateAsUpdate(seed, before), + 'legacy-document' + ) + await store.catchUp(NAME) + expect(attached.getText('body').toString()).toBe('legacy peer') + store.detachRoom(NAME) + } else { + await expect(store.attachRoom(NAME, attached)).rejects.toThrow('replaced') + expect(storeInternals(store).rooms.has(NAME)).toBe(false) + } + seed.destroy() + attached.destroy() + } + ) + + it('replays stream history in bounded pages', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 40 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 40 + const store = await newStore() + + await expect(store.getStreamState(NAME)).resolves.not.toBeNull() + + expect(state.backing!.maxRangeCount).toBe(4) + }) + + it('fails safely when an uncompacted stream exceeds the replay entry budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 2_001 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 2_001 + const store = await newStore() + + await expect(store.getStreamState(NAME)).rejects.toThrow('replay exceeded its safety limit') + }) + + it('never exposes a partially replayed document when room attachment exceeds its budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 2_001 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 2_001 + const store = await newStore() + const doc = new Y.Doc() + + await expect(store.attachRoom(NAME, doc)).rejects.toThrow('replay exceeded its safety limit') + + expect(doc.getText('body').toString()).toBe('') + expect(storeInternals(store).rooms.has(NAME)).toBe(false) + doc.destroy() + }) + + it('recovers from compaction that trims unread pages during replay', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 8 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 8 + state.backing!.onRange = (call, key) => { + if (call !== 2 || key !== streamKey) return + state.backing!.streams.set(streamKey, [ + { + id: '9-0', + message: { u: Buffer.from(updateFor('compacted')).toString('base64'), s: '1' }, + }, + ]) + state.backing!.seq = 9 + state.backing!.onRange = undefined + } + const store = await newStore() + + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('compacted') + recovered.destroy() + }) + + it.each(['headless', 'attached'] as const)( + 'does not recount a replacement snapshot near the byte budget during %s replay', + async (mode) => { + const streamKey = `filedoc:stream:${NAME}` + const source = new Y.Doc() + source.getText('body').insert(0, 'x'.repeat(10 * 1024 * 1024)) + const initial = Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64') + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 8 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: index === 0 ? initial : noop }, + })) + ) + source.getText('body').insert(source.getText('body').length, ' joined') + const compacted = Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64') + state.backing!.seq = 8 + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(4-0') return + state.backing!.streams.set(streamKey, [{ id: '9-0', message: { u: compacted, s: '1' } }]) + state.backing!.seq = 9 + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + if (mode === 'attached') await store.attachRoom(NAME, recovered) + else Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe(source.getText('body').toString()) + } finally { + store.detachRoom(NAME) + recovered.destroy() + source.destroy() + } + } + ) + + it('does not recount retained entries when compaction meets the exact entry budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + const entries = Array.from({ length: 1_999 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 1_999 + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(1996-0') return + state.backing!.streams.set(streamKey, [ + ...entries.slice(1_996), + { + id: '2000-0', + message: { u: Buffer.from(updateFor('complete')).toString('base64'), s: '1' }, + }, + ]) + state.backing!.seq = 2_000 + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('complete') + } finally { + recovered.destroy() + } + }) + + it('reads the replacement snapshot when peer deltas cross the old replay tail', async () => { + const streamKey = `filedoc:stream:${NAME}` + const source = new Y.Doc() + const entries: Array<{ id: string; message: Record }> = [] + source.on('update', (update: Uint8Array) => { + entries.push({ + id: `${entries.length + 1}-0`, + message: { u: Buffer.from(update).toString('base64') }, + }) + }) + for (let i = 1; i <= 8; i++) + source.getText('body').insert(source.getText('body').length, String(i)) + const snapshot = Y.encodeStateAsUpdate(source) + state.backing!.streams.set(streamKey, entries.slice()) + for (let i = 9; i <= 11; i++) + source.getText('body').insert(source.getText('body').length, String(i)) + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(4-0') return + state.backing!.streams.set(streamKey, [ + ...entries.slice(7), + { id: '12-0', message: { u: Buffer.from(snapshot).toString('base64'), s: '1' } }, + ]) + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe(source.getText('body').toString()) + } finally { + source.destroy() + recovered.destroy() + } + }) + it('attachRoom catches a fresh task up to the current shared state', async () => { const a = await newStore() - a.publish(NAME, updateFor('already here')) + await a.seedIfEmpty(NAME, seedFor('already here')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) // A second task opens the same file: its doc must load the existing content, not start empty. @@ -325,6 +964,7 @@ describe('FileDocStore', () => { }) it('converges a peer task via the tailer after attach', async () => { + seedLegacyStream() const a = await newStore() const b = await newStore() const bDoc = new Y.Doc() @@ -363,16 +1003,18 @@ describe('FileDocStore', () => { const a = await newStore() // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the // two peer entries. Inject that lagging room directly (a real edit was integrated → realEdited). - internals(a).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + lastDeltaBytes: 0, compactRetryAfter: 0, - pendingDeltas: new Map(), + compacting: false, seededObserved: true, realEdited: true, }) - await internals(a).maybeCompact(NAME) + await storeInternals(a).maybeCompact(NAME) // A fresh catch-up must still reconstruct the peer content — compaction must not have trimmed 401/402. const doc = new Y.Doc() @@ -387,6 +1029,7 @@ describe('FileDocStore', () => { }) it('tags an agent-streamed frame so a peer tailer applies it as REDIS_AGENT_ORIGIN (never persisted)', async () => { + seedLegacyStream() const streamKey = `filedoc:stream:${NAME}` const a = await newStore() const b = await newStore() @@ -412,244 +1055,188 @@ describe('FileDocStore', () => { }) it('latches realEdited synchronously so a concurrent compaction can never mislabel a real edit', async () => { + seedLegacyStream() // The data-loss race: a real edit sits in room.doc synchronously, but if realEdited were set only // AFTER appendUpdate's awaits, a concurrent agent-triggered compaction could snapshot that content and // stamp it an agent (no-persist) frame — losing the edit. The latch must be set in the same tick. const a = await newStore() const doc = new Y.Doc() await a.attachRoom(NAME, doc) - const room = internals(a).rooms.get(NAME)! + const room = storeInternals(a).rooms.get(NAME)! expect(room.realEdited).toBe(false) // Kick off a real (non-agent) append but do NOT await it: realEdited must already be true before the // xAdd/expire awaits resolve, so any compaction racing on the awaits sees the real edit. - const pending = internals(a).appendUpdate(NAME, updateFor('real user edit')) + const pending = storeInternals(a).appendUpdate(NAME, updateFor('real user edit')) expect(room.realEdited).toBe(true) await pending doc.destroy() }) - it('compacts on appended bytes, before the entry threshold is anywhere near reached', async () => { - const streamKey = `filedoc:stream:${NAME}` - const a = await newStore() + it('compacts a burst of large edits below the entry threshold without losing content', async () => { + seedLegacyStream() + const store = await newStore() const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - - // A handful of large pastes: far below COMPACT_THRESHOLD entries, far above the byte ceiling. - // Before bytes were counted this stream held tens of megabytes and never compacted. - const updates: Uint8Array[] = [] - doc.on('update', (u: Uint8Array) => updates.push(u)) - for (let i = 0; i < 4; i++) { - doc.getText('body').insert(0, 'x'.repeat(3 * 1024 * 1024)) - } - for (const update of updates) { - await a.publishAndWait(NAME, update) + await store.attachRoom(NAME, doc) + const source = new Y.Doc() + for (let index = 0; index < 4; index++) { + const before = Y.encodeStateVector(source) + source.getText('body').insert(0, 'x'.repeat(3 * 1024 * 1024)) + await store.publishAndWait(NAME, Y.encodeStateAsUpdate(source, before)) } - await vi.waitFor( - () => { - const stream = state.backing!.streams.get(streamKey)! - expect(stream.length).toBeLessThan(COMPACT_THRESHOLD_ENTRIES) - expect(stream.some((entry) => entry.message.s === '1')).toBe(true) - }, - { timeout: 5000 } - ) - - // Compaction must be lossless: the whole document is still reconstructable from what remains. + await vi.waitFor(() => { + const stream = state.backing!.streams.get(`filedoc:stream:${NAME}`)! + expect(stream.length).toBeLessThan(400) + expect(stream.some((entry) => entry.message.c === '1')).toBe(true) + }) const rebuilt = new Y.Doc() - Y.applyUpdate(rebuilt, (await a.getStreamState(NAME))!) - expect(rebuilt.getText('body').length).toBe(4 * 3 * 1024 * 1024) + Y.applyUpdate(rebuilt, (await store.getStreamState(NAME))!) + expect(rebuilt.getText('body').length).toBe(12 * 1024 * 1024) + store.detachRoom(NAME) rebuilt.destroy() + source.destroy() doc.destroy() }) - it('does not re-compact on every publish once the document itself exceeds the byte ceiling', async () => { - const streamKey = `filedoc:stream:${NAME}` - const a = await newStore() + it('does not append a full snapshot per small edit after growing beyond the byte threshold', async () => { + seedLegacyStream() + const store = await newStore() const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - - const updates: Uint8Array[] = [] - doc.on('update', (u: Uint8Array) => updates.push(u)) - // Grow the document past the byte ceiling so its own snapshot exceeds it, then keep editing. - // Counting the snapshot as appended bytes would leave the threshold permanently breached and - // force a full snapshot append per keystroke — the amplification the threshold exists to stop. - doc.getText('body').insert(0, 'x'.repeat(12 * 1024 * 1024)) - for (let i = 0; i < 30; i++) doc.getText('body').insert(0, 'tiny') - for (const update of updates) { - await a.publishAndWait(NAME, update) + await store.attachRoom(NAME, doc) + const source = new Y.Doc() + for (let index = 0; index < 33; index++) { + const before = Y.encodeStateVector(source) + source.getText('body').insert(0, index < 3 ? 'x'.repeat(3 * 1024 * 1024) : 'tiny') + await store.publishAndWait(NAME, Y.encodeStateAsUpdate(source, before)) + await store.catchUp(NAME) } + const streamKey = `filedoc:stream:${NAME}` await vi.waitFor(() => { - const stream = state.backing!.streams.get(streamKey)! - expect(stream.some((entry) => entry.message.s === '1')).toBe(true) + expect(storeInternals(store).rooms.get(NAME)!.compacting).toBe(false) + expect(state.backing!.streams.get(streamKey)!.some((entry) => entry.message.c === '1')).toBe( + true + ) }) - - const snapshots = state - .backing!.streams.get(streamKey)! - .filter((entry) => entry.message.s === '1').length - expect(snapshots).toBeLessThanOrEqual(2) - + expect( + state.backing!.streams.get(streamKey)!.filter((entry) => entry.message.c === '1').length + ).toBeLessThanOrEqual(2) const rebuilt = new Y.Doc() - Y.applyUpdate(rebuilt, (await a.getStreamState(NAME))!) + Y.applyUpdate(rebuilt, (await store.getStreamState(NAME))!) + expect(rebuilt.getText('body').length).toBe(9 * 1024 * 1024 + 30 * 4) expect(rebuilt.getText('body').toString().startsWith('tiny')).toBe(true) - expect(rebuilt.getText('body').length).toBe(12 * 1024 * 1024 + 30 * 4) + store.detachRoom(NAME) + source.destroy() rebuilt.destroy() doc.destroy() }) - it('keeps the byte trigger armed when compaction fails', async () => { - const a = await newStore() - const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - const room = internals(a).rooms.get(NAME)! - room.pendingDeltas = new Map([['1-0', 9 * 1024 * 1024]]) - room.realEdited = true - - const write = internals(a).write - const original = write.xTrim.bind(write) - write.xTrim = async () => { - throw new Error('redis blip') + it.each([false, true])( + 'adopts and compacts an oversized legacy stream (agent: %s)', + async (agent) => { + const source = new Y.Doc() + const updates: Uint8Array[] = [] + source.on('update', (update: Uint8Array) => updates.push(update)) + source.getText('body').insert(0, 'x'.repeat(7 * 1024 * 1024)) + source.getText('body').insert(0, 'tail') + const streamKey = `filedoc:stream:${NAME}` + state.backing!.streams.set( + streamKey, + updates.map((update, index) => ({ + id: `${index + 1}-0`, + message: { u: Buffer.from(update).toString('base64'), ...(agent ? { a: '1' } : {}) }, + })) + ) + state.backing!.seq = updates.length + const store = await newStore() + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) + + await vi.waitFor(() => + expect( + state.backing!.streams.get(streamKey)!.some((entry) => entry.message.c === '1') + ).toBe(true) + ) + const rebuilt = new Y.Doc() + Y.applyUpdate(rebuilt, (await store.getStreamState(NAME))!) + expect(rebuilt.getText('body').length).toBe(7 * 1024 * 1024 + 4) + store.detachRoom(NAME) + rebuilt.destroy() + source.destroy() + doc.destroy() } - await internals(a).maybeCompact(NAME, true) - - // A failed fold must not disarm the trigger — otherwise the stream stays oversized until - // this task happens to append another full threshold's worth of deltas. - expect([...room.pendingDeltas]).toEqual([['1-0', 9 * 1024 * 1024]]) - - // But it must not retry immediately either: the snapshot XADD lands before the XTRIM, so a - // persistent trim failure would append a full-document snapshot on every attempt. - const snapshotsAfterFailure = state.backing!.streams.get(`filedoc:stream:${NAME}`)?.length ?? 0 - await internals(a).maybeCompact(NAME, true) - await internals(a).maybeCompact(NAME, true) - expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)?.length ?? 0).toBe( - snapshotsAfterFailure - ) + ) - write.xTrim = original - doc.destroy() - }) - - it('keeps counting deltas the trim retained because they sit past the fold boundary', async () => { - const a = await newStore() - const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - const room = internals(a).rooms.get(NAME)! - room.realEdited = true - // The tailer has integrated up to 5-0, so `MINID 5-0` retains both 5-0 (the boundary is - // INCLUSIVE) and 9-0. Their bytes are still in Redis, and dropping them would disarm the - // byte trigger while the stream kept growing. - room.lastId = '5-0' - room.pendingDeltas = new Map([ - ['3-0', 4 * 1024 * 1024], - ['5-0', 6 * 1024 * 1024], - ['9-0', 7 * 1024 * 1024], - ]) - - await internals(a).maybeCompact(NAME, true) - - expect([...room.pendingDeltas]).toEqual([ - ['5-0', 6 * 1024 * 1024], - ['9-0', 7 * 1024 * 1024], - ]) - doc.destroy() - }) - - it('adopts accounting for a stream it takes over, and folds it if already over the ceiling', async () => { - const streamKey = `filedoc:stream:${NAME}` - // A stream left behind by a previous task: two entries, so far under the entry threshold, and - // far over the byte ceiling. A fresh room starting from an empty ledger would never fold it, - // while its own heartbeat kept refreshing the TTL. - const seedDoc = new Y.Doc() - const updates: Uint8Array[] = [] - seedDoc.on('update', (u: Uint8Array) => updates.push(u)) - seedDoc.getText('body').insert(0, 'x'.repeat(9 * 1024 * 1024)) - seedDoc.getText('body').insert(0, 'tail') - state.backing!.streams.set( - streamKey, - updates.map((update, index) => ({ - id: `${index + 1}-0`, - message: { u: Buffer.from(update).toString('base64') }, - })) - ) - state.backing!.seq = updates.length - - const a = await newStore() + it('keeps failed compaction accounting armed without repeated snapshot appends', async () => { + seedLegacyStream() + const store = await newStore() const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - - // Either marker counts as a fold: this room only ever replayed entries, so it never observed - // a real edit and its snapshot is stamped as an agent frame (the no-persist guarantee). - await vi.waitFor(() => { - const stream = state.backing!.streams.get(streamKey)! - expect(stream.some((entry) => entry.message.s === '1' || entry.message.a === '1')).toBe(true) - }) - - // Lossless: the adopted content survives the fold it triggered. - const rebuilt = new Y.Doc() - Y.applyUpdate(rebuilt, (await a.getStreamState(NAME))!) - expect(rebuilt.getText('body').length).toBe(9 * 1024 * 1024 + 4) - rebuilt.destroy() + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + room.uncompactedDeltaBytes = 9 * 1024 * 1024 + state.backing!.failSnapshotTrim = true + + await storeInternals(store).maybeCompact(NAME) + + expect(room.uncompactedDeltaBytes).toBe(9 * 1024 * 1024) + expect(room.compactRetryAfter).toBeGreaterThan(Date.now()) + const entriesAfterFailure = state.backing!.streams.get(`filedoc:stream:${NAME}`)!.length + await storeInternals(store).maybeCompact(NAME) + await storeInternals(store).maybeCompact(NAME) + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(entriesAfterFailure) + + state.backing!.failSnapshotTrim = false + room.compactRetryAfter = 0 + await storeInternals(store).maybeCompact(NAME) + expect(room.uncompactedDeltaBytes).toBeLessThan(9 * 1024 * 1024) + store.detachRoom(NAME) doc.destroy() - seedDoc.destroy() }) - it('counts agent preview deltas, which share a marker with an agent-only snapshot', async () => { - const streamKey = `filedoc:stream:${NAME}` - // Agent preview frames are the LARGE ones — a copilot file edit re-serialising a document is - // what filled Redis. They carry the same marker as a fold of an agent-only stream, so keying - // exclusion on that marker would drop exactly the payloads this bound exists for. - const seedDoc = new Y.Doc() - const updates: Uint8Array[] = [] - seedDoc.on('update', (u: Uint8Array) => updates.push(u)) - seedDoc.getText('body').insert(0, 'x'.repeat(9 * 1024 * 1024)) - seedDoc.getText('body').insert(0, 'tail') - state.backing!.streams.set( - streamKey, - updates.map((update, index) => ({ - id: `${index + 1}-0`, - message: { u: Buffer.from(update).toString('base64'), a: '1' }, - })) - ) - state.backing!.seq = updates.length - - const a = await newStore() + it('retains the last delta without repeatedly folding an unreclaimable boundary', async () => { + seedLegacyStream() + const store = await newStore() const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - - await vi.waitFor(() => { - const stream = state.backing!.streams.get(streamKey)! - expect(stream.some((entry) => entry.message.c === '1')).toBe(true) - }) - - const rebuilt = new Y.Doc() - Y.applyUpdate(rebuilt, (await a.getStreamState(NAME))!) - expect(rebuilt.getText('body').length).toBe(9 * 1024 * 1024 + 4) - rebuilt.destroy() + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + room.uncompactedDeltaBytes = 9 * 1024 * 1024 + room.lastDeltaBytes = room.uncompactedDeltaBytes + await storeInternals(store).maybeCompact(NAME) + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(1) + expect(room.uncompactedDeltaBytes).toBe(9 * 1024 * 1024) + store.detachRoom(NAME) doc.destroy() - seedDoc.destroy() }) - it("never counts a fold's own output, so a large document cannot arm the trigger against itself", async () => { - const a = await newStore() + it('excludes older agent compaction output from delta bytes', async () => { + seedLegacyStream() + const store = await newStore() const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - const room = internals(a).rooms.get(NAME)! - - internals(a).applyEntry(room, '7-0', { u: 'x'.repeat(9 * 1024 * 1024), a: '1', c: '1' }) - - expect(room.pendingDeltas.has('7-0')).toBe(false) + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + const priorBytes = room.uncompactedDeltaBytes + const encoded = Buffer.from(updateFor('agent snapshot')).toString('base64') + storeInternals(store).applyEntry(NAME, room, '7-0', { u: encoded, a: '1', c: '1' }) + expect(room.uncompactedDeltaBytes).toBe(priorBytes) + expect(room.lastDeltaBytes).toBe(0) + expect(doc.getText('body').toString()).toBe('agent snapshot') + store.detachRoom(NAME) doc.destroy() }) - it('counts a delta published by a peer task, which this room only ever tails', async () => { - const a = await newStore() + it('counts peer deltas once using ordered replay without retaining an entry ledger', async () => { + seedLegacyStream() + const store = await newStore() const doc = new Y.Doc() - await a.attachRoom(NAME, doc) - const room = internals(a).rooms.get(NAME)! - - // Never published locally, so publish-side accounting would miss it entirely. - internals(a).applyEntry(room, '4-0', { u: 'x'.repeat(1024) }) - - expect(room.pendingDeltas.get('4-0')).toBe(1024) + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + const priorBytes = room.uncompactedDeltaBytes + const encoded = Buffer.from(updateFor('peer')).toString('base64') + storeInternals(store).applyEntry(NAME, room, '4-0', { u: encoded }) + storeInternals(store).applyEntry(NAME, room, '4-0', { u: encoded }) + storeInternals(store).applyEntry(NAME, room, '3-0', { u: encoded }) + expect(room.uncompactedDeltaBytes).toBe(priorBytes + encoded.length) + expect(room.lastDeltaBytes).toBe(encoded.length) + store.detachRoom(NAME) doc.destroy() }) @@ -666,16 +1253,18 @@ describe('FileDocStore', () => { state.backing!.seq = 400 const a = await newStore() - internals(a).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: agentDoc, lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + lastDeltaBytes: 0, compactRetryAfter: 0, - pendingDeltas: new Map(), + compacting: false, seededObserved: true, realEdited: false, }) - await internals(a).maybeCompact(NAME) + await storeInternals(a).maybeCompact(NAME) // The snapshot must carry the AGENT marker, NOT the snapshot marker, so a peer catch-up applies it as // REDIS_AGENT_ORIGIN and never marks the doc edited — the no-persist guarantee survives compaction. @@ -692,6 +1281,7 @@ describe('FileDocStore', () => { }) it('retries a transient append failure so the edit is not lost from the shared log', async () => { + seedLegacyStream() const a = await newStore() state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed a.publish(NAME, updateFor('resilient')) @@ -706,10 +1296,306 @@ describe('FileDocStore', () => { ) }) + it('deduplicates acknowledged client retries by update id', async () => { + seedLegacyStream() + const store = await newStore() + const update = updateFor('retry-safe') + + await store.publishClientUpdateAndWait(NAME, 'update-1', update) + await store.publishClientUpdateAndWait(NAME, 'update-1', update) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + }) + + it('does not drop different payloads that reuse an acknowledged update id', async () => { + seedLegacyStream() + const store = await newStore() + + await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('first')) + await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('second')) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + }) + + it('uses unambiguous acknowledged-update deduplication keys', async () => { + seedLegacyStream() + const store = await newStore() + + await store.publishClientUpdateAndWait(NAME, 'a', new Uint8Array([0, 98])) + await store.publishClientUpdateAndWait(NAME, 'a\0', new Uint8Array([98])) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + }) + + it('bounds acknowledged-update deduplication independently of stream traffic', async () => { + seedLegacyStream() + const store = await newStore() + const update = updateFor('bounded') + + for (let index = 0; index <= 16_384; index += 1) { + await store.publishClientUpdateAndWait(NAME, `update-${index}`, update) + } + + expect(state.backing!.dedupe.get(`filedoc:updates:${NAME}`)).toHaveLength(16_384) + }) + + it('limits every multiplexed read to four streams and one entry per stream', async () => { + const store = await newStore() + const docs = Array.from({ length: 9 }, () => new Y.Doc()) + await Promise.all(docs.map((doc, index) => store.attachRoom(`${NAME}-${index}`, doc))) + state.backing!.maxReadStreams = 0 + state.backing!.maxReadCount = 0 + const readsBefore = state.backing!.reads + + await vi.waitFor(() => expect(state.backing!.reads).toBeGreaterThan(readsBefore)) + + expect(state.backing!.maxReadStreams).toBeLessThanOrEqual(4) + expect(state.backing!.maxReadCount).toBe(1) + docs.forEach((doc, index) => { + store.detachRoom(`${NAME}-${index}`) + doc.destroy() + }) + }) + + it.each(['client', 'server'] as const)( + 'does not delay a durable %s append for pending compaction', + async (publisher) => { + const store = await newStore() + await store.seedIfEmpty(NAME, seedFor('base')) + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + room.publishes = 63 + let finishCompaction!: () => void + const compaction = new Promise((resolve) => { + finishCompaction = resolve + }) + state.backing!.onLength = vi.fn(() => compaction) + const publish = (id: string) => + publisher === 'client' + ? store.publishClientUpdateAndWait(NAME, id, updateFor(id), 'doc-base') + : store.publishAndWait(NAME, updateFor(id), 'doc-base') + let accepted = false + const pending = publish('first').then(() => { + accepted = true + }) + try { + await vi.waitFor(() => expect(accepted).toBe(true)) + expect(room.compacting).toBe(true) + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + room.publishes = 127 + await publish('second') + expect(state.backing!.onLength).toHaveBeenCalledOnce() + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + } finally { + finishCompaction() + await pending + await vi.waitFor(() => expect(room.compacting).toBe(false)) + store.detachRoom(NAME) + doc.destroy() + } + } + ) + + it('compacts on retained bytes before the entry-count threshold can exhaust replay', async () => { + const streamKey = `filedoc:stream:${NAME}` + const snapshot = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + state.backing!.streams.set(streamKey, [{ id: '1-0', message: { u: snapshot } }]) + state.backing!.seq = 1 + const store = await newStore() + storeInternals(store).rooms.set(NAME, { + doc: new Y.Doc(), + lastId: '1-0', + publishes: 0, + uncompactedDeltaBytes: 12 * 1024 * 1024, + lastDeltaBytes: 0, + compactRetryAfter: 0, + compacting: false, + seededObserved: true, + realEdited: true, + }) + + await storeInternals(store).maybeCompact(NAME) + + const stream = state.backing!.streams.get(streamKey)! + expect(stream).toHaveLength(2) + expect(stream.at(-1)?.message.s).toBe('1') + }) + + it('does not compact a large snapshot again while continuing to accept small edits', async () => { + const store = await newStore() + const source = docWithText('x'.repeat(9 * 1024 * 1024)) + source.getMap('config').set('initialContentLoaded', true) + source.getMap('config').set('docId', 'large-document') + const streamKey = `filedoc:stream:${NAME}` + state.backing!.kv.set(`filedoc:generation:${NAME}`, 'large-document') + state.backing!.streams.set(streamKey, [ + { + id: '1-0', + message: { + u: Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64'), + s: '1', + g: 'large-document', + }, + }, + ]) + state.backing!.seq = 1 + const loaded = new Y.Doc() + await store.attachRoom(NAME, loaded) + expect(storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes).toBe(0) + + let deltaBytes = 0 + for (let index = 0; index < 30; index++) { + const before = Y.encodeStateVector(source) + source.getText('body').insert(source.getText('body').length, 'y') + const update = Y.encodeStateAsUpdate(source, before) + deltaBytes += Buffer.from(update).toString('base64').length + await store.publishClientUpdateAndWait(NAME, `small-${index}`, update, 'large-document') + await store.catchUp(NAME) + } + expect(state.backing!.streams.get(streamKey)?.filter((entry) => entry.message.s)).toHaveLength( + 1 + ) + expect(storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes).toBe(deltaBytes) + expect(loaded.getText('body').length).toBe(9 * 1024 * 1024 + 30) + store.detachRoom(NAME) + source.destroy() + loaded.destroy() + }) + + it('preserves the inclusive barrier and delta bytes observed during compaction', async () => { + seedLegacyStream() + const store = await newStore() + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + room.uncompactedDeltaBytes = 12 * 1024 * 1024 + const retainedBarrierBytes = room.lastDeltaBytes + const lateUpdate = updateFor('concurrent edit') + state.backing!.onSnapshot = async () => { + await store.publishAndWait(NAME, lateUpdate) + await store.catchUp(NAME) + } + await storeInternals(store).maybeCompact(NAME) + expect(room.uncompactedDeltaBytes).toBe( + retainedBarrierBytes + Buffer.from(lateUpdate).toString('base64').length + ) + expect(doc.getText('body').toString()).toBe('concurrent edit') + store.detachRoom(NAME) + doc.destroy() + }) + + it('does not trim a replacement stream recreated in the same millisecond as its compaction barrier', async () => { + const store = await newStore() + const replacer = await newStore() + const oldDoc = docWithText('old') + oldDoc.getMap('config').set('docId', 'old-generation') + state.backing!.nextIds = ['1000-0', '1000-1', '1000-2'] + await store.seedIfEmpty(NAME, Y.encodeStateAsUpdate(oldDoc), 10) + await store.publishAndWait(NAME, updateFor('first edit'), 'old-generation') + await store.publishAndWait(NAME, updateFor('second edit'), 'old-generation') + await store.attachRoom(NAME, oldDoc) + storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes = 12 * 1024 * 1024 + + const freshDoc = docWithText('fresh') + freshDoc.getMap('config').set('docId', 'new-generation') + const freshSeed = Y.encodeStateAsUpdate(freshDoc) + const beforeEdit = Y.encodeStateVector(freshDoc) + freshDoc.getText('body').insert(5, ' accepted edit') + const freshEdit = Y.encodeStateAsUpdate(freshDoc, beforeEdit) + const streamKey = `filedoc:stream:${NAME}` + state.backing!.nextIds = ['1000-3', '1000-0', '1000-1'] + state.backing!.onSnapshot = async () => { + await replacer.invalidateDocument(NAME, 20) + await replacer.seedIfEmpty(NAME, freshSeed, 20) + await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation') + expect(state.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ + '1000-0', + '1000-1', + ]) + } + + await storeInternals(store).maybeCompact(NAME) + + expect(state.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ + '1000-0', + '1000-1', + ]) + await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation') + expect(state.backing!.streams.get(streamKey)).toHaveLength(2) + const persisted = await replacer.getStreamState(NAME) + expect(persisted).not.toBeNull() + const replayed = new Y.Doc() + Y.applyUpdate(replayed, persisted!) + expect(replayed.getText('body').toString()).toBe('fresh accepted edit') + store.detachRoom(NAME) + oldDoc.destroy() + freshDoc.destroy() + replayed.destroy() + }) + + it('expires idle single-replica invalidation watermarks', async () => { + vi.useFakeTimers() + const store = new FileDocStore(undefined) + try { + for (let index = 0; index < 100; index++) { + await store.invalidateDocument(`closed-${index}`, 10) + } + expect(storeInternals(store).localInvalidations.size).toBe(100) + await vi.advanceTimersByTimeAsync(660_000) + expect(storeInternals(store).localInvalidations.size).toBe(0) + } finally { + await store.shutdown() + vi.useRealTimers() + } + }) + + it('deduplicates single-replica invalidations across same-version seeds and room reopen', async () => { + const store = new FileDocStore(undefined) + const staleDoc = new Y.Doc() + await store.attachRoom(NAME, staleDoc) + await store.invalidateDocument(NAME, 20) + await expect(store.seedIfEmpty(NAME, seedFor('stale fetched seed'), 10)).resolves.toBe(false) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(false) + expect(storeInternals(store).localInvalidations.size).toBe(1) + await expect(store.seedIfEmpty(NAME, seedFor('same-version seed'), 20)).resolves.toBe(true) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(true) + + store.detachRoom(NAME) + expect(storeInternals(store).localInvalidations.size).toBe(1) + const freshDoc = new Y.Doc() + await store.attachRoom(NAME, freshDoc) + await expect(store.seedIfEmpty(NAME, seedFor('fresh authoritative seed'), 20)).resolves.toBe( + true + ) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(true) + await store.invalidateDocument(NAME, 40) + await store.shutdown() + expect(storeInternals(store).localInvalidations.size).toBe(0) + staleDoc.destroy() + freshDoc.destroy() + }) + + it('fails closed when a Redis-backed store has not initialized', async () => { + const store = new FileDocStore(REDIS_URL) + const doc = new Y.Doc() + + await expect(store.attachRoom(NAME, doc)).rejects.toThrow('not initialized') + await expect( + store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('x')) + ).rejects.toThrow('not initialized') + await expect(store.seedIfEmpty(NAME, seedFor('seed'))).rejects.toThrow('not initialized') + await expect(store.getStreamState(NAME)).rejects.toThrow('not initialized') + expect(await store.acquireMergeSlot(NAME, 1_000)).toBeNull() + doc.destroy() + }) + it('streamHasContent fences a seed apply against an already-seeded stream', async () => { const a = await newStore() expect(await a.streamHasContent(NAME)).toBe(false) - a.publish(NAME, updateFor('seeded')) + await a.seedIfEmpty(NAME, seedFor('seeded')) await vi.waitFor(async () => expect(await a.streamHasContent(NAME)).toBe(true)) }) @@ -742,12 +1628,60 @@ describe('FileDocStore', () => { doc.destroy() }) + it.each([ + { redis: true, docId: undefined }, + { redis: true, docId: '' }, + { redis: false, docId: undefined }, + { redis: false, docId: '' }, + ])('rejects an unnamed seed before publication (%j)', async ({ redis, docId }) => { + const store = redis ? await newStore() : new FileDocStore(undefined) + const doc = new Y.Doc() + const config = doc.getMap(FILE_DOC_SEED.configMap) + config.set(FILE_DOC_SEED.flag, true) + if (docId !== undefined) config.set(FILE_DOC_SEED.docIdKey, docId) + try { + await expect(store.seedIfEmpty(NAME, Y.encodeStateAsUpdate(doc), 1)).rejects.toThrow( + 'missing its accepted document identity' + ) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + } finally { + doc.destroy() + if (!redis) await store.shutdown() + } + }) + + it('uses the same accepted identity for the seed owner and a replaying peer', async () => { + const owner = await newStore() + const peer = await newStore() + const ownerDoc = new Y.Doc() + const peerDoc = new Y.Doc() + try { + await owner.attachRoom(NAME, ownerDoc) + const seed = seedFor('shared identity') + expect(await owner.seedIfEmpty(NAME, seed, 1)).toBe(true) + Y.applyUpdate(ownerDoc, seed) + await peer.attachRoom(NAME, peerDoc) + for (const [store, doc] of [ + [owner, ownerDoc], + [peer, peerDoc], + ] as const) { + const docId = doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + expect(docId).toBe('doc-shared identity') + expect(await store.getDocumentGeneration(NAME)).toBe(docId) + expect(await store.isDocumentGenerationCurrent(NAME, 'doc-shared identity')).toBe(true) + } + } finally { + ownerDoc.destroy() + peerDoc.destroy() + } + }) + it('seedIfEmpty writes the seed once and reports it, then refuses a non-empty stream', async () => { const a = await newStore() - expect(await a.seedIfEmpty(NAME, updateFor('first'))).toBe(true) + expect(await a.seedIfEmpty(NAME, seedFor('first'))).toBe(true) // A second seed attempt (any task) must be refused — the stream already holds content. const b = await newStore() - expect(await b.seedIfEmpty(NAME, updateFor('second'))).toBe(false) + expect(await b.seedIfEmpty(NAME, seedFor('second'))).toBe(false) const doc = new Y.Doc() Y.applyUpdate(doc, (await a.getStreamState(NAME))!) expect(doc.getText('body').toString()).toBe('first') @@ -769,8 +1703,8 @@ describe('FileDocStore', () => { expect(tokenB).toBeTruthy() // Both tasks now race to seed with distinct client ids. const [seededA, seededB] = await Promise.all([ - a.seedIfEmpty(NAME, updateFor('SEED-A')), - b.seedIfEmpty(NAME, updateFor('SEED-B')), + a.seedIfEmpty(NAME, seedFor('SEED-A')), + b.seedIfEmpty(NAME, seedFor('SEED-B')), ]) expect([seededA, seededB].filter(Boolean)).toHaveLength(1) // Exactly one seed is in the stream — the reconstructed text is a single seed, never a duplicated @@ -790,7 +1724,7 @@ describe('FileDocStore', () => { author.getText('body').insert(4, 'peer') const a = await newStore() - a.publish(NAME, updates[0]) // 'base' + seedLegacyStream(updates[0]) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) // Task B attaches; while its synchronous catch-up runs, task A publishes the second edit. The tailer @@ -834,25 +1768,29 @@ describe('FileDocStore', () => { const b = await newStore() const docA = new Y.Doc() Y.applyUpdate(docA, peerUpdates[0]) // A integrated up to 401 - internals(a).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: docA, lastId: '401-0', publishes: 0, + uncompactedDeltaBytes: 0, + lastDeltaBytes: 0, compactRetryAfter: 0, - pendingDeltas: new Map(), + compacting: false, seededObserved: true, realEdited: true, }) - internals(b).rooms.set(NAME, { + storeInternals(b).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + lastDeltaBytes: 0, compactRetryAfter: 0, - pendingDeltas: new Map(), + compacting: false, seededObserved: true, realEdited: true, }) - await Promise.all([internals(a).maybeCompact(NAME), internals(b).maybeCompact(NAME)]) + await Promise.all([storeInternals(a).maybeCompact(NAME), storeInternals(b).maybeCompact(NAME)]) const doc = new Y.Doc() Y.applyUpdate(doc, (await a.getStreamState(NAME))!) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 43332963feb..3fa761671ed 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -34,8 +34,10 @@ * * @module */ + +import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' -import { FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_LIMITS, FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' @@ -61,7 +63,38 @@ const RELEASE_LOCK_SCRIPT = * Returns 1 if THIS call wrote the seed, 0 if the stream already had content. */ const SEED_IF_EMPTY_SCRIPT = - "if redis.call('xlen', KEYS[1]) == 0 then redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]); return 1 else return 0 end" + "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[6]) then return 0 end; if redis.call('xlen', KEYS[1]) == 0 then redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[4]); if ARGV[6] ~= '0' then redis.call('set', KEYS[3], ARGV[6], 'EX', ARGV[4]) end; redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[5], ARGV[3]); redis.call('expire', KEYS[1], ARGV[4]); redis.call('expire', KEYS[4], ARGV[4]); return 1 else return 0 end" + +/** Orders a durable replacement with seeds and merges, and fences publishers in the same transaction. */ +const INVALIDATE_DOCUMENT_SCRIPT = + "local invalidated = redis.call('get', KEYS[6]); if invalidated and tonumber(invalidated) >= tonumber(ARGV[1]) then return false end; local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[1]) then return false end; local generation = redis.call('get', KEYS[2]) or ''; if version == ARGV[1] and generation == ARGV[3] then return false end; redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[2]); redis.call('set', KEYS[3], ARGV[1], 'EX', ARGV[2]); redis.call('set', KEYS[6], ARGV[1], 'EX', ARGV[2]); redis.call('del', KEYS[1], KEYS[4], KEYS[5]); return generation" + +/** Upgrades an existing pre-negotiation stream without ever resurrecting a missing stream. */ +const ADOPT_GENERATION_SCRIPT = + "local generation = redis.call('get', KEYS[2]); if generation then return generation end; if redis.call('xlen', KEYS[1]) == 0 then return false end; redis.call('set', KEYS[2], ARGV[1], 'EX', ARGV[2]); return ARGV[1]" + +/** Atomically fence XADD so stale rooms cannot recreate a replaced or expired stream. Returns false when fenced. */ +const APPEND_UPDATE_SCRIPT = + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; for _, key in ipairs(KEYS) do redis.call('expire', key, ARGV[5]) end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" + +/** Renew stream metadata atomically, so an invalidation watermark cannot expire ahead of its stream. */ +const REFRESH_DOCUMENT_TTLS_SCRIPT = + "for _, key in ipairs(KEYS) do redis.call('expire', key, ARGV[1]) end; return 1" + +/** + * Append and trim under one generation fence: invalidation can recreate the stream with lower IDs + * within the same millisecond, so a separate trim could delete the replacement's seed and edits. + * Carry the seed's generation forward and keep every entry at or beyond the captured prefix barrier. + */ +const APPEND_SNAPSHOT_SCRIPT = + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; local id = redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1', ARGV[5], ARGV[4], ARGV[7], '1'); redis.call('xtrim', KEYS[1], 'MINID', ARGV[6]); return id" + +/** + * Atomically deduplicate and append an acknowledged client update. Socket acknowledgements can be + * lost, so a retry with the same id must not inflate the stream or its compaction counters. + */ +const APPEND_CLIENT_UPDATE_SCRIPT = + "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] or redis.call('exists', KEYS[1]) == 0 then return -1 end; redis.call('expire', KEYS[4], ARGV[5]); if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" /** * Monotonic set of the synced-version token: overwrite ONLY when the new value is greater than the @@ -73,7 +106,7 @@ const SEED_IF_EMPTY_SCRIPT = * comfortably within a Lua double, so the numeric compare is exact. */ const SET_VERSION_IF_NEWER_SCRIPT = - "local c = redis.call('get', KEYS[1]); if c == false or tonumber(c) < tonumber(ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) else redis.call('expire', KEYS[1], ARGV[2]) end; return 1" + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[3] then return 0 end; local c = redis.call('get', KEYS[1]); if c == false or tonumber(c) < tonumber(ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) else redis.call('expire', KEYS[1], ARGV[2]) end; return 1" /** * The transaction origin the store stamps on updates it applies from the stream. The relay's @@ -103,6 +136,10 @@ export const REDIS_SNAPSHOT_ORIGIN = Symbol('file-doc-redis-snapshot') export const REDIS_AGENT_ORIGIN = Symbol('file-doc-redis-agent') const STREAM_PREFIX = 'filedoc:stream:' +const CLIENT_UPDATE_PREFIX = 'filedoc:updates:' +const GENERATION_PREFIX = 'filedoc:generation:' +/** Retries must remain idempotent after a seed replaces the generation tombstone. */ +const INVALIDATION_VERSION_PREFIX = 'filedoc:invalidatedver:' /** Cluster-wide "durable version the live doc is synced to" (the persist If-Match token). */ const SYNC_VERSION_PREFIX = 'filedoc:syncver:' const SEED_LOCK_PREFIX = 'filedoc:seedlock:' @@ -123,20 +160,11 @@ const SNAPSHOT_FIELD = 's' /** Marks a stream entry as an AGENT-STREAMED preview frame, so the tailer applies it with * {@link REDIS_AGENT_ORIGIN} (never marks the doc edited). Present only on agent-frame entries. */ const AGENT_FIELD = 'a' -/** - * Marks a stream entry as the OUTPUT of a compaction, for byte accounting only. - * - * {@link SNAPSHOT_FIELD} cannot serve this purpose: a fold of an agent-only stream is stamped - * {@link AGENT_FIELD} instead, so it is indistinguishable from an ordinary agent preview frame — - * and those are the large ones. Excluding both markers would drop preview deltas from accounting; - * excluding neither would count a snapshot as something a fold can reclaim, arming the trigger - * against its own output. A separate field settles it without touching origin selection, which - * must keep treating an agent-only fold as an agent frame to preserve the no-persist guarantee. - * - * Entries written before this field existed carry no marker and are counted as deltas. That - * over-arms by at most one fold, which then trims them. - */ +/** Distinguishes compacted agent snapshots from ordinary agent deltas, including older writers. */ const COMPACTION_FIELD = 'c' +/** Identifies a seed's document generation, allowing old rooms to reject every later update. */ +const GENERATION_FIELD = 'g' +const INVALIDATED_GENERATION = '__invalidated__' /** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it @@ -150,40 +178,26 @@ const READ_BLOCK_MS = 1_000 /** Idle poll cadence when NO room is open on this task, so a freshly-attached room is picked up fast * without busy-spinning an empty task. */ const IDLE_POLL_MS = 250 -/** Max entries drained per stream per read. */ -const READ_COUNT = 200 +/** Max entries drained per stream per read, bounding one Redis response even for maximum-size edits. */ +const READ_COUNT = 1 +/** Maximum streams passed to one XREAD, bounding response memory independently of open-room count. */ +const READ_STREAM_BATCH_SIZE = 4 +/** Replay streams incrementally instead of materializing their complete history in one response. */ +const REPLAY_PAGE_COUNT = 4 +/** Compaction normally holds a stream near 400 entries; fail safely if that invariant is badly broken. */ +const REPLAY_MAX_ENTRIES = 2_000 +/** Base64 bytes accepted during one replay, including a full snapshot plus a bounded edit backlog. */ +const REPLAY_MAX_ENCODED_BYTES = FILE_DOC_LIMITS.updateBytes * 6 +/** Compact before a handful of individually valid large updates can exhaust the replay byte budget. */ +const COMPACT_ENCODED_BYTES = 8 * 1024 * 1024 /** Compact a stream once it exceeds this many entries (snapshot + trim). */ const COMPACT_THRESHOLD = 400 -/** - * Compact a stream once its appended deltas exceed this many bytes, whichever comes first. - * - * The entry threshold alone bounds how many entries a stream holds and says nothing about - * how large each one is: one pasted block is a single entry carrying megabytes, so a stream - * can sit at a few dozen entries and hundreds of megabytes and never reach - * {@link COMPACT_THRESHOLD} before its TTL. Folding by bytes as well keeps a stream's cost - * proportional to its document rather than to the size of the edits that produced it. - * - * Compaction is the only safe way to shrink one of these streams: a task attaching later - * replays every entry to rebuild the doc, so dropping the oldest entries — what a native - * `MAXLEN` retention bound would do — loses edits outright. A snapshot folds them first. - * - * Measured over deltas appended since the last fold, never over the resulting snapshot, so a - * stream settles at roughly one document snapshot plus this much churn. - */ -const COMPACT_BYTES_THRESHOLD = 8 * 1024 * 1024 /** Check whether compaction is due only every Nth local publish, to avoid an XLEN per keystroke. */ const COMPACT_CHECK_EVERY = 64 /** Compaction critical section (snapshot + xAdd + xTrim) is fast; a generous TTL covers a slow Redis * round-trip without risking expiry mid-compact. Released via compare-and-delete regardless. */ const COMPACT_LOCK_TTL_MS = 10_000 -/** - * Quiet period after a failed fold before another may be forced. - * - * A failed fold deliberately leaves the trigger armed so the bytes are not forgotten, but the - * snapshot `XADD` lands before the `XTRIM` — so if the trim is what failed, retrying immediately - * appends another full-document snapshot each time, turning a Redis blip into exactly the write - * amplification the threshold exists to prevent. The entry-count path is unaffected. - */ +/** Avoid repeated snapshot appends when a failed compaction leaves the byte trigger armed. */ const COMPACT_RETRY_COOLDOWN_MS = 30_000 /** Retry a failed stream append this many times before giving up, so a transient Redis blip doesn't * silently drop an edit from the shared log (which no peer would then ever see). */ @@ -206,25 +220,42 @@ const RECONNECT_MAX_DELAY_MS = 3_000 const READER_RETRY_MAX_MS = 10_000 /** After the first failure of a streak, log one reader failure in this many. */ const READER_ERROR_LOG_EVERY = 20 +const CLIENT_UPDATE_DEDUPE_CAPACITY = 16_384 const streamKey = (name: string) => `${STREAM_PREFIX}${name}` +const generationKey = (name: string) => `${GENERATION_PREFIX}${name}` +const documentKeys = (name: string) => [ + streamKey(name), + generationKey(name), + `${SYNC_VERSION_PREFIX}${name}`, + `${INVALIDATION_VERSION_PREFIX}${name}`, +] -/** - * Unfolded delta bytes a compaction could actually reclaim right now. - * - * Only entries strictly before `room.lastId` count. A fold trims with `MINID upTo`, which is - * inclusive, so everything from `upTo` onward survives it; counting those would re-arm the trigger - * the moment a fold finished and force a full snapshot append per publish that reclaims nothing. - * They stay in `pendingDeltas` and start counting once the tailer has moved past them. - */ -function foldableDeltaBytes(room: StoreRoom): number { - let bytes = 0 - for (const [id, deltaBytes] of room.pendingDeltas) { - // Strictly before the boundary: MINID is inclusive, so the entry AT `lastId` survives the - // trim and folding cannot reclaim it. - if (isAfterStreamId(room.lastId, id)) bytes += deltaBytes - } - return bytes +export class FileDocInvalidatedError extends Error { + constructor() { + super('The live file document was replaced by a newer durable version') + this.name = 'FileDocInvalidatedError' + } +} + +function assertUpdateWithinLimit(update: Uint8Array): void { + if (update.byteLength === 0 || update.byteLength > FILE_DOC_LIMITS.updateBytes) { + throw new Error(`File document update is outside the ${FILE_DOC_LIMITS.updateBytes}-byte limit`) + } +} + +function generationOfSeed(update: Uint8Array): string { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, update) + const docId = doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (typeof docId !== 'string' || docId.length === 0) { + throw new Error('File document seed is missing its accepted document identity') + } + return docId + } finally { + doc.destroy() + } } /** @@ -276,25 +307,16 @@ interface StoreRoom { lastId: string /** Local publish count, to pace compaction checks. */ publishes: number - /** Epoch ms before which no forced fold is attempted, after one failed. */ + /** Non-snapshot bytes observed since this replica last compacted. */ + uncompactedDeltaBytes: number + /** MINID retains the last applied entry; its bytes cannot trigger a fold until the cursor advances. */ + lastDeltaBytes: number compactRetryAfter: number - /** - * Unfolded delta bytes in the shared stream, by entry id. - * - * Recorded in {@link FileDocStore.applyEntry}, so it covers EVERY entry this room's tailer - * observes — this task's own appends, a peer task's, and one published with no room attached - * anywhere. Accounting on publish instead would see only this task's writes. - * - * Keyed by id rather than summed, because a fold trims to `room.lastId` and retains anything - * from that boundary on. Those bytes are still in Redis, so dropping them would disarm the - * trigger while the stream kept growing; entries are removed only once an `XTRIM` provably - * removed them. - * - * Excludes what a fold produces (see {@link COMPACTION_FIELD}) — a snapshot is a function of - * document size rather than edit volume, and counting one would make a large document breach - * the threshold permanently. - */ - pendingDeltas: Map + compacting: boolean + /** Document generation read from the seed entry; every later append is fenced against it. */ + generation: string | null + /** A newer seed was observed; this old room must ignore all entries until the relay replaces it. */ + generationInvalidated: boolean /** Set once the doc has been observed seeded, so the seed transition itself is never mistaken for an * edit (mirrors the relay's `seededObserved`). */ seededObserved: boolean @@ -316,6 +338,7 @@ export class FileDocStore { /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ private read: RedisClientType | null = null private readonly rooms = new Map() + private readonly localInvalidations = new Map() private running = false private heartbeat: ReturnType | null = null @@ -339,7 +362,10 @@ export class FileDocStore { * connection it can rebuild is always worth rebuilding. */ reconnectStrategy: (retries: number) => - backoffWithJitter(retries + 1, null, { baseMs: 100, maxMs: RECONNECT_MAX_DELAY_MS }), + backoffWithJitter(retries + 1, null, { + baseMs: 100, + maxMs: RECONNECT_MAX_DELAY_MS, + }), }, } this.write = createClient(options) @@ -361,60 +387,78 @@ export class FileDocStore { await Promise.all([this.write?.quit().catch(() => {}), this.read?.quit().catch(() => {})]) this.write = null this.read = null + this.rooms.clear() + this.localInvalidations.clear() } /** * Register a locally-opened room and load the shared state into its doc ({@link catchUp}). A * brand-new file has an empty stream and loads nothing (it is seeded shortly after, via - * {@link shouldSeed}). No-op when disabled. + * {@link shouldSeed}). Single-replica rooms are tracked only for invalidation lifecycle. */ async attachRoom(name: string, doc: Y.Doc): Promise { - if (!this.enabled || !this.write) return + if (this.enabled && !this.write) throw new Error('FileDocStore is not initialized') // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — // the tailer resumes from `lastId`, which the catch-up advances. const room: StoreRoom = { doc, lastId: '0', publishes: 0, + uncompactedDeltaBytes: 0, + lastDeltaBytes: 0, compactRetryAfter: 0, - pendingDeltas: new Map(), + compacting: false, + generation: null, + generationInvalidated: false, seededObserved: false, realEdited: false, } this.rooms.set(name, room) - await this.catchUp(name) + if (!this.enabled) return + try { + await this.catchUp(name) + } catch (error) { + if (this.rooms.get(name) === room) this.rooms.delete(name) + throw error + } } /** - * PULL the shared state into a registered room: read the stream and apply every entry the doc has - * not integrated yet (origin {@link REDIS_ORIGIN}), advancing `lastId` so the tailer resumes exactly - * after it. This is the ONLY way a room loads shared state, so a caller that must not depend on the - * tailer's asynchronous push — the join, which may not serve a client a half-assembled document — - * can converge on demand. Idempotent and safe to call repeatedly; no-op when disabled or the room is - * not registered (a fast open→close detached it). Never throws. + * Completes shared replay before applying entries, so joins never receive a partial document. + * Repeated calls skip integrated entries; detached rooms and disabled stores are ignored. */ async catchUp(name: string): Promise { - if (!this.enabled || !this.write) return + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') const room = this.rooms.get(name) if (!room) return try { - const entries = await this.write.xRange(streamKey(name), '-', '+') - for (const entry of entries) { + const entries: Array<{ id: string; message: Record }> = [] + await this.replayEntries(name, room.lastId, (entry) => { // The room can be detached + its doc destroyed while the read is in flight (a fast // open→close); stop touching it the moment that happens. - if (this.rooms.get(name) !== room) return - // Applying a Yjs update twice is a no-op, but `applyEntry`'s bookkeeping is not: re-applying - // the SEED after `seededObserved` latched would count it as a post-seed edit and let a - // compaction snapshot claim content no user ever typed. Skip what this room already holds. - if (!isAfterStreamId(entry.id, room.lastId)) continue - this.applyEntry(room, entry.id, entry.message) + if (this.rooms.get(name) !== room) return false + entries.push(entry) + return true + }) + if (this.rooms.get(name) !== room) return + for (const entry of entries) this.applyEntry(name, room, entry.id, entry.message) + if (room.generationInvalidated) throw new FileDocInvalidatedError() + const docId = room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (room.generation === null && typeof docId === 'string') { + const adopted = await this.write.eval(ADOPT_GENERATION_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [docId, String(STREAM_TTL_SEC)], + }) + if (adopted !== docId) throw new FileDocInvalidatedError() + room.generation = docId } - await this.write.expire(streamKey(name), STREAM_TTL_SEC) - // A stream taken over may already be past the ceiling, and nothing else re-checks until the - // next local publish — which a read-only participant never makes. - if (foldableDeltaBytes(room) >= COMPACT_BYTES_THRESHOLD) void this.maybeCompact(name, true) + await this.refreshDocumentTtls(name) } catch (error) { - logger.warn(`FileDocStore catch-up failed for ${name}`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore catch-up failed for ${name}`, { + error: getErrorMessage(error), + }) + throw error } } @@ -426,11 +470,17 @@ export class FileDocStore { /** * Append a locally-applied update to the shared stream so every task converges, AWAITING the write * and retrying a transient failure ({@link PUBLISH_MAX_RETRIES}) so a Redis blip can't silently drop - * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are - * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. + * an edit from the shared log. The append and metadata TTL renewal are atomic; post-write + * compaction never re-triggers the append. Throws if the append ultimately fails. */ - private async appendUpdate(name: string, update: Uint8Array, agent = false): Promise { + private async appendUpdate( + name: string, + update: Uint8Array, + agent = false, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { if (!this.write) return + assertUpdateWithinLimit(update) // Latch realEdited SYNCHRONOUSLY — before the first await — for a real (non-agent) publish. The edit // already sits in room.doc (applied in doc.on('update') before publish was called), so if this set // were deferred past the xAdd/expire awaits a CONCURRENT agent-frame-triggered maybeCompact could read @@ -443,15 +493,21 @@ export class FileDocStore { if (editedRoom) editedRoom.realEdited = true } const encoded = Buffer.from(update).toString('base64') - const fields: Record = { [UPDATE_FIELD]: encoded } - if (agent) fields[AGENT_FIELD] = '1' + const marker = agent ? AGENT_FIELD : '' for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { - await this.write.xAdd(streamKey(name), '*', fields) + const id = await this.write.eval(APPEND_UPDATE_SCRIPT, { + keys: documentKeys(name), + arguments: [UPDATE_FIELD, encoded, marker, expectedGeneration, String(STREAM_TTL_SEC)], + }) + if (id === null || id === false) throw new FileDocInvalidatedError() break } catch (error) { + if (error instanceof FileDocInvalidatedError) throw error if (attempt === PUBLISH_MAX_RETRIES) { - logger.error(`FileDocStore append failed for ${name}`, { error: getErrorMessage(error) }) + logger.error(`FileDocStore append failed for ${name}`, { + error: getErrorMessage(error), + }) throw error } // Snappy backoff — a stream append is a fast op; a transient blip clears in tens of ms. @@ -459,15 +515,15 @@ export class FileDocStore { await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) } } - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) const room = this.rooms.get(name) - if (!room) return - // Bytes are checked every publish: one entry can cross the ceiling on its own, so pacing this - // check the way the entry count is paced would let a stream sit far over the ceiling for up to - // COMPACT_CHECK_EVERY more appends. The check itself is a local sum over unfolded entries. - const overBytes = foldableDeltaBytes(room) >= COMPACT_BYTES_THRESHOLD - if (overBytes || ++room.publishes % COMPACT_CHECK_EVERY === 0) { - void this.maybeCompact(name, overBytes) + if (room) { + room.publishes += 1 + if ( + room.uncompactedDeltaBytes - room.lastDeltaBytes >= COMPACT_ENCODED_BYTES || + room.publishes % COMPACT_CHECK_EVERY === 0 + ) { + void this.maybeCompact(name) + } } } @@ -478,7 +534,11 @@ export class FileDocStore { */ publish(name: string, update: Uint8Array, agent = false): void { if (!this.enabled || !this.write) return - void this.appendUpdate(name, update, agent).catch(() => {}) // already logged inside appendUpdate + void this.appendUpdate(name, update, agent).catch((error) => { + logger.warn(`FileDocStore rejected a non-durable legacy update for ${name}`, { + error: getErrorMessage(error), + }) + }) } /** @@ -486,9 +546,80 @@ export class FileDocStore { * — the copilot merge, so the cross-task merge lock is not released before the diff is committed * (else the next task would diff a stale base). Throws on ultimate failure. No-op when disabled. */ - async publishAndWait(name: string, update: Uint8Array): Promise { - if (!this.enabled || !this.write) return - await this.appendUpdate(name, update) + async publishAndWait( + name: string, + update: Uint8Array, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') + await this.appendUpdate(name, update, false, expectedGeneration) + } + + /** + * Waits for Redis acceptance before the relay acknowledges the client. Retries are deduplicated + * within the bounded window; clients retain their journal until the acknowledgement arrives. + */ + async publishClientUpdateAndWait( + name: string, + updateId: string, + update: Uint8Array, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') + assertUpdateWithinLimit(update) + const encoded = Buffer.from(update).toString('base64') + const dedupeMember = createHash('sha256') + .update(String(Buffer.byteLength(updateId))) + .update(':') + .update(updateId) + .update(update) + .digest('hex') + const room = this.rooms.get(name) + + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { + try { + const appended = await this.write.eval(APPEND_CLIENT_UPDATE_SCRIPT, { + keys: [ + streamKey(name), + `${CLIENT_UPDATE_PREFIX}${name}`, + generationKey(name), + `${INVALIDATION_VERSION_PREFIX}${name}`, + ], + arguments: [ + dedupeMember, + UPDATE_FIELD, + encoded, + String(CLIENT_UPDATE_DEDUPE_CAPACITY), + String(STREAM_TTL_SEC), + expectedGeneration, + ], + }) + if (appended === -1) throw new FileDocInvalidatedError() + if (appended === 1 && room) { + room.realEdited = true + room.publishes += 1 + if ( + room.uncompactedDeltaBytes - room.lastDeltaBytes >= COMPACT_ENCODED_BYTES || + room.publishes % COMPACT_CHECK_EVERY === 0 + ) { + void this.maybeCompact(name) + } + } + return + } catch (error) { + if (error instanceof FileDocInvalidatedError) throw error + if (attempt === PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore acknowledged append failed for ${name}`, { + updateId, + error: getErrorMessage(error), + }) + throw error + } + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } } /** @@ -501,20 +632,41 @@ export class FileDocStore { * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. Disabled → * true (single-replica: seed locally, no stream). */ - async seedIfEmpty(name: string, update: Uint8Array): Promise { - if (!this.enabled || !this.write) return true + async seedIfEmpty(name: string, update: Uint8Array, version = 0): Promise { + assertUpdateWithinLimit(update) + const generation = generationOfSeed(update) + if (!this.enabled) { + const invalidation = this.localInvalidations.get(name) + if (invalidation && invalidation.expiresAt > Date.now() && invalidation.version > version) + return false + const room = this.rooms.get(name) + if (room) room.generationInvalidated = false + return true + } + if (!this.write) throw new Error('FileDocStore is not initialized') const encoded = Buffer.from(update).toString('base64') for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { const wrote = await this.write.eval(SEED_IF_EMPTY_SCRIPT, { - keys: [streamKey(name)], - arguments: [UPDATE_FIELD, encoded], + keys: documentKeys(name), + arguments: [ + UPDATE_FIELD, + encoded, + generation, + String(STREAM_TTL_SEC), + GENERATION_FIELD, + String(version), + ], }) - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + await this.refreshDocumentTtls(name).catch(() => {}) + const room = this.rooms.get(name) + if (wrote === 1 && room) room.generation = generation return wrote === 1 } catch (error) { if (attempt === PUBLISH_MAX_RETRIES) { - logger.error(`FileDocStore seed failed for ${name}`, { error: getErrorMessage(error) }) + logger.error(`FileDocStore seed failed for ${name}`, { + error: getErrorMessage(error), + }) throw error } await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) @@ -523,6 +675,64 @@ export class FileDocStore { return false } + /** + * Fences an unsupported durable replacement before deleting its stream. The next authoritative + * seed replaces the tombstone. A separate version watermark deduplicates retries across reseeds; + * both expire with the stream TTL once the document is idle. + */ + async invalidateDocument( + name: string, + version: number + ): Promise<{ status: 'applied'; docId?: string } | { status: 'stale' }> { + if (!this.enabled) { + const now = Date.now() + const previous = this.localInvalidations.get(name) + if (previous && previous.expiresAt > now && previous.version >= version) + return { status: 'stale' } + this.localInvalidations.set(name, { version, expiresAt: now + STREAM_TTL_SEC * 1_000 }) + const room = this.rooms.get(name) + const docId = room?.generationInvalidated + ? undefined + : room?.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (room) room.generationInvalidated = true + if (!this.heartbeat) { + this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) + this.heartbeat.unref() + } + return { status: 'applied', ...(typeof docId === 'string' ? { docId } : {}) } + } + if (!this.write) throw new Error('FileDocStore is not initialized') + const generation = await this.write.eval(INVALIDATE_DOCUMENT_SCRIPT, { + keys: [ + streamKey(name), + generationKey(name), + `${SYNC_VERSION_PREFIX}${name}`, + `${CLIENT_UPDATE_PREFIX}${name}`, + `${AGENT_STREAM_PREFIX}${name}`, + `${INVALIDATION_VERSION_PREFIX}${name}`, + ], + arguments: [String(version), String(STREAM_TTL_SEC), INVALIDATED_GENERATION], + }) + if (typeof generation !== 'string') return { status: 'stale' } + return { + status: 'applied', + ...(generation && generation !== INVALIDATED_GENERATION ? { docId: generation } : {}), + } + } + + async getDocumentGeneration(name: string): Promise { + if (!this.enabled) return '' + if (!this.write) throw new Error('FileDocStore is not initialized') + return (await this.write.get(generationKey(name))) ?? '' + } + + async isDocumentGenerationCurrent(name: string, generation?: string): Promise { + if (!this.enabled) return !this.rooms.get(name)?.generationInvalidated + if (!this.write) throw new Error('FileDocStore is not initialized') + const current = await this.write.get(generationKey(name)) + return current === null ? !generation : current === generation + } + /** * Whether the file's stream already holds content — an EFFICIENCY recheck in {@link shouldSeed} that * skips the seed fetch when a prior holder already seeded (the split-brain guard itself is the atomic @@ -549,12 +759,15 @@ export class FileDocStore { * disabled store return a truthy token so callers proceed single-replica without special-casing. */ private async acquireLock(key: string, ttlMs: number): Promise { - if (!this.enabled || !this.write) return DISABLED_LOCK_TOKEN + if (!this.enabled) return DISABLED_LOCK_TOKEN + if (!this.write) return null const token = generateId() try { return (await this.write.set(key, token, { NX: true, PX: ttlMs })) === 'OK' ? token : null } catch (error) { - logger.warn(`FileDocStore lock ${key} failed`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore lock ${key} failed`, { + error: getErrorMessage(error), + }) return null } } @@ -593,19 +806,78 @@ export class FileDocStore { * `null` when the stream is empty — i.e. no doc is (or was recently) live, so there is nothing to * merge into and the caller should fall back to a direct file write. Disabled → always null. */ - async getStreamState(name: string): Promise { - if (!this.enabled || !this.write) return null - const entries = await this.write.xRange(streamKey(name), '-', '+') - if (entries.length === 0) return null + async getStreamState(name: string, expectedGeneration?: string): Promise { + if (!this.enabled) return null + if (!this.write) throw new Error('FileDocStore is not initialized') const doc = new Y.Doc() try { - for (const entry of entries) applyEntryToDoc(doc, entry.id, entry.message) + const generation = await this.getDocumentGeneration(name) + if (expectedGeneration !== undefined && generation !== expectedGeneration) { + throw new FileDocInvalidatedError() + } + const count = await this.replayEntries(name, '0', (entry) => { + if (entry.message[GENERATION_FIELD] && entry.message[GENERATION_FIELD] !== generation) { + throw new FileDocInvalidatedError() + } + applyEntryToDoc(doc, entry.id, entry.message) + return true + }) + if ((await this.getDocumentGeneration(name)) !== generation) { + throw new FileDocInvalidatedError() + } + if (count === 0) return null return Y.encodeStateAsUpdate(doc) } finally { doc.destroy() } } + private async replayEntries( + name: string, + afterId: string, + visit: (entry: { id: string; message: Record }) => boolean + ): Promise { + if (!this.write) return 0 + const key = streamKey(name) + let firstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id + if (!firstId) return 0 + const tail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) + if (tail.length === 0) throw new FileDocInvalidatedError() + let endId = tail[0].id + let cursor = afterId.includes('-') ? afterId : `${afterId}-0` + let entriesRead = 0 + let encodedBytes = 0 + + while (true) { + while (isAfterStreamId(endId, cursor)) { + const page = await this.write.xRange(key, `(${cursor}`, '+', { + COUNT: REPLAY_PAGE_COUNT, + }) + if (page.length === 0) { + throw new Error(`File document replay lost its completion barrier for ${name}`) + } + for (const entry of page) { + entriesRead += 1 + encodedBytes += entry.message[UPDATE_FIELD]?.length ?? 0 + if (entriesRead > REPLAY_MAX_ENTRIES || encodedBytes > REPLAY_MAX_ENCODED_BYTES) { + throw new Error(`File document replay exceeded its safety limit for ${name}`) + } + cursor = entry.id + if (!visit(entry)) return entriesRead + } + } + + /** Compaction appends its snapshot before trimming; extend the barrier without rereading it. */ + const currentFirstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id + if (!currentFirstId) throw new FileDocInvalidatedError() + if (currentFirstId === firstId) return entriesRead + firstId = currentFirstId + const currentTail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) + if (currentTail.length === 0) throw new FileDocInvalidatedError() + endId = currentTail[0].id + } + } + /** Release the seed lock (compare-and-delete) once the seed has been published or a seed attempt failed. */ async releaseSeedLock(name: string, token: string): Promise { await this.releaseLock(`${SEED_LOCK_PREFIX}${name}`, token) @@ -669,7 +941,11 @@ export class FileDocStore { * new value exceeds the stored one ({@link SET_VERSION_IF_NEWER_SCRIPT}), so an out-of-order * fire-and-forget write can't regress the token. Best-effort; TTL-bounded like the stream so an idle * file's key can't outlive its room. No-op when disabled (single-pod fallback). */ - async setSyncedVersion(name: string, version: number): Promise { + async setSyncedVersion( + name: string, + version: number, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { if (!this.enabled || !this.write) return // Retry a transient failure (bounded) rather than swallow it: this token is the ONLY way a // peer-seeded task learns the durable version, so a dropped write would leave that peer's persists @@ -678,8 +954,8 @@ export class FileDocStore { for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { await this.write.eval(SET_VERSION_IF_NEWER_SCRIPT, { - keys: [`${SYNC_VERSION_PREFIX}${name}`], - arguments: [String(version), String(STREAM_TTL_SEC)], + keys: [`${SYNC_VERSION_PREFIX}${name}`, generationKey(name)], + arguments: [String(version), String(STREAM_TTL_SEC), expectedGeneration], }) return } catch (error) { @@ -726,14 +1002,35 @@ export class FileDocStore { await this.releaseLock(`${MERGE_LOCK_PREFIX}${name}`, token) } - private applyEntry(room: StoreRoom, id: string, message: Record): void { + private applyEntry( + name: string, + room: StoreRoom, + id: string, + message: Record + ): void { + if (!isAfterStreamId(id, room.lastId)) return room.lastId = id - // Account for every entry the tailer sees, whoever wrote it — this is the only point that - // observes peer and roomless appends. A fold's own output is excluded so it cannot arm the - // trigger against itself. - if (!message[COMPACTION_FIELD]) { - room.pendingDeltas.set(id, message[UPDATE_FIELD]?.length ?? 0) + const generation = message[GENERATION_FIELD] + if (generation) { + if ( + room.generationInvalidated || + (room.generation !== null && room.generation !== generation) || + (room.generation === null && + room.seededObserved && + room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) !== generation) + ) { + room.generationInvalidated = true + return + } + room.generation = generation } + if (room.generationInvalidated) return + const isSnapshot = + message[GENERATION_FIELD] !== undefined || + message[SNAPSHOT_FIELD] !== undefined || + message[COMPACTION_FIELD] !== undefined + room.lastDeltaBytes = isSnapshot ? 0 : (message[UPDATE_FIELD]?.length ?? 0) + room.uncompactedDeltaBytes += room.lastDeltaBytes // A compaction snapshot folds seed + edits into one frame; stamp it so the relay's edit-tracker // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). An // agent-streamed preview frame is stamped separately so the tracker NEVER marks it edited. @@ -752,6 +1049,9 @@ export class FileDocStore { if (origin === REDIS_SNAPSHOT_ORIGIN || (origin === REDIS_ORIGIN && seededBefore)) { room.realEdited = true } + if (room.uncompactedDeltaBytes - room.lastDeltaBytes >= COMPACT_ENCODED_BYTES) { + void this.maybeCompact(name) + } } /** @@ -760,6 +1060,7 @@ export class FileDocStore { */ private async runReader(): Promise { let failures = 0 + let blockingBatchIndex = 0 while (this.running && this.read) { const snapshot = new Map(this.rooms) if (snapshot.size === 0) { @@ -767,31 +1068,40 @@ export class FileDocStore { continue } try { - const res = await this.read.xRead( - [...snapshot].map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), - { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT } - ) - // The streak ends HERE, on the read returning at all — not further down once entries are - // applied. A blocking read that times out with nothing new is the idle steady state, and it - // proves the connection works just as well as one carrying messages; leaving the streak - // standing through it would keep an old outage's count alive indefinitely, so the next - // unrelated blip would open at the backoff cap and log a failure count it never earned. - failures = 0 - if (!res) continue - for (const stream of res) { - const name = stream.name.slice(STREAM_PREFIX.length) - const room = this.rooms.get(name) - // Skip if detached mid-read, OR replaced by a close→reopen (a DIFFERENT StoreRoom): applying - // entries read against the OLD room's lastId to the new one could regress its lastId (harmless - // but wasteful re-delivery). The new room caught itself up via xRange already. - if (!room || room !== snapshot.get(name)) continue - for (const entry of stream.messages) this.applyEntry(room, entry.id, entry.message) - // Foldability is decided by `lastId`, which only the tailer advances — so a burst of - // large edits followed by silence would otherwise sit unfolded until the next publish - // happened to re-evaluate the trigger. Re-check it where the boundary actually moved. - if (foldableDeltaBytes(room) >= COMPACT_BYTES_THRESHOLD) - void this.maybeCompact(name, true) + const rooms = [...snapshot] + const batches: Array = [] + for (let index = 0; index < rooms.length; index += READ_STREAM_BATCH_SIZE) { + batches.push(rooms.slice(index, index + READ_STREAM_BATCH_SIZE)) + } + const applyResults = (results: Awaited>): boolean => { + if (!results) return false + for (const stream of results) { + const name = stream.name.slice(STREAM_PREFIX.length) + const room = this.rooms.get(name) + if (!room || room !== snapshot.get(name)) continue + for (const entry of stream.messages) + this.applyEntry(name, room, entry.id, entry.message) + } + return true + } + let received = false + for (const batch of batches) { + const streams = batch.map(([name, room]) => ({ + key: streamKey(name), + id: room.lastId, + })) + received = applyResults(await this.read.xRead(streams, { COUNT: READ_COUNT })) || received + } + if (!received) { + const batch = batches[blockingBatchIndex % batches.length] + blockingBatchIndex = (blockingBatchIndex + 1) % batches.length + const streams = batch.map(([name, room]) => ({ + key: streamKey(name), + id: room.lastId, + })) + applyResults(await this.read.xRead(streams, { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT })) } + failures = 0 } catch (error) { if (!this.running) break await this.recoverReader(++failures, error) @@ -818,7 +1128,12 @@ export class FileDocStore { error: getErrorMessage(error), }) } - await sleep(backoffWithJitter(failures, null, { baseMs: 500, maxMs: READER_RETRY_MAX_MS })) + await sleep( + backoffWithJitter(failures, null, { + baseMs: 500, + maxMs: READER_RETRY_MAX_MS, + }) + ) if (this.running && this.read && !this.read.isOpen) { await this.read.connect().catch((reconnectError) => { logger.warn('FileDocStore could not re-open the reader connection', { @@ -834,17 +1149,26 @@ export class FileDocStore { * only one task compacts a given stream at a time (concurrent snapshot+trim would race). Trims only up * to what the snapshot provably contains — never un-integrated peer entries (see below). */ - private async maybeCompact(name: string, force = false): Promise { + private async maybeCompact(name: string): Promise { if (!this.write) return const room = this.rooms.get(name) - if (!room) return + if (!room || room.compacting || Date.now() < room.compactRetryAfter) return + room.compacting = true try { - if (force && Date.now() < room.compactRetryAfter) return - if (!force && (await this.write.xLen(streamKey(name))) < COMPACT_THRESHOLD) return + const streamLength = await this.write.xLen(streamKey(name)) + if ( + streamLength < COMPACT_THRESHOLD && + room.uncompactedDeltaBytes - room.lastDeltaBytes < COMPACT_ENCODED_BYTES + ) { + return + } const key = `${COMPACT_LOCK_PREFIX}${name}` const token = await this.acquireLock(key, COMPACT_LOCK_TTL_MS) if (!token) return try { + /** Integrate the completed stream prefix before capturing the snapshot and compaction barrier. */ + await this.catchUp(name) + if (this.rooms.get(name) !== room) return // Capture the snapshot AND the id it covers in one synchronous step (no await between): the // snapshot is `room.doc`, which holds exactly what this task's tailer has integrated — every // entry up to `room.lastId`. Entries a peer task published AFTER that (id > lastId) are NOT in @@ -852,42 +1176,64 @@ export class FileDocStore { // them — only entries the snapshot provably subsumes (id <= lastId). Trimming to the freshly // appended snapshot id instead would silently drop those un-integrated peer entries. const upTo = room.lastId + /** Ordered, deduplicated replay lets two counters represent the prefix without a per-entry map. */ + const deltaBytesAtBarrier = room.uncompactedDeltaBytes - room.lastDeltaBytes const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') // Stamp the snapshot by what it folds: a real edit → SNAPSHOT_FIELD (a fresh catch-up treats it // as edited content, not a bare seed). An agent-ONLY stream (no real edit yet) → AGENT_FIELD, so a // peer catching up applies it as REDIS_AGENT_ORIGIN and never marks the doc edited — preserving // the no-persist guarantee even when a long copilot stream alone crosses the compaction threshold. const marker = room.realEdited ? SNAPSHOT_FIELD : AGENT_FIELD - await this.write.xAdd(streamKey(name), '*', { - [UPDATE_FIELD]: snapshot, - [marker]: '1', - [COMPACTION_FIELD]: '1', + const snapshotId = await this.write.eval(APPEND_SNAPSHOT_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [ + UPDATE_FIELD, + snapshot, + marker, + room.generation ?? '', + GENERATION_FIELD, + upTo, + COMPACTION_FIELD, + ], }) - // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and - // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. - await this.write.xTrim(streamKey(name), 'MINID', upTo) - // Drop exactly what the trim removed, which `MINID upTo` being INCLUSIVE makes `id < upTo` - // — the entry at the boundary survives, and its bytes are still in Redis. Run after the - // trim, so a failed fold leaves the ledger intact and the trigger armed. - for (const id of room.pendingDeltas.keys()) { - if (isAfterStreamId(upTo, id)) room.pendingDeltas.delete(id) - } + if (typeof snapshotId !== 'string') return + /** MINID retains the barrier entry and later deltas, including those observed during the await. */ + room.uncompactedDeltaBytes = Math.max(0, room.uncompactedDeltaBytes - deltaBytesAtBarrier) } finally { await this.releaseLock(key, token) } } catch (error) { room.compactRetryAfter = Date.now() + COMPACT_RETRY_COOLDOWN_MS - logger.warn(`FileDocStore compaction failed for ${name}`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore compaction failed for ${name}`, { + error: getErrorMessage(error), + }) + } finally { + room.compacting = false } } + private async refreshDocumentTtls(name: string): Promise { + await this.write?.eval(REFRESH_DOCUMENT_TTLS_SCRIPT, { + keys: documentKeys(name), + arguments: [String(STREAM_TTL_SEC)], + }) + } + private async refreshTtls(): Promise { - if (!this.write) return + if (!this.write) { + const now = Date.now() + for (const [name, invalidation] of this.localInvalidations) { + if (this.rooms.has(name)) invalidation.expiresAt = now + STREAM_TTL_SEC * 1_000 + else if (invalidation.expiresAt <= now) this.localInvalidations.delete(name) + } + if (this.localInvalidations.size === 0 && this.heartbeat) { + clearInterval(this.heartbeat) + this.heartbeat = null + } + return + } for (const name of this.rooms.keys()) { - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) - // Keep the synced-version key alive as long as its stream, so an open-but-idle doc's persist - // If-Match token can't expire out from under it (which would force a needless reconcile). - await this.write.expire(`${SYNC_VERSION_PREFIX}${name}`, STREAM_TTL_SEC).catch(() => {}) + await this.refreshDocumentTtls(name).catch(() => {}) } } } diff --git a/apps/realtime/src/handlers/file-doc.join-readiness.test.ts b/apps/realtime/src/handlers/file-doc.join-readiness.test.ts index 9b7b6a1c7ec..4938c0680f4 100644 --- a/apps/realtime/src/handlers/file-doc.join-readiness.test.ts +++ b/apps/realtime/src/handlers/file-doc.join-readiness.test.ts @@ -63,12 +63,19 @@ vi.mock('redis', () => { for (let i = 0; i < backing.readDelayTicks; i++) await Promise.resolve() return (backing.streams.get(key) ?? []).map((e) => ({ ...e })) }, + xRevRange: async (key: string) => + [...(backing.streams.get(key) ?? [])] + .reverse() + .slice(0, 1) + .map((entry) => ({ ...entry })), xLen: async (key: string) => (backing.streams.get(key) ?? []).length, - xRead: async (streams: { key: string; id: string }[]) => { + xRead: async (streams: { key: string; id: string }[], options?: { COUNT?: number }) => { const res: { name: string; messages: { id: string; message: Record }[] }[] = [] for (const { key, id } of streams) { - const after = (backing.streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + const after = (backing.streams.get(key) ?? []) + .filter((e) => seqOf(e.id) > seqOf(id)) + .slice(0, options?.COUNT) if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) } if (res.length) return res diff --git a/apps/realtime/src/handlers/file-doc.multireplica.test.ts b/apps/realtime/src/handlers/file-doc.multireplica.test.ts index cfffe81e857..cb24909618f 100644 --- a/apps/realtime/src/handlers/file-doc.multireplica.test.ts +++ b/apps/realtime/src/handlers/file-doc.multireplica.test.ts @@ -25,6 +25,7 @@ const fakeStore = { versions: new Map(), acquireMergeSlot: vi.fn(async () => 'token'), releaseMergeSlot: vi.fn(async () => {}), + getDocumentGeneration: vi.fn(async () => 'shared-generation'), getStreamState: vi.fn(async () => new Uint8Array([1])), publishAndWait: vi.fn(async () => {}), getSyncedVersion: vi.fn(async (name: string) => fakeStore.versions.get(name) ?? null), @@ -67,7 +68,13 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(await applyMarkdownToLiveFileDoc('file-1', '# durable', { version: 100 })).toBe( 'applied' ) - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100, 'shared-generation') + expect(fakeStore.getStreamState).toHaveBeenCalledWith(ROOM_NAME, 'shared-generation') + expect(fakeStore.publishAndWait).toHaveBeenCalledWith( + ROOM_NAME, + expect.any(Uint8Array), + 'shared-generation' + ) mockFetchFileDocMerge.mockClear() // A durable write with an OLDER version than the SHARED synced version is stale — rejected under the @@ -81,7 +88,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(await applyMarkdownToLiveFileDoc('file-1', '# durable again', { version: 150 })).toBe( 'applied' ) - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150) + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150, 'shared-generation') // setSyncedVersion fired only for the two applied durable writes, never for the stale one. expect(fakeStore.setSyncedVersion).toHaveBeenCalledTimes(2) }) @@ -98,7 +105,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' ).toBe('applied') expect(mockFetchFileDocMerge).not.toHaveBeenCalled() // content deferred to the client expect(fakeStore.publishAndWait).not.toHaveBeenCalled() - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) // version still recorded + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100, 'shared-generation') // version still recorded // Once streaming stops the flag clears and the (now near-noop) durable merge resumes normally. fakeStore.isAgentStreaming.mockResolvedValue(false) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index c0878d90129..3bc5b90c5c4 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -3,6 +3,7 @@ */ import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, FILE_DOC_SEED, } from '@sim/realtime-protocol/file-doc' @@ -37,12 +38,20 @@ vi.mock('@/handlers/file-doc-app', () => ({ import { applyMarkdownToLiveFileDoc, cleanupFileDocForSocket, + fileDocAdmissionRoom, flushAllFileDocRooms, + invalidateLiveFileDocument, setupWorkspaceFileDocHandlers, } from '@/handlers/file-doc' -import { beginRoomPermissionRead, commitRoomPermission } from '@/middleware/permissions' +import { FileDocInvalidatedError, getFileDocStore } from '@/handlers/file-doc-store' +import * as permissions from '@/middleware/permissions' +import { + beginRoomPermissionRead, + commitRoomPermission, + ROLE_REVALIDATION_TTL_MS, +} from '@/middleware/permissions' -type Handler = (payload?: unknown) => Promise | void +type Handler = (...payload: unknown[]) => Promise | void const ROOM_NAME = 'workspace-file-doc:file-1' @@ -54,16 +63,19 @@ interface SentMessage { } /** An `io` mock that records every server-originated emit with its target/except. */ -function createIo() { +function createIo(deliver?: (message: SentMessage) => void) { const sent: SentMessage[] = [] + const emit = (message: SentMessage) => { + sent.push(message) + deliver?.(message) + } /** Records `io.in(socketId).socketsLeave(room)` — a socket forced out of a room from outside. */ const left: { socketId: string; room: string }[] = [] const to = vi.fn((target: string) => ({ except: (exclude: string) => ({ - emit: (event: string, payload: unknown) => - sent.push({ target, except: exclude, event, payload }), + emit: (event: string, payload: unknown) => emit({ target, except: exclude, event, payload }), }), - emit: (event: string, payload: unknown) => sent.push({ target, event, payload }), + emit: (event: string, payload: unknown) => emit({ target, event, payload }), })) const inFn = vi.fn((socketId: string) => ({ socketsLeave: (room: string) => { @@ -133,10 +145,14 @@ async function flushMicrotasks(): Promise { * An encoded Yjs update shaped like the server seed builder's output: some content in the shared * `default` type plus the {@link FILE_DOC_SEED} flag, so applying it marks the doc seeded. */ -function seedResult(content: string): { update: Uint8Array; version: number } { +function seedResult( + content: string, + docId = 'doc-default' +): { update: Uint8Array; version: number } { const doc = new Y.Doc() doc.getText(FILE_DOC_FIELD).insert(0, content) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + if (docId) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, docId) return { update: Y.encodeStateAsUpdate(doc), version: 1 } } @@ -186,9 +202,7 @@ describe('setupWorkspaceFileDocHandlers', () => { workspaceId: 'ws-1', workspacePermission: 'write', }) - // Default: the server seed builder returns no content (empty file). Tests that - // exercise seeding override this per-case with an encoded Yjs update. - mockFetchFileDocSeed.mockResolvedValue(null) + mockFetchFileDocSeed.mockResolvedValue(seedResult('')) // Default: the merge builder returns a valid no-op (empty-doc) update. Tests exercising copilot // merges override it. mockFetchFileDocMerge.mockResolvedValue(Y.encodeStateAsUpdate(new Y.Doc())) @@ -196,13 +210,14 @@ describe('setupWorkspaceFileDocHandlers', () => { mockFetchFileDocPersist.mockResolvedValue({ status: 'persisted', version: 1 }) }) - afterEach(() => { + afterEach(async () => { // The room store is module-global; drop every room the test's sockets opened. const { io } = createIo() // Simulate a full disconnect between tests (`endOfLife`) so the module-global join-generation // map is cleared and never bleeds a counter into the next test. for (const id of createdSocketIds) cleanupFileDocForSocket(id, io, true) createdSocketIds.clear() + await getFileDocStore().shutdown() }) it('rejects join when the socket is not authenticated', async () => { @@ -217,6 +232,24 @@ describe('setupWorkspaceFileDocHandlers', () => { ) }) + it('fails closed when authorization does not resolve a workspace context', async () => { + mockAuthorizeRoom.mockResolvedValueOnce({ + allowed: true, + status: 200, + workspacePermission: 'write', + }) + const { io } = createIo() + const { socket, handlers } = setup('socket-no-workspace', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'JOIN_FAILED', retryable: true }) + ) + expect(socket.join).not.toHaveBeenCalled() + }) + it('rejects join with a retryable error when realtime is unavailable', async () => { const { io } = createIo() const { socket, handlers } = createSocket('socket-1') @@ -248,6 +281,264 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockAuthorizeRoom).not.toHaveBeenCalled() }) + it('rejects an incompatible collaborative-document schema before authorizing', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-schema', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ + fileId: 'file-1', + clientId: 1, + schemaVersion: 99, + }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'SCHEMA_VERSION_MISMATCH', retryable: false }) + ) + expect(mockAuthorizeRoom).not.toHaveBeenCalled() + }) + + it('acknowledges user updates only after applying them to the joined document', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io, sent } = createIo() + const { handlers } = setup('socket-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'acknowledged edit') + const acknowledge = vi.fn() + sent.length = 0 + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-1', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + + await vi.waitFor(() => + expect(acknowledge).toHaveBeenCalledWith({ status: 'accepted', updateId: 'update-1' }) + ) + expect(sent).toContainEqual( + expect.objectContaining({ + target: ROOM_NAME, + event: FILE_DOC_EVENTS.MESSAGE, + }) + ) + source.destroy() + }) + + it('rejects an update for a replaced document without applying it', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-current')) + const { io, sent } = createIo() + const { handlers } = setup('socket-replaced', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const acknowledge = vi.fn() + sent.length = 0 + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-stale', + updateId: 'update-stale', + update: Y.encodeStateAsUpdate(new Y.Doc()), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-stale', + }) + expect(sent).toHaveLength(0) + }) + + it('rejects malformed Yjs updates without retrying them', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io } = createIo() + const { handlers } = setup('socket-malformed-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const acknowledge = vi.fn() + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-malformed', + update: new Uint8Array([255]), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'INVALID_UPDATE', + retryable: false, + updateId: 'update-malformed', + }) + }) + + it('ignores an acknowledged-update event without a callable acknowledgement', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io } = createIo() + const { handlers } = setup('socket-missing-ack', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(() => + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-1', + update: Y.encodeStateAsUpdate(new Y.Doc()), + }, + { not: 'a function' } + ) + ).not.toThrow() + }) + + it('keeps a room alive until an acknowledged update finishes appending', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + let resolveAppend: () => void = () => {} + const append = new Promise((resolve) => { + resolveAppend = resolve + }) + const publish = vi + .spyOn(getFileDocStore(), 'publishClientUpdateAndWait') + .mockReturnValue(append) + const { io } = createIo() + const { handlers } = setup('socket-update-leave', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'accepted before leave') + const acknowledge = vi.fn() + + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-leave', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + await vi.waitFor(() => expect(publish).toHaveBeenCalledTimes(1)) + handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAppend() + + await vi.waitFor(() => + expect(acknowledge).toHaveBeenCalledWith({ + status: 'accepted', + updateId: 'update-leave', + }) + ) + publish.mockRestore() + source.destroy() + }) + + it('rejects a generation-fenced update as a durable document replacement', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const publish = vi + .spyOn(getFileDocStore(), 'publishClientUpdateAndWait') + .mockRejectedValue(new FileDocInvalidatedError()) + const { io } = createIo() + const { handlers } = setup('socket-replaced-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'stale edit') + const acknowledge = vi.fn() + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-replaced', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-replaced', + }) + publish.mockRestore() + source.destroy() + }) + + it.each(['before append', 'during append', 'during append and reseed'] as const)( + 'rejects a document invalidated %s without applying or relaying its stale update', + async (timing) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('Original', 'doc-race')) + const { io, sent } = createIo() + const { handlers } = setup('socket-generation-race', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const store = getFileDocStore() + let finishAppend!: () => void + const pendingAppend = new Promise((resolve) => { + finishAppend = resolve + }) + const publish = + timing !== 'before append' + ? vi.spyOn(store, 'publishClientUpdateAndWait').mockReturnValueOnce(pendingAppend) + : undefined + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'Stale text') + const acknowledge = vi.fn() + try { + if (timing === 'before append') await invalidateLiveFileDocument('file-1', 2) + sent.length = 0 + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-race', + updateId: 'update-race', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + if (timing !== 'before append') { + expect(publish).toHaveBeenCalledTimes(1) + await invalidateLiveFileDocument('file-1', 2) + if (timing === 'during append and reseed') { + mockFetchFileDocSeed.mockResolvedValue({ + ...seedResult('Replacement', 'doc-new'), + version: 2, + }) + const fresh = setup('socket-generation-fresh', io) + await fresh.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(fresh.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ docId: 'doc-new' }) + ) + sent.length = 0 + } + finishAppend() + } + await vi.waitFor(() => + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-race', + }) + ) + expect(sent).toHaveLength(0) + } finally { + finishAppend() + publish?.mockRestore() + source.destroy() + } + } + ) + it('does not re-enter the room when access was revoked while the join was in flight', async () => { // The sweep records a revocation before it evicts, so a join whose authorize // completed just before that must not put the socket back in the document. @@ -369,6 +660,125 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockFetchFileDocPersist).toHaveBeenCalled() }) + it('awaits final persistence of an already removed room during shutdown', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-final-persist', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'last edit') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(edit)) + ) + ) + let finishPersist!: (result: { status: 'persisted'; version: number }) => void + mockFetchFileDocPersist.mockReturnValueOnce( + new Promise((resolve) => { + finishPersist = resolve + }) + ) + cleanupFileDocForSocket('socket-final-persist', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(1) + const completed = vi.fn() + const flush = flushAllFileDocRooms().then(completed) + await flushMicrotasks() + expect(completed).not.toHaveBeenCalled() + finishPersist({ status: 'persisted', version: 2 }) + await flush + expect(completed).toHaveBeenCalledTimes(1) + edit.destroy() + }) + + it('drains an accepted update still appending when the last socket closes', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server', 'shutdown-doc')) + const { io } = createIo() + const { handlers } = setup('socket-closing-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + let finishAppend!: () => void + const append = vi.spyOn(getFileDocStore(), 'publishClientUpdateAndWait').mockReturnValueOnce( + new Promise((resolve) => { + finishAppend = resolve + }) + ) + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'accepted before socket close') + const ack = vi.fn() + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'shutdown-doc', + updateId: 'shutdown-update', + update: Y.encodeStateAsUpdate(edit), + }, + ack + ) + cleanupFileDocForSocket('socket-closing-update', io, true) + const completed = vi.fn() + const flush = flushAllFileDocRooms().then(completed) + await flushMicrotasks() + expect(completed).not.toHaveBeenCalled() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + finishAppend() + await flush + expect(ack).toHaveBeenCalledWith({ status: 'accepted', updateId: 'shutdown-update' }) + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(1) + const persisted = new Y.Doc() + Y.applyUpdate(persisted, mockFetchFileDocPersist.mock.calls[0][3]) + expect(persisted.getText(FILE_DOC_FIELD).toString()).toContain('accepted before socket close') + append.mockRestore() + edit.destroy() + persisted.destroy() + }) + + it.each(['persist', 'invalidate', 'leave'] as const)( + 'retries a throttled persist safely until %s', + async (outcome) => { + vi.useFakeTimers() + const store = getFileDocStore() + const claim = vi + .spyOn(store, 'tryClaimPersistWindow') + .mockResolvedValueOnce(false) + .mockResolvedValue(true) + try { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-throttled', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'pending durable edit') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(edit)) + ) + ) + await vi.advanceTimersByTimeAsync(5_000) + expect(claim).toHaveBeenCalledTimes(1) + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + if (outcome === 'invalidate') await store.invalidateDocument(ROOM_NAME, 2) + if (outcome === 'leave') { + cleanupFileDocForSocket('socket-throttled', io, true) + await vi.advanceTimersByTimeAsync(0) + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(1) + mockFetchFileDocPersist.mockClear() + } + await vi.advanceTimersByTimeAsync(5_000) + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(outcome === 'persist' ? 1 : 0) + if (outcome === 'persist') { + const persisted = new Y.Doc() + Y.applyUpdate(persisted, mockFetchFileDocPersist.mock.calls[0][3]) + expect(persisted.getText(FILE_DOC_FIELD).toString()).toContain('pending durable edit') + persisted.destroy() + } + edit.destroy() + } finally { + claim.mockRestore() + vi.useRealTimers() + } + } + ) + it('drops document frames and evicts once the editor loses write access mid-session', async () => { // The join-time check is not a standing right: a collaborator downgraded to `read` // (or removed) must stop landing durable edits on the socket they already hold. @@ -550,6 +960,10 @@ describe('setupWorkspaceFileDocHandlers', () => { FILE_DOC_EVENTS.JOIN_SUCCESS, expect.objectContaining({ fileId: 'file-1', clientId: 1 }) ) + const joinSuccess = socket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.JOIN_SUCCESS + )?.[1] as Record + expect(joinSuccess).not.toHaveProperty('acknowledgedUpdates') // A binary sync-step-1 message (type tag 0) is sent to kick off the handshake. const syncMessage = socket.emit.mock.calls.find( @@ -575,6 +989,479 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# From server') }) + it('discards a fenced in-memory generation before serving the next join', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io, left } = createIo() + const first = setup('socket-old-generation', io) + await first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + await getFileDocStore().invalidateDocument(ROOM_NAME, 1) + mockFetchFileDocSeed.mockResolvedValue(seedResult('# New', 'doc-new')) + const second = setup('socket-new-generation', io) + await second.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + expect(left).toContainEqual({ socketId: 'socket-old-generation', room: ROOM_NAME }) + second.socket.emit.mockClear() + second.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeSyncStep1(encoder, new Y.Doc()) + ) + ) + const reply = second.socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# New') + clientDoc.destroy() + }) + + it.each([false, true])( + 'rejects a pending join invalidated during permission with existing room %s', + async (existingRoom) => { + const seed = seedResult('# Old', 'doc-old') + mockFetchFileDocSeed.mockResolvedValue(seed) + const { io, sent } = createIo() + if (existingRoom) { + const first = setup('socket-first-invalidation', io) + await first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + } + let resolvePermission!: (permission: string) => void + const permission = new Promise((resolve) => { + resolvePermission = resolve + }) + const permissionCheck = vi + .spyOn(permissions, 'resolveCurrentRoomPermission') + .mockImplementationOnce(() => permission) + try { + const pending = setup('socket-pending-invalidation', io) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await vi.waitFor(() => expect(permissionCheck).toHaveBeenCalledOnce()) + expect(await invalidateLiveFileDocument('file-1', 2)).toMatchObject({ status: 'applied' }) + io.to(ROOM_NAME).emit(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1' }) + resolvePermission('write') + await joining + + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ retryable: true }) + ) + const staleClient = new Y.Doc() + Y.applyUpdate(staleClient, seed.update) + const vector = Y.encodeStateVector(staleClient) + staleClient.getText(FILE_DOC_FIELD).insert(0, 'must not accept ') + sent.length = 0 + pending.socket.emit.mockClear() + pending.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(staleClient, vector)) + ) + ) + expect(sent).toHaveLength(0) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.any(Uint8Array) + ) + await flushAllFileDocRooms() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + staleClient.destroy() + + mockFetchFileDocSeed.mockResolvedValue({ ...seedResult('# New', 'doc-new'), version: 2 }) + await pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ docId: 'doc-new' }) + ) + } finally { + resolvePermission('write') + permissionCheck.mockRestore() + } + } + ) + + it.each(['write', 'read', null] as const)( + 'withholds document and presence broadcasts until final authorization resolves to %s', + async (permission) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Private', 'doc-private')) + const memberships = new Set() + let pending: ReturnType + const { io } = createIo(({ target, event, payload }) => { + if (memberships.has(target)) pending.socket.emit(event, payload) + }) + pending = setup('socket-pending-authorization', io, { + join: vi.fn((name: string) => { + memberships.add(name) + }), + leave: vi.fn((name: string) => { + memberships.delete(name) + }), + }) + let resolvePermission!: (value: 'write' | 'read' | null) => void + const authorization = new Promise<'write' | 'read' | null>((resolve) => { + resolvePermission = resolve + }) + const guard = vi + .spyOn(permissions, 'resolveCurrentRoomPermission') + .mockResolvedValueOnce('write') + .mockImplementationOnce(() => authorization) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + try { + await vi.waitFor(() => expect(guard).toHaveBeenCalledTimes(2)) + const content = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, seedResult('# Private update').update) + ) + io.local.to(ROOM_NAME).emit(FILE_DOC_EVENTS.MESSAGE, content) + io.to(ROOM_NAME).emit( + FILE_DOC_EVENTS.MESSAGE, + new Uint8Array([FILE_DOC_MESSAGE_TYPE.AWARENESS]) + ) + io.to(ROOM_NAME).emit(FILE_DOC_EVENTS.PRESENCE, [{ userId: 'private-peer' }]) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.anything() + ) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.PRESENCE, + expect.anything() + ) + expect(memberships.has(ROOM_NAME)).toBe(false) + resolvePermission(permission) + await joining + expect(memberships.has(ROOM_NAME)).toBe(permission === 'write') + if (permission === 'write') { + expect(joinSuccessFileId(pending.socket)).toBe('file-1') + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.any(Uint8Array) + ) + } else { + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + } + } finally { + resolvePermission(permission) + await joining + guard.mockRestore() + } + } + ) + + it('rejects a generation invalidated while final authorization is pending', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io } = createIo() + const pending = setup('socket-final-authorization-invalidation', io) + let finishAuthorization!: (permission: 'write') => void + const authorization = new Promise<'write'>((resolve) => { + finishAuthorization = resolve + }) + const guard = vi + .spyOn(permissions, 'resolveCurrentRoomPermission') + .mockResolvedValueOnce('write') + .mockImplementationOnce(() => authorization) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + try { + await vi.waitFor(() => expect(guard).toHaveBeenCalledTimes(2)) + await getFileDocStore().invalidateDocument(ROOM_NAME, 2) + finishAuthorization('write') + await joining + expect(pending.socket.join).not.toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'JOIN_FAILED', retryable: true }) + ) + expect(pending.socket.leave).toHaveBeenCalledWith(fileDocAdmissionRoom('file-1')) + } finally { + finishAuthorization('write') + await joining + guard.mockRestore() + } + }) + + it.each(['revoked', 'expired'] as const)( + 'rejects access %s while the final generation check is pending', + async (access) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Private', 'doc-private')) + const { io } = createIo() + const pending = setup('socket-generation-authorization-revocation', io) + let finishGeneration!: (current: boolean) => void + const generation = new Promise((resolve) => { + finishGeneration = resolve + }) + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockImplementationOnce(() => generation) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const clock = vi.spyOn(Date, 'now') + try { + await vi.waitFor(() => expect(guard).toHaveBeenCalledOnce()) + if (access === 'revoked') { + commitRoomPermission( + 'user-1', + { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-1' }, + 'read', + beginRoomPermissionRead() + ) + } else { + clock.mockReturnValue(Date.now() + ROLE_REVALIDATION_TTL_MS + 1) + } + finishGeneration(true) + await joining + expect(pending.socket.join).not.toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ + code: access === 'revoked' ? 'ACCESS_DENIED' : 'JOIN_FAILED', + retryable: access === 'expired', + }) + ) + } finally { + clock.mockRestore() + finishGeneration(true) + await joining + guard.mockRestore() + } + } + ) + + it('receives invalidation while the subscribed generation check is pending', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const memberships = new Set() + let pending: ReturnType + const { io } = createIo(({ target, event, payload }) => { + if (memberships.has(target)) pending.socket.emit(event, payload) + }) + pending = setup('socket-subscribed-invalidation', io, { + join: vi.fn((name: string) => { + memberships.add(name) + }), + leave: vi.fn((name: string) => { + memberships.delete(name) + }), + }) + let resolveGeneration!: (current: boolean) => void + const currentGeneration = new Promise((resolve) => { + resolveGeneration = resolve + }) + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockImplementationOnce(() => currentGeneration) + try { + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await vi.waitFor(() => expect(guard).toHaveBeenCalledOnce()) + expect(memberships.has(ROOM_NAME)).toBe(false) + expect(memberships.has(fileDocAdmissionRoom('file-1'))).toBe(true) + await getFileDocStore().invalidateDocument(ROOM_NAME, 2) + io.to(fileDocAdmissionRoom('file-1')).emit(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1' }) + expect(pending.socket.emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + }) + await pending.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveGeneration(true) + await joining + + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(memberships.has(ROOM_NAME)).toBe(false) + expect(memberships.has(fileDocAdmissionRoom('file-1'))).toBe(false) + } finally { + resolveGeneration(true) + guard.mockRestore() + } + }) + + it.each(['invalidation', 'revocation', 'leave'] as const)( + 'rolls back an asynchronous room subscription interrupted by %s', + async (interruption) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io } = createIo() + let finishSubscription!: () => void + const subscription = new Promise((resolve) => { + finishSubscription = resolve + }) + const pending = setup('socket-async-subscription', io, { + join: vi.fn(() => subscription), + }) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await vi.waitFor(() => expect(pending.socket.join).toHaveBeenCalledOnce()) + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + if (interruption === 'invalidation') { + await getFileDocStore().invalidateDocument(ROOM_NAME, 2) + } else if (interruption === 'revocation') { + commitRoomPermission( + 'user-1', + { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-1' }, + 'read', + beginRoomPermissionRead() + ) + } else { + await pending.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + } + finishSubscription() + await joining + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.leave).toHaveBeenCalledWith(ROOM_NAME) + } + ) + + it.each(['revoked', 'expired', 'unchanged'] as const)( + 'checks %s access after an asynchronous content-room join', + async (access) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Private', 'doc-private')) + const { io } = createIo() + const memberships = new Set() + let finishSubscription!: () => void + const subscription = new Promise((resolve) => { + finishSubscription = resolve + }) + const pending = setup('socket-content-subscription-access', io, { + join: vi.fn((name: string) => { + if (name === ROOM_NAME) + return subscription.then(() => { + memberships.add(name) + }) + memberships.add(name) + }), + leave: vi.fn((name: string) => memberships.delete(name)), + }) + const clock = vi.spyOn(Date, 'now') + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + try { + await vi.waitFor(() => expect(pending.socket.join).toHaveBeenCalledWith(ROOM_NAME)) + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + if (access === 'revoked') { + commitRoomPermission( + 'user-1', + { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-1' }, + 'read', + beginRoomPermissionRead() + ) + } else if (access === 'expired') { + clock.mockReturnValue(Date.now() + ROLE_REVALIDATION_TTL_MS + 1) + } + finishSubscription() + await joining + expect(memberships.has(fileDocAdmissionRoom('file-1'))).toBe(false) + expect(memberships.has(ROOM_NAME)).toBe(access === 'unchanged') + if (access === 'unchanged') { + expect(joinSuccessFileId(pending.socket)).toBe('file-1') + } else { + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ + code: access === 'revoked' ? 'ACCESS_DENIED' : 'JOIN_FAILED', + retryable: access === 'expired', + }) + ) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.anything() + ) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.PRESENCE, + expect.anything() + ) + } + } finally { + clock.mockRestore() + finishSubscription() + await joining + } + } + ) + + it('keeps a shared provisional subscription until the other provider finishes joining', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Shared', 'doc-shared')) + const { io } = createIo() + const pending = setup('socket-shared-subscription', io) + const checks: Array<(current: boolean) => void> = [] + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockImplementation(() => new Promise((resolve) => checks.push(resolve))) + try { + const first = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const second = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await vi.waitFor(() => expect(checks).toHaveLength(2)) + checks[0](false) + await first + expect(pending.socket.leave).not.toHaveBeenCalled() + checks[1](true) + await second + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ clientId: 2, docId: 'doc-shared' }) + ) + expect(pending.socket.leave).not.toHaveBeenCalledWith(ROOM_NAME) + expect(pending.socket.leave).toHaveBeenCalledWith(fileDocAdmissionRoom('file-1')) + } finally { + for (const resolve of checks) resolve(true) + guard.mockRestore() + } + }) + + it('preserves the committed binding when a co-mounted provider join fails', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Shared', 'doc-shared')) + const { io } = createIo() + const current = setup('socket-existing-subscription', io) + await current.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockRejectedValueOnce(new Error('Temporary generation read failure')) + try { + await current.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(current.socket.leave).not.toHaveBeenCalledWith(ROOM_NAME) + current.socket.emit.mockClear() + current.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeSyncStep1(encoder, new Y.Doc()) + ) + ) + expect(current.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.any(Uint8Array) + ) + } finally { + guard.mockRestore() + } + }) + + it('does not discard a rebuilt room after a delayed check of its predecessor', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io, left } = createIo() + const original = setup('socket-original', io) + await original.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const checks: Array<(current: boolean) => void> = [] + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockResolvedValue(true) + .mockImplementationOnce(() => new Promise((resolve) => checks.push(resolve))) + .mockImplementationOnce(() => new Promise((resolve) => checks.push(resolve))) + try { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# New', 'doc-new')) + const first = setup('socket-first-new', io) + const second = setup('socket-second-new', io) + const firstJoin = first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + const secondJoin = second.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 3 }) + await vi.waitFor(() => expect(checks).toHaveLength(2)) + checks[0](false) + await firstJoin + checks[1](false) + await secondJoin + + expect(joinSuccessFileId(first.socket)).toBe('file-1') + expect(joinSuccessFileId(second.socket)).toBe('file-1') + expect(left).toContainEqual({ socketId: 'socket-original', room: ROOM_NAME }) + expect(left).not.toContainEqual({ socketId: 'socket-first-new', room: ROOM_NAME }) + } finally { + guard.mockRestore() + } + }) + it('seeds once across concurrent joiners, and every one of them waits for that seed', async () => { // Keep the first seed fetch IN FLIGHT so the doc is still unseeded when the second socket joins: // that forces the dedup onto the in-flight seed rather than `isDocSeeded`. Both joins must WAIT @@ -613,16 +1500,18 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# From server') }) - it('marks an empty/absent-file doc seeded so clients still reach readiness', async () => { - // A genuinely absent file yields a null seed (a read error would throw, not return null). The - // relay must still flip `initialContentLoaded` so the client's `synced && seeded` gate opens. - mockFetchFileDocSeed.mockResolvedValue(null) + it('seeds an existing empty file with its accepted document identity', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('', 'empty-doc')) const { io } = createIo() const { socket, handlers } = setup('socket-1', io) await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) await flushMicrotasks() + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ docId: 'empty-doc', version: 1 }) + ) socket.emit.mockClear() handlers[FILE_DOC_EVENTS.MESSAGE]( frame(FILE_DOC_MESSAGE_TYPE.SYNC, (e) => syncProtocol.writeSyncStep1(e, new Y.Doc())) @@ -633,7 +1522,39 @@ describe('setupWorkspaceFileDocHandlers', () => { const clientDoc = new Y.Doc() applySyncReply(reply?.[1] as Uint8Array, clientDoc) expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(clientDoc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey)).toBe('empty-doc') expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('') + clientDoc.destroy() + }) + + it('rejects a missing seed without publishing an editable blank room and releases its lock', async () => { + mockFetchFileDocSeed.mockResolvedValueOnce(null) + const { io, sent } = createIo() + const { socket, handlers } = setup('socket-missing-seed', io) + const release = vi.spyOn(getFileDocStore(), 'releaseSeedLock') + const seed = vi.spyOn(getFileDocStore(), 'seedIfEmpty') + try { + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'NOT_FOUND', retryable: false }) + ) + expect(socket.emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN_SUCCESS, expect.anything()) + expect(socket.emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.MESSAGE, expect.anything()) + expect(socket.join).not.toHaveBeenCalled() + expect(sent.some(({ event }) => event === FILE_DOC_EVENTS.PRESENCE)).toBe(false) + expect(seed).not.toHaveBeenCalled() + expect(release).toHaveBeenCalledOnce() + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ docId: 'doc-default' }) + ) + } finally { + release.mockRestore() + seed.mockRestore() + } }) it('makes one seed attempt and releases the guard on failure so a later join retries', async () => { @@ -918,7 +1839,8 @@ describe('setupWorkspaceFileDocHandlers', () => { FILE_DOC_EVENTS.JOIN_ERROR, expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) ) - expect(b.socket.join).not.toHaveBeenCalled() + expect(b.socket.leave).toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(b.socket)).toBeUndefined() }) it('reclaims a client id for the SAME user reconnecting (reused Yjs client id)', async () => { @@ -1030,6 +1952,31 @@ describe('setupWorkspaceFileDocHandlers', () => { ).not.toThrow() }) + it('drops a legacy frame that cannot fit the durable stream budget', async () => { + const { io, sent } = createIo() + const a = setup('socket-oversized-legacy', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + sent.length = 0 + + expect(() => + a.handlers[FILE_DOC_EVENTS.MESSAGE](new Uint8Array(FILE_DOC_LIMITS.updateBytes + 65)) + ).not.toThrow() + expect(sent).toHaveLength(0) + }) + + it('preflights the inner legacy update before applying a framing-sized overflow', async () => { + const { io, sent } = createIo() + const a = setup('socket-inner-oversized-legacy', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + sent.length = 0 + const oversized = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, new Uint8Array(FILE_DOC_LIMITS.updateBytes + 1)) + ) + + expect(() => a.handlers[FILE_DOC_EVENTS.MESSAGE](oversized)).not.toThrow() + expect(sent).toHaveLength(0) + }) + it('drops the document when the last editor leaves, re-seeding a fresh joiner from the server', async () => { const { io } = createIo() const a = setup('socket-a', io) @@ -1053,12 +2000,22 @@ describe('setupWorkspaceFileDocHandlers', () => { let resolveFirst: (v: unknown) => void = () => {} mockAuthorizeRoom .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) - .mockResolvedValueOnce({ allowed: true, status: 200, workspacePermission: 'write' }) + .mockResolvedValueOnce({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) const s = setup('socket-a', io) const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) - resolveFirst({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveFirst({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending // The socket is bound only to the newer file, never cross-bound to file-1. @@ -1075,7 +2032,12 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) s.socket.disconnected = true cleanupFileDocForSocket('socket-a', io, true) // disconnect cleanup — no-op, nothing registered yet - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(s.socket.join).not.toHaveBeenCalled() @@ -1095,7 +2057,12 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) // A stale leave for a DIFFERENT file must not invalidate the in-flight join. s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(joinSuccessFileId(s.socket)).toBe('file-2') @@ -1119,7 +2086,12 @@ describe('setupWorkspaceFileDocHandlers', () => { // map (`undefined !== generation`) and abort the join the client actually wants. s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(joinSuccessFileId(s.socket)).toBe('file-2') @@ -1241,7 +2213,8 @@ describe('setupWorkspaceFileDocHandlers', () => { ) // The rejected switch must leave file-1 intact — a is not torn out of its current document. expect(a.socket.leave).not.toHaveBeenCalledWith('workspace-file-doc:file-1') - expect(a.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-2') + expect(a.socket.leave).toHaveBeenCalledWith('workspace-file-doc:file-2') + expect(joinSuccessFileId(a.socket)).toBe('file-1') }) it('broadcasts a server-authenticated presence roster on join, one entry per session', async () => { diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 28ef6686f7a..994a19ce014 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -27,10 +27,15 @@ import { createLogger } from '@sim/logger' import { ROOM_MEMBERSHIP_ACTIONS, satisfiesRoomMembership } from '@sim/platform-authz/room-policy' import { FILE_DOC_EVENTS, + FILE_DOC_LEGACY_SCHEMA_VERSION, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, FILE_DOC_TIMEOUTS, type FileDocPresenceUser, + type FileDocUpdateAck, + type FileDocUpdatePayload, type JoinFileDocPayload, type LeaveFileDocPayload, toFileDocBytes, @@ -47,6 +52,7 @@ import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' import { + FileDocInvalidatedError, getFileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN, @@ -176,7 +182,7 @@ interface FileDocRoom { agentStreamingUntil: number /** * Resolves once this room's doc reflects the file's shared stream (see {@link FileDocStore.catchUp}). - * Never rejects — the catch-up logs and gives up — so awaiting it can never fail a join. + * Rejects when replay cannot complete so the join fails closed rather than serving partial state. */ hydrated: Promise /** @@ -185,10 +191,14 @@ interface FileDocRoom { * document being assembled. A room with a join in flight is not idle. */ pendingJoins: number + /** Acknowledged updates currently waiting for their durable stream append. */ + pendingUpdates: number } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ const fileDocRooms = new Map() +const pendingFileDocPersists = new Set>() +const pendingFileDocUpdates = new Set>() /** socketId → its current file-doc room name (a socket edits at most one doc). */ const socketToRoomName = new Map() /** @@ -217,14 +227,40 @@ const fileDocRoom = (fileId: string): RoomRef => ({ id: fileId, }) +/** Pending admissions receive invalidations here, never document or presence frames. */ +export function fileDocAdmissionRoom(fileId: string): string { + return `file-doc-admission:${fileId}` +} + /** * A `y-protocols` transaction/awareness origin is the emitting socket id (a * string) when it came from a client, and something else (`null` / `'local'` / * `'timeout'`) for server-internal changes. Returns the socket id to exclude * from a relay, or `null` to broadcast to the whole room. */ +interface ClientUpdateOrigin { + kind: 'client-update' + socketId: string +} + +const MAX_CLIENT_UPDATE_ID_LENGTH = 128 + +function clientUpdateOrigin(socketId: string): ClientUpdateOrigin { + return { kind: 'client-update', socketId } +} + +function isClientUpdateOrigin(origin: unknown): origin is ClientUpdateOrigin { + return ( + typeof origin === 'object' && + origin !== null && + (origin as Partial).kind === 'client-update' && + typeof (origin as Partial).socketId === 'string' + ) +} + function originSocketId(origin: unknown): string | null { - return typeof origin === 'string' ? origin : null + if (typeof origin === 'string') return origin + return isClientUpdateOrigin(origin) ? origin.socketId : null } /** @@ -235,6 +271,28 @@ function originSocketId(origin: unknown): string | null { * on its own echo because the operations are already applied locally. */ const AGENT_SYNC_ORIGIN = Symbol('file-doc-agent-sync') +/** Maximum legacy framed message size: raw update budget plus small Yjs framing headroom. */ +const MAX_LEGACY_FRAME_BYTES = FILE_DOC_LIMITS.updateBytes + 64 + +/** + * Checks the inner update before readSyncMessage mutates the room: the legacy outer-frame limit + * includes framing headroom, which must not allow an update too large for the shared stream. + */ +function hasOversizedLegacyUpdate(bytes: Uint8Array): boolean { + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + if ( + messageType !== FILE_DOC_MESSAGE_TYPE.SYNC && + messageType !== FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST + ) { + return false + } + const syncType = decoding.readVarUint(decoder) + if (syncType !== syncProtocol.messageYjsSyncStep2 && syncType !== syncProtocol.messageYjsUpdate) { + return false + } + return decoding.readVarUint8Array(decoder).byteLength > FILE_DOC_LIMITS.updateBytes +} /** * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness @@ -295,10 +353,19 @@ function schedulePersist(name: string, room: FileDocRoom): void { * fallback before any await, so a `void flushPersist(name, room, true)` fired immediately before the * caller destroys `room.doc` never encodes a destroyed doc, and the disabled path stays authoritative. */ -async function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { +function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { + const pending = persistRoom(name, room, final).finally(() => + pendingFileDocPersists.delete(pending) + ) + pendingFileDocPersists.add(pending) + return pending +} + +async function persistRoom(name: string, room: FileDocRoom, final: boolean): Promise { // Never project a doc no user actually edited back over the file (see {@link FileDocRoom.edited}). if (!room.edited || !room.workspaceId || !room.lastEditorUserId) return const store = getFileDocStore() + const generation = docIdOf(room.doc) const workspaceId = room.workspaceId const userId = room.lastEditorUserId // Synchronous fallback capture — before any await, since the caller may destroy `room.doc` the moment @@ -321,6 +388,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr try { return (await store.getStreamState(name)) ?? localState } catch (streamError) { + if (streamError instanceof FileDocInvalidatedError) throw streamError // A transient Redis read must NOT drop the write when we already hold a valid local snapshot — // else the last-disconnect flush loses the session's edits as the room is torn down. But once a // reconcile has run, `localState` is NULLED (it predates the merged-in out-of-band edit), so a @@ -344,8 +412,11 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr } try { - if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) + if (!(await store.isDocumentGenerationCurrent(name, generation))) return + if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) { + if (fileDocRooms.get(name) === room) schedulePersist(name, room) return + } // The If-Match token: the durable content version the live doc is synced to. let ifMatch = await currentVersion() @@ -367,6 +438,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // out-of-band edit. A single attempt — on conflict we STOP rather than retry (see below). const docState = await captureState() if (!docState) return // nothing seeded/authoritative to persist yet + if (!(await store.isDocumentGenerationCurrent(name, generation))) return const result = await fetchFileDocPersist(workspaceId, room.fileId, userId, docState, ifMatch) if (result.status === 'missing') return // the file was deleted; nothing to write if (result.status === 'deferred') { @@ -382,23 +454,21 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // here means a task that exits in the moments after a write comes back holding a version older // than the file's, and — since a conflict neither writes nor advances the token — never persists // that document again. One round trip after a blob write is not a cost worth that. - await store.setSyncedVersion(name, result.version) + await store.setSyncedVersion(name, result.version, generation) return } - // status === 'conflict': the durable file advanced out-of-band since our If-Match token. We do NOT - // re-persist against the current stream: an external write commits durable BEFORE its chokepoint merge - // (`mergeEditIntoLiveFileDoc`) reaches the stream, so a re-persist landing in that window would CAS-pass - // with a stream that still lacks the external content and clobber the committed write. Instead leave the - // durable content authoritative — the chokepoint merges the change into the stream and, ONLY once it is - // actually there, advances the synced version (via the merge's own `recordVersion`); a later flush - // (a subsequent debounced persist, or the final flush) then projects the converged stream with a token - // that matches. The session's edits stay in the stream meanwhile. Deliberately do NOT advance the synced - // version here: before the stream reflects the durable content, that would let the next flush clobber it. + /** + * External writes commit before merging into the stream. Retrying or advancing the synced + * version here could overwrite content not yet merged; leave the durable file authoritative + * until the merge advances the version, then let a later flush persist the converged state. + */ logger.warn( `Persist conflict for file ${room.fileId}; durable content advanced out-of-band, left authoritative` ) } catch (error) { - logger.warn(`Persist failed for file ${room.fileId}`, { error: getErrorMessage(error) }) + logger.warn(`Persist failed for file ${room.fileId}`, { + error: getErrorMessage(error), + }) } } @@ -469,7 +539,7 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] { */ function destroyRoomIfIdle(name: string) { const room = fileDocRooms.get(name) - if (!room || room.owners.size > 0 || room.pendingJoins > 0) return + if (!room || room.owners.size > 0 || room.pendingJoins > 0 || room.pendingUpdates > 0) return room.persistDeadline = null if (room.persistTimer) { clearTimeout(room.persistTimer) @@ -484,6 +554,27 @@ function destroyRoomIfIdle(name: string) { fileDocRooms.delete(name) } +/** + * Drop a seeded in-memory generation after an out-of-band durable replacement. It must not flush: the + * durable replacement is newer, and persisting this superseded document would only create a conflict. + * Existing clients are removed from the room before the next join creates and seeds a fresh document. + */ +function discardInvalidatedRoom(name: string, io: Server): void { + const room = fileDocRooms.get(name) + if (!room) return + room.persistDeadline = null + if (room.persistTimer) clearTimeout(room.persistTimer) + room.persistTimer = null + for (const socketId of room.owners.keys()) { + if (socketToRoomName.get(socketId) === name) socketToRoomName.delete(socketId) + io.in(socketId).socketsLeave(name) + } + getFileDocStore().detachRoom(name) + room.awareness.destroy() + room.doc.destroy() + fileDocRooms.delete(name) +} + /** * Flush every open, edited room's converged doc to durable markdown, AWAITING the writes. Called on * graceful shutdown (rolling deploy / scale-in) so edits since the last debounce aren't left only in the @@ -492,18 +583,17 @@ function destroyRoomIfIdle(name: string) { * process is exiting); only their durable state is secured. */ export async function flushAllFileDocRooms(): Promise { + await Promise.all([...pendingFileDocUpdates]) const flushes: Promise[] = [] for (const [name, room] of fileDocRooms) { if (room.edited) flushes.push(flushPersist(name, room, true)) } - await Promise.all(flushes) + await Promise.all([...pendingFileDocPersists, ...flushes]) } /** - * Bring a room's document to its AUTHORITATIVE state — reflecting the file's shared stream and - * carrying its seed — so the join can attach a client to a document that is already whole. Never - * rejects: a room that cannot be seeded is served unseeded, which the client's readiness deadline - * turns into its read-only fallback, exactly as an unreachable relay does. + * Waits for shared hydration and authoritative seeding before joining; failures must not expose + * an editable partial document. */ async function ensureRoomReady( name: string, @@ -513,22 +603,18 @@ async function ensureRoomReady( await room.hydrated // The room can be dropped and re-created while the catch-up is in flight (a fast open→close); the // join re-checks identity after this and abandons a stale room rather than serving from it. - if (fileDocRooms.get(name) !== room || !workspaceId) return + if (fileDocRooms.get(name) !== room) return + if (!workspaceId) throw new Error(`File document ${room.fileId} has no workspace context`) await ensureServerSeed(name, room, workspaceId) + if (fileDocRooms.get(name) === room && !isDocSeeded(room.doc)) { + throw new Error(`File document ${room.fileId} could not be seeded`) + } } /** - * Seed a room's document server-side, once: ask the app to build the seed (the file's current markdown - * → Yjs, through the exact editor engine) and apply it. No client is elected to import content. - * - * MEMOIZED on the room, so concurrent joins await the same seed instead of the second one being served - * an empty document while the first one's fetch is still in flight. Cleared when it settles: a failed - * seed is re-attempted by the next join (a genuinely empty file stays empty and needs no retry). - * - * `isDocSeeded` is the sufficient guard: content only ever reaches the doc alongside the seed flag - * (this seed, or a client's offline fallback), so an unseeded doc is genuinely empty and safe to seed. - * A genuinely empty/missing file returns `null` (a read error throws instead), so still set the flag — - * an empty doc must reach readiness, not wait forever. + * Share one authoritative seed attempt across concurrent joins so none observes an unseeded doc. + * Clear settled attempts to allow retry after transient failures. Existing empty files have a named + * seed; a missing file must fail admission rather than create an editable blank room. */ function ensureServerSeed(name: string, room: FileDocRoom, workspaceId: string): Promise { if (isDocSeeded(room.doc)) return Promise.resolve() @@ -548,6 +634,8 @@ function ensureServerSeed(name: string, room: FileDocRoom, workspaceId: string): */ const SEED_WAIT_RETRY_MS = 150 +class FileDocNotFoundError extends Error {} + async function runServerSeed(name: string, room: FileDocRoom, workspaceId: string): Promise { const store = getFileDocStore() const deadline = Date.now() + FILE_DOC_TIMEOUTS.seedRequestMs @@ -580,32 +668,23 @@ async function seedUnderLock( try { const seed = await fetchFileDocSeed(workspaceId, room.fileId) if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return - // Build the seed (file content + seed flag, or just the flag for an empty/missing file) and write it - // to the shared stream ATOMICALLY, iff the stream is still empty. This — NOT the seed lock — is the - // split-brain guard: two tasks racing (even both past an expired lock) can never both seed, because - // the emptiness check and the append are one Redis-side step. Publish-before-apply: the doc is marked - // seeded (via the local apply) only once the seed is durably in the stream, so a failed write leaves - // the doc unseeded and the stream empty for a clean retry. SEED_ORIGIN keeps `doc.on('update')` from - // re-publishing it. - const seedUpdate = seed?.update ?? emptySeedUpdate() - const didSeed = await store.seedIfEmpty(name, seedUpdate) - // Record the durable version the moment THIS task's seed is in the stream — BEFORE the liveness/ - // seeded guard below. Recording it only now that our seed WON (not from the fetch, before knowing who - // won) keeps it in step with the stream's actual content: a newer own-fetch version could otherwise - // shadow a peer's winning seed and let a later persist clobber an out-of-band edit. But it must not - // sit AFTER the guard: the tailer can integrate our just-appended seed during the await above, so - // `isDocSeeded` may already be true here — an early return would then leave the stream holding seed - // content with NO cluster If-Match token, and later persists would defer and strand session edits. - // Cluster-wide (Redis) so any task's persist reads it; the live room is the single-pod fallback / the - // read-through-cache seed. (No version for an empty/missing file — nothing durable to guard.) - if (didSeed && seed) { + if (!seed) throw new FileDocNotFoundError('File not found') + /** + * Publish before local apply: the atomic empty-stream append, not the expiring lock, prevents + * independent Yjs histories from entering the same room. + */ + const didSeed = await store.seedIfEmpty(name, seed.update, seed.version) + /** + * Record only our winning seed's version before the readiness guard: the tailer may already have + * applied it during the append, but persistence still needs the matching local version. + */ + if (didSeed) { const live = fileDocRooms.get(name) if (live) live.syncedVersion = Math.max(live.syncedVersion ?? 0, seed.version) - void store.setSyncedVersion(name, seed.version) } if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return if (didSeed) { - Y.applyUpdate(room.doc, seedUpdate, SEED_ORIGIN) + Y.applyUpdate(room.doc, seed.update, SEED_ORIGIN) } else { // A peer won the atomic append: we must NOT apply our own — a second, different-client-id seed IS // the split-brain. Read THEIRS out of the stream instead of waiting for the tailer to deliver it, @@ -613,24 +692,13 @@ async function seedUnderLock( await store.catchUp(name) } } catch (error) { + if (error instanceof FileDocNotFoundError) throw error logger.warn(`Server seed failed for file ${room.fileId} (workspace ${workspaceId})`, error) } finally { await store.releaseSeedLock(name, token) } } -/** The seed update for an empty/missing file: just the `initialContentLoaded` flag, so an empty doc - * still reaches readiness (and its emptiness is durably shared like any seed). */ -function emptySeedUpdate(): Uint8Array { - const doc = new Y.Doc() - doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) - try { - return Y.encodeStateAsUpdate(doc) - } finally { - doc.destroy() - } -} - /** Serializes live merges per file so overlapping calls never race the same doc (see below). */ const fileDocMergeChains = new Map>() @@ -671,18 +739,50 @@ export function applyMarkdownToLiveFileDoc( order: MergeOrder = {} ): Promise<'applied' | 'no-live-room' | 'merge-unavailable' | 'stale'> { const name = roomName(fileDocRoom(fileId)) + return serializeFileDocMutation(name, () => mergeMarkdownIntoRoom(name, fileId, markdown, order)) +} + +function serializeFileDocMutation(name: string, operation: () => Promise): Promise { const prior = fileDocMergeChains.get(name) ?? Promise.resolve() - // `.catch` so a failed prior merge doesn't reject this one — each merge is independent. - const run = prior.catch(() => {}).then(() => mergeMarkdownIntoRoom(name, fileId, markdown, order)) - fileDocMergeChains.set( - name, - run.finally(() => { + const run = prior + .catch(() => {}) + .then(operation) + .finally(() => { if (fileDocMergeChains.get(name) === run) fileDocMergeChains.delete(name) }) - ) + fileDocMergeChains.set(name, run) return run } +async function acquireFileDocMergeSlot(name: string): Promise { + const store = getFileDocStore() + let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { + await sleep(MERGE_LOCK_RETRY_MS) + token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + } + return token +} + +/** Serializes and version-orders an unsupported durable replacement with live Markdown merges. */ +export function invalidateLiveFileDocument( + fileId: string, + version: number +): Promise<{ status: 'applied'; docId?: string } | { status: 'stale' }> { + const name = roomName(fileDocRoom(fileId)) + return serializeFileDocMutation(name, async () => { + const store = getFileDocStore() + const token = await acquireFileDocMergeSlot(name) + if (!token) throw new Error('Live document invalidation slot is temporarily unavailable') + try { + if ((fileDocRooms.get(name)?.syncedVersion ?? 0) > version) return { status: 'stale' } + return await store.invalidateDocument(name, version) + } finally { + await store.releaseMergeSlot(name, token) + } + }) +} + async function mergeMarkdownIntoRoom( name: string, fileId: string, @@ -695,14 +795,16 @@ async function mergeMarkdownIntoRoom( // in Redis for multi-task, plus this task's room) so the persist If-Match guard treats this write as // synced rather than an out-of-band conflict. AWAITED so the version is durable before the merge lock // releases, so the next lock holder's staleness check (below) reads a consistent value. - const recordVersion = async () => { + const recordVersion = async (generation?: string) => { if (version === undefined) return const room = fileDocRooms.get(name) // Never regress the token: merges/seeds/persists all write it, so a lower value arriving out of // order must not shadow a higher one the doc already incorporates (the Redis side is guarded // identically by SET_VERSION_IF_NEWER_SCRIPT). - if (room) room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) - await store.setSyncedVersion(name, version) + if (room && (generation === undefined || docIdOf(room.doc) === generation)) { + room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) + } + await store.setSyncedVersion(name, version, generation) } // Order this merge on the file's version line, where `current` is the durable version the doc already @@ -723,11 +825,7 @@ async function mergeMarkdownIntoRoom( // always releases (or its lock expires) first and we acquire — never merging against a shared base // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable // file write stands) rather than race. - let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) - for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { - await sleep(MERGE_LOCK_RETRY_MS) - token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) - } + const token = await acquireFileDocMergeSlot(name) if (!token) { logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) return 'merge-unavailable' @@ -738,13 +836,14 @@ async function mergeMarkdownIntoRoom( const shared = await store.getSyncedVersion(name) const current = Math.max(shared ?? 0, fileDocRooms.get(name)?.syncedVersion ?? 0) if (isStale(current)) return 'stale' + const generation = await store.getDocumentGeneration(name) // Defer to an actively-streaming client: it is applying this SAME agent edit into the shared doc // frame-by-frame, so also publishing a whole-document merge here would double-write the content (the // client's private shadow never observes this merge, so it re-inserts what we added → duplication). // Still record the durable version so the persist If-Match stays correct; the client owns the bytes, // and once streaming stops the flag clears and the final durable merge lands as a near-noop. if (await store.isAgentStreaming(name)) { - await recordVersion() + await recordVersion(generation) return 'applied' } // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc @@ -752,11 +851,11 @@ async function mergeMarkdownIntoRoom( // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is // durably in the stream before we release the lock (else the next task would diff a stale base). - const base = await store.getStreamState(name) + const base = await store.getStreamState(name, generation) if (!base) return 'no-live-room' const diff = await fetchFileDocMerge(fileId, base, markdown) - await store.publishAndWait(name, diff) - await recordVersion() + await store.publishAndWait(name, diff, generation) + await recordVersion(generation) return 'applied' } finally { await store.releaseMergeSlot(name, token) @@ -815,6 +914,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { agentStreamingUntil: 0, hydrated, pendingJoins: 0, + pendingUpdates: 0, } // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. fileDocRooms.set(name, room) @@ -839,7 +939,8 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { origin !== REDIS_ORIGIN && origin !== REDIS_SNAPSHOT_ORIGIN && origin !== REDIS_AGENT_ORIGIN && - origin !== SEED_ORIGIN + origin !== SEED_ORIGIN && + !isClientUpdateOrigin(origin) ) getFileDocStore().publish(name, update, origin === AGENT_SYNC_ORIGIN) // A locally-originated agent frame (this task's stream leader) means a client is applying this agent @@ -979,10 +1080,24 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { const bytes = toFileDocBytes(data) if (!bytes) return + if (bytes.byteLength > MAX_LEGACY_FRAME_BYTES) { + logger.warn('Dropping an oversized legacy file-doc frame', { + socketId: socket.id, + bytes: bytes.byteLength, + }) + return + } // A malformed frame from any client must never escape as a process-level // exception; drop it and keep the relay running. try { + if (hasOversizedLegacyUpdate(bytes)) { + logger.warn('Dropping a legacy file-doc update outside the durable stream budget', { + socketId: socket.id, + bytes: bytes.byteLength, + }) + return + } const decoder = decoding.createDecoder(bytes) const messageType = decoding.readVarUint(decoder) @@ -1027,7 +1142,9 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { // owned by this socket. const owned = room.owners.get(socket.id) if (owned === undefined || awarenessUpdateClientIds(update).some((id) => !owned.has(id))) { - logger.warn('Dropping awareness frame for an unowned client id', { socketId: socket.id }) + logger.warn('Dropping awareness frame for an unowned client id', { + socketId: socket.id, + }) return } awarenessProtocol.applyAwarenessUpdate(room.awareness, update, socket.id) @@ -1037,7 +1154,114 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { logger.warn('Unknown file-doc message type', { messageType }) } } catch (error) { - logger.warn('Dropping malformed file-doc frame', { socketId: socket.id, error }) + logger.warn('Dropping malformed file-doc frame', { + socketId: socket.id, + error, + }) + } +} + +async function handleClientUpdate( + socket: AuthenticatedSocket, + io: Server, + data: unknown, + acknowledge: (result: FileDocUpdateAck) => void +): Promise { + const reject = ( + code: Extract['code'], + retryable: boolean, + updateId?: string + ) => acknowledge({ status: 'rejected', code, retryable, updateId }) + + if (typeof data !== 'object' || data === null) { + reject('INVALID_UPDATE', false) + return + } + + const candidate = data as Partial + const update = toFileDocBytes(candidate.update) + if ( + typeof candidate.fileId !== 'string' || + candidate.fileId.length === 0 || + typeof candidate.docId !== 'string' || + candidate.docId.length === 0 || + typeof candidate.updateId !== 'string' || + candidate.updateId.length === 0 || + candidate.updateId.length > MAX_CLIENT_UPDATE_ID_LENGTH || + !update || + update.byteLength === 0 || + update.byteLength > FILE_DOC_LIMITS.updateBytes + ) { + reject('INVALID_UPDATE', false, candidate.updateId) + return + } + + const name = socketToRoomName.get(socket.id) + if (!name || name !== roomName(fileDocRoom(candidate.fileId))) { + reject('NOT_JOINED', true, candidate.updateId) + return + } + const room = fileDocRooms.get(name) + if (!room) { + reject('NOT_JOINED', true, candidate.updateId) + return + } + if (!isFileDocWriteAllowed(socket, io, name)) { + reject('ACCESS_REVOKED', false, candidate.updateId) + return + } + if (docIdOf(room.doc) !== candidate.docId) { + reject('DOCUMENT_REPLACED', false, candidate.updateId) + return + } + + const validationDoc = new Y.Doc() + try { + Y.applyUpdate(validationDoc, update) + } catch (error) { + logger.warn('Dropping malformed acknowledged file-doc update', { + socketId: socket.id, + fileId: candidate.fileId, + updateId: candidate.updateId, + error, + }) + reject('INVALID_UPDATE', false, candidate.updateId) + return + } finally { + validationDoc.destroy() + } + + const editor = room.owners.get(socket.id)?.values().next().value?.userId + if (editor) room.lastEditorUserId = editor + room.pendingUpdates += 1 + try { + const store = getFileDocStore() + await store.publishClientUpdateAndWait(name, candidate.updateId, update, candidate.docId) + if ( + !(await store.isDocumentGenerationCurrent(name, candidate.docId)) || + fileDocRooms.get(name) !== room + ) { + throw new FileDocInvalidatedError() + } + Y.applyUpdate(room.doc, update, clientUpdateOrigin(socket.id)) + room.edited = true + schedulePersist(name, room) + acknowledge({ status: 'accepted', updateId: candidate.updateId }) + } catch (error) { + if (error instanceof FileDocInvalidatedError) { + reject('DOCUMENT_REPLACED', false, candidate.updateId) + return + } + logger.error('Failed to accept acknowledged file-doc update', { + socketId: socket.id, + fileId: candidate.fileId, + updateId: candidate.updateId, + error, + }) + reject('TEMPORARY_FAILURE', true, candidate.updateId) + } finally { + room.pendingUpdates -= 1 + destroyRoomIfIdle(name) } } @@ -1102,11 +1326,15 @@ export function setupWorkspaceFileDocHandlers( // awaiting authorization can't complete after the client left and register a ghost owner. A // leave for a DIFFERENT file must NOT cancel it (a document switch), mirroring workspace-files. let currentFileId: string | null = null + /** Co-mounted providers share invalidation membership until their last admission settles. */ + const pendingMemberships = new Map() - socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { + socket.on(FILE_DOC_EVENTS.JOIN, async (payload: JoinFileDocPayload) => { + const { fileId, clientId } = payload // Hoisted so the catch can tell whether this join was superseded (a switch to another file) // before surfacing a retryable error for the abandoned one. let generation: number | undefined + let registered = false try { const userId = socket.userId const userName = socket.userName @@ -1142,6 +1370,17 @@ export function setupWorkspaceFileDocHandlers( emitJoinError(socket, fileId, clientId, 'Invalid join payload', 'INVALID_PAYLOAD', false) return } + if ((payload.schemaVersion ?? FILE_DOC_LEGACY_SCHEMA_VERSION) !== FILE_DOC_SCHEMA_VERSION) { + emitJoinError( + socket, + fileId, + clientId, + 'This document version is not supported', + 'SCHEMA_VERSION_MISMATCH', + false + ) + return + } // A generation represents the socket's intended FILE, not an individual provider. Co-mounted // providers for the same file must be allowed to join concurrently; switching files advances the @@ -1156,6 +1395,7 @@ export function setupWorkspaceFileDocHandlers( const room = fileDocRoom(fileId) const name = roomName(room) + const admissionName = fileDocAdmissionRoom(fileId) const authorized = await resolveRoomJoinAuth({ userId, @@ -1177,6 +1417,17 @@ export function setupWorkspaceFileDocHandlers( // awareness). Resolved here so the generation guard below also covers this await. const avatarUrl = await resolveAvatarUrl(socket, userId) + const store = getFileDocStore() + const existing = fileDocRooms.get(name) + if ( + existing && + isDocSeeded(existing.doc) && + !(await store.isDocumentGenerationCurrent(name, docIdOf(existing.doc))) && + fileDocRooms.get(name) === existing + ) { + discardInvalidatedRoom(name, io) + } + const entry = getOrCreateRoom(io, room) // The workspace the server-side persist writes back to — and what the seed is built from, so it // must be captured BEFORE the room is prepared below. @@ -1185,6 +1436,25 @@ export function setupWorkspaceFileDocHandlers( // Hold the room open across the awaits below: it has no owner until this join commits, so a // concurrent last-leave would otherwise tear down the very document being prepared. entry.pendingJoins += 1 + let subscribed = false + const isCurrentJoin = () => + !socket.disconnected && + joinGeneration.get(socket.id) === generation && + fileDocRooms.get(name) === entry + const canRegisterJoin = () => { + if (!isCurrentJoin()) return false + const permission = peekRoomPermission(userId, room) + if (satisfiesRoomMembership(permission ?? null, ROOM_TYPES.WORKSPACE_FILE_DOC)) return true + emitJoinError( + socket, + fileId, + clientId, + 'File access changed while joining', + permission === undefined ? 'JOIN_FAILED' : 'ACCESS_DENIED', + permission === undefined + ) + return false + } try { // A client is attached to a WHOLE document or to nothing. A room assembles itself from the // shared stream and the server seed, and both land in the same Y.Doc that fans every update out @@ -1209,18 +1479,40 @@ export function setupWorkspaceFileDocHandlers( emitJoinError(socket, fileId, clientId, 'Access denied to file', 'ACCESS_DENIED', false) return } - - // Abort a JOIN superseded while the room was being prepared: the socket disconnected, a newer - // JOIN (a document switch) bumped the generation, or the room was dropped and re-created. - // Registering here would leak a dead socket's room, bind the socket to the wrong document, or - // attach it to a doc no longer registered. Last await before the commit, so nothing can - // interleave between the access re-check above and the registration below. - if ( - socket.disconnected || - joinGeneration.get(socket.id) !== generation || - fileDocRooms.get(name) !== entry - ) + if (!isCurrentJoin()) return + + /** + * Watch invalidations before checking the generation, including broadcasts from another + * replica. Pending clients must not receive document or presence frames before authorization. + */ + pendingMemberships.set(name, (pendingMemberships.get(name) ?? 0) + 1) + subscribed = true + await socket.join(admissionName) + const joinedVersion = + Math.max(entry.syncedVersion ?? 0, (await store.getSyncedVersion(name)) ?? 0) || undefined + /** Adapter membership can wait; resolve access again before checking the final generation. */ + const finalPermission = await resolveCurrentRoomPermission(userId, room, FILE_DOC_ACTION) + if (!satisfiesRoomMembership(finalPermission, ROOM_TYPES.WORKSPACE_FILE_DOC)) { + emitJoinError(socket, fileId, clientId, 'Access denied to file', 'ACCESS_DENIED', false) return + } + const currentDocument = await store.isDocumentGenerationCurrent(name, docIdOf(entry.doc)) + if (!isCurrentJoin()) return + if (!currentDocument) { + emitJoinError( + socket, + fileId, + clientId, + 'Document changed while joining', + 'JOIN_FAILED', + true + ) + return + } + if (!canRegisterJoin()) return + await socket.join(name) + /** Adapter joins can wait; recheck access and liveness before ownership or synchronization. */ + if (!canRegisterJoin()) return // A client id must be owned by at most one user, or a peer could bind an active // collaborator's id and pass the per-frame ownership check to spoof/clear its caret. @@ -1282,7 +1574,7 @@ export function setupWorkspaceFileDocHandlers( } clientMap.set(clientId, { clientId, userId, userName, avatarUrl }) socketToRoomName.set(socket.id, name) - socket.join(name) + registered = true // Attribution for the server-side persist, refreshed to the actual editor on each edit in // `handleMessage`. @@ -1295,6 +1587,9 @@ export function setupWorkspaceFileDocHandlers( fileId, clientId, docId: docIdOf(entry.doc), + version: joinedVersion, + schemaVersion: FILE_DOC_SCHEMA_VERSION, + ...(store.enabled ? { acknowledgedUpdates: true as const } : {}), }) // Server-authenticated roster → everyone in the room, including this joiner. broadcastFileDocPresence(io, name, entry) @@ -1324,19 +1619,28 @@ export function setupWorkspaceFileDocHandlers( // A join that returned without registering may have left behind the room it created; drop it // if nothing else claimed it. A no-op once this join committed (the room then has an owner). destroyRoomIfIdle(name) + if (subscribed) { + const remaining = (pendingMemberships.get(name) ?? 1) - 1 + if (remaining > 0) pendingMemberships.set(name, remaining) + else { + pendingMemberships.delete(name) + await socket.leave(admissionName) + if (socketToRoomName.get(socket.id) !== name) await socket.leave(name) + } + } } } catch (error) { logger.error('Error joining file-doc room:', error) try { const name = roomName(fileDocRoom(fileId)) - socket.leave(name) - // Roll back ONLY this join's target room. cleanupFileDocForSocket keys off socketToRoomName, - // which — if the join failed before rebinding to the target (e.g. a switch that threw during - // client-id reclaim) — still points at the socket's PRIOR, valid document. Running it then - // would tear down a document the socket is validly in. So only run it when the binding - // already points at the target; otherwise the socket never registered as an owner of this - // room and the only leftover is a freshly-created empty room, dropped below. - if (socketToRoomName.get(socket.id) === name) cleanupFileDocForSocket(socket.id, io) + /** + * Roll back ownership only if this attempt committed it. A failed provisional admission must + * preserve a previous file's binding and any co-mounted provider already in the target room. + */ + if (registered && socketToRoomName.get(socket.id) === name) { + socket.leave(name) + cleanupFileDocForSocket(socket.id, io) + } destroyRoomIfIdle(name) } catch {} // Suppress the client-facing error when this join was already superseded (a switch to another @@ -1348,12 +1652,27 @@ export function setupWorkspaceFileDocHandlers( (generation !== undefined && joinGeneration.get(socket.id) !== generation) ) return - emitJoinError(socket, fileId, clientId, 'Failed to join file document', 'JOIN_FAILED', true) + if (error instanceof FileDocNotFoundError) { + emitJoinError(socket, fileId, clientId, 'File not found', 'NOT_FOUND', false) + } else { + emitJoinError(socket, fileId, clientId, 'Failed to join file document', 'JOIN_FAILED', true) + } } }) socket.on(FILE_DOC_EVENTS.MESSAGE, (data: unknown) => handleMessage(socket, io, data)) + socket.on( + FILE_DOC_EVENTS.UPDATE, + (data: unknown, acknowledge?: (result: FileDocUpdateAck) => void) => { + if (typeof acknowledge !== 'function') return + const pending = handleClientUpdate(socket, io, data, acknowledge) + .catch((error) => logger.error('Unhandled acknowledged file-doc update failure:', error)) + .finally(() => pendingFileDocUpdates.delete(pending)) + pendingFileDocUpdates.add(pending) + } + ) + socket.on(FILE_DOC_EVENTS.LEAVE, (payload?: LeaveFileDocPayload) => { try { // Cancel an in-flight join whose file the client is now leaving (or an unscoped leave): a diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 80663141334..29240c4e656 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -6,6 +6,7 @@ import { createSocketIOServer, shutdownSocketIOAdapter } from '@/config/socket' import { assertSchemaCompatibility } from '@/database/preflight' import { env } from '@/env' import { setupAllHandlers } from '@/handlers' +import { waitForConnectionCleanup } from '@/handlers/connection' import { flushAllFileDocRooms } from '@/handlers/file-doc' import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import { type AuthenticatedSocket, authenticateSocket } from '@/middleware/auth' @@ -121,6 +122,11 @@ async function main() { shuttingDown = true logger.info('Shutting down Socket.IO server...') + const shutdownTimer = setTimeout(() => { + logger.error('Forced shutdown after timeout') + process.exit(1) + }, SHUTDOWN_TIMEOUT_MS) + accessRevalidation.stop() // Flush open collaborative docs to durable markdown BEFORE tearing down Redis/the store — the @@ -132,6 +138,15 @@ async function main() { logger.error('Error flushing collaborative documents on shutdown:', error) } + /** Transport closure permits reconnection; a namespace DISCONNECT intentionally does not. */ + try { + await io.close() + await waitForConnectionCleanup() + await flushAllFileDocRooms() + } catch (error) { + logger.error('Error draining socket connections on shutdown:', error) + } + try { await roomManager.shutdown() logger.info('RoomManager shutdown complete') @@ -151,24 +166,9 @@ async function main() { logger.error('Error during FileDocStore shutdown:', error) } - // Close local client connections so `httpServer.close()` can complete its callback and exit - // gracefully — otherwise open websockets keep it hanging until the forced-exit timer below. - // Local-only: a rolling deploy must not disconnect clients pinned to other pods. - try { - io.local.disconnectSockets(true) - } catch (error) { - logger.error('Error disconnecting sockets on shutdown:', error) - } - - httpServer.close(() => { - logger.info('Socket.IO server closed') - process.exit(0) - }) - - setTimeout(() => { - logger.error('Forced shutdown after timeout') - process.exit(1) - }, SHUTDOWN_TIMEOUT_MS) + clearTimeout(shutdownTimer) + logger.info('Socket.IO server closed') + process.exit(0) } process.on('SIGINT', shutdown) diff --git a/apps/realtime/src/routes/http.test.ts b/apps/realtime/src/routes/http.test.ts index 725341deac9..e2fe3a476d4 100644 --- a/apps/realtime/src/routes/http.test.ts +++ b/apps/realtime/src/routes/http.test.ts @@ -1,6 +1,15 @@ import type { IncomingMessage, ServerResponse } from 'http' import { describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' + +const { mockInvalidateDocument } = vi.hoisted(() => ({ mockInvalidateDocument: vi.fn() })) + +vi.mock('@/handlers/file-doc', () => ({ + applyMarkdownToLiveFileDoc: vi.fn(), + fileDocAdmissionRoom: (fileId: string) => `file-doc-admission:${fileId}`, + invalidateLiveFileDocument: mockInvalidateDocument, +})) + import { createHttpHandler } from '@/routes/http' function createMocks(req: Partial) { @@ -8,9 +17,13 @@ function createMocks(req: Partial) { const writeHead = vi.fn() const end = vi.fn() const logger = { info: vi.fn(), error: vi.fn(), debug: vi.fn(), warn: vi.fn() } + const emit = vi.fn() + const to = vi.fn(() => ({ emit })) const roomManager = { + io: { to }, getTotalActiveConnections: vi.fn().mockResolvedValue(0), isReady: vi.fn().mockReturnValue(true), + emitToRoom: vi.fn(), } as unknown as IRoomManager return { @@ -20,9 +33,27 @@ function createMocks(req: Partial) { setHeader, writeHead, end, + roomManager, + to, + emit, } } +function requestWithBody(url: string, body: unknown): Partial { + const text = JSON.stringify(body) + const request = { + method: 'POST', + url, + headers: { 'x-api-key': 'test-internal-api-secret-at-least-32-chars' }, + on(event: string, callback: (value?: Buffer) => void) { + if (event === 'data') callback(Buffer.from(text)) + if (event === 'end') callback() + return request + }, + } + return request as unknown as Partial +} + describe('createHttpHandler', () => { /** * `/health` is the only route on this server that returns 200 with a body, so @@ -58,4 +89,43 @@ describe('createHttpHandler', () => { expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) }) + + it('invalidates the shared generation before notifying every open editor', async () => { + mockInvalidateDocument.mockResolvedValueOnce({ status: 'applied', docId: 'old-document' }) + const { handler, req, res, writeHead, to, emit } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1', version: 100 }) + ) + + await handler(req, res) + + expect(mockInvalidateDocument).toHaveBeenCalledWith('file-1', 100) + expect(to).toHaveBeenCalledWith(['workspace-file-doc:file-1', 'file-doc-admission:file-1']) + expect(emit).toHaveBeenCalledWith( + 'file-doc-invalidated', + expect.objectContaining({ fileId: 'file-1', version: 100, docId: 'old-document' }) + ) + expect(mockInvalidateDocument.mock.invocationCallOrder[0]).toBeLessThan( + emit.mock.invocationCallOrder[0] + ) + expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) + }) + + it('does not evict editors for a superseded invalidation', async () => { + mockInvalidateDocument.mockResolvedValueOnce({ status: 'stale' }) + const { handler, req, res, end, roomManager, to } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1', version: 100 }) + ) + await handler(req, res) + expect(roomManager.emitToRoom).not.toHaveBeenCalled() + expect(to).not.toHaveBeenCalled() + expect(end).toHaveBeenCalledWith(JSON.stringify({ status: 'stale' })) + }) + + it('requires a durable version for invalidation', async () => { + const { handler, req, res, writeHead } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1' }) + ) + await handler(req, res) + expect(writeHead).toHaveBeenCalledWith(400, { 'Content-Type': 'application/json' }) + }) }) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 19d2401e37e..da26b68aa3b 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,8 +1,13 @@ import type { IncomingMessage, ServerResponse } from 'http' -import { WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { FILE_DOC_EVENTS, type FileDocInvalidated } from '@sim/realtime-protocol/file-doc' +import { ROOM_TYPES, roomName, WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' -import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' +import { + applyMarkdownToLiveFileDoc, + fileDocAdmissionRoom, + invalidateLiveFileDocument, +} from '@/handlers/file-doc' import { type IRoomManager, WorkflowRoomService } from '@/rooms' interface Logger { @@ -207,7 +212,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { version: typeof version === 'number' ? version : undefined, }) res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ applied: result === 'applied' })) + res.end(JSON.stringify({ applied: result === 'applied', status: result })) } catch (error) { logger.error('Error applying copilot edit to live file-doc:', error) sendError(res, 'Failed to apply edit to live document') @@ -215,6 +220,35 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + if (req.method === 'POST' && req.url === '/api/file-doc/invalidate') { + try { + const body = await readRequestBody(req) + const { fileId, version } = JSON.parse(body) + if (!isNonEmptyString(fileId)) return sendError(res, 'Invalid fileId', 400) + if (!Number.isSafeInteger(version) || version <= 0) { + return sendError(res, 'Invalid version', 400) + } + const room = { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: fileId } as const + const result = await invalidateLiveFileDocument(fileId, version) + const payload: FileDocInvalidated = { + fileId, + version, + ...(result.status === 'applied' && result.docId ? { docId: result.docId } : {}), + message: 'This file changed outside the editor. Reload to continue editing.', + } + if (result.status === 'applied') + roomManager.io + .to([roomName(room), fileDocAdmissionRoom(fileId)]) + .emit(FILE_DOC_EVENTS.INVALIDATED, payload) + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ status: result.status })) + } catch (error) { + logger.error('Error invalidating live file-doc:', error) + sendError(res, 'Failed to invalidate live document') + } + return + } + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } diff --git a/apps/sim/app/api/internal/file-doc/seed/route.test.ts b/apps/sim/app/api/internal/file-doc/seed/route.test.ts index f64504f9ded..2f071395033 100644 --- a/apps/sim/app/api/internal/file-doc/seed/route.test.ts +++ b/apps/sim/app/api/internal/file-doc/seed/route.test.ts @@ -19,7 +19,7 @@ vi.mock('@/lib/collab-doc/seed', () => ({ buildFileDocSeed: mockBuildFileDocSeed, })) -import { POST } from './route' +import { POST } from '@/app/api/internal/file-doc/seed/route' function seedRequest(body: unknown) { return createMockRequest('POST', body, { 'x-api-key': 'internal' }) @@ -31,9 +31,7 @@ describe('POST /api/internal/file-doc/seed', () => { mockCheckInternalApiKey.mockReturnValue({ success: true }) }) - // Regression guard for the auth-helper choice: the realtime relay authenticates with - // `x-api-key: INTERNAL_API_SECRET`, so this route MUST gate on `checkInternalApiKey`. Wiring the - // Bearer-JWT-only `checkInternalAuth` (which forbids `x-api-key`) 401s every real seed fetch. + /** The relay authenticates with the shared internal API key, not a Bearer JWT. */ it('401s when the internal api key is rejected, without building a seed', async () => { mockCheckInternalApiKey.mockReturnValue({ success: false }) const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) @@ -43,10 +41,11 @@ describe('POST /api/internal/file-doc/seed', () => { it('returns the seed as base64 for an authorized request', async () => { mockBuildFileDocSeed.mockResolvedValue({ update: new Uint8Array([1, 2, 3, 4]) }) - const res = await POST(seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' })) + const request = seedRequest({ workspaceId: 'ws-1', fileId: 'file-1' }) + const res = await POST(request) expect(res.status).toBe(200) expect((await res.json()).update).toBe(Buffer.from([1, 2, 3, 4]).toString('base64')) - expect(mockBuildFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1') + expect(mockBuildFileDocSeed).toHaveBeenCalledWith('ws-1', 'file-1', request.signal) }) it('returns update:null for a genuinely absent file', async () => { diff --git a/apps/sim/app/api/internal/file-doc/seed/route.ts b/apps/sim/app/api/internal/file-doc/seed/route.ts index aaf051da094..75cc13f3864 100644 --- a/apps/sim/app/api/internal/file-doc/seed/route.ts +++ b/apps/sim/app/api/internal/file-doc/seed/route.ts @@ -25,7 +25,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const { workspaceId, fileId } = parsed.data.body try { - const seed = await buildFileDocSeed(workspaceId, fileId) + const seed = await buildFileDocSeed(workspaceId, fileId, request.signal) return NextResponse.json({ update: seed ? Buffer.from(seed.update).toString('base64') : null, version: seed ? seed.version : null, diff --git a/apps/sim/app/api/webhooks/outbox/process/route.ts b/apps/sim/app/api/webhooks/outbox/process/route.ts index ea6746ff8f8..b4acb132c12 100644 --- a/apps/sim/app/api/webhooks/outbox/process/route.ts +++ b/apps/sim/app/api/webhooks/outbox/process/route.ts @@ -14,6 +14,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { directGrantOutboxHandlers } from '@/lib/invitations/direct-grant' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { workspaceFileLiveDocOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' import { workspaceFileStorageCleanupOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' import { workflowDeploymentOutboxHandlers } from '@/lib/workflows/deployment-outbox' import { invitationMigrationOutboxHandlers } from '@/lib/workspaces/admin-move' @@ -34,6 +35,7 @@ const handlers = { ...invitationMigrationOutboxHandlers, ...directGrantOutboxHandlers, ...knowledgeDocumentProcessingOutboxHandlers, + ...workspaceFileLiveDocOutboxHandlers, ...workspaceFileStorageCleanupOutboxHandlers, ...workflowDeploymentOutboxHandlers, } as const diff --git a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx index 1a4c4db0f15..da99f963646 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx @@ -10,6 +10,7 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/emcn', () => ({ + cn: (...classes: Array) => classes.filter(Boolean).join(' '), Button: ({ children, ...props }: { children: ReactNode } & Record) => ( - ) : undefined - } - /> - {/* Always mounted, reserving its width: rendering it only once there is a +
{ + if (event.key !== 'Escape') return + event.stopPropagation() + if (event.nativeEvent.isComposing || event.keyCode === 229) return + event.preventDefault() + onClose() + }} + className={cn( + 'absolute top-2 right-2 z-[var(--z-dropdown)] flex max-w-[calc(100%_-_1rem)] flex-col gap-1 rounded-lg border border-[var(--border)] bg-[var(--surface-1)] p-1 shadow-medium', + replace && 'w-[min(400px,calc(100%_-_1rem))]' + )} + > +
+ {replace && ( + + )} + onQueryChange(e.target.value)} + onKeyDown={handleKeyDown} + endAdornment={ + query.length > 0 ? ( + + ) : undefined + } + /> + {/* Always mounted, reserving its width: rendering it only once there is a query would resize the bar on the first keystroke, and a live region inserted together with its text is announced unreliably. */} - - {counterContent()} - - - - + + {counterContent()} + + + + +
+ {replace && showReplace && ( +
+ + replace.onChange(event.target.value)} + onKeyDown={(event) => { + if (event.nativeEvent.isComposing || event.keyCode === 229) return + if (event.key === 'Enter' && replace.canReplace) { + event.preventDefault() + replace.onReplace() + } + }} + /> + + +
+ )}
) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 7d420e6da3f..1bc27bcf55c 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -3,25 +3,60 @@ */ import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, + type FileDocUpdateAck, } from '@sim/realtime-protocol/file-doc' +import { update as updateJournalStorage } from 'idb-keyval' +import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' import type { Socket } from 'socket.io-client' import { describe, expect, it, vi } from 'vitest' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' -import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' -import { FileDocProvider } from './file-doc-provider' +import { AGENT_STREAM_ORIGIN } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' + +const journalStorage = vi.hoisted(() => new Map()) + +vi.mock('idb-keyval', () => ({ + get: vi.fn((key: string) => journalStorage.get(key)), + update: vi.fn((key: string, updater: (value: unknown) => unknown) => { + journalStorage.set(key, updater(journalStorage.get(key))) + }), + del: vi.fn((key: string) => { + journalStorage.delete(key) + }), +})) + +const UPDATE_BATCH_TEST_WINDOW_MS = 100 /** A minimal fake Socket.IO client whose server→client events can be fired in tests. */ function createSocket(connected = true) { const listeners = new Map void>>() const emit = vi.fn() + const timeout = vi.fn((delay: number) => ({ + emit( + event: string, + payload: unknown, + acknowledge: (error: Error | null, ack?: FileDocUpdateAck) => void + ) { + const timer = setTimeout(() => acknowledge(new Error('operation has timed out')), delay) + emit(event, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + clearTimeout(timer) + acknowledge(error, ack) + }) + }, + })) const socket = { connected, emit, + timeout, on(event: string, cb: (...args: unknown[]) => void) { let set = listeners.get(event) if (!set) { @@ -39,23 +74,29 @@ function createSocket(connected = true) { if (event === 'disconnect') socket.connected = false for (const cb of listeners.get(event) ?? []) cb(...args) } - return { socket: socket as unknown as Socket, emit, fire } + return { socket: socket as unknown as Socket, emit, fire, timeout } } function createProvider(connected = true) { - const { socket, emit, fire } = createSocket(connected) + const { socket, emit, fire, timeout } = createSocket(connected) const doc = new Y.Doc() const awareness = new awarenessProtocol.Awareness(doc) const provider = new FileDocProvider(socket, 'file-1', doc, awareness) - return { provider, doc, awareness, emit, fire } + return { provider, doc, awareness, emit, fire, timeout } } function acceptJoin( fire: (event: string, ...args: unknown[]) => void, clientId: number, - docId?: string + docId?: string, + acknowledgedUpdates = true ) { - fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1', clientId, docId }) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId, + docId, + acknowledgedUpdates: acknowledgedUpdates ? true : undefined, + }) } /** Messages emitted to the server, decoded to their `{ type, bytes }`. */ @@ -67,12 +108,20 @@ function emittedMessages(emit: ReturnType) { .map(([, payload]) => payload as Uint8Array) } +function syncStep1Frame(doc: Y.Doc): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(encoder, doc) + return encoding.toUint8Array(encoder) +} + describe('FileDocProvider', () => { it('joins immediately with its client id when the socket is already connected', () => { const { doc, emit } = createProvider(true) expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, { fileId: 'file-1', clientId: doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION, }) }) @@ -206,6 +255,17 @@ describe('FileDocProvider', () => { expect(joinError).toHaveBeenCalledTimes(1) }) + it('fails closed when a seeded legacy tab has no identity but the server does', () => { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + emit.mockClear() + + acceptJoin(fire, doc.clientID, 'doc-current') + + expect(emittedMessages(emit)).toHaveLength(0) + expect(provider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED', retryable: false }) + }) + it('syncs when the room holds the document it already has', () => { const { doc, emit, fire } = createProvider(true) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-original') @@ -247,16 +307,1334 @@ describe('FileDocProvider', () => { expect(synced).toHaveBeenCalledWith(true) }) - it('sends local document edits to the server as sync updates', () => { + it('routes local differences through the acknowledged channel instead of the sync handshake', async () => { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getText('default').insert(0, 'local') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc) + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + + const syncReplies = emittedMessages(emit).filter((message) => { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + return decoding.readVarUint(decoder) === syncProtocol.messageYjsSyncStep2 + }) + expect(syncReplies).toHaveLength(0) + await vi.waitFor(() => { + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(true) + }) + const updatePayload = emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE + )?.[1] as { + update: Uint8Array + } + const serverDoc = new Y.Doc() + Y.applyUpdate(serverDoc, updatePayload.update) + expect(serverDoc.getText('default').toString()).toBe('local') + serverDoc.destroy() + provider.destroy() + }) + + it('keeps standard Yjs sync behavior with an older relay during a rolling deployment', () => { const { doc, emit, fire } = createProvider(true) - acceptJoin(fire, doc.clientID) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getText('default').insert(0, 'local') + acceptJoin(fire, doc.clientID, 'doc-1', false) emit.mockClear() - doc.getText('default').insert(0, 'x') + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + doc.getText('default').insert(5, ' edit') const messages = emittedMessages(emit) - expect(messages.length).toBe(1) - expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + expect(messages.length).toBeGreaterThanOrEqual(2) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + }) + + it('does not enable acknowledged updates unless the relay also supplies a document identity', () => { + const { doc, emit, fire } = createProvider(true) + doc.getText('default').insert(0, 'local') + acceptJoin(fire, doc.clientID, undefined, true) + emit.mockClear() + + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + + expect(emittedMessages(emit).length).toBeGreaterThan(0) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + }) + + it('protects only unsent changes when a legacy relay provides no document identity', () => { + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const { provider, doc, awareness, emit, fire } = createProvider(true) + const serverDoc = new Y.Doc() + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + doc.getText('default').insert(0, 'before join') + expect(unloadIsPrevented()).toBe(true) + acceptJoin(fire, doc.clientID, undefined, false) + expect(unloadIsPrevented()).toBe(false) + doc.getText('default').insert(11, ' online') + expect(unloadIsPrevented()).toBe(false) + + fire('disconnect') + doc.getText('default').insert(18, ' and offline') + expect(unloadIsPrevented()).toBe(true) + fire('connect') + acceptJoin(fire, doc.clientID, undefined, false) + emit.mockClear() + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + + for (const message of emittedMessages(emit)) { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + } + expect(serverDoc.getText('default').toString()).toBe('before join online and offline') + expect(unloadIsPrevented()).toBe(true) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.unstubAllGlobals() + } + }) + + it.each(['acknowledged', 'legacy-offline', 'legacy-rejoining'] as const)( + 'recovers an unacknowledged %s edit after restart and clears it only after acceptance', + async (mode) => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const serverDoc = new Y.Doc() + const serverConfig = serverDoc.getMap(FILE_DOC_SEED.configMap) + serverConfig.set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverConfig.set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'base') + + const firstSocket = createSocket(true) + const firstDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(serverDoc)) + const firstProvider = new FileDocProvider( + firstSocket.socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc), + scope + ) + acceptJoin(firstSocket.fire, firstDoc.clientID, 'doc-1', mode === 'acknowledged') + await vi.waitFor(() => expect(emittedMessages(firstSocket.emit).length).toBeGreaterThan(0)) + if (mode !== 'acknowledged') firstSocket.fire('disconnect') + if (mode === 'legacy-rejoining') firstSocket.fire('connect') + firstSocket.emit.mockClear() + firstDoc.getText('default').insert(4, ' local') + const firstJournal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + expect(await firstJournal.load('doc-1')).not.toBeNull() + }) + expect(firstSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( + mode === 'acknowledged' + ) + firstProvider.destroy() + + const secondSocket = createSocket(true) + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + secondSocket.socket, + 'file-1', + secondDoc, + new awarenessProtocol.Awareness(secondDoc), + scope + ) + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + const syncEncoder = encoding.createEncoder() + encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(syncEncoder, serverDoc) + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) + + await vi.waitFor(() => { + expect(secondDoc.getText('default').toString()).toBe('base local') + expect( + secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE) + ).toBe(true) + }) + const updateCall = secondSocket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE + ) + const payload = updateCall?.[1] as { updateId: string } + const acknowledge = updateCall?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + Y.applyUpdate(serverDoc, (updateCall?.[1] as { update: Uint8Array }).update) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load()).resolves.toBeNull() + }) + + vi.useFakeTimers() + try { + secondSocket.fire('disconnect') + secondSocket.fire('connect') + secondSocket.emit.mockClear() + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect( + secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE) + ).toBe(false) + } finally { + vi.useRealTimers() + } + secondProvider.destroy() + firstDoc.destroy() + secondDoc.destroy() + serverDoc.destroy() + } + ) + + it('batches local document edits into the acknowledged update channel', async () => { + const { doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'x') + + await vi.waitFor(() => { + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.UPDATE, + expect.objectContaining({ fileId: 'file-1', docId: 'doc-1' }), + expect.any(Function) + ) + }) + expect(emittedMessages(emit)).toHaveLength(0) + }) + + it('serializes journal flushes so an edit made during storage never becomes stranded', async () => { + vi.useFakeTimers() + const firstSave = Promise.withResolvers<{ + pendingUpdate: Uint8Array + status: 'saved' + }>() + let saveCalls = 0 + let firstPendingUpdate: Uint8Array | null = null + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => { + saveCalls += 1 + if (saveCalls === 1) { + firstPendingUpdate = pendingUpdate + return firstSave.promise + } + return { pendingUpdate, status: 'saved' } + }) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + doc.getText('default').insert(5, ' second') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(save).toHaveBeenCalledOnce() + + firstSave.resolve({ + pendingUpdate: firstPendingUpdate!, + status: 'saved', + }) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(save).toHaveBeenCalledTimes(2) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, save.mock.calls[0][2]) + expect(recovered.getText('default').toString()).toBe('first') + Y.applyUpdate(recovered, save.mock.calls[1][1]) + expect(recovered.getText('default').toString()).toBe('first second') + recovered.destroy() + await vi.advanceTimersByTimeAsync(1_000) + expect(save).toHaveBeenCalledTimes(2) + const firstUpdate = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const firstPayload = firstUpdate?.[1] as { updateId: string } + const acknowledge = firstUpdate?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + + expect(save).toHaveBeenCalledTimes(3) + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength(2) + provider.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } + }) + + it('retries an unacknowledged update with the same idempotency key', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire, timeout } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'kept until acknowledged') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(first).toBeDefined() + expect(timeout).toHaveBeenCalledWith(FILE_DOC_TIMEOUTS.updateAckMs) + + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 2_000) + const updates = emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(updates.length).toBeGreaterThan(1) + expect((updates[1][1] as { updateId: string }).updateId).toBe( + (first?.[1] as { updateId: string }).updateId + ) + provider.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('rejoins before retrying an update rejected because the room membership went stale', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + doc.getText('default').insert(0, 'edit') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const payload = first?.[1] as { updateId: string } + const acknowledge = first?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + + acknowledge(null, { + status: 'rejected', + updateId: payload.updateId, + code: 'NOT_JOINED', + retryable: true, + }) + await vi.advanceTimersByTimeAsync(1_000) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.JOIN)).toBe(true) + + emit.mockClear() + acceptJoin(fire, doc.clientID, 'doc-1') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(true) + provider.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('ignores expired acknowledgements after the provider is destroyed', async () => { + vi.useFakeTimers() + const { provider, doc, awareness, emit, fire, timeout } = createProvider(true) + try { + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + doc.getText('default').insert(0, 'pending') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(timeout).toHaveBeenCalledWith(FILE_DOC_TIMEOUTS.updateAckMs) + + provider.destroy() + emit.mockClear() + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + expect(emit).not.toHaveBeenCalled() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + vi.useRealTimers() + } + }) + + it('preserves pending acknowledged edits across a downgrade without calling legacy sync an acceptance', async () => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const clear = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'clear') + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + const serverDoc = new Y.Doc() + Y.applyUpdate(serverDoc, Y.encodeStateAsUpdate(doc)) + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'pending acknowledged edit') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const firstPayload = first?.[1] as { updateId: string } + expect(first).toBeDefined() + + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + for (const message of emittedMessages(emit)) { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + } + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + + expect(serverDoc.getText('default').toString()).toBe('pending acknowledged edit') + expect(provider.synced).toBe(true) + expect(unloadIsPrevented()).toBe(true) + expect(await journal.load('doc-1')).not.toBeNull() + expect(clear).not.toHaveBeenCalled() + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + const retry = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(retry?.[1]).toMatchObject({ updateId: firstPayload.updateId }) + const acknowledge = retry?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(unloadIsPrevented()).toBe(false) + expect(await journal.load('doc-1')).toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + clear.mockRestore() + vi.unstubAllGlobals() + vi.useRealTimers() + } + }) + + it('protects a recovered pending journal even when the new relay has no acknowledged channel', async () => { + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const recoveredDoc = new Y.Doc() + recoveredDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + recoveredDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + recoveredDoc.getText('default').insert(0, 'recover me') + const update = Y.encodeStateAsUpdate(recoveredDoc) + await journal.save('doc-1', update, update) + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.waitFor(() => expect(doc.getText('default').toString()).toBe('recover me')) + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + expect(event.defaultPrevented).toBe(true) + expect(await journal.load('doc-1')).not.toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + recoveredDoc.destroy() + vi.unstubAllGlobals() + } + }) + + it('journals an edit made while disconnected before page teardown', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + fire('disconnect') + + doc.getText('default').insert(0, 'offline edit') + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load('doc-1')).resolves.not.toBeNull() + }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + }) + + it('preserves pending recovery through page teardown and destroy', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + fire('disconnect') + doc.getText('default').insert(0, 'preserve me') + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => expect(await journal.load('doc-1')).not.toBeNull()) + + ;(provider as unknown as { handlePageHide: () => void }).handlePageHide() + provider.destroy() + + const stored = await journal.load('doc-1') + expect(stored).not.toBeNull() + const recovered = new Y.Doc() + Y.applyUpdate(recovered, stored!.recoverySnapshot!) + Y.applyUpdate(recovered, stored!.pendingUpdate) + expect(recovered.getText('default').toString()).toBe('preserve me') + recovered.destroy() + doc.destroy() + }) + + it.each(['pagehide', 'destroy'] as const)( + 'preserves new edits over malformed recovery on %s before joining', + async (lifecycle) => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const { socket, emit } = createSocket(false) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const snapshot = Y.encodeStateAsUpdate(doc) + const invalid = new Uint8Array([255]) + await journal.save('doc-1', invalid, snapshot) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + const restored = new Y.Doc() + try { + doc.getText('default').insert(0, 'edits before reconnecting') + + if (lifecycle === 'pagehide') browserWindow.dispatchEvent(new Event('pagehide')) + else provider.destroy() + await vi.advanceTimersByTimeAsync(0) + + const recovered = await journal.load('doc-1') + expect(recovered).not.toBeNull() + Y.applyUpdate(restored, recovered!.recoverySnapshot!) + Y.applyUpdate(restored, recovered!.pendingUpdate) + expect(restored.getText('default').toString()).toBe('edits before reconnecting') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + expect([...journalStorage.values()]).toEqual([ + expect.objectContaining({ + documents: expect.arrayContaining([ + expect.objectContaining({ + pendingUpdate: invalid, + recoverySnapshot: snapshot, + quarantined: true, + }), + ]), + }), + ]) + } finally { + provider.destroy() + await vi.advanceTimersByTimeAsync(0) + awareness.destroy() + doc.destroy() + restored.destroy() + vi.useRealTimers() + vi.unstubAllGlobals() + } + } + ) + + it('reopens the current generation without installing an incompatible recovery draft', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + oldDoc.getText('default').insert(0, 'complete draft') + const recoverySnapshot = Y.encodeStateAsUpdate(oldDoc) + const stateVector = Y.encodeStateVector(oldDoc) + oldDoc.getText('default').insert('complete draft'.length, ' plus pending') + const pendingUpdate = Y.encodeStateAsUpdate(oldDoc, stateVector) + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'old-doc', + pendingUpdate, + recoverySnapshot + ) + for (let mount = 0; mount < 3; mount++) { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + acceptJoin(fire, doc.clientID, 'current-doc') + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + awareness.destroy() + doc.destroy() + } + const retained = await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load( + 'old-doc' + ) + expect(retained?.pendingUpdate).toEqual(pendingUpdate) + expect(retained?.recoverySnapshot).toEqual(recoverySnapshot) + oldDoc.destroy() + }) + + it.each([false, true])( + 'does not install disk recovery without a negotiated identity (local identity: %s)', + async (hasLocalIdentity) => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const draft = new Y.Doc() + draft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + draft.getText('default').insert(0, 'retained draft') + const snapshot = Y.encodeStateAsUpdate(draft) + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await journal.save('old-doc', snapshot, snapshot) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + if (hasLocalIdentity) Y.applyUpdate(doc, snapshot) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + acceptJoin(fire, doc.clientID, undefined, false) + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe(hasLocalIdentity ? 'retained draft' : '') + expect(await journal.load('old-doc')).not.toBeNull() + provider.destroy() + awareness.destroy() + doc.destroy() + draft.destroy() + } + ) + + it('admits the final online batch before leaving while its relay publication is still pending', () => { + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness) + acceptJoin(fire, doc.clientID, 'doc-1') + let joined = true + const publications: Array<() => void> = [] + emit.mockImplementation((event, payload, acknowledge) => { + if (event === FILE_DOC_EVENTS.LEAVE) joined = false + if (event !== FILE_DOC_EVENTS.UPDATE) return + expect(joined).toBe(true) + publications.push(() => { + Y.applyUpdate(serverDoc, payload.update) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + }) + }) + + doc.getText('default').insert(0, 'last edit') + provider.destroy() + + expect(joined).toBe(false) + expect(publications).toHaveLength(1) + expect(serverDoc.getText('default').toString()).toBe('') + publications[0]() + expect(serverDoc.getText('default').toString()).toBe('last edit') + awareness.destroy() + doc.destroy() + serverDoc.destroy() + }) + + it('does not send an oversized aggregate batch during teardown', () => { + const { provider, doc, awareness, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + const bytes = new Uint8Array(FILE_DOC_LIMITS.updateBytes / 2) + doc.getArray('binary').insert(0, [bytes]) + doc.getArray('binary').insert(1, [bytes]) + emit.mockClear() + + provider.destroy() + + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + awareness.destroy() + doc.destroy() + }) + + it.each(['destroy', 'file-switch'] as const)( + 'drains an edit being journaled and later edits before %s', + async (navigation) => { + vi.useFakeTimers() + journalStorage.clear() + const storageGate = Promise.withResolvers() + vi.mocked(updateJournalStorage).mockImplementationOnce(async (key, updater) => { + await storageGate.promise + journalStorage.set(String(key), updater(journalStorage.get(String(key)))) + }) + const clear = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'clear') + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + const nextDoc = new Y.Doc() + const nextAwareness = new awarenessProtocol.Awareness(nextDoc) + let nextProvider: FileDocProvider | undefined + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + doc.getText('default').insert(5, ' second') + emit.mockClear() + if (navigation === 'file-switch') { + nextProvider = new FileDocProvider(socket, 'file-2', nextDoc, nextAwareness, scope) + } + provider.destroy() + awareness.destroy() + doc.destroy() + + const updateIndex = emit.mock.calls.findIndex(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const membershipEvent = + navigation === 'file-switch' ? FILE_DOC_EVENTS.JOIN : FILE_DOC_EVENTS.LEAVE + const membershipIndex = emit.mock.calls.findIndex(([event]) => event === membershipEvent) + expect(updateIndex).toBeGreaterThanOrEqual(0) + expect(updateIndex).toBeLessThan(membershipIndex) + const [, payload, acknowledge] = emit.mock.calls[updateIndex] + Y.applyUpdate(serverDoc, payload.update) + expect(serverDoc.getText('default').toString()).toBe('first second') + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + expect(clear).not.toHaveBeenCalled() + storageGate.resolve() + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(clear).toHaveBeenCalledOnce() + expect( + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-1') + ).toBeNull() + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength( + 1 + ) + } finally { + storageGate.resolve() + provider.destroy() + nextProvider?.destroy() + awareness.destroy() + doc.destroy() + nextAwareness.destroy() + nextDoc.destroy() + serverDoc.destroy() + clear.mockRestore() + vi.useRealTimers() + } + } + ) + + it.each(['accepted', 'rejected', 'timeout'] as const)( + 'retains the final combined draft until its own %s acknowledgment', + async (outcome) => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(first).toBeDefined() + doc.getText('default').insert(5, ' second') + provider.destroy() + const updates = emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(updates).toHaveLength(2) + const [, finalPayload, finalAcknowledge] = updates[1] + expect(finalPayload.updateId).not.toBe(first?.[1].updateId) + Y.applyUpdate(serverDoc, finalPayload.update) + expect(serverDoc.getText('default').toString()).toBe('first second') + first?.[2](null, { status: 'accepted', updateId: first[1].updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(await journal.load('doc-1')).not.toBeNull() + if (outcome === 'accepted') { + finalAcknowledge(null, { status: 'accepted', updateId: finalPayload.updateId }) + } else if (outcome === 'rejected') { + finalAcknowledge(null, { + status: 'rejected', + updateId: finalPayload.updateId, + retryable: false, + code: 'ACCESS_REVOKED', + }) + } + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + const recovery = await journal.load('doc-1') + if (outcome === 'accepted') expect(recovery).toBeNull() + else expect(recovery?.pendingUpdate).toEqual(finalPayload.update) + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength( + 2 + ) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.useRealTimers() + } + } + ) + + it.each(['disconnected', 'invalidated', 'not-joined'] as const)( + 'keeps the final draft local when %s', + async (state) => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + if (state !== 'not-joined') { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + } + doc.getText('default').insert(0, 'retained draft') + if (state === 'disconnected') fire('disconnect') + if (state === 'invalidated') { + fire(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1', message: 'Replaced' }) + } + emit.mockClear() + provider.destroy() + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + expect(emittedMessages(emit)).toHaveLength(0) + expect( + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-1') + ).not.toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + vi.useRealTimers() + } + } + ) + + it('delivers a rejoined legacy batch before leaving without treating it as acknowledged', async () => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + fire('disconnect') + doc.getText('default').insert(0, 'legacy draft') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + provider.destroy() + const frame = emittedMessages(emit)[0] + const decoder = decoding.createDecoder(frame) + expect(decoding.readVarUint(decoder)).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + expect(serverDoc.getText('default').toString()).toBe('legacy draft') + expect( + emit.mock.calls.findIndex(([event]) => event === FILE_DOC_EVENTS.MESSAGE) + ).toBeLessThan(emit.mock.calls.findIndex(([event]) => event === FILE_DOC_EVENTS.LEAVE)) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect( + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-1') + ).not.toBeNull() + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.useRealTimers() + } + }) + + it('never falls back to a different document identity when loading local recovery', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-old') + oldDoc.getText('default').insert(0, 'old draft') + const oldSnapshot = Y.encodeStateAsUpdate(oldDoc) + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'doc-old', + oldSnapshot, + oldSnapshot + ) + + const { socket, fire } = createSocket(true) + const currentDoc = new Y.Doc() + currentDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-current') + currentDoc.getText('default').insert(0, 'current content') + const provider = new FileDocProvider( + socket, + 'file-1', + currentDoc, + new awarenessProtocol.Awareness(currentDoc), + scope + ) + acceptJoin(fire, currentDoc.clientID, 'doc-current') + + await vi.waitFor(() => expect(provider.joinError).toBeNull()) + expect(currentDoc.getText('default').toString()).toBe('current content') + await expect( + new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-old') + ).resolves.not.toBeNull() + provider.destroy() + currentDoc.destroy() + oldDoc.destroy() + }) + + it('recovers the negotiated generation even when an incompatible draft is newer', async () => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const currentDraft = new Y.Doc() + currentDraft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'current-doc') + currentDraft.getText('default').insert(0, 'current draft') + const currentSnapshot = Y.encodeStateAsUpdate(currentDraft) + await journal.save('current-doc', currentSnapshot, currentSnapshot) + await vi.advanceTimersByTimeAsync(1) + const oldDraft = new Y.Doc() + oldDraft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + oldDraft.getText('default').insert(0, 'incompatible draft') + const oldSnapshot = Y.encodeStateAsUpdate(oldDraft) + await journal.save('old-doc', oldSnapshot, oldSnapshot) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'current-doc') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('current draft') + const update = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(update).toBeDefined() + update?.[2](null, { status: 'accepted', updateId: update[1].updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(await journal.load('current-doc')).toBeNull() + expect((await journal.load('old-doc'))?.recoverySnapshot).toEqual(oldSnapshot) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + currentDraft.destroy() + oldDraft.destroy() + vi.useRealTimers() + } + }) + + it('rejects an in-memory generation mismatch before applying a matching server draft', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const serverDraft = new Y.Doc() + serverDraft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'server-doc') + serverDraft.getText('default').insert(0, 'server draft') + const snapshot = Y.encodeStateAsUpdate(serverDraft) + await journal.save('server-doc', snapshot, snapshot) + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'local-doc') + doc.getText('default').insert(0, 'local draft') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + acceptJoin(fire, doc.clientID, 'server-doc') + await vi.waitFor(() => expect(provider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED' })) + expect(doc.getText('default').toString()).toBe('local draft') + expect((await journal.load('server-doc'))?.recoverySnapshot).toEqual(snapshot) + provider.destroy() + awareness.destroy() + doc.destroy() + serverDraft.destroy() + }) + + it('syncs after malformed recovery without replaying it on subsequent mounts', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getText('default').insert(0, 'quarantined snapshot') + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'doc-1', + new Uint8Array([255]), + Y.encodeStateAsUpdate(oldDoc) + ) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'server content') + const frame = encoding.createEncoder() + encoding.writeVarUint(frame, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(frame, serverDoc) + for (let mount = 0; mount < 2; mount++) { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(frame)) + + expect(provider.joinError).toBeNull() + expect(provider.synced).toBe(true) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(doc.getText('default').toString()).toBe('server content') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + doc.destroy() + } + oldDoc.destroy() + serverDoc.destroy() + }) + + it('ignores an obsolete schema rejection when recovery finishes after reconnecting', async () => { + const recovery = Promise.withResolvers() + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(recovery.promise) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, { + workspaceId: 'workspace-1', + userId: 'user-1', + }) + try { + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'doc-1', + schemaVersion: FILE_DOC_SCHEMA_VERSION + 1, + }) + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + recovery.resolve(null) + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + load.mockRestore() + } + }) + + it('fails closed when hydration buffers more than its bounded message count', async () => { + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(new Promise(() => {})) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + const message = syncStep1Frame(new Y.Doc()) + + for (let index = 0; index < 129; index += 1) { + fire(FILE_DOC_EVENTS.MESSAGE, message) + } + + expect(provider.joinError).toMatchObject({ code: 'HYDRATION_BUFFER_OVERFLOW' }) + provider.destroy() + load.mockRestore() + }) + + it('fails closed when hydration buffers more than its bounded byte budget', () => { + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(new Promise(() => {})) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + + fire(FILE_DOC_EVENTS.MESSAGE, new Uint8Array(FILE_DOC_LIMITS.updateBytes * 2 + 1)) + + expect(provider.joinError).toMatchObject({ code: 'HYDRATION_BUFFER_OVERFLOW' }) + provider.destroy() + doc.destroy() + load.mockRestore() + }) + + it('makes an older different-file provider terminal before unscoped frames can cross documents', async () => { + const { socket, emit, fire } = createSocket(true) + const firstDoc = new Y.Doc() + const firstProvider = new FileDocProvider( + socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc) + ) + acceptJoin(fire, firstDoc.clientID) + + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + socket, + 'file-2', + secondDoc, + new awarenessProtocol.Awareness(secondDoc) + ) + expect(firstProvider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED' }) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-2', + clientId: secondDoc.clientID, + acknowledgedUpdates: true, + }) + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + + const remote = new Y.Doc() + remote.getText('default').insert(0, 'second-file content') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(remote)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(firstDoc.getText('default').toString()).toBe('') + expect(secondDoc.getText('default').toString()).toBe('second-file content') + firstProvider.destroy() + secondProvider.destroy() + firstDoc.destroy() + secondDoc.destroy() + remote.destroy() + }) + + it.each(['acknowledged', 'legacy-offline'] as const)( + 'stops editing when the complete %s recovery snapshot cannot be persisted', + async (mode) => { + vi.useFakeTimers() + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ + pendingUpdate, + status: 'limit-exceeded', + })) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1', mode === 'acknowledged') + await vi.advanceTimersByTimeAsync(0) + if (mode === 'legacy-offline') fire('disconnect') + emit.mockClear() + + doc.getText('default').insert(0, 'must remain visible') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + + expect(provider.joinError).toMatchObject({ code: 'PENDING_UPDATE_LIMIT' }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + doc.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } + } + ) + + it.each(['saved', 'unavailable'] as const)( + 'warns before unloading pending edits and continues acknowledged saves when storage is %s', + async (status) => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ pendingUpdate, status })) + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, { + workspaceId: 'workspace-1', + userId: 'user-1', + }) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + expect(unloadIsPrevented()).toBe(false) + + doc.getText('default').insert(0, 'pending edit') + expect(unloadIsPrevented()).toBe(true) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(provider.joinError).toBeNull() + const update = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(update).toBeDefined() + const payload = update?.[1] as { updateId: string } + const acknowledge = update?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + expect(unloadIsPrevented()).toBe(true) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + expect(unloadIsPrevented()).toBe(false) + + doc.getText('default').insert(0, 'another ') + expect(unloadIsPrevented()).toBe(true) + provider.destroy() + expect(unloadIsPrevented()).toBe(false) + awareness.destroy() + doc.destroy() + } finally { + save.mockRestore() + vi.unstubAllGlobals() + vi.useRealTimers() + } + } + ) + + it('keeps retrying sync without fatally timing out a previously healthy reconnect', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + acceptJoin(fire, doc.clientID, 'doc-1') + const serverDoc = new Y.Doc() + const config = serverDoc.getMap(FILE_DOC_SEED.configMap) + config.set(FILE_DOC_SEED.docIdKey, 'doc-1') + config.set(FILE_DOC_SEED.flag, true) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + emit.mockClear() + fire('disconnect') + fire('connect') + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.JOIN)).toHaveLength(1) + acceptJoin(fire, doc.clientID, 'doc-1') + + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + expect(provider.joinError).toBeNull() + expect(emittedMessages(emit).length).toBeGreaterThan(1) + provider.destroy() + serverDoc.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('retries an accepted sync handshake that never receives a response', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + acceptJoin(fire, doc.clientID) + emit.mockClear() + + await vi.advanceTimersByTimeAsync(6_000) + + expect(emittedMessages(emit).length).toBeGreaterThan(1) + expect(provider.joinError).toBeNull() + provider.destroy() + } finally { + vi.useRealTimers() + } }) it('tags agent-streamed edits as SYNC_NO_PERSIST so the relay skips the durable persist', () => { @@ -359,6 +1737,31 @@ describe('FileDocProvider', () => { expect(provider.joinError).toEqual(error) }) + it('becomes terminal when a durable replacement invalidates its document generation', () => { + const { provider, doc, emit, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + message: 'This file changed outside the editor. Reload to continue editing.', + }) + + expect(provider.joinError).toMatchObject({ + code: 'DOCUMENT_REPLACED', + retryable: false, + }) + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'DOCUMENT_REPLACED', retryable: false }) + ) + + emit.mockClear() + fire('connect') + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(doc.getText('default').toString()).toBe('') + }) + it('scopes join errors to the matching provider on a shared socket', () => { const { socket, fire } = createSocket(true) const firstDoc = new Y.Doc() @@ -624,28 +2027,38 @@ describe('FileDocProvider', () => { expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.LEAVE, { fileId: 'file-b' }) }) - it('gives up with a non-retryable join-error when the first sync never arrives (offline)', () => { + it('keeps an unseeded document retryable after the readiness deadline and accepts late server content', () => { vi.useFakeTimers() try { - const { provider, emit, fire } = createProvider(false) // socket never connects + const { provider, doc, emit, fire } = createProvider(false) const onError = vi.fn() provider.on('join-error', onError) vi.advanceTimersByTime(12_000) - // Surfaces the same non-retryable rejection the fatal path uses, so the editor falls back to - // showing the file read-only. expect(onError).toHaveBeenCalledWith( - expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: true }) ) expect(provider.joinError).toEqual( - expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: true }) ) - // Latched fatal: a later connect must not re-join (which could sync server state in and - // duplicate the locally-seeded content). emit.mockClear() fire('connect') - expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBeUndefined() + acceptJoin(fire, doc.clientID) + const remote = new Y.Doc() + remote.getText('default').insert(0, 'authoritative body') + remote.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('authoritative body') + provider.destroy() + remote.destroy() } finally { vi.useRealTimers() } @@ -698,7 +2111,7 @@ describe('FileDocProvider', () => { // The readiness deadline still fires → the editor falls back to the stored content read-only, // and `synced` is dropped so the `synced && seeded` gate stays closed (read-only, not editable). expect(onError).toHaveBeenCalledWith( - expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: true }) ) expect(provider.synced).toBe(false) } finally { @@ -706,30 +2119,93 @@ describe('FileDocProvider', () => { } }) - it('ignores a late SyncStep2 that arrives after the readiness deadline (no merge, stays gated)', () => { + it('accepts a late authoritative SyncStep2 after the readiness deadline without a local fallback seed', () => { vi.useFakeTimers() try { const { provider, doc, fire } = createProvider(true) acceptJoin(fire, doc.clientID) - // Deadline lapses with no first sync → fatal fallback (editor falls back to a read-only seed). vi.advanceTimersByTime(12_000) expect(provider.joinError).toEqual(expect.objectContaining({ code: 'READINESS_TIMEOUT' })) - // A delayed SyncStep2 finally arrives. Applying it would merge server content into the - // already-seeded doc (duplication) and flip synced→true (un-gating autosave), so it MUST be - // dropped once fatal. const remote = new Y.Doc() remote.getText('default').insert(0, 'server content') + remote.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeSyncStep2(encoder, remote) fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) - expect(provider.synced).toBe(false) - expect(doc.getText('default').toString()).toBe('') + expect(provider.synced).toBe(true) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('server content') + provider.destroy() + remote.destroy() } finally { vi.useRealTimers() } }) + + it.each(['old-document', undefined])( + 'ignores delayed invalidation of %s after a newer authoritative JOIN', + (invalidatedDocId) => { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'new-document', + version: 30, + }) + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + docId: invalidatedDocId, + version: 20, + message: 'Old replacement', + }) + expect(provider.joinError).toBeNull() + provider.destroy() + } + ) + + it('keeps matching-generation invalidation terminal even when its version equals JOIN', () => { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'current-document', + version: 20, + }) + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + docId: 'current-document', + version: 20, + message: 'Current replacement', + }) + expect(provider.joinError?.code).toBe('DOCUMENT_REPLACED') + provider.destroy() + }) + + it('does not let a newer tombstone notification spare an older joined document', () => { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'current-document', + version: 10, + }) + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + version: 30, + message: 'Second unsupported replacement', + }) + expect(provider.joinError?.code).toBe('DOCUMENT_REPLACED') + provider.destroy() + }) + + it('fails closed on invalidation before reliable JOIN metadata exists', () => { + const { provider, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1', version: 20, message: 'Replacement' }) + expect(provider.joinError?.code).toBe('DOCUMENT_REPLACED') + provider.destroy() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index b3a04d50b96..17c3e07ad13 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -4,14 +4,20 @@ import { } from '@sim/realtime-protocol/events' import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, FILE_DOC_TIMEOUTS, + type FileDocInvalidated, + type FileDocUpdateAck, + type FileDocUpdatePayload, type JoinFileDocError, type JoinFileDocSuccess, toFileDocBytes, } from '@sim/realtime-protocol/file-doc' import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { generateShortId } from '@sim/utils/id' import { backoffWithJitter } from '@sim/utils/retry' import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' @@ -19,8 +25,9 @@ import { ObservableV2 } from 'lib0/observable' import type { Socket } from 'socket.io-client' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' -import type * as Y from 'yjs' -import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' +import * as Y from 'yjs' +import { AGENT_STREAM_ORIGIN } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' /** * Events emitted by {@link FileDocProvider}. @@ -33,21 +40,34 @@ interface FileDocProviderEvents { } /** - * How long to wait to reach a USABLE editor — connected, synced, AND seeded (`initialContentLoaded` - * set by the server seed) — before giving up. It guards two failure modes with one timer: - * - the realtime server is unreachable, so the first sync never arrives; and - * - the socket syncs an empty doc but the server-side seed never lands (its build persistently fails - * / exhausts its retries), which `synced` alone would wrongly treat as "connected, all good". - * - * On the deadline the provider latches fatal and surfaces a non-retryable `join-error` — the exact - * path a fatal rejection uses — so the editor falls back to showing the file's stored content - * read-only instead of a permanently blank pane. Generous enough to clear a slow connect + seed - * round-trip; a healthy cold open reaches readiness well within it. Shared with (and must exceed) the - * relay's seed-fetch timeout — see `FILE_DOC_TIMEOUTS` and its ordering test. + * Report delayed connection or seeding without abandoning recovery. The stored-content preview + * stays separate from the authoritative Y.Doc, preventing duplicate content on a late sync. + * Must outlast the relay's seed-fetch timeout; see FILE_DOC_TIMEOUTS and its ordering test. */ const READINESS_DEADLINE_MS = FILE_DOC_TIMEOUTS.readinessDeadlineMs const JOIN_RETRY_BASE_MS = 500 const JOIN_RETRY_MAX_MS = 5_000 +const UPDATE_BATCH_MS = 50 +const UPDATE_RETRY_BASE_MS = 250 +const UPDATE_RETRY_MAX_MS = 5_000 +const MAX_HYDRATION_MESSAGES = 128 +const MAX_HYDRATION_BYTES = FILE_DOC_LIMITS.updateBytes * 2 +const RECOVERY_ORIGIN = Symbol('file-doc-recovery') + +function hasYjsUpdateContent(update: Uint8Array): boolean { + const decoded = Y.decodeUpdate(update) + return decoded.structs.length > 0 || decoded.ds.clients.size > 0 +} + +interface FileDocProviderScope { + workspaceId: string + userId: string +} + +interface PendingClientUpdate { + updateId: string + update: Uint8Array +} /** * Live-provider counts per file, per shared socket. Two surfaces in one tab (the Files editor and the @@ -101,11 +121,18 @@ function releaseRoomMembership(socket: Socket, fileId: string): boolean { * reconnect) without discarding local edits. */ export class FileDocProvider extends ObservableV2 { + /** Socket.IO carries unscoped Yjs frames, so opening a different file terminalizes providers for the + * previous file; multiple providers for the same file may coexist. */ + private static readonly activeProviders = new WeakMap< + Socket, + { fileId: string; providers: Set } + >() + synced = false /** - * The latched non-retryable join rejection, or `null`. The `join-error` event is - * transient and can fire before a consumer subscribes, - * so consumers read this on subscription to detect a fatal failure they missed. + * The current readiness failure, or `null`. Retryable timeouts clear once authoritative sync + * completes; terminal rejections remain latched. Consumers read it when subscribing so an earlier + * event is not missed. */ joinError: JoinFileDocError | null = null @@ -116,18 +143,47 @@ export class FileDocProvider extends ObservableV2 { /** Deadline for reaching readiness (synced + seeded); fires the fallback if it is never reached. */ private readinessTimer: ReturnType | null = null private joinAccepted = false + private joinedDocument: Pick | null = null + private updateMode: 'negotiating' | 'legacy' | 'acknowledged' = 'negotiating' private joinPending = false private joinRetryAttempt = 0 private joinRetryTimer: ReturnType | null = null + private joinAckTimer: ReturnType | null = null + private syncRetryTimer: ReturnType | null = null + private syncRetryAttempt = 0 + private joinHydrating = false + private connectionGeneration = 0 + private bufferedMessages: Uint8Array[] = [] + private bufferedMessageBytes = 0 + private pendingUpdateBatch: Uint8Array[] = [] + private inFlightUpdate: PendingClientUpdate | null = null + private updateBatchTimer: ReturnType | null = null + private updateRetryTimer: ReturnType | null = null + private updateRetryAttempt = 0 + private updateFlushInProgress = false + private flushingUpdate: Uint8Array | null = null + private pendingUpdatesDrained = false + private recoveryApplied = false + private recoveryQueued = false + private beforeUnloadProtected = false + private readonly journal: PendingFileDocUpdateJournal | null + private recoveryLoad: { + docId: string + promise: ReturnType + } | null = null constructor( private readonly socket: Socket, private readonly fileId: string, readonly doc: Y.Doc, - readonly awareness: awarenessProtocol.Awareness + readonly awareness: awarenessProtocol.Awareness, + scope?: FileDocProviderScope ) { super() + this.journal = scope ? new PendingFileDocUpdateJournal({ ...scope, fileId: this.fileId }) : null + this.registerActiveProvider() + // Restore an empty local awareness state if it has been cleared. A fresh // Awareness starts with `{}`, but a *reused* one whose local state was removed // (a prior provider's `destroy()` clears it, and so does `Awareness.destroy()`) @@ -143,11 +199,13 @@ export class FileDocProvider extends ObservableV2 { socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + socket.on(FILE_DOC_EVENTS.INVALIDATED, this.handleInvalidated) socket.on(ROOM_ACCESS_REVOKED_EVENT, this.handleAccessRevoked) socket.on('connect', this.handleConnect) socket.on('disconnect', this.handleDisconnect) doc.on('update', this.handleDocUpdate) awareness.on('update', this.handleAwarenessUpdate) + if (typeof window !== 'undefined') window.addEventListener('pagehide', this.handlePageHide) // Watch the seed flag so reaching "seeded" (server seed applied) can clear the readiness deadline. doc.getMap(FILE_DOC_SEED.configMap).observe(this.handleConfigChange) @@ -157,8 +215,7 @@ export class FileDocProvider extends ObservableV2 { if (socket.connected) this.join() - // Arm the fallback: if we don't reach readiness (synced + seeded) before the deadline, give up. - this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + this.armReadinessDeadline() } /** Whether the server seed has recorded the initial content on the doc. */ @@ -168,25 +225,33 @@ export class FileDocProvider extends ObservableV2 { /** Clear the readiness deadline once the editor is usable (synced AND seeded). */ private handleConfigChange = () => { - if (this.synced && this.isSeeded()) this.clearReadinessTimer() + if (this.synced && this.isSeeded()) { + this.clearReadinessTimer() + if (this.joinError?.retryable) this.joinError = null + } + if (this.updateMode === 'acknowledged' && this.docId() && this.pendingUpdateBatch.length > 0) { + this.scheduleUpdateFlush(0) + } } /** * Readiness was never reached within {@link READINESS_DEADLINE_MS} — either the realtime server is - * unreachable (never synced) or it synced but the server-side seed never landed (synced yet - * unseeded). Reset `synced` (so the editor gates read-only), latch fatal (so a late reconnect or - * seed can't sync server state in and merge-duplicate the content the editor is about to render - * locally), and surface a synthetic non-retryable join-error — the exact path a fatal rejection - * uses — so the owner falls back to the read-only view of the file's stored content instead of a - * blank pane. No-op if we already reached readiness, already failed fatally, or were torn down. + * unreachable or its authoritative seed is delayed. Keep retrying with the existing bounded + * join/sync backoff. The owner's stored-content preview must remain separate from this Y.Doc. */ private handleReadinessDeadline = () => { this.readinessTimer = null - if (this.synced && this.isSeeded()) return - // Dropping `synced` (see {@link failFatally}) is what keeps the editor's `synced && seeded` gate - // closed, so the fallback renders the stored content read-only rather than becoming editable on a - // document the server never seeded. - this.failFatally('Realtime document was not ready in time', 'READINESS_TIMEOUT') + if (this.disposed || this.fatal || (this.synced && this.isSeeded())) return + this.joinError = { + fileId: this.fileId, + error: 'Realtime document was not ready in time', + code: 'READINESS_TIMEOUT', + retryable: true, + } + this.setSynced(false) + this.emit('join-error', [this.joinError]) + if (this.joinAccepted) this.scheduleSyncRetry() + else if (!this.joinPending) this.scheduleJoinRetry() } private clearReadinessTimer() { @@ -196,6 +261,11 @@ export class FileDocProvider extends ObservableV2 { } } + private armReadinessDeadline() { + this.clearReadinessTimer() + this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + } + private clearJoinRetryTimer() { if (this.joinRetryTimer !== null) { clearTimeout(this.joinRetryTimer) @@ -203,11 +273,45 @@ export class FileDocProvider extends ObservableV2 { } } + private clearJoinAckTimer() { + if (this.joinAckTimer !== null) { + clearTimeout(this.joinAckTimer) + this.joinAckTimer = null + } + } + + private clearSyncRetryTimer() { + if (this.syncRetryTimer !== null) { + clearTimeout(this.syncRetryTimer) + this.syncRetryTimer = null + } + } + + private clearUpdateTimers() { + if (this.updateBatchTimer !== null) clearTimeout(this.updateBatchTimer) + if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) + this.updateBatchTimer = null + this.updateRetryTimer = null + } + /** Join the room, binding our client id so the server only accepts awareness we own. */ private join = () => { if (this.fatal || this.disposed || !this.socket.connected || this.joinPending) return this.joinPending = true - this.socket.emit(FILE_DOC_EVENTS.JOIN, { fileId: this.fileId, clientId: this.doc.clientID }) + this.clearJoinAckTimer() + this.joinAckTimer = setTimeout(() => { + this.joinAckTimer = null + if (!this.joinPending || this.fatal || this.disposed) return + this.joinPending = false + this.joinAccepted = false + this.setSynced(false) + this.scheduleJoinRetry() + }, FILE_DOC_TIMEOUTS.joinAckMs) + this.socket.emit(FILE_DOC_EVENTS.JOIN, { + fileId: this.fileId, + clientId: this.doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION, + }) } private scheduleJoinRetry() { @@ -232,7 +336,10 @@ export class FileDocProvider extends ObservableV2 { */ private handleConnect = () => { if (this.fatal) return + this.connectionGeneration += 1 this.clearJoinRetryTimer() + this.clearSyncRetryTimer() + this.syncRetryAttempt = 0 this.joinAccepted = false this.joinPending = false this.joinRetryAttempt = 0 @@ -241,8 +348,15 @@ export class FileDocProvider extends ObservableV2 { } private handleDisconnect = () => { + this.connectionGeneration += 1 + this.clearBufferedMessages() this.clearJoinRetryTimer() + this.clearJoinAckTimer() + this.clearSyncRetryTimer() + if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) + this.updateRetryTimer = null this.joinAccepted = false + this.joinHydrating = false this.joinPending = false this.setSynced(false) @@ -268,7 +382,6 @@ export class FileDocProvider extends ObservableV2 { * rebuilt only when the room AND the shared stream are both gone (a tab that slept through it), which * is precisely when a stale tab reconnects. There is no way to un-merge afterwards, so the sync never * happens: take the fatal path, which leaves the editor read-only on the content it already shows. - * A reload binds a fresh document and recovers. */ private handleJoinSuccess = (data: JoinFileDocSuccess) => { if ( @@ -277,20 +390,115 @@ export class FileDocProvider extends ObservableV2 { (data.clientId !== undefined && data.clientId !== this.doc.clientID) ) return + this.clearJoinAckTimer() this.joinPending = false this.joinRetryAttempt = 0 this.clearJoinRetryTimer() + this.joinedDocument = { docId: data.docId, version: data.version } + if (data.acknowledgedUpdates === true && data.docId !== undefined) { + this.updateMode = 'acknowledged' + } + this.joinHydrating = true + const generation = this.connectionGeneration + if (!this.journal || data.docId === undefined) { + this.finishAcceptJoin(data, generation, null) + return + } + const recoveryDocId = data.docId + if (!this.recoveryLoad || this.recoveryLoad.docId !== recoveryDocId) { + this.recoveryLoad = { + docId: recoveryDocId, + promise: this.journal.load(recoveryDocId), + } + } + void this.recoveryLoad.promise.then((recovered) => { + this.finishAcceptJoin(data, generation, recovered) + }) + } + + private finishAcceptJoin( + data: JoinFileDocSuccess, + generation: number, + recovered: Awaited> + ): void { + if ( + this.disposed || + this.fatal || + !this.socket.connected || + generation !== this.connectionGeneration || + !this.joinHydrating + ) + return + + const serverSchemaVersion = data.schemaVersion ?? 1 + if (serverSchemaVersion !== FILE_DOC_SCHEMA_VERSION) { + this.failFatally( + 'This document version is not supported; refresh to continue editing', + 'SCHEMA_VERSION_MISMATCH' + ) + return + } + const local = this.docId() - if (local !== undefined && data.docId !== undefined && data.docId !== local) { + if ( + data.docId !== undefined && + ((local !== undefined && data.docId !== local) || (local === undefined && this.isSeeded())) + ) { + this.failFatally( + 'This document was reloaded on the server; refresh to continue editing', + 'DOCUMENT_REPLACED' + ) + return + } + + if (recovered !== null && !this.recoveryApplied) { + try { + if (recovered.recoverySnapshot) { + Y.applyUpdate(this.doc, recovered.recoverySnapshot, RECOVERY_ORIGIN) + } + Y.applyUpdate(this.doc, recovered.pendingUpdate, RECOVERY_ORIGIN) + } catch { + this.failFatally('The local recovery copy could not be restored.', 'INVALID_UPDATE') + return + } + this.recoveryApplied = true + } + + if ( + recovered !== null && + (data.docId !== recovered.docId || this.docId() !== recovered.docId) + ) { this.failFatally( 'This document was reloaded on the server; refresh to continue editing', 'DOCUMENT_REPLACED' ) return } + + const updateMode = + data.acknowledgedUpdates === true && data.docId !== undefined ? 'acknowledged' : 'legacy' + /** Pre-negotiation deltas stay in Y.Doc for legacy sync; existing recovery is never acknowledged here. */ + if (updateMode === 'legacy' && this.updateMode === 'negotiating') this.pendingUpdateBatch = [] + this.updateMode = updateMode + + if (recovered !== null && !this.recoveryQueued) { + this.queuePendingUpdate(recovered.pendingUpdate) + this.recoveryQueued = true + } + this.updateBeforeUnloadProtection() + + this.joinHydrating = false this.joinAccepted = true this.sendSyncStep1() + this.scheduleSyncRetry() this.sendLocalAwareness() + const bufferedMessages = this.bufferedMessages + this.clearBufferedMessages() + for (const message of bufferedMessages) this.applyMessage(message) + if (this.updateMode === 'acknowledged') { + if (this.inFlightUpdate) this.sendInFlightUpdate() + else if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) + } } /** The identity of the document we hold, once the server seed has named one. */ @@ -313,15 +521,49 @@ export class FileDocProvider extends ObservableV2 { retryable: false, } this.fatal = true + this.clearBufferedMessages() this.joinError = error + void this.persistPendingSnapshot() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearUpdateTimers() + this.clearSyncRetryTimer() this.joinAccepted = false this.joinPending = false + this.joinHydrating = false + this.clearJoinAckTimer() this.setSynced(false) this.emit('join-error', [error]) } + private registerActiveProvider(): void { + const active = FileDocProvider.activeProviders.get(this.socket) + if (active?.fileId === this.fileId) { + active.providers.add(this) + return + } + if (active) { + for (const provider of active.providers) { + provider.drainPendingUpdates() + provider.failFatally( + 'Another file was opened in this tab. Reload this file to resume editing it.', + 'DOCUMENT_REPLACED' + ) + } + } + FileDocProvider.activeProviders.set(this.socket, { + fileId: this.fileId, + providers: new Set([this]), + }) + } + + private unregisterActiveProvider(): void { + const active = FileDocProvider.activeProviders.get(this.socket) + if (active?.fileId !== this.fileId) return + active.providers.delete(this) + if (active.providers.size === 0) FileDocProvider.activeProviders.delete(this.socket) + } + /** * Handle a join rejection. A non-retryable rejection (access denied, invalid) * won't succeed on retry, so latch {@link fatal} to stop (re)joining and let the @@ -336,11 +578,16 @@ export class FileDocProvider extends ObservableV2 { return this.joinAccepted = false this.joinPending = false + this.joinHydrating = false + this.clearJoinAckTimer() if (data.retryable === false) { this.fatal = true this.joinError = data + void this.persistPendingSnapshot() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearUpdateTimers() + this.clearSyncRetryTimer() this.setSynced(false) } else { this.setSynced(false) @@ -362,14 +609,46 @@ export class FileDocProvider extends ObservableV2 { this.failFatally(data.message, 'ACCESS_REVOKED') } + private handleInvalidated = (data: FileDocInvalidated) => { + if (data.fileId !== this.fileId) return + const joined = this.joinedDocument + if (data.docId && joined?.docId && data.docId !== joined.docId) return + if ( + !data.docId && + data.version !== undefined && + joined?.version !== undefined && + data.version < joined.version + ) + return + this.failFatally(data.message, 'DOCUMENT_REPLACED') + } + private handleMessage = (data: unknown) => { - // Once we've given up (a non-retryable rejection, or the connect deadline lapsed and the editor - // fell back to a read-only local seed), ignore ALL inbound frames. A late SyncStep2 arriving - // after the deadline would otherwise merge the server's state into the already-seeded doc — - // duplicating content — and flip `synced` true, which un-gates autosave and would persist the - // duplicate back to the real file. `fatal` guarding (re)join alone is not enough; it must also - // stop applying sync here. - if (this.fatal || !this.joinAccepted) return + /** A terminal authorization or generation failure must never accept late document frames. */ + if (this.fatal) return + if (this.joinHydrating) { + const bytes = toFileDocBytes(data) + if (!bytes) return + if ( + this.bufferedMessages.length >= MAX_HYDRATION_MESSAGES || + this.bufferedMessageBytes + bytes.byteLength > MAX_HYDRATION_BYTES + ) { + this.failFatally( + 'Realtime document hydration exceeded its safety limit', + 'HYDRATION_BUFFER_OVERFLOW' + ) + return + } + const buffered = new Uint8Array(bytes) + this.bufferedMessages.push(buffered) + this.bufferedMessageBytes += buffered.byteLength + return + } + if (!this.joinAccepted) return + this.applyMessage(data) + } + + private applyMessage(data: unknown) { const bytes = toFileDocBytes(data) if (!bytes) return @@ -384,7 +663,19 @@ export class FileDocProvider extends ObservableV2 { // re-sending updates we just applied from the server. const syncType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) if (encoding.length(encoder) > 1) { - this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + const response = encoding.toUint8Array(encoder) + if (this.updateMode === 'acknowledged' && syncType === syncProtocol.messageYjsSyncStep1) { + const responseDecoder = decoding.createDecoder(response) + decoding.readVarUint(responseDecoder) + decoding.readVarUint(responseDecoder) + const update = new Uint8Array(decoding.readVarUint8Array(responseDecoder)) + if (hasYjsUpdateContent(update)) { + this.queuePendingUpdate(update) + this.scheduleUpdateFlush(0) + } + } else { + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, response) + } } if (syncType === syncProtocol.messageYjsSyncStep2 && !this.synced) this.setSynced(true) break @@ -401,26 +692,278 @@ export class FileDocProvider extends ObservableV2 { } private handleDocUpdate = (update: Uint8Array, origin: unknown) => { - // Once fatal (a non-retryable rejection, or the readiness deadline lapsed), the editor may render - // the stored content into the doc locally as its read-only fallback. Never relay those local - // writes — the server never seeded this doc, so echoing them would push unseeded content to peers - // (and each fallen-back client would do so, union-duplicating). A fatal client is fully local. - if (this.fatal || !this.joinAccepted || !this.socket.connected) return - // Updates we applied from the server carry `this` as origin — don't echo them. - if (origin === this) return + /** A terminal document cannot publish; inbound and recovery updates must not echo. */ + if (this.fatal || origin === this || origin === RECOVERY_ORIGIN) return // Agent-streamed frames must reach peers (so a collaborator sees the stream live) but must NOT be // treated by the server as a durable user edit — the copilot's final `edit_content` write is the // authoritative persist. Tag them so the relay applies + fans out but skips persist bookkeeping. - const messageType = - origin === AGENT_STREAM_ORIGIN - ? FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST - : FILE_DOC_MESSAGE_TYPE.SYNC + if (origin === AGENT_STREAM_ORIGIN) { + if (!this.joinAccepted || !this.socket.connected) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + return + } + + if (!this.joinAccepted || !this.socket.connected) { + this.queuePendingUpdate(update) + if (this.updateMode !== 'negotiating') this.scheduleUpdateFlush(UPDATE_BATCH_MS) + return + } + + if (this.updateMode !== 'acknowledged') { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + return + } + + this.queuePendingUpdate(update) + this.scheduleUpdateFlush(UPDATE_BATCH_MS) + } + + private queuePendingUpdate(update: Uint8Array): void { + this.pendingUpdateBatch.push(update) + this.updateBeforeUnloadProtection() + } + + private scheduleUpdateFlush(delay: number) { + if (this.updateBatchTimer !== null || this.updateFlushInProgress || this.disposed || this.fatal) + return + this.updateBatchTimer = setTimeout(() => { + this.updateBatchTimer = null + void this.flushPendingUpdates() + }, delay) + } + + private async flushPendingUpdates(): Promise { + if ( + this.updateMode === 'negotiating' || + this.pendingUpdateBatch.length === 0 || + this.disposed || + this.fatal + ) + return + const docId = this.docId() + if (!docId) return + + this.updateFlushInProgress = true + let hasUnjournaledUpdates = false + try { + const update = Y.mergeUpdates(this.pendingUpdateBatch) + this.flushingUpdate = update + this.pendingUpdateBatch = [] + const journalUpdate = this.inFlightUpdate + ? Y.mergeUpdates([this.inFlightUpdate.update, update]) + : update + const saved = await this.journal?.save(docId, journalUpdate, Y.encodeStateAsUpdate(this.doc)) + hasUnjournaledUpdates = this.pendingUpdateBatch.length > 0 + if (this.pendingUpdatesDrained) return + if (this.disposed || this.fatal) { + this.queuePendingUpdate(update) + return + } + if (saved?.status === 'limit-exceeded') { + this.queuePendingUpdate(update) + this.failFatally('Local edits exceeded the safe recovery limit.', 'PENDING_UPDATE_LIMIT') + return + } + const durableUpdate = saved?.pendingUpdate ?? update + + if (this.inFlightUpdate) { + this.queuePendingUpdate(update) + return + } + this.inFlightUpdate = { updateId: generateShortId(), update: durableUpdate } + this.updateRetryAttempt = 0 + this.sendInFlightUpdate() + } finally { + this.flushingUpdate = null + this.updateFlushInProgress = false + this.updateBeforeUnloadProtection() + if (this.pendingUpdateBatch.length > 0 && (hasUnjournaledUpdates || !this.inFlightUpdate)) { + this.scheduleUpdateFlush(0) + } + } + } + + private sendInFlightUpdate() { + const pending = this.inFlightUpdate + const docId = this.docId() + if ( + !pending || + !docId || + this.updateMode !== 'acknowledged' || + this.disposed || + this.fatal || + !this.socket.connected || + !this.joinAccepted + ) + return + + const generation = this.connectionGeneration + const payload: FileDocUpdatePayload = { + fileId: this.fileId, + docId, + updateId: pending.updateId, + update: pending.update, + } + this.socket + .timeout(FILE_DOC_TIMEOUTS.updateAckMs) + .emit(FILE_DOC_EVENTS.UPDATE, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + if (this.disposed || this.fatal || this.inFlightUpdate !== pending) return + if (error) { + if (generation === this.connectionGeneration) this.scheduleUpdateRetry() + return + } + if (ack) this.handleUpdateAck(ack) + }) + } + + private handleUpdateAck(ack: FileDocUpdateAck) { + const pending = this.inFlightUpdate + if (!pending || ack.updateId !== pending.updateId || this.disposed || this.fatal) return + + if (ack.status === 'accepted') { + const docId = this.docId() + this.inFlightUpdate = null + this.updateRetryAttempt = 0 + this.updateBeforeUnloadProtection() + if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) + else if (!this.updateFlushInProgress && docId) void this.journal?.clear(docId, pending.update) + return + } + + if (!ack.retryable) { + const message = + ack.code === 'ACCESS_REVOKED' + ? 'Your access to this document has been revoked' + : 'This document changed while this tab was disconnected; refresh to continue editing' + this.failFatally(message, ack.code) + return + } + if (ack.code === 'NOT_JOINED') { + this.setSynced(false) + this.joinAccepted = false + this.joinPending = false + this.clearSyncRetryTimer() + this.scheduleJoinRetry() + return + } + this.scheduleUpdateRetry() + } + + private scheduleUpdateRetry() { + if (this.updateRetryTimer !== null || this.disposed || this.fatal || !this.socket.connected) + return + this.updateRetryAttempt += 1 + this.updateRetryTimer = setTimeout( + () => { + this.updateRetryTimer = null + this.sendInFlightUpdate() + }, + backoffWithJitter(this.updateRetryAttempt, null, { + baseMs: UPDATE_RETRY_BASE_MS, + maxMs: UPDATE_RETRY_MAX_MS, + }) + ) + } + + private pendingJournalUpdate(): Uint8Array | null { + const updates = [ + ...(this.inFlightUpdate ? [this.inFlightUpdate.update] : []), + ...(this.flushingUpdate ? [this.flushingUpdate] : []), + ...this.pendingUpdateBatch, + ] + return updates.length > 0 ? Y.mergeUpdates(updates) : null + } + + private persistPendingSnapshot(): Promise | undefined { + const update = this.pendingJournalUpdate() + const docId = this.docId() + if (!update || !docId || !this.journal) return + return this.journal.save(docId, update, Y.encodeStateAsUpdate(this.doc)).then(() => undefined) + } + + /** + * Transfer the final batch before LEAVE or a different file's JOIN changes socket membership. + * The relay admits UPDATE synchronously and pins its room until publication finishes. Only the + * immutable bytes and journal survive teardown; an acceptance clears them after all queued saves. + */ + private drainPendingUpdates(): void { + if ( + this.disposed || + this.fatal || + this.pendingUpdatesDrained || + !this.joinAccepted || + !this.socket.connected + ) + return + const update = this.pendingJournalUpdate() + if (!update || update.byteLength > FILE_DOC_LIMITS.updateBytes) return + const docId = this.docId() + if (this.updateMode === 'acknowledged' && !docId) return + + const updateId = + this.inFlightUpdate && !this.flushingUpdate && this.pendingUpdateBatch.length === 0 + ? this.inFlightUpdate.updateId + : generateShortId() + const journal = this.journal + const snapshotSaved = this.persistPendingSnapshot() + this.pendingUpdatesDrained = true + this.pendingUpdateBatch = [] + this.inFlightUpdate = null + this.flushingUpdate = null + this.clearUpdateTimers() + + if (this.updateMode === 'acknowledged' && docId) { + const payload: FileDocUpdatePayload = { fileId: this.fileId, docId, updateId, update } + this.socket + .timeout(FILE_DOC_TIMEOUTS.updateAckMs) + .emit(FILE_DOC_EVENTS.UPDATE, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + if (error || ack?.status !== 'accepted' || ack.updateId !== updateId) return + if (journal && snapshotSaved) { + void snapshotSaved.then(() => journal.clear(docId, update)) + } + }) + return + } + const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageType) + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeUpdate(encoder, update) this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } + private handlePageHide = () => { + void this.persistPendingSnapshot() + } + + private handleBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault() + event.returnValue = '' + } + + private updateBeforeUnloadProtection(): void { + if (typeof window === 'undefined') return + const shouldProtect = + !this.disposed && + (this.pendingUpdateBatch.length > 0 || + this.inFlightUpdate !== null || + this.updateFlushInProgress) + if (shouldProtect === this.beforeUnloadProtected) return + this.beforeUnloadProtected = shouldProtect + if (shouldProtect) window.addEventListener('beforeunload', this.handleBeforeUnload) + else window.removeEventListener('beforeunload', this.handleBeforeUnload) + } + + private clearBufferedMessages(): void { + this.bufferedMessages = [] + this.bufferedMessageBytes = 0 + } + private handleAwarenessUpdate = ( { added, updated, removed }: { added: number[]; updated: number[]; removed: number[] }, origin: unknown @@ -452,6 +995,25 @@ export class FileDocProvider extends ObservableV2 { this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } + private scheduleSyncRetry() { + this.clearSyncRetryTimer() + if (this.synced || this.fatal || this.disposed || !this.socket.connected || !this.joinAccepted) + return + this.syncRetryAttempt += 1 + this.syncRetryTimer = setTimeout( + () => { + this.syncRetryTimer = null + if (this.synced || this.fatal || this.disposed || !this.joinAccepted) return + this.sendSyncStep1() + this.scheduleSyncRetry() + }, + backoffWithJitter(this.syncRetryAttempt, null, { + baseMs: 1_000, + maxMs: JOIN_RETRY_MAX_MS, + }) + ) + } + private sendLocalAwareness() { if (this.awareness.getLocalState() === null) return const encoder = encoding.createEncoder() @@ -466,9 +1028,16 @@ export class FileDocProvider extends ObservableV2 { private setSynced(synced: boolean) { if (this.synced === synced) return this.synced = synced + if (synced) { + this.clearSyncRetryTimer() + this.syncRetryAttempt = 0 + } // Readiness needs synced AND seeded; only clear the deadline when both hold (the seed may have // arrived first, or may still be pending — `handleConfigChange` clears it if seeded arrives later). - if (synced && this.isSeeded()) this.clearReadinessTimer() + if (synced && this.isSeeded()) { + this.clearReadinessTimer() + if (this.joinError?.retryable) this.joinError = null + } this.emit('synced', [synced]) } @@ -482,9 +1051,17 @@ export class FileDocProvider extends ObservableV2 { super.destroy() return } + this.drainPendingUpdates() + void this.persistPendingSnapshot() this.disposed = true + this.updateBeforeUnloadProtection() + this.unregisterActiveProvider() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearJoinAckTimer() + this.clearSyncRetryTimer() + this.clearUpdateTimers() + this.clearBufferedMessages() this.joinPending = false // Publish our final awareness removal while this provider is still admitted. A co-mounted sibling @@ -500,12 +1077,14 @@ export class FileDocProvider extends ObservableV2 { this.socket.off(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) this.socket.off(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) this.socket.off(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + this.socket.off(FILE_DOC_EVENTS.INVALIDATED, this.handleInvalidated) this.socket.off(ROOM_ACCESS_REVOKED_EVENT, this.handleAccessRevoked) this.socket.off('connect', this.handleConnect) this.socket.off('disconnect', this.handleDisconnect) this.doc.off('update', this.handleDocUpdate) this.doc.getMap(FILE_DOC_SEED.configMap).unobserve(this.handleConfigChange) this.awareness.off('update', this.handleAwarenessUpdate) + if (typeof window !== 'undefined') window.removeEventListener('pagehide', this.handlePageHide) super.destroy() } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts new file mode 100644 index 00000000000..f9b553f4f06 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts @@ -0,0 +1,427 @@ +/** + * @vitest-environment node + */ +import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' +import { get, update as updateValue } from 'idb-keyval' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +const storage = vi.hoisted(() => new Map()) + +vi.mock('idb-keyval', () => ({ + get: vi.fn(async (key: string) => storage.get(key)), + update: vi.fn((key: string, updater: (value: unknown) => unknown) => { + storage.set(key, updater(storage.get(key))) + }), +})) + +import { + type PendingDocumentRecovery, + PendingFileDocUpdateJournal, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' + +function journal(): PendingFileDocUpdateJournal { + return new PendingFileDocUpdateJournal({ + workspaceId: 'workspace-1', + fileId: 'file-1', + userId: 'user-1', + }) +} + +function updateWith(text: string): Uint8Array { + const doc = new Y.Doc() + doc.getText('body').insert(0, text) + return Y.encodeStateAsUpdate(doc) +} + +describe('PendingFileDocUpdateJournal', () => { + beforeEach(() => { + storage.clear() + vi.mocked(updateValue) + .mockReset() + .mockImplementation(async (key, updater) => { + storage.set(String(key), updater(storage.get(String(key)))) + }) + }) + + it('stores a full recovery snapshot separately from the pending wire update', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + const recoverySnapshot = updateWith('complete local draft') + + await subject.save('doc-1', pendingUpdate, recoverySnapshot) + + await expect(subject.load('doc-1')).resolves.toEqual( + expect.objectContaining({ docId: 'doc-1', pendingUpdate, recoverySnapshot }) + ) + }) + + it('reports when the current full recovery snapshot cannot be stored', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + + const result = await subject.save( + 'doc-1', + pendingUpdate, + new Uint8Array(FILE_DOC_LIMITS.updateBytes * 2 + 1) + ) + + expect(result).toMatchObject({ status: 'limit-exceeded' }) + }) + + it('loads an existing draft without requiring a writable transaction', async () => { + const subject = journal() + const pendingUpdate = updateWith('recoverable draft') + await subject.save('doc-1', pendingUpdate, pendingUpdate) + const writes = vi.mocked(updateValue).mock.calls.length + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Read-only storage')) + + await expect(subject.load('doc-1')).resolves.toMatchObject({ docId: 'doc-1', pendingUpdate }) + expect(updateValue).toHaveBeenCalledTimes(writes) + }) + + it.each(['pendingUpdate', 'recoverySnapshot'] as const)( + 'isolates malformed %s bytes without replaying or deleting them', + async (field) => { + const subject = journal() + const valid = updateWith('preserved snapshot') + const invalid = new Uint8Array([255]) + const pending = field === 'pendingUpdate' ? invalid : valid + const snapshot = field === 'recoverySnapshot' ? invalid : valid + await subject.save('doc-1', pending, snapshot) + + await expect(subject.load('doc-1')).resolves.toBeNull() + await expect(journal().load()).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [ + expect.objectContaining({ + docId: 'doc-1', + pendingUpdate: pending, + recoverySnapshot: snapshot, + quarantined: true, + }), + ], + }), + ]) + + const newUpdate = updateWith('new edits') + await expect(subject.save('doc-1', newUpdate, newUpdate)).resolves.toMatchObject({ + status: 'saved', + pendingUpdate: newUpdate, + }) + await expect(subject.load('doc-1')).resolves.toMatchObject({ pendingUpdate: newUpdate }) + await subject.clear('doc-1', newUpdate) + await expect(subject.load()).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [expect.objectContaining({ pendingUpdate: pending, quarantined: true })], + }), + ]) + } + ) + + it('ignores malformed recovery even if browser storage cannot be updated', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const before = structuredClone([...storage.values()]) + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) + + await expect(subject.load()).resolves.toBeNull() + expect([...storage.values()]).toEqual(before) + }) + + it.each(['pendingUpdate', 'recoverySnapshot'] as const)( + 'saves new edits before loading a malformed existing %s without deleting the original', + async (field) => { + vi.useFakeTimers() + const restored = new Y.Doc() + try { + const subject = journal() + const valid = updateWith('recoverable original half') + const invalid = new Uint8Array([255]) + await subject.save( + 'doc-1', + field === 'pendingUpdate' ? invalid : valid, + field === 'recoverySnapshot' ? invalid : valid + ) + const original = ( + structuredClone([...storage.values()][0]) as { documents: PendingDocumentRecovery[] } + ).documents[0] + await vi.advanceTimersByTimeAsync(1_000) + const current = updateWith('current edits') + + await expect(subject.save('doc-1', current, current)).resolves.toEqual({ + status: 'saved', + pendingUpdate: current, + }) + await expect(journal().load('doc-1')).resolves.toMatchObject({ pendingUpdate: current }) + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [ + expect.objectContaining({ pendingUpdate: current }), + { ...original, quarantined: true }, + ], + }), + ]) + + const peer = journal() + const later = updateWith('later peer edits') + const combined = await peer.save('doc-1', later, later) + expect(combined.status).toBe('saved') + const recovery = await subject.load('doc-1') + Y.applyUpdate(restored, recovery!.recoverySnapshot!) + Y.applyUpdate(restored, recovery!.pendingUpdate) + expect(restored.getText('body').toString()).toContain('current edits') + expect(restored.getText('body').toString()).toContain('later peer edits') + + await peer.clear('doc-1', combined.pendingUpdate) + await expect(subject.load('doc-1')).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ documents: [{ ...original, quarantined: true }] }), + ]) + } finally { + restored.destroy() + vi.useRealTimers() + } + } + ) + + it.each(['pendingUpdate', 'recoverySnapshot'] as const)( + 'preserves valid existing recovery when the incoming %s is malformed', + async (field) => { + const subject = journal() + const existing = updateWith('existing valid history') + await subject.save('doc-1', existing, existing) + const before = structuredClone([...storage.values()]) + const current = updateWith('current edits') + const invalid = new Uint8Array([255]) + + await expect( + subject.save( + 'doc-1', + field === 'pendingUpdate' ? invalid : current, + field === 'recoverySnapshot' ? invalid : current + ) + ).resolves.toMatchObject({ status: 'unavailable' }) + + expect([...storage.values()]).toEqual(before) + await expect(subject.load('doc-1')).resolves.toMatchObject({ pendingUpdate: existing }) + } + ) + + it('does not replace malformed existing recovery with malformed incoming recovery', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const before = structuredClone([...storage.values()]) + + await expect(subject.save('doc-1', invalid, invalid)).resolves.toMatchObject({ + status: 'unavailable', + }) + + expect([...storage.values()]).toEqual(before) + }) + + it('prioritizes valid records when isolating malformed recovery during a save', async () => { + const subject = journal() + const valid = updateWith('valid recovery') + await subject.save('first', valid, valid) + await subject.save('second', valid, valid) + await subject.save('current', new Uint8Array([255]), valid) + + await expect(subject.save('current', valid, valid)).resolves.toMatchObject({ status: 'saved' }) + + for (const docId of ['first', 'second', 'current']) { + await expect(subject.load(docId)).resolves.toMatchObject({ docId }) + } + expect((storage.values().next().value as { documents: unknown[] }).documents).toHaveLength(3) + }) + + it('does not quarantine a record that another tab replaced after the read', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const stale = structuredClone([...storage.values()][0]) + storage.clear() + const valid = updateWith('concurrent valid edits') + await subject.save('doc-1', valid, valid) + vi.mocked(get).mockResolvedValueOnce(stale) + + await expect(subject.load()).resolves.toBeNull() + await expect(subject.load()).resolves.toMatchObject({ pendingUpdate: valid }) + }) + + it('prioritizes valid recovery within the existing record cap', async () => { + const subject = journal() + const valid = updateWith('valid') + await subject.save('first', valid, valid) + const invalid = new Uint8Array([255]) + await subject.save('invalid', invalid, invalid) + await expect(subject.load('invalid')).resolves.toBeNull() + await subject.save('second', valid, valid) + await subject.save('third', valid, valid) + + for (const docId of ['first', 'second', 'third']) { + await expect(subject.load(docId)).resolves.toMatchObject({ docId }) + } + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: expect.arrayContaining([ + expect.objectContaining({ docId: 'first' }), + expect.objectContaining({ docId: 'second' }), + expect.objectContaining({ docId: 'third' }), + ]), + }), + ]) + expect((storage.values().next().value as { documents: unknown[] }).documents).toHaveLength(3) + }) + + it('does not extend malformed recovery retention while quarantining it', async () => { + vi.useFakeTimers() + try { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('invalid', invalid, invalid) + await vi.advanceTimersByTimeAsync(6 * 24 * 60 * 60 * 1_000) + await expect(subject.load()).resolves.toBeNull() + await vi.advanceTimersByTimeAsync(2 * 24 * 60 * 60 * 1_000) + const valid = updateWith('new edits') + await subject.save('current', valid, valid) + + expect([...storage.values()]).toEqual([ + expect.objectContaining({ documents: [expect.objectContaining({ docId: 'current' })] }), + ]) + } finally { + vi.useRealTimers() + } + }) + + it('distinguishes unavailable browser storage from a configured size limit', async () => { + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) + const pendingUpdate = updateWith('pending') + + await expect(journal().save('doc-1', pendingUpdate, pendingUpdate)).resolves.toEqual({ + pendingUpdate, + status: 'unavailable', + }) + }) + + it('atomically preserves concurrent providers until their aggregate is acknowledged', async () => { + const first = journal() + const second = journal() + const firstUpdate = updateWith('a') + const secondUpdate = updateWith('b') + + await first.save('doc-1', firstUpdate, firstUpdate) + const combined = await second.save('doc-1', secondUpdate, secondUpdate) + await first.clear('doc-1', firstUpdate) + await expect(first.load('doc-1')).resolves.not.toBeNull() + + const recovered = new Y.Doc() + Y.applyUpdate(recovered, combined.pendingUpdate) + expect(recovered.getText('body').toString()).toHaveLength(2) + + await second.clear('doc-1', combined.pendingUpdate) + await expect(first.load('doc-1')).resolves.toBeNull() + }) + + it('preserves the snapshot dependencies of pending edits from concurrent tabs', async () => { + const first = journal() + const second = journal() + const base = new Y.Doc() + base.getText('body').insert(0, 'base') + const firstDoc = new Y.Doc() + const secondDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(base)) + Y.applyUpdate(secondDoc, Y.encodeStateAsUpdate(base)) + + firstDoc.getText('body').insert(4, ' acknowledged') + const firstVector = Y.encodeStateVector(firstDoc) + firstDoc.getText('body').insert(17, ' pending-first') + await first.save( + 'doc-1', + Y.encodeStateAsUpdate(firstDoc, firstVector), + Y.encodeStateAsUpdate(firstDoc) + ) + + const secondVector = Y.encodeStateVector(secondDoc) + secondDoc.getText('body').insert(4, ' pending-second') + await second.save( + 'doc-1', + Y.encodeStateAsUpdate(secondDoc, secondVector), + Y.encodeStateAsUpdate(secondDoc) + ) + + const stored = await journal().load('doc-1') + expect(stored).not.toBeNull() + const recovered = new Y.Doc() + Y.applyUpdate(recovered, stored!.recoverySnapshot!) + Y.applyUpdate(recovered, stored!.pendingUpdate) + + const expected = new Y.Doc() + Y.applyUpdate(expected, Y.encodeStateAsUpdate(firstDoc)) + Y.applyUpdate(expected, Y.encodeStateAsUpdate(secondDoc)) + expect(recovered.getText('body').toString()).toBe(expected.getText('body').toString()) + expect(recovered.getText('body').toString()).toContain('pending-first') + expect(recovered.getText('body').toString()).toContain('pending-second') + for (const doc of [base, firstDoc, secondDoc, recovered, expected]) doc.destroy() + }) + + it('bounds the combined snapshots without overwriting the previous recovery copy', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + const firstSnapshot = updateWith('a'.repeat(FILE_DOC_LIMITS.updateBytes)) + const secondSnapshot = updateWith('b'.repeat(FILE_DOC_LIMITS.updateBytes)) + await subject.save('doc-1', pendingUpdate, firstSnapshot) + + await expect(subject.save('doc-1', pendingUpdate, secondSnapshot)).resolves.toMatchObject({ + status: 'limit-exceeded', + }) + const recovered = await subject.load('doc-1') + expect(recovered?.recoverySnapshot).toBeInstanceOf(Uint8Array) + expect(Buffer.from(recovered!.recoverySnapshot!).equals(Buffer.from(firstSnapshot))).toBe(true) + }) + + it('retains bounded recovery records for separate document identities', async () => { + const subject = journal() + for (const docId of ['doc-1', 'doc-2', 'doc-3', 'doc-4']) { + const update = updateWith(docId) + await subject.save(docId, update, update) + } + + await expect(subject.load('doc-4')).resolves.toMatchObject({ docId: 'doc-4' }) + await expect(subject.load('doc-2')).resolves.toMatchObject({ docId: 'doc-2' }) + await expect(subject.load('doc-1')).resolves.toBeNull() + await expect(subject.load()).resolves.toMatchObject({ docId: 'doc-4' }) + }) + + it('clears only the acknowledged document identity', async () => { + const subject = journal() + const oldUpdate = updateWith('old') + const currentUpdate = updateWith('current') + await subject.save('old-doc', oldUpdate, oldUpdate) + await subject.save('current-doc', currentUpdate, currentUpdate) + + await subject.clear('old-doc', oldUpdate) + + await expect(subject.load('old-doc')).resolves.toBeNull() + await expect(subject.load('current-doc')).resolves.toMatchObject({ docId: 'current-doc' }) + }) + + it('isolates records by user, workspace, and file', async () => { + const first = journal() + const otherUser = new PendingFileDocUpdateJournal({ + workspaceId: 'workspace-1', + fileId: 'file-1', + userId: 'user-2', + }) + const update = updateWith('draft') + + await first.save('doc-1', update, update) + + await expect(first.load()).resolves.not.toBeNull() + await expect(otherUser.load()).resolves.toBeNull() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts new file mode 100644 index 00000000000..e4e91a2dd45 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts @@ -0,0 +1,267 @@ +'use client' + +import { createLogger } from '@sim/logger' +import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' +import { get, update as updateValue } from 'idb-keyval' +import * as Y from 'yjs' + +const logger = createLogger('PendingFileDocUpdateJournal') +const JOURNAL_VERSION = 1 +const JOURNAL_TTL_MS = 7 * 24 * 60 * 60 * 1_000 +const MAX_DOCUMENTS = 3 +const RECOVERY_SNAPSHOT_MAX_BYTES = FILE_DOC_LIMITS.updateBytes * 2 + +export interface PendingDocumentRecovery { + docId: string + pendingUpdate: Uint8Array + recoverySnapshot: Uint8Array | null + updatedAt: number +} + +interface PendingUpdateJournalRecord { + version: typeof JOURNAL_VERSION + documents: JournalDocument[] +} + +interface JournalDocument extends PendingDocumentRecovery { + quarantined?: boolean +} + +interface PendingUpdateJournalScope { + workspaceId: string + fileId: string + userId: string +} + +interface JournalSaveResult { + pendingUpdate: Uint8Array + status: 'saved' | 'limit-exceeded' | 'unavailable' +} + +function isRecovery(value: unknown): value is JournalDocument { + if (typeof value !== 'object' || value === null) return false + const candidate = value as Partial + return ( + typeof candidate.docId === 'string' && + candidate.docId.length > 0 && + candidate.pendingUpdate instanceof Uint8Array && + candidate.pendingUpdate.byteLength > 0 && + candidate.pendingUpdate.byteLength <= FILE_DOC_LIMITS.updateBytes && + (candidate.recoverySnapshot === null || + (candidate.recoverySnapshot instanceof Uint8Array && + candidate.recoverySnapshot.byteLength > 0 && + candidate.recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES)) && + typeof candidate.updatedAt === 'number' && + Number.isFinite(candidate.updatedAt) && + (candidate.quarantined === undefined || typeof candidate.quarantined === 'boolean') + ) +} + +function liveDocuments(value: unknown, now: number): JournalDocument[] { + if (typeof value !== 'object' || value === null) return [] + const candidate = value as Partial + if (candidate.version !== JOURNAL_VERSION || !Array.isArray(candidate.documents)) return [] + return candidate.documents + .filter(isRecovery) + .filter((document) => now - document.updatedAt <= JOURNAL_TTL_MS) + .sort( + (left, right) => + Number(left.quarantined === true) - Number(right.quarantined === true) || + right.updatedAt - left.updatedAt + ) + .slice(0, MAX_DOCUMENTS) +} + +function record(documents: JournalDocument[]): PendingUpdateJournalRecord { + return { version: JOURNAL_VERSION, documents } +} + +function sameUpdate(left: Uint8Array | null, right: Uint8Array | null): boolean { + if (left === null || right === null) return left === right + if (left.byteLength !== right.byteLength) return false + return left.every((byte, index) => byte === right[index]) +} + +function validateRecovery({ + pendingUpdate, + recoverySnapshot, +}: Pick): void { + const validationDoc = new Y.Doc() + try { + if (recoverySnapshot) Y.applyUpdate(validationDoc, recoverySnapshot) + Y.applyUpdate(validationDoc, pendingUpdate) + } finally { + validationDoc.destroy() + } +} + +/** + * A bounded crash-recovery journal for user edits the relay has not acknowledged. One atomic + * file-scoped envelope retains up to three recent Yjs document identities, so rebuilding a live + * document cannot overwrite an older local draft. The pending delta is wire-bounded separately from + * the full recovery snapshot: only the delta is ever replayed to a matching server document. + */ +export class PendingFileDocUpdateJournal { + private readonly key: string + private mutationQueue = Promise.resolve() + + constructor({ workspaceId, fileId, userId }: PendingUpdateJournalScope) { + const origin = typeof location === 'undefined' ? 'server' : location.origin + this.key = [ + 'sim', + 'file-doc-pending', + JOURNAL_VERSION, + origin, + userId, + workspaceId, + fileId, + ].join(':') + } + + async load(preferredDocId?: string): Promise { + try { + await this.mutationQueue + const documents = liveDocuments(await get(this.key), Date.now()).filter( + (document) => !document.quarantined + ) + const recovered = preferredDocId + ? (documents.find((document) => document.docId === preferredDocId) ?? null) + : (documents[0] ?? null) + if (!recovered) return null + try { + validateRecovery(recovered) + return recovered + } catch (error) { + logger.warn('Isolating malformed pending file edits', { error }) + await this.quarantine(recovered) + return null + } + } catch (error) { + logger.warn('Failed to load pending file edits', { error }) + return null + } + } + + save( + docId: string, + pendingUpdate: Uint8Array, + recoverySnapshot: Uint8Array + ): Promise { + const pendingWithinLimit = + pendingUpdate.byteLength > 0 && pendingUpdate.byteLength <= FILE_DOC_LIMITS.updateBytes + const snapshotWithinLimit = + recoverySnapshot.byteLength > 0 && recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES + const limited: JournalSaveResult = { pendingUpdate, status: 'limit-exceeded' } + if (!pendingWithinLimit || !snapshotWithinLimit) return Promise.resolve(limited) + + return this.enqueue( + async () => { + let result = limited + await updateValue(this.key, (value) => { + const now = Date.now() + const documents = liveDocuments(value, now) + const existing = documents.find( + (document) => document.docId === docId && !document.quarantined + ) + let merged = pendingUpdate + let mergedSnapshot = recoverySnapshot + if (existing) { + try { + merged = Y.mergeUpdates([existing.pendingUpdate, pendingUpdate]) + if (merged.byteLength > FILE_DOC_LIMITS.updateBytes) return record(documents) + if (existing.recoverySnapshot) { + mergedSnapshot = Y.mergeUpdates([existing.recoverySnapshot, recoverySnapshot]) + } + } catch (error) { + let existingIsValid = true + try { + validateRecovery(existing) + } catch { + existingIsValid = false + } + if (existingIsValid) throw error + validateRecovery({ pendingUpdate, recoverySnapshot }) + logger.warn('Isolating malformed pending file edits during save', { error }) + documents.splice(documents.indexOf(existing), 1) + documents.push({ ...existing, quarantined: true }) + merged = pendingUpdate + mergedSnapshot = recoverySnapshot + } + } + if (mergedSnapshot.byteLength > RECOVERY_SNAPSHOT_MAX_BYTES) return record(documents) + + const next: PendingDocumentRecovery = { + docId, + pendingUpdate: merged, + recoverySnapshot: mergedSnapshot, + updatedAt: now, + } + const retained = [ + next, + ...documents.filter((document) => document.docId !== docId || document.quarantined), + ].slice(0, MAX_DOCUMENTS) + result = { + pendingUpdate: merged, + status: 'saved', + } + return record(retained) + }) + if (result.status === 'limit-exceeded') { + logger.warn('Pending file edits exceeded the crash-recovery journal limit') + } + return result + }, + { pendingUpdate, status: 'unavailable' } + ) + } + + clear(docId: string, acknowledgedUpdate: Uint8Array): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => { + const documents = liveDocuments(value, Date.now()) + return record( + documents.filter( + (document) => + document.quarantined || + document.docId !== docId || + !sameUpdate(document.pendingUpdate, acknowledgedUpdate) + ) + ) + }), + undefined + ) + } + + /** Retain invalid bytes within the journal's existing bounds without replaying or merging them. */ + private quarantine(recovered: PendingDocumentRecovery): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => + record( + liveDocuments(value, Date.now()).map((document) => + document.docId === recovered.docId && + document.updatedAt === recovered.updatedAt && + sameUpdate(document.pendingUpdate, recovered.pendingUpdate) && + sameUpdate(document.recoverySnapshot, recovered.recoverySnapshot) + ? { ...document, quarantined: true } + : document + ) + ) + ), + undefined + ) + } + + private enqueue(operation: () => Promise, fallback: T): Promise { + const result = this.mutationQueue.then(operation, operation) + this.mutationQueue = result.then( + () => undefined, + () => undefined + ) + return result.catch((error) => { + logger.warn('Failed to persist pending file edits', { error }) + return fallback + }) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts index 6164e03085c..394e2140e2f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts @@ -11,18 +11,17 @@ import { const at = (input: Partial): CollabReadinessInputs => ({ synced: false, seeded: false, - offlineSeed: false, fatal: false, ...input, }) describe('isCollabReady', () => { it('is not ready before syncing or seeding', () => { - expect(isCollabReady(at({ synced: false, seeded: false, offlineSeed: false }))).toBe(false) + expect(isCollabReady(at({ synced: false, seeded: false }))).toBe(false) }) it('is not ready when synced but not yet seeded', () => { - expect(isCollabReady(at({ synced: true, seeded: false, offlineSeed: false }))).toBe(false) + expect(isCollabReady(at({ synced: true, seeded: false }))).toBe(false) }) it('is ready only when the current session is synced and the server seed is present', () => { @@ -33,10 +32,6 @@ describe('isCollabReady', () => { expect(isCollabReady(at({ synced: false, seeded: true }))).toBe(false) }) - it('stays read-only for an offline (local) seed that never reached the server', () => { - expect(isCollabReady(at({ synced: true, seeded: true, offlineSeed: true }))).toBe(false) - }) - it('revokes readiness when a live document turns fatal', () => { expect(isCollabReady(at({ synced: true, seeded: true, fatal: true }))).toBe(false) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts index 38d59aee26f..8169f9bed31 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts @@ -5,17 +5,11 @@ export interface CollabReadinessInputs { synced: boolean /** Whether the shared doc carries the seed flag. */ seeded: boolean - /** Whether the seed flag was set by the offline fallback (no server sync) rather than the server. */ - offlineSeed: boolean - /** - * Whether the provider has GIVEN UP on this document — a non-retryable rejection, an access - * revocation, or the readiness deadline lapsing. A fatal provider ignores every inbound frame and - * never rejoins, so nothing typed after this point reaches the server. - */ + /** A terminal rejection or access revocation prevents further synchronization and editing. */ fatal: boolean } /** A document is writable only while this connection has synced the server-seeded Yjs document. */ export function isCollabReady(input: CollabReadinessInputs): boolean { - return input.synced && input.seeded && !input.offlineSeed && !input.fatal + return input.synced && input.seeded && !input.fatal } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts index 7dc9eaaa530..2bad83b13da 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -5,9 +5,9 @@ import { FILE_DOC_EVENTS, type FileDocPresence } from '@sim/realtime-protocol/fi import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' import { getUserColor } from '@/lib/workspaces/colors' +import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' +import { useReportFileDocOthers } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context' import { useSocket } from '@/app/workspace/providers/socket-provider' -import { FileDocProvider } from './file-doc-provider' -import { useReportFileDocOthers } from './file-doc-room-context' /** The live collaboration binding the editor wires into TipTap's Collaboration * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ @@ -32,6 +32,7 @@ export interface FileDocCollaboration { } interface UseFileDocCollaborationParams { + workspaceId: string fileId: string userId: string userName: string @@ -51,6 +52,7 @@ interface UseFileDocCollaborationParams { * realtime relay over the shared socket. Returns `null` while disabled. */ export function useFileDocCollaboration({ + workspaceId, fileId, userId, userName, @@ -102,13 +104,16 @@ export function useFileDocCollaboration({ // (see above), so this always binds the same doc/awareness the editor froze at mount. const doc = docRef.current as Y.Doc const awareness = awarenessRef.current as Awareness - const fileProvider = new FileDocProvider(socket, fileId, doc, awareness) + const fileProvider = new FileDocProvider(socket, fileId, doc, awareness, { + workspaceId, + userId, + }) setProvider(fileProvider) return () => { fileProvider.destroy() setProvider(null) } - }, [enabled, socket, fileId]) + }, [enabled, socket, fileId, workspaceId, userId]) const reportOthers = useReportFileDocOthers() const reportOthersRef = useRef(reportOthers) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx index 914a0643d63..b5c3e053e1c 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx @@ -8,6 +8,7 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' +import { SIM_SELECTION_MIME } from '@/lib/copilot/chat/selection-clipboard' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' import { ImageUploadPlaceholders } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-upload' @@ -22,7 +23,10 @@ const { collaborationRef, uploadFile } = vi.hoisted(() => ({ uploadFile: vi.fn(), })) -vi.mock('next/navigation', () => ({ useRouter: () => ({ push: vi.fn() }) })) +vi.mock('next/navigation', () => ({ + usePathname: () => '/workspace/workspace-1/files', + useRouter: () => ({ push: vi.fn() }), +})) vi.mock('@/lib/auth/auth-client', () => ({ useSession: () => ({ data: null, isPending: false }) })) vi.mock('@/hooks/queries/workspace-files', () => ({ useUploadWorkspaceFile: () => ({ mutateAsync: uploadFile }), @@ -36,10 +40,6 @@ vi.mock( '@/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content', () => ({ useEditableFileContent: vi.fn() }) ) -vi.mock( - '@/app/workspace/[workspaceId]/files/components/file-viewer/use-selection-copy-bridge', - () => ({ useSelectionCopyBridge: vi.fn() }) -) vi.mock('@/app/workspace/[workspaceId]/files/components/file-viewer/text-editor', () => ({ TextEditor: () => null, })) @@ -76,6 +76,10 @@ vi.mock( '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu', () => ({ TableBubbleMenu: () => null }) ) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu', + () => ({ ImageBubbleMenu: () => null }) +) vi.mock( '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-hover-card', () => ({ LinkHoverCard: () => null }) @@ -223,6 +227,88 @@ afterEach(async () => { }) describe('loaded rich editor lifecycle', () => { + it.each(['connecting', 'timeout', 'fatal'] as const)( + 'copies selection context from the visible %s preview and switches to the live editor on sync', + async (status) => { + const provider = new FakeFileDocProvider() + const doc = new Y.Doc() + collaborationRef.current = { + doc, + awareness: new Awareness(doc), + provider, + user: { name: 'User', color: '#000000', clientId: doc.clientID }, + } + await render('stored preview body', 'stored preview body', true, { collaborative: true }) + if (status !== 'connecting') { + await act(async () => + provider.fail({ + fileId: FILE.id, + error: status, + code: status === 'timeout' ? 'READINESS_TIMEOUT' : 'ACCESS_DENIED', + retryable: status === 'timeout', + }) + ) + } + + const preview = getEditor() + expect(preview.view.dom.getAttribute('aria-label')).toBe('Document preview') + expect(preview.isEditable).toBe(false) + const hiddenEditor = container.querySelector( + '.hidden .tiptap' + )!.editor + await act(async () => { + hiddenEditor.commands.setContent('

stale hidden selection

') + hiddenEditor.commands.setTextSelection({ from: 1, to: 6 }) + preview.commands.setTextSelection({ from: 1, to: 7 }) + }) + + const copy = (editor: Editor) => { + const written: Record = {} + const event = new Event('copy', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'clipboardData', { + value: { + clearData: () => { + for (const key of Object.keys(written)) delete written[key] + }, + setData: (type: string, value: string) => { + written[type] = value + }, + }, + }) + editor.view.dom.dispatchEvent(event) + return written + } + const previewCopy = copy(preview) + expect(previewCopy['text/plain']).toBe('stored') + expect(previewCopy[SIM_SELECTION_MIME]).toBeDefined() + expect(JSON.parse(previewCopy[SIM_SELECTION_MIME])).toMatchObject({ + sourceWorkspaceId: FILE.workspaceId, + context: { kind: 'file_selection', fileId: FILE.id, fileName: FILE.name, text: 'stored' }, + }) + expect(doc.getXmlFragment('default').length).toBe(0) + await act(async () => preview.commands.setTextSelection(1)) + expect(copy(preview)[SIM_SELECTION_MIME]).toBeUndefined() + + await act(async () => { + provider.joinError = null + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + provider.setSynced(true) + }) + const editor = getEditor() + expect(editor).not.toBe(preview) + expect(container.querySelector('[aria-label="Document preview"]')).toBeNull() + await act(async () => { + editor.commands.setContent('

live content

') + editor.commands.setTextSelection({ from: 1, to: 5 }) + }) + const liveCopy = copy(editor) + expect(liveCopy['text/plain']).toBe('live') + expect(JSON.parse(liveCopy[SIM_SELECTION_MIME])).toMatchObject({ + context: { fileId: FILE.id, text: 'live' }, + }) + } + ) + it('pauses editing while reconnecting and resumes after the document resyncs', async () => { const provider = new FakeFileDocProvider() const doc = new Y.Doc() @@ -253,9 +339,13 @@ describe('loaded rich editor lifecycle', () => { expect(editor.isEditable).toBe(true) expect(editor.view.dom.getAttribute('aria-readonly')).toBe('false') expect(container.textContent).not.toContain('Reconnecting…') + expect(container.querySelector('[role="status"]')).toBeNull() + expect(container.querySelector('[role="alert"]')).toBeNull() + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() }) - it('keeps the live document visible and read-only after a fatal collaboration error', async () => { + it('keeps revoked pending edits visible and read-only without draft-management prompts', async () => { const provider = new FakeFileDocProvider() const doc = new Y.Doc() doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) @@ -275,7 +365,7 @@ describe('loaded rich editor lifecycle', () => { provider.fail({ fileId: 'file-1', error: 'Access denied', - code: 'ACCESS_DENIED', + code: 'ACCESS_REVOKED', retryable: false, }) ) @@ -286,6 +376,13 @@ describe('loaded rich editor lifecycle', () => { expect(editor.view.dom.closest('.hidden')).toBeNull() expect(container.textContent).not.toContain('stale opening snapshot') expect(container.textContent).not.toContain('Reconnecting…') + expect(container.querySelector('[role="status"]')?.textContent).toBe( + 'You no longer have edit access to this document.' + ) + expect(container.querySelector('button')).toBeNull() + expect(container.querySelector('[role="alert"], [role="dialog"]')).toBeNull() + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() }) it('shows stored content read-only when collaboration fails before the first sync', async () => { @@ -316,6 +413,81 @@ describe('loaded rich editor lifecycle', () => { expect(container.textContent).not.toContain('Reconnecting…') }) + it('keeps timeout preview separate from the authoritative document and recovers on late sync', async () => { + const provider = new FakeFileDocProvider() + const doc = new Y.Doc() + collaborationRef.current = { + doc, + awareness: new Awareness(doc), + provider, + user: { name: 'User', color: '#000000', clientId: doc.clientID }, + } + await render('stored preview body', 'stored preview body', true, { collaborative: true }) + await act(async () => + provider.fail({ + fileId: 'file-1', + error: 'Not ready', + code: 'READINESS_TIMEOUT', + retryable: true, + }) + ) + expect(container.textContent).toContain('stored preview body') + expect(container.textContent).toContain('Reconnecting…') + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBeUndefined() + expect(doc.getXmlFragment('default').length).toBe(0) + const editors = [...container.querySelectorAll('.tiptap')].map( + (element) => (element as HTMLElement & { editor: Editor }).editor + ) + expect(editors.every((editor) => !editor.isEditable)).toBe(true) + await act(async () => { + provider.joinError = null + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + provider.setSynced(true) + }) + expect(container.textContent).not.toContain('stored preview body') + expect(container.textContent).not.toContain('Reconnecting…') + expect(getEditor().isEditable).toBe(true) + expect(onClientAutosaveChange).not.toHaveBeenCalledWith(true) + }) + + it.each(['DOCUMENT_REPLACED', 'PENDING_UPDATE_LIMIT', 'INVALID_UPDATE'])( + 'preserves pending edits with only a passive status for %s', + async (code) => { + const provider = new FakeFileDocProvider() + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + collaborationRef.current = { + doc, + awareness: new Awareness(doc), + provider, + user: { name: 'User', color: '#000000', clientId: doc.clientID }, + } + await render('stored body', 'stored body', true, { collaborative: true }) + + await act(async () => provider.setSynced(true)) + await act(async () => getEditor().commands.insertContent('preserved local change')) + await act(async () => + provider.fail({ + fileId: 'file-1', + error: 'Local recovery required', + code, + retryable: false, + }) + ) + + expect(container.querySelector('[role="status"]')?.textContent).toBe( + 'Live editing is unavailable.' + ) + expect(container.querySelector('button')).toBeNull() + expect(container.querySelector('[role="alert"], [role="dialog"]')).toBeNull() + expect(container.textContent).not.toContain('Reconnecting…') + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() + expect(getEditor().isEditable).toBe(false) + expect(getEditor().getText()).toContain('preserved local change') + } + ) + it('explains a picker selection whose insertion anchor was invalidated', async () => { await render('before TARGET after') const editor = getEditor() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/field-lifecycle.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/field-lifecycle.test.tsx new file mode 100644 index 00000000000..066c036cc5f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/field-lifecycle.test.tsx @@ -0,0 +1,207 @@ +/** @vitest-environment jsdom */ +import { act, type ComponentProps, StrictMode, Suspense, startTransition } from 'react' +import type { Editor } from '@tiptap/core' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { RichMarkdownField } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field' + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention', + () => ({ useEditorMentions: vi.fn() }) +) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu', + () => ({ EditorBubbleMenu: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu', + () => ({ ImageBubbleMenu: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-hover-card', + () => ({ LinkHoverCard: () => null }) +) + +let host: HTMLDivElement +let root: Root +const pending = new Promise(() => {}) +const suspended = vi.fn() +interface BlockerProps { + active: boolean +} + +function Blocker({ active }: BlockerProps) { + if (active) { + suspended() + throw pending + } + return null +} +beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.useFakeTimers() + suspended.mockClear() + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) +afterEach(async () => { + await act(async () => root.unmount()) + await vi.advanceTimersByTimeAsync(10) + host.remove() + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +describe('editability synchronization', () => { + async function renderField(props: ComponentProps) { + await act(async () => + root.render( + + + + ) + ) + await act(async () => vi.advanceTimersByTimeAsync(10)) + return host.querySelector('.tiptap')!.editor + } + + it.each([ + { label: 'start streaming', initial: {}, next: { isStreaming: true, value: 'streamed' } }, + { + label: 'finish streaming', + initial: { isStreaming: true }, + next: { isStreaming: false, value: 'final' }, + }, + { label: 'disable', initial: {}, next: { disabled: true } }, + { label: 'enable', initial: { disabled: true }, next: { disabled: false } }, + ])('does not report a local edit when props $label', async ({ initial, next }) => { + const props = { value: 'body', onChange: vi.fn(), ...initial } + const owner = await renderField(props) + props.onChange.mockClear() + expect(await renderField({ ...props, ...next })).toBe(owner) + expect(owner.getText()).toBe(next.value ?? 'body') + expect(props.onChange).not.toHaveBeenCalled() + }) + + it('continues reporting actual edits after streaming completes', async () => { + const props = { value: 'body', onChange: vi.fn(), isStreaming: true } + const owner = await renderField(props) + await renderField({ ...props, value: 'final', isStreaming: false }) + props.onChange.mockClear() + await act(async () => + owner.commands.insertContentAt(owner.state.doc.content.size - 1, ' edited') + ) + expect(props.onChange).toHaveBeenCalledExactlyOnceWith('final edited') + }) + + it('continues reporting successful uploads after editability changes', async () => { + const pending = Promise.withResolvers<{ url: string; alt: string }>() + const props = { + value: 'body', + onChange: vi.fn(), + disabled: true, + uploadImage: vi.fn(() => pending.promise), + } + const owner = await renderField(props) + await renderField({ ...props, disabled: false }) + props.onChange.mockClear() + const event = new Event('paste', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'clipboardData', { + value: { + files: [new File(['image'], 'image.png', { type: 'image/png' })], + items: [], + types: ['Files'], + getData: () => '', + }, + }) + await act(async () => owner.view.dom.dispatchEvent(event)) + await act(async () => pending.resolve({ url: 'https://sim.ai/valid.png', alt: 'valid' })) + expect(host.querySelector('img')?.getAttribute('alt')).toBe('valid') + expect(props.onChange).toHaveBeenCalledOnce() + expect(props.onChange.mock.calls[0][0]).toContain('https://sim.ai/valid.png') + }) +}) + +describe('field callbacks remain tied to the committed render', () => { + for (const action of ['edit', 'upload'] as const) + for (const suspend of [false, true]) { + it(`${action}, suspended=${suspend}`, async () => { + const originalChange = vi.fn() + const nextChange = vi.fn() + const originalUpload = vi.fn().mockResolvedValue(null) + const nextUpload = vi.fn().mockResolvedValue(null) + const render = (next: boolean) => + root.render( + + + + + ) + await act(async () => render(false)) + await act(async () => vi.advanceTimersByTimeAsync(10)) + const owner = host.querySelector('.tiptap')!.editor + await act(async () => { + if (suspend) startTransition(() => render(true)) + else render(true) + }) + if (suspend) expect(suspended).toHaveBeenCalled() + expect(host.querySelector('.tiptap')!.editor).toBe(owner) + expect(owner.getText()).toBe('body') + if (action === 'edit') await act(async () => owner.commands.insertContentAt(1, 'typed ')) + else { + const event = new Event('paste', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'clipboardData', { + value: { + files: [new File(['image'], 'image.png', { type: 'image/png' })], + items: [], + types: ['Files'], + getData: () => '', + }, + }) + await act(async () => owner.view.dom.dispatchEvent(event)) + } + const original = action === 'edit' ? originalChange : originalUpload + const next = action === 'edit' ? nextChange : nextUpload + expect({ committed: original.mock.calls.length, next: next.mock.calls.length }).toEqual( + suspend ? { committed: 1, next: 0 } : { committed: 0, next: 1 } + ) + }) + } + + it('ignores completion after React unmount before TipTap delayed destruction', async () => { + const change = vi.fn() + const pendingUpload = Promise.withResolvers<{ url: string; alt: string } | null>() + await act(async () => + root.render( + pendingUpload.promise} + /> + ) + ) + await act(async () => vi.advanceTimersByTimeAsync(10)) + const owner = host.querySelector('.tiptap')!.editor + const event = new Event('paste', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'clipboardData', { + value: { + files: [new File(['image'], 'image.png', { type: 'image/png' })], + items: [], + types: ['Files'], + getData: () => '', + }, + }) + await act(async () => owner.view.dom.dispatchEvent(event)) + const before = owner.getJSON() + await act(async () => root.render(null)) + expect(owner.isDestroyed).toBe(false) + await act(async () => pendingUpload.resolve({ url: 'https://sim.ai/image.png', alt: 'late' })) + expect(owner.getJSON()).toEqual(before) + expect(change).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/field-upload.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/field-upload.test.tsx new file mode 100644 index 00000000000..80439c8b042 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/field-upload.test.tsx @@ -0,0 +1,206 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import type { Editor } from '@tiptap/core' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { RichMarkdownField } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field' + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention', + () => ({ useEditorMentions: vi.fn() }) +) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu', + () => ({ EditorBubbleMenu: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-hover-card', + () => ({ LinkHoverCard: () => null }) +) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu', + () => ({ ImageBubbleMenu: () => null }) +) + +let host: HTMLDivElement +let root: Root +let onChange: ReturnType +let upload: ReturnType +beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.useFakeTimers() + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) + onChange = vi.fn() + upload = vi.fn() +}) +afterEach(async () => { + await act(async () => root.unmount()) + await vi.advanceTimersByTimeAsync(10) + host.remove() + vi.useRealTimers() + vi.unstubAllGlobals() + vi.restoreAllMocks() +}) +async function render( + value = 'before TARGET after', + disabled = false, + streaming = false, + identity = 'initial' +) { + await act(async () => + root.render( + + ) + ) + await act(async () => vi.advanceTimersByTimeAsync(10)) +} +function editor() { + return host.querySelector('.tiptap')!.editor +} +async function submit( + method: 'paste' | 'drop', + target: Editor, + files = [new File(['image'], 'image.png', { type: 'image/png' })] +) { + await act(async () => target.commands.setTextSelection(8)) + if (method === 'drop') vi.spyOn(target.view, 'posAtCoords').mockReturnValue({ pos: 8, inside: 0 }) + const transfer = { files, items: [], types: ['Files'], getData: () => '' } + const event = new Event(method, { bubbles: true, cancelable: true }) + Object.defineProperty(event, method === 'paste' ? 'clipboardData' : 'dataTransfer', { + value: transfer, + }) + Object.defineProperties(event, { clientX: { value: 0 }, clientY: { value: 0 } }) + await act(async () => target.view.dom.dispatchEvent(event)) + expect(event.defaultPrevented).toBe(true) + expect(upload).toHaveBeenCalledTimes(1) +} + +describe('field upload completion boundary', () => { + it('invalidates an upload even when streaming ends before completion', async () => { + const pending = Promise.withResolvers<{ url: string; alt: string }>() + upload.mockReturnValueOnce(pending.promise) + await render() + const owner = editor() + await submit('paste', owner) + await render('replacement streamed content', false, true) + await render('replacement streamed content') + expect(owner.isEditable).toBe(true) + const before = owner.getJSON() + onChange.mockClear() + await act(async () => pending.resolve({ url: 'https://sim.ai/late.png', alt: 'Late' })) + expect(owner.getJSON()).toEqual(before) + expect(onChange).not.toHaveBeenCalled() + }) + + it('maps an upload anchor through edits without showing upload controls', async () => { + const pending = Promise.withResolvers<{ url: string; alt: string }>() + upload.mockReturnValueOnce(pending.promise) + await render() + const owner = editor() + await submit('paste', owner) + expect(host.querySelector('[data-image-upload-placeholder]')).toBeNull() + await act(async () => owner.commands.insertContentAt(1, 'new prefix ')) + await act(async () => pending.resolve({ url: 'https://sim.ai/mapped.png', alt: 'Mapped' })) + expect(owner.getJSON().content?.map((node) => node.type)).toEqual([ + 'paragraph', + 'image', + 'paragraph', + ]) + expect(owner.getJSON().content?.[0].content?.[0].text).toBe('new prefix before ') + expect(owner.getJSON().content?.[2].content?.[0].text).toBe('TARGET after') + }) + + it('does not insert after the upload anchor is deleted', async () => { + const pending = Promise.withResolvers<{ url: string; alt: string }>() + upload.mockReturnValueOnce(pending.promise) + await render() + const owner = editor() + await submit('paste', owner) + await act(async () => owner.commands.deleteRange({ from: 1, to: 15 })) + const before = owner.getJSON() + await act(async () => pending.resolve({ url: 'https://sim.ai/deleted.png', alt: 'Deleted' })) + expect(owner.getJSON()).toEqual(before) + }) + + it('keeps successful images in batch order when another upload fails', async () => { + const first = Promise.withResolvers<{ url: string; alt: string }>() + const last = Promise.withResolvers<{ url: string; alt: string }>() + upload + .mockReturnValueOnce(first.promise) + .mockRejectedValueOnce(new Error('upload failed')) + .mockReturnValueOnce(last.promise) + await render() + const owner = editor() + await submit( + 'paste', + owner, + ['first', 'failed', 'last'].map( + (name) => new File(['image'], `${name}.png`, { type: 'image/png' }) + ) + ) + await act(async () => first.resolve({ url: 'https://sim.ai/first.png', alt: 'First' })) + await act(async () => last.resolve({ url: 'https://sim.ai/last.png', alt: 'Last' })) + expect( + Array.from(host.querySelectorAll('img')).map((image) => image.getAttribute('alt')) + ).toEqual(['First', 'Last']) + expect(upload).toHaveBeenCalledTimes(3) + }) + + it.each( + (['paste', 'drop'] as const).flatMap((method) => + (['disabled', 'streaming', 'unmount', 'identity', 'rejection'] as const).map((action) => ({ + method, + action, + })) + ) + )( + '$method result after $action cannot change the current document', + async ({ method, action }) => { + const pending = Promise.withResolvers<{ url: string; alt: string } | null>() + upload.mockReturnValueOnce(pending.promise) + await render() + const original = editor() + await submit(method, original) + if (action === 'disabled') await render('before TARGET after', true) + else if (action === 'streaming') await render('replacement streamed content', false, true) + else if (action === 'unmount') { + await act(async () => root.render(null)) + await act(async () => vi.advanceTimersByTimeAsync(10)) + } else if (action === 'identity') await render('new identity content', false, false, 'next') + const current = action === 'unmount' ? original : editor() + const before = current.getJSON() + onChange.mockClear() + await act(async () => { + if (action === 'rejection') pending.reject(new Error('late upload failure')) + else pending.resolve({ url: 'https://sim.ai/completed.png', alt: 'Uploaded image' }) + }) + expect(current.getJSON()).toEqual(before) + expect(onChange).not.toHaveBeenCalled() + if (action === 'disabled' || action === 'streaming') expect(current.isEditable).toBe(false) + if (action === 'unmount') expect(original.isDestroyed).toBe(true) + if (action === 'identity') expect(current).not.toBe(original) + } + ) + it.each(['paste', 'drop'] as const)( + '%s successful completion inserts into an unchanged editable host', + async (method) => { + const pending = Promise.withResolvers<{ url: string; alt: string }>() + upload.mockReturnValueOnce(pending.promise) + await render() + const owner = editor() + await submit(method, owner) + await act(async () => pending.resolve({ url: 'https://sim.ai/success.png', alt: 'Success' })) + expect(host.querySelector('img')?.getAttribute('src')).toBe('https://sim.ai/success.png') + expect(onChange).toHaveBeenCalledOnce() + } + ) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts index 2f730b40771..32afc078615 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts @@ -1,11 +1,26 @@ /** * @vitest-environment jsdom */ + +import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import { Editor } from '@tiptap/core' -import { undoDepth } from '@tiptap/pm/history' -import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { getFindTally, RichMarkdownFind, setFindQuery, stepFindMatch } from './find-extension' +import { redoDepth, undoDepth } from '@tiptap/pm/history' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import type * as Y from 'yjs' +import { markdownToYDoc } from '@/lib/collab-doc/converter' +import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + getFindTally, + RichMarkdownFind, + replaceActiveFindMatch, + replaceAllFindMatches, + setFindQuery, + stepFindMatch, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension' +import { FIND_MATCH_LIMIT } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' let editor: Editor | null = null afterEach(() => { @@ -129,4 +144,133 @@ describe('RichMarkdownFind', () => { // instead of their real last edit. expect(undoDepth(instance.state)).toBe(undoBefore) }) + + it('replaces the active match while preserving its inline marks', () => { + const instance = mountEditor('**alpha** and alpha') + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'beta')).toBe(true) + expect(instance.getMarkdown()).toBe('**beta** and alpha') + expect(getFindTally(instance.state).matches).toHaveLength(1) + }) + + it('uses the matched text formatting instead of an unrelated typing mark', () => { + const instance = mountEditor('alpha and beta') + instance.commands.setTextSelection(instance.state.doc.content.size - 1) + instance.commands.toggleBold() + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'gamma')).toBe(true) + expect(instance.getMarkdown()).toBe('gamma and beta') + }) + + it.each([ + ['he**llo**', 'world'], + ['**he**llo', '**world**'], + ])('follows native ProseMirror replacement formatting for %s', (source, expected) => { + const instance = mountEditor(source) + setFindQuery(instance, 'hello') + const { from, to } = getFindTally(instance.state).matches[0] + const nativeResult = instance.state.tr.setStoredMarks(null).insertText('world', from, to).doc + + expect(replaceActiveFindMatch(instance, 'world')).toBe(true) + expect(instance.state.doc.eq(nativeResult)).toBe(true) + expect(instance.getMarkdown()).toBe(expected) + }) + + it('preserves each matched range formatting during Replace All', () => { + const instance = mountEditor('**alpha** and alpha and *alpha*') + instance.commands.setTextSelection(instance.state.doc.content.size - 1) + instance.commands.toggleStrike() + setFindQuery(instance, 'alpha') + + expect(replaceAllFindMatches(instance, 'beta')).toBe(3) + expect(instance.getMarkdown()).toBe('**beta** and beta and *beta*') + }) + + it('supports deleting matches with an empty replacement', () => { + const instance = mountEditor('alpha beta alpha') + setFindQuery(instance, 'alpha ') + + expect(replaceActiveFindMatch(instance, '')).toBe(true) + expect(instance.getMarkdown()).toBe('beta alpha') + }) + + it('rejects oversized individual and aggregate replacements before dispatching a transaction', () => { + const instance = mountEditor(Array.from({ length: FIND_MATCH_LIMIT }, () => 'x').join(' ')) + setFindQuery(instance, 'x') + const onLimitExceeded = vi.fn() + const dispatch = vi.spyOn(instance.view, 'dispatch') + const documentBefore = instance.state.doc + + expect( + replaceActiveFindMatch( + instance, + 'y'.repeat(PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS), + onLimitExceeded + ) + ).toBe(false) + expect(replaceAllFindMatches(instance, 'y'.repeat(600), onLimitExceeded)).toBe(0) + + expect(onLimitExceeded).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + expect(instance.state.doc).toBe(documentBefore) + }) + + it('advances past a replacement that still contains the search term', () => { + const instance = mountEditor('alpha alpha') + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'alphaX')).toBe(true) + expect(replaceActiveFindMatch(instance, 'alphaX')).toBe(true) + + expect(instance.getMarkdown()).toBe('alphaX alphaX') + }) + + it('keeps each collaborative replacement as a separate undo item', () => { + const doc = markdownToYDoc('alpha alpha') + const awareness = new Awareness(doc) + editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#fff' } }, + }), + }) + const history = yUndoPluginKey.getState(editor.state) as { undoManager: Y.UndoManager } + history.undoManager.clear() + setFindQuery(editor, 'alpha') + + replaceActiveFindMatch(editor, 'beta') + replaceActiveFindMatch(editor, 'gamma') + expect(editor.getMarkdown()).toBe('beta gamma') + + expect(editor.commands.undo()).toBe(true) + expect(editor.getMarkdown()).toBe('beta alpha') + editor.destroy() + editor = null + awareness.destroy() + doc.destroy() + }) + + it('replaces every match in one undo step', () => { + const instance = mountEditor('alpha alpha alpha') + setFindQuery(instance, 'alpha') + const undoBefore = undoDepth(instance.state) + + expect(replaceAllFindMatches(instance, 'beta')).toBe(3) + expect(instance.getMarkdown()).toBe('beta beta beta') + expect(undoDepth(instance.state)).toBe(undoBefore + 1) + expect(redoDepth(instance.state)).toBe(0) + expect(instance.commands.undo()).toBe(true) + expect(instance.getMarkdown()).toBe('alpha alpha alpha') + }) + + it('refuses to label a capped partial replacement as replace all', () => { + const instance = mountEditor(Array.from({ length: FIND_MATCH_LIMIT + 1 }, () => 'x').join(' ')) + setFindQuery(instance, 'x') + expect(getFindTally(instance.state).truncated).toBe(true) + + expect(replaceAllFindMatches(instance, 'y')).toBe(0) + expect(instance.getMarkdown().startsWith('x x x')).toBe(true) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts index e2ac3f0a20a..d3c9805f47a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts @@ -1,9 +1,16 @@ +import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Editor } from '@tiptap/core' import { Extension } from '@tiptap/core' +import { closeHistory } from '@tiptap/pm/history' import type { EditorState } from '@tiptap/pm/state' import { Plugin, PluginKey } from '@tiptap/pm/state' import { Decoration, DecorationSet } from '@tiptap/pm/view' -import { EMPTY_FIND_RESULT, type FindMatch, findMatches } from './find-matches' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { + EMPTY_FIND_RESULT, + type FindMatch, + findMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' /** Class on every match. The active one carries {@link ACTIVE_MATCH_CLASS} as well. */ const MATCH_CLASS = 'rich-find-match' @@ -149,3 +156,87 @@ export function setFindQuery(editor: Editor, query: string): void { export function stepFindMatch(editor: Editor, delta: number): void { dispatchFindMeta(editor, { activeIndex: getFindTally(editor.state).activeIndex + delta }) } + +function stopUndoCapture(editor: Editor): void { + const state = yUndoPluginKey.getState(editor.state) as + | { undoManager?: { stopCapturing: () => void } } + | undefined + state?.undoManager?.stopCapturing() +} + +function isolateReplacement(editor: Editor, transaction: EditorState['tr']): void { + stopUndoCapture(editor) + editor.view.dispatch(closeHistory(transaction).scrollIntoView()) + stopUndoCapture(editor) + editor.view.dispatch(closeHistory(editor.state.tr)) +} + +/** Bounds aggregate growth before Replace All can materialize hundreds of large insertions. */ +function replacementExceedsLimit( + editor: Editor, + matches: readonly FindMatch[], + replacement: string +): boolean { + const currentSize = editor.state.doc.content.size + const nextSize = matches.reduce( + (size, match) => size + replacement.length - (match.to - match.from), + currentSize + ) + return nextSize > Math.max(currentSize, PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS) +} + +/** Uses the target range's marks, independent of formatting armed at the editor's caret. */ +function replaceMatch(transaction: EditorState['tr'], match: FindMatch, replacement: string): void { + const marks = transaction.doc.resolve(match.from).marksAcross(transaction.doc.resolve(match.to)) + transaction.replaceWith( + match.from, + match.to, + replacement ? transaction.doc.type.schema.text(replacement, marks) : [] + ) +} + +export function replaceActiveFindMatch( + editor: Editor, + replacement: string, + onLimitExceeded?: () => void +): boolean { + if (!editor.isEditable) return false + const findState = RICH_FIND_PLUGIN_KEY.getState(editor.state) ?? INITIAL_STATE + const { matches, activeIndex } = findState + const match = matches[activeIndex] + if (!match) return false + if (replacementExceedsLimit(editor, [match], replacement)) { + onLimitExceeded?.() + return false + } + const transaction = editor.state.tr + replaceMatch(transaction, match, replacement) + const remaining = findMatches(transaction.doc, findState.query).matches + const insertionEnd = match.from + replacement.length + const nextIndex = remaining.findIndex((candidate) => candidate.from >= insertionEnd) + transaction.setMeta(RICH_FIND_PLUGIN_KEY, { activeIndex: nextIndex === -1 ? 0 : nextIndex }) + isolateReplacement(editor, transaction) + return true +} + +/** Replaces every collected match in one undo step; capped searches must first be narrowed. */ +export function replaceAllFindMatches( + editor: Editor, + replacement: string, + onLimitExceeded?: () => void +): number { + if (!editor.isEditable) return 0 + const { matches, truncated } = getFindTally(editor.state) + if (truncated || matches.length === 0) return 0 + if (replacementExceedsLimit(editor, matches, replacement)) { + onLimitExceeded?.() + return 0 + } + const transaction = closeHistory(editor.state.tr) + for (let index = matches.length - 1; index >= 0; index -= 1) { + const match = matches[index] + replaceMatch(transaction, match, replacement) + } + isolateReplacement(editor, transaction) + return matches.length +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts index b62f392cea7..ed0cebdf333 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts @@ -3,8 +3,11 @@ */ import { Editor } from '@tiptap/core' import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { FIND_MATCH_LIMIT, findMatches } from './find-matches' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + FIND_MATCH_LIMIT, + findMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' let editor: Editor | null = null afterEach(() => { @@ -59,6 +62,24 @@ describe('findMatches', () => { expect(matchedText('ab\n\ncd', 'abcd')).toEqual([]) }) + it.each(['\uFFFF', 'a\uFFFFb'])('never matches an inline atom using %j', (query) => { + const doc = docFor('a
b') + expect(() => doc.check()).not.toThrow() + expect(findMatches(doc, query)).toEqual({ matches: [], truncated: false }) + }) + + it('does not count atom placeholders toward the match limit', () => { + const doc = docFor('a
b\uFFFF') + expect(() => doc.check()).not.toThrow() + const { matches, truncated } = findMatches(doc, '\uFFFF', 1) + expect(matches.map(({ from, to }) => doc.textBetween(from, to))).toEqual(['\uFFFF']) + expect(truncated).toBe(false) + }) + + it('keeps real non-character text searchable across a formatting boundary', () => { + expect(matchedText('a**\uFFFF**b', 'a\uFFFFb')).toEqual(['a\uFFFFb']) + }) + it('never matches across an inline atom', () => { // The image between them occupies a position; joining `a` to `b` would be a phantom match. expect(matchedText('a![alt](https://x.com/i.png)b', 'ab')).toEqual([]) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts index 5133557d52f..933313c8ec2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts @@ -25,8 +25,8 @@ export const EMPTY_FIND_RESULT: FindResult = { matches: [], truncated: false } /** * Stands in for one position of a non-text inline node (an image, a mention chip) so a match can * never span one — searching `ab` must not join the `a` before an image to the `b` after it. U+FFFF - * is a permanent Unicode non-character, so no query can contain it and match the placeholder itself, - * and it is not whitespace, so the shared scan's whitespace folding leaves it alone. + * is not whitespace, so the shared scan's whitespace folding leaves it alone. Segment checks exclude + * atoms even when a query contains this character, without excluding genuine U+FFFF text. */ const ATOM_PLACEHOLDER = '￿' @@ -34,6 +34,7 @@ const ATOM_PLACEHOLDER = ' interface TextSegment { textStart: number docStart: number + isText: boolean } /** @@ -74,7 +75,7 @@ export function findMatches( if (soleText === null) { const built: TextSegment[] = [] node.forEach((child, offset) => { - built.push({ textStart: text.length, docStart: pos + 1 + offset }) + built.push({ textStart: text.length, docStart: pos + 1 + offset, isText: child.isText }) text += child.isText && child.text ? child.text : ATOM_PLACEHOLDER.repeat(child.nodeSize) }) segments = built @@ -83,6 +84,21 @@ export function findMatches( let segmentIndex = 0 forEachSearchOccurrence(text, query, (start, end) => { if (truncated) return + if (segments) { + while ( + segmentIndex + 1 < segments.length && + segments[segmentIndex + 1].textStart <= start + ) { + segmentIndex++ + } + for ( + let index = segmentIndex; + index < segments.length && segments[index].textStart < end; + index++ + ) { + if (!segments[index].isText) return + } + } if (matches.length >= limit) { truncated = true return @@ -91,10 +107,6 @@ export function findMatches( matches.push({ from: pos + 1 + start, to: pos + 1 + end }) return } - // Segments are ordered and occurrences arrive left to right, so the cursor only moves forward. - while (segmentIndex + 1 < segments.length && segments[segmentIndex + 1].textStart <= start) { - segmentIndex++ - } const segment = segments[segmentIndex] const from = segment.docStart + (start - segment.textStart) matches.push({ from, to: from + (end - start) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts index b0a97d5a6f6..d3a103a69c1 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts @@ -2,9 +2,17 @@ import type React from 'react' import { useCallback, useEffect, useRef, useState } from 'react' +import { toast } from '@sim/emcn' import type { Editor } from '@tiptap/react' import { useFindShortcut } from '@/app/workspace/[workspaceId]/components' -import { ACTIVE_MATCH_CLASS, getFindTally, setFindQuery, stepFindMatch } from './find-extension' +import { + ACTIVE_MATCH_CLASS, + getFindTally, + replaceActiveFindMatch, + replaceAllFindMatches, + setFindQuery, + stepFindMatch, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension' /** What the surface hands `FindBar`, plus the open state the shortcut drives. */ export interface MarkdownFindController { @@ -14,9 +22,13 @@ export interface MarkdownFindController { currentIndex: number truncated: boolean inputRef: React.RefObject + replacement: string setQuery: (query: string) => void + setReplacement: (replacement: string) => void next: () => void prev: () => void + replaceCurrent: () => void + replaceAll: () => void close: () => void } @@ -29,6 +41,12 @@ interface FindTally { const EMPTY_TALLY: FindTally = { count: 0, currentIndex: 0, truncated: false } +function warnReplacementLimit(): void { + toast.warning('Replacement is too large', { + description: 'Use the source editor for changes that exceed the rich-text editing limit.', + }) +} + interface UseMarkdownFindOptions { editor: Editor | null /** @@ -55,6 +73,7 @@ export function useMarkdownFind({ }: UseMarkdownFindOptions): MarkdownFindController { const [isOpen, setIsOpen] = useState(false) const [query, setQueryState] = useState('') + const [replacement, setReplacement] = useState('') const [tally, setTally] = useState(EMPTY_TALLY) const inputRef = useRef(null) const editorRef = useRef(editor) @@ -141,13 +160,29 @@ export function useMarkdownFind({ const next = useCallback(() => step(1), [step]) const prev = useCallback(() => step(-1), [step]) + const replaceCurrent = useCallback(() => { + const current = editorRef.current + if (!current || !replaceActiveFindMatch(current, replacement, warnReplacementLimit)) return + revealActiveMatch() + }, [replacement, revealActiveMatch]) + + const replaceAll = useCallback(() => { + const current = editorRef.current + if (!current) return + replaceAllFindMatches(current, replacement, warnReplacementLimit) + }, [replacement]) + /** Closing ends the search: term, highlights and active match all go. */ const close = useCallback(() => { setIsOpen(false) setQueryState('') + setReplacement('') setTally(EMPTY_TALLY) const current = editorRef.current if (current) setFindQuery(current, '') + requestAnimationFrame(() => { + if (current && !current.isDestroyed) current.commands.focus() + }) }, []) const open = useCallback(() => setIsOpen(true), []) @@ -156,13 +191,17 @@ export function useMarkdownFind({ return { isOpen, query, + replacement, count: tally.count, currentIndex: tally.currentIndex, truncated: tally.truncated, inputRef, setQuery, + setReplacement, next, prev, + replaceCurrent, + replaceAll, close, } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/frontmatter.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/frontmatter.test.ts new file mode 100644 index 00000000000..a9dd185a0f0 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/frontmatter.test.ts @@ -0,0 +1,82 @@ +/** @vitest-environment jsdom */ +import { Editor } from '@tiptap/core' +import { describe, expect, it } from 'vitest' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + applyFrontmatter, + postProcessSerializedMarkdown, + splitFrontmatter, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { + parseMarkdownToDoc, + serializeMarkdownDocument, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' +import { isRoundTripSafe } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety' + +describe('frontmatter preservation', () => { + it.each([ + ['leading comment', '# Document metadata\ntitle: Hello'], + ['leading blank and comment', '\n# Document metadata\ntitle: Hello'], + ['multiple comments', '# First\n\n# Second\ntitle: Hello'], + ['double-quoted key', '"title": Hello'], + ['single-quoted key', "'title': Hello"], + ['escaped double quote', '"the \\"title\\"": Hello'], + ['escaped single quote', "'author''s title': Hello"], + ['quoted Unicode key', '"标题": Hello'], + ['double-quoted colon', '"namespace:field": Hello'], + ['single-quoted colon', "'namespace:field': Hello"], + ['double-quoted backslash', '"path\\\\name": Hello'], + ['single-quoted backslash', "'path\\name': Hello"], + ['empty quoted key', '"": Hello'], + ['quoted hash', '"# heading-like key": Hello'], + ['comment before quoted key', '# Metadata\n"page title": Hello'], + ])('keeps %s byte-exact when the body is edited', (_name, metadata) => { + const prefix = `---\n${metadata}\n---\n\n` + const source = `${prefix}Body remains text.` + expect(isRoundTripSafe(source)).toBe(true) + expect(splitFrontmatter(source)).toEqual({ + frontmatter: prefix, + body: 'Body remains text.', + }) + + const { frontmatter, body } = splitFrontmatter(source) + const editor = new Editor({ + extensions: createMarkdownContentExtensions(), + content: parseMarkdownToDoc(body), + }) + try { + expect(editor.commands.insertContentAt(1, 'Edited ')).toBe(true) + const saved = applyFrontmatter( + frontmatter, + postProcessSerializedMarkdown(editor.getMarkdown()) + ) + expect(saved).toBe(`${prefix}Edited Body remains text.`) + expect(serializeMarkdownDocument(saved)).toBe(saved) + } finally { + editor.destroy() + } + }) + + it('preserves comments, quoted keys, BOM, CRLF and the exact body separator together', () => { + const prefix = '\uFEFF---\r\n# Metadata\r\n"title": Hello\r\n--- \t\r\n\r\n\r\n' + const { frontmatter, body } = splitFrontmatter(`${prefix}Body`) + expect(frontmatter).toBe(prefix) + expect(body).toBe('Body') + expect(serializeMarkdownDocument(`${frontmatter}Edited`)).toBe(`${prefix}Edited`) + }) + + it.each([ + ['heading', '---\n# Heading\n---\nBody'], + ['heading and prose', '---\n# Heading\n\nProse\n---\nBody'], + ['several headings', '---\n# Heading\n## Subheading\n---\nBody'], + ['quoted prose', '---\n"A quotation"\n---\nBody'], + ['quoted prose with a colon', '---\n"A quotation: with colon"\n---\nBody'], + ['unclosed double quote', '---\n"title: Hello\n---\nBody'], + ['unclosed single quote', "---\n'author's title': Hello\n---\nBody"], + ['delimiter prefix', '---\n"title": Hello\n---not-a-delimiter\nBody'], + ['delimiter prefix after a plain key', '---\ntitle: Hello\n---not-a-delimiter\nBody'], + ['delimiter prefix as body', '---\n---not-a-delimiter\n---\nBody'], + ])('keeps a leading thematic break and %s visible', (_name, source) => { + expect(splitFrontmatter(source)).toEqual({ frontmatter: '', body: source }) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-collaboration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-collaboration.test.tsx new file mode 100644 index 00000000000..62a5d465328 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-collaboration.test.tsx @@ -0,0 +1,658 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { Tooltip } from '@sim/emcn' +import Collaboration from '@tiptap/extension-collaboration' +import { Editor, EditorContent } from '@tiptap/react' +import StarterKit from '@tiptap/starter-kit' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' +import { BlockMover } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/block-mover' +import { ResizableImage } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image' +import { moveDraggedImageNode } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-drag-move' +import { ImageBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu' + +let host: HTMLDivElement +let root: Root +let local: Editor +let peer: Editor +let localDoc: Y.Doc +let peerDoc: Y.Doc + +beforeEach(async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.useFakeTimers() + localDoc = new Y.Doc() + peerDoc = new Y.Doc() + const createEditor = (document: Y.Doc) => + new Editor({ + extensions: [ + StarterKit.configure({ undoRedo: false }), + BlockMover, + ResizableImage, + Collaboration.configure({ document }), + ], + editorProps: { handleScrollToSelection: () => true }, + }) + local = createEditor(localDoc) + local.commands.setContent( + '

Earlier heading

Original

After image

' + ) + Y.applyUpdate(peerDoc, Y.encodeStateAsUpdate(localDoc)) + peer = createEditor(peerDoc) + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) + vi.spyOn(local.view, 'coordsAtPos').mockReturnValue({ top: 10, bottom: 30, left: 10, right: 50 }) + await act(async () => { + root.render( + + + + + ) + }) + await act(async () => local.commands.setNodeSelection(imagePosition(local))) +}) + +afterEach(async () => { + await act(async () => { + root.unmount() + local.destroy() + peer.destroy() + }) + localDoc.destroy() + peerDoc.destroy() + host.remove() + vi.clearAllTimers() + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +function imagePosition(editor: Editor, alt?: string): number { + let position = -1 + editor.state.doc.descendants((node, pos) => { + if (node.type.name === 'image' && (alt === undefined || node.attrs.alt === alt)) position = pos + }) + return position +} + +function imageAttributes(editor: Editor) { + const position = imagePosition(editor) + return position < 0 ? null : editor.state.doc.nodeAt(position)?.attrs +} + +async function receivePeerUpdate(): Promise { + await act(async () => Y.applyUpdate(localDoc, Y.encodeStateAsUpdate(peerDoc))) +} + +function pointer(target: EventTarget, type: string, clientX: number): void { + const event = new MouseEvent(type, { bubbles: true, cancelable: true, button: 0, clientX }) + Object.defineProperty(event, 'pointerId', { value: 7 }) + act(() => target.dispatchEvent(event)) +} + +function beginResize(): void { + const image = host.querySelector('img')! + const handle = host.querySelector('button[aria-label="Resize image"]')! + Object.defineProperty(image, 'offsetWidth', { value: 200, configurable: true }) + Object.assign(handle, { + setPointerCapture: vi.fn(), + hasPointerCapture: vi.fn(() => true), + releasePointerCapture: vi.fn(), + }) + pointer(handle, 'pointerdown', 100) + pointer(window, 'pointermove', 160) + expect(host.querySelector('img')).toBe(image) + expect(handle.setPointerCapture).toHaveBeenCalledWith(7) + expect(image.style.width).toBe('260px') +} + +function changeDraft(value: string): HTMLInputElement { + const input = host.querySelector('[aria-label="Image editing"] input')! + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + return input +} + +async function addPeerSibling(sameSource = true): Promise { + const position = local.state.doc.firstChild!.nodeSize + peer.commands.insertContentAt(position + 1, { + type: 'image', + attrs: { + src: sameSource ? 'https://sim.ai/image.png' : 'https://sim.ai/second.png', + alt: 'Peer image', + title: 'Sibling identity', + width: '400', + height: '300', + }, + }) + await receivePeerUpdate() + act(() => local.commands.setNodeSelection(position)) + return position +} + +function movePeerImage(from: number, to: number): void { + const image = peer.state.doc.nodeAt(from)! + peer.commands.setNodeSelection(from) + vi.spyOn(peer.view, 'posAtCoords').mockReturnValue({ pos: to, inside: 0 }) + expect( + moveDraggedImageNode( + peer.view, + new MouseEvent('drop', { clientX: 0, clientY: 0, cancelable: true }) as DragEvent, + { images: [], html: `` } + ) + ).toBe(true) +} + +async function setNestedImages(depth: number): Promise { + const wrap = (content: string) => + `${'
'.repeat(depth)}${content}${'
'.repeat(depth)}` + peer.commands.setContent( + '

Earlier heading

' + + wrap( + '

Original group

Original' + ) + + wrap( + '

Peer group

Peer image' + ) + + '

After image

' + ) + await receivePeerUpdate() + await act(async () => local.commands.setNodeSelection(imagePosition(local, 'Original'))) +} + +describe('image interactions during real peer Yjs updates', () => { + it.each( + (['alt', 'href', 'resize'] as const).flatMap((interaction) => + [1, 2].flatMap((depth) => + [false, true].flatMap((queued) => + ['target', 'peer'].map((moved) => ({ interaction, depth, queued, moved })) + ) + ) + ) + )( + 'cancels $interaction after moving the $moved containing block at depth $depth (queued: $queued)', + async ({ interaction, depth, queued, moved }) => { + await setNestedImages(depth) + let input: HTMLInputElement | undefined + if (interaction === 'resize') beginResize() + else { + const label = interaction === 'alt' ? 'alt text' : 'link' + act(() => + host.querySelector(`[aria-label="Edit image ${label}"]`)!.click() + ) + input = changeDraft( + interaction === 'alt' ? 'Draft for original' : 'https://sim.ai/for-original' + ) + } + peer.commands.setNodeSelection( + imagePosition(peer, moved === 'target' ? 'Original' : 'Peer image') + ) + expect(moved === 'target' ? peer.commands.moveBlockDown() : peer.commands.moveBlockUp()).toBe( + true + ) + const finish = () => { + if (input) + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + else pointer(window, 'pointerup', 160) + } + if (queued) { + await act(async () => { + Y.applyUpdate(localDoc, Y.encodeStateAsUpdate(peerDoc)) + finish() + }) + } else { + await receivePeerUpdate() + await act(async () => finish()) + } + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it.each(['alt', 'href', 'resize'] as const)( + 'preserves %s alongside selected metadata and peer text inside nested image containers', + async (interaction) => { + await setNestedImages(2) + let input: HTMLInputElement | undefined + if (interaction === 'resize') beginResize() + else { + const label = interaction === 'alt' ? 'alt text' : 'link' + act(() => + host.querySelector(`[aria-label="Edit image ${label}"]`)!.click() + ) + input = changeDraft( + interaction === 'alt' ? 'Local corrected alt' : 'https://sim.ai/local-link' + ) + } + peer.commands.setNodeSelection(imagePosition(peer, 'Original')) + peer.commands.updateAttributes( + 'image', + interaction === 'alt' ? { href: 'https://sim.ai/peer-link' } : { alt: 'Peer corrected alt' } + ) + peer.commands.insertContentAt('Earlier heading'.length + 1, ' PEER') + peer.commands.insertContentAt(imagePosition(peer, 'Peer image') - 2, ' PEER') + await receivePeerUpdate() + if (input) { + expect(host.querySelector('[aria-label="Image editing"] input')).toBe(input) + await act(async () => + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + ) + } else pointer(window, 'pointerup', 160) + const alt = interaction === 'alt' ? 'Local corrected alt' : 'Peer corrected alt' + expect(local.state.doc.nodeAt(imagePosition(local, alt))?.attrs).toMatchObject({ + alt, + href: + interaction === 'href' + ? 'https://sim.ai/local-link' + : interaction === 'alt' + ? 'https://sim.ai/peer-link' + : null, + width: interaction === 'resize' ? '260' : '200', + }) + await act(async () => Y.applyUpdate(peerDoc, Y.encodeStateAsUpdate(localDoc))) + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it.each(['delete', 'replace'] as const)( + 'rejects a queued draft after the peer %ss its containing block', + async (action) => { + await setNestedImages(2) + act(() => + host.querySelector('[aria-label="Edit image alt text"]')!.click() + ) + const input = changeDraft('Uncommitted draft') + const from = peer.state.doc.firstChild!.nodeSize + const parent = peer.state.doc.child(1) + peer.commands.deleteRange({ from, to: from + parent.nodeSize }) + if (action === 'replace') peer.commands.insertContentAt(from, parent.toJSON()) + await act(async () => { + Y.applyUpdate(localDoc, Y.encodeStateAsUpdate(peerDoc)) + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it.each(['alt', 'href', 'resize'] as const)( + 'cancels %s conservatively when another container image changes', + async (interaction) => { + await setNestedImages(2) + let input: HTMLInputElement | undefined + if (interaction === 'resize') beginResize() + else { + const label = interaction === 'alt' ? 'alt text' : 'link' + act(() => + host.querySelector(`[aria-label="Edit image ${label}"]`)!.click() + ) + input = changeDraft('https://sim.ai/uncommitted') + } + peer.commands.setNodeSelection(imagePosition(peer, 'Peer image')) + peer.commands.updateAttributes('image', { alt: 'Peer corrected alt', width: '480' }) + await receivePeerUpdate() + if (input) { + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + await act(async () => + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + ) + } else pointer(window, 'pointerup', 160) + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it.each( + (['alt', 'href', 'resize'] as const).flatMap((interaction) => + [false, true].flatMap((sameSource) => + ['target', 'sibling'].map((moved) => ({ interaction, sameSource, moved })) + ) + ) + )( + 'cancels $interaction after a peer moves the $moved image (same source: $sameSource)', + async ({ interaction, sameSource, moved }) => { + const position = await addPeerSibling(sameSource) + let input: HTMLInputElement | undefined + if (interaction === 'resize') beginResize() + else { + const label = interaction === 'alt' ? 'alt text' : 'link' + act(() => + host.querySelector(`[aria-label="Edit image ${label}"]`)!.click() + ) + input = changeDraft( + interaction === 'alt' ? 'Draft for original' : 'https://sim.ai/for-original' + ) + } + if (moved === 'target') movePeerImage(position, position + 2) + else movePeerImage(position + 1, position) + await receivePeerUpdate() + if (input) { + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + await act(async () => { + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + } else pointer(window, 'pointerup', 160) + expect(local.state.doc.nodeAt(position)?.attrs.alt).toBe('Peer image') + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it.each(['alt text', 'link'])( + 'rejects queued %s Apply before React renders a same-source reorder', + async (field) => { + const position = await addPeerSibling() + act(() => + host.querySelector(`[aria-label="Edit image ${field}"]`)!.click() + ) + const input = changeDraft('https://sim.ai/stale-draft') + movePeerImage(position + 1, position) + await act(async () => { + Y.applyUpdate(localDoc, Y.encodeStateAsUpdate(peerDoc)) + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + expect(local.getJSON()).toEqual(peer.getJSON()) + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + } + ) + + it('does not revive a draft after images are reordered back', async () => { + const position = await addPeerSibling() + act(() => host.querySelector('[aria-label="Edit image alt text"]')!.click()) + changeDraft('Stale draft') + movePeerImage(position + 1, position) + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + movePeerImage(position + 1, position) + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + expect(local.getJSON()).toEqual(peer.getJSON()) + }) + + it('preserves target metadata edits and text edits around an unchanged same-source sibling', async () => { + const position = await addPeerSibling() + act(() => host.querySelector('[aria-label="Edit image link"]')!.click()) + const input = changeDraft('https://sim.ai/local-link') + peer.commands.setNodeSelection(position) + peer.commands.updateAttributes('image', { alt: 'Peer corrected alt' }) + peer.commands.insertContentAt('Earlier heading'.length + 1, ' PEER') + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBe(input) + await act(async () => { + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + const currentPosition = local.state.doc.firstChild!.nodeSize + expect(local.state.doc.nodeAt(currentPosition)?.attrs).toMatchObject({ + alt: 'Peer corrected alt', + href: 'https://sim.ai/local-link', + }) + expect(local.state.doc.nodeAt(currentPosition + 1)?.attrs.alt).toBe('Peer image') + await act(async () => { + Y.applyUpdate(peerDoc, Y.encodeStateAsUpdate(localDoc)) + }) + expect(local.getJSON()).toEqual(peer.getJSON()) + }) + + it.each(['alt', 'width', 'src'])( + 'cancels conservatively when a sibling image changes its %s', + async (field) => { + const position = await addPeerSibling() + act(() => + host.querySelector('[aria-label="Edit image alt text"]')!.click() + ) + const input = changeDraft('Stale draft') + peer.commands.setNodeSelection(position + 1) + peer.commands.updateAttributes('image', { + [field]: field === 'width' ? '500' : 'https://sim.ai/peer-change', + }) + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + await act(async () => { + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it.each( + (['alt', 'href', 'resize'] as const).flatMap((interaction) => + [false, true].flatMap((identical) => + ['before', 'after'].map((side) => ({ interaction, identical, side })) + ) + ) + )( + 'cancels $interaction after a peer inserts $side the image (identical: $identical)', + async ({ interaction, identical, side }) => { + let input: HTMLInputElement | undefined + if (interaction === 'resize') beginResize() + else { + const label = interaction === 'alt' ? 'alt text' : 'link' + act(() => + host.querySelector(`[aria-label="Edit image ${label}"]`)!.click() + ) + input = changeDraft(interaction === 'alt' ? 'Local draft' : 'https://sim.ai/local-draft') + } + const originalTarget = localDoc.getXmlFragment('default').get(1) + peer.commands.insertContentAt(imagePosition(peer) + (side === 'after' ? 1 : 0), { + type: 'image', + attrs: identical + ? imageAttributes(peer) + : { src: 'https://sim.ai/inserted.png', alt: 'Inserted', width: '400' }, + }) + await receivePeerUpdate() + expect(localDoc.getXmlFragment('default').get(1)).toBe(originalTarget) + if (input) { + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + act(() => + input?.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + ) + } else pointer(window, 'pointerup', 160) + expect(local.getJSON()).toEqual(peer.getJSON()) + } + ) + + it('rejects a queued Apply before React renders the peer insertion', async () => { + act(() => host.querySelector('[aria-label="Edit image alt text"]')!.click()) + const input = changeDraft('Stale draft') + peer.commands.insertContentAt(imagePosition(peer), { + type: 'image', + attrs: imageAttributes(peer), + }) + await act(async () => { + Y.applyUpdate(localDoc, Y.encodeStateAsUpdate(peerDoc)) + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + expect(local.getJSON()).toEqual(peer.getJSON()) + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + }) + + it('does not revive a canceled draft when the peer removes their inserted image', async () => { + act(() => host.querySelector('[aria-label="Edit image alt text"]')!.click()) + changeDraft('Stale draft') + const position = imagePosition(peer) + peer.commands.insertContentAt(position, { type: 'image', attrs: imageAttributes(peer) }) + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + peer.commands.deleteRange({ from: position, to: position + 1 }) + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + expect(local.getJSON()).toEqual(peer.getJSON()) + }) + + it.each(['cancel', 'apply', 'unmount'] as const)( + 'removes the draft guard listener on %s', + (finish) => { + const subscribe = vi.spyOn(local, 'on') + const unsubscribe = vi.spyOn(local, 'off') + act(() => + host.querySelector('[aria-label="Edit image alt text"]')!.click() + ) + const listener = subscribe.mock.calls.find(([event]) => event === 'transaction')?.[1] + expect(listener).toBeTypeOf('function') + if (finish === 'unmount') act(() => root.unmount()) + else { + const key = finish === 'cancel' ? 'Escape' : 'Enter' + act(() => + host + .querySelector('input')! + .dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: true })) + ) + } + expect(unsubscribe).toHaveBeenCalledWith('transaction', listener) + } + ) + + it.each(['pointerup', 'pointercancel', 'blur', 'unmount'])( + 'removes the resize transaction listener after %s', + (finish) => { + const subscribe = vi.spyOn(local, 'on') + const unsubscribe = vi.spyOn(local, 'off') + beginResize() + const listener = subscribe.mock.calls.find(([event]) => event === 'transaction')?.[1] + expect(listener).toBeTypeOf('function') + + if (finish === 'unmount') act(() => root.unmount()) + else pointer(window, finish, 160) + + expect(unsubscribe).toHaveBeenCalledWith('transaction', listener) + } + ) + + it('preserves peer alt text when only the local link draft changes', async () => { + act(() => + host.querySelector('button[aria-label="Edit image link"]')!.click() + ) + const input = host.querySelector('input[aria-label="Image link URL"]')! + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call( + input, + 'https://sim.ai/local-link' + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + peer.commands.setNodeSelection(imagePosition(peer)) + peer.commands.updateAttributes('image', { alt: 'Peer corrected alt' }) + await receivePeerUpdate() + expect(imageAttributes(local)?.alt).toBe('Peer corrected alt') + expect(host.querySelector('input[aria-label="Image link URL"]')).toBe(input) + await act(async () => { + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + + expect(imageAttributes(local)).toMatchObject({ + alt: 'Peer corrected alt', + href: 'https://sim.ai/local-link', + }) + }) + + it('keeps resizing the same image after a peer heading and metadata edit', async () => { + const originalImage = localDoc.getXmlFragment('default').get(1) + beginResize() + peer.commands.insertContentAt('Earlier heading'.length + 1, ' PEER') + peer.commands.setNodeSelection(imagePosition(peer)) + peer.commands.updateAttributes('image', { alt: 'Peer corrected alt' }) + await receivePeerUpdate() + + expect(localDoc.getXmlFragment('default').get(1)).toBe(originalImage) + pointer(window, 'pointerup', 160) + expect(imageAttributes(local)).toMatchObject({ + alt: 'Peer corrected alt', + width: '260', + height: null, + }) + expect(local.state.doc.firstChild?.textContent).toBe('Earlier heading PEER') + }) + + it.each(['alt', 'href', 'unchanged', 'reverted'] as const)( + 'preserves peer fields and follows the image through preceding edits: %s', + async (change) => { + const field = change === 'href' ? 'link' : 'alt text' + act(() => + host.querySelector(`button[aria-label="Edit image ${field}"]`)!.click() + ) + const input = changeDraft( + change === 'href' + ? 'https://sim.ai/local' + : change === 'unchanged' + ? 'Original' + : 'Local alt' + ) + if (change === 'reverted') changeDraft('Original') + peer.commands.insertContentAt('Earlier heading'.length + 1, ' PEER') + peer.commands.setNodeSelection(imagePosition(peer)) + peer.commands.updateAttributes('image', { alt: 'Peer alt', href: 'https://sim.ai/peer' }) + await receivePeerUpdate() + expect(host.querySelector('[aria-label="Image editing"] input')).toBe(input) + await act(async () => + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + ) + expect(imageAttributes(local)).toMatchObject({ + alt: change === 'alt' ? 'Local alt' : 'Peer alt', + href: change === 'href' ? 'https://sim.ai/local' : 'https://sim.ai/peer', + }) + expect(local.state.doc.firstChild?.textContent).toBe('Earlier heading PEER') + } + ) + + it.each(['delete', 'replace', 'identical replacement'] as const)( + 'never applies an open draft to a peer replacement: %s', + async (action) => { + act(() => + host.querySelector('[aria-label="Edit image alt text"]')!.click() + ) + const input = changeDraft('Uncommitted draft') + const position = imagePosition(peer) + const originalAttributes = imageAttributes(peer) + peer.commands.deleteRange({ from: position, to: position + 1 }) + if (action !== 'delete') + peer.commands.insertContentAt(position, { + type: 'image', + attrs: + action === 'replace' + ? { ...originalAttributes, alt: 'Replacement' } + : originalAttributes, + }) + await receivePeerUpdate() + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))) + expect(host.querySelector('[aria-label="Image editing"] input')).toBeNull() + expect(imageAttributes(local)?.alt ?? null).toBe( + action === 'delete' ? null : action === 'replace' ? 'Replacement' : 'Original' + ) + } + ) + + it.each([false, true])( + 'cancels a resize when the peer replaces the actual image node (identical attributes: %s)', + async (identicalAttributes) => { + const originalImage = localDoc.getXmlFragment('default').get(1) + const replacement = identicalAttributes + ? { ...imageAttributes(peer) } + : { src: 'https://sim.ai/replacement.png', alt: 'Replacement', width: '400', height: '300' } + beginResize() + const position = imagePosition(peer) + peer.commands.deleteRange({ from: position, to: position + 1 }) + peer.commands.insertContentAt(position, { type: 'image', attrs: replacement }) + await receivePeerUpdate() + + expect(localDoc.getXmlFragment('default').get(1)).not.toBe(originalImage) + expect(host.querySelector('img')?.style.width).toBe( + identicalAttributes ? '200px' : '400px' + ) + pointer(window, 'pointerup', 160) + expect(imageAttributes(local)).toMatchObject(replacement) + } + ) + + it('cancels a resize when the peer deletes the image', async () => { + beginResize() + const position = imagePosition(peer) + peer.commands.deleteRange({ from: position, to: position + 1 }) + await receivePeerUpdate() + pointer(window, 'pointerup', 160) + + expect(host.querySelector('img')).toBeNull() + expect(local.getHTML()).toBe('

Earlier heading

After image

') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-input-rule.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-input-rule.test.ts new file mode 100644 index 00000000000..34e252318d6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-input-rule.test.ts @@ -0,0 +1,413 @@ +/** @vitest-environment jsdom */ +import { Editor, type EditorOptions } from '@tiptap/core' +import { closeHistory } from '@tiptap/pm/history' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { markdownToYDoc } from '@/lib/collab-doc/converter' +import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' + +const cleanups: Array<() => void> = [] +afterEach(() => { + cleanups.splice(0).forEach((cleanup) => cleanup()) + vi.useRealTimers() +}) + +function createPeer(seed: Y.Doc) { + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(seed)) + const awareness = new Awareness(doc) + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#ffffff' } }, + }), + editorProps: { handleScrollToSelection: () => true }, + }) + cleanups.push(() => { + editor.destroy() + awareness.destroy() + doc.destroy() + }) + return { doc, editor } +} + +/** Exercises the same input-rule ordering as character-by-character browser typing. */ +function typeText(editor: Editor, text: string): void { + for (const character of text) { + inputText(editor, character) + } +} + +function inputText( + editor: Editor, + text: string, + { from, to }: { from: number; to: number } = editor.state.selection +): void { + const handled = editor.view.someProp('handleTextInput', (handler) => + handler(editor.view, from, to, text, () => editor.state.tr.insertText(text, from, to)) + ) + if (!handled) editor.view.dispatch(editor.state.tr.insertText(text, from, to)) +} + +function createEditors( + collaborative: boolean, + content: string, + options: Partial = {} +) { + if (collaborative) { + const seed = markdownToYDoc('') + const a = createPeer(seed) + const b = createPeer(seed) + seed.destroy() + a.editor.commands.setContent(content) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + return { + editor: a.editor, + assertSynced: () => { + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + }, + } + } + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ placeholder: '' }), + content, + editorProps: { handleScrollToSelection: () => true }, + ...options, + }) + cleanups.push(() => editor.destroy()) + return { editor, assertSynced: () => {} } +} + +function selectText(editor: Editor, text: string) { + let from = -1 + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text?.includes(text)) from = pos + node.text.indexOf(text) + }) + expect(from).toBeGreaterThanOrEqual(0) + editor.commands.setTextSelection({ from, to: from + text.length }) +} + +const IMAGE_SOURCE = '![Audit image](https://example.com/logo.png "Brand")' +const INPUT_CHUNKS = [ + { name: 'characters', chunks: Array.from(IMAGE_SOURCE) }, + { name: 'whole syntax', chunks: [IMAGE_SOURCE] }, + { name: 'closing delimiter', chunks: [IMAGE_SOURCE.slice(0, -1), ')'] }, + { name: 'alt chunk', chunks: [IMAGE_SOURCE.slice(0, 6), IMAGE_SOURCE.slice(6)] }, + { name: 'after bang', chunks: ['!', IMAGE_SOURCE.slice(1)] }, +] + +it.each(['

TARGET

', '

TARGET

'])( + 'requests caret scrolling after image conversion in %s', + (content) => { + const { editor } = createEditors(false, `${content}

After

`) + selectText(editor, 'TARGET') + const requests: boolean[] = [] + editor.on('transaction', ({ transaction }) => { + if (transaction.docChanged) requests.push(transaction.scrolledIntoView) + }) + inputText(editor, IMAGE_SOURCE) + expect(requests).toContain(true) + } +) + +describe.each([false, true])('image input rules (collaborative=%s)', (collaborative) => { + describe.each([ + { name: 'paragraph', html: '

TARGET

' }, + { name: 'heading', html: '

TARGET

' }, + { name: 'blockquote', html: '

TARGET

' }, + { name: 'list', html: '

  • TARGET

' }, + ])('$name', ({ html }) => { + it.each(INPUT_CHUNKS)('converts $name without dropping text', ({ chunks }) => { + const { editor, assertSynced } = createEditors( + collaborative, + `

Earlier document

${html}

Later document

` + ) + selectText(editor, 'TARGET') + for (const chunk of chunks) inputText(editor, chunk) + const images: unknown[] = [] + editor.state.doc.descendants((node) => { + if (node.type.name === 'image') images.push(node.attrs) + }) + expect(images).toMatchObject([ + { src: 'https://example.com/logo.png', alt: 'Audit image', title: 'Brand' }, + ]) + expect(editor.getText()).not.toContain('![') + expect(editor.getText()).not.toContain('TARGET') + expect(editor.getText()).toContain('Earlier document') + expect(editor.getText()).toContain('Later document') + expect(editor.getMarkdown()).toContain(IMAGE_SOURCE) + assertSynced() + }) + }) + + it.each(INPUT_CHUNKS)('preserves neighboring marks and undoes $name', ({ chunks }) => { + const { editor, assertSynced } = createEditors( + collaborative, + '

Before TARGET After

' + ) + selectText(editor, 'TARGET') + for (const chunk of chunks) inputText(editor, chunk) + expect(editor.state.doc.firstChild?.firstChild?.text?.startsWith('Before ')).toBe(true) + expect(editor.state.doc.firstChild?.firstChild?.marks.map((mark) => mark.type.name)).toContain( + 'bold' + ) + expect(editor.getHTML()).toContain(' After') + expect(editor.state.selection.$from.parent.type.name).not.toBe('image') + assertSynced() + expect(editor.commands.undoInputRule()).toBe(true) + expect(editor.getText()).toBe(`Before ${IMAGE_SOURCE} After`) + expect(editor.state.doc.firstChild?.firstChild?.text?.startsWith('Before ')).toBe(true) + expect(editor.state.doc.firstChild?.firstChild?.marks.map((mark) => mark.type.name)).toContain( + 'bold' + ) + expect(editor.getHTML()).toContain(' After') + assertSynced() + }) + + it('preserves a whole burst prefix while replacing the selection', () => { + const { editor, assertSynced } = createEditors(collaborative, '

TARGET suffix

') + selectText(editor, 'TARGET') + inputText(editor, `New prefix ${IMAGE_SOURCE}`) + expect(editor.getText()).toBe('New prefix \n\n\n\n suffix') + expect(editor.getMarkdown()).toContain(IMAGE_SOURCE) + assertSynced() + }) +}) + +describe.each([ + { name: 'image', extension: 'image', source: '![Audit image](https://example.com/logo.png)' }, + { name: 'link', extension: 'markdownLinkInputRule', source: '[Audit link](https://example.com)' }, +])('$name input-rule event boundaries', ({ name, extension, source }) => { + function assertConverted(editor: Editor) { + expect(editor.getHTML()).toContain( + name === 'image' ? 'src="https://example.com/logo.png"' : 'href="https://example.com"' + ) + expect(editor.getText()).not.toContain(source) + } + + it.each([false, true])( + 'replaces a marked selection and preserves the incoming prefix (collaborative=%s)', + (collaborative) => { + const selected = `oldpre ${source.slice(0, -1)}${' obsolete'.repeat(20)}` + const { editor, assertSynced } = createEditors( + collaborative, + `

Before ${selected} After

` + ) + selectText(editor, selected) + inputText(editor, `newpre ${source}`) + assertConverted(editor) + expect(editor.getText()).toContain('newpre ') + expect(editor.getText()).not.toContain('oldpre') + expect(editor.getText()).not.toContain('obsolete') + expect(editor.getHTML()).toContain('Before ') + expect(editor.getHTML()).toContain(' After') + assertSynced() + expect(editor.commands.undoInputRule()).toBe(true) + expect(editor.getText()).toBe(`Before newpre ${source} After`) + assertSynced() + } + ) + + it('uses the event range rather than an unrelated current selection', () => { + const { editor } = createEditors(false, '

oldpre ![a](src

After

') + selectText(editor, 'After') + inputText(editor, `newpre ${source}`, { from: 1, to: 1 + 'oldpre ![a](src'.length }) + assertConverted(editor) + expect(editor.getText()).toContain('newpre ') + expect(editor.getText()).not.toContain('oldpre') + expect(editor.getText()).toContain('After') + }) + + it('preserves a trailing newline instead of matching before it', () => { + const { editor } = createEditors(false, '

TARGET

') + selectText(editor, 'TARGET') + inputText(editor, `${source}\n`) + expect(editor.state.doc.firstChild?.textContent).toBe(`${source}\n`) + }) + + it('keeps Enter separate from already inserted literal syntax', () => { + const { editor } = createEditors(false, '

TARGET

') + selectText(editor, 'TARGET') + editor.commands.insertContent(source) + editor.view.someProp('handleKeyDown', (handler) => + handler(editor.view, new KeyboardEvent('keydown', { key: 'Enter' })) + ) + expect(editor.state.doc.firstChild?.textContent).toBe(source) + }) + + it.each(['
TARGET
', '

TARGET

'])( + 'leaves code literal in %s', + (content) => { + const { editor } = createEditors(false, content) + selectText(editor, 'TARGET') + inputText(editor, source) + expect(editor.state.doc.firstChild?.textContent).toBe(source) + expect(editor.getHTML()).not.toContain(' { + const { editor } = createEditors(false, '

TARGET

After

', { enableInputRules }) + selectText(editor, 'TARGET') + inputText(editor, source) + if (Array.isArray(enableInputRules) && enableInputRules.includes(extension)) + assertConverted(editor) + else expect(editor.getText()).toBe(`${source}\n\nAfter`) + } + ) + + it('converts completed composition without inserting its text twice', () => { + vi.useFakeTimers() + const { editor } = createEditors(false, '

TARGET

After

') + selectText(editor, 'TARGET') + const composing = vi.spyOn(editor.view, 'composing', 'get').mockReturnValue(true) + inputText(editor, `Prefix ${source}`) + expect(editor.getText()).toBe(`Prefix ${source}\n\nAfter`) + composing.mockRestore() + editor.view.someProp('handleDOMEvents', (handlers) => + handlers.compositionend?.(editor.view, new CompositionEvent('compositionend')) + ) + vi.runOnlyPendingTimers() + assertConverted(editor) + expect(editor.getText()).toContain('Prefix ') + expect(editor.commands.undoInputRule()).toBe(true) + expect(editor.getText()).toBe(`Prefix ${source}\n\nAfter`) + }) + + it('converts insertContent applyInputRules without reinserting materialized text', () => { + vi.useFakeTimers() + const { editor } = createEditors(false, '

TARGET

After

') + selectText(editor, 'TARGET') + editor.commands.insertContent(`Prefix ${source}`, { applyInputRules: true }) + vi.runOnlyPendingTimers() + assertConverted(editor) + expect(editor.getText()).toContain('Prefix ') + expect(editor.getText()).toContain('After') + expect(editor.commands.undoInputRule()).toBe(true) + expect(editor.getText()).toBe(`Prefix ${source}\n\nAfter`) + }) + + it('supports ordinary history undo and redo', () => { + const { editor } = createEditors(false, '

TARGET

After

') + selectText(editor, 'TARGET') + editor.view.dispatch(closeHistory(editor.state.tr)) + inputText(editor, `Prefix ${source}`) + const converted = editor.getJSON() + assertConverted(editor) + expect(editor.commands.undo()).toBe(true) + expect(editor.getText()).toBe('TARGET\n\nAfter') + expect(editor.commands.redo()).toBe(true) + expect(editor.getJSON()).toEqual(converted) + }) + + it('supports ordinary undo and redo when conversion is at the document end', () => { + const { editor } = createEditors(false, '

TARGET

') + selectText(editor, 'TARGET') + editor.view.dispatch(closeHistory(editor.state.tr)) + inputText(editor, source) + const converted = editor.getJSON() + assertConverted(editor) + expect(editor.commands.undo()).toBe(true) + expect(editor.getText()).toBe('TARGET') + expect(editor.commands.redo()).toBe(true) + expect(editor.getJSON()).toEqual(converted) + }) + + it('undoes a collaborative conversion without removing peer text', () => { + const seed = markdownToYDoc('TARGET\n\nAfter') + const a = createPeer(seed) + const b = createPeer(seed) + seed.destroy() + selectText(a.editor, 'TARGET') + yUndoPluginKey.getState(a.editor.state).undoManager.stopCapturing() + inputText(a.editor, `Prefix ${source}`) + assertConverted(a.editor) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + selectText(b.editor, 'After') + b.editor.commands.insertContent('Peer text') + Y.applyUpdate(a.doc, Y.encodeStateAsUpdate(b.doc)) + expect(a.editor.commands.undo()).toBe(true) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + expect(a.editor.getText()).toBe('TARGET\n\nPeer text') + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + expect(a.editor.commands.redo()).toBe(true) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + assertConverted(a.editor) + expect(a.editor.getText()).toContain('Peer text') + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + }) +}) + +describe('typed images with the collaborative editor extensions', () => { + it.each([ + { alt: 'Audit image', title: null }, + { alt: '', title: null }, + { alt: 'Logo', title: 'Brand' }, + ])('creates an image, not a bang plus a link ($alt, $title)', ({ alt, title }) => { + const seed = markdownToYDoc('') + const a = createPeer(seed) + const b = createPeer(seed) + seed.destroy() + const source = `![${alt}](https://example.com/logo.png${title ? ` "${title}"` : ''})` + typeText(a.editor, source) + + expect(a.editor.getJSON().content?.filter((node) => node.type === 'image')).toMatchObject([ + { type: 'image', attrs: { src: 'https://example.com/logo.png', alt, title } }, + ]) + expect(a.editor.getText()).not.toContain('!') + expect(a.editor.getMarkdown().trim()).toBe(source) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + }) + + it('continues to create ordinary links during typing', () => { + const seed = markdownToYDoc('') + const { editor } = createPeer(seed) + seed.destroy() + typeText(editor, '[Audit link](https://example.com)') + expect(editor.getJSON().content?.[0]).toMatchObject({ + type: 'paragraph', + content: [{ type: 'text', text: 'Audit link', marks: [{ type: 'link' }] }], + }) + }) + + it('preserves mixed marks in an already typed link label', () => { + const { editor } = createEditors( + false, + '

[Bold and plain](https://example.com

' + ) + editor.commands.setTextSelection(editor.state.doc.firstChild!.nodeSize - 1) + inputText(editor, ')') + expect(editor.getText()).toBe('Bold and plain') + expect(editor.state.doc.firstChild?.content.content).toMatchObject([ + { text: 'Bold', marks: [{ type: { name: 'link' } }, { type: { name: 'bold' } }] }, + { text: ' and plain', marks: [{ type: { name: 'link' } }] }, + ]) + }) + + it('leaves refused link schemes literal and clears the previous input event', () => { + const { editor } = createEditors(false, '

TARGET

After

') + selectText(editor, 'TARGET') + inputText(editor, '[Unsafe](javascript:alert)') + expect(editor.getText()).toContain('[Unsafe](javascript:alert)') + selectText(editor, 'After') + inputText(editor, '[Safe](https://example.com)') + expect(editor.getText()).toBe('[Unsafe](javascript:alert)\n\nSafe') + expect(editor.getHTML()).toContain('href="https://example.com"') + }) + + it('preserves Unicode image labels and surrounding text in a burst', () => { + const { editor } = createEditors(false, '

TARGET suffix

') + selectText(editor, 'TARGET') + inputText(editor, '😃 prefix ![图 😃](https://example.com/a.png)') + expect(editor.getText()).toContain('😃 prefix ') + expect(editor.getText()).toContain(' suffix') + expect(editor.getHTML()).toContain('alt="图 😃"') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx new file mode 100644 index 00000000000..620fabae053 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx @@ -0,0 +1,191 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import type { ReactNodeViewProps } from '@tiptap/react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@tiptap/react', () => ({ + NodeViewWrapper: 'div', + ReactNodeViewRenderer: vi.fn(), +})) + +vi.mock('@tiptap/y-tiptap', () => ({ + ySyncPluginKey: { getState: vi.fn() }, +})) + +vi.mock('@/hooks/use-file-content-source', () => ({ + useFileContentSource: () => ({ + resolveImageSrc: (src: string) => src, + getImageDimensions: () => null, + }), +})) + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/use-editor-editable', + () => ({ useEditorEditable: () => true }) +) + +import { ResizableImageView } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image' + +let host: HTMLDivElement +let root: Root +const editor = { isEditable: true, isDestroyed: false, commands: { focus: vi.fn() } } + +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }) + vi.clearAllMocks() + editor.isEditable = true + editor.isDestroyed = false + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) + +afterEach(() => { + act(() => root.unmount()) + host.remove() +}) + +function pointerEvent( + type: string, + { pointerId, clientX = 0, button = 0 }: { pointerId: number; clientX?: number; button?: number } +): Event { + const event = new Event(type, { bubbles: true, cancelable: true }) + Object.defineProperties(event, { + pointerId: { value: pointerId }, + clientX: { value: clientX }, + button: { value: button }, + pointerType: { value: 'touch' }, + }) + return event +} + +function renderImage( + updateAttributes: ReturnType, + dimensions: { width?: string | null; height?: string | null } = {} +): HTMLButtonElement { + const props = { + node: { + attrs: { + src: '/image.png', + alt: '', + title: null, + width: null, + height: '100', + ...dimensions, + href: null, + }, + }, + updateAttributes, + selected: true, + editor, + } as unknown as ReactNodeViewProps + act(() => root.render()) + const image = host.querySelector('img') + const handle = host.querySelector('button[aria-label="Resize image"]') + if (!image || !handle) throw new Error('Resizable image did not render') + Object.defineProperty(image, 'offsetWidth', { configurable: true, value: 200 }) + Object.assign(handle, { + setPointerCapture: vi.fn(), + hasPointerCapture: vi.fn(() => true), + releasePointerCapture: vi.fn(), + }) + return handle +} + +describe('ResizableImageView', () => { + it('renders a height-only image proportionally without fixing its responsive height', () => { + renderImage(vi.fn()) + const image = host.querySelector('img') + if (!image) throw new Error('Missing image') + Object.defineProperties(image, { + naturalWidth: { configurable: true, value: 400 }, + naturalHeight: { configurable: true, value: 200 }, + }) + act(() => image.dispatchEvent(new Event('load'))) + + expect(image.style.height).toBe('') + expect(image.style.width).toBe('calc(200px)') + expect(image.style.aspectRatio).toBe('400 / 200') + }) + + it.each([ + { width: '600', height: '400' }, + { width: '600px', height: '400px' }, + { width: '600', height: '400px' }, + ])('uses the authored ratio for responsive pixel dimensions: %j', (dimensions) => { + renderImage(vi.fn(), dimensions) + const image = host.querySelector('img')! + expect(image.style.width).toBe('600px') + expect(image.style.height).toBe('') + expect(image.style.aspectRatio).toBe('600 / 400') + }) + + it('preserves relative dimensions instead of assuming they are pixel ratios', () => { + renderImage(vi.fn(), { width: '50%', height: '100px' }) + const image = host.querySelector('img')! + expect(image.style.width).toBe('50%') + expect(image.style.height).toBe('100px') + }) + + it.each(['50%', 'auto', '10em', 'calc(50% - 10px)', 'min-content', 'inherit'])( + 'preserves the native height-only CSS value %s before and after loading', + (height) => { + renderImage(vi.fn(), { height }) + const image = host.querySelector('img')! + expect(image.style.width).toBe('') + expect(image.style.height).toBe(height) + expect(image.style.maxHeight).toBe('') + + Object.defineProperties(image, { + naturalWidth: { configurable: true, value: 400 }, + naturalHeight: { configurable: true, value: 200 }, + }) + act(() => image.dispatchEvent(new Event('load'))) + + expect(image.style.width).toBe('') + expect(image.style.height).toBe(height) + expect(image.style.maxHeight).toBe('') + } + ) + + it('commits one proportional width change and clears a stale explicit height', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 7, clientX: 160 }))) + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 7, clientX: 160 }))) + + expect(updateAttributes).toHaveBeenCalledOnce() + expect(updateAttributes).toHaveBeenCalledWith({ width: '260', height: null }) + }) + + it('ignores unrelated pointers and cancels without mutating document attributes', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 8, clientX: 180 }))) + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 8, clientX: 180 }))) + act(() => window.dispatchEvent(pointerEvent('pointercancel', { pointerId: 7 }))) + expect(updateAttributes).not.toHaveBeenCalled() + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 9, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 9, clientX: 140 }))) + act(() => window.dispatchEvent(new Event('blur'))) + expect(updateAttributes).not.toHaveBeenCalled() + }) + + it('does not commit a resize after live editing becomes unavailable', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 7, clientX: 160 }))) + editor.isEditable = false + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 7, clientX: 160 }))) + + expect(updateAttributes).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts index dccc926d6b4..51f3889441a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts @@ -1,5 +1,7 @@ -import type { JSONContent } from '@tiptap/core' -import { Image } from '@tiptap/extension-image' +import { InputRule, type JSONContent } from '@tiptap/core' +import { Image, inputRegex } from '@tiptap/extension-image' +import { Lexer, Tokenizer } from 'marked' +import { createTextInputRulePlugins } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/text-input-rule' /** * React-free schema half of the image node. Lives apart from {@link ./image} (its React resize node @@ -16,9 +18,6 @@ import { Image } from '@tiptap/extension-image' * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it * round-trips losslessly (and the file stays editable rather than opening read-only). */ -const LINKED_IMAGE_RE = - /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ - /** Escape a value for safe interpolation into a double-quoted HTML attribute. */ function escapeAttr(value: string): string { return value @@ -28,16 +27,27 @@ function escapeAttr(value: string): string { .replace(/>/g, '>') } +function decodeAttr(value: string): string { + return value + .replace(/"/g, '"') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') +} + +function imageAttrsFromHtml(raw: string): Record | null { + if (!/^ = {} + const attributePattern = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g + for (const match of raw.matchAll(attributePattern)) { + attrs[match[1].toLowerCase()] = decodeAttr(match[2] ?? match[3] ?? match[4] ?? '') + } + return typeof attrs.src === 'string' ? attrs : null +} + /** - * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when - * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to - * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is - * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. - * - * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer - * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on - * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the - * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. + * Markdown has no image dimensions, so sized images use HTML. Links wrap either representation: + * `[![alt](src)](href)` or `[](href)`. */ function imageMarkdown(node: JSONContent): string { const attrs = node.attrs ?? {} @@ -49,9 +59,8 @@ function imageMarkdown(node: JSONContent): string { const width = attrs.width const height = attrs.height let image: string - if ((width || height) && !href) { - const parts = [`src="${escapeAttr(src)}"`] - if (alt) parts.push(`alt="${escapeAttr(alt)}"`) + if (width || height) { + const parts = [`src="${escapeAttr(src)}"`, `alt="${escapeAttr(alt)}"`] if (title) parts.push(`title="${escapeAttr(title)}"`) if (width) parts.push(`width="${escapeAttr(String(width))}"`) if (height) parts.push(`height="${escapeAttr(String(height))}"`) @@ -67,7 +76,8 @@ function imageMarkdown(node: JSONContent): string { // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the // image title escaping above). const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' - return `[${image}](${href}${hrefTitlePart})` + const safeHref = /[\s()]/.test(href) ? `<${href}>` : href + return `[${image}](${safeHref}${hrefTitlePart})` } interface MarkdownImageToken { @@ -78,6 +88,8 @@ interface MarkdownImageToken { /** Built-in image token holds the source URL here; our linked token holds the link target. */ href?: string hrefTitle?: string | null + width?: string | null + height?: string | null /** Built-in image token holds the alt text here. */ text?: string } @@ -94,6 +106,8 @@ function parseImageToken(token: MarkdownImageToken): JSONContent { title: token.title ?? null, href: token.href ?? null, hrefTitle: token.hrefTitle ?? null, + width: token.width ?? null, + height: token.height ?? null, } : { src: token.href ?? '', @@ -101,6 +115,8 @@ function parseImageToken(token: MarkdownImageToken): JSONContent { title: token.title ?? null, href: null, hrefTitle: null, + width: null, + height: null, }, } } @@ -129,6 +145,26 @@ const hrefTitleAttr = { default: null, rendered: false } * round-trip path (no node view) and the live {@link ResizableImage}. */ export const MarkdownImage = Image.extend({ + addInputRules: () => [], + addProseMirrorPlugins() { + return createTextInputRulePlugins( + this.editor, + this.name, + new InputRule({ + find: new RegExp(`${inputRegex.source}(?![\\s\\S])`), + handler: ({ state, range, match }) => { + const [, syntax, alt, src, title] = match + state.tr + .replaceRangeWith( + range.from + match[0].indexOf(syntax), + range.to, + this.type.create({ alt, src, title: title ?? null }) + ) + .scrollIntoView() + }, + }) + ) + }, addAttributes() { return { ...this.parent?.(), @@ -141,18 +177,44 @@ export const MarkdownImage = Image.extend({ markdownTokenizer: { name: 'image', level: 'inline', - start: (src: string) => src.indexOf('[!['), + start: (src: string) => { + const markdown = src.indexOf('[![') + const html = src.search(/\[ { - const match = LINKED_IMAGE_RE.exec(src) - if (!match) return undefined + if (!src.startsWith('[![') && !/^\[ + Array.from( + binding.type.createTreeWalker( + (child) => child instanceof XmlElement && child.nodeName === 'image' + ) + ) + const originalImages = images().map((element) => ({ + element, + node: binding.mapping.get(element), + })) + const source = target?.getAttribute('src') + return (currentNode: Node | null): boolean => { + if ( + !target || + !currentNode || + target.parent !== parent || + binding.mapping.get(target) !== currentNode || + target.getAttribute('src') !== source + ) + return false + const currentImages = images() + return ( + originalImages.length === currentImages.length && + originalImages.every(({ element, node }, index) => { + if (element !== currentImages[index]) return false + if (element === target) return true + const current = binding.mapping.get(element) + return node instanceof Node && current instanceof Node && node.eq(current) + }) + ) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx index a1c8600de67..9e7f4dc9f13 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image.tsx @@ -1,23 +1,32 @@ import { type CSSProperties, useEffect, useMemo, useRef, useState } from 'react' -import { cn } from '@sim/emcn' +import { Button, cn } from '@sim/emcn' import { NodeSelection, Plugin } from '@tiptap/pm/state' import type { ReactNodeViewProps } from '@tiptap/react' import { NodeViewWrapper, ReactNodeViewRenderer } from '@tiptap/react' +import { type ProsemirrorBinding, ySyncPluginKey } from '@tiptap/y-tiptap' +import { MarkdownImage } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema' +import { createImageTargetGuard } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-target' +import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { useEditorEditable } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/use-editor-editable' import { type ImageDimensions, useFileContentSource } from '@/hooks/use-file-content-source' -import { MarkdownImage } from './image-schema' -import { normalizeLinkHref } from './markdown-fidelity' -import { useEditorEditable } from './use-editor-editable' const MIN_WIDTH = 64 -/** A bare pixel count (`"640"`) that needs a `px` suffix, vs. an already-unit'd width (`"50%"`). */ -const BARE_PIXEL_WIDTH = /^\d+$/ +/** A bare pixel count (`"640"`) that needs a `px` suffix, vs. an already-unit'd size (`"50%"`). */ +const BARE_PIXEL_SIZE = /^\d+$/ +const PIXEL_SIZE = /^\d+(?:\.\d+)?px$/ /** * Drag-to-resize image node view (handle at the bottom-right, revealed on selection). Dragging * commits the new pixel width to the `width` attribute, which serializes to ``. */ -function ResizableImageView({ node, updateAttributes, selected, editor }: ReactNodeViewProps) { +export function ResizableImageView({ + node, + updateAttributes, + selected, + editor, + getPos, +}: ReactNodeViewProps) { const source = useFileContentSource() const imageRef = useRef(null) const dragAbortRef = useRef(null) @@ -37,6 +46,7 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN alt?: string title?: string width?: string | null + height?: string | null href?: string | null } @@ -54,8 +64,28 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN const startResize = (event: React.PointerEvent) => { event.preventDefault() + if (event.button !== 0 || dragging) return const image = imageRef.current if (!image) return + const binding: ProsemirrorBinding | undefined = ySyncPluginKey.getState(editor.state)?.binding + const position = binding ? getPos() : undefined + const currentNode = typeof position === 'number' ? editor.state.doc.nodeAt(position) : null + const matchesTarget = + binding && currentNode ? createImageTargetGuard(binding, currentNode) : undefined + /** A node view can be reused for a replacement image, even when every attribute is identical. */ + const isCurrentTarget = () => { + if (!binding) return true + const position = getPos() + return ( + matchesTarget !== undefined && + typeof position === 'number' && + matchesTarget(editor.state.doc.nodeAt(position)) + ) + } + if (!isCurrentTarget()) return + const handle = event.currentTarget + const pointerId = event.pointerId + handle.setPointerCapture(pointerId) const startX = event.clientX const startWidth = image.offsetWidth setDragging(true) @@ -67,29 +97,66 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN window.addEventListener( 'pointermove', (move) => { + if (move.pointerId !== pointerId) return const next = Math.max(MIN_WIDTH, Math.round(startWidth + (move.clientX - startX))) dragWidthRef.current = next setDragWidth(next) }, { signal } ) - const finish = () => { + const finish = (commit: boolean) => { const finalWidth = dragWidthRef.current setDragging(false) setDragWidth(null) dragWidthRef.current = null + if (handle.hasPointerCapture(pointerId)) handle.releasePointerCapture(pointerId) controller.abort() - if (finalWidth !== null) updateAttributes({ width: String(finalWidth) }) + if ( + commit && + finalWidth !== null && + editor.isEditable && + !editor.isDestroyed && + isCurrentTarget() + ) { + updateAttributes({ width: String(finalWidth), height: null }) + } + } + if (binding) { + const onTransaction = () => { + if (!isCurrentTarget()) finish(false) + } + editor.on('transaction', onTransaction) + signal.addEventListener('abort', () => editor.off('transaction', onTransaction), { + once: true, + }) } - window.addEventListener('pointerup', finish, { signal }) - window.addEventListener('pointercancel', finish, { signal }) + window.addEventListener( + 'pointerup', + (up) => { + if (up.pointerId === pointerId) finish(true) + }, + { signal } + ) + window.addEventListener( + 'pointercancel', + (cancel) => { + if (cancel.pointerId === pointerId) finish(false) + }, + { signal } + ) + window.addEventListener('blur', () => finish(false), { signal }) } const committedWidth = attrs.width - ? BARE_PIXEL_WIDTH.test(attrs.width) + ? BARE_PIXEL_SIZE.test(attrs.width) ? `${attrs.width}px` : attrs.width : undefined + const committedHeight = attrs.height + ? BARE_PIXEL_SIZE.test(attrs.height) + ? `${attrs.height}px` + : attrs.height + : undefined // Stored intrinsic dimensions reserve the box on the very first render. Memoized on the src (not the // live drag width) so a resize drag never re-scans the file list. Falls back to what we measured on // load this session for a first-ever view the metadata hasn't caught up on. @@ -101,17 +168,40 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN // stored value is stale (e.g. left over after the file's content was replaced) — so it wins once // available; stored metadata only reserves the box pre-load. Equal in the common case, so no shift. const intrinsicDimensions = measuredDimensions ?? storedDimensions + const hasPixelHeight = committedHeight !== undefined && PIXEL_SIZE.test(committedHeight) + const authoredDimensions = + committedWidth && + committedHeight && + PIXEL_SIZE.test(committedWidth) && + PIXEL_SIZE.test(committedHeight) && + Number.parseFloat(committedWidth) > 0 && + Number.parseFloat(committedHeight) > 0 + ? { width: Number.parseFloat(committedWidth), height: Number.parseFloat(committedHeight) } + : null + const displayDimensions = + dragWidth === null ? (authoredDimensions ?? intrinsicDimensions) : intrinsicDimensions const displayWidth = dragWidth !== null ? `${dragWidth}px` - : (committedWidth ?? (intrinsicDimensions ? `${intrinsicDimensions.width}px` : undefined)) + : (committedWidth ?? + (intrinsicDimensions && (!committedHeight || hasPixelHeight) + ? committedHeight + ? `calc(${committedHeight} * ${intrinsicDimensions.width / intrinsicDimensions.height})` + : `${intrinsicDimensions.width}px` + : undefined)) // width + aspect-ratio (with `max-w-full`/`h-auto` from the class list) reserves a responsive box the // image can't reflow into, per the CLS-avoidance pattern for known-ratio responsive images. React drops // the undefined keys, so an unmeasured image simply gets no reservation (its prior behavior). const imageStyle: CSSProperties = { width: displayWidth, - aspectRatio: intrinsicDimensions - ? `${intrinsicDimensions.width} / ${intrinsicDimensions.height}` + height: + dragWidth === null && !authoredDimensions && (committedWidth || !hasPixelHeight) + ? committedHeight + : undefined, + maxHeight: + dragWidth === null && !committedWidth && hasPixelHeight ? committedHeight : undefined, + aspectRatio: displayDimensions + ? `${displayDimensions.width} / ${displayDimensions.height}` : undefined, } @@ -182,12 +272,16 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN image )} {editable && (selected || dragging) && ( - )} ) @@ -210,6 +304,7 @@ export const ResizableImage = MarkdownImage.extend({ addProseMirrorPlugins() { const nodeName = this.name return [ + ...(this.parent?.() ?? []), new Plugin({ props: { handleClickOn(view, _pos, node, nodePos, event) { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts index 6f59a439523..9b6fdd2bde4 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts @@ -1,12 +1,14 @@ import { Extension, InputRule } from '@tiptap/core' -import { normalizeLinkHref } from './markdown-fidelity' +import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { createTextInputRulePlugins } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/text-input-rule' /** * Typed markdown link: `[text](url)` or `[text](url "title")`, completed by the closing `)`. The URL * is space-free (markdown requires `` for spaces, which this intentionally skips). StarterKit's * Link ships no input rule — only paste/autolink — so without this, typed link syntax stays literal. + * A preceding bang belongs to the image input rule, which must receive the complete syntax. */ -const LINK_INPUT_RULE = /\[([^\]]+)]\(([^)\s]+)(?:\s+"([^"]*)")?\)$/ +const LINK_INPUT_RULE = /(? { - if (state.selection.$from.parent.type.spec.code) return null const linkType = state.schema.marks.link if (!linkType) return null const [fullMatch, text, rawHref, title] = match @@ -33,13 +36,12 @@ export const MarkdownLinkInputRule = Extension.create({ const { tr } = state const textStart = range.from + fullMatch.indexOf(text) const textEnd = textStart + text.length - if (textEnd < range.to) tr.delete(textEnd, range.to) - if (textStart > range.from) tr.delete(range.from, textStart) + tr.replaceWith(range.from, range.to, tr.doc.slice(textStart, textEnd).content) const markEnd = range.from + text.length tr.addMark(range.from, markEnd, linkType.create({ href, title: title || null })) tr.removeStoredMark(linkType) }, - }), - ] + }) + ) }, }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts index fd46d579527..ba3ea95b755 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity.ts @@ -5,7 +5,8 @@ */ const BOM = '\uFEFF' -const FRONTMATTER_REGEX = /^---\r?\n(?:[\s\S]*?\r?\n)?---[ \t]*(?:\r?\n)*/ +const FRONTMATTER_REGEX = /^---\r?\n(?:[\s\S]*?\r?\n)?---[ \t]*(?=\r?\n|$)(?:\r?\n)*/ +const FRONTMATTER_KEY_REGEX = /^(?:[A-Za-z0-9_-]+|'(?:[^']|'')*'|"(?:[^"\\]|\\.)*")[ \t]*:/ const ESCAPED_CALLOUT_REGEX = /^(\s*>(?:\s*>)*\s*)\\\[!([A-Za-z]+)\\\]/gm /** @@ -56,20 +57,24 @@ export function splitFrontmatter(markdown: string): SplitMarkdown { } /** - * A leading `---…---` block is YAML frontmatter unless its first content line is markdown rather than - * a `key:` — so a doc that opens with a `---` thematic break (e.g. a changelog whose next `---` closes - * the regex) stays in the editor body instead of being held out-of-band and hidden. An empty block - * (`---\n---`) is still treated as (empty) frontmatter. + * Recognize mapping-style metadata without parsing or rewriting its values. Comments can precede + * a plain or quoted key; comment-only blocks remain visible because they may be Markdown headings + * between thematic breaks. A genuinely empty block is still treated as frontmatter. */ function isYamlFrontmatterBlock(block: string): boolean { const interior = block.replace(/^---[ \t]*\r?\n/, '') + let hasComment = false for (const rawLine of interior.split('\n')) { const line = rawLine.trim() if (line === '') continue - if (line.startsWith('---')) return true - return /^[A-Za-z0-9_-]+[ \t]*:/.test(line) + if (line === '---') return !hasComment + if (line.startsWith('#')) { + hasComment = true + continue + } + return FRONTMATTER_KEY_REGEX.test(line) } - return true + return !hasComment } export function applyFrontmatter(frontmatter: string, body: string): string { diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts index 83a10d3f63c..3affe48b409 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome.ts @@ -1,9 +1,3 @@ -/** - * Shared chrome for the editor's selection-driven bubble toolbars — the text formatting bar and the - * table bar. Single source of truth so the two read identically; never re-derive this class string - * per consumer. - */ - -/** The floating toolbar: bordered card, popover layer, with the enter animation. */ +/** Shared floating toolbar chrome for text, table, and image selections. */ export const BUBBLE_MENU_CLASS = - 'fade-in-0 z-[var(--z-popover)] flex animate-in items-center gap-0.5 rounded-lg border border-[var(--border)] bg-[var(--bg)] p-1 shadow-xs duration-150 ease-out motion-reduce:animate-none' + 'scrollbar-none fade-in-0 z-[var(--z-popover)] flex max-w-[calc(100%_-_1rem)] animate-in items-center gap-0.5 overflow-x-auto rounded-lg border border-[var(--border)] bg-[var(--bg)] p-1 shadow-xs duration-150 ease-out motion-reduce:animate-none' diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-collaboration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-collaboration.test.tsx new file mode 100644 index 00000000000..abde3c909ec --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-collaboration.test.tsx @@ -0,0 +1,209 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { Tooltip } from '@sim/emcn' +import { Editor } from '@tiptap/core' +import { Plugin } from '@tiptap/pm/state' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { markdownToYDoc } from '@/lib/collab-doc/converter' +import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' +import { EditorBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu' + +interface Peer { + doc: Y.Doc + awareness: Awareness + editor: Editor +} + +let a: Peer +let b: Peer +let root: Root +let viewport: HTMLDivElement + +beforeEach(() => { + vi.useFakeTimers() + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + document.elementFromPoint ??= () => null + const seed = markdownToYDoc('## Before\n\nbefore [format](https://example.com/original) after') + const makePeer = (): Peer => { + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(seed)) + const awareness = new Awareness(doc) + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#ffffff' } }, + }), + editorProps: { handleScrollToSelection: () => true }, + }) + return { doc, awareness, editor } + } + a = makePeer() + b = makePeer() + seed.destroy() + viewport = document.createElement('div') + const host = document.createElement('div') + viewport.append(a.editor.view.dom, host) + document.body.append(viewport) + vi.spyOn(a.editor.view, 'coordsAtPos').mockReturnValue({ + top: 10, + bottom: 30, + left: 10, + right: 50, + }) + root = createRoot(host) + act(() => { + root.render( + + + + ) + }) +}) + +afterEach(() => { + act(() => root.unmount()) + vi.clearAllTimers() + for (const peer of [a, b]) { + peer.editor.destroy() + peer.awareness.destroy() + peer.doc.destroy() + } + viewport.remove() + vi.restoreAllMocks() + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +function textPosition(editor: Editor, text: string): number { + let position = -1 + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text === text) position = pos + }) + expect(position).toBeGreaterThan(-1) + return position +} + +async function openDraft(caret: boolean): Promise { + const from = textPosition(a.editor, 'format') + act(() => { + a.editor.commands.setTextSelection(caret ? from + 2 : { from, to: from + 6 }) + a.editor.view.focus() + a.editor.view.dom.dispatchEvent( + new KeyboardEvent('keydown', { key: 'k', ctrlKey: true, bubbles: true, cancelable: true }) + ) + }) + await act(async () => vi.advanceTimersToNextFrame()) + const input = viewport.querySelector('input[aria-label="Link URL"]') + expect(input).not.toBeNull() + if (!input) throw new Error('Missing link draft') + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call( + input, + 'https://example.com/draft' + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + expect(document.activeElement).toBe(input) + return input +} + +async function receivePeerEdit(): Promise { + await act(async () => Y.applyUpdate(a.doc, Y.encodeStateAsUpdate(b.doc))) +} + +describe('link drafts during actual collaborative updates', () => { + it.each([false, true])( + 'preserves and applies a draft after a peer prefix edit (caret=%s)', + async (caret) => { + const input = await openDraft(caret) + b.editor.commands.insertContentAt(textPosition(b.editor, 'Before') + 6, ' PEER') + await receivePeerEdit() + + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + expect(document.activeElement).toBe(input) + expect(input.value).toBe('https://example.com/draft') + const apply = viewport.querySelector('button[aria-label="Apply link"]') + await act(async () => apply?.click()) + expect(a.editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/draft' + ) + expect(a.editor.getText()).toContain('Before PEER') + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + } + ) + + it('maps the original caret through local and peer edits before canceling', async () => { + const input = await openDraft(true) + act(() => a.editor.view.dispatch(a.editor.state.tr.insertText('LOCAL ', 1))) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + b.editor.commands.insertContentAt(1, 'PEER ') + await receivePeerEdit() + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))) + expect(a.editor.state.selection.empty).toBe(true) + expect(a.editor.state.selection.from).toBe(textPosition(a.editor, 'format') + 2) + act(() => a.editor.commands.insertContent('X')) + expect(a.editor.getText()).toContain('foXrmat') + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) + }) + + it('retains a draft across read-only permission intervals and peer updates', async () => { + const input = await openDraft(false) + const apply = viewport.querySelector('button[aria-label="Apply link"]') + act(() => a.editor.setEditable(false)) + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + b.editor.commands.insertContentAt(1, 'PEER ') + await receivePeerEdit() + act(() => apply?.click()) + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) + act(() => a.editor.setEditable(true)) + act(() => a.editor.view.focus()) + await act(async () => vi.advanceTimersToNextFrame()) + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + await act(async () => apply?.click()) + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/draft' + ) + }) + + it('discards the draft when a peer deletes its entire target', async () => { + const input = await openDraft(false) + const from = textPosition(b.editor, 'format') + b.editor.commands.deleteRange({ from, to: from + 6 }) + await receivePeerEdit() + expect(viewport.contains(input)).toBe(false) + expect(a.editor.view.dom.querySelector('a')).toBeNull() + }) + + it('maps a peer transaction together with a locally appended transaction exactly once', async () => { + const input = await openDraft(false) + let appended = false + a.editor.registerPlugin( + new Plugin({ + appendTransaction: (transactions, _oldState, state) => { + if (appended || !transactions.some((transaction) => transaction.docChanged)) return null + appended = true + return state.tr.insertText('APPENDED ', 1) + }, + }) + ) + b.editor.commands.insertContentAt(1, 'PEER ') + await receivePeerEdit() + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + await act(async () => { + viewport.querySelector('button[aria-label="Apply link"]')?.click() + }) + expect(a.editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/draft' + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx index 1ba7457cc4b..98321f451e4 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx @@ -16,8 +16,11 @@ import { TextQuote, Unlink, } from '@sim/emcn/icons' +import type { MappablePosition } from '@tiptap/core' +import type { Node } from '@tiptap/pm/model' import { PluginKey, + type Selection, type SelectionBookmark, TextSelection, type Transaction, @@ -58,6 +61,56 @@ function revealBubbleMenu(editor: Editor, key: PluginKey): void { editor.commands.setMeta(key, 'updatePosition') } +type CapturedSelection = + | { anchor: MappablePosition; head: MappablePosition; bookmark?: never } + | { bookmark: SelectionBookmark; anchor?: never; head?: never } + +interface LinkSelection { + target: CapturedSelection + original: CapturedSelection +} + +/** Collaborative positions survive the full-document replacements used to apply Yjs updates. */ +function captureSelection(editor: Editor): CapturedSelection { + const { selection } = editor.state + return selection instanceof TextSelection + ? { + anchor: editor.utils.createMappablePosition(selection.anchor), + head: editor.utils.createMappablePosition(selection.head), + } + : { bookmark: selection.getBookmark() } +} + +function mapSelection( + editor: Editor, + selection: CapturedSelection, + transaction: Transaction +): CapturedSelection { + return selection.bookmark + ? { bookmark: selection.bookmark.map(transaction.mapping) } + : { + anchor: editor.utils.getUpdatedPosition(selection.anchor, transaction).position, + head: editor.utils.getUpdatedPosition(selection.head, transaction).position, + } +} + +function resolveSelection(selection: CapturedSelection, doc: Node): Selection { + return selection.bookmark + ? selection.bookmark.resolve(doc) + : TextSelection.between( + doc.resolve(selection.anchor.position), + doc.resolve(selection.head.position) + ) +} + +/** Keep the editing target separate from the selection restored when the user cancels. */ +function captureLinkSelection(editor: Editor): LinkSelection | null { + const original = captureSelection(editor) + if (editor.state.selection.empty) editor.commands.extendMarkRange('link') + const { selection } = editor.state + return selection.empty ? null : { target: captureSelection(editor), original } +} + interface EditorBubbleMenuProps { editor: Editor /** The editor's scrollable viewport, so the toolbar repositions with the selection as the pane scrolls. */ @@ -79,7 +132,7 @@ export function EditorBubbleMenu({ }: EditorBubbleMenuProps) { const [linkValue, setLinkValue] = useState(null) const linkInputRef = useRef(null) - const linkRangeRef = useRef(null) + const linkSelectionRef = useRef(null) const isEditingLink = linkValue !== null const [bubbleMenuKey] = useState(() => new PluginKey('markdownBubbleMenu')) @@ -122,18 +175,25 @@ export function EditorBubbleMenu({ transaction: Transaction appendedTransactions?: Transaction[] }) => { - let bookmark = linkRangeRef.current - if (!bookmark) return - for (const change of [transaction, ...appendedTransactions]) - bookmark = bookmark.map(change.mapping) - const selection = bookmark.resolve(editor.state.doc) - linkRangeRef.current = - selection instanceof TextSelection && !selection.empty ? bookmark : null - if (!linkRangeRef.current) setLinkValue(null) + let captured = linkSelectionRef.current + if (!captured) return + for (const change of [transaction, ...appendedTransactions]) { + captured = { + target: mapSelection(editor, captured.target, change), + original: mapSelection(editor, captured.original, change), + } + } + const selection = resolveSelection(captured.target, editor.state.doc) + linkSelectionRef.current = + selection instanceof TextSelection && !selection.empty ? captured : null + if (!linkSelectionRef.current) setLinkValue(null) } const exitOnCollapse = () => { const { from, to } = editor.state.selection - if (from === to) setLinkValue(null) + if (from === to) { + linkSelectionRef.current = null + setLinkValue(null) + } } editor.on('selectionUpdate', exitOnCollapse) editor.on('transaction', mapLinkRange) @@ -144,10 +204,8 @@ export function EditorBubbleMenu({ }, [editor]) /** - * Linear-style reveal: the toolbar stays hidden while the pointer is down (the drag gate in - * `shouldShow`) and surfaces on release. `mouseup`/`blur` listen on `window` so a release outside - * the editor — or off-screen, where no `mouseup` fires — still clears the drag flag; otherwise it - * could wedge `true` and suppress the toolbar for later keyboard selections. + * Window-level release/cancel/blur handlers clear the drag gate even outside the editor, + * preventing a lost pointer release from suppressing later keyboard selections. */ useEffect(() => { const dom = editor.view.dom @@ -163,19 +221,23 @@ export function EditorBubbleMenu({ const onWindowBlur = () => { isPointerDownRef.current = false } - dom.addEventListener('mousedown', onPointerDown) - window.addEventListener('mouseup', onPointerUp) + dom.addEventListener('pointerdown', onPointerDown) + window.addEventListener('pointerup', onPointerUp) + window.addEventListener('pointercancel', onWindowBlur) window.addEventListener('blur', onWindowBlur) return () => { - dom.removeEventListener('mousedown', onPointerDown) - window.removeEventListener('mouseup', onPointerUp) + dom.removeEventListener('pointerdown', onPointerDown) + window.removeEventListener('pointerup', onPointerUp) + window.removeEventListener('pointercancel', onWindowBlur) window.removeEventListener('blur', onWindowBlur) } }, [editor, bubbleMenuKey]) const openLinkEditor = () => { if (!editor.isEditable || editor.isActive('codeBlock') || editor.isActive('code')) return - linkRangeRef.current = editor.state.selection.getBookmark() + const captured = captureLinkSelection(editor) + if (!captured) return + linkSelectionRef.current = captured setLinkValue(editor.getAttributes('link').href ?? '') } @@ -192,10 +254,11 @@ export function EditorBubbleMenu({ ) return if (event.key?.toLowerCase() !== 'k') return - const { from, to } = editor.state.selection - if (from === to || editor.isActive('codeBlock') || editor.isActive('code')) return + if (editor.isActive('codeBlock') || editor.isActive('code')) return + const captured = captureLinkSelection(editor) + if (!captured) return event.preventDefault() - linkRangeRef.current = editor.state.selection.getBookmark() + linkSelectionRef.current = captured setLinkValue(editor.getAttributes('link').href ?? '') } dom.addEventListener('keydown', openLinkOnShortcut) @@ -206,19 +269,31 @@ export function EditorBubbleMenu({ const commitCapturedLink = (href: string) => { if (editor.isDestroyed || !editor.isEditable) return - const selection = linkRangeRef.current?.resolve(editor.state.doc) + const captured = linkSelectionRef.current + const selection = captured && resolveSelection(captured.target, editor.state.doc) if (selection instanceof TextSelection && !selection.empty) { applyLink( editor.chain().focus().setTextSelection({ from: selection.from, to: selection.to }), href ) } - linkRangeRef.current = null + linkSelectionRef.current = null setLinkValue(null) } const commitLink = () => commitCapturedLink(linkValue ?? '') const removeLink = () => commitCapturedLink('') + const cancelLink = () => { + const captured = linkSelectionRef.current + linkSelectionRef.current = null + setLinkValue(null) + if (!captured || editor.isDestroyed) return + editor.view.dispatch( + editor.state.tr.setSelection(resolveSelection(captured.original, editor.state.doc)) + ) + editor.commands.focus() + } + const { resolveAnchor, appendTo } = useBubbleMenuFloating(editor, scrollContainerRef) const canFocus = useCallback( () => hasFormattableSelection(editor, editor.state.selection.from, editor.state.selection.to), @@ -229,6 +304,7 @@ export function EditorBubbleMenu({ pluginKey: bubbleMenuKey, roving: !isEditingLink, canFocus, + onEscape: isEditingLink ? cancelLink : undefined, }) const shouldShow = useCallback( @@ -267,7 +343,7 @@ export function EditorBubbleMenu({ value={linkValue ?? ''} onChange={setLinkValue} onCommit={commitLink} - onCancel={() => setLinkValue(null)} + onCancel={cancelLink} /> {active.link && ( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx index e57463a6708..817cfbf314a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx @@ -109,6 +109,16 @@ async function openLinkEditor(): Promise { return input } +async function openLinkAtCaret(offset: number): Promise { + select('format', true) + act(() => editor.commands.setTextSelection(editor.state.selection.from + offset)) + expect(key(editor.view.dom, 'k', { ctrlKey: true }).defaultPrevented).toBe(true) + await frame() + const input = linkGroup().querySelector('input[aria-label="Link URL"]') + if (!input) throw new Error('Missing link URL field') + return input +} + function changeUrl(input: HTMLInputElement, value: string): void { act(() => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) @@ -269,6 +279,183 @@ describe('real editor BubbleMenu keyboard integration', () => { expect(key(remove, 'Tab').defaultPrevented).toBe(false) }) + it('edits the complete existing link from a collapsed caret with Cmd/Ctrl+K', async () => { + select('format') + act(() => editor.commands.setLink({ href: 'https://example.com/original' })) + select('format', true) + + expect(key(editor.view.dom, 'k', { ctrlKey: true }).defaultPrevented).toBe(true) + await frame() + const input = linkGroup().querySelector('input[aria-label="Link URL"]') + expect(input).not.toBeNull() + if (!input) return + + changeUrl(input, 'https://example.com/replacement') + key(input, 'Enter') + await frame() + + const link = editor.view.dom.querySelector('a') + expect(link?.textContent).toBe('format') + expect(link?.getAttribute('href')).toBe('https://example.com/replacement') + }) + + it.each([0, 2, 6])( + 'prefills the complete link at caret offset %i without changing it on apply', + async (offset) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const before = editor.getJSON() + const input = await openLinkAtCaret(offset) + + expect(input.value).toBe('https://example.com/original') + expect(button(linkGroup(), 'Remove link').disabled).toBe(false) + key(input, 'Enter') + await frame() + expect(editor.getJSON()).toEqual(before) + } + ) + + it('uses the same adjacent link for the captured range, URL, and update', async () => { + act(() => + editor.commands.setContent( + editorNormalForm( + '[before](https://example.com/first)[format](https://example.com/second) after' + ) + ) + ) + const input = await openLinkAtCaret(0) + expect(input.value).toBe('https://example.com/second') + changeUrl(input, 'https://example.com/replacement') + key(input, 'Enter') + await frame() + + const links = editor.view.dom.querySelectorAll('a') + expect([...links].map((link) => [link.textContent, link.getAttribute('href')])).toEqual([ + ['before', 'https://example.com/first'], + ['format', 'https://example.com/replacement'], + ]) + }) + + it.each([false, true])( + 'keeps a caret-opened link draft through a peer edit with read-only interval %s', + async (readOnly) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const input = await openLinkAtCaret(2) + const group = linkGroup() + const apply = button(group, 'Apply link') + changeUrl(input, 'https://example.com/replacement') + if (readOnly) { + const before = editor.getJSON() + act(() => editor.setEditable(false)) + act(() => apply.click()) + expect(editor.getJSON()).toEqual(before) + } + act(() => editor.view.dispatch(editor.state.tr.insertText('remote ', 1))) + + if (readOnly) act(() => editor.setEditable(true)) + await frame() + expect((readOnly ? group : linkGroup()).querySelector('input')).toBe(input) + expect(input.value).toBe('https://example.com/replacement') + act(() => apply.click()) + await frame() + expect(editor.getText()).toBe('remote before format after') + expect(editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/replacement' + ) + } + ) + + it.each([0, 2, 6])('restores the original caret at link offset %i on cancel', async (offset) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + offset + const input = await openLinkAtCaret(offset) + const before = editor.getJSON() + changeUrl(input, 'https://example.com/cancelled') + key(input, 'Escape') + await frame() + + expect(viewport.contains(input)).toBe(false) + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret) + expect(editor.getJSON()).toEqual(before) + act(() => editor.commands.insertContent('X')) + expect(editor.getText()).toBe( + `before ${'format'.slice(0, offset)}X${'format'.slice(offset)} after` + ) + }) + + it('maps the original caret through peer and appended edits before canceling', async () => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + 2 + const input = await openLinkAtCaret(2) + editor.registerPlugin( + new Plugin({ + appendTransaction: (transactions, _oldState, newState) => + transactions.some((transaction) => transaction.getMeta('toolbar-prefix')) + ? newState.tr.insertText('appended ', 1) + : null, + }) + ) + act(() => editor.setEditable(false)) + act(() => + editor.view.dispatch(editor.state.tr.insertText('peer ', 1).setMeta('toolbar-prefix', true)) + ) + act(() => editor.setEditable(true)) + const before = editor.getJSON() + key(input, 'Escape') + await frame() + + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret + 'appended peer '.length) + expect(editor.getJSON()).toEqual(before) + act(() => editor.commands.insertContent('X')) + expect(editor.getText()).toBe('appended peer before foXrmat after') + }) + + it.each(['Apply link', 'Remove link'])('restores the caret on Escape from %s', async (label) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + 2 + const input = await openLinkAtCaret(2) + changeUrl(input, 'https://example.com/cancelled') + const action = button(linkGroup(), label) + act(() => action.focus()) + key(action, 'Escape') + await frame() + + expect(viewport.contains(input)).toBe(false) + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret) + expect(editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) + }) + it('maps the captured link target through a prefix edit and an appended transaction', async () => { const input = await openLinkEditor() changeUrl(input, 'https://example.com/mapped') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu.test.tsx new file mode 100644 index 00000000000..c623e8ddda6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu.test.tsx @@ -0,0 +1,253 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { Tooltip } from '@sim/emcn' +import { Editor } from '@tiptap/core' +import { NodeSelection } from '@tiptap/pm/state' +import { CellSelection } from '@tiptap/pm/tables' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { BUBBLE_MENU_CLASS } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome' +import { ImageBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu' +import { TableBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu' + +let editor: Editor +let root: Root +let viewport: HTMLDivElement + +beforeEach(async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.useFakeTimers() + viewport = document.createElement('div') + const editorHost = document.createElement('div') + const menuHost = document.createElement('div') + viewport.append(editorHost, menuHost) + document.body.append(viewport) + editor = new Editor({ + element: editorHost, + extensions: createMarkdownContentExtensions(), + content: 'Diagram

After

', + editorProps: { handleScrollToSelection: () => true }, + }) + vi.spyOn(editor.view, 'coordsAtPos').mockReturnValue({ top: 10, bottom: 30, left: 10, right: 50 }) + root = createRoot(menuHost) + await act(async () => { + root.render( + + + + + ) + }) + await act(async () => { + editor.commands.setNodeSelection(0) + editor.view.focus() + vi.advanceTimersToNextFrame() + }) +}) + +afterEach(() => { + act(() => root.unmount()) + editor.destroy() + viewport.remove() + vi.restoreAllMocks() + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +function button(label: string): HTMLButtonElement { + const element = viewport.querySelector(`button[aria-label="${label}"]`) + if (!element) throw new Error(`Missing ${label} button`) + return element +} + +function input(): HTMLInputElement { + const element = viewport.querySelector('[aria-label="Image editing"] input') + if (!element) throw new Error('Missing image toolbar input') + return element +} + +function change(value: string): void { + const field = input() + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(field, value) + field.dispatchEvent(new Event('input', { bubbles: true })) + }) +} + +function key(target: HTMLElement, value: string, options: KeyboardEventInit = {}): void { + act(() => + target.dispatchEvent( + new KeyboardEvent('keydown', { + key: value, + bubbles: true, + cancelable: true, + ...options, + }) + ) + ) +} + +describe('ImageBubbleMenu', () => { + it('uses the shared floating chrome and keyboard navigation without a gear', async () => { + const toolbar = viewport.querySelector('[aria-label="Image editing"]')! + expect(toolbar.parentElement?.className).toBe(BUBBLE_MENU_CLASS) + expect(viewport.querySelector('[aria-label="Edit image details"]')).toBeNull() + key(editor.view.dom, 'F10', { altKey: true }) + await act(async () => vi.advanceTimersToNextFrame()) + expect(document.activeElement).toBe(button('Edit image alt text')) + key(button('Edit image alt text'), 'ArrowRight') + expect(document.activeElement).toBe(button('Edit image link')) + key(button('Edit image link'), 'Escape') + await act(async () => vi.advanceTimersToNextFrame()) + expect(editor.view.hasFocus()).toBe(true) + expect(editor.state.selection).toBeInstanceOf(NodeSelection) + }) + + it.each([ + { key: 'Enter', isComposing: true, keyCode: 13 }, + { key: 'Escape', isComposing: true, keyCode: 27 }, + { key: 'Enter', isComposing: false, keyCode: 229 }, + { key: 'Escape', isComposing: false, keyCode: 229 }, + ])('does not commit or cancel during composition: $key/$keyCode', (keyboard) => { + act(() => button('Edit image alt text').click()) + change('Composition draft') + const field = input() + const parentKeyDown = vi.fn() + viewport.addEventListener('keydown', parentKeyDown) + const event = new KeyboardEvent('keydown', { bubbles: true, cancelable: true, ...keyboard }) + act(() => field.dispatchEvent(event)) + viewport.removeEventListener('keydown', parentKeyDown) + expect(parentKeyDown).not.toHaveBeenCalled() + expect(event.defaultPrevented).toBe(false) + expect(input()).toBe(field) + expect(input().value).toBe('Composition draft') + expect(document.activeElement).toBe(field) + expect(editor.state.doc.firstChild?.attrs.alt).toBe('Diagram') + }) + + it('edits alt text without changing other attributes and cancels without writing', () => { + act(() => button('Edit image alt text').click()) + change('New description') + key(input(), 'Enter') + expect(editor.state.doc.firstChild?.attrs).toMatchObject({ + alt: 'New description', + width: '200', + }) + act(() => button('Edit image alt text').click()) + change('Do not save') + key(input(), 'Escape') + expect(editor.state.doc.firstChild?.attrs.alt).toBe('New description') + expect(viewport.querySelector('[aria-label="Image editing"] input')).toBeNull() + }) + + it('validates links, normalizes a valid URL, and explicitly removes a cleared link', () => { + act(() => button('Edit image link').click()) + change('javascript:alert(1)') + expect(input()).toHaveAttribute('aria-invalid', 'true') + expect(button('Apply image change').disabled).toBe(true) + key(input(), 'Enter') + expect(editor.state.doc.firstChild?.attrs.href).toBeNull() + change(' https://sim.ai/image ') + act(() => button('Apply image change').click()) + expect(editor.state.doc.firstChild?.attrs.href).toBe('https://sim.ai/image') + act(() => button('Edit image link').click()) + change('') + key(input(), 'Enter') + expect(editor.state.doc.firstChild?.attrs).toMatchObject({ alt: 'Diagram', href: null }) + }) + + it('resets dimensions and omits reset for an image without custom dimensions', () => { + act(() => button('Reset image size').click()) + expect(editor.state.doc.firstChild?.attrs).toMatchObject({ + alt: 'Diagram', + width: null, + height: null, + }) + expect(viewport.querySelector('[aria-label="Reset image size"]')).toBeNull() + }) + + it.each(['read-only', 'destroyed'] as const)( + 'rejects queued input and reset actions after the editor becomes %s', + (state) => { + const reset = button('Reset image size') + act(() => button('Edit image alt text').click()) + change('Do not save') + const field = input() + const original = editor.state.doc + act(() => { + if (state === 'read-only') editor.setEditable(false) + else editor.destroy() + field.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + reset.click() + }) + expect(editor.state.doc.eq(original)).toBe(true) + } + ) + + it('drops a draft when selecting a different image and does not resurrect it on return', () => { + act(() => + editor.commands.insertContentAt(editor.state.doc.content.size, { + type: 'image', + attrs: { src: '/other.png', alt: 'Other' }, + }) + ) + act(() => editor.commands.setNodeSelection(0)) + act(() => button('Edit image alt text').click()) + change('Uncommitted draft') + let otherPosition = -1 + editor.state.doc.descendants((node, pos) => { + if (node.attrs.src === '/other.png') otherPosition = pos + }) + act(() => editor.commands.setNodeSelection(otherPosition)) + expect(viewport.querySelector('[aria-label="Image editing"] input')).toBeNull() + act(() => editor.commands.setNodeSelection(0)) + act(() => button('Edit image alt text').click()) + expect(input().value).toBe('Diagram') + }) + + it.each(['read-only', 'destroyed'] as const)( + 'rejects a mounted reset button click before React rerenders for %s', + (state) => { + const reset = button('Reset image size') + const original = editor.state.doc + act(() => { + if (state === 'read-only') editor.setEditable(false) + else editor.destroy() + reset.click() + }) + expect(editor.state.doc.eq(original)).toBe(true) + } + ) + + it('shows only the image toolbar for an image selected inside a table', async () => { + await act(async () => { + editor.commands.setContent( + '

Header

' + ) + let imagePos = -1 + editor.state.doc.descendants((node, pos) => { + if (node.type.name === 'image') imagePos = pos + }) + expect(imagePos).toBeGreaterThan(-1) + editor.commands.setNodeSelection(imagePos) + vi.advanceTimersToNextFrame() + }) + expect(viewport.querySelector('[aria-label="Table editing"]')).toBeNull() + expect(button('Edit image alt text')).toBeTruthy() + key(editor.view.dom, 'F10', { altKey: true }) + await act(async () => vi.advanceTimersToNextFrame()) + expect(document.activeElement).toBe(button('Edit image alt text')) + + await act(async () => { + const cell = editor.state.selection.$from.before(3) + editor.view.dispatch( + editor.state.tr.setSelection(CellSelection.create(editor.state.doc, cell)) + ) + editor.view.focus() + vi.advanceTimersToNextFrame() + }) + expect(viewport.querySelector('[aria-label="Image editing"]')).toBeNull() + expect(viewport.querySelector('[aria-label="Table editing"]')).not.toBeNull() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu.tsx new file mode 100644 index 00000000000..081b4d00f9f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu.tsx @@ -0,0 +1,200 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { Check, Link, RefreshCw, TypeText, X } from '@sim/emcn/icons' +import type { Node } from '@tiptap/pm/model' +import { NodeSelection, PluginKey } from '@tiptap/pm/state' +import { type Editor, useEditorState } from '@tiptap/react' +import { BubbleMenu } from '@tiptap/react/menus' +import { type ProsemirrorBinding, ySyncPluginKey } from '@tiptap/y-tiptap' +import type { XmlElement } from 'yjs' +import { + createImageTargetGuard, + getImageYTarget, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-target' +import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { BUBBLE_MENU_CLASS } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu-chrome' +import { + ToolbarButton, + ToolbarDivider, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button' +import { ToolbarInput } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-input' +import { useBubbleMenuFloating } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-bubble-menu-floating' +import { useEditorToolbar } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar' + +interface ImageBubbleMenuProps { + editor: Editor + scrollContainerRef: React.RefObject +} + +interface ImageDraft { + target: Node | XmlElement + matchesTarget: (node: Node) => boolean + field: 'alt' | 'href' + initial: string + value: string +} + +function selectedImage(editor: Editor) { + if (editor.isDestroyed || !editor.isEditable) return null + const { selection } = editor.state + if (!(selection instanceof NodeSelection) || selection.node.type.name !== 'image') return null + const binding: ProsemirrorBinding | undefined = ySyncPluginKey.getState(editor.state)?.binding + const target = binding ? getImageYTarget(binding, selection.node) : selection.node + return target ? { node: selection.node, target, binding } : null +} + +const shouldShowImageMenu = ({ editor }: { editor: Editor }) => selectedImage(editor) !== null + +/** Image actions use the same floating bar, inline fields, and keyboard navigation as text/table actions. */ +export function ImageBubbleMenu({ editor, scrollContainerRef }: ImageBubbleMenuProps) { + const inputRef = useRef(null) + const [menuKey] = useState(() => new PluginKey('markdownImageMenu')) + const [draft, setDraft] = useState(null) + const selected = useEditorState({ + editor, + selector: ({ editor: current }) => selectedImage(current), + equalityFn: (a, b) => a?.node === b?.node && a?.target === b?.target, + }) + if (draft && draft.target !== selected?.target) setDraft(null) + const currentDraft = draft?.target === selected?.target ? draft : null + const editingField = currentDraft?.field + const hasCustomSize = Boolean(selected?.node.attrs.width || selected?.node.attrs.height) + const normalizedHref = + currentDraft?.field === 'href' ? normalizeLinkHref(currentDraft.value.trim()) : null + const invalidLink = + currentDraft?.field === 'href' && Boolean(currentDraft.value.trim()) && !normalizedHref + + const { appendTo } = useBubbleMenuFloating(editor, scrollContainerRef) + const canFocus = useCallback(() => selectedImage(editor) !== null, [editor]) + const toolbar = useEditorToolbar({ + editor, + pluginKey: menuKey, + canFocus, + roving: !currentDraft, + onEscape: () => setDraft(null), + }) + + const matchesDraftTarget = draft?.matchesTarget + useEffect(() => { + if (!matchesDraftTarget) return + const invalidateDraft = () => { + const image = selectedImage(editor) + if (!image || !matchesDraftTarget(image.node)) setDraft(null) + } + editor.on('transaction', invalidateDraft) + invalidateDraft() + return () => { + editor.off('transaction', invalidateDraft) + } + }, [editor, matchesDraftTarget]) + + useEffect(() => { + if (editingField) inputRef.current?.focus() + }, [editingField]) + + useEffect(() => { + if (!editor.isDestroyed) editor.commands.setMeta(menuKey, 'updatePosition') + }, [editor, menuKey, editingField, hasCustomSize, selected?.target]) + + const close = () => { + setDraft(null) + if (!editor.isDestroyed) editor.commands.focus() + } + const edit = (field: ImageDraft['field']) => { + const image = selectedImage(editor) + if (!image || image.target !== selected?.target) return + const value = typeof image.node.attrs[field] === 'string' ? image.node.attrs[field] : '' + const matchesTarget = image.binding + ? createImageTargetGuard(image.binding, image.node) + : (node: Node) => node === image.node + setDraft({ target: image.target, matchesTarget, field, initial: value, value }) + } + const apply = () => { + if (!currentDraft || invalidLink) return + const image = selectedImage(editor) + if (!image || image.target !== currentDraft.target || !currentDraft.matchesTarget(image.node)) + return + if (currentDraft.value !== currentDraft.initial) { + editor.commands.updateAttributes('image', { + [currentDraft.field]: + currentDraft.field === 'href' ? normalizedHref || null : currentDraft.value, + }) + } + close() + } + + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx index 3ebcc2c6312..3e276a22a13 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx @@ -1,6 +1,7 @@ import type { Ref } from 'react' import type { ChainedCommands } from '@tiptap/core' import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { ToolbarInput } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-input' /** * Applies a link to the chain's current selection: normalizes `rawHref`, expands to the full link @@ -30,40 +31,9 @@ interface LinkUrlInputProps { readOnly?: boolean } -/** - * The inline link-URL field shared by the bubble menu and the link hover card — Enter commits, Escape - * cancels. Styled to sit flush in the 28px floating micro-toolbar (a `ChipInput` would impose its own - * field chrome and break the bar), so this is a deliberate raw ``. - */ -export function LinkUrlInput({ - value, - onChange, - onCommit, - onCancel, - inputRef, - readOnly = false, -}: LinkUrlInputProps) { +/** Inline link field shared by the text-selection toolbar and link hover card. */ +export function LinkUrlInput(props: LinkUrlInputProps) { return ( - onChange(event.target.value)} - onKeyDown={(event) => { - if (event.nativeEvent.isComposing || event.nativeEvent.keyCode === 229) return - if (event.key === 'Enter' && !readOnly) { - event.preventDefault() - onCommit() - } else if (event.key === 'Escape') { - event.preventDefault() - onCancel() - } - }} - placeholder='Paste or type a link…' - className='h-[28px] w-[220px] bg-transparent px-2 text-[var(--text-body)] text-small outline-hidden placeholder:text-[var(--text-subtle)]' - /> + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu.tsx index 13ec131d565..2340a50a6a5 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu.tsx @@ -1,6 +1,6 @@ import { useCallback, useState } from 'react' import { ArrowDown, ArrowLeft, ArrowRight, ArrowUp, Columns3, Rows3, Trash } from '@sim/emcn/icons' -import { PluginKey } from '@tiptap/pm/state' +import { NodeSelection, PluginKey } from '@tiptap/pm/state' import type { Editor } from '@tiptap/react' import { useEditorState } from '@tiptap/react' import { BubbleMenu } from '@tiptap/react/menus' @@ -18,8 +18,14 @@ interface TableBubbleMenuProps { scrollContainerRef: React.RefObject } -const shouldShowTableMenu = ({ editor }: { editor: Editor }) => - editor.isEditable && editor.isActive('table') +const shouldShowTableMenu = ({ editor }: { editor: Editor }) => { + const { selection } = editor.state + return ( + editor.isEditable && + editor.isActive('table') && + !(selection instanceof NodeSelection && selection.node.type.name === 'image') + ) +} /** * Floating toolbar shown whenever the selection is inside a table: row/column insert-before/after, @@ -41,7 +47,7 @@ export function TableBubbleMenu({ editor, scrollContainerRef }: TableBubbleMenuP const { resolveAnchor, appendTo } = useBubbleMenuFloating(editor, scrollContainerRef) const canFocus = useCallback( () => - editor.isActive('table') && + shouldShowTableMenu({ editor }) && editor.state.doc .textBetween(editor.state.selection.from, editor.state.selection.to, ' ') .trim().length === 0, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx index e6bf381b8e8..340a43c1e50 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx @@ -44,4 +44,18 @@ describe('ToolbarButton', () => { expect(button?.className).toContain('size-[28px]') expect(button?.querySelector('svg')?.className.baseVal).toContain('size-[12px]') }) + + it('preserves the editor selection for mouse, pen, and touch activation', () => { + const host = renderButton() + const button = host.querySelector('button[aria-label="Bold"]') + expect(button).not.toBeNull() + if (!button) return + + for (const pointerType of ['mouse', 'pen', 'touch']) { + const event = new Event('pointerdown', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'pointerType', { value: pointerType }) + act(() => button.dispatchEvent(event)) + expect(event.defaultPrevented).toBe(true) + } + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx index 3436913633f..4bdcb5b19d8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx @@ -33,10 +33,10 @@ export function ToolbarButton({ aria-label={label} aria-pressed={isActive} disabled={disabled} - onMouseDown={(event) => event.preventDefault()} + onPointerDown={(event) => event.preventDefault()} onClick={onClick} className={cn( - 'size-[28px] focus-visible:bg-[var(--surface-hover)]', + 'size-10 focus-visible:bg-[var(--surface-hover)] sm:size-[28px]', !isActive && 'hover-hover:bg-[var(--surface-hover)]' )} > @@ -52,5 +52,5 @@ export function ToolbarButton({ /** Thin vertical separator between groups of {@link ToolbarButton}s. */ export function ToolbarDivider() { - return
+ return
} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-input.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-input.tsx new file mode 100644 index 00000000000..30c3917e353 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-input.tsx @@ -0,0 +1,56 @@ +import type { Ref } from 'react' + +interface ToolbarInputProps { + label: string + placeholder: string + inputMode?: 'text' | 'url' + value: string + onChange: (value: string) => void + onCommit: () => void + onCancel: () => void + inputRef: Ref + readOnly?: boolean + invalid?: boolean +} + +/** Flush toolbar field; ChipInput's standalone field chrome would break the shared floating bar. */ +export function ToolbarInput({ + label, + placeholder, + inputMode = 'text', + value, + onChange, + onCommit, + onCancel, + inputRef, + readOnly = false, + invalid = false, +}: ToolbarInputProps) { + return ( + onChange(event.target.value)} + onKeyDown={(event) => { + if (event.nativeEvent.isComposing || event.nativeEvent.keyCode === 229) { + event.stopPropagation() + return + } + if (event.key === 'Enter' && !readOnly) { + event.preventDefault() + onCommit() + } else if (event.key === 'Escape') { + event.preventDefault() + onCancel() + } + }} + placeholder={placeholder} + className='h-10 w-[220px] bg-transparent px-2 text-[var(--text-body)] text-small outline-hidden placeholder:text-[var(--text-subtle)] sm:h-[28px]' + /> + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts index da99cbb72b3..f86e2220fde 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts @@ -15,6 +15,7 @@ interface EditorToolbarOptions { canFocus: () => boolean /** URL editing uses ordinary form tab order so its native arrow keys do not trap action buttons. */ roving?: boolean + onEscape?: () => void } function controls(toolbar: HTMLElement): HTMLElement[] { @@ -33,6 +34,7 @@ export function useEditorToolbar({ pluginKey, canFocus, roving = true, + onEscape, }: EditorToolbarOptions) { const ref = useRef(null) @@ -113,6 +115,7 @@ export function useEditorToolbar({ ) return if (event.key === 'Escape') { + if (!event.defaultPrevented) onEscape?.() event.preventDefault() editor.commands.focus() editor.commands.setMeta(pluginKey, 'hide') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 55384a26ff1..93a07f39619 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -1,8 +1,8 @@ 'use client' -import { memo, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { Chip, cn, toast } from '@sim/emcn' -import { FILE_DOC_SEED, type JoinFileDocError } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { PASTE_LIMITS, PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Extensions, JSONContent, Range } from '@tiptap/core' import { isChangeOrigin } from '@tiptap/extension-collaboration' @@ -58,6 +58,7 @@ import { parseMarkdownToDoc } from '@/app/workspace/[workspaceId]/files/componen import { isPlainTextPaste } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-paste' import { useEditorMentions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention' import { EditorBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu' +import { ImageBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu' import { LinkHoverCard } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-hover-card' import { TableBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/table-menu' import { normalizeMarkdownContent } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/normalize-content' @@ -110,6 +111,25 @@ function warnRichMarkdownPasteLimit(reason?: 'paste' | 'formatting') { const EDITOR_SURFACE_CLASS = 'mx-auto flex w-full max-w-[48rem] flex-1 flex-col px-8 py-6 selection:bg-[var(--selection-bg)] selection:text-[var(--text-primary)] dark:selection:bg-[var(--selection-dark)] dark:selection:text-white' +/** ProseMirror block positions do not correspond to markdown source line numbers. */ +function buildEditorSelectionContext( + editor: Editor | null, + file: Pick +): ChatContext | null { + if (!editor) return null + const { from, to } = editor.state.selection + if (from === to) return null + const text = editor.state.doc.textBetween(from, to, '\n') + if (!text.trim()) return null + return { + kind: 'file_selection', + fileId: file.id, + fileName: file.name, + label: buildFileSelectionLabel(file.name), + text: truncateSelectionText(text), + } +} + /** * Read-only editor that renders the already-fetched markdown while a collaborative doc waits for its * server seed, so the pane shows content instantly instead of blocking blank on the socket round-trip @@ -123,9 +143,12 @@ const EDITOR_SURFACE_CLASS = */ interface ReadOnlyPlaceholderProps { content: JSONContent + file: WorkspaceFileRecord + workspaceId: string } -function ReadOnlyPlaceholder({ content }: ReadOnlyPlaceholderProps) { +function ReadOnlyPlaceholder({ content, file, workspaceId }: ReadOnlyPlaceholderProps) { + const containerRef = useRef(null) const editor = useEditor({ extensions: EXTENSIONS, editable: false, @@ -145,7 +168,12 @@ function ReadOnlyPlaceholder({ content }: ReadOnlyPlaceholderProps) { }, }, }) - return + const buildSelectionContext = useCallback( + () => buildEditorSelectionContext(editor, { id: file.id, name: file.name }), + [editor, file.id, file.name] + ) + useSelectionCopyBridge(containerRef, buildSelectionContext, workspaceId) + return } interface RichMarkdownEditorProps { @@ -446,6 +474,7 @@ export function LoadedRichMarkdownEditor({ const isEditable = canEdit && !isStreaming && (settled?.verdict ?? false) && collabReady const collaboration = useFileDocCollaboration({ + workspaceId, fileId: file.id, userId, userName, @@ -811,12 +840,6 @@ export function LoadedRichMarkdownEditor({ [] ) - /** - * The loaded markdown to seed the shared doc from, held by pointer so the parse - * runs once at seed time rather than every render. - */ - const seedContentRef = useRef(content) - /** The lifetime-stable editor and its async work consume only committed React inputs. */ useLayoutEffect(() => { onChangeRef.current = onChange @@ -831,7 +854,6 @@ export function LoadedRichMarkdownEditor({ insertImagesRef.current = insertImages cloneHostedImageRef.current = cloneHostedImage editorInstanceRef.current = editor - seedContentRef.current = content }) /** @@ -842,11 +864,9 @@ export function LoadedRichMarkdownEditor({ * synced AND seeded — it never imports content itself on the happy path; * - **gate** the parent's autosave until the doc is synced AND seeded, so an * empty/still-syncing doc can never overwrite the real file's markdown mirror; - * - **fall back** on a fatal join: seed the loaded content so it is SHOWN, but - * leave the editor read-only + gated. Every non-retryable failure (auth, access - * denied, not found, client-id conflict) either can't save or is moot, so the - * safe fallback is a read-only view of the content rather than editable-but- - * unsavable — which would silently drop the user's edits. + * - **preview** stored content in a separate read-only editor until authoritative content arrives. + * A retryable timeout never seeds the shared Y.Doc, so late server content cannot duplicate it. + * Terminal failures keep any existing live content visible but never editable. * * `ready` (synced+seeded) gates BOTH the editor's editability (a user must never * type into an empty/unsynced doc) and the parent's autosave. Non-collaborative @@ -864,11 +884,12 @@ export function LoadedRichMarkdownEditor({ * document that was already correct, and it opens mid-flight anyway whenever the updates arrive * more than a frame apart (which is what a remote Redis and a long room history produce). */ - const setReady = (ready: boolean, fatal = false) => { + const setReady = (ready: boolean, fatal = false, retrying = false) => { // Child-local: gates editability (a user must never type into an unsynced/unseeded doc). setCollabStatus((previous) => { if (fatal) return 'fatal' if (ready) return 'ready' + if (retrying) return 'reconnecting' return previous === 'ready' || previous === 'reconnecting' ? 'reconnecting' : 'connecting' }) // Parent: gates CLIENT autosave. In a collaborative session the relay persists the doc to @@ -888,20 +909,6 @@ export function LoadedRichMarkdownEditor({ } const config = doc.getMap(FILE_DOC_SEED.configMap) - let offlineSeed = false - - const seedFromLoaded = () => { - if (config.get(FILE_DOC_SEED.flag) === true) return - offlineSeed = true - doc.transact(() => { - editor.commands.setContent( - parseMarkdownToDoc(splitFrontmatter(seedContentRef.current).body), - { contentType: 'json', emitUpdate: false } - ) - config.set(FILE_DOC_SEED.flag, true) - }) - } - if (!provider) { setReady(false) return @@ -910,26 +917,20 @@ export function LoadedRichMarkdownEditor({ const report = () => { const synced = provider.synced const seeded = config.get(FILE_DOC_SEED.flag) === true - // `joinError` is latched ONLY on the provider's fatal paths (non-retryable rejection, access - // revocation, readiness deadline), so it is exactly "this document is abandoned". - const fatal = provider.joinError !== null - setReady(isCollabReady({ synced, seeded, offlineSeed, fatal }), fatal) - } - /** - * Re-report unconditionally, not just when the fallback seeds. A fatal that arrives on an ALREADY - * seeded doc (access revoked mid-session) leaves `seedFromLoaded` a no-op, so nothing else would - * fire an observer and the editor would stay editable on a document the provider has abandoned. - */ - const onJoinError = (error: JoinFileDocError) => { - if (error.retryable === false) seedFromLoaded() - report() + const fatal = provider.joinError?.retryable === false + setReady( + isCollabReady({ synced, seeded, fatal }), + fatal, + provider.joinError?.retryable === true + ) } + /** Rejections must close the editing gate even if the document was already seeded. */ + const onJoinError = () => report() provider.on('synced', report) provider.on('join-error', onJoinError) config.observe(report) report() - if (provider.joinError) onJoinError(provider.joinError) return () => { provider.off('synced', report) @@ -1282,38 +1283,27 @@ export function LoadedRichMarkdownEditor({ ) const addToChat = useAddToChat() - /** - * No line range: this editor renders a ProseMirror document, whose block - * boundaries do not correspond to markdown source lines (blank lines between - * paragraphs, list markers, heading prefixes and fenced blocks all shift the - * real line). Reporting a derived count would label the chip — and prompt the - * agent — with line numbers that don't exist in the file. - */ - const buildSelectionContext = useCallback((): ChatContext | null => { - if (!editor) return null - const { from, to } = editor.state.selection - if (from === to) return null - const text = editor.state.doc.textBetween(from, to, '\n') - if (!text.trim()) return null - return { - kind: 'file_selection', - fileId: file.id, - fileName: file.name, - label: buildFileSelectionLabel(file.name), - text: truncateSelectionText(text), - } - }, [editor, file.id, file.name]) + const buildSelectionContext = useCallback( + () => buildEditorSelectionContext(editor, { id: file.id, name: file.name }), + [editor, file.id, file.name] + ) const handleAddSelectionToChat = () => { const context = buildSelectionContext() if (context) addToChat(context) } - useSelectionCopyBridge(containerRef, buildSelectionContext, workspaceId) - - /** Use the stored-content placeholder only while the live document is bootstrapping. */ - const showPlaceholder = collaborationEnabled && collabStatus === 'connecting' + /** Stored content belongs to a separate preview, never to an unseeded collaborative document. */ + const showPlaceholder = + collaborationEnabled && + (collabStatus === 'connecting' || + (collabStatus !== 'ready' && + collaboration?.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) !== true)) const showReconnecting = collaborationEnabled && collabStatus === 'reconnecting' + const collabFailure = collaboration?.provider?.joinError ?? null + const showCollabFailure = collaborationEnabled && collabStatus === 'fatal' ? collabFailure : null + + useSelectionCopyBridge(containerRef, buildSelectionContext, workspaceId, !showPlaceholder) /** * Find is off while the placeholder is up. The text on screen then belongs to the placeholder's own @@ -1322,6 +1312,28 @@ export function LoadedRichMarkdownEditor({ * native find reads the rendered placeholder correctly; it becomes ours once the seed lands. */ const find = useMarkdownFind({ editor, enabled: enableFind && !showPlaceholder }) + const replaceControls = useMemo( + () => + isEditable + ? { + value: find.replacement, + onChange: find.setReplacement, + onReplace: find.replaceCurrent, + onReplaceAll: find.replaceAll, + canReplace: find.count > 0, + canReplaceAll: find.count > 0 && !find.truncated, + } + : undefined, + [ + find.count, + find.replaceAll, + find.replaceCurrent, + find.replacement, + find.setReplacement, + find.truncated, + isEditable, + ] + ) return ( // The find bar is a sibling of the scroller, not a child: pinned inside `containerRef` it would @@ -1347,6 +1359,17 @@ export function LoadedRichMarkdownEditor({ Reconnecting…
)} + {showCollabFailure && ( +
+ {showCollabFailure.code === 'ACCESS_REVOKED' || showCollabFailure.code === 'ACCESS_DENIED' + ? 'You no longer have edit access to this document.' + : 'Live editing is unavailable.'} +
+ )} {find.isOpen && ( )}
)} {editor && } + {editor && } {editor && } {showPlaceholder && placeholderContent && ( - + )} null, }) ) +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu', + () => ({ ImageBubbleMenu: () => null }) +) let root: Root let container: HTMLDivElement diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx index a5ec28c9920..9fef6356d62 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-field.tsx @@ -5,6 +5,13 @@ import { ChipTextarea, chipFieldSurfaceClass, cn, toast } from '@sim/emcn' import { formatPasteLimit, PASTE_LIMITS } from '@sim/utils/paste' import type { JSONContent } from '@tiptap/core' import { EditorContent, useEditor } from '@tiptap/react' +import { + beginImageUploads, + findImageUpload, + finishImageUpload, + removeImageUpload, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-upload' +import { ImageBubbleMenu } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/image-menu' import { assessRawMarkdownPaste } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/paste-admission' import { createMarkdownEditorExtensions } from './editor-extensions' import { moveDraggedImageNode } from './image-drag-move' @@ -118,7 +125,7 @@ interface RichMarkdownFieldProps { /** * The WYSIWYG editor for round-trip-safe content (chosen by {@link RichMarkdownField}). The file-less * sibling of {@link RichMarkdownEditor}'s loaded editor: same TipTap extensions, parser, and menus but - * no file loading, autosave, or image upload. + * no file loading or autosave. */ function LoadedRichMarkdownField({ value, @@ -153,13 +160,11 @@ function LoadedRichMarkdownField({ /** The body last reflected into the editor — updated on local edits and on each streamed sync. */ const lastSyncedBodyRef = useRef(initialSplit.body) const onChangeRef = useRef(onChange) - onChangeRef.current = onChange const onPasteTextRef = useRef(onPasteText) - onPasteTextRef.current = onPasteText const uploadImageRef = useRef(uploadImage) - uploadImageRef.current = uploadImage const autoFocusAtRef = useRef(autoFocusAt) const editorInstanceRef = useRef>(null) + const uploadGenerationRef = useRef(0) /** * The `/Image` slash command opens this hidden picker; `pendingImagePosRef` holds the caret @@ -169,36 +174,36 @@ function LoadedRichMarkdownField({ const pendingImagePosRef = useRef(null) /** - * Sequential upload-then-insert, mirroring the file editor's own image flow: - * each image inserts at the evolving position so a multi-image paste lands in - * order, and a failed upload skips its insert without aborting the rest. The - * upload mutation owns user feedback. + * Reuse the file editor's mapped anchors without adding upload chrome to embedded fields. + * A streamed replacement invalidates the batch even if editing resumes before upload completes. */ - const insertImagesRef = useRef<(images: File[], at: number) => Promise>(() => - Promise.resolve() - ) - insertImagesRef.current = async (images, at) => { + async function insertImages(images: File[], at: number) { const upload = uploadImageRef.current const owner = editorInstanceRef.current - if (!upload || !owner) return - let position = at - for (const image of images) { - const result = await upload(image).catch(() => null) - /* Bail if the editor unmounted (note closed) while the upload ran. */ - if (!result || editorInstanceRef.current !== owner || owner.isDestroyed) continue - const safePosition = Math.min(position, owner.state.doc.content.size) - try { - owner - .chain() - .insertContentAt(safePosition, { - type: 'image', - attrs: { src: result.url, alt: result.alt }, - }) - .run() - position = owner.state.selection.to - } catch { - position = owner.state.doc.content.size + if (!upload || !owner || owner.isDestroyed || !owner.isEditable) return + const generation = uploadGenerationRef.current + const anchors = beginImageUploads( + owner, + { from: at, to: at }, + images.map(() => '') + ) + const canInsert = () => + editorInstanceRef.current === owner && + !owner.isDestroyed && + owner.isEditable && + uploadGenerationRef.current === generation + try { + for (const [index, image] of images.entries()) { + if (!canInsert()) break + const anchor = anchors[index] + if (!anchor || findImageUpload(owner, anchor) === null) continue + const result = await upload(image).catch(() => null) + if (!canInsert()) break + if (result) finishImageUpload(owner, anchor, result.url, result.alt) + else removeImageUpload(owner, anchor) } + } finally { + for (const anchor of anchors) removeImageUpload(owner, anchor) } } @@ -260,7 +265,7 @@ function LoadedRichMarkdownField({ const clipboardHtml = event.clipboardData?.getData('text/html') ?? '' if (images.length > 0 && !shouldSkipFileUpload(images, clipboardHtml, isInlineRouteSrc)) { event.preventDefault() - void insertImagesRef.current(images, view.state.selection.from) + void insertImages(images, view.state.selection.from) return true } const handler = onPasteTextRef.current @@ -285,7 +290,7 @@ function LoadedRichMarkdownField({ event.preventDefault() if (images.length > 0) { const dropPos = view.posAtCoords({ left: event.clientX, top: event.clientY })?.pos - void insertImagesRef.current(images, dropPos ?? view.state.selection.from) + void insertImages(images, dropPos ?? view.state.selection.from) } return true } @@ -295,7 +300,6 @@ function LoadedRichMarkdownField({ /* Resolved after creation, not via `autofocus`: mapping a point to a document position needs the editor's DOM laid out. */ onCreate: ({ editor }) => { - editorInstanceRef.current = editor const point = autoFocusAtRef.current if (!point) return const resolved = editor.view.posAtCoords({ left: point.clientX, top: point.clientY }) @@ -309,6 +313,20 @@ function LoadedRichMarkdownField({ }, }) + useLayoutEffect(() => { + onChangeRef.current = onChange + onPasteTextRef.current = onPasteText + uploadImageRef.current = uploadImage + }, [onChange, onPasteText, uploadImage]) + + /** React can unmount the field before TipTap's deferred destruction runs. */ + useLayoutEffect(() => { + editorInstanceRef.current = editor + return () => { + editorInstanceRef.current = null + } + }, [editor]) + /** Mirrors an externally-driven value (AI generation) into the editor, then settles to editable. */ const wasStreamingRef = useRef(isStreaming) useEffect(() => { @@ -317,8 +335,9 @@ function LoadedRichMarkdownField({ frontmatterRef.current = frontmatter if (isStreaming) { + if (!wasStreamingRef.current) uploadGenerationRef.current++ wasStreamingRef.current = true - if (editor.isEditable) editor.setEditable(false) + if (editor.isEditable) editor.setEditable(false, false) if (body === lastSyncedBodyRef.current) return lastSyncedBodyRef.current = body const el = containerRef.current @@ -341,7 +360,7 @@ function LoadedRichMarkdownField({ }) } } - if (editor.isEditable !== !disabled) editor.setEditable(!disabled) + if (editor.isEditable !== !disabled) editor.setEditable(!disabled, false) }, [editor, value, isStreaming, disabled]) /** @@ -393,6 +412,9 @@ function LoadedRichMarkdownField({ /> )} {editor && } + {editor && ( + + )} {uploadImage && ( 0) void insertImagesRef.current(images, at) + if (images.length > 0) void insertImages(images, at) }} /> )} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts index 2fd713b8d62..47336ae7443 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.test.ts @@ -1,11 +1,77 @@ /** * @vitest-environment jsdom */ +import { Editor } from '@tiptap/core' import { describe, expect, it } from 'vitest' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + parseMarkdownToDoc, + serializeMarkdownDocument, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' import { normalizeMarkdownContent } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/normalize-content' import { isRoundTripSafe } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety' describe('isRoundTripSafe', () => { + it.each([ + { field: 'alt', linked: false }, + { field: 'title', linked: false }, + { field: 'alt', linked: true }, + { field: 'title', linked: true }, + ])('keeps a resized image editable with quoted $field (linked: $linked)', ({ field, linked }) => { + const attributes = { + src: '/image.png', + alt: 'Diagram', + [field]: 'A "quoted" diagram', + href: linked ? '/destination' : null, + } + const editor = new Editor({ + extensions: createMarkdownContentExtensions(), + content: { type: 'doc', content: [{ type: 'image', attrs: attributes }] }, + }) + try { + expect(isRoundTripSafe(editor.getMarkdown())).toBe(true) + editor.commands.setNodeSelection(0) + editor.commands.updateAttributes('image', { width: '320', height: null }) + const markdown = editor.getMarkdown() + expect(markdown).toContain('"') + expect(isRoundTripSafe(markdown)).toBe(true) + expect(parseMarkdownToDoc(markdown).content?.[0].attrs).toMatchObject({ + ...attributes, + width: '320', + height: null, + }) + } finally { + editor.destroy() + } + }) + + it.each([ + '![literal "](/outer)', + "![outer](/outer \"literal "\")", + "[text](/link \"literal "\")", + '`"`\n\n![x][id]\n\n[id]: /outer """', + '"outside"\n\n["inside"](/link)', + '["©"](/link)', + '["inside"](/link)', + ])('does not exempt unsafe text or dropped attributes near image quotes: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + }) + + it.each([ + '
\n\n
', + '', + '', + '---\nexample: \'\'\n---\n# Heading', + ])('allows image attributes preserved verbatim in raw content: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(true) + expect(normalizeMarkdownContent(source).trim()).toBe(source) + }) + + it('does not let a preserved raw tag hide an identical image tag that loses attributes', () => { + const tag = '' + expect(isRoundTripSafe(`
\n${tag}\n
\n\n${tag}`)).toBe(false) + }) + it('passes ordinary markdown and lossless normalizations', () => { expect(isRoundTripSafe('# Title\n\nA **bold** word and a [link](https://sim.ai).')).toBe(true) expect(isRoundTripSafe('- one\n- two\n\n```js\nconst x = 1\n```')).toBe(true) @@ -24,6 +90,11 @@ describe('isRoundTripSafe', () => { isRoundTripSafe('[![build](https://img.shields.io/badge/x-green)](https://ci.example.com)') ).toBe(true) expect(isRoundTripSafe('[![alt](https://e.com/i.png "t")](https://e.com "h")')).toBe(true) + expect( + isRoundTripSafe( + '[](https://e.com)' + ) + ).toBe(true) }) it('passes inline code without an interior backtick', () => { @@ -146,11 +217,63 @@ describe('isRoundTripSafe', () => { expect(isRoundTripSafe('')).toBe(true) }) + it('keeps HTML images with unsupported attributes in source mode', () => { + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('a')).toBe( + true + ) + }) + + it.each([ + '[](/link)', + '[](/link)', + "[](/link)", + ])('checks attributes after quoted angle brackets without losing source: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it.each([ + 'first', + '[first](/link)', + '[first](/link)', + '[](/link)', + ])('keeps duplicate image attributes in source mode: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it.each([ + '', + '', + '', + "", + '', + '[](/link)', + ])('keeps valueless image dimensions in source mode: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it('allows supported image attributes containing quoted angle brackets', () => { + expect(isRoundTripSafe('')).toBe(true) + expect(isRoundTripSafe('[a>b](/link)')).toBe(true) + expect( + isRoundTripSafe('[](/link)') + ).toBe(true) + }) + it.each([ '| |\n| --- |\n| body |', '| header |\n| --- |\n| |', '| header |\n| --- |\n| |', '| header |\n| --- |\n| [](/dest) |', + '| header |\n| --- |\n| |', + "| header |\n| --- |\n| |", + '| header |\n| --- |\n| example `code` |', + '| header |\n| --- |\n| |', ])('refuses unsupported HTML images inside GFM tables: %s', (source) => { expect(isRoundTripSafe(source)).toBe(false) }) @@ -159,6 +282,19 @@ describe('isRoundTripSafe', () => { expect(isRoundTripSafe('| header |\n| --- |\n| `` |')).toBe(true) }) + it.each([ + '', + '', + 'text', + ])('preserves literal image markup in table comments and attributes: %s', (cell) => { + for (const source of [`| ${cell} |\n| --- |\n| body |`, `| header |\n| --- |\n| ${cell} |`]) { + const serialized = serializeMarkdownDocument(source) + expect(serialized).toContain(cell) + expect(serializeMarkdownDocument(serialized)).toBe(serialized) + expect(isRoundTripSafe(source)).toBe(true) + } + }) + it('does not flag a fenced block that merely contains html or backticks', () => { expect(isRoundTripSafe('```html\n
hi
\n```')).toBe(true) expect(isRoundTripSafe('````md\n```\ncode\n```\n````')).toBe(true) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts index bab3759891f..3884fc877d5 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts @@ -1,6 +1,6 @@ import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import { decodeHtmlEntities } from '@tiptap/core' -import { Marked, type Token } from 'marked' +import { Lexer, Marked, type Token, Tokenizer } from 'marked' import { extractImgSrcs } from '@/lib/uploads/utils/embedded-image-ref' import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { serializeMarkdownDocument } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' @@ -42,6 +42,43 @@ function stripCode(content: string): string { } const fidelityLexer = new Marked({ gfm: true }) +const SUPPORTED_IMAGE_ATTRIBUTES = new Set(['src', 'alt', 'title', 'width', 'height']) + +/** + * Count tags that lose attributes, and exempt image-local " from the text-entity check: + * the image schema decodes it losslessly, unlike the prose parser. + */ +function inspectHtmlImages(content: string) { + const images = new Map() + let quotedEntities = 0 + const tokenizer = new Tokenizer() + new Lexer({ gfm: true, tokenizer }) + const imagePattern = /])/gi + for (let image = imagePattern.exec(content); image; image = imagePattern.exec(content)) { + const tag = tokenizer.tag(content.slice(image.index)) + if (!tag) continue + imagePattern.lastIndex = image.index + tag.raw.length + quotedEntities += tag.raw.match(/"/g)?.length ?? 0 + const attributes = tag.raw.slice(4, -1) + const seen = new Set() + const pattern = /(?:^|\s)([^\s=/>]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s"'=<>`]+))?/g + for (const attribute of attributes.matchAll(pattern)) { + const name = attribute[1].toLowerCase() + const value = attribute[2] + if ( + !SUPPORTED_IMAGE_ATTRIBUTES.has(name) || + seen.has(name) || + value === undefined || + ((name === 'width' || name === 'height') && (value === '""' || value === "''")) + ) { + images.set(tag.raw, (images.get(tag.raw) ?? 0) + 1) + break + } + seen.add(name) + } + } + return { unsupported: images, quotedEntities } +} function imageSources(token: Token): string[] { if (token.type === 'image') return [token.href] @@ -60,11 +97,28 @@ function imageSources(token: Token): string[] { function inspectMarkdownFidelity(content: string) { const targets = new Map() let hasTaskReference = false + let hasTableHtmlImage = false + let hasQuotedImageMetadata = false + let preservedQuotes = 0 + const body = splitFrontmatter(content).body const add = (kind: 'image' | 'linkedImage', ...destinations: string[]) => { const target = JSON.stringify([kind, ...destinations.map(decodeHtmlEntities)]) targets.set(target, (targets.get(target) ?? 0) + 1) } - fidelityLexer.walkTokens(fidelityLexer.lexer(splitFrontmatter(content).body), (token) => { + fidelityLexer.walkTokens(fidelityLexer.lexer(body), (token) => { + if ( + token.type === 'image' && + [token.raw, token.text, token.title, token.href].some((value) => value?.includes('"')) + ) + hasQuotedImageMetadata = true + if (token.type === 'html') preservedQuotes += inspectHtmlImages(token.raw).quotedEntities + if (token.type === 'code' || token.type === 'codespan') + preservedQuotes += token.raw.match(/"/g)?.length ?? 0 + if (token.type === 'table') { + fidelityLexer.walkTokens([token], (child) => { + if (child.type === 'html' && /^])/i.test(child.raw)) hasTableHtmlImage = true + }) + } for (const src of imageSources(token)) add('image', src) if (token.type === 'link') { fidelityLexer.walkTokens(token.tokens ?? [], (child) => { @@ -83,7 +137,9 @@ function inspectMarkdownFidelity(content: string) { } } }) - return { targets, hasTaskReference } + const hasUnsafeQuotes = + hasQuotedImageMetadata || (body.match(/"/g)?.length ?? 0) > preservedQuotes + return { targets, hasTaskReference, hasTableHtmlImage, hasUnsafeQuotes } } /** @@ -139,12 +195,17 @@ function hasOrphanReferenceDefinition(content: string): boolean { export function isRoundTripSafe(content: string): boolean { if (content.length > PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS) return false const stripped = stripCode(content) - if (STABLE_LOSS_PATTERNS.some((pattern) => pattern.test(stripped))) return false + if (STABLE_LOSS_PATTERNS.some((pattern) => pattern.test(stripped.replaceAll('"', '')))) + return false if (hasOrphanReferenceDefinition(stripped)) return false try { const source = inspectMarkdownFidelity(content) - if (source.hasTaskReference) return false + if (source.hasTaskReference || source.hasTableHtmlImage || source.hasUnsafeQuotes) return false const once = serializeMarkdownDocument(content) + const preservedImages = inspectHtmlImages(stripCode(once)).unsupported + for (const [tag, count] of inspectHtmlImages(stripped).unsupported) { + if ((preservedImages.get(tag) ?? 0) < count) return false + } const serialized = inspectMarkdownFidelity(once) for (const [target, count] of source.targets) { if ((serialized.targets.get(target) ?? 0) < count) return false diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts index 961fd1d86df..bcc50b243eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts @@ -301,6 +301,52 @@ describe('editor markdown round-trip', () => { expect(roundTrip('![a](https://e.com/i.png)')).toContain('![a](https://e.com/i.png)') }) + it('round-trips every sized linked-image attribute without dropping dimensions', () => { + const source = + '[](https://e.com "Details")' + const out = roundTrip(source) + + expect(out).toContain('alt=""') + expect(out).toContain('width="320" height="180"') + expect(out).toContain('](https://e.com "Details")') + expect(roundTrip(out)).toBe(out) + }) + + it('uses empty alt text when a linked HTML image has no alt attribute', () => { + const source = '[](https://e.com)' + const out = roundTrip(source) + + expect(out).toContain('alt=""') + expect(out).not.toContain('alt="<img') + expect(roundTrip(out)).toBe(out) + }) + + it('round-trips linked images with escaped alt text and angle-bracket destinations', () => { + const source = '[![a\\]b]()]( "Details")' + const out = roundTrip(source) + + expect(out).toContain('a\\]b') + expect(out).toContain('') + expect(out).toContain('') + expect(roundTrip(out)).toBe(out) + }) + + it('parses a paragraph of adjacent links and linked images without recursive suffix scans', () => { + const links = Array.from( + { length: 80 }, + (_, index) => `[Link ${index}](https://e.com/${index})` + ) + const images = Array.from( + { length: 40 }, + (_, index) => `[![Image ${index}](https://e.com/${index}.png)](https://e.com/${index})` + ) + const out = roundTrip([...links, ...images].join(' ')) + + for (const link of links) expect(out).toContain(link) + for (const image of images) expect(out).toContain(image) + expect(roundTrip(out)).toBe(out) + }) + it('preserves a sized base64 image and escapes quotes in attributes', () => { const dataUrl = '' expect(roundTrip(dataUrl)).toContain('data:image/png;base64,iVBORw0KGgo=') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/text-input-rule.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/text-input-rule.ts new file mode 100644 index 00000000000..70d0578e1cd --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/text-input-rule.ts @@ -0,0 +1,41 @@ +import { type Editor, InputRule, inputRulesPlugin, isExtensionRulesEnabled } from '@tiptap/core' + +/** + * Materializes the actual text-input event before converting its syntax. Native rule ranges may + * extend past the current document for multi-character input; the native plugin still owns undo + * and composition, whose text is already in the document. + */ +export function createTextInputRulePlugins(editor: Editor, name: string, rule: InputRule) { + const extension = editor.extensionManager.extensions.find((item) => item.name === name) + if (!extension || !isExtensionRulesEnabled(extension, editor.options.enableInputRules)) return [] + + let pendingInput: { from: number; to: number; text: string } | null = null + const plugin = inputRulesPlugin({ + editor, + rules: [ + new InputRule({ + find: rule.find, + undoable: rule.undoable, + handler: (props) => { + if (!pendingInput) return rule.handler(props) + const { from, to, text } = pendingInput + props.state.tr.insertText(text, from, to) + return rule.handler({ + ...props, + range: { from: props.range.from, to: from + text.length }, + }) + }, + }), + ], + }) + const handleTextInput = plugin.props.handleTextInput + plugin.props.handleTextInput = (view, from, to, text, defaultTr) => { + pendingInput = { from, to, text } + try { + return handleTextInput?.call(plugin, view, from, to, text, defaultTr) + } finally { + pendingInput = null + } + } + return [plugin] +} diff --git a/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.test.tsx b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.test.tsx new file mode 100644 index 00000000000..8699d870a01 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.test.tsx @@ -0,0 +1,113 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + params: { workspaceId: 'workspace-1' } as Record, + socket: { + isReconnecting: true, + isRetryingWorkflowJoin: false, + blockedJoinWorkflowId: null as string | null, + }, + hasOperationError: false, + toast: { error: vi.fn(() => 'toast-1'), dismiss: vi.fn() }, + setQueryData: vi.fn(), + refetch: vi.fn(), +})) + +vi.mock('@sim/emcn', () => ({ useToast: () => ({ toast: mocks.toast }) })) +vi.mock('next/navigation', () => ({ useParams: () => mocks.params })) +vi.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ setQueryData: mocks.setQueryData }), +})) +vi.mock('@/app/workspace/providers/socket-provider', () => ({ useSocket: () => mocks.socket })) +vi.mock('@/hooks/queries/workspace', () => ({ + useWorkspacePermissionsQuery: () => ({ + data: null, + isLoading: false, + error: null, + refetch: mocks.refetch, + }), + workspaceKeys: { permissions: (id: string) => ['workspace', id, 'permissions'] }, +})) +vi.mock('@/hooks/use-stable-flag', () => ({ useStableFlag: (value: boolean) => value })) +vi.mock('@/hooks/use-user-permissions', () => ({ + useUserPermissions: () => ({ + canRead: true, + canEdit: true, + canAdmin: true, + userPermissions: 'admin', + isLoading: false, + error: null, + }), +})) +vi.mock('@/stores/operation-queue/store', () => ({ + useOperationQueueStore: (select: (state: { hasOperationError: boolean }) => boolean) => + select({ hasOperationError: mocks.hasOperationError }), +})) + +import { WorkspacePermissionsProvider } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' + +;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + +describe('workspace reconnect notifications', () => { + let host: HTMLDivElement + let root: Root + + function renderProvider() { + act(() => + root.render( + +
+ + ) + ) + } + + beforeEach(() => { + vi.clearAllMocks() + mocks.params = { workspaceId: 'workspace-1' } + mocks.socket.isReconnecting = true + mocks.socket.isRetryingWorkflowJoin = false + mocks.socket.blockedJoinWorkflowId = null + mocks.hasOperationError = false + host = document.createElement('div') + document.body.appendChild(host) + root = createRoot(host) + }) + + afterEach(() => { + act(() => root.unmount()) + host.remove() + }) + + it('leaves file reconnect feedback to the inline editor status', () => { + mocks.params.fileId = 'file-1' + renderProvider() + expect(mocks.toast.error).not.toHaveBeenCalled() + }) + + it('retains reconnect feedback elsewhere in the workspace', () => { + mocks.params.workflowId = 'workflow-1' + renderProvider() + expect(mocks.toast.error).toHaveBeenCalledWith('Reconnecting...', expect.any(Object)) + }) + + it('dismisses the workspace reconnect toast when entering a file', () => { + renderProvider() + mocks.params.fileId = 'file-1' + renderProvider() + expect(mocks.toast.dismiss).toHaveBeenCalledWith('toast-1') + expect(mocks.toast.error).toHaveBeenCalledTimes(1) + }) + + it('does not suppress terminal operation errors inside a file', () => { + mocks.params.fileId = 'file-1' + mocks.hasOperationError = true + renderProvider() + expect(mocks.toast.error).toHaveBeenCalledWith('Connection unavailable', expect.any(Object)) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx index 92100372f02..af4ba896db5 100644 --- a/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx +++ b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx @@ -120,6 +120,7 @@ export function WorkspacePermissionsProvider({ children }: WorkspacePermissionsP const params = useParams() const workspaceId = params?.workspaceId as string const urlWorkflowId = params?.workflowId as string | undefined + const isFileViewer = Boolean(params?.fileId) const queryClient = useQueryClient() const hasOperationError = useOperationQueueStore((state) => state.hasOperationError) @@ -131,13 +132,14 @@ export function WorkspacePermissionsProvider({ children }: WorkspacePermissionsP delayMs: RECONNECTING_TOAST_DELAY_MS, minVisibleMs: RECONNECTING_TOAST_MIN_VISIBLE_MS, }) - const realtimeStatusMessage = isOfflineMode - ? null - : showReconnecting - ? 'Reconnecting...' - : isRetryingWorkflowJoin - ? 'Joining workflow...' - : null + const realtimeStatusMessage = + isOfflineMode || isFileViewer + ? null + : showReconnecting + ? 'Reconnecting...' + : isRetryingWorkflowJoin + ? 'Joining workflow...' + : null usePersistentErrorToast(realtimeStatusMessage) // Offline mode only recovers via workspace switch or refresh; the join block diff --git a/apps/sim/lib/api/contracts/v2/__tests__/files-attribution.test.ts b/apps/sim/lib/api/contracts/v2/__tests__/files-attribution.test.ts new file mode 100644 index 00000000000..d92650f6dc4 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/__tests__/files-attribution.test.ts @@ -0,0 +1,22 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { v2FileSchema } from '@/lib/api/contracts/v2/files' +import { ANONYMOUS_USER } from '@/lib/auth/constants' + +describe('file uploader attribution', () => { + it.each([ANONYMOUS_USER.email, 'ada@example.com', 'ada+files@example.co.uk'])( + 'preserves the stored uploader email %s', + (email) => { + expect(v2FileSchema.shape.uploadedByEmail.parse(email)).toBe(email) + } + ) + + it.each(['', 'not-an-email', 'ada@', '@example.com', 'ada @example.com', 'ada@example..com'])( + 'rejects malformed attribution %s', + (email) => { + expect(v2FileSchema.shape.uploadedByEmail.safeParse(email).success).toBe(false) + } + ) +}) diff --git a/apps/sim/lib/api/contracts/v2/files.ts b/apps/sim/lib/api/contracts/v2/files.ts index 50c919b5ba4..f88101075a7 100644 --- a/apps/sim/lib/api/contracts/v2/files.ts +++ b/apps/sim/lib/api/contracts/v2/files.ts @@ -98,7 +98,7 @@ export const v2FileSchema = z 'Canonical containing-folder path. `/` is the workspace root.' ), uploadedByEmail: z - .email() + .email({ pattern: z.regexes.html5Email }) .describe('Current email address of the uploader.') .meta({ examples: ['jane@example.com'] }), /** ISO-8601 timestamp. */ diff --git a/apps/sim/lib/collab-doc/README.md b/apps/sim/lib/collab-doc/README.md index 4613bb9c3b7..36b4b69d080 100644 --- a/apps/sim/lib/collab-doc/README.md +++ b/apps/sim/lib/collab-doc/README.md @@ -1,53 +1,57 @@ -# `@/lib/collab-doc` — server-side collaborative-document conversion - -Server-side conversion between a file's **markdown** (the durable source of truth) and its -collaborative **Yjs document**, so the server can own the doc: seed it, project it back to -markdown, and let the agent write into it while a user is typing. - -## Why this exists - -Collaborative file editing had two writers with no shared CRDT: copilot `edit_content` wrote -markdown straight to the file while the user typed into an ephemeral, client-seeded Yjs doc. They -couldn't reconcile — the agent's edit didn't stream into the editor, and last-writer clobbered. The -fix is a **server-authoritative Yjs doc** both sides write into, with markdown as a projection. - -## What Stage A (this module) provides - -| Function | Purpose | -|---|---| -| `markdownToYDoc(md)` | Cold-start seed: file markdown → a fresh `Y.Doc`. | -| `yDocToMarkdown(ydoc)` | Projection: `Y.Doc` → the file's canonical markdown. | -| `applyMarkdownToYDoc(ydoc, md)` | Agent write: merge new content into a live `Y.Doc` as a minimal CRDT diff (no clobber). | - -### Design decisions (why it's not hacky) - -- **Parity by construction.** The markdown↔ProseMirror step reuses the *exact* client engine - (`parseMarkdownToDoc` / `serializeDocToMarkdown`, `@tiptap/markdown` on the shared extension set) — - not a second markdown implementation — so the server can never diverge from what the editor - renders. The custom-fidelity constructs (tables, footnotes, raw HTML, `sim:` mentions) are covered - by the same code that covers them in the browser; the round-trip test asserts equivalence. -- **Same Yjs binding as the browser.** ProseMirror↔Yjs uses `@tiptap/y-tiptap` (what TipTap's - Collaboration extension uses), pinned to the same version and sharing the same `prosemirror-model` - / `yjs` instances (peer deps) — so the structure the server produces is byte-compatible with the - client, targeting the same `'default'` fragment. -- **Merge, not replace.** `applyMarkdownToYDoc` uses `updateYFragment` (the primitive `ySyncPlugin` - runs on every keystroke) to apply only the diff, so Yjs reconciles the agent's write with in-flight - remote edits. The test proves an agent write and a concurrent remote edit both survive. -- **Server-only, DOM via jsdom.** The markdown engine builds a (never-mounted) TipTap editor that - needs a DOM; on the server it's backed by a single lazily-created `jsdom` window. Lazy-required so - the client bundle never pulls jsdom in. - -## Server-authoritative seeding (shipped alongside this module) - -The realtime relay seeds each room's document from this module over an internal endpoint -(`buildFileDocSeed` → `POST /api/internal/file-doc/seed` → `ensureServerSeed`), which let the entire -client-seeder subsystem (election / deadlines / `triedSeeders` / `MAX_SEED_ROUNDS` / the -`SEED_REQUEST` handshake) be deleted. The client's connect-deadline offline fallback is deliberately -**kept** — it is unrelated to seeding. No feature flag: the cutover is all-at-once. - -## Remaining stages (future PRs) - -- **Durable persistence.** A DB column for the Yjs binary + debounced snapshotting, so a document - survives with no collaborators connected instead of being re-seeded from markdown on cold open. -- **Copilot into the doc + projection.** `edit_content` calls `applyMarkdownToYDoc` when a doc is - live; a debounced `yDocToMarkdown` projection keeps the file's markdown current. +# Server-side collaborative documents + +This module converts workspace Markdown files to and from the shared TipTap/Yjs document. +Markdown is the durable file content; the persisted Yjs binary retains the causal identities and +deletion history needed to reconnect existing clients. Equal Markdown does not imply equal history. + +## Persistence and seeding + +- Convert with the same editor extensions and Markdown pipeline used by the client. +- Preserve native Yjs snapshots. Normalize the Markdown projection, not a detached shared tree: + deleting an empty paragraph in a saved snapshot can delete text a disconnected peer types there later. +- Persist the relay's native full snapshot. Reject a different document identity; stale content + writes also require a throwaway merge proving that the candidate does not omit durable content. +- Commit the prepared binary and Markdown pointer in the same file-row transaction. The content + version and exact cached binary/source hashes must still match their preparation inputs. +- Cache-only saves and seeds use the same file-row lock and revision check. Unchanged snapshots + validate their revision without rewriting the row. Simultaneous cold seeds adopt the winning identity. +- Keep conversion and blob I/O outside the transaction. Bound loaded and prepared binary states to + 12 MiB; oversized or unavailable cache reads fail rather than masquerading as an absent document. +- Retry content/cache conflicts a bounded number of times from fresh reads. Infrastructure errors + propagate so callers can retry without acknowledging an uncommitted snapshot. + +External Markdown writes reconcile through `applyMarkdownToYDoc`, using the existing +`updateYFragment` binding. Equivalent normalized bodies leave the native tree untouched; actual +content changes apply a diff. This preserves unaffected identities, but is not a guarantee that +arbitrary structural rewrites retain every concurrent edit. + +## Compatibility and limits + +All cache writers must use the shared transaction/revision protocol. Drain older application writers +before relying on its guarantees; an old unconditional writer does not participate in the fence. +This change does not alter the document schema or migrate existing documents. + +Legacy caches can contain private normalization deletions that the live relay never received. +Unconditionally merging those caches into snapshots can delete delayed edits or prevent saving. +The relay remains the snapshot owner, as before; this change does not solve retention of extra +cache-only operations invisible in Markdown. Safely unifying all historical state requires an +explicit legacy compatibility plan, not a hash check or a guess at deletion provenance. + +Native nested-list reparenting can lose concurrent edits to moved content in the current binding. +A stable-parent list representation requires a separately tested schema and offline-update migration; +rebuilding a Y.Doc or changing its identity is not a safe migration. + +Conversion is server-side and uses a lazily initialized jsdom window for TipTap. It must not enter +a client bundle. + +## Precedent + +- [Yjs document updates](https://docs.yjs.dev/api/document-updates): native update merging and encoded state. +- [Hocuspocus persistence](https://tiptap.dev/docs/hocuspocus/guides/persistence): preserve Yjs binary + rather than recreating it from JSON on reconnect. +- [PostgreSQL row locking](https://www.postgresql.org/docs/current/explicit-locking.html): serialize + conflicting commits under the existing file-row lock. + +The repository tests cover conversion, native-history preservation, cache conflicts, seed races, +and file-manager transaction wiring. Real PostgreSQL concurrency and live collaboration require +integration validation in addition to those unit tests. diff --git a/apps/sim/lib/collab-doc/collab-state.test.ts b/apps/sim/lib/collab-doc/collab-state.test.ts new file mode 100644 index 00000000000..e60578a2d24 --- /dev/null +++ b/apps/sim/lib/collab-doc/collab-state.test.ts @@ -0,0 +1,322 @@ +/** @vitest-environment node */ +import { createHash } from 'crypto' +import { db } from '@sim/db' +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@sim/db/schema', () => ({ + ...schemaMock, + workspaceFileCollabState: { + fileId: 'file_id', + docState: 'doc_state', + sourceHash: 'source_hash', + }, +})) + +import { workspaceFileCollabState, workspaceFiles } from '@sim/db/schema' +import { + assertCollabDocStateSize, + CollabDocStateConflictError, + commitCollabDocState, + hashMarkdown, + loadCollabDocState, + MAX_COLLAB_DOC_STATE_BYTES, + type PreparedCollabDocState, + saveCollabDocStateInTx, +} from '@/lib/collab-doc/collab-state' + +const VERSION = new Date('2026-09-07T10:00:00.123Z') + +function preparedState(expectedState: PreparedCollabDocState['expectedState'] = null) { + return { docState: new Uint8Array([0, 0]), sourceHash: 'next-source', expectedState } +} + +function saveState(prepared: PreparedCollabDocState) { + return db.transaction((tx) => saveCollabDocStateInTx(tx, 'file-1', prepared)) +} + +function commitState(prepared = preparedState(), version = VERSION.getTime()) { + return commitCollabDocState('workspace-1', 'file-1', version, prepared) +} + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() +}) + +describe('loadCollabDocState', () => { + it.each([undefined, { maxBytes: 1024 }, { maxBytes: MAX_COLLAB_DOC_STATE_BYTES * 2 }])( + 'bounds binary transfer and hashing in SQL with options %j', + async (options) => { + const docState = Buffer.from([0, 0]) + dbChainMockFns.limit.mockResolvedValueOnce([ + { docState, byteCount: 2, sourceHash: 'source-hash', stateHash: 'state-hash' }, + ]) + + await expect(loadCollabDocState('file-1', options)).resolves.toEqual({ + docState: new Uint8Array(docState), + sourceHash: 'source-hash', + stateHash: 'state-hash', + }) + const maxBytes = Math.min( + options?.maxBytes ?? MAX_COLLAB_DOC_STATE_BYTES, + MAX_COLLAB_DOC_STATE_BYTES + ) + expect(dbChainMockFns.select).toHaveBeenCalledWith({ + byteCount: expect.objectContaining({ strings: ['octet_length(', ')'] }), + docState: expect.objectContaining({ + strings: ['CASE WHEN ', ' <= ', ' THEN ', ' END'], + values: [expect.anything(), maxBytes, 'doc_state'], + }), + sourceHash: 'source_hash', + stateHash: expect.objectContaining({ + strings: ['CASE WHEN ', ' <= ', ' THEN encode(sha256(', "), 'hex') END"], + values: [expect.anything(), maxBytes, 'doc_state'], + }), + }) + expect(dbChainMockFns.where).toHaveBeenCalledWith({ + type: 'eq', + left: 'file_id', + right: 'file-1', + }) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(1) + } + ) + + it('returns null only when the cache row is absent', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([]) + await expect(loadCollabDocState('file-1', { maxBytes: 1024 })).resolves.toBeNull() + }) + + it.each([undefined, { maxBytes: 1024 }])( + 'rejects an oversized existing state: %j', + async (options) => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + docState: null, + stateHash: null, + sourceHash: 'source', + byteCount: MAX_COLLAB_DOC_STATE_BYTES + 1, + }, + ]) + await expect(loadCollabDocState('file-1', options)).rejects.toThrow(RangeError) + } + ) + + it('does not return an incomplete cache token', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { docState: Buffer.from([0, 0]), stateHash: null, sourceHash: 'source', byteCount: 2 }, + ]) + await expect(loadCollabDocState('file-1')).rejects.toThrow(RangeError) + }) + + it('returns an owned snapshot, not the driver buffer', async () => { + const docState = Buffer.from([0, 0]) + dbChainMockFns.limit.mockResolvedValueOnce([ + { docState, byteCount: 2, sourceHash: 'source', stateHash: 'state' }, + ]) + const cached = await loadCollabDocState('file-1') + docState[0] = 255 + expect(cached?.docState).toEqual(new Uint8Array([0, 0])) + }) + + it.each([-1, Number.NaN, 1.5])( + 'rejects invalid byte limit %s before querying', + async (maxBytes) => { + await expect(loadCollabDocState('file-1', { maxBytes })).rejects.toThrow(RangeError) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + } + ) + + it('propagates database errors instead of reporting an absent cache', async () => { + const error = new Error('database unavailable') + dbChainMockFns.limit.mockRejectedValueOnce(error) + await expect(loadCollabDocState('file-1')).rejects.toBe(error) + }) +}) + +describe('saveCollabDocStateInTx', () => { + it('fences an unchanged snapshot without rewriting its binary or timestamp', async () => { + const prepared = preparedState() + prepared.expectedState = { + sourceHash: prepared.sourceHash, + stateHash: createHash('sha256').update(prepared.docState).digest('hex'), + } + queueTableRows(workspaceFileCollabState, [{ fileId: 'file-1' }]) + + await expect(saveState(prepared)).resolves.toBeUndefined() + expect(dbChainMockFns.select).toHaveBeenCalledWith({ fileId: 'file_id' }) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(1) + expect(dbChainMockFns.where).toHaveBeenCalledWith( + expect.objectContaining({ + conditions: expect.arrayContaining([ + { type: 'eq', left: 'file_id', right: 'file-1' }, + { type: 'eq', left: 'source_hash', right: prepared.sourceHash }, + expect.objectContaining({ type: 'eq', right: prepared.expectedState.stateHash }), + ]), + }) + ) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + + it('rejects an unchanged snapshot when its cached revision no longer matches', async () => { + const prepared = preparedState() + prepared.expectedState = { + sourceHash: prepared.sourceHash, + stateHash: createHash('sha256').update(prepared.docState).digest('hex'), + } + + await expect(saveState(prepared)).rejects.toBeInstanceOf(CollabDocStateConflictError) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + + it('inserts an observed absent state without replacing a racing writer', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([{ fileId: 'file-1' }]) + const prepared = preparedState() + await expect(saveState(prepared)).resolves.toBeUndefined() + expect(dbChainMockFns.values).toHaveBeenCalledWith({ + fileId: 'file-1', + docState: Buffer.from(prepared.docState), + sourceHash: 'next-source', + updatedAt: expect.any(Date), + }) + expect(dbChainMockFns.onConflictDoNothing).toHaveBeenCalledWith({ target: 'file_id' }) + expect(dbChainMockFns.onConflictDoUpdate).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('conflicts when another writer inserted the first state', async () => { + await expect(saveState(preparedState())).rejects.toBeInstanceOf(CollabDocStateConflictError) + expect(dbChainMockFns.onConflictDoUpdate).not.toHaveBeenCalled() + }) + + it('requires both the projected source and exact binary history token', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([{ fileId: 'file-1' }]) + await saveState(preparedState({ sourceHash: 'previous-source', stateHash: 'previous-state' })) + expect(dbChainMockFns.where).toHaveBeenCalledWith({ + type: 'and', + conditions: [ + { type: 'eq', left: 'file_id', right: 'file-1' }, + { type: 'eq', left: 'source_hash', right: 'previous-source' }, + { + type: 'eq', + left: expect.objectContaining({ + strings: ['CASE WHEN octet_length(', ') <= ', ' THEN encode(sha256(', "), 'hex') END"], + values: ['doc_state', MAX_COLLAB_DOC_STATE_BYTES, 'doc_state'], + }), + right: 'previous-state', + }, + ], + }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + + it('does not fall back to an insert after a stale token or deleted cache row', async () => { + await expect( + saveState(preparedState({ sourceHash: 'source', stateHash: 'stale' })) + ).rejects.toBeInstanceOf(CollabDocStateConflictError) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + + it('rejects oversized writes before preparing any mutation', async () => { + await expect( + saveState({ + ...preparedState(), + docState: new Uint8Array(MAX_COLLAB_DOC_STATE_BYTES + 1), + }) + ).rejects.toThrow(RangeError) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('propagates a failed cache write so its caller rolls back the transaction', async () => { + const error = new Error('write failed') + dbChainMockFns.returning.mockRejectedValueOnce(error) + await expect(saveState(preparedState())).rejects.toBe(error) + }) +}) + +describe('commitCollabDocState', () => { + it('locks only the active scoped workspace file before committing its cache', async () => { + queueTableRows(workspaceFiles, [{ contentUpdatedAt: VERSION }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ fileId: 'file-1' }]) + await expect(commitState()).resolves.toEqual({ + status: 'committed', + version: VERSION.getTime(), + }) + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + expect(dbChainMockFns.limit).toHaveBeenCalledWith(1) + expect(dbChainMockFns.where).toHaveBeenCalledWith({ + type: 'and', + conditions: [ + { type: 'eq', left: workspaceFiles.id, right: 'file-1' }, + { type: 'eq', left: workspaceFiles.workspaceId, right: 'workspace-1' }, + { type: 'eq', left: workspaceFiles.context, right: 'workspace' }, + { type: 'isNull', column: workspaceFiles.deletedAt }, + ], + }) + expect(dbChainMockFns.for.mock.invocationCallOrder[0]).toBeLessThan( + dbChainMockFns.insert.mock.invocationCallOrder[0] + ) + expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('does not create cache state for a missing, deleted, or differently scoped file', async () => { + await expect(commitState()).resolves.toEqual({ status: 'missing' }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + + it.each([-1, 1])( + 'rejects a changed durable version (%s ms) under the lock', + async (difference) => { + queueTableRows(workspaceFiles, [ + { contentUpdatedAt: new Date(VERSION.getTime() + difference) }, + ]) + await expect(commitState()).resolves.toEqual({ status: 'conflict' }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + } + ) + + it('reports a racing cache revision as a conflict, even with unchanged durable content', async () => { + queueTableRows(workspaceFiles, [{ contentUpdatedAt: VERSION }]) + const prepared = preparedState({ sourceHash: 'next-source', stateHash: 'stale-history' }) + await expect(commitState(prepared)).resolves.toEqual({ status: 'conflict' }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + }) + + it('rejects an oversized state before opening a transaction', async () => { + await expect( + commitState({ + ...preparedState(), + docState: new Uint8Array(MAX_COLLAB_DOC_STATE_BYTES + 1), + }) + ).rejects.toThrow(RangeError) + expect(dbChainMockFns.transaction).not.toHaveBeenCalled() + }) + + it('propagates infrastructure failures instead of misclassifying them as conflicts', async () => { + queueTableRows(workspaceFiles, [{ contentUpdatedAt: VERSION }]) + const error = new Error('database connection lost') + dbChainMockFns.returning.mockRejectedValueOnce(error) + await expect(commitState()).rejects.toBe(error) + }) +}) + +describe('collaborative state byte bounds', () => { + it('accepts the exact maximum and rejects one byte more', () => { + expect(() => assertCollabDocStateSize(new Uint8Array(MAX_COLLAB_DOC_STATE_BYTES))).not.toThrow() + expect(() => assertCollabDocStateSize(new Uint8Array(MAX_COLLAB_DOC_STATE_BYTES + 1))).toThrow( + RangeError + ) + }) + + it('hashes the exact markdown bytes', () => { + expect(hashMarkdown(Buffer.from('abc'))).toBe( + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad' + ) + }) +}) diff --git a/apps/sim/lib/collab-doc/collab-state.ts b/apps/sim/lib/collab-doc/collab-state.ts index 115ab356191..a1e065b9249 100644 --- a/apps/sim/lib/collab-doc/collab-state.ts +++ b/apps/sim/lib/collab-doc/collab-state.ts @@ -1,82 +1,185 @@ import { createHash } from 'crypto' import { db } from '@sim/db' -import { workspaceFileCollabState } from '@sim/db/schema' -import { eq } from 'drizzle-orm' +import { workspaceFileCollabState, workspaceFiles } from '@sim/db/schema' +import { and, eq, isNull, sql } from 'drizzle-orm' +import type { DbTransaction } from '@/lib/db/types' -/** - * The cached-collab-state cache (`workspace_file_collab_state`) lets a cold room open load the file's - * last-persisted Yjs binary directly instead of re-converting markdown → Yjs on every open — the - * Hocuspocus load-document pattern. See {@link workspaceFileCollabState} for the full rationale. - */ +/** Matches the decoded size of the persist endpoint's 16 MiB base64 snapshot limit. */ +export const MAX_COLLAB_DOC_STATE_BYTES = 12 * 1024 * 1024 + +/** Reject oversized snapshots before copying, decoding, or writing them. */ +export function assertCollabDocStateSize(docState: Uint8Array): void { + if (docState.byteLength > MAX_COLLAB_DOC_STATE_BYTES) { + throw new RangeError('Collaborative document state exceeds the 12 MiB limit') + } +} -/** sha256 (hex) of a markdown buffer — the freshness tag matching a cached doc state to the live file. */ +/** SHA-256 freshness tag for the exact durable markdown bytes. */ export function hashMarkdown(markdown: Buffer): string { return createHash('sha256').update(markdown).digest('hex') } -/** A file's stored collaborative document, with the markdown it was last derived from. */ -export interface CachedCollabDocState { +/** Both the projected markdown and the complete binary history identify a cached revision. */ +export interface CollabDocStateToken { + sourceHash: string + stateHash: string +} + +/** The binary document is authoritative for CRDT identity, including deleted content history. */ +export interface CachedCollabDocState extends CollabDocStateToken { + docState: Uint8Array +} + +/** A snapshot prepared against an exact cached revision, or an observed absent cache row. */ +export interface PreparedCollabDocState { docState: Uint8Array - /** Hash of the markdown this binary projects to — `null`s out nothing; compare to decide freshness. */ sourceHash: string + expectedState: CollabDocStateToken | null +} + +/** A different writer has replaced the cached revision used to prepare this snapshot. */ +export class CollabDocStateConflictError extends Error { + constructor(fileId: string) { + super(`Collaborative document state changed for file ${fileId}`) + this.name = 'CollabDocStateConflictError' + } } /** - * Load a file's stored Yjs binary, fresh or not. - * - * FRESH (its `sourceHash` matches the file's current markdown) means it can seed a room verbatim. - * STALE means the markdown moved on out-of-band, and the caller must bring it up to date — but it must - * do so by UPDATING this document, never by building a second one: the stored binary carries the - * document's identity, and a rebuilt document's items carry different client ids, so any client still - * holding the old one would merge the two into duplicated content. Either way this row is the file's - * collaborative document; there is only ever one. + * Load the existing binary without mistaking an oversized state or a database failure for absence. + * CASE bounds both the transferred binary and the hash work before either is materialized. */ -export async function loadCollabDocState(fileId: string): Promise { +export async function loadCollabDocState( + fileId: string, + options?: { maxBytes: number } +): Promise { + const maxBytes = Math.min( + options?.maxBytes ?? MAX_COLLAB_DOC_STATE_BYTES, + MAX_COLLAB_DOC_STATE_BYTES + ) + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Collaborative document state byte limit must be a non-negative integer') + } + const byteCount = sql`octet_length(${workspaceFileCollabState.docState})` const [row] = await db .select({ - docState: workspaceFileCollabState.docState, + byteCount, + docState: sql`CASE WHEN ${byteCount} <= ${maxBytes} THEN ${workspaceFileCollabState.docState} END`, sourceHash: workspaceFileCollabState.sourceHash, + stateHash: sql< + string | null + >`CASE WHEN ${byteCount} <= ${maxBytes} THEN encode(sha256(${workspaceFileCollabState.docState}), 'hex') END`, }) .from(workspaceFileCollabState) .where(eq(workspaceFileCollabState.fileId, fileId)) .limit(1) if (!row) return null - return { docState: new Uint8Array(row.docState), sourceHash: row.sourceHash } + if (row.byteCount > maxBytes || row.docState === null || row.stateHash === null) { + throw new RangeError(`Collaborative document state exceeds the ${maxBytes} byte limit`) + } + return { + docState: new Uint8Array(row.docState), + sourceHash: row.sourceHash, + stateHash: row.stateHash, + } } /** - * The markdown hash this file's cached doc state was derived from — i.e. the exact bytes the live - * document last projected onto the file — or `null` when nothing is cached. Selects only the tag, so a - * caller asking "is what's on disk still our own last write?" never loads the binary to find out. + * Replace only the cached revision used to prepare this snapshot. The caller must already hold the + * workspaceFiles row lock and validate its content version in this transaction. */ -export async function collabDocStateSourceHash(fileId: string): Promise { - const [row] = await db - .select({ sourceHash: workspaceFileCollabState.sourceHash }) - .from(workspaceFileCollabState) - .where(eq(workspaceFileCollabState.fileId, fileId)) - .limit(1) - return row?.sourceHash ?? null +export async function saveCollabDocStateInTx( + tx: DbTransaction, + fileId: string, + prepared: PreparedCollabDocState +): Promise { + assertCollabDocStateSize(prepared.docState) + const expected = prepared.expectedState + const matchesExpected = expected + ? and( + eq(workspaceFileCollabState.fileId, fileId), + eq(workspaceFileCollabState.sourceHash, expected.sourceHash), + eq( + sql< + string | null + >`CASE WHEN octet_length(${workspaceFileCollabState.docState}) <= ${MAX_COLLAB_DOC_STATE_BYTES} THEN encode(sha256(${workspaceFileCollabState.docState}), 'hex') END`, + expected.stateHash + ) + ) + : undefined + + if ( + expected?.sourceHash === prepared.sourceHash && + expected.stateHash === createHash('sha256').update(prepared.docState).digest('hex') + ) { + const [current] = await tx + .select({ fileId: workspaceFileCollabState.fileId }) + .from(workspaceFileCollabState) + .where(matchesExpected) + .limit(1) + if (!current) throw new CollabDocStateConflictError(fileId) + return + } + + const values = { + docState: Buffer.from(prepared.docState), + sourceHash: prepared.sourceHash, + updatedAt: new Date(), + } + const [accepted] = expected + ? await tx + .update(workspaceFileCollabState) + .set(values) + .where(matchesExpected) + .returning({ fileId: workspaceFileCollabState.fileId }) + : await tx + .insert(workspaceFileCollabState) + .values({ fileId, ...values }) + .onConflictDoNothing({ target: workspaceFileCollabState.fileId }) + .returning({ fileId: workspaceFileCollabState.fileId }) + + if (!accepted) throw new CollabDocStateConflictError(fileId) } -/** - * Persist a collaborative doc's Yjs binary as the file's cold-start state, tagged with the hash of the - * markdown it was derived from. Upsert — one row per file. Called from the server-side persist right - * after the markdown is written, so the cached binary and its `sourceHash` are always consistent with - * the file that was just saved. - */ -export async function saveCollabDocState( +export type CommitCollabDocStateResult = + | { status: 'committed'; version: number } + | { status: 'missing' } + | { status: 'conflict' } + +/** Commit a cache-only refresh against the locked file version and the exact cached revision. */ +export async function commitCollabDocState( + workspaceId: string, fileId: string, - docState: Uint8Array, - sourceHash: string -): Promise { - const state = Buffer.from(docState) - const updatedAt = new Date() - await db - .insert(workspaceFileCollabState) - .values({ fileId, docState: state, sourceHash, updatedAt }) - .onConflictDoUpdate({ - target: workspaceFileCollabState.fileId, - set: { docState: state, sourceHash, updatedAt }, + expectedVersion: number, + prepared: PreparedCollabDocState +): Promise { + assertCollabDocStateSize(prepared.docState) + try { + return await db.transaction(async (tx): Promise => { + const [file] = await tx + .select({ contentUpdatedAt: workspaceFiles.contentUpdatedAt }) + .from(workspaceFiles) + .where( + and( + eq(workspaceFiles.id, fileId), + eq(workspaceFiles.workspaceId, workspaceId), + eq(workspaceFiles.context, 'workspace'), + isNull(workspaceFiles.deletedAt) + ) + ) + .for('update') + .limit(1) + + if (!file) return { status: 'missing' } + const version = file.contentUpdatedAt.getTime() + if (version !== expectedVersion) return { status: 'conflict' } + + await saveCollabDocStateInTx(tx, fileId, prepared) + return { status: 'committed', version } }) + } catch (error) { + if (error instanceof CollabDocStateConflictError) return { status: 'conflict' } + throw error + } } diff --git a/apps/sim/lib/collab-doc/converter.test.ts b/apps/sim/lib/collab-doc/converter.test.ts index f31796d2a82..06664d84664 100644 --- a/apps/sim/lib/collab-doc/converter.test.ts +++ b/apps/sim/lib/collab-doc/converter.test.ts @@ -7,6 +7,13 @@ import { prosemirrorJSONToYDoc, yDocToProsemirrorJSON } from '@tiptap/y-tiptap' import { beforeAll, describe, expect, it } from 'vitest' import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' +import { + applyMarkdownToYDoc, + markdownToYDoc, + yDocToFileMarkdown, + yDocToMarkdown, +} from '@/lib/collab-doc/converter' +import { COLLAB_DOC_FIELD } from '@/lib/collab-doc/field' import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' import { @@ -19,14 +26,6 @@ import { parseMarkdownToDoc, serializeMarkdownBody, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' -import { - applyMarkdownToYDoc, - canonicalizeYDoc, - markdownToYDoc, - yDocToFileMarkdown, - yDocToMarkdown, -} from './converter' -import { COLLAB_DOC_FIELD } from './field' /** Representative markdown covering the custom-fidelity constructs (tables, code, lists, marks). */ const SAMPLES = [ @@ -40,6 +39,7 @@ const SAMPLES = [ 'A footnote reference[^1].\n\n[^1]: the footnote body.', 'Before.\n\n
untouched raw html
\n\nAfter.', '- [ ] todo\n- [x] done', + '[](https://e.com)', ] beforeAll(() => { @@ -62,17 +62,7 @@ describe('collab-doc converter', () => { expect(yDocToMarkdown(markdownToYDoc(''))).toBe(serializeMarkdownBody('')) }) - /** - * The Files editor paints a read-only placeholder built from the file's markdown, then swaps in the - * live collaborative doc once the CRDT syncs. Anything the CRDT holds that the markdown projection - * cannot round-trip shows up as the document reflowing its spacing a beat after the file appears. - * - * The case that matters is a state no markdown parse produced: the user presses Enter on an empty - * line, which puts a real empty paragraph in the CRDT. Projecting that to markdown and re-parsing it - * has to give the same blocks back — otherwise the blank line is both invisible on first paint and - * deleted for good once the room goes cold. - */ - describe('placeholder ⇄ live CRDT parity', () => { + describe('Markdown-derived seed and placeholder parity', () => { const shapeOf = (blocks: JSONContent[] | undefined) => (blocks ?? []) .map((n) => (n.type === 'paragraph' && !n.content?.length ? '∅' : n.type)) @@ -107,66 +97,6 @@ describe('collab-doc converter', () => { live.destroy() }) - /** - * Every CRDT state below is one markdown genuinely cannot describe, so the round-trip is NOT the - * identity on it — a run past the parse bound, trailing empties the serializer collapses, an empty - * paragraph in a document that must parse whole. Each one used to reach a room verbatim and reflow - * the doc a beat after it painted. `canonicalizeYDoc` is the single pass that resolves all of them, - * so assert the invariant it establishes rather than enumerating what markdown can hold. - */ - it.each([ - [ - 'a run past the per-gap bound', - () => typedInto('a\n\nb', (f) => f.insert(1, paragraphs(30))), - ], - ['trailing empties', () => typedInto('a\n\nb', (f) => f.insert(f.length, paragraphs(3)))], - ['leading empties', () => typedInto('a\n\nb', (f) => f.insert(0, paragraphs(2)))], - ['between two lists', () => typedInto('- a\n\n- b', (f) => f.insert(1, paragraphs(1)))], - ['between two quotes', () => typedInto('> a\n\n> b', (f) => f.insert(1, paragraphs(1)))], - [ - 'inside a raw-HTML document', - () => typedInto('# H\n\nbody\n\n
x
', (f) => f.insert(1, paragraphs(1))), - ], - [ - 'inside a reference-definition document', - () => - typedInto('# H\n\nsee [y][r]\n\n[r]: https://e.com', (f) => f.insert(1, paragraphs(1))), - ], - ])('canonicalizing restores parity: %s', (_label, build) => { - const live = build() - canonicalizeYDoc(live) - const { crdt, placeholder } = parity(live) - expect(placeholder).toBe(crdt) - // Idempotent: a canonical doc is already its own markdown projection, so a second pass is a no-op. - expect(canonicalizeYDoc(live)).toBe(false) - live.destroy() - }) - - /** - * The point of canonical is that the CRDT and the durable bytes describe the same document, so the - * invariant is stated against the bytes that get WRITTEN — `yDocToFileMarkdown`, post-process and - * all. Converging on the bare serializer output instead would leave that pass's fidelity fixes - * (empty list markers, escaped callout markers) outside the fixed point. - */ - it.each([ - ['a callout the serializer escapes', '# T\n\n> [!NOTE]\n> body\n\ntail'], - ['a list with an empty item', '# T\n\n- parent\n - \n- after'], - ['trailing blank run', '# T\n\nbody\n\n\n\n'], - ['ends on a list', '# T\n\nintro\n\n- a\n- b'], - ])('a canonical doc projects to exactly the file body: %s', (_label, md) => { - const live = markdownToYDoc(md) - canonicalizeYDoc(live) - - const body = splitFrontmatter(yDocToFileMarkdown(live)).body - // Re-reading the written bytes must rebuild the very doc the CRDT holds. - expect(shapeOf(editorNormalForm(body).content)).toBe( - shapeOf(yDocToProsemirrorJSON(live, COLLAB_DOC_FIELD).content) - ) - // And a second pass has nothing left to do. - expect(canonicalizeYDoc(live)).toBe(false) - live.destroy() - }) - it('holds for every representative document', () => { for (const md of [ ...SAMPLES, @@ -184,35 +114,6 @@ describe('collab-doc converter', () => { } }) - /** - * `canonicalizeYDoc`'s return value gates whether both callers re-encode, so it has to be true - * whenever the doc actually moved. Deciding that from the markdown projection could not work: the - * repair is the trailing paragraph, which serializes to a blank line the post-process collapses, so - * every doc ending on a list/heading/table/rule was repaired and still reported unchanged — and the - * cached snapshot kept the unrepaired bytes, reopening the stacking-empties path. - */ - it.each([ - ['ends with a list', '# T\n\nbody\n\n- a\n- b'], - ['ends with a heading', '# T\n\nbody\n\n## Tail'], - ['ends with a table', '# T\n\n| a | b |\n| --- | --- |\n| 1 | 2 |'], - ['ends with a rule', '# T\n\nbody\n\n---'], - ])('reports the repair it actually made: %s', (_label, md) => { - // A doc built from the RAW parse — what a snapshot cached before this normalization looks like. - const doc = prosemirrorJSONToYDoc( - markdownSchemaForTest(), - parseMarkdownToDoc(md), - COLLAB_DOC_FIELD - ) - const before = shapeOf(yDocToProsemirrorJSON(doc, COLLAB_DOC_FIELD).content) - - const reported = canonicalizeYDoc(doc) - - const after = shapeOf(yDocToProsemirrorJSON(doc, COLLAB_DOC_FIELD).content) - expect(after).toBe(`${before},∅`) - expect(reported).toBe(true) - doc.destroy() - }) - /** * Opening a document must not CHANGE it. ProseMirror appends an empty paragraph to any doc that * does not end in one, so a seed ending on a list, heading, table, or rule used to be rewritten by @@ -253,6 +154,104 @@ describe('collab-doc converter', () => { }) }) + describe('equivalent external bodies preserve native Yjs history', () => { + it.each([ + ['trailing newlines', 'base', 'base\n\n'], + ['alternate emphasis syntax', '**base**', '__base__'], + ['alternate bullet marker', '- first\n- second', '* first\n* second'], + ['CRLF line endings', 'first\n\nsecond', 'first\r\n\r\nsecond'], + ])('retains a delayed edit into an empty tail with %s', (_label, body, equivalentBody) => { + const server = markdownToYDoc(body) + const tail = new Y.XmlElement('paragraph') + tail.insert(0, [new Y.XmlText()]) + const fragment = server.getXmlFragment(COLLAB_DOC_FIELD) + fragment.push([tail]) + const remote = new Y.Doc() + const before = Y.encodeStateAsUpdate(server) + Y.applyUpdate(remote, before) + + applyMarkdownToYDoc(server, equivalentBody) + + expect(Y.encodeStateAsUpdate(server)).toEqual(before) + expect(fragment.get(fragment.length - 1)).toBe(tail) + const remoteFragment = remote.getXmlFragment(COLLAB_DOC_FIELD) + const remoteTail = remoteFragment.get(remoteFragment.length - 1) as Y.XmlElement + const remoteText = remoteTail.get(0) as Y.XmlText + remoteText.insert(0, 'late offline text') + Y.applyUpdate(server, Y.encodeStateAsUpdate(remote)) + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server)) + + expect(yDocToFileMarkdown(server)).toContain('late offline text') + expect(yDocToFileMarkdown(remote)).toBe(yDocToFileMarkdown(server)) + remote.destroy() + server.destroy() + }) + + it('updates frontmatter without deleting a delayed body edit target', () => { + const server = markdownToYDoc('base') + const tail = new Y.XmlElement('paragraph') + tail.insert(0, [new Y.XmlText()]) + server.getXmlFragment(COLLAB_DOC_FIELD).push([tail]) + const remote = new Y.Doc() + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server)) + const { frontmatter, body } = splitFrontmatter('---\ntitle: changed\n---\n\nbase') + + applyMarkdownToYDoc(server, body) + server.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.frontmatterKey, frontmatter) + const remoteTail = remote.getXmlFragment(COLLAB_DOC_FIELD).get(1) as Y.XmlElement + const remoteText = remoteTail.get(0) as Y.XmlText + remoteText.insert(0, 'late offline text') + Y.applyUpdate(server, Y.encodeStateAsUpdate(remote)) + Y.applyUpdate(remote, Y.encodeStateAsUpdate(server)) + + expect(yDocToFileMarkdown(server)).toContain('title: changed') + expect(yDocToFileMarkdown(server)).toContain('late offline text') + expect(yDocToFileMarkdown(remote)).toBe(yDocToFileMarkdown(server)) + remote.destroy() + server.destroy() + }) + + it('does not generate repeated repair identities for a legacy heading snapshot', () => { + const doc = prosemirrorJSONToYDoc( + markdownSchemaForTest(), + parseMarkdownToDoc('# Heading'), + COLLAB_DOC_FIELD + ) + const before = Y.encodeStateAsUpdate(doc) + + for (let attempt = 0; attempt < 12; attempt++) { + applyMarkdownToYDoc(doc, '# Heading') + } + + expect(doc.getXmlFragment(COLLAB_DOC_FIELD).length).toBe(1) + expect(Y.encodeStateAsUpdate(doc)).toEqual(before) + doc.destroy() + }) + + it.each([ + ['plain text', 'base', 'changed'], + ['adding a paragraph', 'base', 'base\n\nnew paragraph'], + ['deleting a paragraph', 'base\n\nremoved', 'base'], + ['inline formatting', 'base', '**base**'], + ['link destination', '[base](https://a.test)', '[base](https://b.test)'], + ['list depth', '- first\n- second', '- first\n - second'], + ['task state', '- [ ] task', '- [x] task'], + ['code whitespace', '```\na\n\nb\n```', '```\na\nb\n```'], + ['code language', '```js\nx\n```', '```ts\nx\n```'], + ['image source', '![alt](https://a.test/a.png)', '![alt](https://a.test/b.png)'], + ['clearing the body', 'base', ''], + ])('still applies actual changes to %s', (_label, beforeBody, afterBody) => { + const doc = markdownToYDoc(beforeBody) + const before = yDocToMarkdown(doc) + + applyMarkdownToYDoc(doc, afterBody) + + expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody(afterBody)) + expect(yDocToMarkdown(doc)).not.toBe(before) + doc.destroy() + }) + }) + it('applies new content into an existing doc (agent write)', () => { const ydoc = markdownToYDoc('# Hello\n\nWorld.') applyMarkdownToYDoc(ydoc, '# Hello\n\nWorld and then some more.') diff --git a/apps/sim/lib/collab-doc/converter.ts b/apps/sim/lib/collab-doc/converter.ts index 33aa546d2c3..bc8a6fc7e9a 100644 --- a/apps/sim/lib/collab-doc/converter.ts +++ b/apps/sim/lib/collab-doc/converter.ts @@ -8,6 +8,7 @@ import { yDocToProsemirrorJSON, } from '@tiptap/y-tiptap' import type * as Y from 'yjs' +import { COLLAB_DOC_FIELD } from '@/lib/collab-doc/field' import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' import { applyFrontmatter, @@ -17,7 +18,6 @@ import { editorNormalForm, serializeDocToMarkdown, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' -import { COLLAB_DOC_FIELD } from './field' /** * Server-side conversion between a file's markdown and its collaborative Yjs document. @@ -112,59 +112,19 @@ export function yDocToFileMarkdown(ydoc: Y.Doc): string { } /** - * Converge a collaborative {@link Y.Doc} onto its own markdown projection — the document's CANONICAL - * form — and report whether anything changed. - * - * A ProseMirror document is strictly richer than markdown, so `parse ∘ serialize` is not the identity: - * trailing empty paragraphs are collapsed by `postProcessSerializedMarkdown`, a blank run past the parse - * bound is truncated, and a document that must parse whole (raw HTML, reference definitions) keeps no - * empty paragraphs at all. A CRDT holding any such state describes a document its own markdown cannot - * reproduce — so the file renders one way from the live doc and another from the durable bytes, and the - * difference surfaces as the editor reflowing a beat after it paints, then silently discarding the - * spacing once the room goes cold. - * - * The fix is to keep the CRDT inside the image of the parse. Defining canonical as "what the round-trip - * produces" rather than as a hand-written list of what markdown cannot hold is what makes this - * self-maintaining: every future gap between the two representations is absorbed here automatically, - * with no second place to update. Idempotent by construction — a canonical doc projects to markdown that - * parses back to itself, so a second call is a no-op — and it applies the difference through - * {@link applyMarkdownToYDoc}, so it is a minimal CRDT diff rather than a replacement. - * - * Call this on a DETACHED doc (a decoded snapshot), never on a live room: it is a correctness pass for - * durable artifacts, and converging a document somebody is typing into would move their caret. - * - * "Changed" is decided on the DOCUMENT, not on its markdown. Comparing projections looks equivalent and - * is not: the repairs this pass exists to make are precisely the ones markdown cannot express, so a - * markdown-equality check is blind to them. Concretely, appending the trailing paragraph - * {@link editorNormalForm} requires serializes to a trailing blank line that - * `postProcessSerializedMarkdown` collapses — so every doc ending on a list, heading, table, or rule was - * repaired here and still reported unchanged, and both callers key their re-encode off that flag. The - * cached snapshot then kept the UNREPAIRED bytes, which is the one path back into the stacking-empties - * bug this pass was written to close. - */ -export function canonicalizeYDoc(ydoc: Y.Doc): boolean { - ensureDomForTipTap() - const before = yDocToProsemirrorJSON(ydoc, COLLAB_DOC_FIELD) - // Converge on the body that will actually be WRITTEN, post-process included — the same pass - // `yDocToFileMarkdown` applies. Targeting the bare serializer output would define canonical against a - // string the file never contains, so the fidelity fixes that pass makes (empty list markers that - // re-parse wrong, backslash-escaped callout markers) would sit outside the fixed point this exists to - // establish, and the live doc could settle on a shape the durable bytes do not reproduce. - applyMarkdownToYDoc(ydoc, postProcessSerializedMarkdown(serializeDocToMarkdown(before))) - return JSON.stringify(yDocToProsemirrorJSON(ydoc, COLLAB_DOC_FIELD)) !== JSON.stringify(before) -} - -/** - * Apply new markdown content into an EXISTING collaborative {@link Y.Doc} as a minimal CRDT diff, - * merging with any concurrent user edits rather than replacing the document. This is how the agent - * writes into a live doc: `updateYFragment` computes exactly the changes between the fragment's - * current content and the target and applies them as Yjs operations — the same primitive TipTap's - * `ySyncPlugin` uses on every keystroke — so Yjs reconciles them with in-flight remote edits. + * Apply an external body change through TipTap's CRDT diff. Equivalent Markdown must not + * normalize the native tree: deleting an empty paragraph also deletes the target of delayed edits. */ export function applyMarkdownToYDoc(ydoc: Y.Doc, markdown: string): void { ensureDomForTipTap() const schema = markdownSchema() const target = ProseMirrorNode.fromJSON(schema, editorNormalForm(markdown)) + const currentProjection = ProseMirrorNode.fromJSON( + schema, + editorNormalForm(postProcessSerializedMarkdown(yDocToMarkdown(ydoc))) + ) + if (currentProjection.eq(target)) return + const fragment = ydoc.getXmlFragment(COLLAB_DOC_FIELD) // `updateYFragment` diffs against the fragment's CURRENT content, so it needs the fragment↔PM // binding metadata (the element/mark mapping the live editor's ySyncPlugin normally maintains). diff --git a/apps/sim/lib/collab-doc/persist.test.ts b/apps/sim/lib/collab-doc/persist.test.ts index 735bcd94849..15b4853476e 100644 --- a/apps/sim/lib/collab-doc/persist.test.ts +++ b/apps/sim/lib/collab-doc/persist.test.ts @@ -1,22 +1,25 @@ /** * @vitest-environment node */ + +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' +import * as collabState from '@/lib/collab-doc/collab-state' const { mockGetWorkspaceFile, mockFetchBuffer, mockUpdateContent, - mockSaveState, - mockStateSourceHash, + mockCommitState, + mockLoadState, ContentVersionConflictError, } = vi.hoisted(() => ({ mockGetWorkspaceFile: vi.fn(), mockFetchBuffer: vi.fn(), mockUpdateContent: vi.fn(), - mockSaveState: vi.fn(), - mockStateSourceHash: vi.fn(), + mockCommitState: vi.fn(), + mockLoadState: vi.fn(), ContentVersionConflictError: class ContentVersionConflictError extends Error {}, })) @@ -27,17 +30,24 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ updateWorkspaceFileContent: mockUpdateContent, })) -vi.mock('./collab-state', () => ({ - hashMarkdown: (buffer: Buffer) => `hash:${buffer.toString('utf-8')}`, - saveCollabDocState: mockSaveState, - collabDocStateSourceHash: mockStateSourceHash, -})) +vi.spyOn(collabState, 'loadCollabDocState').mockImplementation(mockLoadState) +vi.spyOn(collabState, 'commitCollabDocState').mockImplementation(mockCommitState) -import { markdownToYDoc, yDocToFileMarkdown } from './converter' -import { persistFileDoc } from './persist' +import type { CachedCollabDocState, PreparedCollabDocState } from '@/lib/collab-doc/collab-state' +import { applyMarkdownToYDoc, markdownToYDoc, yDocToFileMarkdown } from '@/lib/collab-doc/converter' +import { COLLAB_DOC_FIELD } from '@/lib/collab-doc/field' +import { persistFileDoc } from '@/lib/collab-doc/persist' const VERSION = new Date('2026-01-01T00:00:00.000Z') +function cachedState(docState: Uint8Array, markdown: Buffer): CachedCollabDocState { + return { + docState, + sourceHash: collabState.hashMarkdown(markdown), + stateHash: collabState.hashMarkdown(Buffer.from(docState)), + } +} + /** The exact bytes `persistFileDoc` would project from a doc seeded with `md`. */ function projectionOf(md: string): Buffer { const doc = markdownToYDoc(md) @@ -57,11 +67,97 @@ function stateOf(md: string): Uint8Array { } } +function editedState(state: Uint8Array, markdown: string): Uint8Array { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, state) + applyMarkdownToYDoc(doc, markdown) + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } +} + +function stateWithGeneration(markdown: string, generation: string): Uint8Array { + const doc = markdownToYDoc(markdown) + try { + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, generation) + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } +} + +/** Simulate only atomic storage I/O; every candidate, history merge and projection uses real Yjs. */ +function installAtomicStore(markdown: string, initialState: Uint8Array | null = null) { + const store = { + durable: projectionOf(markdown), + version: VERSION.getTime(), + cache: initialState ? cachedState(initialState, projectionOf(markdown)) : null, + accepted: [] as PreparedCollabDocState[], + } + const matches = (prepared: PreparedCollabDocState) => + prepared.expectedState === null + ? store.cache === null + : store.cache?.sourceHash === prepared.expectedState.sourceHash && + store.cache?.stateHash === prepared.expectedState.stateHash + const accept = (prepared: PreparedCollabDocState) => { + store.cache = { + docState: prepared.docState, + sourceHash: prepared.sourceHash, + stateHash: collabState.hashMarkdown(Buffer.from(prepared.docState)), + } + store.accepted.push(prepared) + } + mockGetWorkspaceFile.mockImplementation(async () => ({ + id: 'file-1', + name: 'note.md', + key: 'k', + size: store.durable.length, + contentUpdatedAt: new Date(store.version), + updatedAt: new Date(store.version), + })) + mockFetchBuffer.mockImplementation(async () => store.durable) + mockLoadState.mockImplementation(async () => store.cache) + mockCommitState.mockImplementation( + async (_workspaceId, _fileId, version: number, prepared: PreparedCollabDocState) => { + if (version !== store.version || !matches(prepared)) return { status: 'conflict' } + accept(prepared) + return { status: 'committed', version: store.version } + } + ) + mockUpdateContent.mockImplementation( + async ( + _workspaceId, + _fileId, + _userId, + bytes: Buffer, + _contentType, + options: { expectedUpdatedAt: Date; collabDocState: PreparedCollabDocState } + ) => { + if (options.expectedUpdatedAt.getTime() !== store.version) { + throw new ContentVersionConflictError('Content changed') + } + if (!matches(options.collabDocState)) { + throw new collabState.CollabDocStateConflictError('file-1') + } + accept(options.collabDocState) + store.durable = Buffer.from(bytes) + store.version++ + return { contentUpdatedAt: new Date(store.version), updatedAt: new Date(store.version) } + } + ) + return store +} + describe('persistFileDoc — no-op writes', () => { beforeEach(() => { vi.clearAllMocks() - mockSaveState.mockResolvedValue(undefined) - mockStateSourceHash.mockResolvedValue(null) + mockCommitState.mockImplementation(async (_workspaceId, _fileId, version: number) => ({ + status: 'committed', + version, + })) + mockLoadState.mockResolvedValue(null) }) function stubFile(durable: Buffer) { @@ -108,17 +204,17 @@ describe('persistFileDoc — no-op writes', () => { expect(result).toEqual({ status: 'persisted', version: VERSION.getTime() }) }) - it('still refreshes the cached snapshot on a no-op, so a cold open seeds from the canonical binary', async () => { + it('fences the cached snapshot on a no-op so a cold open resumes accepted binary history', async () => { const md = 'a\n\nb' stubFile(projectionOf(md)) await persistFileDoc('ws-1', 'file-1', 'user-1', stateOf(md), VERSION.getTime()) - expect(mockSaveState).toHaveBeenCalledWith( - 'file-1', - expect.anything(), - `hash:${projectionOf(md).toString('utf-8')}` - ) + expect(mockCommitState).toHaveBeenCalledWith('ws-1', 'file-1', VERSION.getTime(), { + docState: expect.any(Uint8Array), + sourceHash: collabState.hashMarkdown(projectionOf(md)), + expectedState: null, + }) }) it('writes when the content actually changed', async () => { @@ -138,6 +234,15 @@ describe('persistFileDoc — no-op writes', () => { expect(mockUpdateContent).toHaveBeenCalledTimes(1) expect(result).toEqual({ status: 'persisted', version: VERSION.getTime() + 1000 }) + expect(mockUpdateContent.mock.calls[0][5]).toMatchObject({ + syncLiveDoc: false, + expectedUpdatedAt: VERSION, + collabDocState: { + sourceHash: collabState.hashMarkdown(projectionOf('a\n\nb\n\nc')), + expectedState: null, + }, + }) + expect(mockCommitState).not.toHaveBeenCalled() }) it('skips the compare read entirely when the byte count already differs', async () => { @@ -159,7 +264,7 @@ describe('persistFileDoc — no-op writes', () => { expect(mockUpdateContent).toHaveBeenCalledTimes(1) }) - it('falls through to the write when the durable bytes cannot be read', async () => { + it('fails closed when the durable bytes cannot be read', async () => { const md = 'a\n\nb' const durable = projectionOf(md) mockGetWorkspaceFile.mockResolvedValue({ @@ -176,9 +281,12 @@ describe('persistFileDoc — no-op writes', () => { updatedAt: new Date(VERSION.getTime() + 1000), }) - await persistFileDoc('ws-1', 'file-1', 'user-1', stateOf(md), VERSION.getTime()) + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', stateOf(md), VERSION.getTime()) + ).rejects.toThrow('storage unavailable') - expect(mockUpdateContent).toHaveBeenCalledTimes(1) + expect(mockUpdateContent).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() }) }) @@ -194,7 +302,11 @@ describe('persistFileDoc — a stale token is not an out-of-band write', () => { beforeEach(() => { vi.clearAllMocks() - mockSaveState.mockResolvedValue(undefined) + mockCommitState.mockImplementation(async (_workspaceId, _fileId, version: number) => ({ + status: 'committed', + version, + })) + mockLoadState.mockResolvedValue(null) }) /** The file is at `NEWER` (durable = `durableMd`), while the caller still believes `VERSION`. */ @@ -204,7 +316,6 @@ describe('persistFileDoc — a stale token is not an out-of-band write', () => { id: 'file-1', name: 'note.md', key: 'k', - // A different size than the projection under test, so the no-op compare never short-circuits. size: durable.length + 999, updatedAt: NEWER, contentUpdatedAt: NEWER, @@ -217,46 +328,48 @@ describe('persistFileDoc — a stale token is not an out-of-band write', () => { } return { contentUpdatedAt: new Date(NEWER.getTime() + 1), updatedAt: NEWER } }) - return durable + return { durable, docState: stateOf(durableMd) } } it('writes anyway when the file still holds the bytes this document last projected', async () => { - const durable = stubConflict('a\n\nb') - // The cached doc state was tagged with exactly these bytes: nobody else has written since. - mockStateSourceHash.mockResolvedValue(`hash:${durable.toString('utf-8')}`) + const { durable, docState } = stubConflict('a\n\nb') + mockLoadState.mockResolvedValue(cachedState(docState, durable)) const result = await persistFileDoc( 'ws-1', 'file-1', 'user-1', - stateOf('a\n\nb\n\nmoved'), + editedState(docState, 'a\n\nb\n\nmoved'), VERSION.getTime() ) expect(result).toEqual({ status: 'persisted', version: NEWER.getTime() + 1 }) - // Once with the stale token (rejected), once with the file's real version. - expect(mockUpdateContent).toHaveBeenCalledTimes(2) + expect(mockUpdateContent).toHaveBeenCalledTimes(1) }) it('still refuses when the file holds someone else’s content', async () => { - stubConflict('a\n\nb') - mockStateSourceHash.mockResolvedValue('hash:something this document never wrote') + const { docState } = stubConflict('a\n\nb') + mockLoadState.mockResolvedValue({ + docState, + sourceHash: 'hash:something this document never wrote', + stateHash: collabState.hashMarkdown(Buffer.from(docState)), + }) const result = await persistFileDoc( 'ws-1', 'file-1', 'user-1', - stateOf('a\n\nb\n\nmoved'), + editedState(docState, 'a\n\nb\n\nmoved'), VERSION.getTime() ) expect(result).toEqual({ status: 'conflict' }) - expect(mockUpdateContent).toHaveBeenCalledTimes(1) + expect(mockUpdateContent).not.toHaveBeenCalled() }) it('refuses when nothing was ever cached, so there is no proof of authorship', async () => { stubConflict('a\n\nb') - mockStateSourceHash.mockResolvedValue(null) + mockLoadState.mockResolvedValue(null) const result = await persistFileDoc( 'ws-1', @@ -268,4 +381,619 @@ describe('persistFileDoc — a stale token is not an out-of-band write', () => { expect(result).toEqual({ status: 'conflict' }) }) + + it.each([ + ['insertion', 'alpha\n\nbeta\n\npeer edit'], + ['deletion', 'alpha'], + ['formatting', '**alpha**\n\nbeta'], + ])('refuses a snapshot missing a persisted %s', async (_kind, persistedMarkdown) => { + const original = stateOf('alpha\n\nbeta') + const persisted = editedState(original, persistedMarkdown) + const candidate = editedState(original, 'alpha\n\nbeta\n\nlocal edit') + if (_kind === 'deletion') { + expect(Y.encodeStateVectorFromUpdate(persisted)).toEqual( + Y.encodeStateVectorFromUpdate(original) + ) + } + const { durable } = stubConflict(persistedMarkdown) + mockLoadState.mockResolvedValue(cachedState(persisted, durable)) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'conflict' }) + expect(mockUpdateContent).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() + }) + + it.each(['alpha', '**alpha**\n\nbeta'])( + 'recovers after already integrating the persisted changes: %s', + async (persistedMarkdown) => { + const original = stateOf('alpha\n\nbeta') + const persisted = editedState(original, persistedMarkdown) + const candidate = editedState(persisted, `${persistedMarkdown}\n\nlocal edit`) + const { durable } = stubConflict(persistedMarkdown) + mockLoadState.mockResolvedValue(cachedState(persisted, durable)) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'persisted', version: NEWER.getTime() + 1 }) + expect(mockUpdateContent).toHaveBeenCalledTimes(1) + } + ) + + it('uses cached metadata-only history for stale content proof without adding it to the relay snapshot', async () => { + const original = stateOf('## Heading') + const cached = new Y.Doc() + Y.applyUpdate(cached, original) + cached.getMap(FILE_DOC_SEED.configMap).set('metadata', 'accepted peer metadata') + const docState = Y.encodeStateAsUpdate(cached) + cached.destroy() + const { durable } = stubConflict('## Heading') + mockLoadState.mockResolvedValue(cachedState(docState, durable)) + const candidate = editedState(original, '## Heading\n\nlocal edit') + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'persisted', version: NEWER.getTime() + 1 }) + const accepted = new Y.Doc() + try { + const prepared = mockUpdateContent.mock.calls[0][5].collabDocState + expect(prepared.docState).toBe(candidate) + Y.applyUpdate(accepted, prepared.docState) + expect(accepted.getMap(FILE_DOC_SEED.configMap).has('metadata')).toBe(false) + expect(yDocToFileMarkdown(accepted)).toBe(projectionOf('## Heading\n\nlocal edit').toString()) + } finally { + accepted.destroy() + } + }) + + it('never recovers an old generation over a new empty document', async () => { + const original = markdownToYDoc('old content') + const replacement = markdownToYDoc('') + original.getMap('config').set('docId', 'old-generation') + replacement.getMap('config').set('docId', 'new-generation') + const candidate = Y.encodeStateAsUpdate(original) + const persisted = Y.encodeStateAsUpdate(replacement) + original.destroy() + replacement.destroy() + const { durable } = stubConflict('') + mockLoadState.mockResolvedValue(cachedState(persisted, durable)) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'conflict' }) + expect(mockUpdateContent).not.toHaveBeenCalled() + }) + + it('fails closed when the cached binary is invalid', async () => { + const { durable } = stubConflict('base') + mockLoadState.mockResolvedValue(cachedState(new Uint8Array([255]), durable)) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', stateOf('local edit'), VERSION.getTime()) + ).rejects.toThrow() + expect(mockUpdateContent).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() + }) + + it('an older in-flight save cannot overwrite a newer completed save', async () => { + const olderState = stateOf('base\n\nolder local edit') + const newerState = editedState(olderState, 'base\n\nolder local edit\n\npeer edit') + let durable = projectionOf('base') + let version = VERSION.getTime() + let cache: CachedCollabDocState | null = null + const firstWrite = Promise.withResolvers() + const started = Promise.withResolvers() + + mockGetWorkspaceFile.mockImplementation(async () => ({ + id: 'file-1', + name: 'note.md', + key: 'key', + size: durable.length, + contentUpdatedAt: new Date(version), + updatedAt: new Date(version), + })) + mockFetchBuffer.mockImplementation(async () => durable) + mockLoadState.mockImplementation(async () => cache) + let attempts = 0 + mockUpdateContent.mockImplementation( + async ( + _workspaceId: string, + _fileId: string, + _userId: string, + bytes: Buffer, + _contentType: unknown, + options: { expectedUpdatedAt: Date; collabDocState: PreparedCollabDocState } + ) => { + if (attempts++ === 0) { + started.resolve() + await firstWrite.promise + } + if (options.expectedUpdatedAt.getTime() !== version) { + throw new ContentVersionConflictError('A newer snapshot committed') + } + durable = Buffer.from(bytes) + cache = cachedState(options.collabDocState.docState, bytes) + version++ + return { contentUpdatedAt: new Date(version), updatedAt: new Date(version) } + } + ) + + const olderSave = persistFileDoc('ws-1', 'file-1', 'user-1', olderState, VERSION.getTime()) + try { + await started.promise + const newerResult = await persistFileDoc( + 'ws-1', + 'file-1', + 'user-1', + newerState, + VERSION.getTime() + ) + expect(newerResult.status).toBe('persisted') + firstWrite.resolve() + await expect(olderSave).resolves.toEqual({ status: 'conflict' }) + expect(durable.toString()).toContain('peer edit') + expect(mockUpdateContent).toHaveBeenCalledTimes(2) + expect(cache?.sourceHash).toBe(collabState.hashMarkdown(durable)) + expect(cache?.docState).toEqual(mockUpdateContent.mock.calls[1][5].collabDocState.docState) + expect(mockCommitState).not.toHaveBeenCalled() + expect(mockLoadState).toHaveBeenCalledTimes(3) + } finally { + firstWrite.resolve() + await olderSave + } + }) +}) + +describe('persistFileDoc — atomic cache and native snapshot ownership', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCommitState.mockReset() + mockLoadState.mockReset() + mockGetWorkspaceFile.mockReset() + mockFetchBuffer.mockReset() + mockUpdateContent.mockReset() + }) + + it.each(['base', 'base\n\nlocal edit'])( + 'rejects an old generation before any write even with a current content token: %s', + async (candidateMarkdown) => { + const newer = stateWithGeneration('base', 'new-generation') + const store = installAtomicStore('base', newer) + const prior = store.cache + + await expect( + persistFileDoc( + 'ws-1', + 'file-1', + 'user-1', + stateWithGeneration(candidateMarkdown, 'old-generation'), + VERSION.getTime() + ) + ).resolves.toEqual({ status: 'conflict' }) + + expect(mockCommitState).not.toHaveBeenCalled() + expect(mockUpdateContent).not.toHaveBeenCalled() + expect(store.cache).toBe(prior) + expect(store.accepted).toHaveLength(0) + } + ) + + it.each(['content write', 'no-op'] as const)( + 'reloads the exact cache token without changing the relay snapshot after a same-version %s conflict', + async (kind) => { + const base = stateWithGeneration('base', 'shared-generation') + const store = installAtomicStore('base', base) + const originalToken = store.cache?.stateHash + const candidate = kind === 'no-op' ? base : editedState(base, 'base\n\nlocal edit') + const other = new Y.Doc() + Y.applyUpdate(other, base) + other.getMap(FILE_DOC_SEED.configMap).set('metadata', 'other cache writer') + const winner = cachedState(Y.encodeStateAsUpdate(other), store.durable) + other.destroy() + if (kind === 'no-op') { + mockCommitState.mockImplementationOnce(async () => { + store.cache = winner + return { status: 'conflict' } + }) + } else { + mockUpdateContent.mockImplementationOnce(async () => { + store.cache = winner + throw new collabState.CollabDocStateConflictError('file-1') + }) + } + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toMatchObject({ status: 'persisted' }) + + expect(mockLoadState).toHaveBeenCalledTimes(2) + expect(mockGetWorkspaceFile).toHaveBeenCalledTimes(2) + const calls = kind === 'no-op' ? mockCommitState.mock.calls : mockUpdateContent.mock.calls + const prepared = (call: unknown[]) => + (kind === 'no-op' + ? call[3] + : (call[5] as { collabDocState: PreparedCollabDocState }) + .collabDocState) as PreparedCollabDocState + expect(calls).toHaveLength(2) + expect(prepared(calls[0]).expectedState?.stateHash).toBe(originalToken) + expect(prepared(calls[1]).expectedState).toEqual({ + sourceHash: winner.sourceHash, + stateHash: winner.stateHash, + }) + expect(store.accepted).toHaveLength(1) + expect(store.accepted[0].docState).toBe(candidate) + expect(store.cache?.docState).toBe(candidate) + const cold = new Y.Doc() + try { + Y.applyUpdate(cold, store.accepted[0].docState) + expect(Buffer.from(yDocToFileMarkdown(cold))).toEqual(store.durable) + } finally { + cold.destroy() + } + } + ) + + it('rejects a generation replacement found while retrying a no-op cache commit', async () => { + const original = stateWithGeneration('base', 'original-generation') + const store = installAtomicStore('base', original) + const replacement = cachedState( + stateWithGeneration('base', 'replacement-generation'), + store.durable + ) + mockCommitState.mockImplementationOnce(async () => { + store.cache = replacement + return { status: 'conflict' } + }) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', original, VERSION.getTime()) + ).resolves.toEqual({ status: 'conflict' }) + expect(mockCommitState).toHaveBeenCalledTimes(1) + expect(mockUpdateContent).not.toHaveBeenCalled() + expect(store.cache).toBe(replacement) + expect(store.accepted).toHaveLength(0) + }) + + it('retries a content version race only after integrating the winner’s accepted history', async () => { + const base = stateWithGeneration('base', 'shared-generation') + const winner = editedState(base, 'base\n\npeer edit') + const candidate = editedState(winner, 'base\n\npeer edit\n\nlocal edit') + const store = installAtomicStore('base', base) + mockUpdateContent.mockImplementationOnce(async () => { + store.durable = projectionOf('base\n\npeer edit') + store.cache = cachedState(winner, store.durable) + store.version++ + throw new ContentVersionConflictError('A peer committed first') + }) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'persisted', version: VERSION.getTime() + 2 }) + expect(mockUpdateContent).toHaveBeenCalledTimes(2) + expect(mockUpdateContent.mock.calls[1][5].expectedUpdatedAt.getTime()).toBe( + VERSION.getTime() + 1 + ) + expect(store.durable).toEqual(projectionOf('base\n\npeer edit\n\nlocal edit')) + expect(store.accepted).toHaveLength(1) + }) + + it.each(['content write', 'no-op'] as const)( + 'bounds repeated %s conflicts to two attempts', + async (kind) => { + const base = stateOf('base') + const store = installAtomicStore('base', base) + const candidate = kind === 'no-op' ? base : editedState(base, 'base\n\nlocal edit') + if (kind === 'no-op') { + mockCommitState.mockResolvedValue({ status: 'conflict' }) + } else { + mockUpdateContent.mockRejectedValue(new collabState.CollabDocStateConflictError('file-1')) + } + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'conflict' }) + expect(mockGetWorkspaceFile).toHaveBeenCalledTimes(2) + expect(mockLoadState).toHaveBeenCalledTimes(2) + expect(kind === 'no-op' ? mockCommitState : mockUpdateContent).toHaveBeenCalledTimes(2) + expect(store.accepted).toHaveLength(0) + expect(store.durable).toEqual(projectionOf('base')) + } + ) + + it.each(['cache read', 'content write', 'cache-only commit'] as const)( + 'propagates a %s failure without claiming persistence or retrying it as a conflict', + async (kind) => { + const base = stateOf('base') + const store = installAtomicStore('base', base) + const candidate = + kind === 'cache-only commit' ? base : editedState(base, 'base\n\nlocal edit') + const operation = + kind === 'cache read' + ? mockLoadState + : kind === 'content write' + ? mockUpdateContent + : mockCommitState + operation.mockRejectedValue(new Error('storage unavailable')) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).rejects.toThrow('storage unavailable') + expect(operation).toHaveBeenCalledOnce() + expect(store.accepted).toHaveLength(0) + expect(store.durable).toEqual(projectionOf('base')) + if (kind === 'cache read') { + expect(mockUpdateContent).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() + } + } + ) + + it('reports a deleted file observed after a content conflict without touching its cache', async () => { + const base = stateOf('base') + const store = installAtomicStore('base', base) + mockUpdateContent.mockImplementationOnce(async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + throw new ContentVersionConflictError('File changed') + }) + + await expect( + persistFileDoc( + 'ws-1', + 'file-1', + 'user-1', + editedState(base, 'base\n\nlocal edit'), + VERSION.getTime() + ) + ).resolves.toEqual({ status: 'missing' }) + expect(mockLoadState).toHaveBeenCalledOnce() + expect(store.accepted).toHaveLength(0) + }) + + it('reports a file deleted during a no-op commit instead of acknowledging it', async () => { + const base = stateOf('base') + installAtomicStore('base', base) + mockCommitState.mockResolvedValue({ status: 'missing' }) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', base, VERSION.getTime()) + ).resolves.toEqual({ status: 'missing' }) + expect(mockCommitState).toHaveBeenCalledOnce() + expect(mockUpdateContent).not.toHaveBeenCalled() + }) + + it('does not decode, read cache, or write when the relay has no expected version', async () => { + installAtomicStore('base') + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', new Uint8Array([255])) + ).resolves.toEqual({ status: 'deferred' }) + expect(mockLoadState).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() + expect(mockUpdateContent).not.toHaveBeenCalled() + }) + + it('returns missing before processing a snapshot for a nonexistent file', async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', new Uint8Array([255]), VERSION.getTime()) + ).resolves.toEqual({ status: 'missing' }) + expect(mockLoadState).not.toHaveBeenCalled() + }) + + it('rejects an oversized incoming snapshot before decoding or loading its cache', async () => { + installAtomicStore('base') + await expect( + persistFileDoc( + 'ws-1', + 'file-1', + 'user-1', + new Uint8Array(collabState.MAX_COLLAB_DOC_STATE_BYTES + 1), + VERSION.getTime() + ) + ).rejects.toThrow('12 MiB limit') + expect(mockLoadState).not.toHaveBeenCalled() + expect(mockUpdateContent).not.toHaveBeenCalled() + }) + + it('repeatedly persists the same legacy snapshot without accumulating server-created structures', async () => { + const base = markdownToYDoc('# Heading') + const cold = new Y.Doc() + try { + base.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'shared-generation') + const fragment = base.getXmlFragment(COLLAB_DOC_FIELD) + if (fragment.length > 1) fragment.delete(1, fragment.length - 1) + const original = Y.encodeStateAsUpdate(base) + const store = installAtomicStore('# Heading', original) + store.durable = Buffer.from(yDocToFileMarkdown(base)) + store.cache = cachedState(original, store.durable) + + for (let attempt = 0; attempt < 12; attempt++) { + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', original, store.version) + ).resolves.toEqual({ status: 'persisted', version: store.version }) + } + + expect(store.accepted).toHaveLength(12) + expect( + new Set( + store.accepted.map(({ docState }) => collabState.hashMarkdown(Buffer.from(docState))) + ).size + ).toBe(1) + Y.applyUpdate(cold, store.accepted[11].docState) + expect(cold.getXmlFragment(COLLAB_DOC_FIELD).length).toBe(fragment.length) + expect(Y.encodeStateVector(cold)).toEqual(Y.encodeStateVector(base)) + expect(mockUpdateContent).not.toHaveBeenCalled() + } finally { + base.destroy() + cold.destroy() + } + }) + + it('keeps late typing into an existing empty paragraph persistable after saving its snapshot', async () => { + const peer = markdownToYDoc('base') + const cold = new Y.Doc() + try { + peer.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'shared-generation') + const tail = new Y.XmlElement('paragraph') + const text = new Y.XmlText() + tail.insert(0, [text]) + peer.getXmlFragment(COLLAB_DOC_FIELD).push([tail]) + const original = Y.encodeStateAsUpdate(peer) + const store = installAtomicStore('base', original) + store.durable = Buffer.from(yDocToFileMarkdown(peer)) + store.cache = cachedState(original, store.durable) + + const first = await persistFileDoc('ws-1', 'file-1', 'user-1', original, store.version) + expect(first.status).toBe('persisted') + if (first.status !== 'persisted') throw new Error('Initial snapshot was not accepted') + text.insert(0, 'late user text') + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', Y.encodeStateAsUpdate(peer), first.version) + ).resolves.toEqual({ status: 'persisted', version: first.version + 1 }) + expect(store.durable.toString()).toBe('base\n\nlate user text') + expect(store.accepted).toHaveLength(2) + Y.applyUpdate(cold, store.accepted[1].docState) + expect(yDocToFileMarkdown(cold)).toBe(store.durable.toString()) + } finally { + peer.destroy() + cold.destroy() + } + }) + + function legacyCachePair(tailCount = 1) { + const peer = markdownToYDoc('base') + const detached = new Y.Doc() + peer.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'shared-generation') + let text = new Y.XmlText() + for (let index = 0; index < tailCount; index++) { + const paragraph = new Y.XmlElement('paragraph') + text = new Y.XmlText() + paragraph.insert(0, [text]) + peer.getXmlFragment(COLLAB_DOC_FIELD).push([paragraph]) + } + try { + Y.applyUpdate(detached, Y.encodeStateAsUpdate(peer)) + detached.getXmlFragment(COLLAB_DOC_FIELD).delete(tailCount, 1) + return { peer, text, legacyState: Y.encodeStateAsUpdate(detached) } + } finally { + detached.destroy() + } + } + + it('saves exact-version peer typing despite an old detached cache deletion of its paragraph', async () => { + const { peer, text, legacyState } = legacyCachePair() + try { + const store = installAtomicStore('base', legacyState) + text.insert(0, 'late user text') + const candidate = Y.encodeStateAsUpdate(peer) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime()) + ).resolves.toEqual({ status: 'persisted', version: VERSION.getTime() + 1 }) + + expect(store.durable.toString()).toBe('base\n\nlate user text') + expect(store.cache?.docState).toBe(candidate) + expect(store.accepted).toHaveLength(1) + expect(mockUpdateContent).toHaveBeenCalledOnce() + } finally { + peer.destroy() + } + }) + + it.each([VERSION.getTime(), VERSION.getTime() - 1])( + 'keeps the native typing target on a no-op with token %i despite private cache deletions', + async (expectedVersion) => { + const { peer, text, legacyState } = legacyCachePair() + try { + const store = installAtomicStore('base', legacyState) + store.durable = Buffer.from(yDocToFileMarkdown(peer)) + store.cache = cachedState(legacyState, store.durable) + const initial = Y.encodeStateAsUpdate(peer) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', initial, expectedVersion) + ).resolves.toEqual({ status: 'persisted', version: VERSION.getTime() }) + expect(store.cache?.docState).toBe(initial) + expect(mockUpdateContent).not.toHaveBeenCalled() + + text.insert(0, 'next peer text') + const next = Y.encodeStateAsUpdate(peer) + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', next, VERSION.getTime()) + ).resolves.toEqual({ status: 'persisted', version: VERSION.getTime() + 1 }) + expect(store.durable.toString()).toBe('base\n\nnext peer text') + expect(store.cache?.docState).toBe(next) + } finally { + peer.destroy() + } + } + ) + + it('does not retain a stale proof’s invisible private deletion that would erase later peer typing', async () => { + const { peer, text, legacyState } = legacyCachePair(2) + const cold = new Y.Doc() + const proof = new Y.Doc() + try { + const store = installAtomicStore('base', legacyState) + const legacy = new Y.Doc() + try { + Y.applyUpdate(legacy, legacyState) + store.durable = Buffer.from(yDocToFileMarkdown(legacy)) + store.cache = cachedState(legacyState, store.durable) + } finally { + legacy.destroy() + } + const paragraph = new Y.XmlElement('paragraph') + const localText = new Y.XmlText() + localText.insert(0, 'local edit') + paragraph.insert(0, [localText]) + peer.getXmlFragment(COLLAB_DOC_FIELD).insert(1, [paragraph]) + const candidate = Y.encodeStateAsUpdate(peer) + Y.applyUpdate(proof, candidate) + Y.applyUpdate(proof, legacyState) + expect(yDocToFileMarkdown(proof)).toBe(yDocToFileMarkdown(peer)) + + await expect( + persistFileDoc('ws-1', 'file-1', 'user-1', candidate, VERSION.getTime() - 1) + ).resolves.toEqual({ status: 'persisted', version: VERSION.getTime() + 1 }) + expect(store.cache?.docState).toBe(candidate) + + text.insert(0, 'late next text') + const delayed = Y.encodeStateAsUpdate(peer) + Y.applyUpdate(proof, delayed) + expect(yDocToFileMarkdown(proof)).not.toContain('late next text') + Y.applyUpdate(cold, store.accepted[0].docState) + Y.applyUpdate(cold, delayed) + expect(yDocToFileMarkdown(cold)).toContain('late next text') + } finally { + peer.destroy() + cold.destroy() + proof.destroy() + } + }) + + it('keeps stale content fail-closed when a private cached deletion changes the proof’s projection', async () => { + const { peer, text, legacyState } = legacyCachePair() + try { + const store = installAtomicStore('base', legacyState) + text.insert(0, 'late user text') + + await expect( + persistFileDoc( + 'ws-1', + 'file-1', + 'user-1', + Y.encodeStateAsUpdate(peer), + VERSION.getTime() - 1 + ) + ).resolves.toEqual({ status: 'conflict' }) + expect(store.durable.toString()).toBe('base') + expect(store.cache?.docState).toBe(legacyState) + expect(store.accepted).toHaveLength(0) + expect(mockUpdateContent).not.toHaveBeenCalled() + } finally { + peer.destroy() + } + }) }) diff --git a/apps/sim/lib/collab-doc/persist.ts b/apps/sim/lib/collab-doc/persist.ts index 6d8f387ce6d..0551d4bec2e 100644 --- a/apps/sim/lib/collab-doc/persist.ts +++ b/apps/sim/lib/collab-doc/persist.ts @@ -1,6 +1,16 @@ import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import * as Y from 'yjs' +import { + assertCollabDocStateSize, + type CachedCollabDocState, + CollabDocStateConflictError, + commitCollabDocState, + hashMarkdown, + loadCollabDocState, + type PreparedCollabDocState, +} from '@/lib/collab-doc/collab-state' +import { yDocToFileMarkdown } from '@/lib/collab-doc/converter' import { ContentVersionConflictError, fetchWorkspaceFileBuffer, @@ -8,21 +18,11 @@ import { updateWorkspaceFileContent, } from '@/lib/uploads/contexts/workspace' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { collabDocStateSourceHash, hashMarkdown, saveCollabDocState } from './collab-state' -import { canonicalizeYDoc, yDocToFileMarkdown } from './converter' const logger = createLogger('FileDocPersist') +const MAX_PERSIST_ATTEMPTS = 2 -/** - * Outcome of a persist attempt: - * - `persisted` — the live doc was projected to markdown and written; `version` is the new durable - * CONTENT version (`content_updated_at`, epoch ms) the relay records as what its live doc is synced to. - * - `missing` — the file is gone (deleted); nothing to write. - * - `conflict` — the file changed out-of-band since the relay's live doc last synced, so writing the - * projection would clobber that change (RFC 7232 `If-Match` failure). NOT written; the relay leaves the - * durable content authoritative and does not advance its synced version (a later flush reconciles once - * the chokepoint merge lands). No `version` is returned — the relay never reads one on this path. - */ +/** Only an accepted content/cache transaction advances the relay's durable content version. */ export type PersistFileDocResult = | { status: 'persisted'; version: number } | { status: 'missing' } @@ -30,20 +30,9 @@ export type PersistFileDocResult = | { status: 'deferred' } /** - * Project a live collaborative document back to durable markdown and write it to the file. The realtime - * relay owns the live Yjs doc but not the conversion engine or blob/DB access, so it ships the doc state - * here and the app persists it — the server-authoritative durable path that replaces the editor's - * client-side autosave. - * - * `expectedVersion` (the durable CONTENT version, `content_updated_at` epoch ms, the relay's live doc last - * synced from) is the optimistic-concurrency guard: the write commits only if the file is still at that - * content version — a rename/move that only bumps `updatedAt` won't trip it, so a - * projection built from a stale live doc can never silently overwrite an out-of-band edit. On a version - * mismatch this returns `conflict` (the current durable version) instead of writing — the relay adopts - * it as its new If-Match and retries against the current live stream. Omit `expectedVersion` to write - * unconditionally (e.g. the first persist, before any synced version exists). - * - * `userId` is attribution only (blob metadata); the caller is already trusted via the `x-api-key` gate. + * Persist the relay's snapshot and its Markdown projection together. Conversion and blob I/O stay + * outside the file-row transaction; content version and cached-state fences protect its commit. + * `userId` is attribution only: the internal surface authorizes the caller before this operation. */ export async function persistFileDoc( workspaceId: string, @@ -52,165 +41,122 @@ export async function persistFileDoc( docState: Uint8Array, expectedVersion?: number ): Promise { - const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) - if (!record) return { status: 'missing' } - - // Optimistic concurrency needs a version. If none was supplied — the relay's synced-version token was - // momentarily unavailable (a Redis blip on a peer-seeded task) — DEFER rather than write: an - // unconditional write could clobber an out-of-band edit, and a reconcile would wipe live edits even - // when nothing changed out-of-band (the version was merely missing). The edits stay in the stream; a - // later persist writes them once the version is re-established. There is deliberately NO empty-file - // unconditional-write carve-out: every existing file has a `content_updated_at`, so the relay always - // has a real version to send and a missing one is always transient — and `record.size` is read outside - // the write transaction, so trusting it (an empty file "has nothing to clobber") is a TOCTOU race a - // concurrent first content write would lose. - if (expectedVersion === undefined) { - return { status: 'deferred' } - } - - const ydoc = new Y.Doc() - let markdownBuffer: Buffer - // The snapshot cached below seeds a later cold room directly, so it must describe the same document - // the durable markdown does — otherwise a warm open renders structure the markdown cannot reproduce - // and the doc reflows once it settles. `canonicalizeYDoc` converges this DETACHED copy onto its own - // markdown projection, which is exactly that guarantee, and leaves the live room untouched. - let cachedDocState = docState + const initialRecord = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + if (!initialRecord) return { status: 'missing' } + if (expectedVersion === undefined) return { status: 'deferred' } + assertCollabDocStateSize(docState) + const candidate = new Y.Doc() + let markdown: Buffer try { - Y.applyUpdate(ydoc, docState) - if (canonicalizeYDoc(ydoc)) cachedDocState = Y.encodeStateAsUpdate(ydoc) - markdownBuffer = Buffer.from(yDocToFileMarkdown(ydoc), 'utf-8') + Y.applyUpdate(candidate, docState) + markdown = Buffer.from(yDocToFileMarkdown(candidate), 'utf-8') } finally { - ydoc.destroy() + candidate.destroy() } - // A persist that would write the bytes already on disk is skipped entirely. Binding an editor to a - // seeded document emits a Yjs update of its own — y-tiptap normalizes node attributes on bind — so - // simply OPENING a file schedules a save whose projection is byte-identical to the file. Writing it - // is not free: `updateWorkspaceFileContent` uploads under a FRESH storage key, repoints the row, and - // deletes the old object, so every reader still holding the previous key 404s. That is the stray - // not-found a page sees on open, racing its own first content read. - // - // Length is the free reject — a real edit almost never lands on the same byte count — so the compare - // read happens only when a no-op write is actually on the table. Unchanged content means there is - // nothing to clobber, so this reports the file's CURRENT durable version rather than conflicting on a - // stale `expectedVersion`: it resynchronizes the relay's If-Match token instead of stranding it. - if (record.size === markdownBuffer.length) { - // A byte-for-byte equality check: anything longer than what we are comparing against - // cannot match, so the buffer we are about to compare is itself the ceiling. - const current = await fetchWorkspaceFileBuffer(record, { - maxBytes: markdownBuffer.length, - }).catch(() => null) - if (current?.equals(markdownBuffer)) { - // Still refresh the cached snapshot: the markdown is unchanged (so its `sourceHash` tag stays - // valid) but the doc state may have just been canonicalized, and a cold open should seed from - // the repaired binary rather than the one that needed repairing. - try { - await saveCollabDocState(fileId, cachedDocState, hashMarkdown(markdownBuffer)) - } catch (error) { - logger.warn(`Failed to cache collab doc state for file ${fileId}`, { - error: getErrorMessage(error), - }) - } - return { - status: 'persisted', - version: (record.contentUpdatedAt ?? record.updatedAt).getTime(), - } + for (let attempt = 0; attempt < MAX_PERSIST_ATTEMPTS; attempt++) { + const record = + attempt === 0 + ? initialRecord + : await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + if (!record) return { status: 'missing' } + const version = (record.contentUpdatedAt ?? record.updatedAt).getTime() + const cached = await loadCollabDocState(fileId) + let durable: Buffer | null = null + if (record.size === markdown.length || version !== expectedVersion) { + durable = await fetchWorkspaceFileBuffer(record, { + maxBytes: version === expectedVersion ? markdown.length : MAX_BUFFERED_TRANSFER_BYTES, + }) } - } - - const write = async (ifMatch: number): Promise => { - const updated = await updateWorkspaceFileContent( - workspaceId, - fileId, - userId, - markdownBuffer, - undefined, - { - // This write IS the projection of the live doc, so re-merging it into that same doc would loop. - syncLiveDoc: false, - // If-Match: only if the durable file is still at the version the live doc synced from. - expectedUpdatedAt: new Date(ifMatch), - secretProvenancePolicy: { mode: 'preserve' }, - } + const prepared = preparePersistedState( + docState, + cached, + markdown, + version !== expectedVersion && !durable?.equals(markdown) ) + if (!prepared) return { status: 'conflict' } - // Cache the Yjs binary (tagged with the exact markdown just written) so a later cold room open loads - // it directly instead of re-converting. Best-effort — the markdown is the durable source of truth. try { - await saveCollabDocState(fileId, cachedDocState, hashMarkdown(markdownBuffer)) - } catch (error) { - logger.warn(`Failed to cache collab doc state for file ${fileId}`, { - error: getErrorMessage(error), - }) - } + if (durable?.equals(markdown)) { + const result = await commitCollabDocState(workspaceId, fileId, version, prepared) + if (result.status === 'committed') { + return { status: 'persisted', version: result.version } + } + if (result.status === 'missing') return result + continue + } - logger.info( - `Persisted live collaborative document to file ${fileId} (workspace ${workspaceId})` - ) - // Return the CONTENT version (what the CAS/seed/merge all guard on), not `updatedAt` — the relay - // records this as its new If-Match token, so it must be the same field a later persist is checked - // against. (A content write sets both to the same instant; using the wrong one only bites once they - // diverge — e.g. a metadata write bumping `updatedAt` afterward.) - return { - status: 'persisted', - version: (updated.contentUpdatedAt ?? updated.updatedAt).getTime(), + /** A stale relay token may advance only if the cached history accounts for the durable bytes. */ + if ( + version !== expectedVersion && + (!durable || cached?.sourceHash !== hashMarkdown(durable)) + ) { + return { status: 'conflict' } + } + + const updated = await updateWorkspaceFileContent( + workspaceId, + fileId, + userId, + markdown, + undefined, + { + syncLiveDoc: false, + expectedUpdatedAt: new Date(version), + secretProvenancePolicy: { mode: 'preserve' }, + collabDocState: prepared, + } + ) + logger.info(`Persisted collaborative document for file ${fileId}`) + return { + status: 'persisted', + version: (updated.contentUpdatedAt ?? updated.updatedAt).getTime(), + } + } catch (error) { + if ( + !(error instanceof ContentVersionConflictError) && + !(error instanceof CollabDocStateConflictError) + ) { + throw error + } } } - try { - return await write(expectedVersion) - } catch (error) { - if (!(error instanceof ContentVersionConflictError)) throw error - return recoverFromVersionConflict(workspaceId, fileId, markdownBuffer, write) - } + logger.warn(`Persist conflict for file ${fileId}; the file or cached history changed during save`) + return { status: 'conflict' } } /** - * A stale If-Match does not prove someone else wrote the file — so ask the CONTENT, not the clock. - * - * The relay's token is a remembered timestamp: it lives in the room (lost when the room is dropped) and - * in a cluster key written best-effort, so a process that dies in the moments after a successful write - * comes back holding a version older than the file's. Every later persist then fails the CAS, and - * because a conflict deliberately neither writes nor advances the token, the room can never persist - * again: the session's edits stay in the stream, the durable markdown freezes at the last write, and - * every reload renders that stale markdown before the live document corrects it on screen. - * - * The guard exists to protect content the live document has never seen. The file's own bytes settle - * that directly: if they hash to what this document last projected ({@link collabDocStateSourceHash}, - * written with every successful persist), then nothing out-of-band exists and the write is safe — retry - * it once against the file's current version. If they hash to anything else, the change is real, the - * conflict stands, and the durable content stays authoritative exactly as before. + * The relay owns the full snapshot. Legacy caches can contain detached normalization deletions + * never sent to that relay, so they cannot be merged into saved state. For stale content writes, + * use a throwaway merge only to prove the candidate does not omit durable content. */ -async function recoverFromVersionConflict( - workspaceId: string, - fileId: string, - markdownBuffer: Buffer, - write: (ifMatch: number) => Promise -): Promise { - const conflict = (): PersistFileDocResult => { - logger.warn( - `Persist conflict for file ${fileId}; durable content changed out-of-band since sync` - ) - return { status: 'conflict' } - } +function preparePersistedState( + docState: Uint8Array, + cached: CachedCollabDocState | null, + markdown: Buffer, + proveContentIncluded: boolean +): PreparedCollabDocState | null { + if (!cached) return { docState, sourceHash: hashMarkdown(markdown), expectedState: null } + const candidate = new Y.Doc() + const persisted = new Y.Doc() try { - const current = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) - if (!current) return { status: 'missing' } - const durable = await fetchWorkspaceFileBuffer(current, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - if (hashMarkdown(durable) !== (await collabDocStateSourceHash(fileId))) return conflict() - logger.info( - `Persist token for file ${fileId} was stale, not the file; re-syncing and writing the projection` - ) - return await write((current.contentUpdatedAt ?? current.updatedAt).getTime()) - } catch (error) { - // Including a SECOND conflict: something wrote the file during the recovery, which is the very - // change the guard exists for. - if (error instanceof ContentVersionConflictError) return conflict() - logger.warn(`Persist conflict recovery failed for file ${fileId}`, { - error: getErrorMessage(error), - }) - return conflict() + Y.applyUpdate(candidate, docState) + Y.applyUpdate(persisted, cached.docState) + const generation = (doc: Y.Doc) => + doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (generation(candidate) !== generation(persisted)) return null + if (proveContentIncluded) { + Y.applyUpdate(candidate, cached.docState) + if (!Buffer.from(yDocToFileMarkdown(candidate), 'utf-8').equals(markdown)) return null + } + return { + docState, + sourceHash: hashMarkdown(markdown), + expectedState: { stateHash: cached.stateHash, sourceHash: cached.sourceHash }, + } + } finally { + candidate.destroy() + persisted.destroy() } } diff --git a/apps/sim/lib/collab-doc/seed.test.ts b/apps/sim/lib/collab-doc/seed.test.ts index 1e0449204b1..0a7839fce04 100644 --- a/apps/sim/lib/collab-doc/seed.test.ts +++ b/apps/sim/lib/collab-doc/seed.test.ts @@ -1,40 +1,48 @@ /** * @vitest-environment node */ -import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getSchema } from '@tiptap/core' import { prosemirrorJSONToYDoc } from '@tiptap/y-tiptap' import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' - -const { mockGetWorkspaceFile, mockFetchBuffer, mockLoadState, mockSaveState } = vi.hoisted(() => ({ - mockGetWorkspaceFile: vi.fn(), - mockFetchBuffer: vi.fn(), - mockLoadState: vi.fn(), - mockSaveState: vi.fn(), -})) +import * as collabState from '@/lib/collab-doc/collab-state' + +const { mockGetWorkspaceFile, mockFetchBuffer, mockLoadState, mockCommitState } = vi.hoisted( + () => ({ + mockGetWorkspaceFile: vi.fn(), + mockFetchBuffer: vi.fn(), + mockLoadState: vi.fn(), + mockCommitState: vi.fn(), + }) +) vi.mock('@/lib/uploads/contexts/workspace', () => ({ getWorkspaceFile: mockGetWorkspaceFile, fetchWorkspaceFileBuffer: mockFetchBuffer, })) -// The DB-backed cold-start cache is exercised in its own suite; here we default it to a MISS so these -// tests cover the markdown → Yjs conversion path (the cache-hit fast path has its own test below). -vi.mock('./collab-state', () => ({ - hashMarkdown: () => 'test-source-hash', - loadCollabDocState: mockLoadState, - saveCollabDocState: mockSaveState, -})) +vi.spyOn(collabState, 'loadCollabDocState').mockImplementation(mockLoadState) +vi.spyOn(collabState, 'commitCollabDocState').mockImplementation(mockCommitState) +import { markdownToYDoc, yDocToFileMarkdown, yDocToMarkdown } from '@/lib/collab-doc/converter' +import { COLLAB_DOC_FIELD } from '@/lib/collab-doc/field' +import { buildFileDocSeed } from '@/lib/collab-doc/seed' import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' import { parseMarkdownToDoc, serializeMarkdownBody, } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' -import { markdownToYDoc, yDocToMarkdown } from './converter' -import { COLLAB_DOC_FIELD } from './field' -import { buildFileDocSeed } from './seed' + +const VERSION = new Date('2026-01-01T00:00:00.000Z').getTime() + +function cachedState(docState: Uint8Array, markdown: string): collabState.CachedCollabDocState { + return { + docState, + sourceHash: collabState.hashMarkdown(Buffer.from(markdown)), + stateHash: collabState.hashMarkdown(Buffer.from(docState)), + } +} describe('buildFileDocSeed', () => { beforeEach(() => { @@ -46,9 +54,11 @@ describe('buildFileDocSeed', () => { context: 'workspace', updatedAt: new Date('2026-01-01T00:00:00.000Z'), }) - // Default: nothing stored → the conversion path runs (the case these tests cover). mockLoadState.mockResolvedValue(null) - mockSaveState.mockResolvedValue(undefined) + mockCommitState.mockImplementation(async (_workspaceId, _fileId, version: number) => ({ + status: 'committed', + version, + })) }) it('builds a seed whose applied update reproduces the file body (through the client engine)', async () => { @@ -60,21 +70,16 @@ describe('buildFileDocSeed', () => { const doc = new Y.Doc() Y.applyUpdate(doc, seed!.update) expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\nHello **world**.')) + doc.destroy() }) it('cold-start fast path: returns the cached binary directly without re-converting when it is fresh', async () => { - // A cached binary derived from the current markdown → seed returns it verbatim (no conversion), the - // Hocuspocus load-document path that preserves the CRDT's client ids across reopens. Built through - // `markdownToYDoc` so it is in the canonical form persist caches; a hand-rolled doc would be - // repaired on the way through and this would assert the fast path while never taking it. const cachedDoc = markdownToYDoc('# Anything') cachedDoc.getText('marker').insert(0, 'cached') - // Named, as anything the current seed stored would be — an unnamed document is rewritten once to - // give it an identity, which has its own test below. cachedDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-already-named') const cached = Y.encodeStateAsUpdate(cachedDoc) mockFetchBuffer.mockResolvedValue(Buffer.from('# Anything', 'utf-8')) - mockLoadState.mockResolvedValue({ docState: cached, sourceHash: 'test-source-hash' }) + mockLoadState.mockResolvedValue(cachedState(cached, '# Anything')) const seed = await buildFileDocSeed('ws-1', 'file-1') @@ -83,52 +88,39 @@ describe('buildFileDocSeed', () => { Y.applyUpdate(doc, seed!.update) expect(doc.getText('marker').toString()).toBe('cached') cachedDoc.destroy() + doc.destroy() }) - /** - * The freshness tag is a hash of the markdown alone, so a snapshot written under older parse rules - * still reads as fresh and would otherwise be replayed verbatim forever. Repairing it here is the only - * path that ever fixes one — and the repair has to be reported, or the caller hands back the bytes it - * just decided were wrong (which is how the cached path kept reseeding docs that were missing the - * editor's trailing paragraph, letting every binding client stack another). - */ - it('repairs a cached snapshot that is not in the editor normal form', async () => { + it('preserves a named legacy snapshot without introducing an unbroadcast structural repair', async () => { const stale = prosemirrorJSONToYDoc( getSchema(createMarkdownContentExtensions()), - // A raw parse — no trailing paragraph, which is exactly what a pre-normalization snapshot holds. parseMarkdownToDoc('# T\n\nbody\n\n- a\n- b'), COLLAB_DOC_FIELD ) + stale.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'legacy-generation') const cached = Y.encodeStateAsUpdate(stale) mockFetchBuffer.mockResolvedValue(Buffer.from('# T\n\nbody\n\n- a\n- b', 'utf-8')) - mockLoadState.mockResolvedValue({ docState: cached, sourceHash: 'test-source-hash' }) + mockLoadState.mockResolvedValue(cachedState(cached, '# T\n\nbody\n\n- a\n- b')) const seed = await buildFileDocSeed('ws-1', 'file-1') - expect(seed?.update).not.toBe(cached) + expect(seed?.update).toBe(cached) const doc = new Y.Doc() Y.applyUpdate(doc, seed!.update) - const fragment = doc.getXmlFragment(COLLAB_DOC_FIELD) - const last = fragment.get(fragment.length - 1) - expect(last instanceof Y.XmlElement && last.nodeName === 'paragraph' && last.length === 0).toBe( - true + expect(doc.getXmlFragment(COLLAB_DOC_FIELD).toJSON()).toEqual( + stale.getXmlFragment(COLLAB_DOC_FIELD).toJSON() ) + expect(Y.encodeStateVector(doc)).toEqual(Y.encodeStateVector(stale)) stale.destroy() doc.destroy() }) - it('falls through to conversion when the cache read fails (never blocks a cold open)', async () => { - // The cache is a best-effort optimization over the durable markdown we already hold; a transient DB - // error or a not-yet-migrated cache table must convert, not abort the seed. + it('fails closed when the cache read fails instead of creating a conflicting document identity', async () => { mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\ntext.', 'utf-8')) mockLoadState.mockRejectedValue(new Error('cache table missing')) - const seed = await buildFileDocSeed('ws-1', 'file-1') - expect(seed).not.toBeNull() - - const doc = new Y.Doc() - Y.applyUpdate(doc, seed!.update) - expect(yDocToMarkdown(doc)).toBe(serializeMarkdownBody('# Title\n\ntext.')) + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow('cache table missing') + expect(mockCommitState).not.toHaveBeenCalled() }) it('strips frontmatter — only the body seeds the collaborative doc', async () => { @@ -140,6 +132,7 @@ describe('buildFileDocSeed', () => { const md = yDocToMarkdown(doc) expect(md).not.toContain('title: X') expect(md).toBe(serializeMarkdownBody('# Body\n\ntext.')) + doc.destroy() }) it('marks the seeded doc as initial-content-loaded so the client needs no seeder handshake', async () => { @@ -148,6 +141,7 @@ describe('buildFileDocSeed', () => { const doc = new Y.Doc() Y.applyUpdate(doc, seed!.update) expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + doc.destroy() }) it('carries the frontmatter in the config map (not the body)', async () => { @@ -158,8 +152,8 @@ describe('buildFileDocSeed', () => { expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.frontmatterKey)).toContain( 'title: X' ) - // …and the frontmatter is NOT in the collaborative body. expect(yDocToMarkdown(doc)).not.toContain('title: X') + doc.destroy() }) it('returns null for a missing file', async () => { @@ -169,7 +163,6 @@ describe('buildFileDocSeed', () => { it('requests the file with throwOnError so a read failure is not mistaken for an empty file', async () => { mockGetWorkspaceFile.mockRejectedValue(new Error('db down')) - // Propagates instead of returning null — the relay must retry, never seed blank over a real file. await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow('db down') expect(mockGetWorkspaceFile).toHaveBeenCalledWith('ws-1', 'file-1', { throwOnError: true }) }) @@ -194,7 +187,10 @@ describe('buildFileDocSeed — document identity', () => { updatedAt: new Date('2026-01-01T00:00:00.000Z'), }) mockLoadState.mockResolvedValue(null) - mockSaveState.mockResolvedValue(undefined) + mockCommitState.mockImplementation(async (_workspaceId, _fileId, version: number) => ({ + status: 'committed', + version, + })) }) const docIdOf = (update: Uint8Array): unknown => { @@ -208,23 +204,25 @@ describe('buildFileDocSeed — document identity', () => { } it('stores the document it builds, so the next open resumes it rather than building another', async () => { - // Until this row exists, a file that is opened but never edited gets a NEW document on every open. mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\nbody', 'utf-8')) const seed = await buildFileDocSeed('ws-1', 'file-1') - expect(mockSaveState).toHaveBeenCalledWith('file-1', seed!.update, 'test-source-hash') + expect(mockCommitState).toHaveBeenCalledWith('ws-1', 'file-1', VERSION, { + docState: seed!.update, + sourceHash: collabState.hashMarkdown(Buffer.from('# Title\n\nbody')), + expectedState: null, + }) expect(typeof docIdOf(seed!.update)).toBe('string') }) it('keeps the stored document’s identity when the markdown changed out-of-band', async () => { - // A copilot write (or the content API) moved the markdown on, so the stored binary is stale. It must - // be UPDATED, not replaced: the identity — and the client ids under it — have to survive. const stored = markdownToYDoc('# Title\n\nbody') stored.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-original') mockLoadState.mockResolvedValue({ docState: Y.encodeStateAsUpdate(stored), sourceHash: 'a-hash-from-before-the-external-write', + stateHash: collabState.hashMarkdown(Buffer.from(Y.encodeStateAsUpdate(stored))), }) mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\nbody\n\nadded externally', 'utf-8')) @@ -239,8 +237,6 @@ describe('buildFileDocSeed — document identity', () => { }) it('a client holding the resumed document merges it back without duplicating the file', async () => { - // The end-to-end property, stated the way it fails: a tab that outlived its room reconnects and - // syncs. Building a second document here appends the whole file to itself, on both sides. const original = markdownToYDoc('# Title\n\nfirst\n\nsecond') original.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-original') const client = new Y.Doc() @@ -249,6 +245,7 @@ describe('buildFileDocSeed — document identity', () => { mockLoadState.mockResolvedValue({ docState: Y.encodeStateAsUpdate(original), sourceHash: 'stale-after-an-external-write', + stateHash: collabState.hashMarkdown(Buffer.from(Y.encodeStateAsUpdate(original))), }) mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\nfirst\n\nsecond', 'utf-8')) @@ -264,6 +261,7 @@ describe('buildFileDocSeed — document identity', () => { mockLoadState.mockResolvedValue({ docState: new Uint8Array([9, 9, 9, 9]), sourceHash: 'stale', + stateHash: collabState.hashMarkdown(Buffer.from([9, 9, 9, 9])), }) mockFetchBuffer.mockResolvedValue(Buffer.from('# Title\n\nbody', 'utf-8')) @@ -288,30 +286,438 @@ describe('buildFileDocSeed — document identity', () => { mockFetchBuffer.mockResolvedValue(Buffer.from('# Legacy', 'utf-8')) mockLoadState.mockResolvedValue({ docState: Y.encodeStateAsUpdate(legacy), - sourceHash: 'test-source-hash', + sourceHash: collabState.hashMarkdown(Buffer.from('# Legacy')), + stateHash: collabState.hashMarkdown(Buffer.from(Y.encodeStateAsUpdate(legacy))), }) const first = await buildFileDocSeed('ws-1', 'file-1') const docId = docIdOf(first!.update) expect(typeof docId).toBe('string') - expect(mockSaveState).toHaveBeenCalledWith('file-1', first!.update, 'test-source-hash') + expect(mockCommitState).toHaveBeenCalledWith('ws-1', 'file-1', VERSION, { + docState: first!.update, + sourceHash: collabState.hashMarkdown(Buffer.from('# Legacy')), + expectedState: expect.objectContaining({ + stateHash: collabState.hashMarkdown(Buffer.from(Y.encodeStateAsUpdate(legacy))), + }), + }) - // The next open finds it named and hands back the stored bytes untouched. - mockSaveState.mockClear() - mockLoadState.mockResolvedValue({ docState: first!.update, sourceHash: 'test-source-hash' }) + mockCommitState.mockClear() + const stored = cachedState(first!.update, '# Legacy') + mockLoadState.mockResolvedValue(stored) const second = await buildFileDocSeed('ws-1', 'file-1') expect(docIdOf(second!.update)).toBe(docId) - expect(mockSaveState).not.toHaveBeenCalled() + expect(mockCommitState).toHaveBeenCalledOnce() + expect(mockCommitState).toHaveBeenCalledWith('ws-1', 'file-1', VERSION, { + docState: first!.update, + sourceHash: stored.sourceHash, + expectedState: { sourceHash: stored.sourceHash, stateHash: stored.stateHash }, + }) legacy.destroy() }) - it('still seeds when the document cannot be stored (the write is best-effort)', async () => { + it('does not return an unaccepted identity when the cache write fails', async () => { mockFetchBuffer.mockResolvedValue(Buffer.from('# Title', 'utf-8')) - mockSaveState.mockRejectedValue(new Error('db down')) + mockCommitState.mockRejectedValue(new Error('db down')) + + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow('db down') + }) +}) + +describe('buildFileDocSeed — accepted revisions', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetWorkspaceFile.mockReset().mockResolvedValue({ + id: 'file-1', + name: 'note.md', + key: 'k', + context: 'workspace', + updatedAt: new Date(VERSION), + contentUpdatedAt: new Date(VERSION), + }) + mockFetchBuffer.mockReset().mockResolvedValue(Buffer.from('base')) + mockLoadState.mockReset().mockResolvedValue(null) + mockCommitState + .mockReset() + .mockImplementation(async (_workspaceId, _fileId, version: number) => ({ + status: 'committed', + version, + })) + }) + + function namedState(markdown: string, identity: string): Uint8Array { + const doc = markdownToYDoc(markdown) + try { + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, identity) + return Y.encodeStateAsUpdate(doc) + } finally { + doc.destroy() + } + } + + function identityOf(update: Uint8Array): unknown { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, update) + return doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + } finally { + doc.destroy() + } + } + + it('makes simultaneous first-open seeds adopt the single accepted document identity', async () => { + let cache: collabState.CachedCollabDocState | null = null + const started = Promise.withResolvers() + const resume = Promise.withResolvers() + let commits = 0 + let losingUpdate: Uint8Array | undefined + mockLoadState.mockImplementation(async () => cache) + mockCommitState.mockImplementation( + async ( + _workspaceId, + _fileId, + version: number, + prepared: collabState.PreparedCollabDocState + ) => { + if (commits++ === 0) { + losingUpdate = prepared.docState + started.resolve() + await resume.promise + } + if ( + prepared.expectedState === null + ? cache !== null + : cache?.sourceHash !== prepared.expectedState.sourceHash || + cache?.stateHash !== prepared.expectedState.stateHash + ) { + return { status: 'conflict' } + } + cache = cachedState(prepared.docState, 'base') + return { status: 'committed', version } + } + ) + + const firstOpen = buildFileDocSeed('ws-1', 'file-1') + try { + await started.promise + const secondOpen = await buildFileDocSeed('ws-1', 'file-1') + expect(secondOpen).not.toBeNull() + expect(identityOf(secondOpen!.update)).not.toBe(identityOf(losingUpdate!)) + resume.resolve() + const resumedFirst = await firstOpen + + expect(resumedFirst?.update).toBe(secondOpen?.update) + expect(resumedFirst?.update).toBe(cache?.docState) + expect(mockCommitState).toHaveBeenCalledTimes(3) + expect(mockLoadState).toHaveBeenCalledTimes(3) + expect(mockCommitState.mock.calls[2][3].expectedState).toEqual({ + stateHash: cache?.stateHash, + sourceHash: cache?.sourceHash, + }) + } finally { + resume.resolve() + await firstOpen + } + }) + + it('revalidates even an unchanged named snapshot against its exact source and binary token', async () => { + const cached = cachedState(namedState('base', 'existing-document'), 'base') + mockLoadState.mockResolvedValue(cached) const seed = await buildFileDocSeed('ws-1', 'file-1') - expect(seed).not.toBeNull() - expect(typeof docIdOf(seed!.update)).toBe('string') + expect(seed?.update).toBe(cached.docState) + expect(mockCommitState).toHaveBeenCalledWith('ws-1', 'file-1', VERSION, { + docState: cached.docState, + sourceHash: cached.sourceHash, + expectedState: { sourceHash: cached.sourceHash, stateHash: cached.stateHash }, + }) }) + + it('adopts a same-content identity replacement instead of returning an unfenced cache hit', async () => { + const prior = cachedState(namedState('base', 'old-generation'), 'base') + const winner = cachedState(namedState('base', 'new-generation'), 'base') + mockLoadState.mockResolvedValueOnce(prior).mockResolvedValue(winner) + mockCommitState.mockResolvedValueOnce({ status: 'conflict' }) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + + expect(identityOf(seed!.update)).toBe('new-generation') + expect(seed?.update).toBe(winner.docState) + expect(mockGetWorkspaceFile).toHaveBeenCalledTimes(2) + expect(mockLoadState).toHaveBeenCalledTimes(2) + expect(mockCommitState.mock.calls[0][3].expectedState.stateHash).toBe(prior.stateHash) + expect(mockCommitState.mock.calls[1][3].expectedState.stateHash).toBe(winner.stateHash) + }) + + it('rereads content and cache after a content-version race before publishing the winning seed', async () => { + const winner = cachedState(namedState('new content', 'winning-generation'), 'new content') + mockCommitState.mockImplementationOnce(async () => { + mockGetWorkspaceFile.mockResolvedValue({ + id: 'file-1', + name: 'note.md', + key: 'new-key', + context: 'workspace', + contentUpdatedAt: new Date(VERSION + 1), + updatedAt: new Date(VERSION + 1), + }) + mockFetchBuffer.mockResolvedValue(Buffer.from('new content')) + mockLoadState.mockResolvedValue(winner) + return { status: 'conflict' } + }) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + + expect(seed).toEqual({ update: winner.docState, version: VERSION + 1 }) + expect(mockGetWorkspaceFile).toHaveBeenCalledTimes(2) + expect(mockFetchBuffer).toHaveBeenCalledTimes(2) + expect(mockCommitState.mock.calls[1][2]).toBe(VERSION + 1) + expect(mockCommitState.mock.calls[1][3].expectedState).toEqual({ + sourceHash: winner.sourceHash, + stateHash: winner.stateHash, + }) + }) + + it('bounds seed conflicts to three complete read/prepare/commit attempts', async () => { + mockCommitState.mockResolvedValue({ status: 'conflict' }) + + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toBeInstanceOf( + collabState.CollabDocStateConflictError + ) + expect(mockGetWorkspaceFile).toHaveBeenCalledTimes(3) + expect(mockFetchBuffer).toHaveBeenCalledTimes(3) + expect(mockLoadState).toHaveBeenCalledTimes(3) + expect(mockCommitState).toHaveBeenCalledTimes(3) + }) + + it('does not start a seed for an already cancelled request', async () => { + const reason = new Error('request cancelled') + + await expect(buildFileDocSeed('ws-1', 'file-1', AbortSignal.abort(reason))).rejects.toBe(reason) + expect(mockGetWorkspaceFile).not.toHaveBeenCalled() + expect(mockFetchBuffer).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() + }) + + it.each(['file lookup', 'download', 'cache lookup'] as const)( + 'stops after cancellation during %s without preparing or committing a seed', + async (stage) => { + const controller = new AbortController() + const reason = new Error('request cancelled') + const stop = () => controller.abort(reason) + if (stage === 'file lookup') { + mockGetWorkspaceFile.mockImplementationOnce(async () => { + stop() + return null + }) + } else if (stage === 'download') { + mockFetchBuffer.mockImplementationOnce(async (_record, options) => { + stop() + expect(options.signal.aborted).toBe(true) + expect(options.signal.reason).toBe(reason) + return Buffer.from('base') + }) + } else { + mockLoadState.mockImplementationOnce(async () => { + stop() + return null + }) + } + + await expect(buildFileDocSeed('ws-1', 'file-1', controller.signal)).rejects.toBe(reason) + expect(mockCommitState).not.toHaveBeenCalled() + if (stage === 'file lookup') expect(mockFetchBuffer).not.toHaveBeenCalled() + if (stage !== 'cache lookup') expect(mockLoadState).not.toHaveBeenCalled() + } + ) + + it.each(['committed', 'conflict'] as const)( + 'does not publish or retry a %s result after cancellation during commit', + async (status) => { + const controller = new AbortController() + const reason = new Error('request cancelled') + mockCommitState.mockImplementationOnce(async () => { + controller.abort(reason) + return { status, version: VERSION } + }) + + await expect(buildFileDocSeed('ws-1', 'file-1', controller.signal)).rejects.toBe(reason) + expect(mockGetWorkspaceFile).toHaveBeenCalledOnce() + expect(mockCommitState).toHaveBeenCalledOnce() + } + ) + + it('shares one deadline across retries and stops when it expires without caller cancellation', async () => { + const deadline = new AbortController() + const reason = new DOMException('Seed deadline expired', 'TimeoutError') + const timeout = vi.spyOn(AbortSignal, 'timeout').mockReturnValue(deadline.signal) + mockCommitState.mockResolvedValueOnce({ status: 'conflict' }) + mockFetchBuffer.mockResolvedValueOnce(Buffer.from('base')).mockImplementationOnce(async () => { + deadline.abort(reason) + return Buffer.from('base') + }) + + try { + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toBe(reason) + expect(timeout).toHaveBeenCalledExactlyOnceWith(FILE_DOC_TIMEOUTS.seedRequestMs) + expect(mockFetchBuffer).toHaveBeenCalledTimes(2) + for (const [, options] of mockFetchBuffer.mock.calls) { + expect(options.signal).toBe(deadline.signal) + } + expect(mockLoadState).toHaveBeenCalledOnce() + expect(mockCommitState).toHaveBeenCalledOnce() + } finally { + timeout.mockRestore() + } + }) + + it('returns missing if the file is deleted during the commit', async () => { + mockCommitState.mockResolvedValue({ status: 'missing' }) + + await expect(buildFileDocSeed('ws-1', 'file-1')).resolves.toBeNull() + expect(mockCommitState).toHaveBeenCalledOnce() + expect(mockGetWorkspaceFile).toHaveBeenCalledOnce() + }) + + it('returns missing if the file disappears before a retry', async () => { + mockCommitState.mockImplementationOnce(async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + return { status: 'conflict' } + }) + + await expect(buildFileDocSeed('ws-1', 'file-1')).resolves.toBeNull() + expect(mockCommitState).toHaveBeenCalledOnce() + expect(mockLoadState).toHaveBeenCalledOnce() + expect(mockGetWorkspaceFile).toHaveBeenCalledTimes(2) + }) + + it('propagates a durable read error without constructing or committing a new document', async () => { + mockFetchBuffer.mockRejectedValue(new Error('object storage unavailable')) + + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow('object storage unavailable') + expect(mockLoadState).not.toHaveBeenCalled() + expect(mockCommitState).not.toHaveBeenCalled() + }) + + it('keeps the freshness hash tied to raw durable Markdown rather than its canonical projection', async () => { + const markdown = '# Title\r\n\r\nbody\r\n' + mockFetchBuffer.mockResolvedValue(Buffer.from(markdown)) + const seed = await buildFileDocSeed('ws-1', 'file-1') + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, seed!.update) + const canonical = yDocToFileMarkdown(doc) + expect(canonical).not.toBe(markdown) + expect(mockCommitState.mock.calls[0][3].sourceHash).toBe( + collabState.hashMarkdown(Buffer.from(markdown)) + ) + expect(mockCommitState.mock.calls[0][3].sourceHash).not.toBe( + collabState.hashMarkdown(Buffer.from(canonical)) + ) + } finally { + doc.destroy() + } + }) + + it.each(['fresh', 'stale'] as const)( + 'retains deleted history when the %s cached document seeds a cold room', + async (kind) => { + const peer = new Y.Doc() + const cold = new Y.Doc() + try { + Y.applyUpdate(peer, namedState('base', 'shared-generation')) + const paragraph = peer.getXmlFragment(COLLAB_DOC_FIELD).get(0) + if (!(paragraph instanceof Y.XmlElement)) throw new Error('Expected paragraph') + const text = paragraph.get(0) + if (!(text instanceof Y.XmlText)) throw new Error('Expected text') + const start = text.length + text.insert(start, ' transient peer text') + const beforeDeletion = Y.encodeStateAsUpdate(peer) + text.delete(start, text.length - start) + const cache = cachedState(Y.encodeStateAsUpdate(peer), 'base') + mockLoadState.mockResolvedValue(cache) + const durable = kind === 'fresh' ? 'base' : 'base\n\nexternal addition' + mockFetchBuffer.mockResolvedValue(Buffer.from(durable)) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + const accepted = mockCommitState.mock.calls[0][3] as collabState.PreparedCollabDocState + expect(seed?.update).toBe(accepted.docState) + Y.applyUpdate(cold, accepted.docState) + Y.applyUpdate(cold, beforeDeletion) + + expect(yDocToFileMarkdown(cold)).not.toContain('transient peer text') + expect(yDocToMarkdown(cold)).toBe(serializeMarkdownBody(durable)) + expect(identityOf(seed!.update)).toBe('shared-generation') + expect(accepted.expectedState).toEqual({ + sourceHash: cache.sourceHash, + stateHash: cache.stateHash, + }) + } finally { + peer.destroy() + cold.destroy() + } + } + ) + + it('fails closed on an undecodable cache tagged as current rather than returning a fabricated history', async () => { + mockLoadState.mockResolvedValue(cachedState(new Uint8Array([255]), 'base')) + + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toThrow() + expect(mockCommitState).not.toHaveBeenCalled() + }) + + it('fences replacement of an undecodable stale cache against the exact corrupt revision', async () => { + const corrupt = cachedState(new Uint8Array([255]), 'older bytes') + mockLoadState.mockResolvedValue(corrupt) + const seed = await buildFileDocSeed('ws-1', 'file-1') + + expect(typeof identityOf(seed!.update)).toBe('string') + expect(mockCommitState).toHaveBeenCalledWith('ws-1', 'file-1', VERSION, { + docState: seed!.update, + sourceHash: collabState.hashMarkdown(Buffer.from('base')), + expectedState: { sourceHash: corrupt.sourceHash, stateHash: corrupt.stateHash }, + }) + }) + + it('never publishes a rebuilt identity when replacement of a corrupt cache loses every race', async () => { + mockLoadState.mockResolvedValue(cachedState(new Uint8Array([255]), 'older bytes')) + mockCommitState.mockResolvedValue({ status: 'conflict' }) + + await expect(buildFileDocSeed('ws-1', 'file-1')).rejects.toBeInstanceOf( + collabState.CollabDocStateConflictError + ) + expect(mockCommitState).toHaveBeenCalledTimes(3) + }) + + it.each(['fresh', 'frontmatter-only write'] as const)( + 'preserves a peer’s empty-paragraph typing target while preparing a %s seed', + async (kind) => { + const peer = new Y.Doc() + const cold = new Y.Doc() + try { + Y.applyUpdate(peer, namedState('base', 'shared-generation')) + const tail = new Y.XmlElement('paragraph') + const text = new Y.XmlText() + tail.insert(0, [text]) + peer.getXmlFragment(COLLAB_DOC_FIELD).push([tail]) + const cache = cachedState(Y.encodeStateAsUpdate(peer), 'base') + mockLoadState.mockResolvedValue(cache) + const durable = kind === 'fresh' ? 'base' : '---\ntitle: changed\n---\n\nbase' + mockFetchBuffer.mockResolvedValue(Buffer.from(durable)) + + const seed = await buildFileDocSeed('ws-1', 'file-1') + expect(seed?.update).toBe(mockCommitState.mock.calls[0][3].docState) + text.insert(0, 'late offline text') + Y.applyUpdate(cold, seed!.update) + Y.applyUpdate(cold, Y.encodeStateAsUpdate(peer)) + + expect(yDocToFileMarkdown(cold)).toContain('base\n\nlate offline text') + if (kind === 'frontmatter-only write') { + expect(yDocToFileMarkdown(cold)).toContain('title: changed') + } + } finally { + peer.destroy() + cold.destroy() + } + } + ) }) diff --git a/apps/sim/lib/collab-doc/seed.ts b/apps/sim/lib/collab-doc/seed.ts index adc10b1aa4a..00456bb5918 100644 --- a/apps/sim/lib/collab-doc/seed.ts +++ b/apps/sim/lib/collab-doc/seed.ts @@ -1,17 +1,18 @@ import { createLogger } from '@sim/logger' -import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import * as Y from 'yjs' -import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' -import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { - type CachedCollabDocState, + assertCollabDocStateSize, + CollabDocStateConflictError, + commitCollabDocState, hashMarkdown, loadCollabDocState, - saveCollabDocState, -} from './collab-state' -import { applyMarkdownToYDoc, canonicalizeYDoc, markdownToYDoc } from './converter' +} from '@/lib/collab-doc/collab-state' +import { applyMarkdownToYDoc, markdownToYDoc } from '@/lib/collab-doc/converter' +import { fetchWorkspaceFileBuffer, getWorkspaceFile } from '@/lib/uploads/contexts/workspace' +import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' const logger = createLogger('FileDocSeed') @@ -20,55 +21,34 @@ const logger = createLogger('FileDocSeed') * non-collaborative path anyway; converting a huge document server-side would be wasted work. */ const MAX_SEED_BYTES = 5 * 1024 * 1024 +const MAX_SEED_ATTEMPTS = 3 /** A collaborative document's initial state, encoded as a Yjs update the relay can apply. */ export interface FileDocSeed { /** `Y.encodeStateAsUpdate` of the seeded document — apply with `Y.applyUpdate`. */ update: Uint8Array - /** - * The file's durable `updatedAt` (epoch ms) this seed was built from — the version the relay records - * as what its freshly-seeded live doc is synced to, for the persist optimistic-concurrency guard. - */ + /** Durable content version (epoch ms), used by the relay's next persist. */ version: number } /** - * Bring a cached Yjs snapshot into canonical form before it seeds a room, so a warm open and a cold open - * render the same document as the static placeholder (see {@link canonicalizeYDoc}). - * - * The freshness tag is a hash of the markdown alone, carrying no parser version — so a snapshot written - * under older parse rules still reads as fresh and would otherwise be replayed verbatim, with no path - * that ever repairs it. Running the round-trip here is that repair, and it doubles as the bound on this - * branch: the cached path never calls `parseMarkdownToDoc`, so it is the one way into a room that the - * parse-side limits do not cover. Returns the original bytes untouched when the snapshot is already - * canonical AND already named (the common case) — no re-encode — and a fresh encode, preserving the - * CRDT's client ids, when it had to repair or name one. - * - * Naming happens here too, not only where a document is built: a document stored before identities - * existed is returned by this path on every open, so if it were skipped here those files would never - * acquire one and the join-ack guard could never fire for them — which is the population most likely to - * have a tab that outlived its room. `changed` tells the caller to store what it got back, so the - * identity is minted ONCE and every later open agrees with it (a re-minted one would make the guard - * refuse a client holding the very same document). + * Name legacy documents without rewriting their shared tree. Named snapshots retain their original + * bytes. Even unchanged bytes are fenced before returning, so a concurrent seed/reset cannot + * publish an unaccepted identity. */ -function prepareCachedSeed(cached: Uint8Array): { update: Uint8Array; changed: boolean } { +function prepareCachedSeed(cached: Uint8Array): Uint8Array { const doc = new Y.Doc() try { Y.applyUpdate(doc, cached) - const repaired = canonicalizeYDoc(doc) const named = ensureDocumentIdentity(doc) - return repaired || named - ? { update: Y.encodeStateAsUpdate(doc), changed: true } - : { update: cached, changed: false } + return named ? Y.encodeStateAsUpdate(doc) : cached } finally { doc.destroy() } } /** - * Give a document an identity if it has none, and report whether it needed one. A resumed document - * keeps the identity its clients already know it by — re-minting would make the join-ack guard refuse - * a client that holds this exact document. + * Preserve identities known by reconnecting clients; only unnamed legacy documents need one. */ function ensureDocumentIdentity(ydoc: Y.Doc): boolean { const config = ydoc.getMap(FILE_DOC_SEED.configMap) @@ -77,106 +57,69 @@ function ensureDocumentIdentity(ydoc: Y.Doc): boolean { return true } -/** Store the file's collaborative document. Best-effort: the durable markdown is the source of truth. */ -async function storeDocument( - fileId: string, - update: Uint8Array, - sourceHash: string -): Promise { - try { - await saveCollabDocState(fileId, update, sourceHash) - } catch (error) { - logger.warn(`Failed to store the collaborative document for file ${fileId}`, { - error: getErrorMessage(error), - }) - } -} - /** - * Build the server-side seed for a file's collaborative document: load the file's current markdown - * and convert it — through the exact client engine (see {@link markdownToYDoc}) — into a Yjs update. - * - * This is what makes seeding server-authoritative: the realtime relay applies this to a fresh room's - * document instead of electing a client to import the content, so the whole client-seeder subsystem - * (election / deadlines / retries) goes away. The frontmatter is stripped exactly as the client's - * seed did — it is file metadata, not part of the collaborative body. - * - * Returns `null` ONLY when the file is genuinely absent (deleted/never-existed). A transient read - * failure THROWS (`throwOnError`) rather than returning `null`, so the relay retries instead of - * mistaking a DB blip for an empty file and seeding blank content over the real document. + * Return only a seed accepted against both the durable file version and the cached Yjs history. + * Fresh caches retain their tree; external Markdown changes reconcile into the existing history. + * Returns null only for an absent file. Read/write failures throw so the relay retries rather than + * treating an unavailable cache as permission to mint a replacement document. */ export async function buildFileDocSeed( workspaceId: string, - fileId: string + fileId: string, + signal?: AbortSignal ): Promise { - const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) - if (!record) return null - - // The content-scoped version (advances only on content writes, never on rename/move) is the persist - // If-Match token — so a metadata bump can't make a racing persist reconcile stale content and clobber - // live edits. `getWorkspaceFile` always maps it from the NOT NULL column; coalesce is a type guard only. - const version = (record.contentUpdatedAt ?? record.updatedAt).getTime() - const buffer = await fetchWorkspaceFileBuffer(record, { maxBytes: MAX_SEED_BYTES }) - - const sourceHash = hashMarkdown(buffer) - - // Best-effort read: the cache is an optimization over the durable markdown we already hold, so a - // transient DB error (or a not-yet-migrated cache table) must fall through to conversion rather than - // block the cold open — symmetric with the best-effort write below. - let stored: CachedCollabDocState | null = null - try { - stored = await loadCollabDocState(fileId) - } catch (error) { - logger.warn(`Failed to read cached collab doc state for file ${fileId}`, { - error: getErrorMessage(error), + const timeoutSignal = AbortSignal.timeout(FILE_DOC_TIMEOUTS.seedRequestMs) + const seedSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal + for (let attempt = 0; attempt < MAX_SEED_ATTEMPTS; attempt++) { + seedSignal.throwIfAborted() + const record = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) + seedSignal.throwIfAborted() + if (!record) return null + const version = (record.contentUpdatedAt ?? record.updatedAt).getTime() + const buffer = await fetchWorkspaceFileBuffer(record, { + maxBytes: MAX_SEED_BYTES, + signal: seedSignal, }) - } + seedSignal.throwIfAborted() + const sourceHash = hashMarkdown(buffer) - // Cold-start fast path: the stored document already projects to THIS markdown, so apply it verbatim - // (the Hocuspocus load-document pattern) instead of re-converting. - if (stored?.sourceHash === sourceHash) { - const prepared = prepareCachedSeed(stored.docState) - // Store a repair or a freshly-minted identity so the next open finds it done — and, for the - // identity, so every open names the SAME document. - if (prepared.changed) await storeDocument(fileId, prepared.update, sourceHash) - return { update: prepared.update, version } - } + /** An unavailable cache is not an absent document: retry without minting a new history. */ + const stored = await loadCollabDocState(fileId) + seedSignal.throwIfAborted() + let update: Uint8Array + if (stored?.sourceHash === sourceHash) { + update = prepareCachedSeed(stored.docState) + } else { + const { frontmatter, body } = splitFrontmatter(buffer.toString('utf-8')) + const ydoc = resumeDocument(fileId, stored?.docState, body) + try { + const config = ydoc.getMap(FILE_DOC_SEED.configMap) + config.set(FILE_DOC_SEED.flag, true) + config.set(FILE_DOC_SEED.frontmatterKey, frontmatter) + ensureDocumentIdentity(ydoc) + update = Y.encodeStateAsUpdate(ydoc) + } finally { + ydoc.destroy() + } + } + assertCollabDocStateSize(update) + seedSignal.throwIfAborted() - const { frontmatter, body } = splitFrontmatter(buffer.toString('utf-8')) - // The markdown moved on out-of-band (a copilot write, the content API, a file tool) — bring the - // STORED document up to it rather than building a second one. See {@link resumeDocument}. - const ydoc = resumeDocument(fileId, stored?.docState, body) - try { - const config = ydoc.getMap(FILE_DOC_SEED.configMap) - // Mark the document seeded IN the same doc, so the client's readiness gate - // (`synced && initialContentLoaded === true`) recognizes a server-seeded doc without any - // client-seeder handshake, and a stray re-election can never seed on top of it. - config.set(FILE_DOC_SEED.flag, true) - // Carry the frontmatter in the doc (not the body) so it merges across clients and a later - // server-side edit can update it — the editor re-attaches this on autosave. - config.set(FILE_DOC_SEED.frontmatterKey, frontmatter) - ensureDocumentIdentity(ydoc) - const update = Y.encodeStateAsUpdate(ydoc) - // Store it NOW, not at the next persist. Until this row exists every cold open builds the document - // again from markdown, minting a new identity each time — so a file that is opened but never edited - // has a different document on every open, and any client that outlives a room (a laptop that slept - // past the shared stream's TTL) reconnects into one and merges its content in twice. - await storeDocument(fileId, update, sourceHash) - return { update, version } - } finally { - ydoc.destroy() + const result = await commitCollabDocState(workspaceId, fileId, version, { + docState: update, + sourceHash, + expectedState: stored ? { stateHash: stored.stateHash, sourceHash: stored.sourceHash } : null, + }) + seedSignal.throwIfAborted() + if (result.status === 'committed') return { update, version: result.version } + if (result.status === 'missing') return null } + throw new CollabDocStateConflictError(fileId) } /** - * The file's collaborative document, brought up to `body`. - * - * Two Yjs documents built from the same markdown are NOT the same document: their items carry - * different client ids, so merging them appends one to the other — the file, twice. Anything that - * rebuilds a document from markdown therefore mints a new identity, and any client still holding the - * previous one corrupts the file the moment it reconnects. So a document is built exactly once and - * every later change is applied INTO it as a CRDT diff (the same path a copilot edit takes), which is - * what keeps one file to one document for its whole life. + * Reconcile external content into the existing history. Recreating equivalent Markdown in a new + * Y.Doc would assign unrelated item identities and duplicate content when old clients reconnect. */ function resumeDocument(fileId: string, stored: Uint8Array | undefined, body: string): Y.Doc { if (!stored) return markdownToYDoc(body) @@ -186,9 +129,7 @@ function resumeDocument(fileId: string, stored: Uint8Array | undefined, body: st applyMarkdownToYDoc(ydoc, body) return ydoc } catch (error) { - // The stored document is the file's identity, but it is still a CACHE: an undecodable one must not - // take the file's markdown down with it. Build a new document — which mints a new identity, so a - // client still holding the old one is refused rather than merged (see FILE_DOC_SEED.docIdKey). + /** A corrupt binary needs a new identity; the caller must win its cache fence before returning it. */ logger.warn(`Stored collaborative document for file ${fileId} is unusable; rebuilding it`, { error: getErrorMessage(error), }) diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index 9817363630c..64c128226e0 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -255,15 +255,28 @@ describe('processOutboxEvents — empty / no handler', () => { }) }) - it('dead-letters events with no registered handler', async () => { + it('retries events with no registered handler during rolling deployments', async () => { queueTableRows(outboxEvent, [makePendingRow({ eventType: 'unknown.event' })]) holdLease() const result = await processOutboxEvents({}) + expect(result.retried).toBe(1) + const retry = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) + expect(retry).toBeDefined() + expect(retry?.attempts).toBe(1) + }) + + it('dead-letters a missing handler after the configured retry budget', async () => { + queueTableRows(outboxEvent, [ + makePendingRow({ eventType: 'unknown.event', attempts: 2, maxAttempts: 3 }), + ]) + holdLease() + + const result = await processOutboxEvents({}) + expect(result.deadLettered).toBe(1) const terminal = updateSets().find((set) => set.status === 'dead_letter') - expect(terminal).toBeDefined() expect(terminal?.lastError).toMatch(/No handler registered/) }) }) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index a4ede767836..152caecfa5a 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -568,17 +568,15 @@ async function runHandler( const handler = handlers[event.eventType] if (!handler) { - logger.error('No handler registered for outbox event type', { + const reason = `No handler registered for event type '${event.eventType}'` + logger.warn('No handler registered for outbox event type; scheduling a bounded retry', { eventId: event.id, eventType: event.eventType, }) - await updateIfLeaseHeld(event, { - status: 'dead_letter', - lastError: `No handler registered for event type '${event.eventType}'`, - processedAt: new Date(), - lockedAt: null, + return scheduleDeferred(event, { + outcome: 'deferred', + reason, }) - return 'dead_letter' } try { diff --git a/apps/sim/lib/realtime/notify.test.ts b/apps/sim/lib/realtime/notify.test.ts index 98196492257..feb5159e0ab 100644 --- a/apps/sim/lib/realtime/notify.test.ts +++ b/apps/sim/lib/realtime/notify.test.ts @@ -6,18 +6,21 @@ import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/utils/urls', () => ({ getSocketServerUrl: () => 'http://realtime' })) vi.mock('@/lib/core/config/env', () => ({ env: { INTERNAL_API_SECRET: 'secret' } })) -import { mergeEditIntoLiveFileDoc } from './notify' +import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' -describe('mergeEditIntoLiveFileDoc', () => { +describe('applyEditToLiveFileDoc', () => { afterEach(() => { vi.unstubAllGlobals() }) it('POSTs the edit to the realtime apply-edit endpoint with the api key', async () => { - const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ applied: true, status: 'applied' }), + }) vi.stubGlobal('fetch', fetchMock) - await mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) + await applyEditToLiveFileDoc('file-1', '# hello', { version: 42 }) expect(fetchMock).toHaveBeenCalledWith( 'http://realtime/api/file-doc/apply-edit', @@ -30,83 +33,90 @@ describe('mergeEditIntoLiveFileDoc', () => { ) }) - it('never throws when the realtime call fails (best-effort)', async () => { + it('throws when the realtime call fails so the outbox can retry', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('socket pod down'))) - await expect( - mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) - ).resolves.toBeUndefined() + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).rejects.toThrow( + 'socket pod down' + ) }) - it('never throws on a non-2xx response', async () => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 503 })) - await expect( - mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) - ).resolves.toBeUndefined() - }) + it('surfaces retryable delivery failures to durable outbox callers', async () => { + const cancel = vi.fn() + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(new Response(new ReadableStream({ cancel }), { status: 503 })) + ) - it('a later durable merge waits for an in-flight earlier one, then applies last', async () => { - let resolveFirst: (value: { ok: boolean }) => void = () => {} - const fetchMock = vi - .fn() - .mockImplementationOnce(() => new Promise((resolve) => (resolveFirst = resolve))) - .mockResolvedValue({ ok: true }) - vi.stubGlobal('fetch', fetchMock) + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).rejects.toThrow( + 'status 503' + ) + expect(cancel).toHaveBeenCalledOnce() + }) - const first = mergeEditIntoLiveFileDoc('file-durable', 'earlier', { version: 99 }) // in flight - await Promise.resolve() - const durable = mergeEditIntoLiveFileDoc('file-durable', 'final content', { version: 100 }) - await Promise.resolve() - await Promise.resolve() + it('returns the relay reconciliation status to durable outbox callers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ applied: false, status: 'no-live-room' }), + }) + ) - // The later write waits for the in-flight earlier one → its fetch has not fired yet, so it cannot be - // reordered before a straggler and cannot be clobbered by one. - expect(fetchMock).toHaveBeenCalledTimes(1) + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).resolves.toEqual({ + applied: false, + status: 'no-live-room', + }) + }) +}) - resolveFirst({ ok: true }) - await first - await durable +describe('invalidateLiveFileDoc', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) - // Only after the earlier merge completed does the later (final) merge apply — always last. - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1][1].body).toBe( - JSON.stringify({ fileId: 'file-durable', markdown: 'final content', version: 100 }) + it.each([200, 503])('cancels unread response bodies for status %i', async (status) => { + const cancel = vi.fn() + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(new Response(new ReadableStream({ cancel }), { status })) ) + + const result = invalidateLiveFileDoc('file-1', 42) + if (status === 200) { + await expect(result).resolves.toBeUndefined() + } else { + await expect(result).rejects.toThrow('status 503') + } + expect(cancel).toHaveBeenCalledOnce() }) - it('serializes concurrent durable writes to a file strictly in order', async () => { - const applied: number[] = [] - const resolvers: Array<() => void> = [] + it('preserves the HTTP failure when response-body cancellation fails', async () => { + const cancel = vi.fn().mockRejectedValue(new Error('body already errored')) vi.stubGlobal( 'fetch', - vi.fn((_url: string, init: { body: string }) => { - applied.push(JSON.parse(init.body).version) - return new Promise<{ ok: boolean }>((resolve) => - resolvers.push(() => resolve({ ok: true })) - ) + vi.fn().mockResolvedValue(new Response(new ReadableStream({ cancel }), { status: 503 })) + ) + + await expect(invalidateLiveFileDoc('file-1', 42)).rejects.toThrow('status 503') + expect(cancel).toHaveBeenCalledOnce() + }) + + it('POSTs a durability-sensitive invalidation and surfaces delivery failures', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + vi.stubGlobal('fetch', fetchMock) + + await invalidateLiveFileDoc('file-1', 42) + + expect(fetchMock).toHaveBeenCalledWith( + 'http://realtime/api/file-doc/invalidate', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ 'x-api-key': 'secret' }), + body: JSON.stringify({ fileId: 'file-1', version: 42 }), }) ) - const flush = async () => { - for (let i = 0; i < 6; i++) await Promise.resolve() - } - const s = mergeEditIntoLiveFileDoc('file-order', 's', { version: 0 }) // in flight - await flush() - // Two later durable writes arrive while the first merge is in flight — both must chain, not both - // resume-and-fire concurrently. - const a = mergeEditIntoLiveFileDoc('file-order', 'a', { version: 1 }) - const b = mergeEditIntoLiveFileDoc('file-order', 'b', { version: 2 }) - await flush() - expect(applied).toEqual([0]) // A and B queued behind the in-flight first merge - - resolvers[0]() // finish first → A applies next (not B) - await flush() - expect(applied).toEqual([0, 1]) - - resolvers[1]() // finish A → B applies after A - await flush() - expect(applied).toEqual([0, 1, 2]) - - resolvers[2]() - await Promise.all([s, a, b]) + fetchMock.mockResolvedValueOnce({ ok: false, status: 503 }) + await expect(invalidateLiveFileDoc('file-1', 42)).rejects.toThrow('status 503') }) }) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index 2374a8d5580..52945218d41 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -169,79 +169,75 @@ export async function notifyFolderResourceChanged( * How a durable live-doc merge is positioned on the file's monotonic version line. Omit `version` to * apply the merge without ordering it (legacy). */ -interface LiveFileDocMergeOrder { +export interface LiveFileDocMergeOrder { /** A durable write's `contentUpdatedAt` (epoch ms): applied only if newer than the version the doc * already incorporates, AND recorded as the synced version (the persist If-Match guard). */ version?: number } +export type LiveFileDocMergeStatus = 'applied' | 'no-live-room' | 'merge-unavailable' | 'stale' + +interface LiveFileDocMergeResponse { + applied: boolean + status: LiveFileDocMergeStatus +} + /** - * Best-effort: ask the realtime relay to merge a durable copilot/file write into a file's LIVE - * collaborative document, so open editors reconcile to it as a CRDT merge rather than the file changing - * underneath them, and a late joiner is seeded from it. No-op when no doc is (or was recently) live (the - * relay reports `applied: false`). The file itself is written durably by the caller regardless — this - * only drives the live view. Never throws. - * - * (Streaming copilot output is NOT merged here: the open editor applies the stream client-side as minimal - * CRDT diffs — see `applyStreamedMarkdownToLiveDoc` — which renders smoothly and broadcasts to peers. This - * merge is the stream-end durable reconcile, and by then it is usually a noop diff.) - * - * The former clobber gap — an open editor's autosave dropping this edit — is closed: a collaborative - * editor no longer client-autosaves (the relay persists the shared doc to markdown server-side), and the - * relay applies this merge THROUGH the shared Redis stream, so it reaches the live doc on whichever task - * holds it and can't go stale relative to this direct write. - * - * The caller awaits this so the fetch dispatches before the route handler returns. Bounded to - * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. - * - * `order.version` positions the merge so a stale write never regresses the doc: it applies only if newer - * than the version the doc already incorporates, and is recorded as the synced version. Ordering is - * enforced at two scales: within this process, merges for a file run on a single serialized chain (each - * chained after the current tail) so writes never apply concurrently; across processes the relay orders - * by that monotonic version under a cluster-wide lock. + * Applies one durable file version to the live collaboration document and surfaces delivery + * failures to callers that own a retry policy, such as the transactional outbox. */ -export async function mergeEditIntoLiveFileDoc( +export async function applyEditToLiveFileDoc( fileId: string, markdown: string, - order: LiveFileDocMergeOrder = {} -): Promise { - const tail = liveDocMergeChain.get(fileId) ?? Promise.resolve() - const run = tail.then(() => applyLiveFileDocMerge(fileId, markdown, order)) - liveDocMergeChain.set(fileId, run) - try { - await run - } finally { - if (liveDocMergeChain.get(fileId) === run) liveDocMergeChain.delete(fileId) + order: LiveFileDocMergeOrder = {}, + signal?: AbortSignal +): Promise { + const timeoutSignal = AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS) + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, markdown, version: order.version }), + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + throw new Error(`Live document reconciliation failed with status ${response.status}`) } -} -/** Per file, the tail of the serialized merge chain (each merge applies after it); never rejects - * because {@link applyLiveFileDocMerge} never throws. Absent when the file's chain is idle. */ -const liveDocMergeChain = new Map>() + const result = (await response.json()) as unknown + if (typeof result !== 'object' || result === null) { + throw new Error('Live document reconciliation returned an invalid response') + } + const candidate = result as Partial + const validStatus = + candidate.status === 'applied' || + candidate.status === 'no-live-room' || + candidate.status === 'merge-unavailable' || + candidate.status === 'stale' + if (typeof candidate.applied !== 'boolean' || !validStatus) { + throw new Error('Live document reconciliation returned an invalid response') + } + return { applied: candidate.applied, status: candidate.status as LiveFileDocMergeStatus } +} -/** POST the merge to the relay. Never throws (a live-doc merge is best-effort). */ -async function applyLiveFileDocMerge( +/** + * Invalidates one live document after a durable replacement that cannot be merged into the rich + * editor. Unlike list notifications this is durability-sensitive and throws so the outbox retries. + */ +export async function invalidateLiveFileDoc( fileId: string, - markdown: string, - order: LiveFileDocMergeOrder + version: number, + signal?: AbortSignal ): Promise { - try { - const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, - // `version` (durable `contentUpdatedAt`) records the synced version the live doc now incorporates - // (the persist If-Match guard). JSON.stringify drops it when undefined (an unordered legacy merge). - body: JSON.stringify({ - fileId, - markdown, - version: order.version, - }), - signal: AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS), - }) - if (!response.ok) { - logger.warn('file-doc apply-edit failed', { fileId, status: response.status }) - } - } catch (error) { - logger.warn('file-doc apply-edit error', { fileId, error: getErrorMessage(error) }) + const timeoutSignal = AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS) + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/invalidate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, version }), + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + await response.body?.cancel().catch(() => {}) + if (!response.ok) { + throw new Error(`Live document invalidation failed with status ${response.status}`) } } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts new file mode 100644 index 00000000000..3c2bbc43ccb --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockApplyEditToLiveFileDoc, mockDownloadFile, mockInvalidateLiveFileDoc } = vi.hoisted( + () => ({ + mockApplyEditToLiveFileDoc: vi.fn(), + mockDownloadFile: vi.fn(), + mockInvalidateLiveFileDoc: vi.fn(), + }) +) + +vi.mock('@/lib/realtime/notify', () => ({ + applyEditToLiveFileDoc: mockApplyEditToLiveFileDoc, + invalidateLiveFileDoc: mockInvalidateLiveFileDoc, +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: mockDownloadFile, +})) + +import type { OutboxEventContext } from '@/lib/core/outbox/service' +import { + WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + workspaceFileLiveDocOutboxHandlers, +} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' + +const VERSION = new Date('2026-09-04T12:00:00.000Z') +const PAYLOAD = { + workspaceId: 'workspace-1', + fileId: 'file-1', + version: VERSION.getTime(), +} + +function context(): OutboxEventContext { + return { + eventId: 'event-1', + eventType: WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + attempts: 0, + maxAttempts: 10, + signal: new AbortController().signal, + checkpointPayload: vi.fn(), + } +} + +function handler() { + const registered = workspaceFileLiveDocOutboxHandlers[WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT] + if (!registered) throw new Error('Workspace file live-document handler is not registered') + return registered +} + +describe('workspace file live-document outbox', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockDownloadFile.mockResolvedValue(Buffer.from('# Durable content')) + mockApplyEditToLiveFileDoc.mockResolvedValue({ applied: true, status: 'applied' }) + }) + + it('loads the committed version and reconciles it into the live document', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/workspace-1/file.md', context: 'workspace' }) + ) + expect(mockApplyEditToLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + '# Durable content', + { version: VERSION.getTime() }, + expect.any(AbortSignal) + ) + }) + + it('completes a stale event without reading or regressing newer content', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: new Date(VERSION.getTime() + 1), + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).not.toHaveBeenCalled() + }) + + it('defers transient merge-lock contention for an outbox retry', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + mockApplyEditToLiveFileDoc.mockResolvedValueOnce({ + applied: false, + status: 'merge-unavailable', + }) + + await expect(handler()(PAYLOAD, context())).resolves.toEqual( + expect.objectContaining({ outcome: 'deferred' }) + ) + }) + + it('rejects malformed payloads before touching durable state', async () => { + await expect(handler()({ ...PAYLOAD, version: 0 }, context())).rejects.toThrow( + 'invalid version' + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('does not materialize files beyond the collaborative editor boundary', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 6 * 1024 * 1024, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + VERSION.getTime(), + expect.any(AbortSignal) + ) + }) + + it('invalidates a live markdown generation when the durable file changes type', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.bin', + name: 'file.bin', + type: 'application/octet-stream', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + VERSION.getTime(), + expect.any(AbortSignal) + ) + }) + + it('still invalidates after a later binary write supersedes the type-changing event', async () => { + const latestVersion = VERSION.getTime() + 1 + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.bin', + name: 'file.bin', + type: 'application/octet-stream', + sizeBytes: 100, + contentUpdatedAt: new Date(latestVersion), + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + latestVersion, + expect.any(AbortSignal) + ) + }) +}) diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts new file mode 100644 index 00000000000..b7dfe968f71 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts @@ -0,0 +1,116 @@ +import { db } from '@sim/db' +import { workspaceFiles } from '@sim/db/schema' +import { PASTE_LIMITS } from '@sim/utils/paste' +import { and, eq, isNull } from 'drizzle-orm' +import { + deferOutboxHandler, + enqueueOutboxEvent, + type OutboxHandler, + type OutboxHandlerRegistry, + processOutboxEventById, +} from '@/lib/core/outbox/service' +import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' + +export const WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT = 'workspace-file.live-doc.reconcile' + +interface WorkspaceFileLiveDocPayload { + workspaceId: string + fileId: string + version: number +} + +function parsePayload(payload: unknown): WorkspaceFileLiveDocPayload { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + throw new Error('Workspace file live-document outbox payload must be an object') + } + const candidate = payload as Partial + if (typeof candidate.workspaceId !== 'string' || candidate.workspaceId.length === 0) { + throw new Error('Workspace file live-document outbox payload is missing workspaceId') + } + if (typeof candidate.fileId !== 'string' || candidate.fileId.length === 0) { + throw new Error('Workspace file live-document outbox payload is missing fileId') + } + if ( + typeof candidate.version !== 'number' || + !Number.isSafeInteger(candidate.version) || + candidate.version <= 0 + ) { + throw new Error('Workspace file live-document outbox payload has an invalid version') + } + return candidate as WorkspaceFileLiveDocPayload +} + +const reconcileWorkspaceFileLiveDoc: OutboxHandler = async (rawPayload, context) => { + const payload = parsePayload(rawPayload) + context.signal.throwIfAborted() + const [file] = await db + .select({ + key: workspaceFiles.key, + name: workspaceFiles.originalName, + type: workspaceFiles.contentType, + sizeBytes: workspaceFiles.sizeBytes, + contentUpdatedAt: workspaceFiles.contentUpdatedAt, + }) + .from(workspaceFiles) + .where( + and( + eq(workspaceFiles.id, payload.fileId), + eq(workspaceFiles.workspaceId, payload.workspaceId), + eq(workspaceFiles.context, 'workspace'), + isNull(workspaceFiles.deletedAt) + ) + ) + .limit(1) + + if (!file) return + const currentVersion = file.contentUpdatedAt.getTime() + if (currentVersion < payload.version) { + throw new Error('Workspace file live-document reconciliation is ahead of durable content') + } + if ( + !isMarkdownFile(file) || + file.sizeBytes === null || + file.sizeBytes > PASTE_LIMITS.RICH_MARKDOWN_BYTES + ) { + /** Later binary writes do not enqueue reconciliation, so retire the latest unsupported version. */ + await invalidateLiveFileDoc(payload.fileId, currentVersion, context.signal) + return + } + if (currentVersion > payload.version) return + + const content = await downloadFile({ + key: file.key, + context: 'workspace', + maxBytes: PASTE_LIMITS.RICH_MARKDOWN_BYTES, + signal: context.signal, + }) + context.signal.throwIfAborted() + const result = await applyEditToLiveFileDoc( + payload.fileId, + content.toString('utf-8'), + { version: payload.version }, + context.signal + ) + if (result.status === 'merge-unavailable') { + return deferOutboxHandler('Live document merge slot is temporarily unavailable') + } +} + +export const workspaceFileLiveDocOutboxHandlers = { + [WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT]: reconcileWorkspaceFileLiveDoc, +} satisfies OutboxHandlerRegistry + +/** Enqueues live-document reconciliation in the same transaction as the durable file version. */ +export function enqueueWorkspaceFileLiveDocReconciliation( + executor: Pick, + payload: WorkspaceFileLiveDocPayload +): Promise { + return enqueueOutboxEvent(executor, WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, payload) +} + +/** Attempts a newly committed reconciliation immediately; the outbox worker owns retries. */ +export function processWorkspaceFileLiveDocReconciliationNow(eventId: string) { + return processOutboxEventById(eventId, workspaceFileLiveDocOutboxHandlers) +} diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 20a99b69163..b95a58375ba 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -44,6 +44,11 @@ import { maybeNotifyStorageLimitForBillingContext, resolveStorageBillingContext, } from '@/lib/billing/storage' +import { + CollabDocStateConflictError, + type PreparedCollabDocState, + saveCollabDocStateInTx, +} from '@/lib/collab-doc/collab-state' import { normalizeVfsSegment } from '@/lib/copilot/vfs/normalize-segment' import { canonicalWorkspaceFilePath, decodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' @@ -55,8 +60,13 @@ import { acquireFolderMutationLock } from '@/lib/folders/locks' import { parseFolderPath } from '@/lib/folders/paths' import { loadActiveFolderPathIndex, resolveFolderPathFromIndex } from '@/lib/folders/queries' import type { FolderIdScope } from '@/lib/folders/scope' -import { mergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { getServePathPrefix } from '@/lib/uploads' +import type { WorkspaceFileFolderRecord } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager' +import { + enqueueWorkspaceFileLiveDocReconciliation, + processWorkspaceFileLiveDocReconciliationNow, +} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, initializeWorkspaceFileSecretProvenanceInTx, @@ -87,7 +97,6 @@ import { import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid } from '@/executor/constants' import type { UserFile } from '@/executor/types' -import type { WorkspaceFileFolderRecord } from './workspace-file-folder-manager' import { assertWorkspaceFileFolderTarget, buildWorkspaceFileFolderPathMap, @@ -1766,12 +1775,14 @@ export async function updateWorkspaceFileContent( syncLiveDoc?: boolean /** * Optimistic-concurrency guard (RFC 7232 `If-Match` semantics). When set, the write commits only - * if the file's `updatedAt` still equals this value — nothing else wrote in between; otherwise it + * if the file's `contentUpdatedAt` still equals this value — no content write intervened; otherwise it * throws {@link ContentVersionConflictError} without clobbering. Checked against the * `SELECT … FOR UPDATE`-locked row, so it is atomic with the write. Used by the collab persist so * projecting the live doc back to markdown can never silently overwrite an out-of-band edit. */ expectedUpdatedAt?: Date + /** Commit with the markdown projection; requires the expected content version. */ + collabDocState?: PreparedCollabDocState /** * Derived edits must explicitly preserve; trusted whole replacements must explicitly replace. * An omitted policy is classified as unknown rather than inheriting provenance across new bytes. @@ -1779,6 +1790,9 @@ export async function updateWorkspaceFileContent( secretProvenancePolicy?: WorkspaceFileSecretProvenancePolicy } ): Promise { + if (options?.collabDocState && !options.expectedUpdatedAt) { + throw new Error('Collaborative state updates require an expected content version') + } logger.info(`Updating workspace file content: ${fileId} for workspace ${workspaceId}`) const fileRecord = await getWorkspaceFile(workspaceId, fileId) @@ -1816,6 +1830,7 @@ export async function updateWorkspaceFileContent( oldKey: string sizeDiff: number updatedUsage: number | undefined + liveDocEventId: string | undefined } try { finalized = await db.transaction(async (tx) => { @@ -1850,6 +1865,10 @@ export async function updateWorkspaceFileContent( throw new ContentVersionConflictError(fileId) } + if (options?.collabDocState) { + await saveCollabDocStateInTx(tx, fileId, options.collabDocState) + } + const sizeDiff = content.length - getWorkspaceFileSize(currentFile) const now = new Date() // `contentUpdatedAt` is the persist If-Match token, so it MUST be strictly monotonic per file — a @@ -1926,11 +1945,23 @@ export async function updateWorkspaceFileContent( ) } + const liveDocEventId = + options?.syncLiveDoc !== false && + (isMarkdownFile({ type: currentFile.contentType, name: currentFile.originalName }) || + isMarkdownFile({ type: updatedFile.contentType, name: updatedFile.originalName })) + ? await enqueueWorkspaceFileLiveDocReconciliation(tx, { + workspaceId, + fileId, + version: updatedFile.contentUpdatedAt.getTime(), + }) + : undefined + return { file: updatedFile, oldKey: currentFile.key, sizeDiff, updatedUsage, + liveDocEventId, } }) } catch (finalizationError) { @@ -1949,22 +1980,25 @@ export async function updateWorkspaceFileContent( await cleanupWorkspaceStorageObject(finalized.oldKey, 'version replacement') } - // Stream this write into any open collaborative editor as a CRDT merge, so a copilot/tool edit - // shows up live instead of the file silently changing underneath the reader. Gated to markdown (the - // only format the collaborative editor renders) and best-effort (a no-op when nobody has the file - // open; never throws). This is the single chokepoint every external writer shares — the relay's own - // persist and empty-shell creates pass `syncLiveDoc: false` to stay out of it. - if ( - options?.syncLiveDoc !== false && - isMarkdownFile({ type: nextContentType, name: finalized.file.originalName }) - ) { - // Pass the new CONTENT version this write produced, so the relay records that its live doc now - // incorporates this durable version — the collab persist's optimistic-concurrency guard then won't - // treat this (already-merged) write as an out-of-band conflict. Must be the SAME field the CAS - // guards on (`contentUpdatedAt`), not `updatedAt`, or the relay's token wouldn't match the CAS. - await mergeEditIntoLiveFileDoc(fileId, content.toString('utf-8'), { - version: finalized.file.contentUpdatedAt.getTime(), - }) + if (finalized.liveDocEventId) { + try { + const result = await processWorkspaceFileLiveDocReconciliationNow(finalized.liveDocEventId) + if (result !== 'completed') { + logger.warn('Live document reconciliation deferred to outbox retry', { + workspaceId, + fileId, + eventId: finalized.liveDocEventId, + result, + }) + } + } catch (error) { + logger.warn('Live document reconciliation deferred after inline processing error', { + workspaceId, + fileId, + eventId: finalized.liveDocEventId, + error: getErrorMessage(error), + }) + } } const pathPrefix = getServePathPrefix() @@ -1993,7 +2027,12 @@ export async function updateWorkspaceFileContent( // Preserve the typed conflict so callers can catch it and reconcile — it's an expected outcome of // the optimistic-concurrency guard, not a failure to wrap. The orphan upload was already cleaned up // by the inner finalization catch before it propagated here. - if (error instanceof ContentVersionConflictError) throw error + if ( + error instanceof ContentVersionConflictError || + error instanceof CollabDocStateConflictError + ) { + throw error + } // Same reasoning for an already-classified failure: a missing file and a blown storage quota are // caller-fixable outcomes that every surface maps to 404/413 by class. Re-wrapping them in a bare // Error stripped that classification and turned both into a 500. diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index e084217a435..5a080ec06df 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -2,8 +2,9 @@ * @vitest-environment node */ import { workspaceFiles } from '@sim/db/schema' -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { describeError } from '@sim/utils/errors' +import { PASTE_LIMITS } from '@sim/utils/paste' import { eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -11,6 +12,7 @@ const { mockDecrementStorageUsageForBillingContextInTx, mockDeleteFile, mockEnqueueWorkspaceFileStorageCleanup, + mockEnqueueWorkspaceFileLiveDocReconciliation, mockGetWorkspaceWithOwner, mockHasCloudStorage, mockHeadObject, @@ -20,18 +22,20 @@ const { mockLoadActiveFolderPathIndex, mockInitializeWorkspaceFileSecretProvenanceInTx, mockMaybeNotifyStorageLimitForBillingContext, - mockMergeEditIntoLiveFileDoc, mockNotifyWorkspaceFilesChanged, mockProcessWorkspaceFileStorageCleanupNow, + mockProcessWorkspaceFileLiveDocReconciliationNow, mockResolveStorageBillingContext, mockResolveFolderPathFromIndex, mockResolveWorkspaceFileFolderTarget, mockReplaceWorkspaceFileSecretProvenanceInTx, + mockSaveCollabDocStateInTx, mockUploadFile, } = vi.hoisted(() => ({ mockDecrementStorageUsageForBillingContextInTx: vi.fn(), mockDeleteFile: vi.fn(), mockEnqueueWorkspaceFileStorageCleanup: vi.fn(), + mockEnqueueWorkspaceFileLiveDocReconciliation: vi.fn(), mockGetWorkspaceWithOwner: vi.fn(), mockHasCloudStorage: vi.fn(), mockHeadObject: vi.fn(), @@ -41,16 +45,22 @@ const { mockLoadActiveFolderPathIndex: vi.fn(), mockInitializeWorkspaceFileSecretProvenanceInTx: vi.fn(), mockMaybeNotifyStorageLimitForBillingContext: vi.fn(), - mockMergeEditIntoLiveFileDoc: vi.fn(), mockNotifyWorkspaceFilesChanged: vi.fn(), mockProcessWorkspaceFileStorageCleanupNow: vi.fn(), + mockProcessWorkspaceFileLiveDocReconciliationNow: vi.fn(), mockResolveStorageBillingContext: vi.fn(), mockResolveFolderPathFromIndex: vi.fn(), mockResolveWorkspaceFileFolderTarget: vi.fn(), mockReplaceWorkspaceFileSecretProvenanceInTx: vi.fn(), + mockSaveCollabDocStateInTx: vi.fn(), mockUploadFile: vi.fn(), })) +vi.mock('@/lib/collab-doc/collab-state', () => ({ + CollabDocStateConflictError: class extends Error {}, + saveCollabDocStateInTx: mockSaveCollabDocStateInTx, +})) + vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE: { status: 'exact', entries: [] }, initializeWorkspaceFileSecretProvenanceInTx: mockInitializeWorkspaceFileSecretProvenanceInTx, @@ -59,10 +69,14 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () })) vi.mock('@/lib/realtime/notify', () => ({ - mergeEditIntoLiveFileDoc: mockMergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged: mockNotifyWorkspaceFilesChanged, })) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox', () => ({ + enqueueWorkspaceFileLiveDocReconciliation: mockEnqueueWorkspaceFileLiveDocReconciliation, + processWorkspaceFileLiveDocReconciliationNow: mockProcessWorkspaceFileLiveDocReconciliationNow, +})) + vi.mock('@/lib/billing/storage', () => ({ decrementStorageUsageForBillingContextInTx: mockDecrementStorageUsageForBillingContextInTx, incrementStorageUsageForBillingContextInTx: mockIncrementStorageUsageForBillingContextInTx, @@ -111,6 +125,10 @@ vi.mock('@/lib/workspaces/permissions/utils', () => ({ getWorkspaceWithOwner: mockGetWorkspaceWithOwner, })) +import { + CollabDocStateConflictError, + type PreparedCollabDocState, +} from '@/lib/collab-doc/collab-state' import { ContentVersionConflictError, deleteWorkspaceFile, @@ -166,10 +184,12 @@ describe('workspace file metadata and storage accounting', () => { mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined) mockDeleteFile.mockResolvedValue(undefined) mockEnqueueWorkspaceFileStorageCleanup.mockResolvedValue('cleanup-event-1') - mockMergeEditIntoLiveFileDoc.mockResolvedValue(undefined) + mockEnqueueWorkspaceFileLiveDocReconciliation.mockResolvedValue('live-doc-event-1') mockNotifyWorkspaceFilesChanged.mockResolvedValue(undefined) mockProcessWorkspaceFileStorageCleanupNow.mockResolvedValue('completed') + mockProcessWorkspaceFileLiveDocReconciliationNow.mockResolvedValue('completed') mockReplaceWorkspaceFileSecretProvenanceInTx.mockResolvedValue(undefined) + mockSaveCollabDocStateInTx.mockResolvedValue(undefined) }) it('returns the canonical inserted record with the pre-resolved folder path', async () => { @@ -739,8 +759,231 @@ describe('workspace file metadata and storage accounting', () => { }) const MD_ROW = { ...FILE_ROW, originalName: 'note.md', contentType: 'text/markdown' } + const PREPARED_COLLAB_STATE: PreparedCollabDocState = { + docState: new Uint8Array([1, 2, 3]), + sourceHash: 'new-markdown-hash', + expectedState: { sourceHash: 'previous-markdown-hash', stateHash: 'previous-state-hash' }, + } + + it('commits the prepared collab state in the locked content and accounting transaction', async () => { + const transaction = { ...dbChainMock.db } + const replacementKey = `${MD_ROW.key}-replacement` + const content = Buffer.from('# new content') + const updatedFile = { ...MD_ROW, key: replacementKey, sizeBytes: content.length } + let committed = false + dbChainMockFns.transaction.mockImplementationOnce(async (callback) => { + const result = await callback(transaction) + expect(mockSaveCollabDocStateInTx).toHaveBeenCalledWith( + transaction, + MD_ROW.id, + PREPARED_COLLAB_STATE + ) + expect(mockIncrementStorageUsageForBillingContextInTx).toHaveBeenCalledWith( + transaction, + STORAGE_CONTEXT, + content.length - MD_ROW.sizeBytes + ) + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith( + transaction, + expect.objectContaining({ fileId: MD_ROW.id }) + ) + expect(mockDeleteFile).not.toHaveBeenCalled() + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).not.toHaveBeenCalled() + committed = true + return result + }) + mockDeleteFile.mockImplementationOnce(async () => { + expect(committed).toBe(true) + }) + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: replacementKey }) + + await expect( + updateWorkspaceFileContent(MD_ROW.workspaceId, MD_ROW.id, MD_ROW.userId, content, undefined, { + expectedUpdatedAt: MD_ROW.contentUpdatedAt, + collabDocState: PREPARED_COLLAB_STATE, + }) + ).resolves.toMatchObject({ key: replacementKey }) + + expect(mockSaveCollabDocStateInTx).toHaveBeenCalledOnce() + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + expect(mockUploadFile.mock.invocationCallOrder[0]).toBeLessThan( + dbChainMockFns.transaction.mock.invocationCallOrder[0] + ) + expect(dbChainMockFns.limit.mock.invocationCallOrder[1]).toBeLessThan( + mockSaveCollabDocStateInTx.mock.invocationCallOrder[0] + ) + expect(mockSaveCollabDocStateInTx.mock.invocationCallOrder[0]).toBeLessThan( + dbChainMockFns.update.mock.invocationCallOrder[0] + ) + expect(mockDeleteFile).toHaveBeenCalledWith({ key: MD_ROW.key, context: 'workspace' }) + }) + + it.each([ + { expectedState: PREPARED_COLLAB_STATE.expectedState, cleanupFails: false }, + { expectedState: PREPARED_COLLAB_STATE.expectedState, cleanupFails: true }, + { expectedState: null, cleanupFails: false }, + { expectedState: null, cleanupFails: true }, + ])( + 'aborts finalization on a collab-state conflict and cleans only the staged blob: %j', + async ({ expectedState, cleanupFails }) => { + const transaction = { ...dbChainMock.db } + const replacementKey = `${MD_ROW.key}-replacement` + const conflict = new CollabDocStateConflictError(MD_ROW.id) + const preparedState = { ...PREPARED_COLLAB_STATE, expectedState } + let rolledBack = false + dbChainMockFns.transaction.mockImplementationOnce(async (callback) => { + try { + return await callback(transaction) + } catch (error) { + rolledBack = true + throw error + } + }) + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + mockUploadFile.mockResolvedValueOnce({ key: replacementKey }) + mockSaveCollabDocStateInTx.mockRejectedValueOnce(conflict) + mockDeleteFile.mockImplementationOnce(async () => { + expect(rolledBack).toBe(true) + if (cleanupFails) throw new Error('storage unavailable') + }) + + await expect( + updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content'), + undefined, + { expectedUpdatedAt: MD_ROW.contentUpdatedAt, collabDocState: preparedState } + ) + ).rejects.toBe(conflict) + + expect(mockSaveCollabDocStateInTx).toHaveBeenCalledWith(transaction, MD_ROW.id, preparedState) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(mockReplaceWorkspaceFileSecretProvenanceInTx).not.toHaveBeenCalled() + expect(mockIncrementStorageUsageForBillingContextInTx).not.toHaveBeenCalled() + expect(mockDecrementStorageUsageForBillingContextInTx).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).not.toHaveBeenCalled() + expect(mockMaybeNotifyStorageLimitForBillingContext).not.toHaveBeenCalled() + expect(mockDeleteFile).toHaveBeenCalledExactlyOnceWith({ + key: replacementKey, + context: 'workspace', + }) + } + ) + + it('rejects prepared collab state without a content version before any I/O', async () => { + await expect( + updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content'), + undefined, + { collabDocState: PREPARED_COLLAB_STATE } + ) + ).rejects.toThrow('Collaborative state updates require an expected content version') + + expect(dbChainMockFns.select).not.toHaveBeenCalled() + expect(mockUploadFile).not.toHaveBeenCalled() + expect(mockSaveCollabDocStateInTx).not.toHaveBeenCalled() + }) + + it('does not accept the prepared collab state before validating the locked content version', async () => { + const replacementKey = `${MD_ROW.key}-replacement` + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + mockUploadFile.mockResolvedValueOnce({ key: replacementKey }) + + await expect( + updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# stale content'), + undefined, + { + expectedUpdatedAt: new Date('2020-01-01T00:00:00.000Z'), + collabDocState: PREPARED_COLLAB_STATE, + } + ) + ).rejects.toBeInstanceOf(ContentVersionConflictError) + + expect(mockSaveCollabDocStateInTx).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(mockDeleteFile).toHaveBeenCalledExactlyOnceWith({ + key: replacementKey, + context: 'workspace', + }) + }) + + it('rejects the shared transaction when accounting fails after accepting the collab state', async () => { + const transaction = { ...dbChainMock.db } + const replacementKey = `${MD_ROW.key}-replacement` + let committed = false + dbChainMockFns.transaction.mockImplementationOnce(async (callback) => { + const result = await callback(transaction) + committed = true + return result + }) + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...MD_ROW, key: replacementKey, sizeBytes: 13 }, + ]) + mockUploadFile.mockResolvedValueOnce({ key: replacementKey }) + mockIncrementStorageUsageForBillingContextInTx.mockRejectedValueOnce( + new Error('accounting unavailable') + ) + + await expect( + updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.from('# new content'), + undefined, + { expectedUpdatedAt: MD_ROW.contentUpdatedAt, collabDocState: PREPARED_COLLAB_STATE } + ) + ).rejects.toThrow('accounting unavailable') - it('streams a markdown overwrite into any open collaborative editor (the shared merge chokepoint)', async () => { + expect(committed).toBe(false) + expect(mockSaveCollabDocStateInTx).toHaveBeenCalledWith( + transaction, + MD_ROW.id, + PREPARED_COLLAB_STATE + ) + expect(mockIncrementStorageUsageForBillingContextInTx).toHaveBeenCalledWith( + transaction, + STORAGE_CONTEXT, + 8 + ) + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() + expect(mockMaybeNotifyStorageLimitForBillingContext).not.toHaveBeenCalled() + expect(mockDeleteFile).toHaveBeenCalledExactlyOnceWith({ + key: replacementKey, + context: 'workspace', + }) + }) + + it('transactionally enqueues a markdown overwrite for live-document reconciliation', async () => { + const transaction = { ...dbChainMock.db } + let committed = false + dbChainMockFns.transaction.mockImplementationOnce(async (callback) => { + const result = await callback(transaction) + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith( + transaction, + expect.objectContaining({ fileId: MD_ROW.id }) + ) + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).not.toHaveBeenCalled() + committed = true + return result + }) + mockProcessWorkspaceFileLiveDocReconciliationNow.mockImplementationOnce(async () => { + expect(committed).toBe(true) + return 'completed' + }) // Distinct updatedAt vs contentUpdatedAt so the assertion proves the merge carries the CONTENT // version (the persist If-Match token), not `updatedAt` — reverting that wiring would fail here. const updatedFile = { @@ -761,9 +1004,15 @@ describe('workspace file metadata and storage accounting', () => { Buffer.from('# new content', 'utf-8') ) - expect(mockMergeEditIntoLiveFileDoc).toHaveBeenCalledWith(MD_ROW.id, '# new content', { + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(transaction, { + workspaceId: MD_ROW.workspaceId, + fileId: MD_ROW.id, version: updatedFile.contentUpdatedAt.getTime(), }) + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).toHaveBeenCalledWith( + 'live-doc-event-1' + ) + expect(mockSaveCollabDocStateInTx).not.toHaveBeenCalled() }) it('does NOT merge when syncLiveDoc is false (the relay persist / empty-shell opt-out)', async () => { @@ -781,7 +1030,8 @@ describe('workspace file metadata and storage accounting', () => { { syncLiveDoc: false } ) - expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() + expect(mockSaveCollabDocStateInTx).not.toHaveBeenCalled() }) it('does NOT merge a non-markdown write (the collaborative editor only renders markdown)', async () => { @@ -798,7 +1048,57 @@ describe('workspace file metadata and storage accounting', () => { 'application/octet-stream' ) - expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() + }) + + it('enqueues an oversized markdown write so an older live generation is invalidated', async () => { + const size = PASTE_LIMITS.RICH_MARKDOWN_BYTES + 1 + const updatedFile = { ...MD_ROW, size, sizeBytes: size } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.alloc(size) + ) + + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: MD_ROW.workspaceId, + fileId: MD_ROW.id, + version: updatedFile.contentUpdatedAt.getTime(), + }) + }) + + it('enqueues a markdown-to-binary replacement so an older live generation is invalidated', async () => { + const markdownByType = { ...MD_ROW, originalName: 'note.txt' } + const updatedFile = { + ...markdownByType, + contentType: 'application/octet-stream', + size: 12, + sizeBytes: 12, + } + dbChainMockFns.limit + .mockResolvedValueOnce([markdownByType]) + .mockResolvedValueOnce([markdownByType]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: markdownByType.key }) + + await updateWorkspaceFileContent( + markdownByType.workspaceId, + markdownByType.id, + markdownByType.userId, + Buffer.alloc(12), + 'application/octet-stream' + ) + + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: markdownByType.workspaceId, + fileId: markdownByType.id, + version: updatedFile.contentUpdatedAt.getTime(), + }) }) it('writes when the expectedUpdatedAt optimistic-concurrency guard matches', async () => { diff --git a/packages/realtime-protocol/src/file-doc.test.ts b/packages/realtime-protocol/src/file-doc.test.ts index 48fd7db5b8a..ca1d8a495d1 100644 --- a/packages/realtime-protocol/src/file-doc.test.ts +++ b/packages/realtime-protocol/src/file-doc.test.ts @@ -9,5 +9,7 @@ describe('FILE_DOC_TIMEOUTS ordering invariants', () => { // The relay's `/seed` fetch must finish before the client's readiness deadline lapses into its // read-only fallback, or a late-but-successful seed can never reach the client. expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.joinAckMs) + expect(FILE_DOC_TIMEOUTS.joinAckMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) }) }) diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index dde704bee73..c1cef7c4398 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -25,6 +25,13 @@ export const FILE_DOC_EVENTS = { LEAVE: 'leave-file-doc', /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ MESSAGE: 'file-doc-message', + /** + * Client → server: one idempotent batch of user-authored Yjs updates. Unlike the handshake and + * awareness channel, this event is acknowledged only after the shared stream accepts the batch. + */ + UPDATE: 'file-doc-update', + /** Server → client: the durable file was replaced outside this live document's generation. */ + INVALIDATED: 'file-doc-invalidated', /** * Server → client: the roster of collaborators currently in the document * ({@link FileDocPresence}), for the avatar stack. Identity is server-authenticated (from @@ -34,6 +41,11 @@ export const FILE_DOC_EVENTS = { PRESENCE: 'file-doc-presence', } as const +/** Schema assumed for peers from before schema negotiation was added. */ +export const FILE_DOC_LEGACY_SCHEMA_VERSION = 1 + +export const FILE_DOC_SCHEMA_VERSION = 1 + /** * The tag carried in the first varUint of a {@link FILE_DOC_EVENTS.MESSAGE} * payload — the standard Yjs websocket framing distinguishing a document-sync @@ -67,8 +79,8 @@ export const FILE_DOC_MESSAGE_TYPE = { * 1. **Never overwrite content with an unseeded doc.** The markdown-mirror autosave MUST be gated on * the document being both synced AND seeded — otherwise an empty/still-syncing doc could be saved * over the real file (the one true data-loss path). - * 2. **One provider per socket.** Destroy the previous provider before creating the next (document - * switch), so a stale provider's binary-frame listener can't apply another document's updates. + * 2. **One active file per shared socket.** Multiple providers may show the same file, but opening a + * different file must make older providers terminal before its unscoped binary frames can arrive. * 3. **Treat a fatal (`retryable: false`) join error as terminal.** Latch it and fall back to a * read-only view of the file's stored content — do not keep rejoining. The server auto-reclaims a * same-user client-id collision silently (the reconnecting socket succeeds), so `CLIENT_ID_IN_USE` @@ -121,10 +133,17 @@ export const FILE_DOC_TIMEOUTS = { seedRequestMs: 8_000, mergeRequestMs: 3_000, applyEditMs: 6_000, + joinAckMs: 10_000, + updateAckMs: 6_000, readinessDeadlineMs: 12_000, persistRequestMs: 8_000, } as const +export const FILE_DOC_LIMITS = { + /** Leaves framing and acknowledgement headroom under Socket.IO's 8 MiB event ceiling. */ + updateBytes: 6 * 1024 * 1024, +} as const + /** Client → server join request. `fileId` is the `workspace_files.id`. */ export interface JoinFileDocPayload { fileId: string @@ -134,6 +153,8 @@ export interface JoinFileDocPayload { * client — an authenticated peer cannot forge or clear another's presence. */ clientId: number + /** Optional during rolling deploys; absent peers use the original version-1 schema. */ + schemaVersion?: number } /** Server → client acceptance of a {@link FILE_DOC_EVENTS.JOIN}. */ @@ -141,6 +162,10 @@ export interface JoinFileDocSuccess { fileId: string /** The provider whose join was accepted. Optional while older relays are still deployed. */ clientId?: number + /** Whether this relay durably acknowledges client updates. Absent on older relays. */ + acknowledgedUpdates?: true + /** Durable version incorporated by the admitted generation; absent on older relays. */ + version?: number /** * The identity of the document this room holds ({@link FILE_DOC_SEED.docIdKey}), so a client can tell * "the room I left" from "a document built in its place" BEFORE it syncs. Absent for a room whose doc @@ -148,6 +173,8 @@ export interface JoinFileDocSuccess { * exactly the case where there is nothing to compare and the client proceeds. */ docId?: string + /** Optional while older relays are still deployed. */ + schemaVersion?: number } /** Server → client rejection of a {@link FILE_DOC_EVENTS.JOIN}. */ @@ -165,6 +192,38 @@ export interface LeaveFileDocPayload { fileId: string } +/** Server → client invalidation after a durable replacement that cannot merge into the rich editor. */ +export interface FileDocInvalidated { + fileId: string + message: string + /** Generation atomically removed by this invalidation, when one existed. */ + docId?: string + /** Durable replacement version; absent only on older relays. */ + version?: number +} + +/** A bounded, retry-safe batch of user-authored changes. */ +export interface FileDocUpdatePayload { + fileId: string + docId: string + updateId: string + update: Uint8Array +} + +export type FileDocUpdateAck = + | { status: 'accepted'; updateId: string } + | { + status: 'rejected' + updateId?: string + code: + | 'ACCESS_REVOKED' + | 'DOCUMENT_REPLACED' + | 'INVALID_UPDATE' + | 'NOT_JOINED' + | 'TEMPORARY_FAILURE' + retryable: boolean + } + /** One collaborator session in a {@link FileDocPresence} roster — server-authenticated identity. * Keyed per socket (session), not per user: the client excludes its OWN `socketId` and then * dedupes the rest per user for the avatar stack, so a second tab of the same account still