diff --git a/apps/docs/openapi-v2-files-audit.json b/apps/docs/openapi-v2-files-audit.json index 2ae1f1de435..494974f0965 100644 --- a/apps/docs/openapi-v2-files-audit.json +++ b/apps/docs/openapi-v2-files-audit.json @@ -3167,7 +3167,7 @@ "uploadedByEmail": { "type": "string", "format": "email", - "pattern": "^(?!\\.)(?!.*\\.\\.)([A-Za-z0-9_'+\\-\\.]*)[A-Za-z0-9_+-]@([A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$", + "pattern": "^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$", "description": "Current email address of the uploader.", "examples": ["jane@example.com"] }, @@ -4029,7 +4029,7 @@ "uploadedByEmail": { "type": "string", "format": "email", - "pattern": "^(?!\\.)(?!.*\\.\\.)([A-Za-z0-9_'+\\-\\.]*)[A-Za-z0-9_+-]@([A-Za-z0-9][A-Za-z0-9\\-]*\\.)+[A-Za-z]{2,}$", + "pattern": "^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$", "description": "Current email address of the uploader.", "examples": ["jane@example.com"] }, diff --git a/apps/realtime/src/handlers/connection.test.ts b/apps/realtime/src/handlers/connection.test.ts new file mode 100644 index 00000000000..26018de95f0 --- /dev/null +++ b/apps/realtime/src/handlers/connection.test.ts @@ -0,0 +1,73 @@ +/** + * @vitest-environment node + */ +import { createServer, type Server as HttpServer } from 'node:http' +import type { AddressInfo } from 'node:net' +import { Server } from 'socket.io' +import { io as connect, type Socket } from 'socket.io-client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { setupConnectionHandlers, waitForConnectionCleanup } from '@/handlers/connection' +import type { AuthenticatedSocket } from '@/middleware/auth' +import { MemoryRoomManager } from '@/rooms' + +vi.mock('@/handlers/file-doc', () => ({ cleanupFileDocForSocket: vi.fn() })) +vi.mock('@/handlers/subblocks', () => ({ cleanupPendingSubblocksForSocket: vi.fn() })) +vi.mock('@/handlers/variables', () => ({ cleanupPendingVariablesForSocket: vi.fn() })) + +describe('server shutdown connection drain', () => { + let httpServer: HttpServer + let io: Server + let manager: MemoryRoomManager + let client: Socket + + beforeEach(async () => { + httpServer = createServer() + io = new Server(httpServer, { transports: ['websocket'] }) + manager = new MemoryRoomManager(io) + await manager.initialize() + io.on('connection', (socket) => setupConnectionHandlers(socket as AuthenticatedSocket, manager)) + await new Promise((resolve) => httpServer.listen(0, '127.0.0.1', resolve)) + const port = (httpServer.address() as AddressInfo).port + client = connect(`http://127.0.0.1:${port}`, { transports: ['websocket'], autoConnect: false }) + const connected = new Promise((resolve) => client.once('connect', resolve)) + client.connect() + await connected + }) + + afterEach(async () => { + client.disconnect() + await io.close() + await waitForConnectionCleanup() + await manager.shutdown() + vi.restoreAllMocks() + }) + + it('keeps automatic reconnection active after transport shutdown', async () => { + const disconnected = new Promise((resolve) => client.once('disconnect', resolve)) + await io.close() + expect(await disconnected).toBe('transport close') + expect(client.active).toBe(true) + await waitForConnectionCleanup() + }) + + it('waits for asynchronous presence cleanup before releasing its dependencies', async () => { + let finishRemoval: (() => void) | undefined + vi.spyOn(manager, 'removeSocketFromAllRooms').mockImplementation( + () => + new Promise((resolve) => { + finishRemoval = () => resolve([]) + }) + ) + await io.close() + let drained = false + const drain = waitForConnectionCleanup().then(() => { + drained = true + }) + await Promise.resolve() + expect(drained).toBe(false) + expect(finishRemoval).toBeDefined() + finishRemoval?.() + await drain + expect(drained).toBe(true) + }) +}) diff --git a/apps/realtime/src/handlers/connection.ts b/apps/realtime/src/handlers/connection.ts index 33d90b5bfb0..fcbdaade40f 100644 --- a/apps/realtime/src/handlers/connection.ts +++ b/apps/realtime/src/handlers/connection.ts @@ -16,6 +16,13 @@ const logger = createLogger('ConnectionHandlers') */ const PRESENCE_BEARING_TYPES = new Set([ROOM_TYPES.WORKFLOW, ROOM_TYPES.TABLE]) +const pendingDisconnects = new Set>() + +/** Keep Redis available until disconnect listeners finish removing presence. */ +export async function waitForConnectionCleanup(): Promise { + await Promise.all(pendingDisconnects) +} + export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager: IRoomManager) { socket.on('error', (error) => { logger.error(`Socket ${socket.id} error:`, error) @@ -28,7 +35,7 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager // `disconnecting` (not `disconnect`): here `socket.rooms` is still populated and // authoritative, so presence is cleaned up even if the Redis room-set key was // evicted or TTL-expired (which would leave the manager's stored rooms empty). - socket.on('disconnecting', async (reason) => { + const handleDisconnect = async (reason: string) => { try { // Snapshot the live Socket.IO room membership SYNCHRONOUSLY, before any // await: Socket.IO clears `socket.rooms` via leaveAll() as soon as the @@ -91,5 +98,11 @@ export function setupConnectionHandlers(socket: AuthenticatedSocket, roomManager } catch (error) { logger.error(`Error handling disconnect for socket ${socket.id}:`, error) } + } + + socket.on('disconnecting', (reason) => { + const cleanup = handleDisconnect(reason) + pendingDisconnects.add(cleanup) + void cleanup.finally(() => pendingDisconnects.delete(cleanup)) }) } diff --git a/apps/realtime/src/handlers/file-doc-store.test.ts b/apps/realtime/src/handlers/file-doc-store.test.ts index f5159c10cbc..673ba9472ad 100644 --- a/apps/realtime/src/handlers/file-doc-store.test.ts +++ b/apps/realtime/src/handlers/file-doc-store.test.ts @@ -13,7 +13,10 @@ import * as Y from 'yjs' interface Backing { streams: Map }[]> kv: Map + dedupe: Map seq: number + /** Override generated IDs to model a recreated stream restarting its same-millisecond sequence. */ + nextIds?: string[] /** Number of upcoming xAdd calls to fail with a transient error (to exercise publish retry). */ failXAdd: number /** Set to fail every xRead the way node-redis does once a client has been closed. */ @@ -26,17 +29,31 @@ interface Backing { idleReads: number /** `connect()` calls, so a test can prove a closed reader is re-opened rather than abandoned. */ connects: number + /** Largest stream range response requested, proving replay is paginated. */ + maxRangeCount: number + /** Optional deterministic compaction hook invoked before each range page is read. */ + onRange?: (call: number, key: string, start: string) => void + rangeCalls: number + /** Largest multiplexed XREAD request and COUNT observed. */ + maxReadStreams: number + maxReadCount: number + onSnapshot?: () => Promise } const state = vi.hoisted(() => ({ backing: null as Backing | null })) -const seqOf = (id: string) => Number(id.split('-')[0]) +function compareStreamIds(left: string, right: string): bigint { + const [leftMs, leftSequence] = left.split('-').map(BigInt) + const [rightMs, rightSequence] = right.split('-').map(BigInt) + return leftMs === rightMs ? leftSequence - rightSequence : leftMs - rightMs +} function makeClient(): any { const b = () => { if (!state.backing) throw new Error('backing not initialized') return state.backing } + const nextId = () => b().nextIds?.shift() ?? `${++b().seq}-0` const client: any = { isOpen: true, connect: async () => { @@ -51,23 +68,45 @@ function makeClient(): any { b().failXAdd-- throw new Error('transient xAdd failure') } - const id = `${++b().seq}-0` + const id = nextId() const arr = b().streams.get(key) ?? [] arr.push({ id, message: { ...fields } }) b().streams.set(key, arr) return id }, - xRange: async (key: string) => (b().streams.get(key) ?? []).map((e) => ({ ...e })), + xRange: async (key: string, start: string, end: string, options?: { COUNT?: number }) => { + b().rangeCalls++ + b().onRange?.(b().rangeCalls, key, start) + const startId = start.startsWith('(') ? start.slice(1) : start + const entries = (b().streams.get(key) ?? []).filter( + (entry) => + (start === '-' || compareStreamIds(entry.id, startId) > 0n) && + (end === '+' || compareStreamIds(entry.id, end) <= 0n) + ) + const count = options?.COUNT ?? entries.length + b().maxRangeCount = Math.max(b().maxRangeCount, count) + return entries.slice(0, count).map((entry) => ({ ...entry })) + }, + xRevRange: async (key: string, _start: string, _end: string, options?: { COUNT?: number }) => + [...(b().streams.get(key) ?? [])] + .reverse() + .slice(0, options?.COUNT) + .map((entry) => ({ ...entry })), xLen: async (key: string) => (b().streams.get(key) ?? []).length, xTrim: async (key: string, _strategy: string, minid: string) => { const arr = b().streams.get(key) ?? [] b().streams.set( key, - arr.filter((e) => seqOf(e.id) >= seqOf(minid)) + arr.filter((e) => compareStreamIds(e.id, minid) >= 0n) ) }, - xRead: async (streams: { key: string; id: string }[]) => { + xRead: async ( + streams: { key: string; id: string }[], + options?: { BLOCK?: number; COUNT?: number } + ) => { b().reads++ + b().maxReadStreams = Math.max(b().maxReadStreams, streams.length) + b().maxReadCount = Math.max(b().maxReadCount, options?.COUNT ?? 0) if (b().readerClosed) { b().failedReadTimes.push(Date.now()) client.isOpen = false @@ -76,7 +115,9 @@ function makeClient(): any { const res: { name: string; messages: { id: string; message: Record }[] }[] = [] for (const { key, id } of streams) { - const after = (b().streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + const after = (b().streams.get(key) ?? []) + .filter((e) => compareStreamIds(e.id, id) > 0n) + .slice(0, options?.COUNT) if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) } if (res.length) { @@ -92,24 +133,128 @@ function makeClient(): any { b().kv.set(key, val) return 'OK' }, - del: async (key: string) => { - b().kv.delete(key) - return 1 + get: async (key: string) => b().kv.get(key) ?? null, + del: async (keys: string | string[]) => { + const targets = Array.isArray(keys) ? keys : [keys] + for (const key of targets) { + b().kv.delete(key) + b().streams.delete(key) + b().dedupe.delete(key) + } + return targets.length }, eval: async (script: string, opts: { keys: string[]; arguments: string[] }) => { const [key] = opts.keys + if (script.startsWith('for _, key in ipairs(KEYS)')) return 1 + if (script.includes("redis.call('exists', KEYS[1])") && !b().streams.has(key)) { + return script.includes('zscore') ? -1 : false + } + if (script.includes('return ARGV[1]')) { + const generation = b().kv.get(opts.keys[1]) + if (generation !== undefined) return generation + if (!b().streams.get(key)?.length) return false + b().kv.set(opts.keys[1], opts.arguments[0]) + return opts.arguments[0] + } + if (script.includes("redis.call('del', KEYS[1], KEYS[4], KEYS[5])")) { + const [, generationKey, versionKey, dedupeKey, agentKey, invalidationKey] = opts.keys + const [version, , marker] = opts.arguments + const current = b().kv.get(versionKey) + const invalidated = b().kv.get(invalidationKey) + if (invalidated && Number(invalidated) >= Number(version)) return null + if (current && Number(current) > Number(version)) return null + if (current === version && b().kv.get(generationKey) === marker) return null + const generation = b().kv.get(generationKey) ?? '' + b().kv.set(generationKey, marker) + b().kv.set(versionKey, version) + b().kv.set(invalidationKey, version) + b().streams.delete(key) + b().dedupe.delete(dedupeKey) + b().kv.delete(agentKey) + return generation + } + if (script.includes('zscore')) { + const [, dedupeKey, generationKey] = opts.keys + const [member, field, value, capacityText, , expectedGeneration] = opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return -1 + const members = b().dedupe.get(dedupeKey) ?? [] + if (members.includes(member)) return 0 + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value } }) + b().streams.set(key, arr) + members.push(member) + const capacity = Number(capacityText) + if (members.length > capacity) members.splice(0, members.length - capacity) + b().dedupe.set(dedupeKey, members) + return 1 + } // Atomic seed-if-empty (SEED_IF_EMPTY_SCRIPT): append the entry iff the stream is empty, in one // synchronous step — mirroring Redis's atomic Lua execution, so two concurrent evals can never both // append (the second sees a non-empty stream). if (script.includes('xlen')) { - const [field, value] = opts.arguments + const [, generationKey, versionKey] = opts.keys + const [field, value, generation, , generationField, version] = opts.arguments + if (Number(b().kv.get(versionKey) ?? 0) > Number(version)) return 0 const arr = b().streams.get(key) ?? [] if (arr.length > 0) return 0 - const id = `${++b().seq}-0` - arr.push({ id, message: { [field]: value } }) + b().kv.set(generationKey, generation) + if (version !== '0') b().kv.set(versionKey, version) + const id = nextId() + arr.push({ id, message: { [field]: value, [generationField]: generation } }) b().streams.set(key, arr) return 1 } + if (script.includes('ARGV[5], ARGV[4]')) { + const [, generationKey] = opts.keys + const [field, value, marker, expectedGeneration, generationField, upTo] = opts.arguments + const generation = b().kv.get(generationKey) + if ((generation ?? '') !== expectedGeneration) return false + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ + id, + message: { + [field]: value, + [marker]: '1', + [generationField]: expectedGeneration, + }, + }) + b().streams.set(key, arr) + if (script.includes("redis.call('xtrim'")) { + b().streams.set( + key, + arr.filter((entry) => compareStreamIds(entry.id, upTo) >= 0n) + ) + } + await b().onSnapshot?.() + return id + } + if (script.includes("ARGV[3] ~= ''")) { + const [, generationKey] = opts.keys + const generation = b().kv.get(generationKey) + const expectedGeneration = opts.arguments[3] + if ((generation ?? '') !== expectedGeneration) return false + if (b().failXAdd > 0) { + b().failXAdd-- + throw new Error('transient xAdd failure') + } + const [field, value, marker] = opts.arguments + const id = nextId() + const arr = b().streams.get(key) ?? [] + arr.push({ id, message: { [field]: value, ...(marker ? { [marker]: '1' } : {}) } }) + b().streams.set(key, arr) + return id + } + if (script.includes('tonumber(c)')) { + const [value, , expectedGeneration] = opts.arguments + const generation = b().kv.get(opts.keys[1]) + if ((generation ?? '') !== expectedGeneration) return 0 + const current = b().kv.get(key) + if (current === undefined || Number(current) < Number(value)) b().kv.set(key, value) + return 1 + } // Compare-and-delete Lua (RELEASE_LOCK_SCRIPT): del only if the stored value matches the token. const [token] = opts.arguments if (b().kv.get(key) === token) { @@ -130,6 +275,28 @@ import { FileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN } from '@/handlers/file- const REDIS_URL = 'redis://fake' const NAME = 'workspace-file-doc:file-1' +interface StoreTestAccess { + localInvalidations: Map + rooms: Map< + string, + { + doc: Y.Doc + lastId: string + publishes: number + uncompactedDeltaBytes: number + compacting: boolean + seededObserved: boolean + realEdited: boolean + } + > + maybeCompact(name: string): Promise + appendUpdate(name: string, update: Uint8Array): Promise +} + +function storeInternals(store: FileDocStore): StoreTestAccess { + return store as unknown as StoreTestAccess +} + function docWithText(text: string): Y.Doc { const doc = new Y.Doc() doc.getText('body').insert(0, text) @@ -145,6 +312,15 @@ function updateFor(text: string): Uint8Array { } let stores: FileDocStore[] = [] + +/** An existing stream from a relay predating generation markers; modern seeds use seedIfEmpty. */ +function seedLegacyStream(update = updateFor('')): void { + const backing = state.backing! + backing.streams.set(`filedoc:stream:${NAME}`, [ + { id: `${++backing.seq}-0`, message: { u: Buffer.from(update).toString('base64') } }, + ]) +} + async function newStore(): Promise { const store = new FileDocStore(REDIS_URL) await store.init() @@ -157,6 +333,7 @@ describe('FileDocStore', () => { state.backing = { streams: new Map(), kv: new Map(), + dedupe: new Map(), seq: 0, failXAdd: 0, readerClosed: false, @@ -164,6 +341,10 @@ describe('FileDocStore', () => { failedReadTimes: [], idleReads: 0, connects: 0, + maxRangeCount: 0, + rangeCalls: 0, + maxReadStreams: 0, + maxReadCount: 0, } stores = [] }) @@ -264,7 +445,7 @@ describe('FileDocStore', () => { const token = await a.shouldSeed(NAME) expect(token).toBeTruthy() // A seeds and releases its lock. - a.publish(NAME, updateFor('hello')) + await a.seedIfEmpty(NAME, updateFor('hello')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) await a.releaseSeedLock(NAME, token as string) // A different task must NOT seed again — the lock is free but the stream is non-empty. @@ -272,9 +453,31 @@ describe('FileDocStore', () => { expect(await b.shouldSeed(NAME)).toBeNull() }) + it('fences stale publishers after invalidation and lets the next authoritative seed start fresh', async () => { + const store = await newStore() + const original = updateFor('old generation') + await store.seedIfEmpty(NAME, original) + await store.invalidateDocument(NAME, 10) + + await expect(store.getStreamState(NAME)).resolves.toBeNull() + await expect(store.publishAndWait(NAME, updateFor('stale write'))).rejects.toThrow( + 'replaced by a newer durable version' + ) + await expect( + store.publishClientUpdateAndWait(NAME, 'stale-update', updateFor('stale acknowledged write')) + ).rejects.toThrow('replaced by a newer durable version') + + const fresh = updateFor('fresh generation') + await expect(store.seedIfEmpty(NAME, fresh, 11)).resolves.toBe(true) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('fresh generation') + recovered.destroy() + }) + it('getStreamState reconstructs the shared document from the stream', async () => { const a = await newStore() - a.publish(NAME, updateFor('shared content')) + await a.seedIfEmpty(NAME, updateFor('shared content')) let state: Uint8Array | null = null await vi.waitFor(async () => { state = await a.getStreamState(NAME) @@ -286,9 +489,403 @@ describe('FileDocStore', () => { doc.destroy() }) + it('lets a headless replica append against the generation of its shared base', async () => { + const seeded = await newStore() + await seeded.seedIfEmpty(NAME, updateFor('shared'), 20) + const headless = await newStore() + const generation = await headless.getDocumentGeneration(NAME) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await headless.getStreamState(NAME, generation))!) + const before = Y.encodeStateVector(doc) + doc.getText('body').insert(6, ' edit') + await headless.publishAndWait(NAME, Y.encodeStateAsUpdate(doc, before), generation) + const replay = new Y.Doc() + Y.applyUpdate(replay, (await seeded.getStreamState(NAME))!) + expect(replay.getText('body').toString()).toBe('shared edit') + doc.destroy() + replay.destroy() + }) + + it('keeps a newer seeded generation when an older invalidation arrives', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('newest'), 20) + const generation = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 10)).resolves.toEqual({ status: 'stale' }) + expect(await store.getDocumentGeneration(NAME)).toBe(generation) + expect(await store.getSyncedVersion(NAME)).toBe(20) + await expect(store.getStreamState(NAME)).resolves.not.toBeNull() + }) + + it('rejects old seeds and version callbacks after an invalidation', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old'), 10) + const generation = await store.getDocumentGeneration(NAME) + await store.invalidateDocument(NAME, 20) + await expect(store.seedIfEmpty(NAME, updateFor('late stale seed'), 10)).resolves.toBe(false) + await store.setSyncedVersion(NAME, 30, generation) + expect(await store.getSyncedVersion(NAME)).toBe(20) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('does not repeat an invalidation after the same durable version is reseeded', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old'), 10) + await expect(store.invalidateDocument(NAME, 20)).resolves.toMatchObject({ status: 'applied' }) + await expect(store.seedIfEmpty(NAME, updateFor('replacement'), 20)).resolves.toBe(true) + const generation = await store.getDocumentGeneration(NAME) + const doc = new Y.Doc() + Y.applyUpdate(doc, (await store.getStreamState(NAME))!) + const before = Y.encodeStateVector(doc) + doc.getText('body').insert(11, ' accepted') + await store.publishClientUpdateAndWait( + NAME, + 'accepted-edit', + Y.encodeStateAsUpdate(doc, before), + generation + ) + + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + expect(await store.getDocumentGeneration(NAME)).toBe(generation) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('replacement accepted') + expect(state.backing!.dedupe.get(`filedoc:updates:${NAME}`)).toHaveLength(1) + doc.destroy() + recovered.destroy() + }) + + it('applies the first invalidation even when its durable version was already seeded', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('same content, changed eligibility'), 20) + const docId = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'applied', docId }) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('returns the removed generation and qualifies consecutive unsupported replacements', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old'), 10) + const oldId = await store.getDocumentGeneration(NAME) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ + status: 'applied', + docId: oldId, + }) + await expect(store.invalidateDocument(NAME, 30)).resolves.toEqual({ status: 'applied' }) + await store.seedIfEmpty(NAME, updateFor('replacement'), 30) + const replacementId = await store.getDocumentGeneration(NAME) + expect(replacementId).not.toBe(oldId) + await expect(store.invalidateDocument(NAME, 30)).resolves.toEqual({ status: 'stale' }) + await expect(store.invalidateDocument(NAME, 40)).resolves.toEqual({ + status: 'applied', + docId: replacementId, + }) + }) + + it('does not resurrect a tracked stream with a dependency-only update after Redis loses it', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('base'), 10) + const generation = await store.getDocumentGeneration(NAME) + state.backing!.streams.delete(`filedoc:stream:${NAME}`) + state.backing!.kv.delete(`filedoc:generation:${NAME}`) + await expect(store.publishAndWait(NAME, updateFor('stale'), generation)).rejects.toThrow( + 'replaced' + ) + await expect( + store.publishClientUpdateAndWait(NAME, 'lost-stream-update', updateFor('stale'), generation) + ).rejects.toThrow('replaced') + await expect(store.getStreamState(NAME)).resolves.toBeNull() + }) + + it('rejects appends and duplicate acknowledgements when only the stream is lost', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('base'), 10) + const generation = await store.getDocumentGeneration(NAME) + const delta = updateFor('edit') + await store.publishClientUpdateAndWait(NAME, 'accepted-update', delta, generation) + state.backing!.streams.delete(`filedoc:stream:${NAME}`) + + await expect(store.publishAndWait(NAME, delta, generation)).rejects.toThrow('replaced') + await expect( + store.publishClientUpdateAndWait(NAME, 'new-update', delta, generation) + ).rejects.toThrow('replaced') + await expect( + store.publishClientUpdateAndWait(NAME, 'accepted-update', delta, generation) + ).rejects.toThrow('replaced') + expect(state.backing!.streams.has(`filedoc:stream:${NAME}`)).toBe(false) + }) + + it('adopts the identity of a pre-upgrade stream before acknowledging its edits', async () => { + const store = await newStore() + const seed = new Y.Doc() + seed.getMap('config').set('initialContentLoaded', true) + seed.getMap('config').set('docId', 'legacy-document') + seed.getText('body').insert(0, 'legacy') + seedLegacyStream(Y.encodeStateAsUpdate(seed)) + const attached = new Y.Doc() + await store.attachRoom(NAME, attached) + expect(await store.getDocumentGeneration(NAME)).toBe('legacy-document') + const before = Y.encodeStateVector(seed) + seed.getText('body').insert(6, ' edit') + await expect( + store.publishClientUpdateAndWait( + NAME, + 'legacy-edit', + Y.encodeStateAsUpdate(seed, before), + 'legacy-document' + ) + ).resolves.toBeUndefined() + store.detachRoom(NAME) + seed.destroy() + attached.destroy() + }) + + it('rejects a shared replay if the document generation changes between pages', async () => { + const store = await newStore() + await store.seedIfEmpty(NAME, updateFor('old generation'), 10) + state.backing!.onRange = () => { + state.backing!.kv.set(`filedoc:generation:${NAME}`, 'new generation') + } + await expect(store.getStreamState(NAME)).rejects.toThrow('replaced') + }) + + it.each([true, false])( + 'validates a modern snapshot following a legacy seed (same identity: %s)', + async (sameIdentity) => { + const store = await newStore() + const seed = new Y.Doc() + seed.getMap('config').set('initialContentLoaded', true) + seed.getMap('config').set('docId', 'legacy-document') + seed.getText('body').insert(0, 'legacy') + seedLegacyStream(Y.encodeStateAsUpdate(seed)) + const backing = state.backing! + backing.kv.set( + `filedoc:generation:${NAME}`, + sameIdentity ? 'legacy-document' : 'different-document' + ) + backing.streams.get(`filedoc:stream:${NAME}`)!.push({ + id: `${++backing.seq}-0`, + message: { + u: Buffer.from(Y.encodeStateAsUpdate(seed)).toString('base64'), + s: '1', + g: sameIdentity ? 'legacy-document' : 'different-document', + }, + }) + const attached = new Y.Doc() + if (sameIdentity) { + await store.attachRoom(NAME, attached) + const before = Y.encodeStateVector(seed) + seed.getText('body').insert(6, ' peer') + await store.publishClientUpdateAndWait( + NAME, + 'peer-edit', + Y.encodeStateAsUpdate(seed, before), + 'legacy-document' + ) + await store.catchUp(NAME) + expect(attached.getText('body').toString()).toBe('legacy peer') + store.detachRoom(NAME) + } else { + await expect(store.attachRoom(NAME, attached)).rejects.toThrow('replaced') + expect(storeInternals(store).rooms.has(NAME)).toBe(false) + } + seed.destroy() + attached.destroy() + } + ) + + it('replays stream history in bounded pages', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 40 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 40 + const store = await newStore() + + await expect(store.getStreamState(NAME)).resolves.not.toBeNull() + + expect(state.backing!.maxRangeCount).toBe(4) + }) + + it('fails safely when an uncompacted stream exceeds the replay entry budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 2_001 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 2_001 + const store = await newStore() + + await expect(store.getStreamState(NAME)).rejects.toThrow('replay exceeded its safety limit') + }) + + it('never exposes a partially replayed document when room attachment exceeds its budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 2_001 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 2_001 + const store = await newStore() + const doc = new Y.Doc() + + await expect(store.attachRoom(NAME, doc)).rejects.toThrow('replay exceeded its safety limit') + + expect(doc.getText('body').toString()).toBe('') + expect(storeInternals(store).rooms.has(NAME)).toBe(false) + doc.destroy() + }) + + it('recovers from compaction that trims unread pages during replay', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 8 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + ) + state.backing!.seq = 8 + state.backing!.onRange = (call, key) => { + if (call !== 2 || key !== streamKey) return + state.backing!.streams.set(streamKey, [ + { + id: '9-0', + message: { u: Buffer.from(updateFor('compacted')).toString('base64'), s: '1' }, + }, + ]) + state.backing!.seq = 9 + state.backing!.onRange = undefined + } + const store = await newStore() + + const recovered = new Y.Doc() + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('compacted') + recovered.destroy() + }) + + it.each(['headless', 'attached'] as const)( + 'does not recount a replacement snapshot near the byte budget during %s replay', + async (mode) => { + const streamKey = `filedoc:stream:${NAME}` + const source = new Y.Doc() + source.getText('body').insert(0, 'x'.repeat(10 * 1024 * 1024)) + const initial = Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64') + const noop = Buffer.from(updateFor('')).toString('base64') + state.backing!.streams.set( + streamKey, + Array.from({ length: 8 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: index === 0 ? initial : noop }, + })) + ) + source.getText('body').insert(source.getText('body').length, ' joined') + const compacted = Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64') + state.backing!.seq = 8 + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(4-0') return + state.backing!.streams.set(streamKey, [{ id: '9-0', message: { u: compacted, s: '1' } }]) + state.backing!.seq = 9 + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + if (mode === 'attached') await store.attachRoom(NAME, recovered) + else Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe(source.getText('body').toString()) + } finally { + store.detachRoom(NAME) + recovered.destroy() + source.destroy() + } + } + ) + + it('does not recount retained entries when compaction meets the exact entry budget', async () => { + const streamKey = `filedoc:stream:${NAME}` + const noop = Buffer.from(updateFor('')).toString('base64') + const entries = Array.from({ length: 1_999 }, (_, index) => ({ + id: `${index + 1}-0`, + message: { u: noop }, + })) + state.backing!.streams.set(streamKey, entries) + state.backing!.seq = 1_999 + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(1996-0') return + state.backing!.streams.set(streamKey, [ + ...entries.slice(1_996), + { + id: '2000-0', + message: { u: Buffer.from(updateFor('complete')).toString('base64'), s: '1' }, + }, + ]) + state.backing!.seq = 2_000 + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe('complete') + } finally { + recovered.destroy() + } + }) + + it('reads the replacement snapshot when peer deltas cross the old replay tail', async () => { + const streamKey = `filedoc:stream:${NAME}` + const source = new Y.Doc() + const entries: Array<{ id: string; message: Record }> = [] + source.on('update', (update: Uint8Array) => { + entries.push({ + id: `${entries.length + 1}-0`, + message: { u: Buffer.from(update).toString('base64') }, + }) + }) + for (let i = 1; i <= 8; i++) + source.getText('body').insert(source.getText('body').length, String(i)) + const snapshot = Y.encodeStateAsUpdate(source) + state.backing!.streams.set(streamKey, entries.slice()) + for (let i = 9; i <= 11; i++) + source.getText('body').insert(source.getText('body').length, String(i)) + state.backing!.onRange = (_call, key, start) => { + if (key !== streamKey || start !== '(4-0') return + state.backing!.streams.set(streamKey, [ + ...entries.slice(7), + { id: '12-0', message: { u: Buffer.from(snapshot).toString('base64'), s: '1' } }, + ]) + state.backing!.onRange = undefined + } + const store = await newStore() + const recovered = new Y.Doc() + try { + Y.applyUpdate(recovered, (await store.getStreamState(NAME))!) + expect(recovered.getText('body').toString()).toBe(source.getText('body').toString()) + } finally { + source.destroy() + recovered.destroy() + } + }) + it('attachRoom catches a fresh task up to the current shared state', async () => { const a = await newStore() - a.publish(NAME, updateFor('already here')) + await a.seedIfEmpty(NAME, updateFor('already here')) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) // A second task opens the same file: its doc must load the existing content, not start empty. @@ -300,6 +897,7 @@ describe('FileDocStore', () => { }) it('converges a peer task via the tailer after attach', async () => { + seedLegacyStream() const a = await newStore() const b = await newStore() const bDoc = new Y.Doc() @@ -338,14 +936,16 @@ describe('FileDocStore', () => { const a = await newStore() // This task has integrated only up to entry 400 (all no-ops) — its local doc is empty and lags the // two peer entries. Inject that lagging room directly (a real edit was integrated → realEdited). - ;(a as any).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: true, }) - await (a as any).maybeCompact(NAME) + await storeInternals(a).maybeCompact(NAME) // A fresh catch-up must still reconstruct the peer content — compaction must not have trimmed 401/402. const doc = new Y.Doc() @@ -360,6 +960,7 @@ describe('FileDocStore', () => { }) it('tags an agent-streamed frame so a peer tailer applies it as REDIS_AGENT_ORIGIN (never persisted)', async () => { + seedLegacyStream() const streamKey = `filedoc:stream:${NAME}` const a = await newStore() const b = await newStore() @@ -385,17 +986,18 @@ describe('FileDocStore', () => { }) it('latches realEdited synchronously so a concurrent compaction can never mislabel a real edit', async () => { + seedLegacyStream() // The data-loss race: a real edit sits in room.doc synchronously, but if realEdited were set only // AFTER appendUpdate's awaits, a concurrent agent-triggered compaction could snapshot that content and // stamp it an agent (no-persist) frame — losing the edit. The latch must be set in the same tick. const a = await newStore() const doc = new Y.Doc() await a.attachRoom(NAME, doc) - const room = (a as any).rooms.get(NAME) + const room = storeInternals(a).rooms.get(NAME)! expect(room.realEdited).toBe(false) // Kick off a real (non-agent) append but do NOT await it: realEdited must already be true before the // xAdd/expire awaits resolve, so any compaction racing on the awaits sees the real edit. - const pending = (a as any).appendUpdate(NAME, updateFor('real user edit')) + const pending = storeInternals(a).appendUpdate(NAME, updateFor('real user edit')) expect(room.realEdited).toBe(true) await pending doc.destroy() @@ -414,14 +1016,16 @@ describe('FileDocStore', () => { state.backing!.seq = 400 const a = await newStore() - ;(a as any).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: agentDoc, lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: false, }) - await (a as any).maybeCompact(NAME) + await storeInternals(a).maybeCompact(NAME) // The snapshot must carry the AGENT marker, NOT the snapshot marker, so a peer catch-up applies it as // REDIS_AGENT_ORIGIN and never marks the doc edited — the no-persist guarantee survives compaction. @@ -438,6 +1042,7 @@ describe('FileDocStore', () => { }) it('retries a transient append failure so the edit is not lost from the shared log', async () => { + seedLegacyStream() const a = await newStore() state.backing!.failXAdd = 2 // first two xAdd attempts throw; the third must succeed a.publish(NAME, updateFor('resilient')) @@ -452,10 +1057,261 @@ describe('FileDocStore', () => { ) }) + it('deduplicates acknowledged client retries by update id', async () => { + seedLegacyStream() + const store = await newStore() + const update = updateFor('retry-safe') + + await store.publishClientUpdateAndWait(NAME, 'update-1', update) + await store.publishClientUpdateAndWait(NAME, 'update-1', update) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(2) + }) + + it('does not drop different payloads that reuse an acknowledged update id', async () => { + seedLegacyStream() + const store = await newStore() + + await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('first')) + await store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('second')) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + }) + + it('uses unambiguous acknowledged-update deduplication keys', async () => { + seedLegacyStream() + const store = await newStore() + + await store.publishClientUpdateAndWait(NAME, 'a', new Uint8Array([0, 98])) + await store.publishClientUpdateAndWait(NAME, 'a\0', new Uint8Array([98])) + + expect(state.backing!.streams.get(`filedoc:stream:${NAME}`)).toHaveLength(3) + }) + + it('bounds acknowledged-update deduplication independently of stream traffic', async () => { + seedLegacyStream() + const store = await newStore() + const update = updateFor('bounded') + + for (let index = 0; index <= 16_384; index += 1) { + await store.publishClientUpdateAndWait(NAME, `update-${index}`, update) + } + + expect(state.backing!.dedupe.get(`filedoc:updates:${NAME}`)).toHaveLength(16_384) + }) + + it('limits every multiplexed read to four streams and one entry per stream', async () => { + const store = await newStore() + const docs = Array.from({ length: 9 }, () => new Y.Doc()) + await Promise.all(docs.map((doc, index) => store.attachRoom(`${NAME}-${index}`, doc))) + state.backing!.maxReadStreams = 0 + state.backing!.maxReadCount = 0 + const readsBefore = state.backing!.reads + + await vi.waitFor(() => expect(state.backing!.reads).toBeGreaterThan(readsBefore)) + + expect(state.backing!.maxReadStreams).toBeLessThanOrEqual(4) + expect(state.backing!.maxReadCount).toBe(1) + docs.forEach((doc, index) => { + store.detachRoom(`${NAME}-${index}`) + doc.destroy() + }) + }) + + it('compacts on retained bytes before the entry-count threshold can exhaust replay', async () => { + const streamKey = `filedoc:stream:${NAME}` + const snapshot = Buffer.from(Y.encodeStateAsUpdate(new Y.Doc())).toString('base64') + state.backing!.streams.set(streamKey, [{ id: '1-0', message: { u: snapshot } }]) + state.backing!.seq = 1 + const store = await newStore() + storeInternals(store).rooms.set(NAME, { + doc: new Y.Doc(), + lastId: '1-0', + publishes: 0, + uncompactedDeltaBytes: 12 * 1024 * 1024, + compacting: false, + seededObserved: true, + realEdited: true, + }) + + await storeInternals(store).maybeCompact(NAME) + + const stream = state.backing!.streams.get(streamKey)! + expect(stream).toHaveLength(2) + expect(stream.at(-1)?.message.s).toBe('1') + }) + + it('does not compact a large snapshot again while continuing to accept small edits', async () => { + const store = await newStore() + const source = docWithText('x'.repeat(9 * 1024 * 1024)) + source.getMap('config').set('initialContentLoaded', true) + source.getMap('config').set('docId', 'large-document') + const streamKey = `filedoc:stream:${NAME}` + state.backing!.kv.set(`filedoc:generation:${NAME}`, 'large-document') + state.backing!.streams.set(streamKey, [ + { + id: '1-0', + message: { + u: Buffer.from(Y.encodeStateAsUpdate(source)).toString('base64'), + s: '1', + g: 'large-document', + }, + }, + ]) + state.backing!.seq = 1 + const loaded = new Y.Doc() + await store.attachRoom(NAME, loaded) + expect(storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes).toBe(0) + + let deltaBytes = 0 + for (let index = 0; index < 30; index++) { + const before = Y.encodeStateVector(source) + source.getText('body').insert(source.getText('body').length, 'y') + const update = Y.encodeStateAsUpdate(source, before) + deltaBytes += Buffer.from(update).toString('base64').length + await store.publishClientUpdateAndWait(NAME, `small-${index}`, update, 'large-document') + await store.catchUp(NAME) + } + expect(state.backing!.streams.get(streamKey)?.filter((entry) => entry.message.s)).toHaveLength( + 1 + ) + expect(storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes).toBe(deltaBytes) + expect(loaded.getText('body').length).toBe(9 * 1024 * 1024 + 30) + store.detachRoom(NAME) + source.destroy() + loaded.destroy() + }) + + it('preserves exactly the delta bytes observed after a compaction barrier', async () => { + seedLegacyStream() + const store = await newStore() + const doc = new Y.Doc() + await store.attachRoom(NAME, doc) + const room = storeInternals(store).rooms.get(NAME)! + room.uncompactedDeltaBytes = 12 * 1024 * 1024 + const lateUpdate = updateFor('concurrent edit') + state.backing!.onSnapshot = async () => { + await store.publishAndWait(NAME, lateUpdate) + await store.catchUp(NAME) + } + await storeInternals(store).maybeCompact(NAME) + expect(room.uncompactedDeltaBytes).toBe(Buffer.from(lateUpdate).toString('base64').length) + expect(doc.getText('body').toString()).toBe('concurrent edit') + store.detachRoom(NAME) + doc.destroy() + }) + + it('does not trim a replacement stream recreated in the same millisecond as its compaction barrier', async () => { + const store = await newStore() + const replacer = await newStore() + const oldDoc = docWithText('old') + oldDoc.getMap('config').set('docId', 'old-generation') + state.backing!.nextIds = ['1000-0', '1000-1', '1000-2'] + await store.seedIfEmpty(NAME, Y.encodeStateAsUpdate(oldDoc), 10) + await store.publishAndWait(NAME, updateFor('first edit'), 'old-generation') + await store.publishAndWait(NAME, updateFor('second edit'), 'old-generation') + await store.attachRoom(NAME, oldDoc) + storeInternals(store).rooms.get(NAME)!.uncompactedDeltaBytes = 12 * 1024 * 1024 + + const freshDoc = docWithText('fresh') + freshDoc.getMap('config').set('docId', 'new-generation') + const freshSeed = Y.encodeStateAsUpdate(freshDoc) + const beforeEdit = Y.encodeStateVector(freshDoc) + freshDoc.getText('body').insert(5, ' accepted edit') + const freshEdit = Y.encodeStateAsUpdate(freshDoc, beforeEdit) + const streamKey = `filedoc:stream:${NAME}` + state.backing!.nextIds = ['1000-3', '1000-0', '1000-1'] + state.backing!.onSnapshot = async () => { + await replacer.invalidateDocument(NAME, 20) + await replacer.seedIfEmpty(NAME, freshSeed, 20) + await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation') + expect(state.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ + '1000-0', + '1000-1', + ]) + } + + await storeInternals(store).maybeCompact(NAME) + + expect(state.backing!.streams.get(streamKey)?.map((entry) => entry.id)).toEqual([ + '1000-0', + '1000-1', + ]) + await replacer.publishClientUpdateAndWait(NAME, 'fresh-edit', freshEdit, 'new-generation') + expect(state.backing!.streams.get(streamKey)).toHaveLength(2) + const persisted = await replacer.getStreamState(NAME) + expect(persisted).not.toBeNull() + const replayed = new Y.Doc() + Y.applyUpdate(replayed, persisted!) + expect(replayed.getText('body').toString()).toBe('fresh accepted edit') + store.detachRoom(NAME) + oldDoc.destroy() + freshDoc.destroy() + replayed.destroy() + }) + + it('expires idle single-replica invalidation watermarks', async () => { + vi.useFakeTimers() + const store = new FileDocStore(undefined) + try { + for (let index = 0; index < 100; index++) { + await store.invalidateDocument(`closed-${index}`, 10) + } + expect(storeInternals(store).localInvalidations.size).toBe(100) + await vi.advanceTimersByTimeAsync(660_000) + expect(storeInternals(store).localInvalidations.size).toBe(0) + } finally { + await store.shutdown() + vi.useRealTimers() + } + }) + + it('deduplicates single-replica invalidations across same-version seeds and room reopen', async () => { + const store = new FileDocStore(undefined) + const staleDoc = new Y.Doc() + await store.attachRoom(NAME, staleDoc) + await store.invalidateDocument(NAME, 20) + await expect(store.seedIfEmpty(NAME, updateFor('stale fetched seed'), 10)).resolves.toBe(false) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(false) + expect(storeInternals(store).localInvalidations.size).toBe(1) + await expect(store.seedIfEmpty(NAME, updateFor('same-version seed'), 20)).resolves.toBe(true) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(true) + + store.detachRoom(NAME) + expect(storeInternals(store).localInvalidations.size).toBe(1) + const freshDoc = new Y.Doc() + await store.attachRoom(NAME, freshDoc) + await expect(store.seedIfEmpty(NAME, updateFor('fresh authoritative seed'), 20)).resolves.toBe( + true + ) + await expect(store.invalidateDocument(NAME, 20)).resolves.toEqual({ status: 'stale' }) + await expect(store.isDocumentGenerationCurrent(NAME)).resolves.toBe(true) + await store.invalidateDocument(NAME, 40) + await store.shutdown() + expect(storeInternals(store).localInvalidations.size).toBe(0) + staleDoc.destroy() + freshDoc.destroy() + }) + + it('fails closed when a Redis-backed store has not initialized', async () => { + const store = new FileDocStore(REDIS_URL) + const doc = new Y.Doc() + + await expect(store.attachRoom(NAME, doc)).rejects.toThrow('not initialized') + await expect( + store.publishClientUpdateAndWait(NAME, 'update-1', updateFor('x')) + ).rejects.toThrow('not initialized') + await expect(store.seedIfEmpty(NAME, updateFor('seed'))).rejects.toThrow('not initialized') + await expect(store.getStreamState(NAME)).rejects.toThrow('not initialized') + expect(await store.acquireMergeSlot(NAME, 1_000)).toBeNull() + doc.destroy() + }) + it('streamHasContent fences a seed apply against an already-seeded stream', async () => { const a = await newStore() expect(await a.streamHasContent(NAME)).toBe(false) - a.publish(NAME, updateFor('seeded')) + await a.seedIfEmpty(NAME, updateFor('seeded')) await vi.waitFor(async () => expect(await a.streamHasContent(NAME)).toBe(true)) }) @@ -536,7 +1392,7 @@ describe('FileDocStore', () => { author.getText('body').insert(4, 'peer') const a = await newStore() - a.publish(NAME, updates[0]) // 'base' + seedLegacyStream(updates[0]) await vi.waitFor(async () => expect(await a.getStreamState(NAME)).not.toBeNull()) // Task B attaches; while its synchronous catch-up runs, task A publishes the second edit. The tailer @@ -580,21 +1436,25 @@ describe('FileDocStore', () => { const b = await newStore() const docA = new Y.Doc() Y.applyUpdate(docA, peerUpdates[0]) // A integrated up to 401 - ;(a as any).rooms.set(NAME, { + storeInternals(a).rooms.set(NAME, { doc: docA, lastId: '401-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: true, }) - ;(b as any).rooms.set(NAME, { + storeInternals(b).rooms.set(NAME, { doc: new Y.Doc(), lastId: '400-0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, seededObserved: true, realEdited: true, }) - await Promise.all([(a as any).maybeCompact(NAME), (b as any).maybeCompact(NAME)]) + await Promise.all([storeInternals(a).maybeCompact(NAME), storeInternals(b).maybeCompact(NAME)]) const doc = new Y.Doc() Y.applyUpdate(doc, (await a.getStreamState(NAME))!) diff --git a/apps/realtime/src/handlers/file-doc-store.ts b/apps/realtime/src/handlers/file-doc-store.ts index 537f7f4db12..c84ade7cdf4 100644 --- a/apps/realtime/src/handlers/file-doc-store.ts +++ b/apps/realtime/src/handlers/file-doc-store.ts @@ -34,8 +34,10 @@ * * @module */ + +import { createHash } from 'node:crypto' import { createLogger } from '@sim/logger' -import { FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_LIMITS, FILE_DOC_SEED, FILE_DOC_TIMEOUTS } from '@sim/realtime-protocol/file-doc' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' @@ -61,7 +63,38 @@ const RELEASE_LOCK_SCRIPT = * Returns 1 if THIS call wrote the seed, 0 if the stream already had content. */ const SEED_IF_EMPTY_SCRIPT = - "if redis.call('xlen', KEYS[1]) == 0 then redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]); return 1 else return 0 end" + "local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[6]) then return 0 end; if redis.call('xlen', KEYS[1]) == 0 then redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[4]); if ARGV[6] ~= '0' then redis.call('set', KEYS[3], ARGV[6], 'EX', ARGV[4]) end; redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[5], ARGV[3]); redis.call('expire', KEYS[1], ARGV[4]); redis.call('expire', KEYS[4], ARGV[4]); return 1 else return 0 end" + +/** Orders a durable replacement with seeds and merges, and fences publishers in the same transaction. */ +const INVALIDATE_DOCUMENT_SCRIPT = + "local invalidated = redis.call('get', KEYS[6]); if invalidated and tonumber(invalidated) >= tonumber(ARGV[1]) then return false end; local version = redis.call('get', KEYS[3]); if version and tonumber(version) > tonumber(ARGV[1]) then return false end; local generation = redis.call('get', KEYS[2]) or ''; if version == ARGV[1] and generation == ARGV[3] then return false end; redis.call('set', KEYS[2], ARGV[3], 'EX', ARGV[2]); redis.call('set', KEYS[3], ARGV[1], 'EX', ARGV[2]); redis.call('set', KEYS[6], ARGV[1], 'EX', ARGV[2]); redis.call('del', KEYS[1], KEYS[4], KEYS[5]); return generation" + +/** Upgrades an existing pre-negotiation stream without ever resurrecting a missing stream. */ +const ADOPT_GENERATION_SCRIPT = + "local generation = redis.call('get', KEYS[2]); if generation then return generation end; if redis.call('xlen', KEYS[1]) == 0 then return false end; redis.call('set', KEYS[2], ARGV[1], 'EX', ARGV[2]); return ARGV[1]" + +/** Atomically fence XADD so stale rooms cannot recreate a replaced or expired stream. Returns false when fenced. */ +const APPEND_UPDATE_SCRIPT = + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; for _, key in ipairs(KEYS) do redis.call('expire', key, ARGV[5]) end; if ARGV[3] ~= '' then return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1') else return redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2]) end" + +/** Renew stream metadata atomically, so an invalidation watermark cannot expire ahead of its stream. */ +const REFRESH_DOCUMENT_TTLS_SCRIPT = + "for _, key in ipairs(KEYS) do redis.call('expire', key, ARGV[1]) end; return 1" + +/** + * Append and trim under one generation fence: invalidation can recreate the stream with lower IDs + * within the same millisecond, so a separate trim could delete the replacement's seed and edits. + * Carry the seed's generation forward and keep every entry at or beyond the captured prefix barrier. + */ +const APPEND_SNAPSHOT_SCRIPT = + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[4] or redis.call('exists', KEYS[1]) == 0 then return false end; local id = redis.call('xadd', KEYS[1], '*', ARGV[1], ARGV[2], ARGV[3], '1', ARGV[5], ARGV[4]); redis.call('xtrim', KEYS[1], 'MINID', ARGV[6]); return id" + +/** + * Atomically deduplicate and append an acknowledged client update. Socket acknowledgements can be + * lost, so a retry with the same id must not inflate the stream or its compaction counters. + */ +const APPEND_CLIENT_UPDATE_SCRIPT = + "local generation = redis.call('get', KEYS[3]) or ''; if generation ~= ARGV[6] or redis.call('exists', KEYS[1]) == 0 then return -1 end; redis.call('expire', KEYS[4], ARGV[5]); if redis.call('zscore', KEYS[2], ARGV[1]) then redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 0 end; local id = redis.call('xadd', KEYS[1], '*', ARGV[2], ARGV[3]); local score = string.match(id, '^(%d+)'); redis.call('zadd', KEYS[2], score, ARGV[1]); local excess = redis.call('zcard', KEYS[2]) - tonumber(ARGV[4]); if excess > 0 then redis.call('zpopmin', KEYS[2], excess) end; if redis.call('ttl', KEYS[2]) < 0 then redis.call('expire', KEYS[2], ARGV[5]) end; redis.call('expire', KEYS[1], ARGV[5]); redis.call('expire', KEYS[3], ARGV[5]); return 1" /** * Monotonic set of the synced-version token: overwrite ONLY when the new value is greater than the @@ -73,7 +106,7 @@ const SEED_IF_EMPTY_SCRIPT = * comfortably within a Lua double, so the numeric compare is exact. */ const SET_VERSION_IF_NEWER_SCRIPT = - "local c = redis.call('get', KEYS[1]); if c == false or tonumber(c) < tonumber(ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) else redis.call('expire', KEYS[1], ARGV[2]) end; return 1" + "local generation = redis.call('get', KEYS[2]) or ''; if generation ~= ARGV[3] then return 0 end; local c = redis.call('get', KEYS[1]); if c == false or tonumber(c) < tonumber(ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) else redis.call('expire', KEYS[1], ARGV[2]) end; return 1" /** * The transaction origin the store stamps on updates it applies from the stream. The relay's @@ -103,6 +136,10 @@ export const REDIS_SNAPSHOT_ORIGIN = Symbol('file-doc-redis-snapshot') export const REDIS_AGENT_ORIGIN = Symbol('file-doc-redis-agent') const STREAM_PREFIX = 'filedoc:stream:' +const CLIENT_UPDATE_PREFIX = 'filedoc:updates:' +const GENERATION_PREFIX = 'filedoc:generation:' +/** Retries must remain idempotent after a seed replaces the generation tombstone. */ +const INVALIDATION_VERSION_PREFIX = 'filedoc:invalidatedver:' /** Cluster-wide "durable version the live doc is synced to" (the persist If-Match token). */ const SYNC_VERSION_PREFIX = 'filedoc:syncver:' const SEED_LOCK_PREFIX = 'filedoc:seedlock:' @@ -123,6 +160,9 @@ const SNAPSHOT_FIELD = 's' /** Marks a stream entry as an AGENT-STREAMED preview frame, so the tailer applies it with * {@link REDIS_AGENT_ORIGIN} (never marks the doc edited). Present only on agent-frame entries. */ const AGENT_FIELD = 'a' +/** Identifies a seed's document generation, allowing old rooms to reject every later update. */ +const GENERATION_FIELD = 'g' +const INVALIDATED_GENERATION = '__invalidated__' /** Sentinel token a DISABLED store returns from a lock acquire, so single-replica callers proceed * without special-casing; {@link FileDocStore.releaseLock} treats it as a no-op. Not a real UUID, so it @@ -136,8 +176,18 @@ const READ_BLOCK_MS = 1_000 /** Idle poll cadence when NO room is open on this task, so a freshly-attached room is picked up fast * without busy-spinning an empty task. */ const IDLE_POLL_MS = 250 -/** Max entries drained per stream per read. */ -const READ_COUNT = 200 +/** Max entries drained per stream per read, bounding one Redis response even for maximum-size edits. */ +const READ_COUNT = 1 +/** Maximum streams passed to one XREAD, bounding response memory independently of open-room count. */ +const READ_STREAM_BATCH_SIZE = 4 +/** Replay streams incrementally instead of materializing their complete history in one response. */ +const REPLAY_PAGE_COUNT = 4 +/** Compaction normally holds a stream near 400 entries; fail safely if that invariant is badly broken. */ +const REPLAY_MAX_ENTRIES = 2_000 +/** Base64 bytes accepted during one replay, including a full snapshot plus a bounded edit backlog. */ +const REPLAY_MAX_ENCODED_BYTES = FILE_DOC_LIMITS.updateBytes * 6 +/** Compact before a handful of individually valid large updates can exhaust the replay byte budget. */ +const COMPACT_ENCODED_BYTES = FILE_DOC_LIMITS.updateBytes * 2 /** Compact a stream once it exceeds this many entries (snapshot + trim). */ const COMPACT_THRESHOLD = 400 /** Check whether compaction is due only every Nth local publish, to avoid an XLEN per keystroke. */ @@ -166,8 +216,42 @@ const RECONNECT_MAX_DELAY_MS = 3_000 const READER_RETRY_MAX_MS = 10_000 /** After the first failure of a streak, log one reader failure in this many. */ const READER_ERROR_LOG_EVERY = 20 +const CLIENT_UPDATE_DEDUPE_CAPACITY = 16_384 const streamKey = (name: string) => `${STREAM_PREFIX}${name}` +const generationKey = (name: string) => `${GENERATION_PREFIX}${name}` +const documentKeys = (name: string) => [ + streamKey(name), + generationKey(name), + `${SYNC_VERSION_PREFIX}${name}`, + `${INVALIDATION_VERSION_PREFIX}${name}`, +] + +export class FileDocInvalidatedError extends Error { + constructor() { + super('The live file document was replaced by a newer durable version') + this.name = 'FileDocInvalidatedError' + } +} + +function assertUpdateWithinLimit(update: Uint8Array): void { + if (update.byteLength === 0 || update.byteLength > FILE_DOC_LIMITS.updateBytes) { + throw new Error(`File document update is outside the ${FILE_DOC_LIMITS.updateBytes}-byte limit`) + } +} + +function generationOfSeed(update: Uint8Array): string { + const doc = new Y.Doc() + try { + Y.applyUpdate(doc, update) + const docId = doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + return typeof docId === 'string' + ? docId + : `seed:${createHash('sha256').update(update).digest('hex')}` + } finally { + doc.destroy() + } +} /** * Decode one stream entry's base64 Yjs update and apply it to `doc`. A malformed entry is logged and @@ -218,6 +302,13 @@ interface StoreRoom { lastId: string /** Local publish count, to pace compaction checks. */ publishes: number + /** Non-snapshot bytes observed since this replica last compacted. */ + uncompactedDeltaBytes: number + compacting: boolean + /** Document generation read from the seed entry; every later append is fenced against it. */ + generation: string | null + /** A newer seed was observed; this old room must ignore all entries until the relay replaces it. */ + generationInvalidated: boolean /** Set once the doc has been observed seeded, so the seed transition itself is never mistaken for an * edit (mirrors the relay's `seededObserved`). */ seededObserved: boolean @@ -239,6 +330,7 @@ export class FileDocStore { /** Dedicated connection for blocking XREAD (a blocking command monopolizes its connection). */ private read: RedisClientType | null = null private readonly rooms = new Map() + private readonly localInvalidations = new Map() private running = false private heartbeat: ReturnType | null = null @@ -262,7 +354,10 @@ export class FileDocStore { * connection it can rebuild is always worth rebuilding. */ reconnectStrategy: (retries: number) => - backoffWithJitter(retries + 1, null, { baseMs: 100, maxMs: RECONNECT_MAX_DELAY_MS }), + backoffWithJitter(retries + 1, null, { + baseMs: 100, + maxMs: RECONNECT_MAX_DELAY_MS, + }), }, } this.write = createClient(options) @@ -284,55 +379,76 @@ export class FileDocStore { await Promise.all([this.write?.quit().catch(() => {}), this.read?.quit().catch(() => {})]) this.write = null this.read = null + this.rooms.clear() + this.localInvalidations.clear() } /** * Register a locally-opened room and load the shared state into its doc ({@link catchUp}). A * brand-new file has an empty stream and loads nothing (it is seeded shortly after, via - * {@link shouldSeed}). No-op when disabled. + * {@link shouldSeed}). Single-replica rooms are tracked only for invalidation lifecycle. */ async attachRoom(name: string, doc: Y.Doc): Promise { - if (!this.enabled || !this.write) return + if (this.enabled && !this.write) throw new Error('FileDocStore is not initialized') // Register BEFORE the async read so a concurrent publish/tailer for this room can't be missed — // the tailer resumes from `lastId`, which the catch-up advances. const room: StoreRoom = { doc, lastId: '0', publishes: 0, + uncompactedDeltaBytes: 0, + compacting: false, + generation: null, + generationInvalidated: false, seededObserved: false, realEdited: false, } this.rooms.set(name, room) - await this.catchUp(name) + if (!this.enabled) return + try { + await this.catchUp(name) + } catch (error) { + if (this.rooms.get(name) === room) this.rooms.delete(name) + throw error + } } /** - * PULL the shared state into a registered room: read the stream and apply every entry the doc has - * not integrated yet (origin {@link REDIS_ORIGIN}), advancing `lastId` so the tailer resumes exactly - * after it. This is the ONLY way a room loads shared state, so a caller that must not depend on the - * tailer's asynchronous push — the join, which may not serve a client a half-assembled document — - * can converge on demand. Idempotent and safe to call repeatedly; no-op when disabled or the room is - * not registered (a fast open→close detached it). Never throws. + * Completes shared replay before applying entries, so joins never receive a partial document. + * Repeated calls skip integrated entries; detached rooms and disabled stores are ignored. */ async catchUp(name: string): Promise { - if (!this.enabled || !this.write) return + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') const room = this.rooms.get(name) if (!room) return try { - const entries = await this.write.xRange(streamKey(name), '-', '+') - for (const entry of entries) { + const entries: Array<{ id: string; message: Record }> = [] + await this.replayEntries(name, room.lastId, (entry) => { // The room can be detached + its doc destroyed while the read is in flight (a fast // open→close); stop touching it the moment that happens. - if (this.rooms.get(name) !== room) return - // Applying a Yjs update twice is a no-op, but `applyEntry`'s bookkeeping is not: re-applying - // the SEED after `seededObserved` latched would count it as a post-seed edit and let a - // compaction snapshot claim content no user ever typed. Skip what this room already holds. - if (!isAfterStreamId(entry.id, room.lastId)) continue - this.applyEntry(room, entry.id, entry.message) + if (this.rooms.get(name) !== room) return false + entries.push(entry) + return true + }) + if (this.rooms.get(name) !== room) return + for (const entry of entries) this.applyEntry(name, room, entry.id, entry.message) + if (room.generationInvalidated) throw new FileDocInvalidatedError() + const docId = room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (room.generation === null && typeof docId === 'string') { + const adopted = await this.write.eval(ADOPT_GENERATION_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [docId, String(STREAM_TTL_SEC)], + }) + if (adopted !== docId) throw new FileDocInvalidatedError() + room.generation = docId } - await this.write.expire(streamKey(name), STREAM_TTL_SEC) + await this.refreshDocumentTtls(name) } catch (error) { - logger.warn(`FileDocStore catch-up failed for ${name}`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore catch-up failed for ${name}`, { + error: getErrorMessage(error), + }) + throw error } } @@ -344,11 +460,17 @@ export class FileDocStore { /** * Append a locally-applied update to the shared stream so every task converges, AWAITING the write * and retrying a transient failure ({@link PUBLISH_MAX_RETRIES}) so a Redis blip can't silently drop - * an edit from the shared log. Only the `xAdd` is retried; the TTL refresh + compaction check are - * post-write best-effort and never re-trigger the append. Throws if the append ultimately fails. + * an edit from the shared log. The append and metadata TTL renewal are atomic; post-write + * compaction never re-triggers the append. Throws if the append ultimately fails. */ - private async appendUpdate(name: string, update: Uint8Array, agent = false): Promise { + private async appendUpdate( + name: string, + update: Uint8Array, + agent = false, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { if (!this.write) return + assertUpdateWithinLimit(update) // Latch realEdited SYNCHRONOUSLY — before the first await — for a real (non-agent) publish. The edit // already sits in room.doc (applied in doc.on('update') before publish was called), so if this set // were deferred past the xAdd/expire awaits a CONCURRENT agent-frame-triggered maybeCompact could read @@ -361,15 +483,21 @@ export class FileDocStore { if (editedRoom) editedRoom.realEdited = true } const encoded = Buffer.from(update).toString('base64') - const fields: Record = { [UPDATE_FIELD]: encoded } - if (agent) fields[AGENT_FIELD] = '1' + const marker = agent ? AGENT_FIELD : '' for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { - await this.write.xAdd(streamKey(name), '*', fields) + const id = await this.write.eval(APPEND_UPDATE_SCRIPT, { + keys: documentKeys(name), + arguments: [UPDATE_FIELD, encoded, marker, expectedGeneration, String(STREAM_TTL_SEC)], + }) + if (id === null || id === false) throw new FileDocInvalidatedError() break } catch (error) { + if (error instanceof FileDocInvalidatedError) throw error if (attempt === PUBLISH_MAX_RETRIES) { - logger.error(`FileDocStore append failed for ${name}`, { error: getErrorMessage(error) }) + logger.error(`FileDocStore append failed for ${name}`, { + error: getErrorMessage(error), + }) throw error } // Snappy backoff — a stream append is a fast op; a transient blip clears in tens of ms. @@ -377,9 +505,16 @@ export class FileDocStore { await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) } } - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) const room = this.rooms.get(name) - if (room && ++room.publishes % COMPACT_CHECK_EVERY === 0) void this.maybeCompact(name) + if (room) { + room.publishes += 1 + if ( + room.uncompactedDeltaBytes >= COMPACT_ENCODED_BYTES || + room.publishes % COMPACT_CHECK_EVERY === 0 + ) { + await this.maybeCompact(name) + } + } } /** @@ -389,7 +524,11 @@ export class FileDocStore { */ publish(name: string, update: Uint8Array, agent = false): void { if (!this.enabled || !this.write) return - void this.appendUpdate(name, update, agent).catch(() => {}) // already logged inside appendUpdate + void this.appendUpdate(name, update, agent).catch((error) => { + logger.warn(`FileDocStore rejected a non-durable legacy update for ${name}`, { + error: getErrorMessage(error), + }) + }) } /** @@ -397,9 +536,80 @@ export class FileDocStore { * — the copilot merge, so the cross-task merge lock is not released before the diff is committed * (else the next task would diff a stale base). Throws on ultimate failure. No-op when disabled. */ - async publishAndWait(name: string, update: Uint8Array): Promise { - if (!this.enabled || !this.write) return - await this.appendUpdate(name, update) + async publishAndWait( + name: string, + update: Uint8Array, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') + await this.appendUpdate(name, update, false, expectedGeneration) + } + + /** + * Waits for Redis acceptance before the relay acknowledges the client. Retries are deduplicated + * within the bounded window; clients retain their journal until the acknowledgement arrives. + */ + async publishClientUpdateAndWait( + name: string, + updateId: string, + update: Uint8Array, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { + if (!this.enabled) return + if (!this.write) throw new Error('FileDocStore is not initialized') + assertUpdateWithinLimit(update) + const encoded = Buffer.from(update).toString('base64') + const dedupeMember = createHash('sha256') + .update(String(Buffer.byteLength(updateId))) + .update(':') + .update(updateId) + .update(update) + .digest('hex') + const room = this.rooms.get(name) + + for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { + try { + const appended = await this.write.eval(APPEND_CLIENT_UPDATE_SCRIPT, { + keys: [ + streamKey(name), + `${CLIENT_UPDATE_PREFIX}${name}`, + generationKey(name), + `${INVALIDATION_VERSION_PREFIX}${name}`, + ], + arguments: [ + dedupeMember, + UPDATE_FIELD, + encoded, + String(CLIENT_UPDATE_DEDUPE_CAPACITY), + String(STREAM_TTL_SEC), + expectedGeneration, + ], + }) + if (appended === -1) throw new FileDocInvalidatedError() + if (appended === 1 && room) { + room.realEdited = true + room.publishes += 1 + if ( + room.uncompactedDeltaBytes >= COMPACT_ENCODED_BYTES || + room.publishes % COMPACT_CHECK_EVERY === 0 + ) { + await this.maybeCompact(name) + } + } + return + } catch (error) { + if (error instanceof FileDocInvalidatedError) throw error + if (attempt === PUBLISH_MAX_RETRIES) { + logger.error(`FileDocStore acknowledged append failed for ${name}`, { + updateId, + error: getErrorMessage(error), + }) + throw error + } + await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) + } + } } /** @@ -412,20 +622,41 @@ export class FileDocStore { * Retries a transient Redis error like {@link appendUpdate}; throws if it ultimately fails. Disabled → * true (single-replica: seed locally, no stream). */ - async seedIfEmpty(name: string, update: Uint8Array): Promise { - if (!this.enabled || !this.write) return true + async seedIfEmpty(name: string, update: Uint8Array, version = 0): Promise { + if (!this.enabled) { + const invalidation = this.localInvalidations.get(name) + if (invalidation && invalidation.expiresAt > Date.now() && invalidation.version > version) + return false + const room = this.rooms.get(name) + if (room) room.generationInvalidated = false + return true + } + if (!this.write) throw new Error('FileDocStore is not initialized') + assertUpdateWithinLimit(update) const encoded = Buffer.from(update).toString('base64') + const generation = generationOfSeed(update) for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { const wrote = await this.write.eval(SEED_IF_EMPTY_SCRIPT, { - keys: [streamKey(name)], - arguments: [UPDATE_FIELD, encoded], + keys: documentKeys(name), + arguments: [ + UPDATE_FIELD, + encoded, + generation, + String(STREAM_TTL_SEC), + GENERATION_FIELD, + String(version), + ], }) - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) + await this.refreshDocumentTtls(name).catch(() => {}) + const room = this.rooms.get(name) + if (wrote === 1 && room) room.generation = generation return wrote === 1 } catch (error) { if (attempt === PUBLISH_MAX_RETRIES) { - logger.error(`FileDocStore seed failed for ${name}`, { error: getErrorMessage(error) }) + logger.error(`FileDocStore seed failed for ${name}`, { + error: getErrorMessage(error), + }) throw error } await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 50, maxMs: 500 })) @@ -434,6 +665,64 @@ export class FileDocStore { return false } + /** + * Fences an unsupported durable replacement before deleting its stream. The next authoritative + * seed replaces the tombstone. A separate version watermark deduplicates retries across reseeds; + * both expire with the stream TTL once the document is idle. + */ + async invalidateDocument( + name: string, + version: number + ): Promise<{ status: 'applied'; docId?: string } | { status: 'stale' }> { + if (!this.enabled) { + const now = Date.now() + const previous = this.localInvalidations.get(name) + if (previous && previous.expiresAt > now && previous.version >= version) + return { status: 'stale' } + this.localInvalidations.set(name, { version, expiresAt: now + STREAM_TTL_SEC * 1_000 }) + const room = this.rooms.get(name) + const docId = room?.generationInvalidated + ? undefined + : room?.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) + if (room) room.generationInvalidated = true + if (!this.heartbeat) { + this.heartbeat = setInterval(() => void this.refreshTtls(), HEARTBEAT_MS) + this.heartbeat.unref() + } + return { status: 'applied', ...(typeof docId === 'string' ? { docId } : {}) } + } + if (!this.write) throw new Error('FileDocStore is not initialized') + const generation = await this.write.eval(INVALIDATE_DOCUMENT_SCRIPT, { + keys: [ + streamKey(name), + generationKey(name), + `${SYNC_VERSION_PREFIX}${name}`, + `${CLIENT_UPDATE_PREFIX}${name}`, + `${AGENT_STREAM_PREFIX}${name}`, + `${INVALIDATION_VERSION_PREFIX}${name}`, + ], + arguments: [String(version), String(STREAM_TTL_SEC), INVALIDATED_GENERATION], + }) + if (typeof generation !== 'string') return { status: 'stale' } + return { + status: 'applied', + ...(generation && generation !== INVALIDATED_GENERATION ? { docId: generation } : {}), + } + } + + async getDocumentGeneration(name: string): Promise { + if (!this.enabled) return '' + if (!this.write) throw new Error('FileDocStore is not initialized') + return (await this.write.get(generationKey(name))) ?? '' + } + + async isDocumentGenerationCurrent(name: string, generation?: string): Promise { + if (!this.enabled) return !this.rooms.get(name)?.generationInvalidated + if (!this.write) throw new Error('FileDocStore is not initialized') + const current = await this.write.get(generationKey(name)) + return current === null ? !generation : current === generation + } + /** * Whether the file's stream already holds content — an EFFICIENCY recheck in {@link shouldSeed} that * skips the seed fetch when a prior holder already seeded (the split-brain guard itself is the atomic @@ -460,12 +749,15 @@ export class FileDocStore { * disabled store return a truthy token so callers proceed single-replica without special-casing. */ private async acquireLock(key: string, ttlMs: number): Promise { - if (!this.enabled || !this.write) return DISABLED_LOCK_TOKEN + if (!this.enabled) return DISABLED_LOCK_TOKEN + if (!this.write) return null const token = generateId() try { return (await this.write.set(key, token, { NX: true, PX: ttlMs })) === 'OK' ? token : null } catch (error) { - logger.warn(`FileDocStore lock ${key} failed`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore lock ${key} failed`, { + error: getErrorMessage(error), + }) return null } } @@ -504,19 +796,78 @@ export class FileDocStore { * `null` when the stream is empty — i.e. no doc is (or was recently) live, so there is nothing to * merge into and the caller should fall back to a direct file write. Disabled → always null. */ - async getStreamState(name: string): Promise { - if (!this.enabled || !this.write) return null - const entries = await this.write.xRange(streamKey(name), '-', '+') - if (entries.length === 0) return null + async getStreamState(name: string, expectedGeneration?: string): Promise { + if (!this.enabled) return null + if (!this.write) throw new Error('FileDocStore is not initialized') const doc = new Y.Doc() try { - for (const entry of entries) applyEntryToDoc(doc, entry.id, entry.message) + const generation = await this.getDocumentGeneration(name) + if (expectedGeneration !== undefined && generation !== expectedGeneration) { + throw new FileDocInvalidatedError() + } + const count = await this.replayEntries(name, '0', (entry) => { + if (entry.message[GENERATION_FIELD] && entry.message[GENERATION_FIELD] !== generation) { + throw new FileDocInvalidatedError() + } + applyEntryToDoc(doc, entry.id, entry.message) + return true + }) + if ((await this.getDocumentGeneration(name)) !== generation) { + throw new FileDocInvalidatedError() + } + if (count === 0) return null return Y.encodeStateAsUpdate(doc) } finally { doc.destroy() } } + private async replayEntries( + name: string, + afterId: string, + visit: (entry: { id: string; message: Record }) => boolean + ): Promise { + if (!this.write) return 0 + const key = streamKey(name) + let firstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id + if (!firstId) return 0 + const tail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) + if (tail.length === 0) throw new FileDocInvalidatedError() + let endId = tail[0].id + let cursor = afterId.includes('-') ? afterId : `${afterId}-0` + let entriesRead = 0 + let encodedBytes = 0 + + while (true) { + while (isAfterStreamId(endId, cursor)) { + const page = await this.write.xRange(key, `(${cursor}`, '+', { + COUNT: REPLAY_PAGE_COUNT, + }) + if (page.length === 0) { + throw new Error(`File document replay lost its completion barrier for ${name}`) + } + for (const entry of page) { + entriesRead += 1 + encodedBytes += entry.message[UPDATE_FIELD]?.length ?? 0 + if (entriesRead > REPLAY_MAX_ENTRIES || encodedBytes > REPLAY_MAX_ENCODED_BYTES) { + throw new Error(`File document replay exceeded its safety limit for ${name}`) + } + cursor = entry.id + if (!visit(entry)) return entriesRead + } + } + + /** Compaction appends its snapshot before trimming; extend the barrier without rereading it. */ + const currentFirstId = (await this.write.xRange(key, '-', '+', { COUNT: 1 }))[0]?.id + if (!currentFirstId) throw new FileDocInvalidatedError() + if (currentFirstId === firstId) return entriesRead + firstId = currentFirstId + const currentTail = await this.write.xRevRange(key, '+', '-', { COUNT: 1 }) + if (currentTail.length === 0) throw new FileDocInvalidatedError() + endId = currentTail[0].id + } + } + /** Release the seed lock (compare-and-delete) once the seed has been published or a seed attempt failed. */ async releaseSeedLock(name: string, token: string): Promise { await this.releaseLock(`${SEED_LOCK_PREFIX}${name}`, token) @@ -580,7 +931,11 @@ export class FileDocStore { * new value exceeds the stored one ({@link SET_VERSION_IF_NEWER_SCRIPT}), so an out-of-order * fire-and-forget write can't regress the token. Best-effort; TTL-bounded like the stream so an idle * file's key can't outlive its room. No-op when disabled (single-pod fallback). */ - async setSyncedVersion(name: string, version: number): Promise { + async setSyncedVersion( + name: string, + version: number, + expectedGeneration = this.rooms.get(name)?.generation ?? '' + ): Promise { if (!this.enabled || !this.write) return // Retry a transient failure (bounded) rather than swallow it: this token is the ONLY way a // peer-seeded task learns the durable version, so a dropped write would leave that peer's persists @@ -589,8 +944,8 @@ export class FileDocStore { for (let attempt = 0; attempt <= PUBLISH_MAX_RETRIES; attempt++) { try { await this.write.eval(SET_VERSION_IF_NEWER_SCRIPT, { - keys: [`${SYNC_VERSION_PREFIX}${name}`], - arguments: [String(version), String(STREAM_TTL_SEC)], + keys: [`${SYNC_VERSION_PREFIX}${name}`, generationKey(name)], + arguments: [String(version), String(STREAM_TTL_SEC), expectedGeneration], }) return } catch (error) { @@ -637,8 +992,32 @@ export class FileDocStore { await this.releaseLock(`${MERGE_LOCK_PREFIX}${name}`, token) } - private applyEntry(room: StoreRoom, id: string, message: Record): void { + private applyEntry( + name: string, + room: StoreRoom, + id: string, + message: Record + ): void { + if (!isAfterStreamId(id, room.lastId)) return room.lastId = id + const generation = message[GENERATION_FIELD] + if (generation) { + if ( + room.generationInvalidated || + (room.generation !== null && room.generation !== generation) || + (room.generation === null && + room.seededObserved && + room.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.docIdKey) !== generation) + ) { + room.generationInvalidated = true + return + } + room.generation = generation + } + if (room.generationInvalidated) return + const isSnapshot = + message[GENERATION_FIELD] !== undefined || message[SNAPSHOT_FIELD] !== undefined + if (!isSnapshot) room.uncompactedDeltaBytes += message[UPDATE_FIELD]?.length ?? 0 // A compaction snapshot folds seed + edits into one frame; stamp it so the relay's edit-tracker // treats a fresh catch-up from it as edited (a snapshot only exists once real edits accumulated). An // agent-streamed preview frame is stamped separately so the tracker NEVER marks it edited. @@ -657,6 +1036,9 @@ export class FileDocStore { if (origin === REDIS_SNAPSHOT_ORIGIN || (origin === REDIS_ORIGIN && seededBefore)) { room.realEdited = true } + if (!isSnapshot && room.uncompactedDeltaBytes >= COMPACT_ENCODED_BYTES) { + void this.maybeCompact(name) + } } /** @@ -665,6 +1047,7 @@ export class FileDocStore { */ private async runReader(): Promise { let failures = 0 + let blockingBatchIndex = 0 while (this.running && this.read) { const snapshot = new Map(this.rooms) if (snapshot.size === 0) { @@ -672,26 +1055,40 @@ export class FileDocStore { continue } try { - const res = await this.read.xRead( - [...snapshot].map(([name, room]) => ({ key: streamKey(name), id: room.lastId })), - { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT } - ) - // The streak ends HERE, on the read returning at all — not further down once entries are - // applied. A blocking read that times out with nothing new is the idle steady state, and it - // proves the connection works just as well as one carrying messages; leaving the streak - // standing through it would keep an old outage's count alive indefinitely, so the next - // unrelated blip would open at the backoff cap and log a failure count it never earned. - failures = 0 - if (!res) continue - for (const stream of res) { - const name = stream.name.slice(STREAM_PREFIX.length) - const room = this.rooms.get(name) - // Skip if detached mid-read, OR replaced by a close→reopen (a DIFFERENT StoreRoom): applying - // entries read against the OLD room's lastId to the new one could regress its lastId (harmless - // but wasteful re-delivery). The new room caught itself up via xRange already. - if (!room || room !== snapshot.get(name)) continue - for (const entry of stream.messages) this.applyEntry(room, entry.id, entry.message) + const rooms = [...snapshot] + const batches: Array = [] + for (let index = 0; index < rooms.length; index += READ_STREAM_BATCH_SIZE) { + batches.push(rooms.slice(index, index + READ_STREAM_BATCH_SIZE)) + } + const applyResults = (results: Awaited>): boolean => { + if (!results) return false + for (const stream of results) { + const name = stream.name.slice(STREAM_PREFIX.length) + const room = this.rooms.get(name) + if (!room || room !== snapshot.get(name)) continue + for (const entry of stream.messages) + this.applyEntry(name, room, entry.id, entry.message) + } + return true + } + let received = false + for (const batch of batches) { + const streams = batch.map(([name, room]) => ({ + key: streamKey(name), + id: room.lastId, + })) + received = applyResults(await this.read.xRead(streams, { COUNT: READ_COUNT })) || received } + if (!received) { + const batch = batches[blockingBatchIndex % batches.length] + blockingBatchIndex = (blockingBatchIndex + 1) % batches.length + const streams = batch.map(([name, room]) => ({ + key: streamKey(name), + id: room.lastId, + })) + applyResults(await this.read.xRead(streams, { BLOCK: READ_BLOCK_MS, COUNT: READ_COUNT })) + } + failures = 0 } catch (error) { if (!this.running) break await this.recoverReader(++failures, error) @@ -718,7 +1115,12 @@ export class FileDocStore { error: getErrorMessage(error), }) } - await sleep(backoffWithJitter(failures, null, { baseMs: 500, maxMs: READER_RETRY_MAX_MS })) + await sleep( + backoffWithJitter(failures, null, { + baseMs: 500, + maxMs: READER_RETRY_MAX_MS, + }) + ) if (this.running && this.read && !this.read.isOpen) { await this.read.connect().catch((reconnectError) => { logger.warn('FileDocStore could not re-open the reader connection', { @@ -737,13 +1139,20 @@ export class FileDocStore { private async maybeCompact(name: string): Promise { if (!this.write) return const room = this.rooms.get(name) - if (!room) return + if (!room || room.compacting) return + room.compacting = true try { - if ((await this.write.xLen(streamKey(name))) < COMPACT_THRESHOLD) return + const streamLength = await this.write.xLen(streamKey(name)) + if (streamLength < COMPACT_THRESHOLD && room.uncompactedDeltaBytes < COMPACT_ENCODED_BYTES) { + return + } const key = `${COMPACT_LOCK_PREFIX}${name}` const token = await this.acquireLock(key, COMPACT_LOCK_TTL_MS) if (!token) return try { + /** Integrate the completed stream prefix before capturing the snapshot and compaction barrier. */ + await this.catchUp(name) + if (this.rooms.get(name) !== room) return // Capture the snapshot AND the id it covers in one synchronous step (no await between): the // snapshot is `room.doc`, which holds exactly what this task's tailer has integrated — every // entry up to `room.lastId`. Entries a peer task published AFTER that (id > lastId) are NOT in @@ -751,34 +1160,61 @@ export class FileDocStore { // them — only entries the snapshot provably subsumes (id <= lastId). Trimming to the freshly // appended snapshot id instead would silently drop those un-integrated peer entries. const upTo = room.lastId + const deltaBytesAtBarrier = room.uncompactedDeltaBytes const snapshot = Buffer.from(Y.encodeStateAsUpdate(room.doc)).toString('base64') // Stamp the snapshot by what it folds: a real edit → SNAPSHOT_FIELD (a fresh catch-up treats it // as edited content, not a bare seed). An agent-ONLY stream (no real edit yet) → AGENT_FIELD, so a // peer catching up applies it as REDIS_AGENT_ORIGIN and never marks the doc edited — preserving // the no-persist guarantee even when a long copilot stream alone crosses the compaction threshold. const marker = room.realEdited ? SNAPSHOT_FIELD : AGENT_FIELD - await this.write.xAdd(streamKey(name), '*', { - [UPDATE_FIELD]: snapshot, - [marker]: '1', + const snapshotId = await this.write.eval(APPEND_SNAPSHOT_SCRIPT, { + keys: [streamKey(name), generationKey(name)], + arguments: [ + UPDATE_FIELD, + snapshot, + marker, + room.generation ?? '', + GENERATION_FIELD, + upTo, + ], }) - // MINID keeps entries with id >= upTo: the snapshot, any un-integrated peer entries, and - // `upTo` itself (redundant with the snapshot, harmless); it drops only the folded older deltas. - await this.write.xTrim(streamKey(name), 'MINID', upTo) + if (typeof snapshotId !== 'string') return + /** Deltas observed after the barrier survive MINID; snapshots never contribute to this count. */ + room.uncompactedDeltaBytes = Math.max(0, room.uncompactedDeltaBytes - deltaBytesAtBarrier) } finally { await this.releaseLock(key, token) } } catch (error) { - logger.warn(`FileDocStore compaction failed for ${name}`, { error: getErrorMessage(error) }) + logger.warn(`FileDocStore compaction failed for ${name}`, { + error: getErrorMessage(error), + }) + } finally { + room.compacting = false } } + private async refreshDocumentTtls(name: string): Promise { + await this.write?.eval(REFRESH_DOCUMENT_TTLS_SCRIPT, { + keys: documentKeys(name), + arguments: [String(STREAM_TTL_SEC)], + }) + } + private async refreshTtls(): Promise { - if (!this.write) return + if (!this.write) { + const now = Date.now() + for (const [name, invalidation] of this.localInvalidations) { + if (this.rooms.has(name)) invalidation.expiresAt = now + STREAM_TTL_SEC * 1_000 + else if (invalidation.expiresAt <= now) this.localInvalidations.delete(name) + } + if (this.localInvalidations.size === 0 && this.heartbeat) { + clearInterval(this.heartbeat) + this.heartbeat = null + } + return + } for (const name of this.rooms.keys()) { - await this.write.expire(streamKey(name), STREAM_TTL_SEC).catch(() => {}) - // Keep the synced-version key alive as long as its stream, so an open-but-idle doc's persist - // If-Match token can't expire out from under it (which would force a needless reconcile). - await this.write.expire(`${SYNC_VERSION_PREFIX}${name}`, STREAM_TTL_SEC).catch(() => {}) + await this.refreshDocumentTtls(name).catch(() => {}) } } } diff --git a/apps/realtime/src/handlers/file-doc.join-readiness.test.ts b/apps/realtime/src/handlers/file-doc.join-readiness.test.ts index 9b7b6a1c7ec..4938c0680f4 100644 --- a/apps/realtime/src/handlers/file-doc.join-readiness.test.ts +++ b/apps/realtime/src/handlers/file-doc.join-readiness.test.ts @@ -63,12 +63,19 @@ vi.mock('redis', () => { for (let i = 0; i < backing.readDelayTicks; i++) await Promise.resolve() return (backing.streams.get(key) ?? []).map((e) => ({ ...e })) }, + xRevRange: async (key: string) => + [...(backing.streams.get(key) ?? [])] + .reverse() + .slice(0, 1) + .map((entry) => ({ ...entry })), xLen: async (key: string) => (backing.streams.get(key) ?? []).length, - xRead: async (streams: { key: string; id: string }[]) => { + xRead: async (streams: { key: string; id: string }[], options?: { COUNT?: number }) => { const res: { name: string; messages: { id: string; message: Record }[] }[] = [] for (const { key, id } of streams) { - const after = (backing.streams.get(key) ?? []).filter((e) => seqOf(e.id) > seqOf(id)) + const after = (backing.streams.get(key) ?? []) + .filter((e) => seqOf(e.id) > seqOf(id)) + .slice(0, options?.COUNT) if (after.length) res.push({ name: key, messages: after.map((e) => ({ ...e })) }) } if (res.length) return res diff --git a/apps/realtime/src/handlers/file-doc.multireplica.test.ts b/apps/realtime/src/handlers/file-doc.multireplica.test.ts index cfffe81e857..cb24909618f 100644 --- a/apps/realtime/src/handlers/file-doc.multireplica.test.ts +++ b/apps/realtime/src/handlers/file-doc.multireplica.test.ts @@ -25,6 +25,7 @@ const fakeStore = { versions: new Map(), acquireMergeSlot: vi.fn(async () => 'token'), releaseMergeSlot: vi.fn(async () => {}), + getDocumentGeneration: vi.fn(async () => 'shared-generation'), getStreamState: vi.fn(async () => new Uint8Array([1])), publishAndWait: vi.fn(async () => {}), getSyncedVersion: vi.fn(async (name: string) => fakeStore.versions.get(name) ?? null), @@ -67,7 +68,13 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(await applyMarkdownToLiveFileDoc('file-1', '# durable', { version: 100 })).toBe( 'applied' ) - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100, 'shared-generation') + expect(fakeStore.getStreamState).toHaveBeenCalledWith(ROOM_NAME, 'shared-generation') + expect(fakeStore.publishAndWait).toHaveBeenCalledWith( + ROOM_NAME, + expect.any(Uint8Array), + 'shared-generation' + ) mockFetchFileDocMerge.mockClear() // A durable write with an OLDER version than the SHARED synced version is stale — rejected under the @@ -81,7 +88,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' expect(await applyMarkdownToLiveFileDoc('file-1', '# durable again', { version: 150 })).toBe( 'applied' ) - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150) + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 150, 'shared-generation') // setSyncedVersion fired only for the two applied durable writes, never for the stale one. expect(fakeStore.setSyncedVersion).toHaveBeenCalledTimes(2) }) @@ -98,7 +105,7 @@ describe('applyMarkdownToLiveFileDoc — multi-replica (store-enabled) ordering' ).toBe('applied') expect(mockFetchFileDocMerge).not.toHaveBeenCalled() // content deferred to the client expect(fakeStore.publishAndWait).not.toHaveBeenCalled() - expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100) // version still recorded + expect(fakeStore.setSyncedVersion).toHaveBeenCalledWith(ROOM_NAME, 100, 'shared-generation') // version still recorded // Once streaming stops the flag clears and the (now near-noop) durable merge resumes normally. fakeStore.isAgentStreaming.mockResolvedValue(false) diff --git a/apps/realtime/src/handlers/file-doc.test.ts b/apps/realtime/src/handlers/file-doc.test.ts index c0878d90129..ebe1a4502cf 100644 --- a/apps/realtime/src/handlers/file-doc.test.ts +++ b/apps/realtime/src/handlers/file-doc.test.ts @@ -3,6 +3,7 @@ */ import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, FILE_DOC_SEED, } from '@sim/realtime-protocol/file-doc' @@ -37,12 +38,16 @@ vi.mock('@/handlers/file-doc-app', () => ({ import { applyMarkdownToLiveFileDoc, cleanupFileDocForSocket, + fileDocAdmissionRoom, flushAllFileDocRooms, + invalidateLiveFileDocument, setupWorkspaceFileDocHandlers, } from '@/handlers/file-doc' +import { FileDocInvalidatedError, getFileDocStore } from '@/handlers/file-doc-store' +import * as permissions from '@/middleware/permissions' import { beginRoomPermissionRead, commitRoomPermission } from '@/middleware/permissions' -type Handler = (payload?: unknown) => Promise | void +type Handler = (...payload: unknown[]) => Promise | void const ROOM_NAME = 'workspace-file-doc:file-1' @@ -54,16 +59,19 @@ interface SentMessage { } /** An `io` mock that records every server-originated emit with its target/except. */ -function createIo() { +function createIo(deliver?: (message: SentMessage) => void) { const sent: SentMessage[] = [] + const emit = (message: SentMessage) => { + sent.push(message) + deliver?.(message) + } /** Records `io.in(socketId).socketsLeave(room)` — a socket forced out of a room from outside. */ const left: { socketId: string; room: string }[] = [] const to = vi.fn((target: string) => ({ except: (exclude: string) => ({ - emit: (event: string, payload: unknown) => - sent.push({ target, except: exclude, event, payload }), + emit: (event: string, payload: unknown) => emit({ target, except: exclude, event, payload }), }), - emit: (event: string, payload: unknown) => sent.push({ target, event, payload }), + emit: (event: string, payload: unknown) => emit({ target, event, payload }), })) const inFn = vi.fn((socketId: string) => ({ socketsLeave: (room: string) => { @@ -133,10 +141,11 @@ async function flushMicrotasks(): Promise { * An encoded Yjs update shaped like the server seed builder's output: some content in the shared * `default` type plus the {@link FILE_DOC_SEED} flag, so applying it marks the doc seeded. */ -function seedResult(content: string): { update: Uint8Array; version: number } { +function seedResult(content: string, docId?: string): { update: Uint8Array; version: number } { const doc = new Y.Doc() doc.getText(FILE_DOC_FIELD).insert(0, content) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + if (docId) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, docId) return { update: Y.encodeStateAsUpdate(doc), version: 1 } } @@ -196,13 +205,14 @@ describe('setupWorkspaceFileDocHandlers', () => { mockFetchFileDocPersist.mockResolvedValue({ status: 'persisted', version: 1 }) }) - afterEach(() => { + afterEach(async () => { // The room store is module-global; drop every room the test's sockets opened. const { io } = createIo() // Simulate a full disconnect between tests (`endOfLife`) so the module-global join-generation // map is cleared and never bleeds a counter into the next test. for (const id of createdSocketIds) cleanupFileDocForSocket(id, io, true) createdSocketIds.clear() + await getFileDocStore().shutdown() }) it('rejects join when the socket is not authenticated', async () => { @@ -217,6 +227,24 @@ describe('setupWorkspaceFileDocHandlers', () => { ) }) + it('fails closed when authorization does not resolve a workspace context', async () => { + mockAuthorizeRoom.mockResolvedValueOnce({ + allowed: true, + status: 200, + workspacePermission: 'write', + }) + const { io } = createIo() + const { socket, handlers } = setup('socket-no-workspace', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'JOIN_FAILED', retryable: true }) + ) + expect(socket.join).not.toHaveBeenCalled() + }) + it('rejects join with a retryable error when realtime is unavailable', async () => { const { io } = createIo() const { socket, handlers } = createSocket('socket-1') @@ -248,6 +276,196 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockAuthorizeRoom).not.toHaveBeenCalled() }) + it('rejects an incompatible collaborative-document schema before authorizing', async () => { + const { io } = createIo() + const { socket, handlers } = setup('socket-schema', io) + + await handlers[FILE_DOC_EVENTS.JOIN]({ + fileId: 'file-1', + clientId: 1, + schemaVersion: 99, + }) + + expect(socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'SCHEMA_VERSION_MISMATCH', retryable: false }) + ) + expect(mockAuthorizeRoom).not.toHaveBeenCalled() + }) + + it('acknowledges user updates only after applying them to the joined document', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io, sent } = createIo() + const { handlers } = setup('socket-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'acknowledged edit') + const acknowledge = vi.fn() + sent.length = 0 + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-1', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ status: 'accepted', updateId: 'update-1' }) + expect(sent).toContainEqual( + expect.objectContaining({ + target: ROOM_NAME, + event: FILE_DOC_EVENTS.MESSAGE, + }) + ) + source.destroy() + }) + + it('rejects an update for a replaced document without applying it', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-current')) + const { io, sent } = createIo() + const { handlers } = setup('socket-replaced', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const acknowledge = vi.fn() + sent.length = 0 + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-stale', + updateId: 'update-stale', + update: Y.encodeStateAsUpdate(new Y.Doc()), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-stale', + }) + expect(sent).toHaveLength(0) + }) + + it('rejects malformed Yjs updates without retrying them', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io } = createIo() + const { handlers } = setup('socket-malformed-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const acknowledge = vi.fn() + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-malformed', + update: new Uint8Array([255]), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'INVALID_UPDATE', + retryable: false, + updateId: 'update-malformed', + }) + }) + + it('ignores an acknowledged-update event without a callable acknowledgement', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const { io } = createIo() + const { handlers } = setup('socket-missing-ack', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + expect(() => + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-1', + update: Y.encodeStateAsUpdate(new Y.Doc()), + }, + { not: 'a function' } + ) + ).not.toThrow() + }) + + it('keeps a room alive until an acknowledged update finishes appending', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + let resolveAppend: () => void = () => {} + const append = new Promise((resolve) => { + resolveAppend = resolve + }) + const publish = vi + .spyOn(getFileDocStore(), 'publishClientUpdateAndWait') + .mockReturnValue(append) + const { io } = createIo() + const { handlers } = setup('socket-update-leave', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'accepted before leave') + const acknowledge = vi.fn() + + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-leave', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + await vi.waitFor(() => expect(publish).toHaveBeenCalledTimes(1)) + handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveAppend() + + await vi.waitFor(() => + expect(acknowledge).toHaveBeenCalledWith({ + status: 'accepted', + updateId: 'update-leave', + }) + ) + publish.mockRestore() + source.destroy() + }) + + it('rejects a generation-fenced update as a durable document replacement', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Original', 'doc-1')) + const publish = vi + .spyOn(getFileDocStore(), 'publishClientUpdateAndWait') + .mockRejectedValue(new FileDocInvalidatedError()) + const { io } = createIo() + const { handlers } = setup('socket-replaced-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const source = new Y.Doc() + source.getText(FILE_DOC_FIELD).insert(0, 'stale edit') + const acknowledge = vi.fn() + + await handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'doc-1', + updateId: 'update-replaced', + update: Y.encodeStateAsUpdate(source), + }, + acknowledge + ) + + expect(acknowledge).toHaveBeenCalledWith({ + status: 'rejected', + code: 'DOCUMENT_REPLACED', + retryable: false, + updateId: 'update-replaced', + }) + publish.mockRestore() + source.destroy() + }) + it('does not re-enter the room when access was revoked while the join was in flight', async () => { // The sweep records a revocation before it evicts, so a join whose authorize // completed just before that must not put the socket back in the document. @@ -369,6 +587,125 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(mockFetchFileDocPersist).toHaveBeenCalled() }) + it('awaits final persistence of an already removed room during shutdown', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-final-persist', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'last edit') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(edit)) + ) + ) + let finishPersist!: (result: { status: 'persisted'; version: number }) => void + mockFetchFileDocPersist.mockReturnValueOnce( + new Promise((resolve) => { + finishPersist = resolve + }) + ) + cleanupFileDocForSocket('socket-final-persist', io, true) + await flushMicrotasks() + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(1) + const completed = vi.fn() + const flush = flushAllFileDocRooms().then(completed) + await flushMicrotasks() + expect(completed).not.toHaveBeenCalled() + finishPersist({ status: 'persisted', version: 2 }) + await flush + expect(completed).toHaveBeenCalledTimes(1) + edit.destroy() + }) + + it('drains an accepted update still appending when the last socket closes', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server', 'shutdown-doc')) + const { io } = createIo() + const { handlers } = setup('socket-closing-update', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + let finishAppend!: () => void + const append = vi.spyOn(getFileDocStore(), 'publishClientUpdateAndWait').mockReturnValueOnce( + new Promise((resolve) => { + finishAppend = resolve + }) + ) + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'accepted before socket close') + const ack = vi.fn() + handlers[FILE_DOC_EVENTS.UPDATE]( + { + fileId: 'file-1', + docId: 'shutdown-doc', + updateId: 'shutdown-update', + update: Y.encodeStateAsUpdate(edit), + }, + ack + ) + cleanupFileDocForSocket('socket-closing-update', io, true) + const completed = vi.fn() + const flush = flushAllFileDocRooms().then(completed) + await flushMicrotasks() + expect(completed).not.toHaveBeenCalled() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + finishAppend() + await flush + expect(ack).toHaveBeenCalledWith({ status: 'accepted', updateId: 'shutdown-update' }) + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(1) + const persisted = new Y.Doc() + Y.applyUpdate(persisted, mockFetchFileDocPersist.mock.calls[0][3]) + expect(persisted.getText(FILE_DOC_FIELD).toString()).toContain('accepted before socket close') + append.mockRestore() + edit.destroy() + persisted.destroy() + }) + + it.each(['persist', 'invalidate', 'leave'] as const)( + 'retries a throttled persist safely until %s', + async (outcome) => { + vi.useFakeTimers() + const store = getFileDocStore() + const claim = vi + .spyOn(store, 'tryClaimPersistWindow') + .mockResolvedValueOnce(false) + .mockResolvedValue(true) + try { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# From server')) + const { io } = createIo() + const { handlers } = setup('socket-throttled', io) + await handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const edit = new Y.Doc() + edit.getText(FILE_DOC_FIELD).insert(0, 'pending durable edit') + handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(edit)) + ) + ) + await vi.advanceTimersByTimeAsync(5_000) + expect(claim).toHaveBeenCalledTimes(1) + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + if (outcome === 'invalidate') await store.invalidateDocument(ROOM_NAME, 2) + if (outcome === 'leave') { + cleanupFileDocForSocket('socket-throttled', io, true) + await vi.advanceTimersByTimeAsync(0) + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(1) + mockFetchFileDocPersist.mockClear() + } + await vi.advanceTimersByTimeAsync(5_000) + expect(mockFetchFileDocPersist).toHaveBeenCalledTimes(outcome === 'persist' ? 1 : 0) + if (outcome === 'persist') { + const persisted = new Y.Doc() + Y.applyUpdate(persisted, mockFetchFileDocPersist.mock.calls[0][3]) + expect(persisted.getText(FILE_DOC_FIELD).toString()).toContain('pending durable edit') + persisted.destroy() + } + edit.destroy() + } finally { + claim.mockRestore() + vi.useRealTimers() + } + } + ) + it('drops document frames and evicts once the editor loses write access mid-session', async () => { // The join-time check is not a standing right: a collaborator downgraded to `read` // (or removed) must stop landing durable edits on the socket they already hold. @@ -550,6 +887,10 @@ describe('setupWorkspaceFileDocHandlers', () => { FILE_DOC_EVENTS.JOIN_SUCCESS, expect.objectContaining({ fileId: 'file-1', clientId: 1 }) ) + const joinSuccess = socket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.JOIN_SUCCESS + )?.[1] as Record + expect(joinSuccess).not.toHaveProperty('acknowledgedUpdates') // A binary sync-step-1 message (type tag 0) is sent to kick off the handshake. const syncMessage = socket.emit.mock.calls.find( @@ -575,6 +916,333 @@ describe('setupWorkspaceFileDocHandlers', () => { expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# From server') }) + it('discards a fenced in-memory generation before serving the next join', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io, left } = createIo() + const first = setup('socket-old-generation', io) + await first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + + await getFileDocStore().invalidateDocument(ROOM_NAME, 1) + mockFetchFileDocSeed.mockResolvedValue(seedResult('# New', 'doc-new')) + const second = setup('socket-new-generation', io) + await second.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + + expect(left).toContainEqual({ socketId: 'socket-old-generation', room: ROOM_NAME }) + second.socket.emit.mockClear() + second.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeSyncStep1(encoder, new Y.Doc()) + ) + ) + const reply = second.socket.emit.mock.calls.find( + ([event, payload]) => event === FILE_DOC_EVENTS.MESSAGE && payload instanceof Uint8Array + ) + const clientDoc = new Y.Doc() + applySyncReply(reply?.[1] as Uint8Array, clientDoc) + expect(clientDoc.getText(FILE_DOC_FIELD).toString()).toBe('# New') + clientDoc.destroy() + }) + + it.each([false, true])( + 'rejects a pending join invalidated during permission with existing room %s', + async (existingRoom) => { + const seed = seedResult('# Old', 'doc-old') + mockFetchFileDocSeed.mockResolvedValue(seed) + const { io, sent } = createIo() + if (existingRoom) { + const first = setup('socket-first-invalidation', io) + await first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + } + let resolvePermission!: (permission: string) => void + const permission = new Promise((resolve) => { + resolvePermission = resolve + }) + const permissionCheck = vi + .spyOn(permissions, 'resolveCurrentRoomPermission') + .mockImplementationOnce(() => permission) + try { + const pending = setup('socket-pending-invalidation', io) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await vi.waitFor(() => expect(permissionCheck).toHaveBeenCalledOnce()) + expect(await invalidateLiveFileDocument('file-1', 2)).toMatchObject({ status: 'applied' }) + io.to(ROOM_NAME).emit(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1' }) + resolvePermission('write') + await joining + + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ retryable: true }) + ) + const staleClient = new Y.Doc() + Y.applyUpdate(staleClient, seed.update) + const vector = Y.encodeStateVector(staleClient) + staleClient.getText(FILE_DOC_FIELD).insert(0, 'must not accept ') + sent.length = 0 + pending.socket.emit.mockClear() + pending.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(staleClient, vector)) + ) + ) + expect(sent).toHaveLength(0) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.any(Uint8Array) + ) + await flushAllFileDocRooms() + expect(mockFetchFileDocPersist).not.toHaveBeenCalled() + staleClient.destroy() + + mockFetchFileDocSeed.mockResolvedValue({ ...seedResult('# New', 'doc-new'), version: 2 }) + await pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ docId: 'doc-new' }) + ) + } finally { + resolvePermission('write') + permissionCheck.mockRestore() + } + } + ) + + it.each(['write', 'read', null] as const)( + 'withholds document and presence broadcasts until final authorization resolves to %s', + async (permission) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Private', 'doc-private')) + const memberships = new Set() + let pending: ReturnType + const { io } = createIo(({ target, event, payload }) => { + if (memberships.has(target)) pending.socket.emit(event, payload) + }) + pending = setup('socket-pending-authorization', io, { + join: vi.fn((name: string) => { + memberships.add(name) + }), + leave: vi.fn((name: string) => { + memberships.delete(name) + }), + }) + let resolvePermission!: (value: 'write' | 'read' | null) => void + const authorization = new Promise<'write' | 'read' | null>((resolve) => { + resolvePermission = resolve + }) + const guard = vi + .spyOn(permissions, 'resolveCurrentRoomPermission') + .mockResolvedValueOnce('write') + .mockImplementationOnce(() => authorization) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + try { + await vi.waitFor(() => expect(guard).toHaveBeenCalledTimes(2)) + const content = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, seedResult('# Private update').update) + ) + io.local.to(ROOM_NAME).emit(FILE_DOC_EVENTS.MESSAGE, content) + io.to(ROOM_NAME).emit( + FILE_DOC_EVENTS.MESSAGE, + new Uint8Array([FILE_DOC_MESSAGE_TYPE.AWARENESS]) + ) + io.to(ROOM_NAME).emit(FILE_DOC_EVENTS.PRESENCE, [{ userId: 'private-peer' }]) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.anything() + ) + expect(pending.socket.emit).not.toHaveBeenCalledWith( + FILE_DOC_EVENTS.PRESENCE, + expect.anything() + ) + expect(memberships.has(ROOM_NAME)).toBe(false) + resolvePermission(permission) + await joining + expect(memberships.has(ROOM_NAME)).toBe(permission === 'write') + if (permission === 'write') { + expect(joinSuccessFileId(pending.socket)).toBe('file-1') + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.any(Uint8Array) + ) + } else { + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_ERROR, + expect.objectContaining({ code: 'ACCESS_DENIED', retryable: false }) + ) + } + } finally { + resolvePermission(permission) + await joining + guard.mockRestore() + } + } + ) + + it('receives invalidation while the subscribed generation check is pending', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const memberships = new Set() + let pending: ReturnType + const { io } = createIo(({ target, event, payload }) => { + if (memberships.has(target)) pending.socket.emit(event, payload) + }) + pending = setup('socket-subscribed-invalidation', io, { + join: vi.fn((name: string) => { + memberships.add(name) + }), + leave: vi.fn((name: string) => { + memberships.delete(name) + }), + }) + let resolveGeneration!: (current: boolean) => void + const currentGeneration = new Promise((resolve) => { + resolveGeneration = resolve + }) + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockImplementationOnce(() => currentGeneration) + try { + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await vi.waitFor(() => expect(guard).toHaveBeenCalledOnce()) + expect(memberships.has(ROOM_NAME)).toBe(false) + expect(memberships.has(fileDocAdmissionRoom('file-1'))).toBe(true) + await getFileDocStore().invalidateDocument(ROOM_NAME, 2) + io.to(fileDocAdmissionRoom('file-1')).emit(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1' }) + expect(pending.socket.emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + }) + await pending.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + resolveGeneration(true) + await joining + + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(memberships.has(ROOM_NAME)).toBe(false) + expect(memberships.has(fileDocAdmissionRoom('file-1'))).toBe(false) + } finally { + resolveGeneration(true) + guard.mockRestore() + } + }) + + it.each(['invalidation', 'revocation', 'leave'] as const)( + 'rolls back an asynchronous room subscription interrupted by %s', + async (interruption) => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io } = createIo() + let finishSubscription!: () => void + const subscription = new Promise((resolve) => { + finishSubscription = resolve + }) + const pending = setup('socket-async-subscription', io, { + join: vi.fn(() => subscription), + }) + const joining = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + await vi.waitFor(() => expect(pending.socket.join).toHaveBeenCalledOnce()) + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + if (interruption === 'invalidation') { + await getFileDocStore().invalidateDocument(ROOM_NAME, 2) + } else if (interruption === 'revocation') { + commitRoomPermission( + 'user-1', + { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: 'file-1' }, + 'read', + beginRoomPermissionRead() + ) + } else { + await pending.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) + } + finishSubscription() + await joining + expect(joinSuccessFileId(pending.socket)).toBeUndefined() + expect(pending.socket.leave).toHaveBeenCalledWith(ROOM_NAME) + } + ) + + it('keeps a shared provisional subscription until the other provider finishes joining', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Shared', 'doc-shared')) + const { io } = createIo() + const pending = setup('socket-shared-subscription', io) + const checks: Array<(current: boolean) => void> = [] + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockImplementation(() => new Promise((resolve) => checks.push(resolve))) + try { + const first = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const second = pending.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + await vi.waitFor(() => expect(checks).toHaveLength(2)) + checks[0](false) + await first + expect(pending.socket.leave).not.toHaveBeenCalled() + checks[1](true) + await second + expect(pending.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.JOIN_SUCCESS, + expect.objectContaining({ clientId: 2, docId: 'doc-shared' }) + ) + expect(pending.socket.leave).not.toHaveBeenCalledWith(ROOM_NAME) + expect(pending.socket.leave).toHaveBeenCalledWith(fileDocAdmissionRoom('file-1')) + } finally { + for (const resolve of checks) resolve(true) + guard.mockRestore() + } + }) + + it('preserves the committed binding when a co-mounted provider join fails', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Shared', 'doc-shared')) + const { io } = createIo() + const current = setup('socket-existing-subscription', io) + await current.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockRejectedValueOnce(new Error('Temporary generation read failure')) + try { + await current.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + expect(current.socket.leave).not.toHaveBeenCalledWith(ROOM_NAME) + current.socket.emit.mockClear() + current.handlers[FILE_DOC_EVENTS.MESSAGE]( + frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeSyncStep1(encoder, new Y.Doc()) + ) + ) + expect(current.socket.emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.MESSAGE, + expect.any(Uint8Array) + ) + } finally { + guard.mockRestore() + } + }) + + it('does not discard a rebuilt room after a delayed check of its predecessor', async () => { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# Old', 'doc-old')) + const { io, left } = createIo() + const original = setup('socket-original', io) + await original.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + const checks: Array<(current: boolean) => void> = [] + const guard = vi + .spyOn(getFileDocStore(), 'isDocumentGenerationCurrent') + .mockResolvedValue(true) + .mockImplementationOnce(() => new Promise((resolve) => checks.push(resolve))) + .mockImplementationOnce(() => new Promise((resolve) => checks.push(resolve))) + try { + mockFetchFileDocSeed.mockResolvedValue(seedResult('# New', 'doc-new')) + const first = setup('socket-first-new', io) + const second = setup('socket-second-new', io) + const firstJoin = first.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 2 }) + const secondJoin = second.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 3 }) + await vi.waitFor(() => expect(checks).toHaveLength(2)) + checks[0](false) + await firstJoin + checks[1](false) + await secondJoin + + expect(joinSuccessFileId(first.socket)).toBe('file-1') + expect(joinSuccessFileId(second.socket)).toBe('file-1') + expect(left).toContainEqual({ socketId: 'socket-original', room: ROOM_NAME }) + expect(left).not.toContainEqual({ socketId: 'socket-first-new', room: ROOM_NAME }) + } finally { + guard.mockRestore() + } + }) + it('seeds once across concurrent joiners, and every one of them waits for that seed', async () => { // Keep the first seed fetch IN FLIGHT so the doc is still unseeded when the second socket joins: // that forces the dedup onto the in-flight seed rather than `isDocSeeded`. Both joins must WAIT @@ -918,7 +1586,8 @@ describe('setupWorkspaceFileDocHandlers', () => { FILE_DOC_EVENTS.JOIN_ERROR, expect.objectContaining({ code: 'CLIENT_ID_IN_USE' }) ) - expect(b.socket.join).not.toHaveBeenCalled() + expect(b.socket.leave).toHaveBeenCalledWith(ROOM_NAME) + expect(joinSuccessFileId(b.socket)).toBeUndefined() }) it('reclaims a client id for the SAME user reconnecting (reused Yjs client id)', async () => { @@ -1030,6 +1699,31 @@ describe('setupWorkspaceFileDocHandlers', () => { ).not.toThrow() }) + it('drops a legacy frame that cannot fit the durable stream budget', async () => { + const { io, sent } = createIo() + const a = setup('socket-oversized-legacy', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + sent.length = 0 + + expect(() => + a.handlers[FILE_DOC_EVENTS.MESSAGE](new Uint8Array(FILE_DOC_LIMITS.updateBytes + 65)) + ).not.toThrow() + expect(sent).toHaveLength(0) + }) + + it('preflights the inner legacy update before applying a framing-sized overflow', async () => { + const { io, sent } = createIo() + const a = setup('socket-inner-oversized-legacy', io) + await a.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) + sent.length = 0 + const oversized = frame(FILE_DOC_MESSAGE_TYPE.SYNC, (encoder) => + syncProtocol.writeUpdate(encoder, new Uint8Array(FILE_DOC_LIMITS.updateBytes + 1)) + ) + + expect(() => a.handlers[FILE_DOC_EVENTS.MESSAGE](oversized)).not.toThrow() + expect(sent).toHaveLength(0) + }) + it('drops the document when the last editor leaves, re-seeding a fresh joiner from the server', async () => { const { io } = createIo() const a = setup('socket-a', io) @@ -1053,12 +1747,22 @@ describe('setupWorkspaceFileDocHandlers', () => { let resolveFirst: (v: unknown) => void = () => {} mockAuthorizeRoom .mockReturnValueOnce(new Promise((resolve) => (resolveFirst = resolve))) - .mockResolvedValueOnce({ allowed: true, status: 200, workspacePermission: 'write' }) + .mockResolvedValueOnce({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) const s = setup('socket-a', io) const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) await s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) - resolveFirst({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveFirst({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending // The socket is bound only to the newer file, never cross-bound to file-1. @@ -1075,7 +1779,12 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-1', clientId: 1 }) s.socket.disconnected = true cleanupFileDocForSocket('socket-a', io, true) // disconnect cleanup — no-op, nothing registered yet - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(s.socket.join).not.toHaveBeenCalled() @@ -1095,7 +1804,12 @@ describe('setupWorkspaceFileDocHandlers', () => { const pending = s.handlers[FILE_DOC_EVENTS.JOIN]({ fileId: 'file-2', clientId: 1 }) // A stale leave for a DIFFERENT file must not invalidate the in-flight join. s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(joinSuccessFileId(s.socket)).toBe('file-2') @@ -1119,7 +1833,12 @@ describe('setupWorkspaceFileDocHandlers', () => { // map (`undefined !== generation`) and abort the join the client actually wants. s.handlers[FILE_DOC_EVENTS.LEAVE]({ fileId: 'file-1' }) - resolveAuth({ allowed: true, status: 200, workspacePermission: 'write' }) + resolveAuth({ + allowed: true, + status: 200, + workspacePermission: 'write', + workspaceId: 'ws-1', + }) await pending expect(joinSuccessFileId(s.socket)).toBe('file-2') @@ -1241,7 +1960,8 @@ describe('setupWorkspaceFileDocHandlers', () => { ) // The rejected switch must leave file-1 intact — a is not torn out of its current document. expect(a.socket.leave).not.toHaveBeenCalledWith('workspace-file-doc:file-1') - expect(a.socket.join).not.toHaveBeenCalledWith('workspace-file-doc:file-2') + expect(a.socket.leave).toHaveBeenCalledWith('workspace-file-doc:file-2') + expect(joinSuccessFileId(a.socket)).toBe('file-1') }) it('broadcasts a server-authenticated presence roster on join, one entry per session', async () => { diff --git a/apps/realtime/src/handlers/file-doc.ts b/apps/realtime/src/handlers/file-doc.ts index 28ef6686f7a..b3b1020e24e 100644 --- a/apps/realtime/src/handlers/file-doc.ts +++ b/apps/realtime/src/handlers/file-doc.ts @@ -27,10 +27,15 @@ import { createLogger } from '@sim/logger' import { ROOM_MEMBERSHIP_ACTIONS, satisfiesRoomMembership } from '@sim/platform-authz/room-policy' import { FILE_DOC_EVENTS, + FILE_DOC_LEGACY_SCHEMA_VERSION, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, FILE_DOC_TIMEOUTS, type FileDocPresenceUser, + type FileDocUpdateAck, + type FileDocUpdatePayload, type JoinFileDocPayload, type LeaveFileDocPayload, toFileDocBytes, @@ -47,6 +52,7 @@ import * as Y from 'yjs' import { resolveAvatarUrl } from '@/handlers/avatar' import { fetchFileDocMerge, fetchFileDocPersist, fetchFileDocSeed } from '@/handlers/file-doc-app' import { + FileDocInvalidatedError, getFileDocStore, REDIS_AGENT_ORIGIN, REDIS_ORIGIN, @@ -176,7 +182,7 @@ interface FileDocRoom { agentStreamingUntil: number /** * Resolves once this room's doc reflects the file's shared stream (see {@link FileDocStore.catchUp}). - * Never rejects — the catch-up logs and gives up — so awaiting it can never fail a join. + * Rejects when replay cannot complete so the join fails closed rather than serving partial state. */ hydrated: Promise /** @@ -185,10 +191,14 @@ interface FileDocRoom { * document being assembled. A room with a join in flight is not idle. */ pendingJoins: number + /** Acknowledged updates currently waiting for their durable stream append. */ + pendingUpdates: number } /** Live documents keyed by Socket.IO room name. Module-global: one Y.Doc per file. */ const fileDocRooms = new Map() +const pendingFileDocPersists = new Set>() +const pendingFileDocUpdates = new Set>() /** socketId → its current file-doc room name (a socket edits at most one doc). */ const socketToRoomName = new Map() /** @@ -217,14 +227,40 @@ const fileDocRoom = (fileId: string): RoomRef => ({ id: fileId, }) +/** Pending admissions receive invalidations here, never document or presence frames. */ +export function fileDocAdmissionRoom(fileId: string): string { + return `file-doc-admission:${fileId}` +} + /** * A `y-protocols` transaction/awareness origin is the emitting socket id (a * string) when it came from a client, and something else (`null` / `'local'` / * `'timeout'`) for server-internal changes. Returns the socket id to exclude * from a relay, or `null` to broadcast to the whole room. */ +interface ClientUpdateOrigin { + kind: 'client-update' + socketId: string +} + +const MAX_CLIENT_UPDATE_ID_LENGTH = 128 + +function clientUpdateOrigin(socketId: string): ClientUpdateOrigin { + return { kind: 'client-update', socketId } +} + +function isClientUpdateOrigin(origin: unknown): origin is ClientUpdateOrigin { + return ( + typeof origin === 'object' && + origin !== null && + (origin as Partial).kind === 'client-update' && + typeof (origin as Partial).socketId === 'string' + ) +} + function originSocketId(origin: unknown): string | null { - return typeof origin === 'string' ? origin : null + if (typeof origin === 'string') return origin + return isClientUpdateOrigin(origin) ? origin.socketId : null } /** @@ -235,6 +271,28 @@ function originSocketId(origin: unknown): string | null { * on its own echo because the operations are already applied locally. */ const AGENT_SYNC_ORIGIN = Symbol('file-doc-agent-sync') +/** Maximum legacy framed message size: raw update budget plus small Yjs framing headroom. */ +const MAX_LEGACY_FRAME_BYTES = FILE_DOC_LIMITS.updateBytes + 64 + +/** + * Checks the inner update before readSyncMessage mutates the room: the legacy outer-frame limit + * includes framing headroom, which must not allow an update too large for the shared stream. + */ +function hasOversizedLegacyUpdate(bytes: Uint8Array): boolean { + const decoder = decoding.createDecoder(bytes) + const messageType = decoding.readVarUint(decoder) + if ( + messageType !== FILE_DOC_MESSAGE_TYPE.SYNC && + messageType !== FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST + ) { + return false + } + const syncType = decoding.readVarUint(decoder) + if (syncType !== syncProtocol.messageYjsSyncStep2 && syncType !== syncProtocol.messageYjsUpdate) { + return false + } + return decoding.readVarUint8Array(decoder).byteLength > FILE_DOC_LIMITS.updateBytes +} /** * Broadcast an AWARENESS frame to the room ACROSS tasks via the Socket.IO Redis adapter. Awareness @@ -295,10 +353,19 @@ function schedulePersist(name: string, room: FileDocRoom): void { * fallback before any await, so a `void flushPersist(name, room, true)` fired immediately before the * caller destroys `room.doc` never encodes a destroyed doc, and the disabled path stays authoritative. */ -async function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { +function flushPersist(name: string, room: FileDocRoom, final: boolean): Promise { + const pending = persistRoom(name, room, final).finally(() => + pendingFileDocPersists.delete(pending) + ) + pendingFileDocPersists.add(pending) + return pending +} + +async function persistRoom(name: string, room: FileDocRoom, final: boolean): Promise { // Never project a doc no user actually edited back over the file (see {@link FileDocRoom.edited}). if (!room.edited || !room.workspaceId || !room.lastEditorUserId) return const store = getFileDocStore() + const generation = docIdOf(room.doc) const workspaceId = room.workspaceId const userId = room.lastEditorUserId // Synchronous fallback capture — before any await, since the caller may destroy `room.doc` the moment @@ -321,6 +388,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr try { return (await store.getStreamState(name)) ?? localState } catch (streamError) { + if (streamError instanceof FileDocInvalidatedError) throw streamError // A transient Redis read must NOT drop the write when we already hold a valid local snapshot — // else the last-disconnect flush loses the session's edits as the room is torn down. But once a // reconcile has run, `localState` is NULLED (it predates the merged-in out-of-band edit), so a @@ -344,8 +412,11 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr } try { - if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) + if (!(await store.isDocumentGenerationCurrent(name, generation))) return + if (!final && !(await store.tryClaimPersistWindow(name, FILE_DOC_TIMEOUTS.persistRequestMs))) { + if (fileDocRooms.get(name) === room) schedulePersist(name, room) return + } // The If-Match token: the durable content version the live doc is synced to. let ifMatch = await currentVersion() @@ -367,6 +438,7 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // out-of-band edit. A single attempt — on conflict we STOP rather than retry (see below). const docState = await captureState() if (!docState) return // nothing seeded/authoritative to persist yet + if (!(await store.isDocumentGenerationCurrent(name, generation))) return const result = await fetchFileDocPersist(workspaceId, room.fileId, userId, docState, ifMatch) if (result.status === 'missing') return // the file was deleted; nothing to write if (result.status === 'deferred') { @@ -382,23 +454,21 @@ async function flushPersist(name: string, room: FileDocRoom, final: boolean): Pr // here means a task that exits in the moments after a write comes back holding a version older // than the file's, and — since a conflict neither writes nor advances the token — never persists // that document again. One round trip after a blob write is not a cost worth that. - await store.setSyncedVersion(name, result.version) + await store.setSyncedVersion(name, result.version, generation) return } - // status === 'conflict': the durable file advanced out-of-band since our If-Match token. We do NOT - // re-persist against the current stream: an external write commits durable BEFORE its chokepoint merge - // (`mergeEditIntoLiveFileDoc`) reaches the stream, so a re-persist landing in that window would CAS-pass - // with a stream that still lacks the external content and clobber the committed write. Instead leave the - // durable content authoritative — the chokepoint merges the change into the stream and, ONLY once it is - // actually there, advances the synced version (via the merge's own `recordVersion`); a later flush - // (a subsequent debounced persist, or the final flush) then projects the converged stream with a token - // that matches. The session's edits stay in the stream meanwhile. Deliberately do NOT advance the synced - // version here: before the stream reflects the durable content, that would let the next flush clobber it. + /** + * External writes commit before merging into the stream. Retrying or advancing the synced + * version here could overwrite content not yet merged; leave the durable file authoritative + * until the merge advances the version, then let a later flush persist the converged state. + */ logger.warn( `Persist conflict for file ${room.fileId}; durable content advanced out-of-band, left authoritative` ) } catch (error) { - logger.warn(`Persist failed for file ${room.fileId}`, { error: getErrorMessage(error) }) + logger.warn(`Persist failed for file ${room.fileId}`, { + error: getErrorMessage(error), + }) } } @@ -469,7 +539,7 @@ function awarenessUpdateClientIds(update: Uint8Array): number[] { */ function destroyRoomIfIdle(name: string) { const room = fileDocRooms.get(name) - if (!room || room.owners.size > 0 || room.pendingJoins > 0) return + if (!room || room.owners.size > 0 || room.pendingJoins > 0 || room.pendingUpdates > 0) return room.persistDeadline = null if (room.persistTimer) { clearTimeout(room.persistTimer) @@ -484,6 +554,27 @@ function destroyRoomIfIdle(name: string) { fileDocRooms.delete(name) } +/** + * Drop a seeded in-memory generation after an out-of-band durable replacement. It must not flush: the + * durable replacement is newer, and persisting this superseded document would only create a conflict. + * Existing clients are removed from the room before the next join creates and seeds a fresh document. + */ +function discardInvalidatedRoom(name: string, io: Server): void { + const room = fileDocRooms.get(name) + if (!room) return + room.persistDeadline = null + if (room.persistTimer) clearTimeout(room.persistTimer) + room.persistTimer = null + for (const socketId of room.owners.keys()) { + if (socketToRoomName.get(socketId) === name) socketToRoomName.delete(socketId) + io.in(socketId).socketsLeave(name) + } + getFileDocStore().detachRoom(name) + room.awareness.destroy() + room.doc.destroy() + fileDocRooms.delete(name) +} + /** * Flush every open, edited room's converged doc to durable markdown, AWAITING the writes. Called on * graceful shutdown (rolling deploy / scale-in) so edits since the last debounce aren't left only in the @@ -492,18 +583,17 @@ function destroyRoomIfIdle(name: string) { * process is exiting); only their durable state is secured. */ export async function flushAllFileDocRooms(): Promise { + await Promise.all([...pendingFileDocUpdates]) const flushes: Promise[] = [] for (const [name, room] of fileDocRooms) { if (room.edited) flushes.push(flushPersist(name, room, true)) } - await Promise.all(flushes) + await Promise.all([...pendingFileDocPersists, ...flushes]) } /** - * Bring a room's document to its AUTHORITATIVE state — reflecting the file's shared stream and - * carrying its seed — so the join can attach a client to a document that is already whole. Never - * rejects: a room that cannot be seeded is served unseeded, which the client's readiness deadline - * turns into its read-only fallback, exactly as an unreachable relay does. + * Waits for shared hydration and authoritative seeding before joining; failures must not expose + * an editable partial document. */ async function ensureRoomReady( name: string, @@ -513,8 +603,12 @@ async function ensureRoomReady( await room.hydrated // The room can be dropped and re-created while the catch-up is in flight (a fast open→close); the // join re-checks identity after this and abandons a stale room rather than serving from it. - if (fileDocRooms.get(name) !== room || !workspaceId) return + if (fileDocRooms.get(name) !== room) return + if (!workspaceId) throw new Error(`File document ${room.fileId} has no workspace context`) await ensureServerSeed(name, room, workspaceId) + if (fileDocRooms.get(name) === room && !isDocSeeded(room.doc)) { + throw new Error(`File document ${room.fileId} could not be seeded`) + } } /** @@ -588,7 +682,7 @@ async function seedUnderLock( // the doc unseeded and the stream empty for a clean retry. SEED_ORIGIN keeps `doc.on('update')` from // re-publishing it. const seedUpdate = seed?.update ?? emptySeedUpdate() - const didSeed = await store.seedIfEmpty(name, seedUpdate) + const didSeed = await store.seedIfEmpty(name, seedUpdate, seed?.version) // Record the durable version the moment THIS task's seed is in the stream — BEFORE the liveness/ // seeded guard below. Recording it only now that our seed WON (not from the fetch, before knowing who // won) keeps it in step with the stream's actual content: a newer own-fetch version could otherwise @@ -601,7 +695,6 @@ async function seedUnderLock( if (didSeed && seed) { const live = fileDocRooms.get(name) if (live) live.syncedVersion = Math.max(live.syncedVersion ?? 0, seed.version) - void store.setSyncedVersion(name, seed.version) } if (fileDocRooms.get(name) !== room || isDocSeeded(room.doc)) return if (didSeed) { @@ -671,18 +764,50 @@ export function applyMarkdownToLiveFileDoc( order: MergeOrder = {} ): Promise<'applied' | 'no-live-room' | 'merge-unavailable' | 'stale'> { const name = roomName(fileDocRoom(fileId)) + return serializeFileDocMutation(name, () => mergeMarkdownIntoRoom(name, fileId, markdown, order)) +} + +function serializeFileDocMutation(name: string, operation: () => Promise): Promise { const prior = fileDocMergeChains.get(name) ?? Promise.resolve() - // `.catch` so a failed prior merge doesn't reject this one — each merge is independent. - const run = prior.catch(() => {}).then(() => mergeMarkdownIntoRoom(name, fileId, markdown, order)) - fileDocMergeChains.set( - name, - run.finally(() => { + const run = prior + .catch(() => {}) + .then(operation) + .finally(() => { if (fileDocMergeChains.get(name) === run) fileDocMergeChains.delete(name) }) - ) + fileDocMergeChains.set(name, run) return run } +async function acquireFileDocMergeSlot(name: string): Promise { + const store = getFileDocStore() + let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { + await sleep(MERGE_LOCK_RETRY_MS) + token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) + } + return token +} + +/** Serializes and version-orders an unsupported durable replacement with live Markdown merges. */ +export function invalidateLiveFileDocument( + fileId: string, + version: number +): Promise<{ status: 'applied'; docId?: string } | { status: 'stale' }> { + const name = roomName(fileDocRoom(fileId)) + return serializeFileDocMutation(name, async () => { + const store = getFileDocStore() + const token = await acquireFileDocMergeSlot(name) + if (!token) throw new Error('Live document invalidation slot is temporarily unavailable') + try { + if ((fileDocRooms.get(name)?.syncedVersion ?? 0) > version) return { status: 'stale' } + return await store.invalidateDocument(name, version) + } finally { + await store.releaseMergeSlot(name, token) + } + }) +} + async function mergeMarkdownIntoRoom( name: string, fileId: string, @@ -695,14 +820,16 @@ async function mergeMarkdownIntoRoom( // in Redis for multi-task, plus this task's room) so the persist If-Match guard treats this write as // synced rather than an out-of-band conflict. AWAITED so the version is durable before the merge lock // releases, so the next lock holder's staleness check (below) reads a consistent value. - const recordVersion = async () => { + const recordVersion = async (generation?: string) => { if (version === undefined) return const room = fileDocRooms.get(name) // Never regress the token: merges/seeds/persists all write it, so a lower value arriving out of // order must not shadow a higher one the doc already incorporates (the Redis side is guarded // identically by SET_VERSION_IF_NEWER_SCRIPT). - if (room) room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) - await store.setSyncedVersion(name, version) + if (room && (generation === undefined || docIdOf(room.doc) === generation)) { + room.syncedVersion = Math.max(room.syncedVersion ?? 0, version) + } + await store.setSyncedVersion(name, version, generation) } // Order this merge on the file's version line, where `current` is the durable version the doc already @@ -723,11 +850,7 @@ async function mergeMarkdownIntoRoom( // always releases (or its lock expires) first and we acquire — never merging against a shared base // while a peer holds the lock. If somehow still unavailable, skip the live merge (copilot's durable // file write stands) rather than race. - let token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) - for (let i = 0; !token && i < MERGE_LOCK_RETRIES; i++) { - await sleep(MERGE_LOCK_RETRY_MS) - token = await store.acquireMergeSlot(name, MERGE_LOCK_TTL_MS) - } + const token = await acquireFileDocMergeSlot(name) if (!token) { logger.warn(`Merge lock unavailable for file ${fileId}; skipping live merge`) return 'merge-unavailable' @@ -738,13 +861,14 @@ async function mergeMarkdownIntoRoom( const shared = await store.getSyncedVersion(name) const current = Math.max(shared ?? 0, fileDocRooms.get(name)?.syncedVersion ?? 0) if (isStale(current)) return 'stale' + const generation = await store.getDocumentGeneration(name) // Defer to an actively-streaming client: it is applying this SAME agent edit into the shared doc // frame-by-frame, so also publishing a whole-document merge here would double-write the content (the // client's private shadow never observes this merge, so it re-inserts what we added → duplication). // Still record the durable version so the persist If-Match stays correct; the client owns the bytes, // and once streaming stops the flag clears and the final durable merge lands as a near-noop. if (await store.isAgentStreaming(name)) { - await recordVersion() + await recordVersion(generation) return 'applied' } // Compute the diff against the committed SHARED state and PUBLISH it — every task with the doc @@ -752,11 +876,11 @@ async function mergeMarkdownIntoRoom( // merge reaches the live doc no matter which task the apply-edit call landed on. An empty stream // means no doc is (or was recently) live → nothing to merge into. AWAIT the publish so the diff is // durably in the stream before we release the lock (else the next task would diff a stale base). - const base = await store.getStreamState(name) + const base = await store.getStreamState(name, generation) if (!base) return 'no-live-room' const diff = await fetchFileDocMerge(fileId, base, markdown) - await store.publishAndWait(name, diff) - await recordVersion() + await store.publishAndWait(name, diff, generation) + await recordVersion(generation) return 'applied' } finally { await store.releaseMergeSlot(name, token) @@ -815,6 +939,7 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { agentStreamingUntil: 0, hydrated, pendingJoins: 0, + pendingUpdates: 0, } // Register synchronously BEFORE the async catch-up so a concurrent join sees this room, not a second. fileDocRooms.set(name, room) @@ -839,7 +964,8 @@ function getOrCreateRoom(io: Server, ref: RoomRef): FileDocRoom { origin !== REDIS_ORIGIN && origin !== REDIS_SNAPSHOT_ORIGIN && origin !== REDIS_AGENT_ORIGIN && - origin !== SEED_ORIGIN + origin !== SEED_ORIGIN && + !isClientUpdateOrigin(origin) ) getFileDocStore().publish(name, update, origin === AGENT_SYNC_ORIGIN) // A locally-originated agent frame (this task's stream leader) means a client is applying this agent @@ -979,10 +1105,24 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { const bytes = toFileDocBytes(data) if (!bytes) return + if (bytes.byteLength > MAX_LEGACY_FRAME_BYTES) { + logger.warn('Dropping an oversized legacy file-doc frame', { + socketId: socket.id, + bytes: bytes.byteLength, + }) + return + } // A malformed frame from any client must never escape as a process-level // exception; drop it and keep the relay running. try { + if (hasOversizedLegacyUpdate(bytes)) { + logger.warn('Dropping a legacy file-doc update outside the durable stream budget', { + socketId: socket.id, + bytes: bytes.byteLength, + }) + return + } const decoder = decoding.createDecoder(bytes) const messageType = decoding.readVarUint(decoder) @@ -1027,7 +1167,9 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { // owned by this socket. const owned = room.owners.get(socket.id) if (owned === undefined || awarenessUpdateClientIds(update).some((id) => !owned.has(id))) { - logger.warn('Dropping awareness frame for an unowned client id', { socketId: socket.id }) + logger.warn('Dropping awareness frame for an unowned client id', { + socketId: socket.id, + }) return } awarenessProtocol.applyAwarenessUpdate(room.awareness, update, socket.id) @@ -1037,7 +1179,112 @@ function handleMessage(socket: AuthenticatedSocket, io: Server, data: unknown) { logger.warn('Unknown file-doc message type', { messageType }) } } catch (error) { - logger.warn('Dropping malformed file-doc frame', { socketId: socket.id, error }) + logger.warn('Dropping malformed file-doc frame', { + socketId: socket.id, + error, + }) + } +} + +async function handleClientUpdate( + socket: AuthenticatedSocket, + io: Server, + data: unknown, + acknowledge: (result: FileDocUpdateAck) => void +): Promise { + const reject = ( + code: Extract['code'], + retryable: boolean, + updateId?: string + ) => acknowledge({ status: 'rejected', code, retryable, updateId }) + + if (typeof data !== 'object' || data === null) { + reject('INVALID_UPDATE', false) + return + } + + const candidate = data as Partial + const update = toFileDocBytes(candidate.update) + if ( + typeof candidate.fileId !== 'string' || + candidate.fileId.length === 0 || + typeof candidate.docId !== 'string' || + candidate.docId.length === 0 || + typeof candidate.updateId !== 'string' || + candidate.updateId.length === 0 || + candidate.updateId.length > MAX_CLIENT_UPDATE_ID_LENGTH || + !update || + update.byteLength === 0 || + update.byteLength > FILE_DOC_LIMITS.updateBytes + ) { + reject('INVALID_UPDATE', false, candidate.updateId) + return + } + + const name = socketToRoomName.get(socket.id) + if (!name || name !== roomName(fileDocRoom(candidate.fileId))) { + reject('NOT_JOINED', true, candidate.updateId) + return + } + const room = fileDocRooms.get(name) + if (!room) { + reject('NOT_JOINED', true, candidate.updateId) + return + } + if (!isFileDocWriteAllowed(socket, io, name)) { + reject('ACCESS_REVOKED', false, candidate.updateId) + return + } + if (docIdOf(room.doc) !== candidate.docId) { + reject('DOCUMENT_REPLACED', false, candidate.updateId) + return + } + + const validationDoc = new Y.Doc() + try { + Y.applyUpdate(validationDoc, update) + } catch (error) { + logger.warn('Dropping malformed acknowledged file-doc update', { + socketId: socket.id, + fileId: candidate.fileId, + updateId: candidate.updateId, + error, + }) + reject('INVALID_UPDATE', false, candidate.updateId) + return + } finally { + validationDoc.destroy() + } + + const editor = room.owners.get(socket.id)?.values().next().value?.userId + if (editor) room.lastEditorUserId = editor + room.pendingUpdates += 1 + try { + await getFileDocStore().publishClientUpdateAndWait( + name, + candidate.updateId, + update, + candidate.docId + ) + Y.applyUpdate(room.doc, update, clientUpdateOrigin(socket.id)) + room.edited = true + schedulePersist(name, room) + acknowledge({ status: 'accepted', updateId: candidate.updateId }) + } catch (error) { + if (error instanceof FileDocInvalidatedError) { + reject('DOCUMENT_REPLACED', false, candidate.updateId) + return + } + logger.error('Failed to accept acknowledged file-doc update', { + socketId: socket.id, + fileId: candidate.fileId, + updateId: candidate.updateId, + error, + }) + reject('TEMPORARY_FAILURE', true, candidate.updateId) + } finally { + room.pendingUpdates -= 1 + destroyRoomIfIdle(name) } } @@ -1102,11 +1349,15 @@ export function setupWorkspaceFileDocHandlers( // awaiting authorization can't complete after the client left and register a ghost owner. A // leave for a DIFFERENT file must NOT cancel it (a document switch), mirroring workspace-files. let currentFileId: string | null = null + /** Co-mounted providers share invalidation membership until their last admission settles. */ + const pendingMemberships = new Map() - socket.on(FILE_DOC_EVENTS.JOIN, async ({ fileId, clientId }: JoinFileDocPayload) => { + socket.on(FILE_DOC_EVENTS.JOIN, async (payload: JoinFileDocPayload) => { + const { fileId, clientId } = payload // Hoisted so the catch can tell whether this join was superseded (a switch to another file) // before surfacing a retryable error for the abandoned one. let generation: number | undefined + let registered = false try { const userId = socket.userId const userName = socket.userName @@ -1142,6 +1393,17 @@ export function setupWorkspaceFileDocHandlers( emitJoinError(socket, fileId, clientId, 'Invalid join payload', 'INVALID_PAYLOAD', false) return } + if ((payload.schemaVersion ?? FILE_DOC_LEGACY_SCHEMA_VERSION) !== FILE_DOC_SCHEMA_VERSION) { + emitJoinError( + socket, + fileId, + clientId, + 'This document version is not supported', + 'SCHEMA_VERSION_MISMATCH', + false + ) + return + } // A generation represents the socket's intended FILE, not an individual provider. Co-mounted // providers for the same file must be allowed to join concurrently; switching files advances the @@ -1156,6 +1418,7 @@ export function setupWorkspaceFileDocHandlers( const room = fileDocRoom(fileId) const name = roomName(room) + const admissionName = fileDocAdmissionRoom(fileId) const authorized = await resolveRoomJoinAuth({ userId, @@ -1177,6 +1440,17 @@ export function setupWorkspaceFileDocHandlers( // awareness). Resolved here so the generation guard below also covers this await. const avatarUrl = await resolveAvatarUrl(socket, userId) + const store = getFileDocStore() + const existing = fileDocRooms.get(name) + if ( + existing && + isDocSeeded(existing.doc) && + !(await store.isDocumentGenerationCurrent(name, docIdOf(existing.doc))) && + fileDocRooms.get(name) === existing + ) { + discardInvalidatedRoom(name, io) + } + const entry = getOrCreateRoom(io, room) // The workspace the server-side persist writes back to — and what the seed is built from, so it // must be captured BEFORE the room is prepared below. @@ -1185,6 +1459,11 @@ export function setupWorkspaceFileDocHandlers( // Hold the room open across the awaits below: it has no owner until this join commits, so a // concurrent last-leave would otherwise tear down the very document being prepared. entry.pendingJoins += 1 + let subscribed = false + const isCurrentJoin = () => + !socket.disconnected && + joinGeneration.get(socket.id) === generation && + fileDocRooms.get(name) === entry try { // A client is attached to a WHOLE document or to nothing. A room assembles itself from the // shared stream and the server seed, and both land in the same Y.Doc that fans every update out @@ -1209,18 +1488,42 @@ export function setupWorkspaceFileDocHandlers( emitJoinError(socket, fileId, clientId, 'Access denied to file', 'ACCESS_DENIED', false) return } - - // Abort a JOIN superseded while the room was being prepared: the socket disconnected, a newer - // JOIN (a document switch) bumped the generation, or the room was dropped and re-created. - // Registering here would leak a dead socket's room, bind the socket to the wrong document, or - // attach it to a doc no longer registered. Last await before the commit, so nothing can - // interleave between the access re-check above and the registration below. - if ( - socket.disconnected || - joinGeneration.get(socket.id) !== generation || - fileDocRooms.get(name) !== entry - ) + if (!isCurrentJoin()) return + + /** + * Watch invalidations before checking the generation, including broadcasts from another + * replica. Pending clients must not receive document or presence frames before authorization. + */ + pendingMemberships.set(name, (pendingMemberships.get(name) ?? 0) + 1) + subscribed = true + await socket.join(admissionName) + const joinedVersion = + Math.max(entry.syncedVersion ?? 0, (await store.getSyncedVersion(name)) ?? 0) || undefined + const currentDocument = await store.isDocumentGenerationCurrent(name, docIdOf(entry.doc)) + if (!isCurrentJoin()) return + if (!currentDocument) { + emitJoinError( + socket, + fileId, + clientId, + 'Document changed while joining', + 'JOIN_FAILED', + true + ) return + } + /** Adapter membership and generation reads can wait; resolve access again before commit. */ + const finalPermission = await resolveCurrentRoomPermission(userId, room, FILE_DOC_ACTION) + if (!satisfiesRoomMembership(finalPermission, ROOM_TYPES.WORKSPACE_FILE_DOC)) { + emitJoinError(socket, fileId, clientId, 'Access denied to file', 'ACCESS_DENIED', false) + return + } + + /** Commit content membership only after the final authorization decision. */ + if (!isCurrentJoin()) return + await socket.join(name) + /** An asynchronous adapter join can be superseded by a leave, switch, or disconnect. */ + if (!isCurrentJoin()) return // A client id must be owned by at most one user, or a peer could bind an active // collaborator's id and pass the per-frame ownership check to spoof/clear its caret. @@ -1282,7 +1585,7 @@ export function setupWorkspaceFileDocHandlers( } clientMap.set(clientId, { clientId, userId, userName, avatarUrl }) socketToRoomName.set(socket.id, name) - socket.join(name) + registered = true // Attribution for the server-side persist, refreshed to the actual editor on each edit in // `handleMessage`. @@ -1295,6 +1598,9 @@ export function setupWorkspaceFileDocHandlers( fileId, clientId, docId: docIdOf(entry.doc), + version: joinedVersion, + schemaVersion: FILE_DOC_SCHEMA_VERSION, + ...(store.enabled ? { acknowledgedUpdates: true as const } : {}), }) // Server-authenticated roster → everyone in the room, including this joiner. broadcastFileDocPresence(io, name, entry) @@ -1324,19 +1630,28 @@ export function setupWorkspaceFileDocHandlers( // A join that returned without registering may have left behind the room it created; drop it // if nothing else claimed it. A no-op once this join committed (the room then has an owner). destroyRoomIfIdle(name) + if (subscribed) { + const remaining = (pendingMemberships.get(name) ?? 1) - 1 + if (remaining > 0) pendingMemberships.set(name, remaining) + else { + pendingMemberships.delete(name) + await socket.leave(admissionName) + if (socketToRoomName.get(socket.id) !== name) await socket.leave(name) + } + } } } catch (error) { logger.error('Error joining file-doc room:', error) try { const name = roomName(fileDocRoom(fileId)) - socket.leave(name) - // Roll back ONLY this join's target room. cleanupFileDocForSocket keys off socketToRoomName, - // which — if the join failed before rebinding to the target (e.g. a switch that threw during - // client-id reclaim) — still points at the socket's PRIOR, valid document. Running it then - // would tear down a document the socket is validly in. So only run it when the binding - // already points at the target; otherwise the socket never registered as an owner of this - // room and the only leftover is a freshly-created empty room, dropped below. - if (socketToRoomName.get(socket.id) === name) cleanupFileDocForSocket(socket.id, io) + /** + * Roll back ownership only if this attempt committed it. A failed provisional admission must + * preserve a previous file's binding and any co-mounted provider already in the target room. + */ + if (registered && socketToRoomName.get(socket.id) === name) { + socket.leave(name) + cleanupFileDocForSocket(socket.id, io) + } destroyRoomIfIdle(name) } catch {} // Suppress the client-facing error when this join was already superseded (a switch to another @@ -1354,6 +1669,17 @@ export function setupWorkspaceFileDocHandlers( socket.on(FILE_DOC_EVENTS.MESSAGE, (data: unknown) => handleMessage(socket, io, data)) + socket.on( + FILE_DOC_EVENTS.UPDATE, + (data: unknown, acknowledge?: (result: FileDocUpdateAck) => void) => { + if (typeof acknowledge !== 'function') return + const pending = handleClientUpdate(socket, io, data, acknowledge) + .catch((error) => logger.error('Unhandled acknowledged file-doc update failure:', error)) + .finally(() => pendingFileDocUpdates.delete(pending)) + pendingFileDocUpdates.add(pending) + } + ) + socket.on(FILE_DOC_EVENTS.LEAVE, (payload?: LeaveFileDocPayload) => { try { // Cancel an in-flight join whose file the client is now leaving (or an unscoped leave): a diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index 80663141334..29240c4e656 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -6,6 +6,7 @@ import { createSocketIOServer, shutdownSocketIOAdapter } from '@/config/socket' import { assertSchemaCompatibility } from '@/database/preflight' import { env } from '@/env' import { setupAllHandlers } from '@/handlers' +import { waitForConnectionCleanup } from '@/handlers/connection' import { flushAllFileDocRooms } from '@/handlers/file-doc' import { getFileDocStore, initFileDocStore } from '@/handlers/file-doc-store' import { type AuthenticatedSocket, authenticateSocket } from '@/middleware/auth' @@ -121,6 +122,11 @@ async function main() { shuttingDown = true logger.info('Shutting down Socket.IO server...') + const shutdownTimer = setTimeout(() => { + logger.error('Forced shutdown after timeout') + process.exit(1) + }, SHUTDOWN_TIMEOUT_MS) + accessRevalidation.stop() // Flush open collaborative docs to durable markdown BEFORE tearing down Redis/the store — the @@ -132,6 +138,15 @@ async function main() { logger.error('Error flushing collaborative documents on shutdown:', error) } + /** Transport closure permits reconnection; a namespace DISCONNECT intentionally does not. */ + try { + await io.close() + await waitForConnectionCleanup() + await flushAllFileDocRooms() + } catch (error) { + logger.error('Error draining socket connections on shutdown:', error) + } + try { await roomManager.shutdown() logger.info('RoomManager shutdown complete') @@ -151,24 +166,9 @@ async function main() { logger.error('Error during FileDocStore shutdown:', error) } - // Close local client connections so `httpServer.close()` can complete its callback and exit - // gracefully — otherwise open websockets keep it hanging until the forced-exit timer below. - // Local-only: a rolling deploy must not disconnect clients pinned to other pods. - try { - io.local.disconnectSockets(true) - } catch (error) { - logger.error('Error disconnecting sockets on shutdown:', error) - } - - httpServer.close(() => { - logger.info('Socket.IO server closed') - process.exit(0) - }) - - setTimeout(() => { - logger.error('Forced shutdown after timeout') - process.exit(1) - }, SHUTDOWN_TIMEOUT_MS) + clearTimeout(shutdownTimer) + logger.info('Socket.IO server closed') + process.exit(0) } process.on('SIGINT', shutdown) diff --git a/apps/realtime/src/routes/http.test.ts b/apps/realtime/src/routes/http.test.ts index 725341deac9..e2fe3a476d4 100644 --- a/apps/realtime/src/routes/http.test.ts +++ b/apps/realtime/src/routes/http.test.ts @@ -1,6 +1,15 @@ import type { IncomingMessage, ServerResponse } from 'http' import { describe, expect, it, vi } from 'vitest' import type { IRoomManager } from '@/rooms' + +const { mockInvalidateDocument } = vi.hoisted(() => ({ mockInvalidateDocument: vi.fn() })) + +vi.mock('@/handlers/file-doc', () => ({ + applyMarkdownToLiveFileDoc: vi.fn(), + fileDocAdmissionRoom: (fileId: string) => `file-doc-admission:${fileId}`, + invalidateLiveFileDocument: mockInvalidateDocument, +})) + import { createHttpHandler } from '@/routes/http' function createMocks(req: Partial) { @@ -8,9 +17,13 @@ function createMocks(req: Partial) { const writeHead = vi.fn() const end = vi.fn() const logger = { info: vi.fn(), error: vi.fn(), debug: vi.fn(), warn: vi.fn() } + const emit = vi.fn() + const to = vi.fn(() => ({ emit })) const roomManager = { + io: { to }, getTotalActiveConnections: vi.fn().mockResolvedValue(0), isReady: vi.fn().mockReturnValue(true), + emitToRoom: vi.fn(), } as unknown as IRoomManager return { @@ -20,9 +33,27 @@ function createMocks(req: Partial) { setHeader, writeHead, end, + roomManager, + to, + emit, } } +function requestWithBody(url: string, body: unknown): Partial { + const text = JSON.stringify(body) + const request = { + method: 'POST', + url, + headers: { 'x-api-key': 'test-internal-api-secret-at-least-32-chars' }, + on(event: string, callback: (value?: Buffer) => void) { + if (event === 'data') callback(Buffer.from(text)) + if (event === 'end') callback() + return request + }, + } + return request as unknown as Partial +} + describe('createHttpHandler', () => { /** * `/health` is the only route on this server that returns 200 with a body, so @@ -58,4 +89,43 @@ describe('createHttpHandler', () => { expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) }) + + it('invalidates the shared generation before notifying every open editor', async () => { + mockInvalidateDocument.mockResolvedValueOnce({ status: 'applied', docId: 'old-document' }) + const { handler, req, res, writeHead, to, emit } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1', version: 100 }) + ) + + await handler(req, res) + + expect(mockInvalidateDocument).toHaveBeenCalledWith('file-1', 100) + expect(to).toHaveBeenCalledWith(['workspace-file-doc:file-1', 'file-doc-admission:file-1']) + expect(emit).toHaveBeenCalledWith( + 'file-doc-invalidated', + expect.objectContaining({ fileId: 'file-1', version: 100, docId: 'old-document' }) + ) + expect(mockInvalidateDocument.mock.invocationCallOrder[0]).toBeLessThan( + emit.mock.invocationCallOrder[0] + ) + expect(writeHead).toHaveBeenCalledWith(200, { 'Content-Type': 'application/json' }) + }) + + it('does not evict editors for a superseded invalidation', async () => { + mockInvalidateDocument.mockResolvedValueOnce({ status: 'stale' }) + const { handler, req, res, end, roomManager, to } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1', version: 100 }) + ) + await handler(req, res) + expect(roomManager.emitToRoom).not.toHaveBeenCalled() + expect(to).not.toHaveBeenCalled() + expect(end).toHaveBeenCalledWith(JSON.stringify({ status: 'stale' })) + }) + + it('requires a durable version for invalidation', async () => { + const { handler, req, res, writeHead } = createMocks( + requestWithBody('/api/file-doc/invalidate', { fileId: 'file-1' }) + ) + await handler(req, res) + expect(writeHead).toHaveBeenCalledWith(400, { 'Content-Type': 'application/json' }) + }) }) diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 19d2401e37e..da26b68aa3b 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -1,8 +1,13 @@ import type { IncomingMessage, ServerResponse } from 'http' -import { WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { FILE_DOC_EVENTS, type FileDocInvalidated } from '@sim/realtime-protocol/file-doc' +import { ROOM_TYPES, roomName, WORKSPACE_LIST_ROOM_TYPES } from '@sim/realtime-protocol/rooms' import { safeCompare } from '@sim/security/compare' import { env } from '@/env' -import { applyMarkdownToLiveFileDoc } from '@/handlers/file-doc' +import { + applyMarkdownToLiveFileDoc, + fileDocAdmissionRoom, + invalidateLiveFileDocument, +} from '@/handlers/file-doc' import { type IRoomManager, WorkflowRoomService } from '@/rooms' interface Logger { @@ -207,7 +212,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { version: typeof version === 'number' ? version : undefined, }) res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ applied: result === 'applied' })) + res.end(JSON.stringify({ applied: result === 'applied', status: result })) } catch (error) { logger.error('Error applying copilot edit to live file-doc:', error) sendError(res, 'Failed to apply edit to live document') @@ -215,6 +220,35 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { return } + if (req.method === 'POST' && req.url === '/api/file-doc/invalidate') { + try { + const body = await readRequestBody(req) + const { fileId, version } = JSON.parse(body) + if (!isNonEmptyString(fileId)) return sendError(res, 'Invalid fileId', 400) + if (!Number.isSafeInteger(version) || version <= 0) { + return sendError(res, 'Invalid version', 400) + } + const room = { type: ROOM_TYPES.WORKSPACE_FILE_DOC, id: fileId } as const + const result = await invalidateLiveFileDocument(fileId, version) + const payload: FileDocInvalidated = { + fileId, + version, + ...(result.status === 'applied' && result.docId ? { docId: result.docId } : {}), + message: 'This file changed outside the editor. Reload to continue editing.', + } + if (result.status === 'applied') + roomManager.io + .to([roomName(room), fileDocAdmissionRoom(fileId)]) + .emit(FILE_DOC_EVENTS.INVALIDATED, payload) + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify({ status: result.status })) + } catch (error) { + logger.error('Error invalidating live file-doc:', error) + sendError(res, 'Failed to invalidate live document') + } + return + } + res.writeHead(404, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ error: 'Not found' })) } diff --git a/apps/sim/app/api/webhooks/outbox/process/route.ts b/apps/sim/app/api/webhooks/outbox/process/route.ts index ea6746ff8f8..b4acb132c12 100644 --- a/apps/sim/app/api/webhooks/outbox/process/route.ts +++ b/apps/sim/app/api/webhooks/outbox/process/route.ts @@ -14,6 +14,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { directGrantOutboxHandlers } from '@/lib/invitations/direct-grant' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { workspaceFileLiveDocOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' import { workspaceFileStorageCleanupOutboxHandlers } from '@/lib/uploads/contexts/workspace/workspace-file-storage-cleanup-outbox' import { workflowDeploymentOutboxHandlers } from '@/lib/workflows/deployment-outbox' import { invitationMigrationOutboxHandlers } from '@/lib/workspaces/admin-move' @@ -34,6 +35,7 @@ const handlers = { ...invitationMigrationOutboxHandlers, ...directGrantOutboxHandlers, ...knowledgeDocumentProcessingOutboxHandlers, + ...workspaceFileLiveDocOutboxHandlers, ...workspaceFileStorageCleanupOutboxHandlers, ...workflowDeploymentOutboxHandlers, } as const diff --git a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx index 1a4c4db0f15..180dde64feb 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/find-bar/find-bar.test.tsx @@ -10,6 +10,7 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/emcn', () => ({ + cn: (...classes: Array) => classes.filter(Boolean).join(' '), Button: ({ children, ...props }: { children: ReactNode } & Record) => ( - ) : undefined - } - /> - {/* Always mounted, reserving its width: rendering it only once there is a +
+
+ {replace && ( + + )} + onQueryChange(e.target.value)} + onKeyDown={handleKeyDown} + endAdornment={ + query.length > 0 ? ( + + ) : undefined + } + /> + {/* Always mounted, reserving its width: rendering it only once there is a query would resize the bar on the first keystroke, and a live region inserted together with its text is announced unreliably. */} - - {counterContent()} - - - - + + {counterContent()} + + + + +
+ {replace && showReplace && ( +
+ + replace.onChange(event.target.value)} + onKeyDown={(event) => { + if (event.nativeEvent.isComposing || event.keyCode === 229) return + if (event.key === 'Enter' && replace.canReplace) { + event.preventDefault() + replace.onReplace() + } else if (event.key === 'Escape') { + event.preventDefault() + onClose() + } + }} + /> + + +
+ )}
) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts index 7d420e6da3f..f518778cd56 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.test.ts @@ -3,25 +3,60 @@ */ import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, + FILE_DOC_TIMEOUTS, + type FileDocUpdateAck, } from '@sim/realtime-protocol/file-doc' +import { update as updateJournalStorage } from 'idb-keyval' +import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' import type { Socket } from 'socket.io-client' import { describe, expect, it, vi } from 'vitest' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' import * as Y from 'yjs' -import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' -import { FileDocProvider } from './file-doc-provider' +import { AGENT_STREAM_ORIGIN } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' + +const journalStorage = vi.hoisted(() => new Map()) + +vi.mock('idb-keyval', () => ({ + get: vi.fn((key: string) => journalStorage.get(key)), + update: vi.fn((key: string, updater: (value: unknown) => unknown) => { + journalStorage.set(key, updater(journalStorage.get(key))) + }), + del: vi.fn((key: string) => { + journalStorage.delete(key) + }), +})) + +const UPDATE_BATCH_TEST_WINDOW_MS = 100 /** A minimal fake Socket.IO client whose server→client events can be fired in tests. */ function createSocket(connected = true) { const listeners = new Map void>>() const emit = vi.fn() + const timeout = vi.fn((delay: number) => ({ + emit( + event: string, + payload: unknown, + acknowledge: (error: Error | null, ack?: FileDocUpdateAck) => void + ) { + const timer = setTimeout(() => acknowledge(new Error('operation has timed out')), delay) + emit(event, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + clearTimeout(timer) + acknowledge(error, ack) + }) + }, + })) const socket = { connected, emit, + timeout, on(event: string, cb: (...args: unknown[]) => void) { let set = listeners.get(event) if (!set) { @@ -39,23 +74,29 @@ function createSocket(connected = true) { if (event === 'disconnect') socket.connected = false for (const cb of listeners.get(event) ?? []) cb(...args) } - return { socket: socket as unknown as Socket, emit, fire } + return { socket: socket as unknown as Socket, emit, fire, timeout } } function createProvider(connected = true) { - const { socket, emit, fire } = createSocket(connected) + const { socket, emit, fire, timeout } = createSocket(connected) const doc = new Y.Doc() const awareness = new awarenessProtocol.Awareness(doc) const provider = new FileDocProvider(socket, 'file-1', doc, awareness) - return { provider, doc, awareness, emit, fire } + return { provider, doc, awareness, emit, fire, timeout } } function acceptJoin( fire: (event: string, ...args: unknown[]) => void, clientId: number, - docId?: string + docId?: string, + acknowledgedUpdates = true ) { - fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { fileId: 'file-1', clientId, docId }) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId, + docId, + acknowledgedUpdates: acknowledgedUpdates ? true : undefined, + }) } /** Messages emitted to the server, decoded to their `{ type, bytes }`. */ @@ -67,12 +108,20 @@ function emittedMessages(emit: ReturnType) { .map(([, payload]) => payload as Uint8Array) } +function syncStep1Frame(doc: Y.Doc): Uint8Array { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep1(encoder, doc) + return encoding.toUint8Array(encoder) +} + describe('FileDocProvider', () => { it('joins immediately with its client id when the socket is already connected', () => { const { doc, emit } = createProvider(true) expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, { fileId: 'file-1', clientId: doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION, }) }) @@ -206,6 +255,17 @@ describe('FileDocProvider', () => { expect(joinError).toHaveBeenCalledTimes(1) }) + it('fails closed when a seeded legacy tab has no identity but the server does', () => { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + emit.mockClear() + + acceptJoin(fire, doc.clientID, 'doc-current') + + expect(emittedMessages(emit)).toHaveLength(0) + expect(provider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED', retryable: false }) + }) + it('syncs when the room holds the document it already has', () => { const { doc, emit, fire } = createProvider(true) doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-original') @@ -247,16 +307,1278 @@ describe('FileDocProvider', () => { expect(synced).toHaveBeenCalledWith(true) }) - it('sends local document edits to the server as sync updates', () => { + it('routes local differences through the acknowledged channel instead of the sync handshake', async () => { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getText('default').insert(0, 'local') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc) + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + + const syncReplies = emittedMessages(emit).filter((message) => { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + return decoding.readVarUint(decoder) === syncProtocol.messageYjsSyncStep2 + }) + expect(syncReplies).toHaveLength(0) + await vi.waitFor(() => { + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(true) + }) + const updatePayload = emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE + )?.[1] as { + update: Uint8Array + } + const serverDoc = new Y.Doc() + Y.applyUpdate(serverDoc, updatePayload.update) + expect(serverDoc.getText('default').toString()).toBe('local') + serverDoc.destroy() + provider.destroy() + }) + + it('keeps standard Yjs sync behavior with an older relay during a rolling deployment', () => { const { doc, emit, fire } = createProvider(true) - acceptJoin(fire, doc.clientID) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getText('default').insert(0, 'local') + acceptJoin(fire, doc.clientID, 'doc-1', false) emit.mockClear() - doc.getText('default').insert(0, 'x') + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + doc.getText('default').insert(5, ' edit') const messages = emittedMessages(emit) - expect(messages.length).toBe(1) - expect(messages[0][0]).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + expect(messages.length).toBeGreaterThanOrEqual(2) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + }) + + it('does not enable acknowledged updates unless the relay also supplies a document identity', () => { + const { doc, emit, fire } = createProvider(true) + doc.getText('default').insert(0, 'local') + acceptJoin(fire, doc.clientID, undefined, true) + emit.mockClear() + + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + + expect(emittedMessages(emit).length).toBeGreaterThan(0) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + }) + + it('protects only unsent changes when a legacy relay provides no document identity', () => { + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const { provider, doc, awareness, emit, fire } = createProvider(true) + const serverDoc = new Y.Doc() + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + doc.getText('default').insert(0, 'before join') + expect(unloadIsPrevented()).toBe(true) + acceptJoin(fire, doc.clientID, undefined, false) + expect(unloadIsPrevented()).toBe(false) + doc.getText('default').insert(11, ' online') + expect(unloadIsPrevented()).toBe(false) + + fire('disconnect') + doc.getText('default').insert(18, ' and offline') + expect(unloadIsPrevented()).toBe(true) + fire('connect') + acceptJoin(fire, doc.clientID, undefined, false) + emit.mockClear() + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + + for (const message of emittedMessages(emit)) { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + } + expect(serverDoc.getText('default').toString()).toBe('before join online and offline') + expect(unloadIsPrevented()).toBe(true) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.unstubAllGlobals() + } + }) + + it.each(['acknowledged', 'legacy-offline', 'legacy-rejoining'] as const)( + 'recovers an unacknowledged %s edit after restart and clears it only after acceptance', + async (mode) => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const serverDoc = new Y.Doc() + const serverConfig = serverDoc.getMap(FILE_DOC_SEED.configMap) + serverConfig.set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverConfig.set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'base') + + const firstSocket = createSocket(true) + const firstDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(serverDoc)) + const firstProvider = new FileDocProvider( + firstSocket.socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc), + scope + ) + acceptJoin(firstSocket.fire, firstDoc.clientID, 'doc-1', mode === 'acknowledged') + await vi.waitFor(() => expect(emittedMessages(firstSocket.emit).length).toBeGreaterThan(0)) + if (mode !== 'acknowledged') firstSocket.fire('disconnect') + if (mode === 'legacy-rejoining') firstSocket.fire('connect') + firstSocket.emit.mockClear() + firstDoc.getText('default').insert(4, ' local') + const firstJournal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + expect(await firstJournal.load('doc-1')).not.toBeNull() + }) + expect(firstSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe( + mode === 'acknowledged' + ) + firstProvider.destroy() + + const secondSocket = createSocket(true) + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + secondSocket.socket, + 'file-1', + secondDoc, + new awarenessProtocol.Awareness(secondDoc), + scope + ) + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + const syncEncoder = encoding.createEncoder() + encoding.writeVarUint(syncEncoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(syncEncoder, serverDoc) + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(syncEncoder)) + + await vi.waitFor(() => { + expect(secondDoc.getText('default').toString()).toBe('base local') + expect( + secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE) + ).toBe(true) + }) + const updateCall = secondSocket.emit.mock.calls.find( + ([event]) => event === FILE_DOC_EVENTS.UPDATE + ) + const payload = updateCall?.[1] as { updateId: string } + const acknowledge = updateCall?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + Y.applyUpdate(serverDoc, (updateCall?.[1] as { update: Uint8Array }).update) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load()).resolves.toBeNull() + }) + + vi.useFakeTimers() + try { + secondSocket.fire('disconnect') + secondSocket.fire('connect') + secondSocket.emit.mockClear() + acceptJoin(secondSocket.fire, secondDoc.clientID, 'doc-1') + secondSocket.fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect( + secondSocket.emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE) + ).toBe(false) + } finally { + vi.useRealTimers() + } + secondProvider.destroy() + firstDoc.destroy() + secondDoc.destroy() + serverDoc.destroy() + } + ) + + it('batches local document edits into the acknowledged update channel', async () => { + const { doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'x') + + await vi.waitFor(() => { + expect(emit).toHaveBeenCalledWith( + FILE_DOC_EVENTS.UPDATE, + expect.objectContaining({ fileId: 'file-1', docId: 'doc-1' }), + expect.any(Function) + ) + }) + expect(emittedMessages(emit)).toHaveLength(0) + }) + + it('serializes journal flushes so an edit made during storage never becomes stranded', async () => { + vi.useFakeTimers() + const firstSave = Promise.withResolvers<{ + pendingUpdate: Uint8Array + status: 'saved' + }>() + let saveCalls = 0 + let firstPendingUpdate: Uint8Array | null = null + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => { + saveCalls += 1 + if (saveCalls === 1) { + firstPendingUpdate = pendingUpdate + return firstSave.promise + } + return { pendingUpdate, status: 'saved' } + }) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + doc.getText('default').insert(5, ' second') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(save).toHaveBeenCalledOnce() + + firstSave.resolve({ + pendingUpdate: firstPendingUpdate!, + status: 'saved', + }) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(save).toHaveBeenCalledTimes(2) + const recovered = new Y.Doc() + Y.applyUpdate(recovered, save.mock.calls[1][2]) + expect(recovered.getText('default').toString()).toBe('first second') + recovered.destroy() + await vi.advanceTimersByTimeAsync(1_000) + expect(save).toHaveBeenCalledTimes(2) + const firstUpdate = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const firstPayload = firstUpdate?.[1] as { updateId: string } + const acknowledge = firstUpdate?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + + expect(save).toHaveBeenCalledTimes(3) + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength(2) + provider.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } + }) + + it('retries an unacknowledged update with the same idempotency key', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire, timeout } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + + doc.getText('default').insert(0, 'kept until acknowledged') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(first).toBeDefined() + expect(timeout).toHaveBeenCalledWith(FILE_DOC_TIMEOUTS.updateAckMs) + + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 2_000) + const updates = emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(updates.length).toBeGreaterThan(1) + expect((updates[1][1] as { updateId: string }).updateId).toBe( + (first?.[1] as { updateId: string }).updateId + ) + provider.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('rejoins before retrying an update rejected because the room membership went stale', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + doc.getText('default').insert(0, 'edit') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const payload = first?.[1] as { updateId: string } + const acknowledge = first?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + + acknowledge(null, { + status: 'rejected', + updateId: payload.updateId, + code: 'NOT_JOINED', + retryable: true, + }) + await vi.advanceTimersByTimeAsync(1_000) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.JOIN)).toBe(true) + + emit.mockClear() + acceptJoin(fire, doc.clientID, 'doc-1') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(true) + provider.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('ignores expired acknowledgements after the provider is destroyed', async () => { + vi.useFakeTimers() + const { provider, doc, awareness, emit, fire, timeout } = createProvider(true) + try { + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + doc.getText('default').insert(0, 'pending') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(timeout).toHaveBeenCalledWith(FILE_DOC_TIMEOUTS.updateAckMs) + + provider.destroy() + emit.mockClear() + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + expect(emit).not.toHaveBeenCalled() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + vi.useRealTimers() + } + }) + + it('preserves pending acknowledged edits across a downgrade without calling legacy sync an acceptance', async () => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const clear = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'clear') + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + const serverDoc = new Y.Doc() + Y.applyUpdate(serverDoc, Y.encodeStateAsUpdate(doc)) + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'pending acknowledged edit') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const firstPayload = first?.[1] as { updateId: string } + expect(first).toBeDefined() + + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(serverDoc)) + for (const message of emittedMessages(emit)) { + const decoder = decoding.createDecoder(message) + decoding.readVarUint(decoder) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + } + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + + expect(serverDoc.getText('default').toString()).toBe('pending acknowledged edit') + expect(provider.synced).toBe(true) + expect(unloadIsPrevented()).toBe(true) + expect(await journal.load('doc-1')).not.toBeNull() + expect(clear).not.toHaveBeenCalled() + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + const retry = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(retry?.[1]).toMatchObject({ updateId: firstPayload.updateId }) + const acknowledge = retry?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + acknowledge(null, { status: 'accepted', updateId: firstPayload.updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(unloadIsPrevented()).toBe(false) + expect(await journal.load('doc-1')).toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + clear.mockRestore() + vi.unstubAllGlobals() + vi.useRealTimers() + } + }) + + it('protects a recovered pending journal even when the new relay has no acknowledged channel', async () => { + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const recoveredDoc = new Y.Doc() + recoveredDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + recoveredDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + recoveredDoc.getText('default').insert(0, 'recover me') + const update = Y.encodeStateAsUpdate(recoveredDoc) + await journal.save('doc-1', update, update) + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.waitFor(() => expect(doc.getText('default').toString()).toBe('recover me')) + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + expect(event.defaultPrevented).toBe(true) + expect(await journal.load('doc-1')).not.toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + recoveredDoc.destroy() + vi.unstubAllGlobals() + } + }) + + it('journals an edit made while disconnected before page teardown', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + fire('disconnect') + + doc.getText('default').insert(0, 'offline edit') + + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => { + await expect(journal.load('doc-1')).resolves.not.toBeNull() + }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + }) + + it('preserves pending recovery through page teardown and destroy', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + fire('disconnect') + doc.getText('default').insert(0, 'preserve me') + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await vi.waitFor(async () => expect(await journal.load('doc-1')).not.toBeNull()) + + ;(provider as unknown as { handlePageHide: () => void }).handlePageHide() + provider.destroy() + + const stored = await journal.load('doc-1') + expect(stored).not.toBeNull() + const recovered = new Y.Doc() + Y.applyUpdate(recovered, stored!.recoverySnapshot!) + Y.applyUpdate(recovered, stored!.pendingUpdate) + expect(recovered.getText('default').toString()).toBe('preserve me') + recovered.destroy() + doc.destroy() + }) + + it('reopens the current generation without installing an incompatible recovery draft', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + oldDoc.getText('default').insert(0, 'complete draft') + const recoverySnapshot = Y.encodeStateAsUpdate(oldDoc) + const stateVector = Y.encodeStateVector(oldDoc) + oldDoc.getText('default').insert('complete draft'.length, ' plus pending') + const pendingUpdate = Y.encodeStateAsUpdate(oldDoc, stateVector) + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'old-doc', + pendingUpdate, + recoverySnapshot + ) + for (let mount = 0; mount < 3; mount++) { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + acceptJoin(fire, doc.clientID, 'current-doc') + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + awareness.destroy() + doc.destroy() + } + const retained = await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load( + 'old-doc' + ) + expect(retained?.pendingUpdate).toEqual(pendingUpdate) + expect(retained?.recoverySnapshot).toEqual(recoverySnapshot) + oldDoc.destroy() + }) + + it.each([false, true])( + 'does not install disk recovery without a negotiated identity (local identity: %s)', + async (hasLocalIdentity) => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const draft = new Y.Doc() + draft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + draft.getText('default').insert(0, 'retained draft') + const snapshot = Y.encodeStateAsUpdate(draft) + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + await journal.save('old-doc', snapshot, snapshot) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + if (hasLocalIdentity) Y.applyUpdate(doc, snapshot) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + acceptJoin(fire, doc.clientID, undefined, false) + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe(hasLocalIdentity ? 'retained draft' : '') + expect(await journal.load('old-doc')).not.toBeNull() + provider.destroy() + awareness.destroy() + doc.destroy() + draft.destroy() + } + ) + + it('admits the final online batch before leaving while its relay publication is still pending', () => { + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness) + acceptJoin(fire, doc.clientID, 'doc-1') + let joined = true + const publications: Array<() => void> = [] + emit.mockImplementation((event, payload, acknowledge) => { + if (event === FILE_DOC_EVENTS.LEAVE) joined = false + if (event !== FILE_DOC_EVENTS.UPDATE) return + expect(joined).toBe(true) + publications.push(() => { + Y.applyUpdate(serverDoc, payload.update) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + }) + }) + + doc.getText('default').insert(0, 'last edit') + provider.destroy() + + expect(joined).toBe(false) + expect(publications).toHaveLength(1) + expect(serverDoc.getText('default').toString()).toBe('') + publications[0]() + expect(serverDoc.getText('default').toString()).toBe('last edit') + awareness.destroy() + doc.destroy() + serverDoc.destroy() + }) + + it('does not send an oversized aggregate batch during teardown', () => { + const { provider, doc, awareness, emit, fire } = createProvider(true) + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + acceptJoin(fire, doc.clientID, 'doc-1') + const bytes = new Uint8Array(FILE_DOC_LIMITS.updateBytes / 2) + doc.getArray('binary').insert(0, [bytes]) + doc.getArray('binary').insert(1, [bytes]) + emit.mockClear() + + provider.destroy() + + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + awareness.destroy() + doc.destroy() + }) + + it.each(['destroy', 'file-switch'] as const)( + 'drains an edit being journaled and later edits before %s', + async (navigation) => { + vi.useFakeTimers() + journalStorage.clear() + const storageGate = Promise.withResolvers() + vi.mocked(updateJournalStorage).mockImplementationOnce(async (key, updater) => { + await storageGate.promise + journalStorage.set(String(key), updater(journalStorage.get(String(key)))) + }) + const clear = vi.spyOn(PendingFileDocUpdateJournal.prototype, 'clear') + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + const nextDoc = new Y.Doc() + const nextAwareness = new awarenessProtocol.Awareness(nextDoc) + let nextProvider: FileDocProvider | undefined + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + doc.getText('default').insert(5, ' second') + emit.mockClear() + if (navigation === 'file-switch') { + nextProvider = new FileDocProvider(socket, 'file-2', nextDoc, nextAwareness, scope) + } + provider.destroy() + awareness.destroy() + doc.destroy() + + const updateIndex = emit.mock.calls.findIndex(([event]) => event === FILE_DOC_EVENTS.UPDATE) + const membershipEvent = + navigation === 'file-switch' ? FILE_DOC_EVENTS.JOIN : FILE_DOC_EVENTS.LEAVE + const membershipIndex = emit.mock.calls.findIndex(([event]) => event === membershipEvent) + expect(updateIndex).toBeGreaterThanOrEqual(0) + expect(updateIndex).toBeLessThan(membershipIndex) + const [, payload, acknowledge] = emit.mock.calls[updateIndex] + Y.applyUpdate(serverDoc, payload.update) + expect(serverDoc.getText('default').toString()).toBe('first second') + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + expect(clear).not.toHaveBeenCalled() + storageGate.resolve() + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(clear).toHaveBeenCalledOnce() + expect( + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-1') + ).toBeNull() + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength( + 1 + ) + } finally { + storageGate.resolve() + provider.destroy() + nextProvider?.destroy() + awareness.destroy() + doc.destroy() + nextAwareness.destroy() + nextDoc.destroy() + serverDoc.destroy() + clear.mockRestore() + vi.useRealTimers() + } + } + ) + + it.each(['accepted', 'rejected', 'timeout'] as const)( + 'retains the final combined draft until its own %s acknowledgment', + async (outcome) => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + doc.getText('default').insert(0, 'first') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + const first = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(first).toBeDefined() + doc.getText('default').insert(5, ' second') + provider.destroy() + const updates = emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(updates).toHaveLength(2) + const [, finalPayload, finalAcknowledge] = updates[1] + expect(finalPayload.updateId).not.toBe(first?.[1].updateId) + Y.applyUpdate(serverDoc, finalPayload.update) + expect(serverDoc.getText('default').toString()).toBe('first second') + first?.[2](null, { status: 'accepted', updateId: first[1].updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(await journal.load('doc-1')).not.toBeNull() + if (outcome === 'accepted') { + finalAcknowledge(null, { status: 'accepted', updateId: finalPayload.updateId }) + } else if (outcome === 'rejected') { + finalAcknowledge(null, { + status: 'rejected', + updateId: finalPayload.updateId, + retryable: false, + code: 'ACCESS_REVOKED', + }) + } + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.updateAckMs + 6_000) + const recovery = await journal.load('doc-1') + if (outcome === 'accepted') expect(recovery).toBeNull() + else expect(recovery?.pendingUpdate).toEqual(finalPayload.update) + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toHaveLength( + 2 + ) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.useRealTimers() + } + } + ) + + it.each(['disconnected', 'invalidated', 'not-joined'] as const)( + 'keeps the final draft local when %s', + async (state) => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + if (state !== 'not-joined') { + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + } + doc.getText('default').insert(0, 'retained draft') + if (state === 'disconnected') fire('disconnect') + if (state === 'invalidated') { + fire(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1', message: 'Replaced' }) + } + emit.mockClear() + provider.destroy() + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + expect(emittedMessages(emit)).toHaveLength(0) + expect( + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-1') + ).not.toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + vi.useRealTimers() + } + } + ) + + it('delivers a rejoined legacy batch before leaving without treating it as acknowledged', async () => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, emit, fire } = createSocket(true) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(serverDoc)) + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + fire('disconnect') + doc.getText('default').insert(0, 'legacy draft') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1', false) + await vi.advanceTimersByTimeAsync(0) + emit.mockClear() + provider.destroy() + const frame = emittedMessages(emit)[0] + const decoder = decoding.createDecoder(frame) + expect(decoding.readVarUint(decoder)).toBe(FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), serverDoc, null) + expect(serverDoc.getText('default').toString()).toBe('legacy draft') + expect( + emit.mock.calls.findIndex(([event]) => event === FILE_DOC_EVENTS.MESSAGE) + ).toBeLessThan(emit.mock.calls.findIndex(([event]) => event === FILE_DOC_EVENTS.LEAVE)) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect( + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-1') + ).not.toBeNull() + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + serverDoc.destroy() + vi.useRealTimers() + } + }) + + it('never falls back to a different document identity when loading local recovery', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-old') + oldDoc.getText('default').insert(0, 'old draft') + const oldSnapshot = Y.encodeStateAsUpdate(oldDoc) + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'doc-old', + oldSnapshot, + oldSnapshot + ) + + const { socket, fire } = createSocket(true) + const currentDoc = new Y.Doc() + currentDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-current') + currentDoc.getText('default').insert(0, 'current content') + const provider = new FileDocProvider( + socket, + 'file-1', + currentDoc, + new awarenessProtocol.Awareness(currentDoc), + scope + ) + acceptJoin(fire, currentDoc.clientID, 'doc-current') + + await vi.waitFor(() => expect(provider.joinError).toBeNull()) + expect(currentDoc.getText('default').toString()).toBe('current content') + await expect( + new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).load('doc-old') + ).resolves.not.toBeNull() + provider.destroy() + currentDoc.destroy() + oldDoc.destroy() + }) + + it('recovers the negotiated generation even when an incompatible draft is newer', async () => { + vi.useFakeTimers() + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const currentDraft = new Y.Doc() + currentDraft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'current-doc') + currentDraft.getText('default').insert(0, 'current draft') + const currentSnapshot = Y.encodeStateAsUpdate(currentDraft) + await journal.save('current-doc', currentSnapshot, currentSnapshot) + await vi.advanceTimersByTimeAsync(1) + const oldDraft = new Y.Doc() + oldDraft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'old-doc') + oldDraft.getText('default').insert(0, 'incompatible draft') + const oldSnapshot = Y.encodeStateAsUpdate(oldDraft) + await journal.save('old-doc', oldSnapshot, oldSnapshot) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + try { + acceptJoin(fire, doc.clientID, 'current-doc') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('current draft') + const update = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(update).toBeDefined() + update?.[2](null, { status: 'accepted', updateId: update[1].updateId }) + await vi.advanceTimersByTimeAsync(0) + expect(await journal.load('current-doc')).toBeNull() + expect((await journal.load('old-doc'))?.recoverySnapshot).toEqual(oldSnapshot) + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + currentDraft.destroy() + oldDraft.destroy() + vi.useRealTimers() + } + }) + + it('rejects an in-memory generation mismatch before applying a matching server draft', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const journal = new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }) + const serverDraft = new Y.Doc() + serverDraft.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'server-doc') + serverDraft.getText('default').insert(0, 'server draft') + const snapshot = Y.encodeStateAsUpdate(serverDraft) + await journal.save('server-doc', snapshot, snapshot) + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'local-doc') + doc.getText('default').insert(0, 'local draft') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, scope) + acceptJoin(fire, doc.clientID, 'server-doc') + await vi.waitFor(() => expect(provider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED' })) + expect(doc.getText('default').toString()).toBe('local draft') + expect((await journal.load('server-doc'))?.recoverySnapshot).toEqual(snapshot) + provider.destroy() + awareness.destroy() + doc.destroy() + serverDraft.destroy() + }) + + it('syncs after malformed recovery without replaying it on subsequent mounts', async () => { + journalStorage.clear() + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const oldDoc = new Y.Doc() + oldDoc.getText('default').insert(0, 'quarantined snapshot') + await new PendingFileDocUpdateJournal({ ...scope, fileId: 'file-1' }).save( + 'doc-1', + new Uint8Array([255]), + Y.encodeStateAsUpdate(oldDoc) + ) + const serverDoc = new Y.Doc() + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + serverDoc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + serverDoc.getText('default').insert(0, 'server content') + const frame = encoding.createEncoder() + encoding.writeVarUint(frame, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(frame, serverDoc) + for (let mount = 0; mount < 2; mount++) { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(frame)) + + expect(provider.joinError).toBeNull() + expect(provider.synced).toBe(true) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBe(true) + expect(doc.getText('default').toString()).toBe('server content') + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + doc.destroy() + } + oldDoc.destroy() + serverDoc.destroy() + }) + + it('ignores an obsolete schema rejection when recovery finishes after reconnecting', async () => { + const recovery = Promise.withResolvers() + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(recovery.promise) + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, { + workspaceId: 'workspace-1', + userId: 'user-1', + }) + try { + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'doc-1', + schemaVersion: FILE_DOC_SCHEMA_VERSION + 1, + }) + fire('disconnect') + fire('connect') + acceptJoin(fire, doc.clientID, 'doc-1') + emit.mockClear() + recovery.resolve(null) + + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + expect(provider.joinError).toBeNull() + } finally { + provider.destroy() + awareness.destroy() + doc.destroy() + load.mockRestore() + } + }) + + it('fails closed when hydration buffers more than its bounded message count', async () => { + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(new Promise(() => {})) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + const message = syncStep1Frame(new Y.Doc()) + + for (let index = 0; index < 129; index += 1) { + fire(FILE_DOC_EVENTS.MESSAGE, message) + } + + expect(provider.joinError).toMatchObject({ code: 'HYDRATION_BUFFER_OVERFLOW' }) + provider.destroy() + load.mockRestore() + }) + + it('fails closed when hydration buffers more than its bounded byte budget', () => { + const load = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'load') + .mockReturnValue(new Promise(() => {})) + const scope = { workspaceId: 'workspace-1', userId: 'user-1' } + const { socket, fire } = createSocket(true) + const doc = new Y.Doc() + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + scope + ) + acceptJoin(fire, doc.clientID, 'doc-1') + + fire(FILE_DOC_EVENTS.MESSAGE, new Uint8Array(FILE_DOC_LIMITS.updateBytes * 2 + 1)) + + expect(provider.joinError).toMatchObject({ code: 'HYDRATION_BUFFER_OVERFLOW' }) + provider.destroy() + doc.destroy() + load.mockRestore() + }) + + it('makes an older different-file provider terminal before unscoped frames can cross documents', async () => { + const { socket, emit, fire } = createSocket(true) + const firstDoc = new Y.Doc() + const firstProvider = new FileDocProvider( + socket, + 'file-1', + firstDoc, + new awarenessProtocol.Awareness(firstDoc) + ) + acceptJoin(fire, firstDoc.clientID) + + const secondDoc = new Y.Doc() + const secondProvider = new FileDocProvider( + socket, + 'file-2', + secondDoc, + new awarenessProtocol.Awareness(secondDoc) + ) + expect(firstProvider.joinError).toMatchObject({ code: 'DOCUMENT_REPLACED' }) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-2', + clientId: secondDoc.clientID, + acknowledgedUpdates: true, + }) + await vi.waitFor(() => expect(emittedMessages(emit).length).toBeGreaterThan(0)) + + const remote = new Y.Doc() + remote.getText('default').insert(0, 'second-file content') + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(remote)) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + + expect(firstDoc.getText('default').toString()).toBe('') + expect(secondDoc.getText('default').toString()).toBe('second-file content') + firstProvider.destroy() + secondProvider.destroy() + firstDoc.destroy() + secondDoc.destroy() + remote.destroy() + }) + + it.each(['acknowledged', 'legacy-offline'] as const)( + 'stops editing when the complete %s recovery snapshot cannot be persisted', + async (mode) => { + vi.useFakeTimers() + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ + pendingUpdate, + status: 'limit-exceeded', + })) + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const provider = new FileDocProvider( + socket, + 'file-1', + doc, + new awarenessProtocol.Awareness(doc), + { workspaceId: 'workspace-1', userId: 'user-1' } + ) + acceptJoin(fire, doc.clientID, 'doc-1', mode === 'acknowledged') + await vi.advanceTimersByTimeAsync(0) + if (mode === 'legacy-offline') fire('disconnect') + emit.mockClear() + + doc.getText('default').insert(0, 'must remain visible') + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + + expect(provider.joinError).toMatchObject({ code: 'PENDING_UPDATE_LIMIT' }) + expect(emit.mock.calls.some(([event]) => event === FILE_DOC_EVENTS.UPDATE)).toBe(false) + provider.destroy() + doc.destroy() + } finally { + save.mockRestore() + vi.useRealTimers() + } + } + ) + + it.each(['saved', 'unavailable'] as const)( + 'warns before unloading pending edits and continues acknowledged saves when storage is %s', + async (status) => { + vi.useFakeTimers() + journalStorage.clear() + const browserWindow = new EventTarget() + vi.stubGlobal('window', browserWindow) + const save = vi + .spyOn(PendingFileDocUpdateJournal.prototype, 'save') + .mockImplementation(async (_docId, pendingUpdate) => ({ pendingUpdate, status })) + const unloadIsPrevented = () => { + const event = new Event('beforeunload', { cancelable: true }) + Object.defineProperty(event, 'returnValue', { value: '', writable: true }) + browserWindow.dispatchEvent(event) + return event.defaultPrevented + } + try { + const { socket, emit, fire } = createSocket(true) + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.docIdKey, 'doc-1') + const awareness = new awarenessProtocol.Awareness(doc) + const provider = new FileDocProvider(socket, 'file-1', doc, awareness, { + workspaceId: 'workspace-1', + userId: 'user-1', + }) + acceptJoin(fire, doc.clientID, 'doc-1') + await vi.advanceTimersByTimeAsync(0) + expect(unloadIsPrevented()).toBe(false) + + doc.getText('default').insert(0, 'pending edit') + expect(unloadIsPrevented()).toBe(true) + await vi.advanceTimersByTimeAsync(UPDATE_BATCH_TEST_WINDOW_MS) + expect(provider.joinError).toBeNull() + const update = emit.mock.calls.find(([event]) => event === FILE_DOC_EVENTS.UPDATE) + expect(update).toBeDefined() + const payload = update?.[1] as { updateId: string } + const acknowledge = update?.[2] as (error: Error | null, ack: FileDocUpdateAck) => void + expect(unloadIsPrevented()).toBe(true) + acknowledge(null, { status: 'accepted', updateId: payload.updateId }) + expect(unloadIsPrevented()).toBe(false) + + doc.getText('default').insert(0, 'another ') + expect(unloadIsPrevented()).toBe(true) + provider.destroy() + expect(unloadIsPrevented()).toBe(false) + awareness.destroy() + doc.destroy() + } finally { + save.mockRestore() + vi.unstubAllGlobals() + vi.useRealTimers() + } + } + ) + + it('keeps retrying sync without fatally timing out a previously healthy reconnect', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + acceptJoin(fire, doc.clientID, 'doc-1') + const serverDoc = new Y.Doc() + const config = serverDoc.getMap(FILE_DOC_SEED.configMap) + config.set(FILE_DOC_SEED.docIdKey, 'doc-1') + config.set(FILE_DOC_SEED.flag, true) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, serverDoc) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + + emit.mockClear() + fire('disconnect') + fire('connect') + expect(emit.mock.calls.filter(([event]) => event === FILE_DOC_EVENTS.JOIN)).toHaveLength(1) + acceptJoin(fire, doc.clientID, 'doc-1') + + await vi.advanceTimersByTimeAsync(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + expect(provider.joinError).toBeNull() + expect(emittedMessages(emit).length).toBeGreaterThan(1) + provider.destroy() + serverDoc.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('retries an accepted sync handshake that never receives a response', async () => { + vi.useFakeTimers() + try { + const { provider, doc, emit, fire } = createProvider(true) + acceptJoin(fire, doc.clientID) + emit.mockClear() + + await vi.advanceTimersByTimeAsync(6_000) + + expect(emittedMessages(emit).length).toBeGreaterThan(1) + expect(provider.joinError).toBeNull() + provider.destroy() + } finally { + vi.useRealTimers() + } }) it('tags agent-streamed edits as SYNC_NO_PERSIST so the relay skips the durable persist', () => { @@ -359,6 +1681,31 @@ describe('FileDocProvider', () => { expect(provider.joinError).toEqual(error) }) + it('becomes terminal when a durable replacement invalidates its document generation', () => { + const { provider, doc, emit, fire } = createProvider(true) + const onError = vi.fn() + provider.on('join-error', onError) + + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + message: 'This file changed outside the editor. Reload to continue editing.', + }) + + expect(provider.joinError).toMatchObject({ + code: 'DOCUMENT_REPLACED', + retryable: false, + }) + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'DOCUMENT_REPLACED', retryable: false }) + ) + + emit.mockClear() + fire('connect') + fire(FILE_DOC_EVENTS.MESSAGE, syncStep1Frame(new Y.Doc())) + expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(doc.getText('default').toString()).toBe('') + }) + it('scopes join errors to the matching provider on a shared socket', () => { const { socket, fire } = createSocket(true) const firstDoc = new Y.Doc() @@ -624,28 +1971,38 @@ describe('FileDocProvider', () => { expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.LEAVE, { fileId: 'file-b' }) }) - it('gives up with a non-retryable join-error when the first sync never arrives (offline)', () => { + it('keeps an unseeded document retryable after the readiness deadline and accepts late server content', () => { vi.useFakeTimers() try { - const { provider, emit, fire } = createProvider(false) // socket never connects + const { provider, doc, emit, fire } = createProvider(false) const onError = vi.fn() provider.on('join-error', onError) vi.advanceTimersByTime(12_000) - // Surfaces the same non-retryable rejection the fatal path uses, so the editor falls back to - // showing the file read-only. expect(onError).toHaveBeenCalledWith( - expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: true }) ) expect(provider.joinError).toEqual( - expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: true }) ) - // Latched fatal: a later connect must not re-join (which could sync server state in and - // duplicate the locally-seeded content). emit.mockClear() fire('connect') - expect(emit).not.toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(emit).toHaveBeenCalledWith(FILE_DOC_EVENTS.JOIN, expect.anything()) + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBeUndefined() + acceptJoin(fire, doc.clientID) + const remote = new Y.Doc() + remote.getText('default').insert(0, 'authoritative body') + remote.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeSyncStep2(encoder, remote) + fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + expect(provider.synced).toBe(true) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('authoritative body') + provider.destroy() + remote.destroy() } finally { vi.useRealTimers() } @@ -698,7 +2055,7 @@ describe('FileDocProvider', () => { // The readiness deadline still fires → the editor falls back to the stored content read-only, // and `synced` is dropped so the `synced && seeded` gate stays closed (read-only, not editable). expect(onError).toHaveBeenCalledWith( - expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: false }) + expect.objectContaining({ code: 'READINESS_TIMEOUT', retryable: true }) ) expect(provider.synced).toBe(false) } finally { @@ -706,30 +2063,93 @@ describe('FileDocProvider', () => { } }) - it('ignores a late SyncStep2 that arrives after the readiness deadline (no merge, stays gated)', () => { + it('accepts a late authoritative SyncStep2 after the readiness deadline without a local fallback seed', () => { vi.useFakeTimers() try { const { provider, doc, fire } = createProvider(true) acceptJoin(fire, doc.clientID) - // Deadline lapses with no first sync → fatal fallback (editor falls back to a read-only seed). vi.advanceTimersByTime(12_000) expect(provider.joinError).toEqual(expect.objectContaining({ code: 'READINESS_TIMEOUT' })) - // A delayed SyncStep2 finally arrives. Applying it would merge server content into the - // already-seeded doc (duplication) and flip synced→true (un-gating autosave), so it MUST be - // dropped once fatal. const remote = new Y.Doc() remote.getText('default').insert(0, 'server content') + remote.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) const encoder = encoding.createEncoder() encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeSyncStep2(encoder, remote) fire(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) - expect(provider.synced).toBe(false) - expect(doc.getText('default').toString()).toBe('') + expect(provider.synced).toBe(true) + expect(provider.joinError).toBeNull() + expect(doc.getText('default').toString()).toBe('server content') + provider.destroy() + remote.destroy() } finally { vi.useRealTimers() } }) + + it.each(['old-document', undefined])( + 'ignores delayed invalidation of %s after a newer authoritative JOIN', + (invalidatedDocId) => { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'new-document', + version: 30, + }) + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + docId: invalidatedDocId, + version: 20, + message: 'Old replacement', + }) + expect(provider.joinError).toBeNull() + provider.destroy() + } + ) + + it('keeps matching-generation invalidation terminal even when its version equals JOIN', () => { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'current-document', + version: 20, + }) + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + docId: 'current-document', + version: 20, + message: 'Current replacement', + }) + expect(provider.joinError?.code).toBe('DOCUMENT_REPLACED') + provider.destroy() + }) + + it('does not let a newer tombstone notification spare an older joined document', () => { + const { provider, doc, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.JOIN_SUCCESS, { + fileId: 'file-1', + clientId: doc.clientID, + docId: 'current-document', + version: 10, + }) + fire(FILE_DOC_EVENTS.INVALIDATED, { + fileId: 'file-1', + version: 30, + message: 'Second unsupported replacement', + }) + expect(provider.joinError?.code).toBe('DOCUMENT_REPLACED') + provider.destroy() + }) + + it('fails closed on invalidation before reliable JOIN metadata exists', () => { + const { provider, fire } = createProvider(true) + fire(FILE_DOC_EVENTS.INVALIDATED, { fileId: 'file-1', version: 20, message: 'Replacement' }) + expect(provider.joinError?.code).toBe('DOCUMENT_REPLACED') + provider.destroy() + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts index b3a04d50b96..17c3e07ad13 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider.ts @@ -4,14 +4,20 @@ import { } from '@sim/realtime-protocol/events' import { FILE_DOC_EVENTS, + FILE_DOC_LIMITS, FILE_DOC_MESSAGE_TYPE, + FILE_DOC_SCHEMA_VERSION, FILE_DOC_SEED, FILE_DOC_TIMEOUTS, + type FileDocInvalidated, + type FileDocUpdateAck, + type FileDocUpdatePayload, type JoinFileDocError, type JoinFileDocSuccess, toFileDocBytes, } from '@sim/realtime-protocol/file-doc' import { ROOM_TYPES } from '@sim/realtime-protocol/rooms' +import { generateShortId } from '@sim/utils/id' import { backoffWithJitter } from '@sim/utils/retry' import * as decoding from 'lib0/decoding' import * as encoding from 'lib0/encoding' @@ -19,8 +25,9 @@ import { ObservableV2 } from 'lib0/observable' import type { Socket } from 'socket.io-client' import * as awarenessProtocol from 'y-protocols/awareness' import * as syncProtocol from 'y-protocols/sync' -import type * as Y from 'yjs' -import { AGENT_STREAM_ORIGIN } from './apply-streamed-markdown' +import * as Y from 'yjs' +import { AGENT_STREAM_ORIGIN } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/apply-streamed-markdown' +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' /** * Events emitted by {@link FileDocProvider}. @@ -33,21 +40,34 @@ interface FileDocProviderEvents { } /** - * How long to wait to reach a USABLE editor — connected, synced, AND seeded (`initialContentLoaded` - * set by the server seed) — before giving up. It guards two failure modes with one timer: - * - the realtime server is unreachable, so the first sync never arrives; and - * - the socket syncs an empty doc but the server-side seed never lands (its build persistently fails - * / exhausts its retries), which `synced` alone would wrongly treat as "connected, all good". - * - * On the deadline the provider latches fatal and surfaces a non-retryable `join-error` — the exact - * path a fatal rejection uses — so the editor falls back to showing the file's stored content - * read-only instead of a permanently blank pane. Generous enough to clear a slow connect + seed - * round-trip; a healthy cold open reaches readiness well within it. Shared with (and must exceed) the - * relay's seed-fetch timeout — see `FILE_DOC_TIMEOUTS` and its ordering test. + * Report delayed connection or seeding without abandoning recovery. The stored-content preview + * stays separate from the authoritative Y.Doc, preventing duplicate content on a late sync. + * Must outlast the relay's seed-fetch timeout; see FILE_DOC_TIMEOUTS and its ordering test. */ const READINESS_DEADLINE_MS = FILE_DOC_TIMEOUTS.readinessDeadlineMs const JOIN_RETRY_BASE_MS = 500 const JOIN_RETRY_MAX_MS = 5_000 +const UPDATE_BATCH_MS = 50 +const UPDATE_RETRY_BASE_MS = 250 +const UPDATE_RETRY_MAX_MS = 5_000 +const MAX_HYDRATION_MESSAGES = 128 +const MAX_HYDRATION_BYTES = FILE_DOC_LIMITS.updateBytes * 2 +const RECOVERY_ORIGIN = Symbol('file-doc-recovery') + +function hasYjsUpdateContent(update: Uint8Array): boolean { + const decoded = Y.decodeUpdate(update) + return decoded.structs.length > 0 || decoded.ds.clients.size > 0 +} + +interface FileDocProviderScope { + workspaceId: string + userId: string +} + +interface PendingClientUpdate { + updateId: string + update: Uint8Array +} /** * Live-provider counts per file, per shared socket. Two surfaces in one tab (the Files editor and the @@ -101,11 +121,18 @@ function releaseRoomMembership(socket: Socket, fileId: string): boolean { * reconnect) without discarding local edits. */ export class FileDocProvider extends ObservableV2 { + /** Socket.IO carries unscoped Yjs frames, so opening a different file terminalizes providers for the + * previous file; multiple providers for the same file may coexist. */ + private static readonly activeProviders = new WeakMap< + Socket, + { fileId: string; providers: Set } + >() + synced = false /** - * The latched non-retryable join rejection, or `null`. The `join-error` event is - * transient and can fire before a consumer subscribes, - * so consumers read this on subscription to detect a fatal failure they missed. + * The current readiness failure, or `null`. Retryable timeouts clear once authoritative sync + * completes; terminal rejections remain latched. Consumers read it when subscribing so an earlier + * event is not missed. */ joinError: JoinFileDocError | null = null @@ -116,18 +143,47 @@ export class FileDocProvider extends ObservableV2 { /** Deadline for reaching readiness (synced + seeded); fires the fallback if it is never reached. */ private readinessTimer: ReturnType | null = null private joinAccepted = false + private joinedDocument: Pick | null = null + private updateMode: 'negotiating' | 'legacy' | 'acknowledged' = 'negotiating' private joinPending = false private joinRetryAttempt = 0 private joinRetryTimer: ReturnType | null = null + private joinAckTimer: ReturnType | null = null + private syncRetryTimer: ReturnType | null = null + private syncRetryAttempt = 0 + private joinHydrating = false + private connectionGeneration = 0 + private bufferedMessages: Uint8Array[] = [] + private bufferedMessageBytes = 0 + private pendingUpdateBatch: Uint8Array[] = [] + private inFlightUpdate: PendingClientUpdate | null = null + private updateBatchTimer: ReturnType | null = null + private updateRetryTimer: ReturnType | null = null + private updateRetryAttempt = 0 + private updateFlushInProgress = false + private flushingUpdate: Uint8Array | null = null + private pendingUpdatesDrained = false + private recoveryApplied = false + private recoveryQueued = false + private beforeUnloadProtected = false + private readonly journal: PendingFileDocUpdateJournal | null + private recoveryLoad: { + docId: string + promise: ReturnType + } | null = null constructor( private readonly socket: Socket, private readonly fileId: string, readonly doc: Y.Doc, - readonly awareness: awarenessProtocol.Awareness + readonly awareness: awarenessProtocol.Awareness, + scope?: FileDocProviderScope ) { super() + this.journal = scope ? new PendingFileDocUpdateJournal({ ...scope, fileId: this.fileId }) : null + this.registerActiveProvider() + // Restore an empty local awareness state if it has been cleared. A fresh // Awareness starts with `{}`, but a *reused* one whose local state was removed // (a prior provider's `destroy()` clears it, and so does `Awareness.destroy()`) @@ -143,11 +199,13 @@ export class FileDocProvider extends ObservableV2 { socket.on(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) socket.on(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) socket.on(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + socket.on(FILE_DOC_EVENTS.INVALIDATED, this.handleInvalidated) socket.on(ROOM_ACCESS_REVOKED_EVENT, this.handleAccessRevoked) socket.on('connect', this.handleConnect) socket.on('disconnect', this.handleDisconnect) doc.on('update', this.handleDocUpdate) awareness.on('update', this.handleAwarenessUpdate) + if (typeof window !== 'undefined') window.addEventListener('pagehide', this.handlePageHide) // Watch the seed flag so reaching "seeded" (server seed applied) can clear the readiness deadline. doc.getMap(FILE_DOC_SEED.configMap).observe(this.handleConfigChange) @@ -157,8 +215,7 @@ export class FileDocProvider extends ObservableV2 { if (socket.connected) this.join() - // Arm the fallback: if we don't reach readiness (synced + seeded) before the deadline, give up. - this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + this.armReadinessDeadline() } /** Whether the server seed has recorded the initial content on the doc. */ @@ -168,25 +225,33 @@ export class FileDocProvider extends ObservableV2 { /** Clear the readiness deadline once the editor is usable (synced AND seeded). */ private handleConfigChange = () => { - if (this.synced && this.isSeeded()) this.clearReadinessTimer() + if (this.synced && this.isSeeded()) { + this.clearReadinessTimer() + if (this.joinError?.retryable) this.joinError = null + } + if (this.updateMode === 'acknowledged' && this.docId() && this.pendingUpdateBatch.length > 0) { + this.scheduleUpdateFlush(0) + } } /** * Readiness was never reached within {@link READINESS_DEADLINE_MS} — either the realtime server is - * unreachable (never synced) or it synced but the server-side seed never landed (synced yet - * unseeded). Reset `synced` (so the editor gates read-only), latch fatal (so a late reconnect or - * seed can't sync server state in and merge-duplicate the content the editor is about to render - * locally), and surface a synthetic non-retryable join-error — the exact path a fatal rejection - * uses — so the owner falls back to the read-only view of the file's stored content instead of a - * blank pane. No-op if we already reached readiness, already failed fatally, or were torn down. + * unreachable or its authoritative seed is delayed. Keep retrying with the existing bounded + * join/sync backoff. The owner's stored-content preview must remain separate from this Y.Doc. */ private handleReadinessDeadline = () => { this.readinessTimer = null - if (this.synced && this.isSeeded()) return - // Dropping `synced` (see {@link failFatally}) is what keeps the editor's `synced && seeded` gate - // closed, so the fallback renders the stored content read-only rather than becoming editable on a - // document the server never seeded. - this.failFatally('Realtime document was not ready in time', 'READINESS_TIMEOUT') + if (this.disposed || this.fatal || (this.synced && this.isSeeded())) return + this.joinError = { + fileId: this.fileId, + error: 'Realtime document was not ready in time', + code: 'READINESS_TIMEOUT', + retryable: true, + } + this.setSynced(false) + this.emit('join-error', [this.joinError]) + if (this.joinAccepted) this.scheduleSyncRetry() + else if (!this.joinPending) this.scheduleJoinRetry() } private clearReadinessTimer() { @@ -196,6 +261,11 @@ export class FileDocProvider extends ObservableV2 { } } + private armReadinessDeadline() { + this.clearReadinessTimer() + this.readinessTimer = setTimeout(this.handleReadinessDeadline, READINESS_DEADLINE_MS) + } + private clearJoinRetryTimer() { if (this.joinRetryTimer !== null) { clearTimeout(this.joinRetryTimer) @@ -203,11 +273,45 @@ export class FileDocProvider extends ObservableV2 { } } + private clearJoinAckTimer() { + if (this.joinAckTimer !== null) { + clearTimeout(this.joinAckTimer) + this.joinAckTimer = null + } + } + + private clearSyncRetryTimer() { + if (this.syncRetryTimer !== null) { + clearTimeout(this.syncRetryTimer) + this.syncRetryTimer = null + } + } + + private clearUpdateTimers() { + if (this.updateBatchTimer !== null) clearTimeout(this.updateBatchTimer) + if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) + this.updateBatchTimer = null + this.updateRetryTimer = null + } + /** Join the room, binding our client id so the server only accepts awareness we own. */ private join = () => { if (this.fatal || this.disposed || !this.socket.connected || this.joinPending) return this.joinPending = true - this.socket.emit(FILE_DOC_EVENTS.JOIN, { fileId: this.fileId, clientId: this.doc.clientID }) + this.clearJoinAckTimer() + this.joinAckTimer = setTimeout(() => { + this.joinAckTimer = null + if (!this.joinPending || this.fatal || this.disposed) return + this.joinPending = false + this.joinAccepted = false + this.setSynced(false) + this.scheduleJoinRetry() + }, FILE_DOC_TIMEOUTS.joinAckMs) + this.socket.emit(FILE_DOC_EVENTS.JOIN, { + fileId: this.fileId, + clientId: this.doc.clientID, + schemaVersion: FILE_DOC_SCHEMA_VERSION, + }) } private scheduleJoinRetry() { @@ -232,7 +336,10 @@ export class FileDocProvider extends ObservableV2 { */ private handleConnect = () => { if (this.fatal) return + this.connectionGeneration += 1 this.clearJoinRetryTimer() + this.clearSyncRetryTimer() + this.syncRetryAttempt = 0 this.joinAccepted = false this.joinPending = false this.joinRetryAttempt = 0 @@ -241,8 +348,15 @@ export class FileDocProvider extends ObservableV2 { } private handleDisconnect = () => { + this.connectionGeneration += 1 + this.clearBufferedMessages() this.clearJoinRetryTimer() + this.clearJoinAckTimer() + this.clearSyncRetryTimer() + if (this.updateRetryTimer !== null) clearTimeout(this.updateRetryTimer) + this.updateRetryTimer = null this.joinAccepted = false + this.joinHydrating = false this.joinPending = false this.setSynced(false) @@ -268,7 +382,6 @@ export class FileDocProvider extends ObservableV2 { * rebuilt only when the room AND the shared stream are both gone (a tab that slept through it), which * is precisely when a stale tab reconnects. There is no way to un-merge afterwards, so the sync never * happens: take the fatal path, which leaves the editor read-only on the content it already shows. - * A reload binds a fresh document and recovers. */ private handleJoinSuccess = (data: JoinFileDocSuccess) => { if ( @@ -277,20 +390,115 @@ export class FileDocProvider extends ObservableV2 { (data.clientId !== undefined && data.clientId !== this.doc.clientID) ) return + this.clearJoinAckTimer() this.joinPending = false this.joinRetryAttempt = 0 this.clearJoinRetryTimer() + this.joinedDocument = { docId: data.docId, version: data.version } + if (data.acknowledgedUpdates === true && data.docId !== undefined) { + this.updateMode = 'acknowledged' + } + this.joinHydrating = true + const generation = this.connectionGeneration + if (!this.journal || data.docId === undefined) { + this.finishAcceptJoin(data, generation, null) + return + } + const recoveryDocId = data.docId + if (!this.recoveryLoad || this.recoveryLoad.docId !== recoveryDocId) { + this.recoveryLoad = { + docId: recoveryDocId, + promise: this.journal.load(recoveryDocId), + } + } + void this.recoveryLoad.promise.then((recovered) => { + this.finishAcceptJoin(data, generation, recovered) + }) + } + + private finishAcceptJoin( + data: JoinFileDocSuccess, + generation: number, + recovered: Awaited> + ): void { + if ( + this.disposed || + this.fatal || + !this.socket.connected || + generation !== this.connectionGeneration || + !this.joinHydrating + ) + return + + const serverSchemaVersion = data.schemaVersion ?? 1 + if (serverSchemaVersion !== FILE_DOC_SCHEMA_VERSION) { + this.failFatally( + 'This document version is not supported; refresh to continue editing', + 'SCHEMA_VERSION_MISMATCH' + ) + return + } + const local = this.docId() - if (local !== undefined && data.docId !== undefined && data.docId !== local) { + if ( + data.docId !== undefined && + ((local !== undefined && data.docId !== local) || (local === undefined && this.isSeeded())) + ) { + this.failFatally( + 'This document was reloaded on the server; refresh to continue editing', + 'DOCUMENT_REPLACED' + ) + return + } + + if (recovered !== null && !this.recoveryApplied) { + try { + if (recovered.recoverySnapshot) { + Y.applyUpdate(this.doc, recovered.recoverySnapshot, RECOVERY_ORIGIN) + } + Y.applyUpdate(this.doc, recovered.pendingUpdate, RECOVERY_ORIGIN) + } catch { + this.failFatally('The local recovery copy could not be restored.', 'INVALID_UPDATE') + return + } + this.recoveryApplied = true + } + + if ( + recovered !== null && + (data.docId !== recovered.docId || this.docId() !== recovered.docId) + ) { this.failFatally( 'This document was reloaded on the server; refresh to continue editing', 'DOCUMENT_REPLACED' ) return } + + const updateMode = + data.acknowledgedUpdates === true && data.docId !== undefined ? 'acknowledged' : 'legacy' + /** Pre-negotiation deltas stay in Y.Doc for legacy sync; existing recovery is never acknowledged here. */ + if (updateMode === 'legacy' && this.updateMode === 'negotiating') this.pendingUpdateBatch = [] + this.updateMode = updateMode + + if (recovered !== null && !this.recoveryQueued) { + this.queuePendingUpdate(recovered.pendingUpdate) + this.recoveryQueued = true + } + this.updateBeforeUnloadProtection() + + this.joinHydrating = false this.joinAccepted = true this.sendSyncStep1() + this.scheduleSyncRetry() this.sendLocalAwareness() + const bufferedMessages = this.bufferedMessages + this.clearBufferedMessages() + for (const message of bufferedMessages) this.applyMessage(message) + if (this.updateMode === 'acknowledged') { + if (this.inFlightUpdate) this.sendInFlightUpdate() + else if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) + } } /** The identity of the document we hold, once the server seed has named one. */ @@ -313,15 +521,49 @@ export class FileDocProvider extends ObservableV2 { retryable: false, } this.fatal = true + this.clearBufferedMessages() this.joinError = error + void this.persistPendingSnapshot() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearUpdateTimers() + this.clearSyncRetryTimer() this.joinAccepted = false this.joinPending = false + this.joinHydrating = false + this.clearJoinAckTimer() this.setSynced(false) this.emit('join-error', [error]) } + private registerActiveProvider(): void { + const active = FileDocProvider.activeProviders.get(this.socket) + if (active?.fileId === this.fileId) { + active.providers.add(this) + return + } + if (active) { + for (const provider of active.providers) { + provider.drainPendingUpdates() + provider.failFatally( + 'Another file was opened in this tab. Reload this file to resume editing it.', + 'DOCUMENT_REPLACED' + ) + } + } + FileDocProvider.activeProviders.set(this.socket, { + fileId: this.fileId, + providers: new Set([this]), + }) + } + + private unregisterActiveProvider(): void { + const active = FileDocProvider.activeProviders.get(this.socket) + if (active?.fileId !== this.fileId) return + active.providers.delete(this) + if (active.providers.size === 0) FileDocProvider.activeProviders.delete(this.socket) + } + /** * Handle a join rejection. A non-retryable rejection (access denied, invalid) * won't succeed on retry, so latch {@link fatal} to stop (re)joining and let the @@ -336,11 +578,16 @@ export class FileDocProvider extends ObservableV2 { return this.joinAccepted = false this.joinPending = false + this.joinHydrating = false + this.clearJoinAckTimer() if (data.retryable === false) { this.fatal = true this.joinError = data + void this.persistPendingSnapshot() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearUpdateTimers() + this.clearSyncRetryTimer() this.setSynced(false) } else { this.setSynced(false) @@ -362,14 +609,46 @@ export class FileDocProvider extends ObservableV2 { this.failFatally(data.message, 'ACCESS_REVOKED') } + private handleInvalidated = (data: FileDocInvalidated) => { + if (data.fileId !== this.fileId) return + const joined = this.joinedDocument + if (data.docId && joined?.docId && data.docId !== joined.docId) return + if ( + !data.docId && + data.version !== undefined && + joined?.version !== undefined && + data.version < joined.version + ) + return + this.failFatally(data.message, 'DOCUMENT_REPLACED') + } + private handleMessage = (data: unknown) => { - // Once we've given up (a non-retryable rejection, or the connect deadline lapsed and the editor - // fell back to a read-only local seed), ignore ALL inbound frames. A late SyncStep2 arriving - // after the deadline would otherwise merge the server's state into the already-seeded doc — - // duplicating content — and flip `synced` true, which un-gates autosave and would persist the - // duplicate back to the real file. `fatal` guarding (re)join alone is not enough; it must also - // stop applying sync here. - if (this.fatal || !this.joinAccepted) return + /** A terminal authorization or generation failure must never accept late document frames. */ + if (this.fatal) return + if (this.joinHydrating) { + const bytes = toFileDocBytes(data) + if (!bytes) return + if ( + this.bufferedMessages.length >= MAX_HYDRATION_MESSAGES || + this.bufferedMessageBytes + bytes.byteLength > MAX_HYDRATION_BYTES + ) { + this.failFatally( + 'Realtime document hydration exceeded its safety limit', + 'HYDRATION_BUFFER_OVERFLOW' + ) + return + } + const buffered = new Uint8Array(bytes) + this.bufferedMessages.push(buffered) + this.bufferedMessageBytes += buffered.byteLength + return + } + if (!this.joinAccepted) return + this.applyMessage(data) + } + + private applyMessage(data: unknown) { const bytes = toFileDocBytes(data) if (!bytes) return @@ -384,7 +663,19 @@ export class FileDocProvider extends ObservableV2 { // re-sending updates we just applied from the server. const syncType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this) if (encoding.length(encoder) > 1) { - this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + const response = encoding.toUint8Array(encoder) + if (this.updateMode === 'acknowledged' && syncType === syncProtocol.messageYjsSyncStep1) { + const responseDecoder = decoding.createDecoder(response) + decoding.readVarUint(responseDecoder) + decoding.readVarUint(responseDecoder) + const update = new Uint8Array(decoding.readVarUint8Array(responseDecoder)) + if (hasYjsUpdateContent(update)) { + this.queuePendingUpdate(update) + this.scheduleUpdateFlush(0) + } + } else { + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, response) + } } if (syncType === syncProtocol.messageYjsSyncStep2 && !this.synced) this.setSynced(true) break @@ -401,26 +692,278 @@ export class FileDocProvider extends ObservableV2 { } private handleDocUpdate = (update: Uint8Array, origin: unknown) => { - // Once fatal (a non-retryable rejection, or the readiness deadline lapsed), the editor may render - // the stored content into the doc locally as its read-only fallback. Never relay those local - // writes — the server never seeded this doc, so echoing them would push unseeded content to peers - // (and each fallen-back client would do so, union-duplicating). A fatal client is fully local. - if (this.fatal || !this.joinAccepted || !this.socket.connected) return - // Updates we applied from the server carry `this` as origin — don't echo them. - if (origin === this) return + /** A terminal document cannot publish; inbound and recovery updates must not echo. */ + if (this.fatal || origin === this || origin === RECOVERY_ORIGIN) return // Agent-streamed frames must reach peers (so a collaborator sees the stream live) but must NOT be // treated by the server as a durable user edit — the copilot's final `edit_content` write is the // authoritative persist. Tag them so the relay applies + fans out but skips persist bookkeeping. - const messageType = - origin === AGENT_STREAM_ORIGIN - ? FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST - : FILE_DOC_MESSAGE_TYPE.SYNC + if (origin === AGENT_STREAM_ORIGIN) { + if (!this.joinAccepted || !this.socket.connected) return + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC_NO_PERSIST) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + return + } + + if (!this.joinAccepted || !this.socket.connected) { + this.queuePendingUpdate(update) + if (this.updateMode !== 'negotiating') this.scheduleUpdateFlush(UPDATE_BATCH_MS) + return + } + + if (this.updateMode !== 'acknowledged') { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) + syncProtocol.writeUpdate(encoder, update) + this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) + return + } + + this.queuePendingUpdate(update) + this.scheduleUpdateFlush(UPDATE_BATCH_MS) + } + + private queuePendingUpdate(update: Uint8Array): void { + this.pendingUpdateBatch.push(update) + this.updateBeforeUnloadProtection() + } + + private scheduleUpdateFlush(delay: number) { + if (this.updateBatchTimer !== null || this.updateFlushInProgress || this.disposed || this.fatal) + return + this.updateBatchTimer = setTimeout(() => { + this.updateBatchTimer = null + void this.flushPendingUpdates() + }, delay) + } + + private async flushPendingUpdates(): Promise { + if ( + this.updateMode === 'negotiating' || + this.pendingUpdateBatch.length === 0 || + this.disposed || + this.fatal + ) + return + const docId = this.docId() + if (!docId) return + + this.updateFlushInProgress = true + let hasUnjournaledUpdates = false + try { + const update = Y.mergeUpdates(this.pendingUpdateBatch) + this.flushingUpdate = update + this.pendingUpdateBatch = [] + const journalUpdate = this.inFlightUpdate + ? Y.mergeUpdates([this.inFlightUpdate.update, update]) + : update + const saved = await this.journal?.save(docId, journalUpdate, Y.encodeStateAsUpdate(this.doc)) + hasUnjournaledUpdates = this.pendingUpdateBatch.length > 0 + if (this.pendingUpdatesDrained) return + if (this.disposed || this.fatal) { + this.queuePendingUpdate(update) + return + } + if (saved?.status === 'limit-exceeded') { + this.queuePendingUpdate(update) + this.failFatally('Local edits exceeded the safe recovery limit.', 'PENDING_UPDATE_LIMIT') + return + } + const durableUpdate = saved?.pendingUpdate ?? update + + if (this.inFlightUpdate) { + this.queuePendingUpdate(update) + return + } + this.inFlightUpdate = { updateId: generateShortId(), update: durableUpdate } + this.updateRetryAttempt = 0 + this.sendInFlightUpdate() + } finally { + this.flushingUpdate = null + this.updateFlushInProgress = false + this.updateBeforeUnloadProtection() + if (this.pendingUpdateBatch.length > 0 && (hasUnjournaledUpdates || !this.inFlightUpdate)) { + this.scheduleUpdateFlush(0) + } + } + } + + private sendInFlightUpdate() { + const pending = this.inFlightUpdate + const docId = this.docId() + if ( + !pending || + !docId || + this.updateMode !== 'acknowledged' || + this.disposed || + this.fatal || + !this.socket.connected || + !this.joinAccepted + ) + return + + const generation = this.connectionGeneration + const payload: FileDocUpdatePayload = { + fileId: this.fileId, + docId, + updateId: pending.updateId, + update: pending.update, + } + this.socket + .timeout(FILE_DOC_TIMEOUTS.updateAckMs) + .emit(FILE_DOC_EVENTS.UPDATE, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + if (this.disposed || this.fatal || this.inFlightUpdate !== pending) return + if (error) { + if (generation === this.connectionGeneration) this.scheduleUpdateRetry() + return + } + if (ack) this.handleUpdateAck(ack) + }) + } + + private handleUpdateAck(ack: FileDocUpdateAck) { + const pending = this.inFlightUpdate + if (!pending || ack.updateId !== pending.updateId || this.disposed || this.fatal) return + + if (ack.status === 'accepted') { + const docId = this.docId() + this.inFlightUpdate = null + this.updateRetryAttempt = 0 + this.updateBeforeUnloadProtection() + if (this.pendingUpdateBatch.length > 0) this.scheduleUpdateFlush(0) + else if (!this.updateFlushInProgress && docId) void this.journal?.clear(docId, pending.update) + return + } + + if (!ack.retryable) { + const message = + ack.code === 'ACCESS_REVOKED' + ? 'Your access to this document has been revoked' + : 'This document changed while this tab was disconnected; refresh to continue editing' + this.failFatally(message, ack.code) + return + } + if (ack.code === 'NOT_JOINED') { + this.setSynced(false) + this.joinAccepted = false + this.joinPending = false + this.clearSyncRetryTimer() + this.scheduleJoinRetry() + return + } + this.scheduleUpdateRetry() + } + + private scheduleUpdateRetry() { + if (this.updateRetryTimer !== null || this.disposed || this.fatal || !this.socket.connected) + return + this.updateRetryAttempt += 1 + this.updateRetryTimer = setTimeout( + () => { + this.updateRetryTimer = null + this.sendInFlightUpdate() + }, + backoffWithJitter(this.updateRetryAttempt, null, { + baseMs: UPDATE_RETRY_BASE_MS, + maxMs: UPDATE_RETRY_MAX_MS, + }) + ) + } + + private pendingJournalUpdate(): Uint8Array | null { + const updates = [ + ...(this.inFlightUpdate ? [this.inFlightUpdate.update] : []), + ...(this.flushingUpdate ? [this.flushingUpdate] : []), + ...this.pendingUpdateBatch, + ] + return updates.length > 0 ? Y.mergeUpdates(updates) : null + } + + private persistPendingSnapshot(): Promise | undefined { + const update = this.pendingJournalUpdate() + const docId = this.docId() + if (!update || !docId || !this.journal) return + return this.journal.save(docId, update, Y.encodeStateAsUpdate(this.doc)).then(() => undefined) + } + + /** + * Transfer the final batch before LEAVE or a different file's JOIN changes socket membership. + * The relay admits UPDATE synchronously and pins its room until publication finishes. Only the + * immutable bytes and journal survive teardown; an acceptance clears them after all queued saves. + */ + private drainPendingUpdates(): void { + if ( + this.disposed || + this.fatal || + this.pendingUpdatesDrained || + !this.joinAccepted || + !this.socket.connected + ) + return + const update = this.pendingJournalUpdate() + if (!update || update.byteLength > FILE_DOC_LIMITS.updateBytes) return + const docId = this.docId() + if (this.updateMode === 'acknowledged' && !docId) return + + const updateId = + this.inFlightUpdate && !this.flushingUpdate && this.pendingUpdateBatch.length === 0 + ? this.inFlightUpdate.updateId + : generateShortId() + const journal = this.journal + const snapshotSaved = this.persistPendingSnapshot() + this.pendingUpdatesDrained = true + this.pendingUpdateBatch = [] + this.inFlightUpdate = null + this.flushingUpdate = null + this.clearUpdateTimers() + + if (this.updateMode === 'acknowledged' && docId) { + const payload: FileDocUpdatePayload = { fileId: this.fileId, docId, updateId, update } + this.socket + .timeout(FILE_DOC_TIMEOUTS.updateAckMs) + .emit(FILE_DOC_EVENTS.UPDATE, payload, (error: Error | null, ack?: FileDocUpdateAck) => { + if (error || ack?.status !== 'accepted' || ack.updateId !== updateId) return + if (journal && snapshotSaved) { + void snapshotSaved.then(() => journal.clear(docId, update)) + } + }) + return + } + const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageType) + encoding.writeVarUint(encoder, FILE_DOC_MESSAGE_TYPE.SYNC) syncProtocol.writeUpdate(encoder, update) this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } + private handlePageHide = () => { + void this.persistPendingSnapshot() + } + + private handleBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault() + event.returnValue = '' + } + + private updateBeforeUnloadProtection(): void { + if (typeof window === 'undefined') return + const shouldProtect = + !this.disposed && + (this.pendingUpdateBatch.length > 0 || + this.inFlightUpdate !== null || + this.updateFlushInProgress) + if (shouldProtect === this.beforeUnloadProtected) return + this.beforeUnloadProtected = shouldProtect + if (shouldProtect) window.addEventListener('beforeunload', this.handleBeforeUnload) + else window.removeEventListener('beforeunload', this.handleBeforeUnload) + } + + private clearBufferedMessages(): void { + this.bufferedMessages = [] + this.bufferedMessageBytes = 0 + } + private handleAwarenessUpdate = ( { added, updated, removed }: { added: number[]; updated: number[]; removed: number[] }, origin: unknown @@ -452,6 +995,25 @@ export class FileDocProvider extends ObservableV2 { this.socket.emit(FILE_DOC_EVENTS.MESSAGE, encoding.toUint8Array(encoder)) } + private scheduleSyncRetry() { + this.clearSyncRetryTimer() + if (this.synced || this.fatal || this.disposed || !this.socket.connected || !this.joinAccepted) + return + this.syncRetryAttempt += 1 + this.syncRetryTimer = setTimeout( + () => { + this.syncRetryTimer = null + if (this.synced || this.fatal || this.disposed || !this.joinAccepted) return + this.sendSyncStep1() + this.scheduleSyncRetry() + }, + backoffWithJitter(this.syncRetryAttempt, null, { + baseMs: 1_000, + maxMs: JOIN_RETRY_MAX_MS, + }) + ) + } + private sendLocalAwareness() { if (this.awareness.getLocalState() === null) return const encoder = encoding.createEncoder() @@ -466,9 +1028,16 @@ export class FileDocProvider extends ObservableV2 { private setSynced(synced: boolean) { if (this.synced === synced) return this.synced = synced + if (synced) { + this.clearSyncRetryTimer() + this.syncRetryAttempt = 0 + } // Readiness needs synced AND seeded; only clear the deadline when both hold (the seed may have // arrived first, or may still be pending — `handleConfigChange` clears it if seeded arrives later). - if (synced && this.isSeeded()) this.clearReadinessTimer() + if (synced && this.isSeeded()) { + this.clearReadinessTimer() + if (this.joinError?.retryable) this.joinError = null + } this.emit('synced', [synced]) } @@ -482,9 +1051,17 @@ export class FileDocProvider extends ObservableV2 { super.destroy() return } + this.drainPendingUpdates() + void this.persistPendingSnapshot() this.disposed = true + this.updateBeforeUnloadProtection() + this.unregisterActiveProvider() this.clearReadinessTimer() this.clearJoinRetryTimer() + this.clearJoinAckTimer() + this.clearSyncRetryTimer() + this.clearUpdateTimers() + this.clearBufferedMessages() this.joinPending = false // Publish our final awareness removal while this provider is still admitted. A co-mounted sibling @@ -500,12 +1077,14 @@ export class FileDocProvider extends ObservableV2 { this.socket.off(FILE_DOC_EVENTS.MESSAGE, this.handleMessage) this.socket.off(FILE_DOC_EVENTS.JOIN_SUCCESS, this.handleJoinSuccess) this.socket.off(FILE_DOC_EVENTS.JOIN_ERROR, this.handleJoinError) + this.socket.off(FILE_DOC_EVENTS.INVALIDATED, this.handleInvalidated) this.socket.off(ROOM_ACCESS_REVOKED_EVENT, this.handleAccessRevoked) this.socket.off('connect', this.handleConnect) this.socket.off('disconnect', this.handleDisconnect) this.doc.off('update', this.handleDocUpdate) this.doc.getMap(FILE_DOC_SEED.configMap).unobserve(this.handleConfigChange) this.awareness.off('update', this.handleAwarenessUpdate) + if (typeof window !== 'undefined') window.removeEventListener('pagehide', this.handlePageHide) super.destroy() } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts new file mode 100644 index 00000000000..5d4a910ccb1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.test.ts @@ -0,0 +1,317 @@ +/** + * @vitest-environment node + */ +import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' +import { get, update as updateValue } from 'idb-keyval' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' + +const storage = vi.hoisted(() => new Map()) + +vi.mock('idb-keyval', () => ({ + get: vi.fn(async (key: string) => storage.get(key)), + update: vi.fn((key: string, updater: (value: unknown) => unknown) => { + storage.set(key, updater(storage.get(key))) + }), +})) + +import { PendingFileDocUpdateJournal } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal' + +function journal(): PendingFileDocUpdateJournal { + return new PendingFileDocUpdateJournal({ + workspaceId: 'workspace-1', + fileId: 'file-1', + userId: 'user-1', + }) +} + +function updateWith(text: string): Uint8Array { + const doc = new Y.Doc() + doc.getText('body').insert(0, text) + return Y.encodeStateAsUpdate(doc) +} + +describe('PendingFileDocUpdateJournal', () => { + beforeEach(() => { + storage.clear() + vi.mocked(updateValue) + .mockReset() + .mockImplementation(async (key, updater) => { + storage.set(String(key), updater(storage.get(String(key)))) + }) + }) + + it('stores a full recovery snapshot separately from the pending wire update', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + const recoverySnapshot = updateWith('complete local draft') + + await subject.save('doc-1', pendingUpdate, recoverySnapshot) + + await expect(subject.load('doc-1')).resolves.toEqual( + expect.objectContaining({ docId: 'doc-1', pendingUpdate, recoverySnapshot }) + ) + }) + + it('reports when the current full recovery snapshot cannot be stored', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + + const result = await subject.save( + 'doc-1', + pendingUpdate, + new Uint8Array(FILE_DOC_LIMITS.updateBytes * 2 + 1) + ) + + expect(result).toMatchObject({ status: 'limit-exceeded' }) + }) + + it('loads an existing draft without requiring a writable transaction', async () => { + const subject = journal() + const pendingUpdate = updateWith('recoverable draft') + await subject.save('doc-1', pendingUpdate, pendingUpdate) + const writes = vi.mocked(updateValue).mock.calls.length + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Read-only storage')) + + await expect(subject.load('doc-1')).resolves.toMatchObject({ docId: 'doc-1', pendingUpdate }) + expect(updateValue).toHaveBeenCalledTimes(writes) + }) + + it.each(['pendingUpdate', 'recoverySnapshot'] as const)( + 'isolates malformed %s bytes without replaying or deleting them', + async (field) => { + const subject = journal() + const valid = updateWith('preserved snapshot') + const invalid = new Uint8Array([255]) + const pending = field === 'pendingUpdate' ? invalid : valid + const snapshot = field === 'recoverySnapshot' ? invalid : valid + await subject.save('doc-1', pending, snapshot) + + await expect(subject.load('doc-1')).resolves.toBeNull() + await expect(journal().load()).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [ + expect.objectContaining({ + docId: 'doc-1', + pendingUpdate: pending, + recoverySnapshot: snapshot, + quarantined: true, + }), + ], + }), + ]) + + const newUpdate = updateWith('new edits') + await expect(subject.save('doc-1', newUpdate, newUpdate)).resolves.toMatchObject({ + status: 'saved', + pendingUpdate: newUpdate, + }) + await expect(subject.load('doc-1')).resolves.toMatchObject({ pendingUpdate: newUpdate }) + await subject.clear('doc-1', newUpdate) + await expect(subject.load()).resolves.toBeNull() + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: [expect.objectContaining({ pendingUpdate: pending, quarantined: true })], + }), + ]) + } + ) + + it('ignores malformed recovery even if browser storage cannot be updated', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const before = structuredClone([...storage.values()]) + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) + + await expect(subject.load()).resolves.toBeNull() + expect([...storage.values()]).toEqual(before) + }) + + it('does not quarantine a record that another tab replaced after the read', async () => { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('doc-1', invalid, invalid) + const stale = structuredClone([...storage.values()][0]) + storage.clear() + const valid = updateWith('concurrent valid edits') + await subject.save('doc-1', valid, valid) + vi.mocked(get).mockResolvedValueOnce(stale) + + await expect(subject.load()).resolves.toBeNull() + await expect(subject.load()).resolves.toMatchObject({ pendingUpdate: valid }) + }) + + it('prioritizes valid recovery within the existing record cap', async () => { + const subject = journal() + const valid = updateWith('valid') + await subject.save('first', valid, valid) + const invalid = new Uint8Array([255]) + await subject.save('invalid', invalid, invalid) + await expect(subject.load('invalid')).resolves.toBeNull() + await subject.save('second', valid, valid) + await subject.save('third', valid, valid) + + for (const docId of ['first', 'second', 'third']) { + await expect(subject.load(docId)).resolves.toMatchObject({ docId }) + } + expect([...storage.values()]).toEqual([ + expect.objectContaining({ + documents: expect.arrayContaining([ + expect.objectContaining({ docId: 'first' }), + expect.objectContaining({ docId: 'second' }), + expect.objectContaining({ docId: 'third' }), + ]), + }), + ]) + expect((storage.values().next().value as { documents: unknown[] }).documents).toHaveLength(3) + }) + + it('does not extend malformed recovery retention while quarantining it', async () => { + vi.useFakeTimers() + try { + const subject = journal() + const invalid = new Uint8Array([255]) + await subject.save('invalid', invalid, invalid) + await vi.advanceTimersByTimeAsync(6 * 24 * 60 * 60 * 1_000) + await expect(subject.load()).resolves.toBeNull() + await vi.advanceTimersByTimeAsync(2 * 24 * 60 * 60 * 1_000) + const valid = updateWith('new edits') + await subject.save('current', valid, valid) + + expect([...storage.values()]).toEqual([ + expect.objectContaining({ documents: [expect.objectContaining({ docId: 'current' })] }), + ]) + } finally { + vi.useRealTimers() + } + }) + + it('distinguishes unavailable browser storage from a configured size limit', async () => { + vi.mocked(updateValue).mockRejectedValueOnce(new Error('Storage denied')) + const pendingUpdate = updateWith('pending') + + await expect(journal().save('doc-1', pendingUpdate, pendingUpdate)).resolves.toEqual({ + pendingUpdate, + status: 'unavailable', + }) + }) + + it('atomically preserves concurrent providers until their aggregate is acknowledged', async () => { + const first = journal() + const second = journal() + const firstUpdate = updateWith('a') + const secondUpdate = updateWith('b') + + await first.save('doc-1', firstUpdate, firstUpdate) + const combined = await second.save('doc-1', secondUpdate, secondUpdate) + await first.clear('doc-1', firstUpdate) + await expect(first.load('doc-1')).resolves.not.toBeNull() + + const recovered = new Y.Doc() + Y.applyUpdate(recovered, combined.pendingUpdate) + expect(recovered.getText('body').toString()).toHaveLength(2) + + await second.clear('doc-1', combined.pendingUpdate) + await expect(first.load('doc-1')).resolves.toBeNull() + }) + + it('preserves the snapshot dependencies of pending edits from concurrent tabs', async () => { + const first = journal() + const second = journal() + const base = new Y.Doc() + base.getText('body').insert(0, 'base') + const firstDoc = new Y.Doc() + const secondDoc = new Y.Doc() + Y.applyUpdate(firstDoc, Y.encodeStateAsUpdate(base)) + Y.applyUpdate(secondDoc, Y.encodeStateAsUpdate(base)) + + firstDoc.getText('body').insert(4, ' acknowledged') + const firstVector = Y.encodeStateVector(firstDoc) + firstDoc.getText('body').insert(17, ' pending-first') + await first.save( + 'doc-1', + Y.encodeStateAsUpdate(firstDoc, firstVector), + Y.encodeStateAsUpdate(firstDoc) + ) + + const secondVector = Y.encodeStateVector(secondDoc) + secondDoc.getText('body').insert(4, ' pending-second') + await second.save( + 'doc-1', + Y.encodeStateAsUpdate(secondDoc, secondVector), + Y.encodeStateAsUpdate(secondDoc) + ) + + const stored = await journal().load('doc-1') + expect(stored).not.toBeNull() + const recovered = new Y.Doc() + Y.applyUpdate(recovered, stored!.recoverySnapshot!) + Y.applyUpdate(recovered, stored!.pendingUpdate) + + const expected = new Y.Doc() + Y.applyUpdate(expected, Y.encodeStateAsUpdate(firstDoc)) + Y.applyUpdate(expected, Y.encodeStateAsUpdate(secondDoc)) + expect(recovered.getText('body').toString()).toBe(expected.getText('body').toString()) + expect(recovered.getText('body').toString()).toContain('pending-first') + expect(recovered.getText('body').toString()).toContain('pending-second') + for (const doc of [base, firstDoc, secondDoc, recovered, expected]) doc.destroy() + }) + + it('bounds the combined snapshots without overwriting the previous recovery copy', async () => { + const subject = journal() + const pendingUpdate = updateWith('pending') + const firstSnapshot = updateWith('a'.repeat(FILE_DOC_LIMITS.updateBytes)) + const secondSnapshot = updateWith('b'.repeat(FILE_DOC_LIMITS.updateBytes)) + await subject.save('doc-1', pendingUpdate, firstSnapshot) + + await expect(subject.save('doc-1', pendingUpdate, secondSnapshot)).resolves.toMatchObject({ + status: 'limit-exceeded', + }) + const recovered = await subject.load('doc-1') + expect(recovered?.recoverySnapshot).toBeInstanceOf(Uint8Array) + expect(Buffer.from(recovered!.recoverySnapshot!).equals(Buffer.from(firstSnapshot))).toBe(true) + }) + + it('retains bounded recovery records for separate document identities', async () => { + const subject = journal() + for (const docId of ['doc-1', 'doc-2', 'doc-3', 'doc-4']) { + const update = updateWith(docId) + await subject.save(docId, update, update) + } + + await expect(subject.load('doc-4')).resolves.toMatchObject({ docId: 'doc-4' }) + await expect(subject.load('doc-2')).resolves.toMatchObject({ docId: 'doc-2' }) + await expect(subject.load('doc-1')).resolves.toBeNull() + await expect(subject.load()).resolves.toMatchObject({ docId: 'doc-4' }) + }) + + it('clears only the acknowledged document identity', async () => { + const subject = journal() + const oldUpdate = updateWith('old') + const currentUpdate = updateWith('current') + await subject.save('old-doc', oldUpdate, oldUpdate) + await subject.save('current-doc', currentUpdate, currentUpdate) + + await subject.clear('old-doc', oldUpdate) + + await expect(subject.load('old-doc')).resolves.toBeNull() + await expect(subject.load('current-doc')).resolves.toMatchObject({ docId: 'current-doc' }) + }) + + it('isolates records by user, workspace, and file', async () => { + const first = journal() + const otherUser = new PendingFileDocUpdateJournal({ + workspaceId: 'workspace-1', + fileId: 'file-1', + userId: 'user-2', + }) + const update = updateWith('draft') + + await first.save('doc-1', update, update) + + await expect(first.load()).resolves.not.toBeNull() + await expect(otherUser.load()).resolves.toBeNull() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts new file mode 100644 index 00000000000..40f12cdfc6d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/pending-update-journal.ts @@ -0,0 +1,241 @@ +'use client' + +import { createLogger } from '@sim/logger' +import { FILE_DOC_LIMITS } from '@sim/realtime-protocol/file-doc' +import { get, update as updateValue } from 'idb-keyval' +import * as Y from 'yjs' + +const logger = createLogger('PendingFileDocUpdateJournal') +const JOURNAL_VERSION = 1 +const JOURNAL_TTL_MS = 7 * 24 * 60 * 60 * 1_000 +const MAX_DOCUMENTS = 3 +const RECOVERY_SNAPSHOT_MAX_BYTES = FILE_DOC_LIMITS.updateBytes * 2 + +export interface PendingDocumentRecovery { + docId: string + pendingUpdate: Uint8Array + recoverySnapshot: Uint8Array | null + updatedAt: number +} + +interface PendingUpdateJournalRecord { + version: typeof JOURNAL_VERSION + documents: JournalDocument[] +} + +interface JournalDocument extends PendingDocumentRecovery { + quarantined?: boolean +} + +interface PendingUpdateJournalScope { + workspaceId: string + fileId: string + userId: string +} + +interface JournalSaveResult { + pendingUpdate: Uint8Array + status: 'saved' | 'limit-exceeded' | 'unavailable' +} + +function isRecovery(value: unknown): value is JournalDocument { + if (typeof value !== 'object' || value === null) return false + const candidate = value as Partial + return ( + typeof candidate.docId === 'string' && + candidate.docId.length > 0 && + candidate.pendingUpdate instanceof Uint8Array && + candidate.pendingUpdate.byteLength > 0 && + candidate.pendingUpdate.byteLength <= FILE_DOC_LIMITS.updateBytes && + (candidate.recoverySnapshot === null || + (candidate.recoverySnapshot instanceof Uint8Array && + candidate.recoverySnapshot.byteLength > 0 && + candidate.recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES)) && + typeof candidate.updatedAt === 'number' && + Number.isFinite(candidate.updatedAt) && + (candidate.quarantined === undefined || typeof candidate.quarantined === 'boolean') + ) +} + +function liveDocuments(value: unknown, now: number): JournalDocument[] { + if (typeof value !== 'object' || value === null) return [] + const candidate = value as Partial + if (candidate.version !== JOURNAL_VERSION || !Array.isArray(candidate.documents)) return [] + return candidate.documents + .filter(isRecovery) + .filter((document) => now - document.updatedAt <= JOURNAL_TTL_MS) + .sort( + (left, right) => + Number(left.quarantined === true) - Number(right.quarantined === true) || + right.updatedAt - left.updatedAt + ) + .slice(0, MAX_DOCUMENTS) +} + +function record(documents: JournalDocument[]): PendingUpdateJournalRecord { + return { version: JOURNAL_VERSION, documents } +} + +function sameUpdate(left: Uint8Array | null, right: Uint8Array | null): boolean { + if (left === null || right === null) return left === right + if (left.byteLength !== right.byteLength) return false + return left.every((byte, index) => byte === right[index]) +} + +/** + * A bounded crash-recovery journal for user edits the relay has not acknowledged. One atomic + * file-scoped envelope retains up to three recent Yjs document identities, so rebuilding a live + * document cannot overwrite an older local draft. The pending delta is wire-bounded separately from + * the full recovery snapshot: only the delta is ever replayed to a matching server document. + */ +export class PendingFileDocUpdateJournal { + private readonly key: string + private mutationQueue = Promise.resolve() + + constructor({ workspaceId, fileId, userId }: PendingUpdateJournalScope) { + const origin = typeof location === 'undefined' ? 'server' : location.origin + this.key = [ + 'sim', + 'file-doc-pending', + JOURNAL_VERSION, + origin, + userId, + workspaceId, + fileId, + ].join(':') + } + + async load(preferredDocId?: string): Promise { + try { + await this.mutationQueue + const documents = liveDocuments(await get(this.key), Date.now()).filter( + (document) => !document.quarantined + ) + const recovered = preferredDocId + ? (documents.find((document) => document.docId === preferredDocId) ?? null) + : (documents[0] ?? null) + if (!recovered) return null + const validationDoc = new Y.Doc() + try { + if (recovered.recoverySnapshot) Y.applyUpdate(validationDoc, recovered.recoverySnapshot) + Y.applyUpdate(validationDoc, recovered.pendingUpdate) + return recovered + } catch (error) { + logger.warn('Isolating malformed pending file edits', { error }) + await this.quarantine(recovered) + return null + } finally { + validationDoc.destroy() + } + } catch (error) { + logger.warn('Failed to load pending file edits', { error }) + return null + } + } + + save( + docId: string, + pendingUpdate: Uint8Array, + recoverySnapshot: Uint8Array + ): Promise { + const pendingWithinLimit = + pendingUpdate.byteLength > 0 && pendingUpdate.byteLength <= FILE_DOC_LIMITS.updateBytes + const snapshotWithinLimit = + recoverySnapshot.byteLength > 0 && recoverySnapshot.byteLength <= RECOVERY_SNAPSHOT_MAX_BYTES + const limited: JournalSaveResult = { pendingUpdate, status: 'limit-exceeded' } + if (!pendingWithinLimit || !snapshotWithinLimit) return Promise.resolve(limited) + + return this.enqueue( + async () => { + let result = limited + await updateValue(this.key, (value) => { + const now = Date.now() + const documents = liveDocuments(value, now) + const existing = documents.find( + (document) => document.docId === docId && !document.quarantined + ) + const merged = existing + ? Y.mergeUpdates([existing.pendingUpdate, pendingUpdate]) + : pendingUpdate + if (merged.byteLength > FILE_DOC_LIMITS.updateBytes) return record(documents) + + const mergedSnapshot = existing?.recoverySnapshot + ? Y.mergeUpdates([existing.recoverySnapshot, recoverySnapshot]) + : recoverySnapshot + if (mergedSnapshot.byteLength > RECOVERY_SNAPSHOT_MAX_BYTES) return record(documents) + + const next: PendingDocumentRecovery = { + docId, + pendingUpdate: merged, + recoverySnapshot: mergedSnapshot, + updatedAt: now, + } + const retained = [ + next, + ...documents.filter((document) => document.docId !== docId || document.quarantined), + ].slice(0, MAX_DOCUMENTS) + result = { + pendingUpdate: merged, + status: 'saved', + } + return record(retained) + }) + if (result.status === 'limit-exceeded') { + logger.warn('Pending file edits exceeded the crash-recovery journal limit') + } + return result + }, + { pendingUpdate, status: 'unavailable' } + ) + } + + clear(docId: string, acknowledgedUpdate: Uint8Array): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => { + const documents = liveDocuments(value, Date.now()) + return record( + documents.filter( + (document) => + document.quarantined || + document.docId !== docId || + !sameUpdate(document.pendingUpdate, acknowledgedUpdate) + ) + ) + }), + undefined + ) + } + + /** Retain invalid bytes within the journal's existing bounds without replaying or merging them. */ + private quarantine(recovered: PendingDocumentRecovery): Promise { + return this.enqueue( + () => + updateValue(this.key, (value) => + record( + liveDocuments(value, Date.now()).map((document) => + document.docId === recovered.docId && + document.updatedAt === recovered.updatedAt && + sameUpdate(document.pendingUpdate, recovered.pendingUpdate) && + sameUpdate(document.recoverySnapshot, recovered.recoverySnapshot) + ? { ...document, quarantined: true } + : document + ) + ) + ), + undefined + ) + } + + private enqueue(operation: () => Promise, fallback: T): Promise { + const result = this.mutationQueue.then(operation, operation) + this.mutationQueue = result.then( + () => undefined, + () => undefined + ) + return result.catch((error) => { + logger.warn('Failed to persist pending file edits', { error }) + return fallback + }) + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts index 6164e03085c..394e2140e2f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.test.ts @@ -11,18 +11,17 @@ import { const at = (input: Partial): CollabReadinessInputs => ({ synced: false, seeded: false, - offlineSeed: false, fatal: false, ...input, }) describe('isCollabReady', () => { it('is not ready before syncing or seeding', () => { - expect(isCollabReady(at({ synced: false, seeded: false, offlineSeed: false }))).toBe(false) + expect(isCollabReady(at({ synced: false, seeded: false }))).toBe(false) }) it('is not ready when synced but not yet seeded', () => { - expect(isCollabReady(at({ synced: true, seeded: false, offlineSeed: false }))).toBe(false) + expect(isCollabReady(at({ synced: true, seeded: false }))).toBe(false) }) it('is ready only when the current session is synced and the server seed is present', () => { @@ -33,10 +32,6 @@ describe('isCollabReady', () => { expect(isCollabReady(at({ synced: false, seeded: true }))).toBe(false) }) - it('stays read-only for an offline (local) seed that never reached the server', () => { - expect(isCollabReady(at({ synced: true, seeded: true, offlineSeed: true }))).toBe(false) - }) - it('revokes readiness when a live document turns fatal', () => { expect(isCollabReady(at({ synced: true, seeded: true, fatal: true }))).toBe(false) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts index 38d59aee26f..8169f9bed31 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/readiness.ts @@ -5,17 +5,11 @@ export interface CollabReadinessInputs { synced: boolean /** Whether the shared doc carries the seed flag. */ seeded: boolean - /** Whether the seed flag was set by the offline fallback (no server sync) rather than the server. */ - offlineSeed: boolean - /** - * Whether the provider has GIVEN UP on this document — a non-retryable rejection, an access - * revocation, or the readiness deadline lapsing. A fatal provider ignores every inbound frame and - * never rejoins, so nothing typed after this point reaches the server. - */ + /** A terminal rejection or access revocation prevents further synchronization and editing. */ fatal: boolean } /** A document is writable only while this connection has synced the server-seeded Yjs document. */ export function isCollabReady(input: CollabReadinessInputs): boolean { - return input.synced && input.seeded && !input.offlineSeed && !input.fatal + return input.synced && input.seeded && !input.fatal } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts index 7dc9eaaa530..2bad83b13da 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -5,9 +5,9 @@ import { FILE_DOC_EVENTS, type FileDocPresence } from '@sim/realtime-protocol/fi import { Awareness } from 'y-protocols/awareness' import * as Y from 'yjs' import { getUserColor } from '@/lib/workspaces/colors' +import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' +import { useReportFileDocOthers } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context' import { useSocket } from '@/app/workspace/providers/socket-provider' -import { FileDocProvider } from './file-doc-provider' -import { useReportFileDocOthers } from './file-doc-room-context' /** The live collaboration binding the editor wires into TipTap's Collaboration * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ @@ -32,6 +32,7 @@ export interface FileDocCollaboration { } interface UseFileDocCollaborationParams { + workspaceId: string fileId: string userId: string userName: string @@ -51,6 +52,7 @@ interface UseFileDocCollaborationParams { * realtime relay over the shared socket. Returns `null` while disabled. */ export function useFileDocCollaboration({ + workspaceId, fileId, userId, userName, @@ -102,13 +104,16 @@ export function useFileDocCollaboration({ // (see above), so this always binds the same doc/awareness the editor froze at mount. const doc = docRef.current as Y.Doc const awareness = awarenessRef.current as Awareness - const fileProvider = new FileDocProvider(socket, fileId, doc, awareness) + const fileProvider = new FileDocProvider(socket, fileId, doc, awareness, { + workspaceId, + userId, + }) setProvider(fileProvider) return () => { fileProvider.destroy() setProvider(null) } - }, [enabled, socket, fileId]) + }, [enabled, socket, fileId, workspaceId, userId]) const reportOthers = useReportFileDocOthers() const reportOthersRef = useRef(reportOthers) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx index 914a0643d63..90829e78ed7 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-lifecycle.test.tsx @@ -22,7 +22,10 @@ const { collaborationRef, uploadFile } = vi.hoisted(() => ({ uploadFile: vi.fn(), })) -vi.mock('next/navigation', () => ({ useRouter: () => ({ push: vi.fn() }) })) +vi.mock('next/navigation', () => ({ + usePathname: () => '/workspace/workspace-1/files', + useRouter: () => ({ push: vi.fn() }), +})) vi.mock('@/lib/auth/auth-client', () => ({ useSession: () => ({ data: null, isPending: false }) })) vi.mock('@/hooks/queries/workspace-files', () => ({ useUploadWorkspaceFile: () => ({ mutateAsync: uploadFile }), @@ -253,9 +256,13 @@ describe('loaded rich editor lifecycle', () => { expect(editor.isEditable).toBe(true) expect(editor.view.dom.getAttribute('aria-readonly')).toBe('false') expect(container.textContent).not.toContain('Reconnecting…') + expect(container.querySelector('[role="status"]')).toBeNull() + expect(container.querySelector('[role="alert"]')).toBeNull() + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() }) - it('keeps the live document visible and read-only after a fatal collaboration error', async () => { + it('keeps revoked pending edits visible and read-only without draft-management prompts', async () => { const provider = new FakeFileDocProvider() const doc = new Y.Doc() doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) @@ -275,7 +282,7 @@ describe('loaded rich editor lifecycle', () => { provider.fail({ fileId: 'file-1', error: 'Access denied', - code: 'ACCESS_DENIED', + code: 'ACCESS_REVOKED', retryable: false, }) ) @@ -286,6 +293,13 @@ describe('loaded rich editor lifecycle', () => { expect(editor.view.dom.closest('.hidden')).toBeNull() expect(container.textContent).not.toContain('stale opening snapshot') expect(container.textContent).not.toContain('Reconnecting…') + expect(container.querySelector('[role="status"]')?.textContent).toBe( + 'You no longer have edit access to this document.' + ) + expect(container.querySelector('button')).toBeNull() + expect(container.querySelector('[role="alert"], [role="dialog"]')).toBeNull() + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() }) it('shows stored content read-only when collaboration fails before the first sync', async () => { @@ -316,6 +330,81 @@ describe('loaded rich editor lifecycle', () => { expect(container.textContent).not.toContain('Reconnecting…') }) + it('keeps timeout preview separate from the authoritative document and recovers on late sync', async () => { + const provider = new FakeFileDocProvider() + const doc = new Y.Doc() + collaborationRef.current = { + doc, + awareness: new Awareness(doc), + provider, + user: { name: 'User', color: '#000000', clientId: doc.clientID }, + } + await render('stored preview body', 'stored preview body', true, { collaborative: true }) + await act(async () => + provider.fail({ + fileId: 'file-1', + error: 'Not ready', + code: 'READINESS_TIMEOUT', + retryable: true, + }) + ) + expect(container.textContent).toContain('stored preview body') + expect(container.textContent).toContain('Reconnecting…') + expect(doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag)).toBeUndefined() + expect(doc.getXmlFragment('default').length).toBe(0) + const editors = [...container.querySelectorAll('.tiptap')].map( + (element) => (element as HTMLElement & { editor: Editor }).editor + ) + expect(editors.every((editor) => !editor.isEditable)).toBe(true) + await act(async () => { + provider.joinError = null + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + provider.setSynced(true) + }) + expect(container.textContent).not.toContain('stored preview body') + expect(container.textContent).not.toContain('Reconnecting…') + expect(getEditor().isEditable).toBe(true) + expect(onClientAutosaveChange).not.toHaveBeenCalledWith(true) + }) + + it.each(['DOCUMENT_REPLACED', 'PENDING_UPDATE_LIMIT', 'INVALID_UPDATE'])( + 'preserves pending edits with only a passive status for %s', + async (code) => { + const provider = new FakeFileDocProvider() + const doc = new Y.Doc() + doc.getMap(FILE_DOC_SEED.configMap).set(FILE_DOC_SEED.flag, true) + collaborationRef.current = { + doc, + awareness: new Awareness(doc), + provider, + user: { name: 'User', color: '#000000', clientId: doc.clientID }, + } + await render('stored body', 'stored body', true, { collaborative: true }) + + await act(async () => provider.setSynced(true)) + await act(async () => getEditor().commands.insertContent('preserved local change')) + await act(async () => + provider.fail({ + fileId: 'file-1', + error: 'Local recovery required', + code, + retryable: false, + }) + ) + + expect(container.querySelector('[role="status"]')?.textContent).toBe( + 'Live editing is unavailable.' + ) + expect(container.querySelector('button')).toBeNull() + expect(container.querySelector('[role="alert"], [role="dialog"]')).toBeNull() + expect(container.textContent).not.toContain('Reconnecting…') + expect(toast.warning).not.toHaveBeenCalled() + expect(toast.info).not.toHaveBeenCalled() + expect(getEditor().isEditable).toBe(false) + expect(getEditor().getText()).toContain('preserved local change') + } + ) + it('explains a picker selection whose insertion anchor was invalidated', async () => { await render('before TARGET after') const editor = getEditor() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts index 2f730b40771..32afc078615 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts @@ -1,11 +1,26 @@ /** * @vitest-environment jsdom */ + +import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import { Editor } from '@tiptap/core' -import { undoDepth } from '@tiptap/pm/history' -import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { getFindTally, RichMarkdownFind, setFindQuery, stepFindMatch } from './find-extension' +import { redoDepth, undoDepth } from '@tiptap/pm/history' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import type * as Y from 'yjs' +import { markdownToYDoc } from '@/lib/collab-doc/converter' +import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + getFindTally, + RichMarkdownFind, + replaceActiveFindMatch, + replaceAllFindMatches, + setFindQuery, + stepFindMatch, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension' +import { FIND_MATCH_LIMIT } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' let editor: Editor | null = null afterEach(() => { @@ -129,4 +144,133 @@ describe('RichMarkdownFind', () => { // instead of their real last edit. expect(undoDepth(instance.state)).toBe(undoBefore) }) + + it('replaces the active match while preserving its inline marks', () => { + const instance = mountEditor('**alpha** and alpha') + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'beta')).toBe(true) + expect(instance.getMarkdown()).toBe('**beta** and alpha') + expect(getFindTally(instance.state).matches).toHaveLength(1) + }) + + it('uses the matched text formatting instead of an unrelated typing mark', () => { + const instance = mountEditor('alpha and beta') + instance.commands.setTextSelection(instance.state.doc.content.size - 1) + instance.commands.toggleBold() + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'gamma')).toBe(true) + expect(instance.getMarkdown()).toBe('gamma and beta') + }) + + it.each([ + ['he**llo**', 'world'], + ['**he**llo', '**world**'], + ])('follows native ProseMirror replacement formatting for %s', (source, expected) => { + const instance = mountEditor(source) + setFindQuery(instance, 'hello') + const { from, to } = getFindTally(instance.state).matches[0] + const nativeResult = instance.state.tr.setStoredMarks(null).insertText('world', from, to).doc + + expect(replaceActiveFindMatch(instance, 'world')).toBe(true) + expect(instance.state.doc.eq(nativeResult)).toBe(true) + expect(instance.getMarkdown()).toBe(expected) + }) + + it('preserves each matched range formatting during Replace All', () => { + const instance = mountEditor('**alpha** and alpha and *alpha*') + instance.commands.setTextSelection(instance.state.doc.content.size - 1) + instance.commands.toggleStrike() + setFindQuery(instance, 'alpha') + + expect(replaceAllFindMatches(instance, 'beta')).toBe(3) + expect(instance.getMarkdown()).toBe('**beta** and beta and *beta*') + }) + + it('supports deleting matches with an empty replacement', () => { + const instance = mountEditor('alpha beta alpha') + setFindQuery(instance, 'alpha ') + + expect(replaceActiveFindMatch(instance, '')).toBe(true) + expect(instance.getMarkdown()).toBe('beta alpha') + }) + + it('rejects oversized individual and aggregate replacements before dispatching a transaction', () => { + const instance = mountEditor(Array.from({ length: FIND_MATCH_LIMIT }, () => 'x').join(' ')) + setFindQuery(instance, 'x') + const onLimitExceeded = vi.fn() + const dispatch = vi.spyOn(instance.view, 'dispatch') + const documentBefore = instance.state.doc + + expect( + replaceActiveFindMatch( + instance, + 'y'.repeat(PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS), + onLimitExceeded + ) + ).toBe(false) + expect(replaceAllFindMatches(instance, 'y'.repeat(600), onLimitExceeded)).toBe(0) + + expect(onLimitExceeded).toHaveBeenCalledTimes(2) + expect(dispatch).not.toHaveBeenCalled() + expect(instance.state.doc).toBe(documentBefore) + }) + + it('advances past a replacement that still contains the search term', () => { + const instance = mountEditor('alpha alpha') + setFindQuery(instance, 'alpha') + + expect(replaceActiveFindMatch(instance, 'alphaX')).toBe(true) + expect(replaceActiveFindMatch(instance, 'alphaX')).toBe(true) + + expect(instance.getMarkdown()).toBe('alphaX alphaX') + }) + + it('keeps each collaborative replacement as a separate undo item', () => { + const doc = markdownToYDoc('alpha alpha') + const awareness = new Awareness(doc) + editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#fff' } }, + }), + }) + const history = yUndoPluginKey.getState(editor.state) as { undoManager: Y.UndoManager } + history.undoManager.clear() + setFindQuery(editor, 'alpha') + + replaceActiveFindMatch(editor, 'beta') + replaceActiveFindMatch(editor, 'gamma') + expect(editor.getMarkdown()).toBe('beta gamma') + + expect(editor.commands.undo()).toBe(true) + expect(editor.getMarkdown()).toBe('beta alpha') + editor.destroy() + editor = null + awareness.destroy() + doc.destroy() + }) + + it('replaces every match in one undo step', () => { + const instance = mountEditor('alpha alpha alpha') + setFindQuery(instance, 'alpha') + const undoBefore = undoDepth(instance.state) + + expect(replaceAllFindMatches(instance, 'beta')).toBe(3) + expect(instance.getMarkdown()).toBe('beta beta beta') + expect(undoDepth(instance.state)).toBe(undoBefore + 1) + expect(redoDepth(instance.state)).toBe(0) + expect(instance.commands.undo()).toBe(true) + expect(instance.getMarkdown()).toBe('alpha alpha alpha') + }) + + it('refuses to label a capped partial replacement as replace all', () => { + const instance = mountEditor(Array.from({ length: FIND_MATCH_LIMIT + 1 }, () => 'x').join(' ')) + setFindQuery(instance, 'x') + expect(getFindTally(instance.state).truncated).toBe(true) + + expect(replaceAllFindMatches(instance, 'y')).toBe(0) + expect(instance.getMarkdown().startsWith('x x x')).toBe(true) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts index e2ac3f0a20a..d3c9805f47a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts @@ -1,9 +1,16 @@ +import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Editor } from '@tiptap/core' import { Extension } from '@tiptap/core' +import { closeHistory } from '@tiptap/pm/history' import type { EditorState } from '@tiptap/pm/state' import { Plugin, PluginKey } from '@tiptap/pm/state' import { Decoration, DecorationSet } from '@tiptap/pm/view' -import { EMPTY_FIND_RESULT, type FindMatch, findMatches } from './find-matches' +import { yUndoPluginKey } from '@tiptap/y-tiptap' +import { + EMPTY_FIND_RESULT, + type FindMatch, + findMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' /** Class on every match. The active one carries {@link ACTIVE_MATCH_CLASS} as well. */ const MATCH_CLASS = 'rich-find-match' @@ -149,3 +156,87 @@ export function setFindQuery(editor: Editor, query: string): void { export function stepFindMatch(editor: Editor, delta: number): void { dispatchFindMeta(editor, { activeIndex: getFindTally(editor.state).activeIndex + delta }) } + +function stopUndoCapture(editor: Editor): void { + const state = yUndoPluginKey.getState(editor.state) as + | { undoManager?: { stopCapturing: () => void } } + | undefined + state?.undoManager?.stopCapturing() +} + +function isolateReplacement(editor: Editor, transaction: EditorState['tr']): void { + stopUndoCapture(editor) + editor.view.dispatch(closeHistory(transaction).scrollIntoView()) + stopUndoCapture(editor) + editor.view.dispatch(closeHistory(editor.state.tr)) +} + +/** Bounds aggregate growth before Replace All can materialize hundreds of large insertions. */ +function replacementExceedsLimit( + editor: Editor, + matches: readonly FindMatch[], + replacement: string +): boolean { + const currentSize = editor.state.doc.content.size + const nextSize = matches.reduce( + (size, match) => size + replacement.length - (match.to - match.from), + currentSize + ) + return nextSize > Math.max(currentSize, PASTE_RENDER_THRESHOLDS.ENHANCED_TEXT_CHARACTERS) +} + +/** Uses the target range's marks, independent of formatting armed at the editor's caret. */ +function replaceMatch(transaction: EditorState['tr'], match: FindMatch, replacement: string): void { + const marks = transaction.doc.resolve(match.from).marksAcross(transaction.doc.resolve(match.to)) + transaction.replaceWith( + match.from, + match.to, + replacement ? transaction.doc.type.schema.text(replacement, marks) : [] + ) +} + +export function replaceActiveFindMatch( + editor: Editor, + replacement: string, + onLimitExceeded?: () => void +): boolean { + if (!editor.isEditable) return false + const findState = RICH_FIND_PLUGIN_KEY.getState(editor.state) ?? INITIAL_STATE + const { matches, activeIndex } = findState + const match = matches[activeIndex] + if (!match) return false + if (replacementExceedsLimit(editor, [match], replacement)) { + onLimitExceeded?.() + return false + } + const transaction = editor.state.tr + replaceMatch(transaction, match, replacement) + const remaining = findMatches(transaction.doc, findState.query).matches + const insertionEnd = match.from + replacement.length + const nextIndex = remaining.findIndex((candidate) => candidate.from >= insertionEnd) + transaction.setMeta(RICH_FIND_PLUGIN_KEY, { activeIndex: nextIndex === -1 ? 0 : nextIndex }) + isolateReplacement(editor, transaction) + return true +} + +/** Replaces every collected match in one undo step; capped searches must first be narrowed. */ +export function replaceAllFindMatches( + editor: Editor, + replacement: string, + onLimitExceeded?: () => void +): number { + if (!editor.isEditable) return 0 + const { matches, truncated } = getFindTally(editor.state) + if (truncated || matches.length === 0) return 0 + if (replacementExceedsLimit(editor, matches, replacement)) { + onLimitExceeded?.() + return 0 + } + const transaction = closeHistory(editor.state.tr) + for (let index = matches.length - 1; index >= 0; index -= 1) { + const match = matches[index] + replaceMatch(transaction, match, replacement) + } + isolateReplacement(editor, transaction) + return matches.length +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts index b62f392cea7..ed0cebdf333 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts @@ -3,8 +3,11 @@ */ import { Editor } from '@tiptap/core' import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { FIND_MATCH_LIMIT, findMatches } from './find-matches' +import { createMarkdownContentExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/extensions' +import { + FIND_MATCH_LIMIT, + findMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches' let editor: Editor | null = null afterEach(() => { @@ -59,6 +62,24 @@ describe('findMatches', () => { expect(matchedText('ab\n\ncd', 'abcd')).toEqual([]) }) + it.each(['\uFFFF', 'a\uFFFFb'])('never matches an inline atom using %j', (query) => { + const doc = docFor('a
b') + expect(() => doc.check()).not.toThrow() + expect(findMatches(doc, query)).toEqual({ matches: [], truncated: false }) + }) + + it('does not count atom placeholders toward the match limit', () => { + const doc = docFor('a
b\uFFFF') + expect(() => doc.check()).not.toThrow() + const { matches, truncated } = findMatches(doc, '\uFFFF', 1) + expect(matches.map(({ from, to }) => doc.textBetween(from, to))).toEqual(['\uFFFF']) + expect(truncated).toBe(false) + }) + + it('keeps real non-character text searchable across a formatting boundary', () => { + expect(matchedText('a**\uFFFF**b', 'a\uFFFFb')).toEqual(['a\uFFFFb']) + }) + it('never matches across an inline atom', () => { // The image between them occupies a position; joining `a` to `b` would be a phantom match. expect(matchedText('a![alt](https://x.com/i.png)b', 'ab')).toEqual([]) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts index 5133557d52f..933313c8ec2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts @@ -25,8 +25,8 @@ export const EMPTY_FIND_RESULT: FindResult = { matches: [], truncated: false } /** * Stands in for one position of a non-text inline node (an image, a mention chip) so a match can * never span one — searching `ab` must not join the `a` before an image to the `b` after it. U+FFFF - * is a permanent Unicode non-character, so no query can contain it and match the placeholder itself, - * and it is not whitespace, so the shared scan's whitespace folding leaves it alone. + * is not whitespace, so the shared scan's whitespace folding leaves it alone. Segment checks exclude + * atoms even when a query contains this character, without excluding genuine U+FFFF text. */ const ATOM_PLACEHOLDER = '￿' @@ -34,6 +34,7 @@ const ATOM_PLACEHOLDER = ' interface TextSegment { textStart: number docStart: number + isText: boolean } /** @@ -74,7 +75,7 @@ export function findMatches( if (soleText === null) { const built: TextSegment[] = [] node.forEach((child, offset) => { - built.push({ textStart: text.length, docStart: pos + 1 + offset }) + built.push({ textStart: text.length, docStart: pos + 1 + offset, isText: child.isText }) text += child.isText && child.text ? child.text : ATOM_PLACEHOLDER.repeat(child.nodeSize) }) segments = built @@ -83,6 +84,21 @@ export function findMatches( let segmentIndex = 0 forEachSearchOccurrence(text, query, (start, end) => { if (truncated) return + if (segments) { + while ( + segmentIndex + 1 < segments.length && + segments[segmentIndex + 1].textStart <= start + ) { + segmentIndex++ + } + for ( + let index = segmentIndex; + index < segments.length && segments[index].textStart < end; + index++ + ) { + if (!segments[index].isText) return + } + } if (matches.length >= limit) { truncated = true return @@ -91,10 +107,6 @@ export function findMatches( matches.push({ from: pos + 1 + start, to: pos + 1 + end }) return } - // Segments are ordered and occurrences arrive left to right, so the cursor only moves forward. - while (segmentIndex + 1 < segments.length && segments[segmentIndex + 1].textStart <= start) { - segmentIndex++ - } const segment = segments[segmentIndex] const from = segment.docStart + (start - segment.textStart) matches.push({ from, to: from + (end - start) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts index b0a97d5a6f6..d3a103a69c1 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts @@ -2,9 +2,17 @@ import type React from 'react' import { useCallback, useEffect, useRef, useState } from 'react' +import { toast } from '@sim/emcn' import type { Editor } from '@tiptap/react' import { useFindShortcut } from '@/app/workspace/[workspaceId]/components' -import { ACTIVE_MATCH_CLASS, getFindTally, setFindQuery, stepFindMatch } from './find-extension' +import { + ACTIVE_MATCH_CLASS, + getFindTally, + replaceActiveFindMatch, + replaceAllFindMatches, + setFindQuery, + stepFindMatch, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension' /** What the surface hands `FindBar`, plus the open state the shortcut drives. */ export interface MarkdownFindController { @@ -14,9 +22,13 @@ export interface MarkdownFindController { currentIndex: number truncated: boolean inputRef: React.RefObject + replacement: string setQuery: (query: string) => void + setReplacement: (replacement: string) => void next: () => void prev: () => void + replaceCurrent: () => void + replaceAll: () => void close: () => void } @@ -29,6 +41,12 @@ interface FindTally { const EMPTY_TALLY: FindTally = { count: 0, currentIndex: 0, truncated: false } +function warnReplacementLimit(): void { + toast.warning('Replacement is too large', { + description: 'Use the source editor for changes that exceed the rich-text editing limit.', + }) +} + interface UseMarkdownFindOptions { editor: Editor | null /** @@ -55,6 +73,7 @@ export function useMarkdownFind({ }: UseMarkdownFindOptions): MarkdownFindController { const [isOpen, setIsOpen] = useState(false) const [query, setQueryState] = useState('') + const [replacement, setReplacement] = useState('') const [tally, setTally] = useState(EMPTY_TALLY) const inputRef = useRef(null) const editorRef = useRef(editor) @@ -141,13 +160,29 @@ export function useMarkdownFind({ const next = useCallback(() => step(1), [step]) const prev = useCallback(() => step(-1), [step]) + const replaceCurrent = useCallback(() => { + const current = editorRef.current + if (!current || !replaceActiveFindMatch(current, replacement, warnReplacementLimit)) return + revealActiveMatch() + }, [replacement, revealActiveMatch]) + + const replaceAll = useCallback(() => { + const current = editorRef.current + if (!current) return + replaceAllFindMatches(current, replacement, warnReplacementLimit) + }, [replacement]) + /** Closing ends the search: term, highlights and active match all go. */ const close = useCallback(() => { setIsOpen(false) setQueryState('') + setReplacement('') setTally(EMPTY_TALLY) const current = editorRef.current if (current) setFindQuery(current, '') + requestAnimationFrame(() => { + if (current && !current.isDestroyed) current.commands.focus() + }) }, []) const open = useCallback(() => setIsOpen(true), []) @@ -156,13 +191,17 @@ export function useMarkdownFind({ return { isOpen, query, + replacement, count: tally.count, currentIndex: tally.currentIndex, truncated: tally.truncated, inputRef, setQuery, + setReplacement, next, prev, + replaceCurrent, + replaceAll, close, } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-collaboration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-collaboration.test.tsx new file mode 100644 index 00000000000..9d140fe0ef7 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-collaboration.test.tsx @@ -0,0 +1,197 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { Tooltip } from '@sim/emcn' +import Collaboration from '@tiptap/extension-collaboration' +import { Editor, EditorContent } from '@tiptap/react' +import StarterKit from '@tiptap/starter-kit' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' +import { ResizableImage } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image' + +let host: HTMLDivElement +let root: Root +let local: Editor +let peer: Editor +let localDoc: Y.Doc +let peerDoc: Y.Doc + +beforeEach(async () => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.useFakeTimers() + localDoc = new Y.Doc() + peerDoc = new Y.Doc() + const createEditor = (document: Y.Doc) => + new Editor({ + extensions: [ + StarterKit.configure({ undoRedo: false }), + ResizableImage, + Collaboration.configure({ document }), + ], + editorProps: { handleScrollToSelection: () => true }, + }) + local = createEditor(localDoc) + local.commands.setContent( + '

Earlier heading

Original

After image

' + ) + Y.applyUpdate(peerDoc, Y.encodeStateAsUpdate(localDoc)) + peer = createEditor(peerDoc) + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) + await act(async () => { + root.render( + + + + ) + }) + await act(async () => local.commands.setNodeSelection(imagePosition(local))) +}) + +afterEach(async () => { + await act(async () => { + root.unmount() + local.destroy() + peer.destroy() + }) + localDoc.destroy() + peerDoc.destroy() + host.remove() + vi.clearAllTimers() + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +function imagePosition(editor: Editor): number { + let position = -1 + editor.state.doc.descendants((node, pos) => { + if (node.type.name === 'image') position = pos + }) + return position +} + +function imageAttributes(editor: Editor) { + const position = imagePosition(editor) + return position < 0 ? null : editor.state.doc.nodeAt(position)?.attrs +} + +async function receivePeerUpdate(): Promise { + await act(async () => Y.applyUpdate(localDoc, Y.encodeStateAsUpdate(peerDoc))) +} + +function pointer(target: EventTarget, type: string, clientX: number): void { + const event = new MouseEvent(type, { bubbles: true, cancelable: true, button: 0, clientX }) + Object.defineProperty(event, 'pointerId', { value: 7 }) + act(() => target.dispatchEvent(event)) +} + +function beginResize(): void { + const image = host.querySelector('img')! + const handle = host.querySelector('button[aria-label="Resize image"]')! + Object.defineProperty(image, 'offsetWidth', { value: 200, configurable: true }) + Object.assign(handle, { + setPointerCapture: vi.fn(), + hasPointerCapture: vi.fn(() => true), + releasePointerCapture: vi.fn(), + }) + pointer(handle, 'pointerdown', 100) + pointer(window, 'pointermove', 160) + expect(image.style.width).toBe('260px') +} + +describe('image interactions during real peer Yjs updates', () => { + it.each(['pointerup', 'pointercancel', 'blur', 'unmount'])( + 'removes the resize transaction listener after %s', + (finish) => { + const subscribe = vi.spyOn(local, 'on') + const unsubscribe = vi.spyOn(local, 'off') + beginResize() + const listener = subscribe.mock.calls.find(([event]) => event === 'transaction')?.[1] + expect(listener).toBeTypeOf('function') + + if (finish === 'unmount') act(() => root.unmount()) + else pointer(window, finish, 160) + + expect(unsubscribe).toHaveBeenCalledWith('transaction', listener) + } + ) + + it('preserves peer alt text when only the local link draft changes', async () => { + act(() => + host.querySelector('button[aria-label="Edit image details"]')!.click() + ) + const input = host.querySelector('input[aria-label="Image link URL"]')! + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call( + input, + 'https://sim.ai/local-link' + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + peer.commands.setNodeSelection(imagePosition(peer)) + peer.commands.updateAttributes('image', { alt: 'Peer corrected alt' }) + await receivePeerUpdate() + expect(imageAttributes(local)?.alt).toBe('Peer corrected alt') + expect(host.querySelector('input[aria-label="Image link URL"]')).toBe(input) + await act(async () => { + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + + expect(imageAttributes(local)).toMatchObject({ + alt: 'Peer corrected alt', + href: 'https://sim.ai/local-link', + }) + }) + + it('keeps resizing the same image after a peer heading and metadata edit', async () => { + const originalImage = localDoc.getXmlFragment('default').get(1) + beginResize() + peer.commands.insertContentAt('Earlier heading'.length + 1, ' PEER') + peer.commands.setNodeSelection(imagePosition(peer)) + peer.commands.updateAttributes('image', { alt: 'Peer corrected alt' }) + await receivePeerUpdate() + + expect(localDoc.getXmlFragment('default').get(1)).toBe(originalImage) + pointer(window, 'pointerup', 160) + expect(imageAttributes(local)).toMatchObject({ + alt: 'Peer corrected alt', + width: '260', + height: null, + }) + expect(local.state.doc.firstChild?.textContent).toBe('Earlier heading PEER') + }) + + it.each([false, true])( + 'cancels a resize when the peer replaces the actual image node (identical attributes: %s)', + async (identicalAttributes) => { + const originalImage = localDoc.getXmlFragment('default').get(1) + const replacement = identicalAttributes + ? { ...imageAttributes(peer) } + : { src: 'https://sim.ai/replacement.png', alt: 'Replacement', width: '400', height: '300' } + beginResize() + const position = imagePosition(peer) + peer.commands.deleteRange({ from: position, to: position + 1 }) + peer.commands.insertContentAt(position, { type: 'image', attrs: replacement }) + await receivePeerUpdate() + + expect(localDoc.getXmlFragment('default').get(1)).not.toBe(originalImage) + expect(host.querySelector('img')?.style.width).toBe( + identicalAttributes ? '200px' : '400px' + ) + pointer(window, 'pointerup', 160) + expect(imageAttributes(local)).toMatchObject(replacement) + } + ) + + it('cancels a resize when the peer deletes the image', async () => { + beginResize() + const position = imagePosition(peer) + peer.commands.deleteRange({ from: position, to: position + 1 }) + await receivePeerUpdate() + pointer(window, 'pointerup', 160) + + expect(host.querySelector('img')).toBeNull() + expect(local.getHTML()).toBe('

Earlier heading

After image

') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-input-rule.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-input-rule.test.ts new file mode 100644 index 00000000000..2f72d9e1d7a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-input-rule.test.ts @@ -0,0 +1,76 @@ +/** @vitest-environment jsdom */ +import { Editor } from '@tiptap/core' +import { afterEach, describe, expect, it } from 'vitest' +import { Awareness } from 'y-protocols/awareness' +import * as Y from 'yjs' +import { markdownToYDoc } from '@/lib/collab-doc/converter' +import { createMarkdownEditorExtensions } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions' + +const cleanups: Array<() => void> = [] +afterEach(() => cleanups.splice(0).forEach((cleanup) => cleanup())) + +function createPeer(seed: Y.Doc) { + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(seed)) + const awareness = new Awareness(doc) + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#ffffff' } }, + }), + editorProps: { handleScrollToSelection: () => true }, + }) + cleanups.push(() => { + editor.destroy() + awareness.destroy() + doc.destroy() + }) + return { doc, editor } +} + +/** Exercises the same input-rule ordering as character-by-character browser typing. */ +function typeText(editor: Editor, text: string): void { + for (const character of text) { + const { from, to } = editor.state.selection + const handled = editor.view.someProp('handleTextInput', (handler) => + handler(editor.view, from, to, character, () => + editor.state.tr.insertText(character, from, to) + ) + ) + if (!handled) editor.view.dispatch(editor.state.tr.insertText(character, from, to)) + } +} + +describe('typed images with the collaborative editor extensions', () => { + it.each([ + { alt: 'Audit image', title: null }, + { alt: '', title: null }, + { alt: 'Logo', title: 'Brand' }, + ])('creates an image, not a bang plus a link ($alt, $title)', ({ alt, title }) => { + const seed = markdownToYDoc('') + const a = createPeer(seed) + const b = createPeer(seed) + seed.destroy() + const source = `![${alt}](https://example.com/logo.png${title ? ` "${title}"` : ''})` + typeText(a.editor, source) + + expect(a.editor.getJSON().content?.filter((node) => node.type === 'image')).toMatchObject([ + { type: 'image', attrs: { src: 'https://example.com/logo.png', alt, title } }, + ]) + expect(a.editor.getText()).not.toContain('!') + expect(a.editor.getMarkdown().trim()).toBe(source) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + }) + + it('continues to create ordinary links during typing', () => { + const seed = markdownToYDoc('') + const { editor } = createPeer(seed) + seed.destroy() + typeText(editor, '[Audit link](https://example.com)') + expect(editor.getJSON().content?.[0]).toMatchObject({ + type: 'paragraph', + content: [{ type: 'text', text: 'Audit link', marks: [{ type: 'link' }] }], + }) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx new file mode 100644 index 00000000000..f710358db14 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.test.tsx @@ -0,0 +1,181 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { Tooltip } from '@sim/emcn' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ImageInspector } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector' + +let host: HTMLDivElement +let root: Root + +beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) + +afterEach(() => { + act(() => root.unmount()) + host.remove() + vi.unstubAllGlobals() +}) + +function button(label: string): HTMLButtonElement { + const element = host.querySelector(`button[aria-label="${label}"]`) + if (!element) throw new Error(`Missing ${label} button`) + return element +} + +function change(input: HTMLInputElement, value: string): void { + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) +} + +describe('ImageInspector', () => { + it.each([ + { key: 'Enter', isComposing: true, keyCode: 13 }, + { key: 'Escape', isComposing: true, keyCode: 27 }, + { key: 'Enter', isComposing: false, keyCode: 229 }, + { key: 'Escape', isComposing: false, keyCode: 229 }, + ])('keeps the draft open for composition key $key/$keyCode', async (keyboard) => { + const onApply = vi.fn() + const onReturnFocus = vi.fn() + act(() => { + root.render( + + + + ) + }) + act(() => button('Edit image details').click()) + const input = host.querySelector('input[aria-label="Image alt text"]')! + change(input, 'Composition draft') + await act(async () => { + input.dispatchEvent(new KeyboardEvent('keydown', { ...keyboard, bubbles: true })) + }) + + expect(host.querySelector('input[aria-label="Image alt text"]')).toBe(input) + expect(input.value).toBe('Composition draft') + expect(document.activeElement).toBe(input) + expect(onApply).not.toHaveBeenCalled() + expect(onReturnFocus).not.toHaveBeenCalled() + }) + + it.each(['alt', 'href', 'neither', 'reverted'] as const)( + 'submits only the locally changed field: %s', + async (changedField) => { + const onApply = vi.fn() + const renderInspector = (alt: string, href: string) => { + act(() => { + root.render( + + + + ) + }) + } + renderInspector('Original alt', 'https://sim.ai/original') + act(() => button('Edit image details').click()) + const alt = host.querySelector('input[aria-label="Image alt text"]')! + const href = host.querySelector('input[aria-label="Image link URL"]')! + if (changedField === 'alt' || changedField === 'reverted') change(alt, 'Local alt') + if (changedField === 'href') change(href, '') + if (changedField === 'reverted') change(alt, 'Original alt') + + renderInspector('Peer alt', 'https://sim.ai/peer') + expect(alt.value).toBe(changedField === 'alt' ? 'Local alt' : 'Original alt') + expect(href.value).toBe(changedField === 'href' ? '' : 'https://sim.ai/original') + await act(async () => { + href.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + }) + + if (changedField === 'alt') expect(onApply).toHaveBeenCalledWith({ alt: 'Local alt' }) + else if (changedField === 'href') expect(onApply).toHaveBeenCalledWith({ href: '' }) + else expect(onApply).not.toHaveBeenCalled() + } + ) + + it('validates and applies accessible image details', async () => { + const onApply = vi.fn() + const onReturnFocus = vi.fn() + act(() => { + root.render( + + + + ) + }) + + act(() => button('Edit image details').click()) + expect(host.firstElementChild).toHaveClass('left-0') + expect(host.firstElementChild).not.toHaveClass('sm:left-1/2', 'sm:-translate-x-1/2') + const alt = host.querySelector('input[aria-label="Image alt text"]') + const href = host.querySelector('input[aria-label="Image link URL"]') + expect(alt?.value).toBe('Diagram') + expect(href?.value).toBe('https://sim.ai/original') + if (!alt || !href) return + + change(alt, 'Updated diagram') + href.focus() + change(href, 'javascript:alert(1)') + expect(document.activeElement).toBe(href) + expect(href).toHaveAttribute('aria-invalid', 'true') + expect(host.querySelector('[role="alert"]')?.textContent).toContain('valid link') + const apply = Array.from(host.querySelectorAll('button')).find((candidate) => + candidate.textContent?.includes('Apply') + ) + expect(apply?.disabled).toBe(true) + + change(href, 'https://sim.ai/updated') + act(() => apply?.click()) + expect(onApply).toHaveBeenCalledWith({ + alt: 'Updated diagram', + href: 'https://sim.ai/updated', + }) + await vi.waitFor(() => expect(onReturnFocus).toHaveBeenCalledTimes(1)) + }) + + it('offers size reset only for explicitly sized images', () => { + const onResetSize = vi.fn() + act(() => { + root.render( + + + + ) + }) + act(() => button('Reset image size').click()) + expect(onResetSize).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx new file mode 100644 index 00000000000..ede3541223d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector.tsx @@ -0,0 +1,136 @@ +import { type KeyboardEvent, useId, useState } from 'react' +import { Button, ChipInput } from '@sim/emcn' +import { Check, RefreshCw, Settings, X } from '@sim/emcn/icons' +import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' +import { ToolbarButton } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button' + +interface ImageDetails { + alt: string + href: string +} + +interface ImageDetailsDraft extends ImageDetails { + initial: ImageDetails +} + +interface ImageInspectorProps extends ImageDetails { + hasCustomSize: boolean + onApply: (details: Partial) => void + onResetSize: () => void + onReturnFocus: () => void +} + +export function ImageInspector({ + alt, + href, + hasCustomSize, + onApply, + onResetSize, + onReturnFocus, +}: ImageInspectorProps) { + const [draft, setDraft] = useState(null) + const errorId = useId() + const normalizedHref = draft ? normalizeLinkHref(draft.href.trim()) : '' + const invalidHref = Boolean(draft?.href.trim()) && !normalizedHref + + const close = () => { + setDraft(null) + queueMicrotask(onReturnFocus) + } + + const apply = () => { + if (!draft || invalidHref) return + const details: Partial = {} + if (draft.alt !== draft.initial.alt) details.alt = draft.alt + if (draft.href !== draft.initial.href) details.href = normalizedHref + if (Object.keys(details).length > 0) onApply(details) + close() + } + + const handleKeyDown = (event: KeyboardEvent) => { + event.stopPropagation() + if (event.nativeEvent.isComposing || event.nativeEvent.keyCode === 229) return + if (event.key === 'Enter') { + event.preventDefault() + apply() + } else if (event.key === 'Escape') { + event.preventDefault() + close() + } + } + + return ( +
event.stopPropagation()} + > + {draft ? ( +
+ + setDraft((current) => ({ ...(current ?? draft), alt: event.target.value })) + } + onKeyDown={handleKeyDown} + /> + + setDraft((current) => ({ ...(current ?? draft), href: event.target.value })) + } + onKeyDown={handleKeyDown} + /> + {invalidHref && ( + + )} +
+ + +
+
+ ) : ( +
+ setDraft({ alt, href, initial: { alt, href } })} + /> + {hasCustomSize && ( + { + onResetSize() + queueMicrotask(onReturnFocus) + }} + /> + )} +
+ )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx new file mode 100644 index 00000000000..5d55ffea7e5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-resize.test.tsx @@ -0,0 +1,239 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import type { ReactNodeViewProps } from '@tiptap/react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@tiptap/react', () => ({ + NodeViewWrapper: 'div', + ReactNodeViewRenderer: vi.fn(), +})) + +vi.mock('@tiptap/y-tiptap', () => ({ + ySyncPluginKey: { getState: vi.fn() }, +})) + +vi.mock('@/hooks/use-file-content-source', () => ({ + useFileContentSource: () => ({ + resolveImageSrc: (src: string) => src, + getImageDimensions: () => null, + }), +})) + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/use-editor-editable', + () => ({ useEditorEditable: () => true }) +) + +vi.mock( + '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector', + () => ({ ImageInspector: vi.fn(() => null) }) +) + +import { ResizableImageView } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image' +import { ImageInspector } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-inspector' + +let host: HTMLDivElement +let root: Root +const editor = { isEditable: true, isDestroyed: false, commands: { focus: vi.fn() } } + +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }) + vi.clearAllMocks() + editor.isEditable = true + editor.isDestroyed = false + host = document.createElement('div') + document.body.append(host) + root = createRoot(host) +}) + +afterEach(() => { + act(() => root.unmount()) + host.remove() +}) + +function pointerEvent( + type: string, + { pointerId, clientX = 0, button = 0 }: { pointerId: number; clientX?: number; button?: number } +): Event { + const event = new Event(type, { bubbles: true, cancelable: true }) + Object.defineProperties(event, { + pointerId: { value: pointerId }, + clientX: { value: clientX }, + button: { value: button }, + pointerType: { value: 'touch' }, + }) + return event +} + +function renderImage( + updateAttributes: ReturnType, + dimensions: { width?: string | null; height?: string | null } = {} +): HTMLButtonElement { + const props = { + node: { + attrs: { + src: '/image.png', + alt: '', + title: null, + width: null, + height: '100', + ...dimensions, + href: null, + }, + }, + updateAttributes, + selected: true, + editor, + } as unknown as ReactNodeViewProps + act(() => root.render()) + const image = host.querySelector('img') + const handle = host.querySelector('button[aria-label="Resize image"]') + if (!image || !handle) throw new Error('Resizable image did not render') + Object.defineProperty(image, 'offsetWidth', { configurable: true, value: 200 }) + Object.assign(handle, { + setPointerCapture: vi.fn(), + hasPointerCapture: vi.fn(() => true), + releasePointerCapture: vi.fn(), + }) + return handle +} + +describe('ResizableImageView', () => { + it.each(['read-only', 'destroyed'] as const)( + 'rejects queued image detail and size changes after the editor becomes %s', + (state) => { + const updateAttributes = vi.fn() + renderImage(updateAttributes) + const inspector = vi.mocked(ImageInspector).mock.calls.at(-1)![0] + if (state === 'read-only') editor.isEditable = false + else editor.isDestroyed = true + + act(() => { + inspector.onApply({ alt: 'changed', href: 'https://example.com' }) + inspector.onResetSize() + }) + expect(updateAttributes).not.toHaveBeenCalled() + } + ) + + it('applies image details and resets dimensions while the editor remains editable', () => { + const updateAttributes = vi.fn() + renderImage(updateAttributes) + const inspector = vi.mocked(ImageInspector).mock.calls.at(-1)![0] + act(() => { + inspector.onApply({ alt: 'changed', href: 'https://example.com' }) + inspector.onResetSize() + }) + expect(updateAttributes.mock.calls).toEqual([ + [{ alt: 'changed', href: 'https://example.com' }], + [{ width: null, height: null }], + ]) + }) + + it('preserves omitted image details and explicitly clears an empty link', () => { + const updateAttributes = vi.fn() + renderImage(updateAttributes) + const inspector = vi.mocked(ImageInspector).mock.calls.at(-1)![0] + act(() => { + inspector.onApply({ alt: 'changed' }) + inspector.onApply({ href: '' }) + }) + expect(updateAttributes.mock.calls).toEqual([[{ alt: 'changed' }], [{ href: null }]]) + }) + + it('renders a height-only image proportionally without fixing its responsive height', () => { + renderImage(vi.fn()) + const image = host.querySelector('img') + if (!image) throw new Error('Missing image') + Object.defineProperties(image, { + naturalWidth: { configurable: true, value: 400 }, + naturalHeight: { configurable: true, value: 200 }, + }) + act(() => image.dispatchEvent(new Event('load'))) + + expect(image.style.height).toBe('') + expect(image.style.width).toBe('calc(200px)') + expect(image.style.aspectRatio).toBe('400 / 200') + }) + + it.each([ + { width: '600', height: '400' }, + { width: '600px', height: '400px' }, + { width: '600', height: '400px' }, + ])('uses the authored ratio for responsive pixel dimensions: %j', (dimensions) => { + renderImage(vi.fn(), dimensions) + const image = host.querySelector('img')! + expect(image.style.width).toBe('600px') + expect(image.style.height).toBe('') + expect(image.style.aspectRatio).toBe('600 / 400') + }) + + it('preserves relative dimensions instead of assuming they are pixel ratios', () => { + renderImage(vi.fn(), { width: '50%', height: '100px' }) + const image = host.querySelector('img')! + expect(image.style.width).toBe('50%') + expect(image.style.height).toBe('100px') + }) + + it.each(['50%', 'auto', '10em', 'calc(50% - 10px)', 'min-content', 'inherit'])( + 'preserves the native height-only CSS value %s before and after loading', + (height) => { + renderImage(vi.fn(), { height }) + const image = host.querySelector('img')! + expect(image.style.width).toBe('') + expect(image.style.height).toBe(height) + expect(image.style.maxHeight).toBe('') + + Object.defineProperties(image, { + naturalWidth: { configurable: true, value: 400 }, + naturalHeight: { configurable: true, value: 200 }, + }) + act(() => image.dispatchEvent(new Event('load'))) + + expect(image.style.width).toBe('') + expect(image.style.height).toBe(height) + expect(image.style.maxHeight).toBe('') + } + ) + + it('commits one proportional width change and clears a stale explicit height', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 7, clientX: 160 }))) + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 7, clientX: 160 }))) + + expect(updateAttributes).toHaveBeenCalledOnce() + expect(updateAttributes).toHaveBeenCalledWith({ width: '260', height: null }) + }) + + it('ignores unrelated pointers and cancels without mutating document attributes', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 8, clientX: 180 }))) + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 8, clientX: 180 }))) + act(() => window.dispatchEvent(pointerEvent('pointercancel', { pointerId: 7 }))) + expect(updateAttributes).not.toHaveBeenCalled() + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 9, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 9, clientX: 140 }))) + act(() => window.dispatchEvent(new Event('blur'))) + expect(updateAttributes).not.toHaveBeenCalled() + }) + + it('does not commit a resize after live editing becomes unavailable', () => { + const updateAttributes = vi.fn() + const handle = renderImage(updateAttributes) + + act(() => handle.dispatchEvent(pointerEvent('pointerdown', { pointerId: 7, clientX: 100 }))) + act(() => window.dispatchEvent(pointerEvent('pointermove', { pointerId: 7, clientX: 160 }))) + editor.isEditable = false + act(() => window.dispatchEvent(pointerEvent('pointerup', { pointerId: 7, clientX: 160 }))) + + expect(updateAttributes).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts index dccc926d6b4..dabebe5bc4d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/image-schema.ts @@ -1,5 +1,6 @@ import type { JSONContent } from '@tiptap/core' import { Image } from '@tiptap/extension-image' +import { Lexer, Tokenizer } from 'marked' /** * React-free schema half of the image node. Lives apart from {@link ./image} (its React resize node @@ -16,9 +17,6 @@ import { Image } from '@tiptap/extension-image' * the whole construct ourselves and hang the link target on the image node's `href` attribute, so it * round-trips losslessly (and the file stays editable rather than opening read-only). */ -const LINKED_IMAGE_RE = - /^\[!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/ - /** Escape a value for safe interpolation into a double-quoted HTML attribute. */ function escapeAttr(value: string): string { return value @@ -28,16 +26,27 @@ function escapeAttr(value: string): string { .replace(/>/g, '>') } +function decodeAttr(value: string): string { + return value + .replace(/"/g, '"') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&') +} + +function imageAttrsFromHtml(raw: string): Record | null { + if (!/^ = {} + const attributePattern = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g + for (const match of raw.matchAll(attributePattern)) { + attrs[match[1].toLowerCase()] = decodeAttr(match[2] ?? match[3] ?? match[4] ?? '') + } + return typeof attrs.src === 'string' ? attrs : null +} + /** - * Serialize an image to markdown when it has no explicit size, and to an HTML `` tag when - * it does — standard markdown has no width syntax, so a resized image must round-trip as HTML to - * preserve its dimensions. Unsized images stay clean `![alt](src)`. An image with an `href` is - * wrapped in a markdown link so a linked badge round-trips as `[![alt](src)](href)`. - * - * A *sized **and** linked* image is the one case markdown can't represent: the linked-image tokenizer - * only recognizes `[![alt](src)](href)`, so emitting `[](href)` would silently drop the link on - * reparse (and the round-trip-safety probe wouldn't catch it). We keep the link and fall back to the - * unsized `[![alt](src)](href)` form — the link matters more than the exact dimensions for a badge. + * Markdown has no image dimensions, so sized images use HTML. Links wrap either representation: + * `[![alt](src)](href)` or `[](href)`. */ function imageMarkdown(node: JSONContent): string { const attrs = node.attrs ?? {} @@ -49,9 +58,8 @@ function imageMarkdown(node: JSONContent): string { const width = attrs.width const height = attrs.height let image: string - if ((width || height) && !href) { - const parts = [`src="${escapeAttr(src)}"`] - if (alt) parts.push(`alt="${escapeAttr(alt)}"`) + if (width || height) { + const parts = [`src="${escapeAttr(src)}"`, `alt="${escapeAttr(alt)}"`] if (title) parts.push(`title="${escapeAttr(title)}"`) if (width) parts.push(`width="${escapeAttr(String(width))}"`) if (height) parts.push(`height="${escapeAttr(String(height))}"`) @@ -67,7 +75,8 @@ function imageMarkdown(node: JSONContent): string { // Escape `"`/`\` so an href title can't break out of the `[…](href "title")` syntax (mirrors the // image title escaping above). const hrefTitlePart = hrefTitle ? ` "${hrefTitle.replace(/["\\]/g, '\\$&')}"` : '' - return `[${image}](${href}${hrefTitlePart})` + const safeHref = /[\s()]/.test(href) ? `<${href}>` : href + return `[${image}](${safeHref}${hrefTitlePart})` } interface MarkdownImageToken { @@ -78,6 +87,8 @@ interface MarkdownImageToken { /** Built-in image token holds the source URL here; our linked token holds the link target. */ href?: string hrefTitle?: string | null + width?: string | null + height?: string | null /** Built-in image token holds the alt text here. */ text?: string } @@ -94,6 +105,8 @@ function parseImageToken(token: MarkdownImageToken): JSONContent { title: token.title ?? null, href: token.href ?? null, hrefTitle: token.hrefTitle ?? null, + width: token.width ?? null, + height: token.height ?? null, } : { src: token.href ?? '', @@ -101,6 +114,8 @@ function parseImageToken(token: MarkdownImageToken): JSONContent { title: token.title ?? null, href: null, hrefTitle: null, + width: null, + height: null, }, } } @@ -141,18 +156,44 @@ export const MarkdownImage = Image.extend({ markdownTokenizer: { name: 'image', level: 'inline', - start: (src: string) => src.indexOf('[!['), + start: (src: string) => { + const markdown = src.indexOf('[![') + const html = src.search(/\[ { - const match = LINKED_IMAGE_RE.exec(src) - if (!match) return undefined + if (!src.startsWith('[![') && !/^\[`. */ -function ResizableImageView({ node, updateAttributes, selected, editor }: ReactNodeViewProps) { +export function ResizableImageView({ + node, + updateAttributes, + selected, + editor, + getPos, +}: ReactNodeViewProps) { const source = useFileContentSource() const imageRef = useRef(null) const dragAbortRef = useRef(null) @@ -37,6 +47,7 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN alt?: string title?: string width?: string | null + height?: string | null href?: string | null } @@ -54,8 +65,33 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN const startResize = (event: React.PointerEvent) => { event.preventDefault() + if (event.button !== 0 || dragging) return const image = imageRef.current if (!image) return + const binding: ProsemirrorBinding | undefined = ySyncPluginKey.getState(editor.state)?.binding + let yTarget: XmlElement | undefined + if (binding) { + for (const [type, mappedNode] of binding.mapping) { + if (mappedNode === node && type instanceof XmlElement) { + yTarget = type + break + } + } + } + /** A node view can be reused for a replacement image, even when every attribute is identical. */ + const isCurrentTarget = () => { + if (!binding) return true + const position = getPos() + return ( + yTarget !== undefined && + typeof position === 'number' && + binding.mapping.get(yTarget) === editor.state.doc.nodeAt(position) + ) + } + if (!isCurrentTarget()) return + const handle = event.currentTarget + const pointerId = event.pointerId + handle.setPointerCapture(pointerId) const startX = event.clientX const startWidth = image.offsetWidth setDragging(true) @@ -67,29 +103,66 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN window.addEventListener( 'pointermove', (move) => { + if (move.pointerId !== pointerId) return const next = Math.max(MIN_WIDTH, Math.round(startWidth + (move.clientX - startX))) dragWidthRef.current = next setDragWidth(next) }, { signal } ) - const finish = () => { + const finish = (commit: boolean) => { const finalWidth = dragWidthRef.current setDragging(false) setDragWidth(null) dragWidthRef.current = null + if (handle.hasPointerCapture(pointerId)) handle.releasePointerCapture(pointerId) controller.abort() - if (finalWidth !== null) updateAttributes({ width: String(finalWidth) }) + if ( + commit && + finalWidth !== null && + editor.isEditable && + !editor.isDestroyed && + isCurrentTarget() + ) { + updateAttributes({ width: String(finalWidth), height: null }) + } + } + if (binding) { + const onTransaction = () => { + if (!isCurrentTarget()) finish(false) + } + editor.on('transaction', onTransaction) + signal.addEventListener('abort', () => editor.off('transaction', onTransaction), { + once: true, + }) } - window.addEventListener('pointerup', finish, { signal }) - window.addEventListener('pointercancel', finish, { signal }) + window.addEventListener( + 'pointerup', + (up) => { + if (up.pointerId === pointerId) finish(true) + }, + { signal } + ) + window.addEventListener( + 'pointercancel', + (cancel) => { + if (cancel.pointerId === pointerId) finish(false) + }, + { signal } + ) + window.addEventListener('blur', () => finish(false), { signal }) } const committedWidth = attrs.width - ? BARE_PIXEL_WIDTH.test(attrs.width) + ? BARE_PIXEL_SIZE.test(attrs.width) ? `${attrs.width}px` : attrs.width : undefined + const committedHeight = attrs.height + ? BARE_PIXEL_SIZE.test(attrs.height) + ? `${attrs.height}px` + : attrs.height + : undefined // Stored intrinsic dimensions reserve the box on the very first render. Memoized on the src (not the // live drag width) so a resize drag never re-scans the file list. Falls back to what we measured on // load this session for a first-ever view the metadata hasn't caught up on. @@ -101,17 +174,40 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN // stored value is stale (e.g. left over after the file's content was replaced) — so it wins once // available; stored metadata only reserves the box pre-load. Equal in the common case, so no shift. const intrinsicDimensions = measuredDimensions ?? storedDimensions + const hasPixelHeight = committedHeight !== undefined && PIXEL_SIZE.test(committedHeight) + const authoredDimensions = + committedWidth && + committedHeight && + PIXEL_SIZE.test(committedWidth) && + PIXEL_SIZE.test(committedHeight) && + Number.parseFloat(committedWidth) > 0 && + Number.parseFloat(committedHeight) > 0 + ? { width: Number.parseFloat(committedWidth), height: Number.parseFloat(committedHeight) } + : null + const displayDimensions = + dragWidth === null ? (authoredDimensions ?? intrinsicDimensions) : intrinsicDimensions const displayWidth = dragWidth !== null ? `${dragWidth}px` - : (committedWidth ?? (intrinsicDimensions ? `${intrinsicDimensions.width}px` : undefined)) + : (committedWidth ?? + (intrinsicDimensions && (!committedHeight || hasPixelHeight) + ? committedHeight + ? `calc(${committedHeight} * ${intrinsicDimensions.width / intrinsicDimensions.height})` + : `${intrinsicDimensions.width}px` + : undefined)) // width + aspect-ratio (with `max-w-full`/`h-auto` from the class list) reserves a responsive box the // image can't reflow into, per the CLS-avoidance pattern for known-ratio responsive images. React drops // the undefined keys, so an unmeasured image simply gets no reservation (its prior behavior). const imageStyle: CSSProperties = { width: displayWidth, - aspectRatio: intrinsicDimensions - ? `${intrinsicDimensions.width} / ${intrinsicDimensions.height}` + height: + dragWidth === null && !authoredDimensions && (committedWidth || !hasPixelHeight) + ? committedHeight + : undefined, + maxHeight: + dragWidth === null && !committedWidth && hasPixelHeight ? committedHeight : undefined, + aspectRatio: displayDimensions + ? `${displayDimensions.width} / ${displayDimensions.height}` : undefined, } @@ -182,11 +278,31 @@ function ResizableImageView({ node, updateAttributes, selected, editor }: ReactN image )} {editable && (selected || dragging) && ( - + )} + {editable && selected && !dragging && ( + { + if (!editor.isEditable || editor.isDestroyed) return + updateAttributes({ ...details, ...(details.href === '' ? { href: null } : {}) }) + }} + onResetSize={() => { + if (!editor.isEditable || editor.isDestroyed) return + updateAttributes({ width: null, height: null }) + }} + onReturnFocus={() => editor.commands.focus()} /> )} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts index 6f59a439523..94e907494dd 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/link-input-rule.ts @@ -1,12 +1,13 @@ import { Extension, InputRule } from '@tiptap/core' -import { normalizeLinkHref } from './markdown-fidelity' +import { normalizeLinkHref } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' /** * Typed markdown link: `[text](url)` or `[text](url "title")`, completed by the closing `)`. The URL * is space-free (markdown requires `` for spaces, which this intentionally skips). StarterKit's * Link ships no input rule — only paste/autolink — so without this, typed link syntax stays literal. + * A preceding bang belongs to the image input rule, which must receive the complete syntax. */ -const LINK_INPUT_RULE = /\[([^\]]+)]\(([^)\s]+)(?:\s+"([^"]*)")?\)$/ +const LINK_INPUT_RULE = /(? { + vi.useFakeTimers() + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + document.elementFromPoint ??= () => null + const seed = markdownToYDoc('## Before\n\nbefore [format](https://example.com/original) after') + const makePeer = (): Peer => { + const doc = new Y.Doc() + Y.applyUpdate(doc, Y.encodeStateAsUpdate(seed)) + const awareness = new Awareness(doc) + const editor = new Editor({ + extensions: createMarkdownEditorExtensions({ + placeholder: '', + collaboration: { doc, awareness, user: { name: 'User', color: '#ffffff' } }, + }), + editorProps: { handleScrollToSelection: () => true }, + }) + return { doc, awareness, editor } + } + a = makePeer() + b = makePeer() + seed.destroy() + viewport = document.createElement('div') + const host = document.createElement('div') + viewport.append(a.editor.view.dom, host) + document.body.append(viewport) + vi.spyOn(a.editor.view, 'coordsAtPos').mockReturnValue({ + top: 10, + bottom: 30, + left: 10, + right: 50, + }) + root = createRoot(host) + act(() => { + root.render( + + + + ) + }) +}) + +afterEach(() => { + act(() => root.unmount()) + vi.clearAllTimers() + for (const peer of [a, b]) { + peer.editor.destroy() + peer.awareness.destroy() + peer.doc.destroy() + } + viewport.remove() + vi.restoreAllMocks() + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +function textPosition(editor: Editor, text: string): number { + let position = -1 + editor.state.doc.descendants((node, pos) => { + if (node.isText && node.text === text) position = pos + }) + expect(position).toBeGreaterThan(-1) + return position +} + +async function openDraft(caret: boolean): Promise { + const from = textPosition(a.editor, 'format') + act(() => { + a.editor.commands.setTextSelection(caret ? from + 2 : { from, to: from + 6 }) + a.editor.view.focus() + a.editor.view.dom.dispatchEvent( + new KeyboardEvent('keydown', { key: 'k', ctrlKey: true, bubbles: true, cancelable: true }) + ) + }) + await act(async () => vi.advanceTimersToNextFrame()) + const input = viewport.querySelector('input[aria-label="Link URL"]') + expect(input).not.toBeNull() + if (!input) throw new Error('Missing link draft') + act(() => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call( + input, + 'https://example.com/draft' + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + expect(document.activeElement).toBe(input) + return input +} + +async function receivePeerEdit(): Promise { + await act(async () => Y.applyUpdate(a.doc, Y.encodeStateAsUpdate(b.doc))) +} + +describe('link drafts during actual collaborative updates', () => { + it.each([false, true])( + 'preserves and applies a draft after a peer prefix edit (caret=%s)', + async (caret) => { + const input = await openDraft(caret) + b.editor.commands.insertContentAt(textPosition(b.editor, 'Before') + 6, ' PEER') + await receivePeerEdit() + + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + expect(document.activeElement).toBe(input) + expect(input.value).toBe('https://example.com/draft') + const apply = viewport.querySelector('button[aria-label="Apply link"]') + await act(async () => apply?.click()) + expect(a.editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/draft' + ) + expect(a.editor.getText()).toContain('Before PEER') + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + expect(b.editor.getJSON()).toEqual(a.editor.getJSON()) + } + ) + + it('maps the original caret through local and peer edits before canceling', async () => { + const input = await openDraft(true) + act(() => a.editor.view.dispatch(a.editor.state.tr.insertText('LOCAL ', 1))) + Y.applyUpdate(b.doc, Y.encodeStateAsUpdate(a.doc)) + b.editor.commands.insertContentAt(1, 'PEER ') + await receivePeerEdit() + act(() => input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }))) + expect(a.editor.state.selection.empty).toBe(true) + expect(a.editor.state.selection.from).toBe(textPosition(a.editor, 'format') + 2) + act(() => a.editor.commands.insertContent('X')) + expect(a.editor.getText()).toContain('foXrmat') + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) + }) + + it('retains a draft across read-only permission intervals and peer updates', async () => { + const input = await openDraft(false) + const apply = viewport.querySelector('button[aria-label="Apply link"]') + act(() => a.editor.setEditable(false)) + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + b.editor.commands.insertContentAt(1, 'PEER ') + await receivePeerEdit() + act(() => apply?.click()) + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) + act(() => a.editor.setEditable(true)) + act(() => a.editor.view.focus()) + await act(async () => vi.advanceTimersToNextFrame()) + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + await act(async () => apply?.click()) + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/draft' + ) + }) + + it('discards the draft when a peer deletes its entire target', async () => { + const input = await openDraft(false) + const from = textPosition(b.editor, 'format') + b.editor.commands.deleteRange({ from, to: from + 6 }) + await receivePeerEdit() + expect(viewport.contains(input)).toBe(false) + expect(a.editor.view.dom.querySelector('a')).toBeNull() + }) + + it('maps a peer transaction together with a locally appended transaction exactly once', async () => { + const input = await openDraft(false) + let appended = false + a.editor.registerPlugin( + new Plugin({ + appendTransaction: (transactions, _oldState, state) => { + if (appended || !transactions.some((transaction) => transaction.docChanged)) return null + appended = true + return state.tr.insertText('APPENDED ', 1) + }, + }) + ) + b.editor.commands.insertContentAt(1, 'PEER ') + await receivePeerEdit() + expect(viewport.querySelector('input[aria-label="Link URL"]')).toBe(input) + await act(async () => { + viewport.querySelector('button[aria-label="Apply link"]')?.click() + }) + expect(a.editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(a.editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/draft' + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx index 1ba7457cc4b..98321f451e4 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/bubble-menu.tsx @@ -16,8 +16,11 @@ import { TextQuote, Unlink, } from '@sim/emcn/icons' +import type { MappablePosition } from '@tiptap/core' +import type { Node } from '@tiptap/pm/model' import { PluginKey, + type Selection, type SelectionBookmark, TextSelection, type Transaction, @@ -58,6 +61,56 @@ function revealBubbleMenu(editor: Editor, key: PluginKey): void { editor.commands.setMeta(key, 'updatePosition') } +type CapturedSelection = + | { anchor: MappablePosition; head: MappablePosition; bookmark?: never } + | { bookmark: SelectionBookmark; anchor?: never; head?: never } + +interface LinkSelection { + target: CapturedSelection + original: CapturedSelection +} + +/** Collaborative positions survive the full-document replacements used to apply Yjs updates. */ +function captureSelection(editor: Editor): CapturedSelection { + const { selection } = editor.state + return selection instanceof TextSelection + ? { + anchor: editor.utils.createMappablePosition(selection.anchor), + head: editor.utils.createMappablePosition(selection.head), + } + : { bookmark: selection.getBookmark() } +} + +function mapSelection( + editor: Editor, + selection: CapturedSelection, + transaction: Transaction +): CapturedSelection { + return selection.bookmark + ? { bookmark: selection.bookmark.map(transaction.mapping) } + : { + anchor: editor.utils.getUpdatedPosition(selection.anchor, transaction).position, + head: editor.utils.getUpdatedPosition(selection.head, transaction).position, + } +} + +function resolveSelection(selection: CapturedSelection, doc: Node): Selection { + return selection.bookmark + ? selection.bookmark.resolve(doc) + : TextSelection.between( + doc.resolve(selection.anchor.position), + doc.resolve(selection.head.position) + ) +} + +/** Keep the editing target separate from the selection restored when the user cancels. */ +function captureLinkSelection(editor: Editor): LinkSelection | null { + const original = captureSelection(editor) + if (editor.state.selection.empty) editor.commands.extendMarkRange('link') + const { selection } = editor.state + return selection.empty ? null : { target: captureSelection(editor), original } +} + interface EditorBubbleMenuProps { editor: Editor /** The editor's scrollable viewport, so the toolbar repositions with the selection as the pane scrolls. */ @@ -79,7 +132,7 @@ export function EditorBubbleMenu({ }: EditorBubbleMenuProps) { const [linkValue, setLinkValue] = useState(null) const linkInputRef = useRef(null) - const linkRangeRef = useRef(null) + const linkSelectionRef = useRef(null) const isEditingLink = linkValue !== null const [bubbleMenuKey] = useState(() => new PluginKey('markdownBubbleMenu')) @@ -122,18 +175,25 @@ export function EditorBubbleMenu({ transaction: Transaction appendedTransactions?: Transaction[] }) => { - let bookmark = linkRangeRef.current - if (!bookmark) return - for (const change of [transaction, ...appendedTransactions]) - bookmark = bookmark.map(change.mapping) - const selection = bookmark.resolve(editor.state.doc) - linkRangeRef.current = - selection instanceof TextSelection && !selection.empty ? bookmark : null - if (!linkRangeRef.current) setLinkValue(null) + let captured = linkSelectionRef.current + if (!captured) return + for (const change of [transaction, ...appendedTransactions]) { + captured = { + target: mapSelection(editor, captured.target, change), + original: mapSelection(editor, captured.original, change), + } + } + const selection = resolveSelection(captured.target, editor.state.doc) + linkSelectionRef.current = + selection instanceof TextSelection && !selection.empty ? captured : null + if (!linkSelectionRef.current) setLinkValue(null) } const exitOnCollapse = () => { const { from, to } = editor.state.selection - if (from === to) setLinkValue(null) + if (from === to) { + linkSelectionRef.current = null + setLinkValue(null) + } } editor.on('selectionUpdate', exitOnCollapse) editor.on('transaction', mapLinkRange) @@ -144,10 +204,8 @@ export function EditorBubbleMenu({ }, [editor]) /** - * Linear-style reveal: the toolbar stays hidden while the pointer is down (the drag gate in - * `shouldShow`) and surfaces on release. `mouseup`/`blur` listen on `window` so a release outside - * the editor — or off-screen, where no `mouseup` fires — still clears the drag flag; otherwise it - * could wedge `true` and suppress the toolbar for later keyboard selections. + * Window-level release/cancel/blur handlers clear the drag gate even outside the editor, + * preventing a lost pointer release from suppressing later keyboard selections. */ useEffect(() => { const dom = editor.view.dom @@ -163,19 +221,23 @@ export function EditorBubbleMenu({ const onWindowBlur = () => { isPointerDownRef.current = false } - dom.addEventListener('mousedown', onPointerDown) - window.addEventListener('mouseup', onPointerUp) + dom.addEventListener('pointerdown', onPointerDown) + window.addEventListener('pointerup', onPointerUp) + window.addEventListener('pointercancel', onWindowBlur) window.addEventListener('blur', onWindowBlur) return () => { - dom.removeEventListener('mousedown', onPointerDown) - window.removeEventListener('mouseup', onPointerUp) + dom.removeEventListener('pointerdown', onPointerDown) + window.removeEventListener('pointerup', onPointerUp) + window.removeEventListener('pointercancel', onWindowBlur) window.removeEventListener('blur', onWindowBlur) } }, [editor, bubbleMenuKey]) const openLinkEditor = () => { if (!editor.isEditable || editor.isActive('codeBlock') || editor.isActive('code')) return - linkRangeRef.current = editor.state.selection.getBookmark() + const captured = captureLinkSelection(editor) + if (!captured) return + linkSelectionRef.current = captured setLinkValue(editor.getAttributes('link').href ?? '') } @@ -192,10 +254,11 @@ export function EditorBubbleMenu({ ) return if (event.key?.toLowerCase() !== 'k') return - const { from, to } = editor.state.selection - if (from === to || editor.isActive('codeBlock') || editor.isActive('code')) return + if (editor.isActive('codeBlock') || editor.isActive('code')) return + const captured = captureLinkSelection(editor) + if (!captured) return event.preventDefault() - linkRangeRef.current = editor.state.selection.getBookmark() + linkSelectionRef.current = captured setLinkValue(editor.getAttributes('link').href ?? '') } dom.addEventListener('keydown', openLinkOnShortcut) @@ -206,19 +269,31 @@ export function EditorBubbleMenu({ const commitCapturedLink = (href: string) => { if (editor.isDestroyed || !editor.isEditable) return - const selection = linkRangeRef.current?.resolve(editor.state.doc) + const captured = linkSelectionRef.current + const selection = captured && resolveSelection(captured.target, editor.state.doc) if (selection instanceof TextSelection && !selection.empty) { applyLink( editor.chain().focus().setTextSelection({ from: selection.from, to: selection.to }), href ) } - linkRangeRef.current = null + linkSelectionRef.current = null setLinkValue(null) } const commitLink = () => commitCapturedLink(linkValue ?? '') const removeLink = () => commitCapturedLink('') + const cancelLink = () => { + const captured = linkSelectionRef.current + linkSelectionRef.current = null + setLinkValue(null) + if (!captured || editor.isDestroyed) return + editor.view.dispatch( + editor.state.tr.setSelection(resolveSelection(captured.original, editor.state.doc)) + ) + editor.commands.focus() + } + const { resolveAnchor, appendTo } = useBubbleMenuFloating(editor, scrollContainerRef) const canFocus = useCallback( () => hasFormattableSelection(editor, editor.state.selection.from, editor.state.selection.to), @@ -229,6 +304,7 @@ export function EditorBubbleMenu({ pluginKey: bubbleMenuKey, roving: !isEditingLink, canFocus, + onEscape: isEditingLink ? cancelLink : undefined, }) const shouldShow = useCallback( @@ -267,7 +343,7 @@ export function EditorBubbleMenu({ value={linkValue ?? ''} onChange={setLinkValue} onCommit={commitLink} - onCancel={() => setLinkValue(null)} + onCancel={cancelLink} /> {active.link && ( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx index e57463a6708..817cfbf314a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/editor-toolbar-integration.test.tsx @@ -109,6 +109,16 @@ async function openLinkEditor(): Promise { return input } +async function openLinkAtCaret(offset: number): Promise { + select('format', true) + act(() => editor.commands.setTextSelection(editor.state.selection.from + offset)) + expect(key(editor.view.dom, 'k', { ctrlKey: true }).defaultPrevented).toBe(true) + await frame() + const input = linkGroup().querySelector('input[aria-label="Link URL"]') + if (!input) throw new Error('Missing link URL field') + return input +} + function changeUrl(input: HTMLInputElement, value: string): void { act(() => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) @@ -269,6 +279,183 @@ describe('real editor BubbleMenu keyboard integration', () => { expect(key(remove, 'Tab').defaultPrevented).toBe(false) }) + it('edits the complete existing link from a collapsed caret with Cmd/Ctrl+K', async () => { + select('format') + act(() => editor.commands.setLink({ href: 'https://example.com/original' })) + select('format', true) + + expect(key(editor.view.dom, 'k', { ctrlKey: true }).defaultPrevented).toBe(true) + await frame() + const input = linkGroup().querySelector('input[aria-label="Link URL"]') + expect(input).not.toBeNull() + if (!input) return + + changeUrl(input, 'https://example.com/replacement') + key(input, 'Enter') + await frame() + + const link = editor.view.dom.querySelector('a') + expect(link?.textContent).toBe('format') + expect(link?.getAttribute('href')).toBe('https://example.com/replacement') + }) + + it.each([0, 2, 6])( + 'prefills the complete link at caret offset %i without changing it on apply', + async (offset) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const before = editor.getJSON() + const input = await openLinkAtCaret(offset) + + expect(input.value).toBe('https://example.com/original') + expect(button(linkGroup(), 'Remove link').disabled).toBe(false) + key(input, 'Enter') + await frame() + expect(editor.getJSON()).toEqual(before) + } + ) + + it('uses the same adjacent link for the captured range, URL, and update', async () => { + act(() => + editor.commands.setContent( + editorNormalForm( + '[before](https://example.com/first)[format](https://example.com/second) after' + ) + ) + ) + const input = await openLinkAtCaret(0) + expect(input.value).toBe('https://example.com/second') + changeUrl(input, 'https://example.com/replacement') + key(input, 'Enter') + await frame() + + const links = editor.view.dom.querySelectorAll('a') + expect([...links].map((link) => [link.textContent, link.getAttribute('href')])).toEqual([ + ['before', 'https://example.com/first'], + ['format', 'https://example.com/replacement'], + ]) + }) + + it.each([false, true])( + 'keeps a caret-opened link draft through a peer edit with read-only interval %s', + async (readOnly) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + const input = await openLinkAtCaret(2) + const group = linkGroup() + const apply = button(group, 'Apply link') + changeUrl(input, 'https://example.com/replacement') + if (readOnly) { + const before = editor.getJSON() + act(() => editor.setEditable(false)) + act(() => apply.click()) + expect(editor.getJSON()).toEqual(before) + } + act(() => editor.view.dispatch(editor.state.tr.insertText('remote ', 1))) + + if (readOnly) act(() => editor.setEditable(true)) + await frame() + expect((readOnly ? group : linkGroup()).querySelector('input')).toBe(input) + expect(input.value).toBe('https://example.com/replacement') + act(() => apply.click()) + await frame() + expect(editor.getText()).toBe('remote before format after') + expect(editor.view.dom.querySelector('a')?.textContent).toBe('format') + expect(editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/replacement' + ) + } + ) + + it.each([0, 2, 6])('restores the original caret at link offset %i on cancel', async (offset) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + offset + const input = await openLinkAtCaret(offset) + const before = editor.getJSON() + changeUrl(input, 'https://example.com/cancelled') + key(input, 'Escape') + await frame() + + expect(viewport.contains(input)).toBe(false) + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret) + expect(editor.getJSON()).toEqual(before) + act(() => editor.commands.insertContent('X')) + expect(editor.getText()).toBe( + `before ${'format'.slice(0, offset)}X${'format'.slice(offset)} after` + ) + }) + + it('maps the original caret through peer and appended edits before canceling', async () => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + 2 + const input = await openLinkAtCaret(2) + editor.registerPlugin( + new Plugin({ + appendTransaction: (transactions, _oldState, newState) => + transactions.some((transaction) => transaction.getMeta('toolbar-prefix')) + ? newState.tr.insertText('appended ', 1) + : null, + }) + ) + act(() => editor.setEditable(false)) + act(() => + editor.view.dispatch(editor.state.tr.insertText('peer ', 1).setMeta('toolbar-prefix', true)) + ) + act(() => editor.setEditable(true)) + const before = editor.getJSON() + key(input, 'Escape') + await frame() + + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret + 'appended peer '.length) + expect(editor.getJSON()).toEqual(before) + act(() => editor.commands.insertContent('X')) + expect(editor.getText()).toBe('appended peer before foXrmat after') + }) + + it.each(['Apply link', 'Remove link'])('restores the caret on Escape from %s', async (label) => { + act(() => + editor.commands.setContent( + editorNormalForm('before [format](https://example.com/original) after') + ) + ) + select('format', true) + const caret = editor.state.selection.from + 2 + const input = await openLinkAtCaret(2) + changeUrl(input, 'https://example.com/cancelled') + const action = button(linkGroup(), label) + act(() => action.focus()) + key(action, 'Escape') + await frame() + + expect(viewport.contains(input)).toBe(false) + expect(document.activeElement).toBe(editor.view.dom) + expect(editor.state.selection.empty).toBe(true) + expect(editor.state.selection.from).toBe(caret) + expect(editor.view.dom.querySelector('a')?.getAttribute('href')).toBe( + 'https://example.com/original' + ) + }) + it('maps the captured link target through a prefix edit and an appended transaction', async () => { const input = await openLinkEditor() changeUrl(input, 'https://example.com/mapped') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx index 3ebcc2c6312..441ecee8a2b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/link-editing.tsx @@ -63,7 +63,7 @@ export function LinkUrlInput({ } }} placeholder='Paste or type a link…' - className='h-[28px] w-[220px] bg-transparent px-2 text-[var(--text-body)] text-small outline-hidden placeholder:text-[var(--text-subtle)]' + className='h-10 w-[220px] bg-transparent px-2 text-[var(--text-body)] text-small outline-hidden placeholder:text-[var(--text-subtle)] sm:h-[28px]' /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx index e6bf381b8e8..340a43c1e50 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.test.tsx @@ -44,4 +44,18 @@ describe('ToolbarButton', () => { expect(button?.className).toContain('size-[28px]') expect(button?.querySelector('svg')?.className.baseVal).toContain('size-[12px]') }) + + it('preserves the editor selection for mouse, pen, and touch activation', () => { + const host = renderButton() + const button = host.querySelector('button[aria-label="Bold"]') + expect(button).not.toBeNull() + if (!button) return + + for (const pointerType of ['mouse', 'pen', 'touch']) { + const event = new Event('pointerdown', { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'pointerType', { value: pointerType }) + act(() => button.dispatchEvent(event)) + expect(event.defaultPrevented).toBe(true) + } + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx index 3436913633f..4bdcb5b19d8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/toolbar-button.tsx @@ -33,10 +33,10 @@ export function ToolbarButton({ aria-label={label} aria-pressed={isActive} disabled={disabled} - onMouseDown={(event) => event.preventDefault()} + onPointerDown={(event) => event.preventDefault()} onClick={onClick} className={cn( - 'size-[28px] focus-visible:bg-[var(--surface-hover)]', + 'size-10 focus-visible:bg-[var(--surface-hover)] sm:size-[28px]', !isActive && 'hover-hover:bg-[var(--surface-hover)]' )} > @@ -52,5 +52,5 @@ export function ToolbarButton({ /** Thin vertical separator between groups of {@link ToolbarButton}s. */ export function ToolbarDivider() { - return
+ return
} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts index da99cbb72b3..f86e2220fde 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/use-editor-toolbar.ts @@ -15,6 +15,7 @@ interface EditorToolbarOptions { canFocus: () => boolean /** URL editing uses ordinary form tab order so its native arrow keys do not trap action buttons. */ roving?: boolean + onEscape?: () => void } function controls(toolbar: HTMLElement): HTMLElement[] { @@ -33,6 +34,7 @@ export function useEditorToolbar({ pluginKey, canFocus, roving = true, + onEscape, }: EditorToolbarOptions) { const ref = useRef(null) @@ -113,6 +115,7 @@ export function useEditorToolbar({ ) return if (event.key === 'Escape') { + if (!event.defaultPrevented) onEscape?.() event.preventDefault() editor.commands.focus() editor.commands.setMeta(pluginKey, 'hide') diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 55384a26ff1..e7d5def9519 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -1,8 +1,8 @@ 'use client' -import { memo, useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' +import { memo, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { Chip, cn, toast } from '@sim/emcn' -import { FILE_DOC_SEED, type JoinFileDocError } from '@sim/realtime-protocol/file-doc' +import { FILE_DOC_SEED } from '@sim/realtime-protocol/file-doc' import { PASTE_LIMITS, PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import type { Extensions, JSONContent, Range } from '@tiptap/core' import { isChangeOrigin } from '@tiptap/extension-collaboration' @@ -446,6 +446,7 @@ export function LoadedRichMarkdownEditor({ const isEditable = canEdit && !isStreaming && (settled?.verdict ?? false) && collabReady const collaboration = useFileDocCollaboration({ + workspaceId, fileId: file.id, userId, userName, @@ -811,12 +812,6 @@ export function LoadedRichMarkdownEditor({ [] ) - /** - * The loaded markdown to seed the shared doc from, held by pointer so the parse - * runs once at seed time rather than every render. - */ - const seedContentRef = useRef(content) - /** The lifetime-stable editor and its async work consume only committed React inputs. */ useLayoutEffect(() => { onChangeRef.current = onChange @@ -831,7 +826,6 @@ export function LoadedRichMarkdownEditor({ insertImagesRef.current = insertImages cloneHostedImageRef.current = cloneHostedImage editorInstanceRef.current = editor - seedContentRef.current = content }) /** @@ -842,11 +836,9 @@ export function LoadedRichMarkdownEditor({ * synced AND seeded — it never imports content itself on the happy path; * - **gate** the parent's autosave until the doc is synced AND seeded, so an * empty/still-syncing doc can never overwrite the real file's markdown mirror; - * - **fall back** on a fatal join: seed the loaded content so it is SHOWN, but - * leave the editor read-only + gated. Every non-retryable failure (auth, access - * denied, not found, client-id conflict) either can't save or is moot, so the - * safe fallback is a read-only view of the content rather than editable-but- - * unsavable — which would silently drop the user's edits. + * - **preview** stored content in a separate read-only editor until authoritative content arrives. + * A retryable timeout never seeds the shared Y.Doc, so late server content cannot duplicate it. + * Terminal failures keep any existing live content visible but never editable. * * `ready` (synced+seeded) gates BOTH the editor's editability (a user must never * type into an empty/unsynced doc) and the parent's autosave. Non-collaborative @@ -864,11 +856,12 @@ export function LoadedRichMarkdownEditor({ * document that was already correct, and it opens mid-flight anyway whenever the updates arrive * more than a frame apart (which is what a remote Redis and a long room history produce). */ - const setReady = (ready: boolean, fatal = false) => { + const setReady = (ready: boolean, fatal = false, retrying = false) => { // Child-local: gates editability (a user must never type into an unsynced/unseeded doc). setCollabStatus((previous) => { if (fatal) return 'fatal' if (ready) return 'ready' + if (retrying) return 'reconnecting' return previous === 'ready' || previous === 'reconnecting' ? 'reconnecting' : 'connecting' }) // Parent: gates CLIENT autosave. In a collaborative session the relay persists the doc to @@ -888,20 +881,6 @@ export function LoadedRichMarkdownEditor({ } const config = doc.getMap(FILE_DOC_SEED.configMap) - let offlineSeed = false - - const seedFromLoaded = () => { - if (config.get(FILE_DOC_SEED.flag) === true) return - offlineSeed = true - doc.transact(() => { - editor.commands.setContent( - parseMarkdownToDoc(splitFrontmatter(seedContentRef.current).body), - { contentType: 'json', emitUpdate: false } - ) - config.set(FILE_DOC_SEED.flag, true) - }) - } - if (!provider) { setReady(false) return @@ -910,26 +889,20 @@ export function LoadedRichMarkdownEditor({ const report = () => { const synced = provider.synced const seeded = config.get(FILE_DOC_SEED.flag) === true - // `joinError` is latched ONLY on the provider's fatal paths (non-retryable rejection, access - // revocation, readiness deadline), so it is exactly "this document is abandoned". - const fatal = provider.joinError !== null - setReady(isCollabReady({ synced, seeded, offlineSeed, fatal }), fatal) - } - /** - * Re-report unconditionally, not just when the fallback seeds. A fatal that arrives on an ALREADY - * seeded doc (access revoked mid-session) leaves `seedFromLoaded` a no-op, so nothing else would - * fire an observer and the editor would stay editable on a document the provider has abandoned. - */ - const onJoinError = (error: JoinFileDocError) => { - if (error.retryable === false) seedFromLoaded() - report() + const fatal = provider.joinError?.retryable === false + setReady( + isCollabReady({ synced, seeded, fatal }), + fatal, + provider.joinError?.retryable === true + ) } + /** Rejections must close the editing gate even if the document was already seeded. */ + const onJoinError = () => report() provider.on('synced', report) provider.on('join-error', onJoinError) config.observe(report) report() - if (provider.joinError) onJoinError(provider.joinError) return () => { provider.off('synced', report) @@ -1311,9 +1284,15 @@ export function LoadedRichMarkdownEditor({ useSelectionCopyBridge(containerRef, buildSelectionContext, workspaceId) - /** Use the stored-content placeholder only while the live document is bootstrapping. */ - const showPlaceholder = collaborationEnabled && collabStatus === 'connecting' + /** Stored content belongs to a separate preview, never to an unseeded collaborative document. */ + const showPlaceholder = + collaborationEnabled && + (collabStatus === 'connecting' || + (collabStatus !== 'ready' && + collaboration?.doc.getMap(FILE_DOC_SEED.configMap).get(FILE_DOC_SEED.flag) !== true)) const showReconnecting = collaborationEnabled && collabStatus === 'reconnecting' + const collabFailure = collaboration?.provider?.joinError ?? null + const showCollabFailure = collaborationEnabled && collabStatus === 'fatal' ? collabFailure : null /** * Find is off while the placeholder is up. The text on screen then belongs to the placeholder's own @@ -1322,6 +1301,28 @@ export function LoadedRichMarkdownEditor({ * native find reads the rendered placeholder correctly; it becomes ours once the seed lands. */ const find = useMarkdownFind({ editor, enabled: enableFind && !showPlaceholder }) + const replaceControls = useMemo( + () => + isEditable + ? { + value: find.replacement, + onChange: find.setReplacement, + onReplace: find.replaceCurrent, + onReplaceAll: find.replaceAll, + canReplace: find.count > 0, + canReplaceAll: find.count > 0 && !find.truncated, + } + : undefined, + [ + find.count, + find.replaceAll, + find.replaceCurrent, + find.replacement, + find.setReplacement, + find.truncated, + isEditable, + ] + ) return ( // The find bar is a sibling of the scroller, not a child: pinned inside `containerRef` it would @@ -1347,6 +1348,17 @@ export function LoadedRichMarkdownEditor({ Reconnecting…
)} + {showCollabFailure && ( +
+ {showCollabFailure.code === 'ACCESS_REVOKED' || showCollabFailure.code === 'ACCESS_DENIED' + ? 'You no longer have edit access to this document.' + : 'Live editing is unavailable.'} +
+ )} {find.isOpen && ( )}
{ + it.each([ + '
\n\n
', + '', + '', + '---\nexample: \'\'\n---\n# Heading', + ])('allows image attributes preserved verbatim in raw content: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(true) + expect(normalizeMarkdownContent(source).trim()).toBe(source) + }) + + it('does not let a preserved raw tag hide an identical image tag that loses attributes', () => { + const tag = '' + expect(isRoundTripSafe(`
\n${tag}\n
\n\n${tag}`)).toBe(false) + }) + it('passes ordinary markdown and lossless normalizations', () => { expect(isRoundTripSafe('# Title\n\nA **bold** word and a [link](https://sim.ai).')).toBe(true) expect(isRoundTripSafe('- one\n- two\n\n```js\nconst x = 1\n```')).toBe(true) @@ -24,6 +39,11 @@ describe('isRoundTripSafe', () => { isRoundTripSafe('[![build](https://img.shields.io/badge/x-green)](https://ci.example.com)') ).toBe(true) expect(isRoundTripSafe('[![alt](https://e.com/i.png "t")](https://e.com "h")')).toBe(true) + expect( + isRoundTripSafe( + '[](https://e.com)' + ) + ).toBe(true) }) it('passes inline code without an interior backtick', () => { @@ -146,6 +166,42 @@ describe('isRoundTripSafe', () => { expect(isRoundTripSafe('')).toBe(true) }) + it('keeps HTML images with unsupported attributes in source mode', () => { + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('')).toBe(false) + expect(isRoundTripSafe('a')).toBe( + true + ) + }) + + it.each([ + '[](/link)', + '[](/link)', + "[](/link)", + ])('checks attributes after quoted angle brackets without losing source: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it.each([ + 'first', + '[first](/link)', + '[first](/link)', + '[](/link)', + ])('keeps duplicate image attributes in source mode: %s', (source) => { + expect(isRoundTripSafe(source)).toBe(false) + expect(normalizeMarkdownContent(source)).toBe(source) + }) + + it('allows supported image attributes containing quoted angle brackets', () => { + expect(isRoundTripSafe('')).toBe(true) + expect(isRoundTripSafe('[a>b](/link)')).toBe(true) + expect( + isRoundTripSafe('[](/link)') + ).toBe(true) + }) + it.each([ '| |\n| --- |\n| body |', '| header |\n| --- |\n| |', diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts index bab3759891f..bc69ce942b0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip-safety.ts @@ -1,6 +1,6 @@ import { PASTE_RENDER_THRESHOLDS } from '@sim/utils/paste' import { decodeHtmlEntities } from '@tiptap/core' -import { Marked, type Token } from 'marked' +import { Lexer, Marked, type Token, Tokenizer } from 'marked' import { extractImgSrcs } from '@/lib/uploads/utils/embedded-image-ref' import { splitFrontmatter } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-fidelity' import { serializeMarkdownDocument } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/markdown-parse' @@ -42,6 +42,35 @@ function stripCode(content: string): string { } const fidelityLexer = new Marked({ gfm: true }) +const SUPPORTED_IMAGE_ATTRIBUTES = new Set(['src', 'alt', 'title', 'width', 'height']) + +/** + * Count tags that image parsing would lose attributes from, including duplicates. Raw snippets + * may preserve these verbatim, so compare the counts before and after the first serialization. + */ +function unsupportedHtmlImages(content: string): Map { + const images = new Map() + const tokenizer = new Tokenizer() + new Lexer({ gfm: true, tokenizer }) + const imagePattern = /])/gi + for (let image = imagePattern.exec(content); image; image = imagePattern.exec(content)) { + const tag = tokenizer.tag(content.slice(image.index)) + if (!tag) continue + imagePattern.lastIndex = image.index + tag.raw.length + const attributes = tag.raw.slice(4, -1) + const seen = new Set() + const pattern = /(?:^|\s)([^\s=/>]+)(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'=<>`]+))?/g + for (const attribute of attributes.matchAll(pattern)) { + const name = attribute[1].toLowerCase() + if (!SUPPORTED_IMAGE_ATTRIBUTES.has(name) || seen.has(name)) { + images.set(tag.raw, (images.get(tag.raw) ?? 0) + 1) + break + } + seen.add(name) + } + } + return images +} function imageSources(token: Token): string[] { if (token.type === 'image') return [token.href] @@ -60,11 +89,13 @@ function imageSources(token: Token): string[] { function inspectMarkdownFidelity(content: string) { const targets = new Map() let hasTaskReference = false + let hasTableHtmlImage = false const add = (kind: 'image' | 'linkedImage', ...destinations: string[]) => { const target = JSON.stringify([kind, ...destinations.map(decodeHtmlEntities)]) targets.set(target, (targets.get(target) ?? 0) + 1) } fidelityLexer.walkTokens(fidelityLexer.lexer(splitFrontmatter(content).body), (token) => { + if (token.type === 'table' && / { @@ -83,7 +114,7 @@ function inspectMarkdownFidelity(content: string) { } } }) - return { targets, hasTaskReference } + return { targets, hasTaskReference, hasTableHtmlImage } } /** @@ -143,8 +174,12 @@ export function isRoundTripSafe(content: string): boolean { if (hasOrphanReferenceDefinition(stripped)) return false try { const source = inspectMarkdownFidelity(content) - if (source.hasTaskReference) return false + if (source.hasTaskReference || source.hasTableHtmlImage) return false const once = serializeMarkdownDocument(content) + const preservedImages = unsupportedHtmlImages(stripCode(once)) + for (const [tag, count] of unsupportedHtmlImages(stripped)) { + if ((preservedImages.get(tag) ?? 0) < count) return false + } const serialized = inspectMarkdownFidelity(once) for (const [target, count] of source.targets) { if ((serialized.targets.get(target) ?? 0) < count) return false diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts index 961fd1d86df..bcc50b243eb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/round-trip.test.ts @@ -301,6 +301,52 @@ describe('editor markdown round-trip', () => { expect(roundTrip('![a](https://e.com/i.png)')).toContain('![a](https://e.com/i.png)') }) + it('round-trips every sized linked-image attribute without dropping dimensions', () => { + const source = + '[](https://e.com "Details")' + const out = roundTrip(source) + + expect(out).toContain('alt=""') + expect(out).toContain('width="320" height="180"') + expect(out).toContain('](https://e.com "Details")') + expect(roundTrip(out)).toBe(out) + }) + + it('uses empty alt text when a linked HTML image has no alt attribute', () => { + const source = '[](https://e.com)' + const out = roundTrip(source) + + expect(out).toContain('alt=""') + expect(out).not.toContain('alt="<img') + expect(roundTrip(out)).toBe(out) + }) + + it('round-trips linked images with escaped alt text and angle-bracket destinations', () => { + const source = '[![a\\]b]()]( "Details")' + const out = roundTrip(source) + + expect(out).toContain('a\\]b') + expect(out).toContain('') + expect(out).toContain('') + expect(roundTrip(out)).toBe(out) + }) + + it('parses a paragraph of adjacent links and linked images without recursive suffix scans', () => { + const links = Array.from( + { length: 80 }, + (_, index) => `[Link ${index}](https://e.com/${index})` + ) + const images = Array.from( + { length: 40 }, + (_, index) => `[![Image ${index}](https://e.com/${index}.png)](https://e.com/${index})` + ) + const out = roundTrip([...links, ...images].join(' ')) + + for (const link of links) expect(out).toContain(link) + for (const image of images) expect(out).toContain(image) + expect(roundTrip(out)).toBe(out) + }) + it('preserves a sized base64 image and escapes quotes in attributes', () => { const dataUrl = '' expect(roundTrip(dataUrl)).toContain('data:image/png;base64,iVBORw0KGgo=') diff --git a/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.test.tsx b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.test.tsx new file mode 100644 index 00000000000..8699d870a01 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.test.tsx @@ -0,0 +1,113 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + params: { workspaceId: 'workspace-1' } as Record, + socket: { + isReconnecting: true, + isRetryingWorkflowJoin: false, + blockedJoinWorkflowId: null as string | null, + }, + hasOperationError: false, + toast: { error: vi.fn(() => 'toast-1'), dismiss: vi.fn() }, + setQueryData: vi.fn(), + refetch: vi.fn(), +})) + +vi.mock('@sim/emcn', () => ({ useToast: () => ({ toast: mocks.toast }) })) +vi.mock('next/navigation', () => ({ useParams: () => mocks.params })) +vi.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ setQueryData: mocks.setQueryData }), +})) +vi.mock('@/app/workspace/providers/socket-provider', () => ({ useSocket: () => mocks.socket })) +vi.mock('@/hooks/queries/workspace', () => ({ + useWorkspacePermissionsQuery: () => ({ + data: null, + isLoading: false, + error: null, + refetch: mocks.refetch, + }), + workspaceKeys: { permissions: (id: string) => ['workspace', id, 'permissions'] }, +})) +vi.mock('@/hooks/use-stable-flag', () => ({ useStableFlag: (value: boolean) => value })) +vi.mock('@/hooks/use-user-permissions', () => ({ + useUserPermissions: () => ({ + canRead: true, + canEdit: true, + canAdmin: true, + userPermissions: 'admin', + isLoading: false, + error: null, + }), +})) +vi.mock('@/stores/operation-queue/store', () => ({ + useOperationQueueStore: (select: (state: { hasOperationError: boolean }) => boolean) => + select({ hasOperationError: mocks.hasOperationError }), +})) + +import { WorkspacePermissionsProvider } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' + +;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + +describe('workspace reconnect notifications', () => { + let host: HTMLDivElement + let root: Root + + function renderProvider() { + act(() => + root.render( + +
+ + ) + ) + } + + beforeEach(() => { + vi.clearAllMocks() + mocks.params = { workspaceId: 'workspace-1' } + mocks.socket.isReconnecting = true + mocks.socket.isRetryingWorkflowJoin = false + mocks.socket.blockedJoinWorkflowId = null + mocks.hasOperationError = false + host = document.createElement('div') + document.body.appendChild(host) + root = createRoot(host) + }) + + afterEach(() => { + act(() => root.unmount()) + host.remove() + }) + + it('leaves file reconnect feedback to the inline editor status', () => { + mocks.params.fileId = 'file-1' + renderProvider() + expect(mocks.toast.error).not.toHaveBeenCalled() + }) + + it('retains reconnect feedback elsewhere in the workspace', () => { + mocks.params.workflowId = 'workflow-1' + renderProvider() + expect(mocks.toast.error).toHaveBeenCalledWith('Reconnecting...', expect.any(Object)) + }) + + it('dismisses the workspace reconnect toast when entering a file', () => { + renderProvider() + mocks.params.fileId = 'file-1' + renderProvider() + expect(mocks.toast.dismiss).toHaveBeenCalledWith('toast-1') + expect(mocks.toast.error).toHaveBeenCalledTimes(1) + }) + + it('does not suppress terminal operation errors inside a file', () => { + mocks.params.fileId = 'file-1' + mocks.hasOperationError = true + renderProvider() + expect(mocks.toast.error).toHaveBeenCalledWith('Connection unavailable', expect.any(Object)) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx index 92100372f02..af4ba896db5 100644 --- a/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx +++ b/apps/sim/app/workspace/[workspaceId]/providers/workspace-permissions-provider.tsx @@ -120,6 +120,7 @@ export function WorkspacePermissionsProvider({ children }: WorkspacePermissionsP const params = useParams() const workspaceId = params?.workspaceId as string const urlWorkflowId = params?.workflowId as string | undefined + const isFileViewer = Boolean(params?.fileId) const queryClient = useQueryClient() const hasOperationError = useOperationQueueStore((state) => state.hasOperationError) @@ -131,13 +132,14 @@ export function WorkspacePermissionsProvider({ children }: WorkspacePermissionsP delayMs: RECONNECTING_TOAST_DELAY_MS, minVisibleMs: RECONNECTING_TOAST_MIN_VISIBLE_MS, }) - const realtimeStatusMessage = isOfflineMode - ? null - : showReconnecting - ? 'Reconnecting...' - : isRetryingWorkflowJoin - ? 'Joining workflow...' - : null + const realtimeStatusMessage = + isOfflineMode || isFileViewer + ? null + : showReconnecting + ? 'Reconnecting...' + : isRetryingWorkflowJoin + ? 'Joining workflow...' + : null usePersistentErrorToast(realtimeStatusMessage) // Offline mode only recovers via workspace switch or refresh; the join block diff --git a/apps/sim/lib/api/contracts/v2/__tests__/files-attribution.test.ts b/apps/sim/lib/api/contracts/v2/__tests__/files-attribution.test.ts new file mode 100644 index 00000000000..d92650f6dc4 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/__tests__/files-attribution.test.ts @@ -0,0 +1,22 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { v2FileSchema } from '@/lib/api/contracts/v2/files' +import { ANONYMOUS_USER } from '@/lib/auth/constants' + +describe('file uploader attribution', () => { + it.each([ANONYMOUS_USER.email, 'ada@example.com', 'ada+files@example.co.uk'])( + 'preserves the stored uploader email %s', + (email) => { + expect(v2FileSchema.shape.uploadedByEmail.parse(email)).toBe(email) + } + ) + + it.each(['', 'not-an-email', 'ada@', '@example.com', 'ada @example.com', 'ada@example..com'])( + 'rejects malformed attribution %s', + (email) => { + expect(v2FileSchema.shape.uploadedByEmail.safeParse(email).success).toBe(false) + } + ) +}) diff --git a/apps/sim/lib/api/contracts/v2/files.ts b/apps/sim/lib/api/contracts/v2/files.ts index 50c919b5ba4..f88101075a7 100644 --- a/apps/sim/lib/api/contracts/v2/files.ts +++ b/apps/sim/lib/api/contracts/v2/files.ts @@ -98,7 +98,7 @@ export const v2FileSchema = z 'Canonical containing-folder path. `/` is the workspace root.' ), uploadedByEmail: z - .email() + .email({ pattern: z.regexes.html5Email }) .describe('Current email address of the uploader.') .meta({ examples: ['jane@example.com'] }), /** ISO-8601 timestamp. */ diff --git a/apps/sim/lib/collab-doc/converter.test.ts b/apps/sim/lib/collab-doc/converter.test.ts index f31796d2a82..20fd26e6436 100644 --- a/apps/sim/lib/collab-doc/converter.test.ts +++ b/apps/sim/lib/collab-doc/converter.test.ts @@ -40,6 +40,7 @@ const SAMPLES = [ 'A footnote reference[^1].\n\n[^1]: the footnote body.', 'Before.\n\n
untouched raw html
\n\nAfter.', '- [ ] todo\n- [x] done', + '[](https://e.com)', ] beforeAll(() => { diff --git a/apps/sim/lib/core/outbox/service.test.ts b/apps/sim/lib/core/outbox/service.test.ts index 9817363630c..64c128226e0 100644 --- a/apps/sim/lib/core/outbox/service.test.ts +++ b/apps/sim/lib/core/outbox/service.test.ts @@ -255,15 +255,28 @@ describe('processOutboxEvents — empty / no handler', () => { }) }) - it('dead-letters events with no registered handler', async () => { + it('retries events with no registered handler during rolling deployments', async () => { queueTableRows(outboxEvent, [makePendingRow({ eventType: 'unknown.event' })]) holdLease() const result = await processOutboxEvents({}) + expect(result.retried).toBe(1) + const retry = updateSets().find((set) => set.status === 'pending' && 'attempts' in set) + expect(retry).toBeDefined() + expect(retry?.attempts).toBe(1) + }) + + it('dead-letters a missing handler after the configured retry budget', async () => { + queueTableRows(outboxEvent, [ + makePendingRow({ eventType: 'unknown.event', attempts: 2, maxAttempts: 3 }), + ]) + holdLease() + + const result = await processOutboxEvents({}) + expect(result.deadLettered).toBe(1) const terminal = updateSets().find((set) => set.status === 'dead_letter') - expect(terminal).toBeDefined() expect(terminal?.lastError).toMatch(/No handler registered/) }) }) diff --git a/apps/sim/lib/core/outbox/service.ts b/apps/sim/lib/core/outbox/service.ts index a4ede767836..152caecfa5a 100644 --- a/apps/sim/lib/core/outbox/service.ts +++ b/apps/sim/lib/core/outbox/service.ts @@ -568,17 +568,15 @@ async function runHandler( const handler = handlers[event.eventType] if (!handler) { - logger.error('No handler registered for outbox event type', { + const reason = `No handler registered for event type '${event.eventType}'` + logger.warn('No handler registered for outbox event type; scheduling a bounded retry', { eventId: event.id, eventType: event.eventType, }) - await updateIfLeaseHeld(event, { - status: 'dead_letter', - lastError: `No handler registered for event type '${event.eventType}'`, - processedAt: new Date(), - lockedAt: null, + return scheduleDeferred(event, { + outcome: 'deferred', + reason, }) - return 'dead_letter' } try { diff --git a/apps/sim/lib/realtime/notify.test.ts b/apps/sim/lib/realtime/notify.test.ts index 98196492257..0955f0766bc 100644 --- a/apps/sim/lib/realtime/notify.test.ts +++ b/apps/sim/lib/realtime/notify.test.ts @@ -6,18 +6,21 @@ import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/core/utils/urls', () => ({ getSocketServerUrl: () => 'http://realtime' })) vi.mock('@/lib/core/config/env', () => ({ env: { INTERNAL_API_SECRET: 'secret' } })) -import { mergeEditIntoLiveFileDoc } from './notify' +import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' -describe('mergeEditIntoLiveFileDoc', () => { +describe('applyEditToLiveFileDoc', () => { afterEach(() => { vi.unstubAllGlobals() }) it('POSTs the edit to the realtime apply-edit endpoint with the api key', async () => { - const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ applied: true, status: 'applied' }), + }) vi.stubGlobal('fetch', fetchMock) - await mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) + await applyEditToLiveFileDoc('file-1', '# hello', { version: 42 }) expect(fetchMock).toHaveBeenCalledWith( 'http://realtime/api/file-doc/apply-edit', @@ -30,83 +33,58 @@ describe('mergeEditIntoLiveFileDoc', () => { ) }) - it('never throws when the realtime call fails (best-effort)', async () => { + it('throws when the realtime call fails so the outbox can retry', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('socket pod down'))) - await expect( - mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) - ).resolves.toBeUndefined() + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).rejects.toThrow( + 'socket pod down' + ) }) - it('never throws on a non-2xx response', async () => { + it('surfaces retryable delivery failures to durable outbox callers', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, status: 503 })) - await expect( - mergeEditIntoLiveFileDoc('file-1', '# hello', { version: 42 }) - ).resolves.toBeUndefined() + + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).rejects.toThrow( + 'status 503' + ) }) - it('a later durable merge waits for an in-flight earlier one, then applies last', async () => { - let resolveFirst: (value: { ok: boolean }) => void = () => {} - const fetchMock = vi - .fn() - .mockImplementationOnce(() => new Promise((resolve) => (resolveFirst = resolve))) - .mockResolvedValue({ ok: true }) - vi.stubGlobal('fetch', fetchMock) + it('returns the relay reconciliation status to durable outbox callers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: true, + json: vi.fn().mockResolvedValue({ applied: false, status: 'no-live-room' }), + }) + ) - const first = mergeEditIntoLiveFileDoc('file-durable', 'earlier', { version: 99 }) // in flight - await Promise.resolve() - const durable = mergeEditIntoLiveFileDoc('file-durable', 'final content', { version: 100 }) - await Promise.resolve() - await Promise.resolve() + await expect(applyEditToLiveFileDoc('file-1', '# hello', { version: 42 })).resolves.toEqual({ + applied: false, + status: 'no-live-room', + }) + }) +}) - // The later write waits for the in-flight earlier one → its fetch has not fired yet, so it cannot be - // reordered before a straggler and cannot be clobbered by one. - expect(fetchMock).toHaveBeenCalledTimes(1) +describe('invalidateLiveFileDoc', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) - resolveFirst({ ok: true }) - await first - await durable + it('POSTs a durability-sensitive invalidation and surfaces delivery failures', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true }) + vi.stubGlobal('fetch', fetchMock) - // Only after the earlier merge completed does the later (final) merge apply — always last. - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1][1].body).toBe( - JSON.stringify({ fileId: 'file-durable', markdown: 'final content', version: 100 }) - ) - }) + await invalidateLiveFileDoc('file-1', 42) - it('serializes concurrent durable writes to a file strictly in order', async () => { - const applied: number[] = [] - const resolvers: Array<() => void> = [] - vi.stubGlobal( - 'fetch', - vi.fn((_url: string, init: { body: string }) => { - applied.push(JSON.parse(init.body).version) - return new Promise<{ ok: boolean }>((resolve) => - resolvers.push(() => resolve({ ok: true })) - ) + expect(fetchMock).toHaveBeenCalledWith( + 'http://realtime/api/file-doc/invalidate', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ 'x-api-key': 'secret' }), + body: JSON.stringify({ fileId: 'file-1', version: 42 }), }) ) - const flush = async () => { - for (let i = 0; i < 6; i++) await Promise.resolve() - } - - const s = mergeEditIntoLiveFileDoc('file-order', 's', { version: 0 }) // in flight - await flush() - // Two later durable writes arrive while the first merge is in flight — both must chain, not both - // resume-and-fire concurrently. - const a = mergeEditIntoLiveFileDoc('file-order', 'a', { version: 1 }) - const b = mergeEditIntoLiveFileDoc('file-order', 'b', { version: 2 }) - await flush() - expect(applied).toEqual([0]) // A and B queued behind the in-flight first merge - - resolvers[0]() // finish first → A applies next (not B) - await flush() - expect(applied).toEqual([0, 1]) - - resolvers[1]() // finish A → B applies after A - await flush() - expect(applied).toEqual([0, 1, 2]) - - resolvers[2]() - await Promise.all([s, a, b]) + + fetchMock.mockResolvedValueOnce({ ok: false, status: 503 }) + await expect(invalidateLiveFileDoc('file-1', 42)).rejects.toThrow('status 503') }) }) diff --git a/apps/sim/lib/realtime/notify.ts b/apps/sim/lib/realtime/notify.ts index 2374a8d5580..df3e13db400 100644 --- a/apps/sim/lib/realtime/notify.ts +++ b/apps/sim/lib/realtime/notify.ts @@ -169,79 +169,73 @@ export async function notifyFolderResourceChanged( * How a durable live-doc merge is positioned on the file's monotonic version line. Omit `version` to * apply the merge without ordering it (legacy). */ -interface LiveFileDocMergeOrder { +export interface LiveFileDocMergeOrder { /** A durable write's `contentUpdatedAt` (epoch ms): applied only if newer than the version the doc * already incorporates, AND recorded as the synced version (the persist If-Match guard). */ version?: number } +export type LiveFileDocMergeStatus = 'applied' | 'no-live-room' | 'merge-unavailable' | 'stale' + +interface LiveFileDocMergeResponse { + applied: boolean + status: LiveFileDocMergeStatus +} + /** - * Best-effort: ask the realtime relay to merge a durable copilot/file write into a file's LIVE - * collaborative document, so open editors reconcile to it as a CRDT merge rather than the file changing - * underneath them, and a late joiner is seeded from it. No-op when no doc is (or was recently) live (the - * relay reports `applied: false`). The file itself is written durably by the caller regardless — this - * only drives the live view. Never throws. - * - * (Streaming copilot output is NOT merged here: the open editor applies the stream client-side as minimal - * CRDT diffs — see `applyStreamedMarkdownToLiveDoc` — which renders smoothly and broadcasts to peers. This - * merge is the stream-end durable reconcile, and by then it is usually a noop diff.) - * - * The former clobber gap — an open editor's autosave dropping this edit — is closed: a collaborative - * editor no longer client-autosaves (the relay persists the shared doc to markdown server-side), and the - * relay applies this merge THROUGH the shared Redis stream, so it reaches the live doc on whichever task - * holds it and can't go stale relative to this direct write. - * - * The caller awaits this so the fetch dispatches before the route handler returns. Bounded to - * {@link APPLY_EDIT_TIMEOUT_MS}, so it adds latency only when the socket pod is unreachable. - * - * `order.version` positions the merge so a stale write never regresses the doc: it applies only if newer - * than the version the doc already incorporates, and is recorded as the synced version. Ordering is - * enforced at two scales: within this process, merges for a file run on a single serialized chain (each - * chained after the current tail) so writes never apply concurrently; across processes the relay orders - * by that monotonic version under a cluster-wide lock. + * Applies one durable file version to the live collaboration document and surfaces delivery + * failures to callers that own a retry policy, such as the transactional outbox. */ -export async function mergeEditIntoLiveFileDoc( +export async function applyEditToLiveFileDoc( fileId: string, markdown: string, - order: LiveFileDocMergeOrder = {} -): Promise { - const tail = liveDocMergeChain.get(fileId) ?? Promise.resolve() - const run = tail.then(() => applyLiveFileDocMerge(fileId, markdown, order)) - liveDocMergeChain.set(fileId, run) - try { - await run - } finally { - if (liveDocMergeChain.get(fileId) === run) liveDocMergeChain.delete(fileId) + order: LiveFileDocMergeOrder = {}, + signal?: AbortSignal +): Promise { + const timeoutSignal = AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS) + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, markdown, version: order.version }), + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (!response.ok) { + throw new Error(`Live document reconciliation failed with status ${response.status}`) } -} -/** Per file, the tail of the serialized merge chain (each merge applies after it); never rejects - * because {@link applyLiveFileDocMerge} never throws. Absent when the file's chain is idle. */ -const liveDocMergeChain = new Map>() + const result = (await response.json()) as unknown + if (typeof result !== 'object' || result === null) { + throw new Error('Live document reconciliation returned an invalid response') + } + const candidate = result as Partial + const validStatus = + candidate.status === 'applied' || + candidate.status === 'no-live-room' || + candidate.status === 'merge-unavailable' || + candidate.status === 'stale' + if (typeof candidate.applied !== 'boolean' || !validStatus) { + throw new Error('Live document reconciliation returned an invalid response') + } + return { applied: candidate.applied, status: candidate.status as LiveFileDocMergeStatus } +} -/** POST the merge to the relay. Never throws (a live-doc merge is best-effort). */ -async function applyLiveFileDocMerge( +/** + * Invalidates one live document after a durable replacement that cannot be merged into the rich + * editor. Unlike list notifications this is durability-sensitive and throws so the outbox retries. + */ +export async function invalidateLiveFileDoc( fileId: string, - markdown: string, - order: LiveFileDocMergeOrder + version: number, + signal?: AbortSignal ): Promise { - try { - const response = await fetch(`${getSocketServerUrl()}/api/file-doc/apply-edit`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, - // `version` (durable `contentUpdatedAt`) records the synced version the live doc now incorporates - // (the persist If-Match guard). JSON.stringify drops it when undefined (an unordered legacy merge). - body: JSON.stringify({ - fileId, - markdown, - version: order.version, - }), - signal: AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS), - }) - if (!response.ok) { - logger.warn('file-doc apply-edit failed', { fileId, status: response.status }) - } - } catch (error) { - logger.warn('file-doc apply-edit error', { fileId, error: getErrorMessage(error) }) + const timeoutSignal = AbortSignal.timeout(APPLY_EDIT_TIMEOUT_MS) + const response = await fetch(`${getSocketServerUrl()}/api/file-doc/invalidate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-api-key': env.INTERNAL_API_SECRET }, + body: JSON.stringify({ fileId, version }), + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (!response.ok) { + throw new Error(`Live document invalidation failed with status ${response.status}`) } } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts new file mode 100644 index 00000000000..3c2bbc43ccb --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockApplyEditToLiveFileDoc, mockDownloadFile, mockInvalidateLiveFileDoc } = vi.hoisted( + () => ({ + mockApplyEditToLiveFileDoc: vi.fn(), + mockDownloadFile: vi.fn(), + mockInvalidateLiveFileDoc: vi.fn(), + }) +) + +vi.mock('@/lib/realtime/notify', () => ({ + applyEditToLiveFileDoc: mockApplyEditToLiveFileDoc, + invalidateLiveFileDoc: mockInvalidateLiveFileDoc, +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFile: mockDownloadFile, +})) + +import type { OutboxEventContext } from '@/lib/core/outbox/service' +import { + WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + workspaceFileLiveDocOutboxHandlers, +} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' + +const VERSION = new Date('2026-09-04T12:00:00.000Z') +const PAYLOAD = { + workspaceId: 'workspace-1', + fileId: 'file-1', + version: VERSION.getTime(), +} + +function context(): OutboxEventContext { + return { + eventId: 'event-1', + eventType: WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, + attempts: 0, + maxAttempts: 10, + signal: new AbortController().signal, + checkpointPayload: vi.fn(), + } +} + +function handler() { + const registered = workspaceFileLiveDocOutboxHandlers[WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT] + if (!registered) throw new Error('Workspace file live-document handler is not registered') + return registered +} + +describe('workspace file live-document outbox', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockDownloadFile.mockResolvedValue(Buffer.from('# Durable content')) + mockApplyEditToLiveFileDoc.mockResolvedValue({ applied: true, status: 'applied' }) + }) + + it('loads the committed version and reconciles it into the live document', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/workspace-1/file.md', context: 'workspace' }) + ) + expect(mockApplyEditToLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + '# Durable content', + { version: VERSION.getTime() }, + expect.any(AbortSignal) + ) + }) + + it('completes a stale event without reading or regressing newer content', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: new Date(VERSION.getTime() + 1), + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).not.toHaveBeenCalled() + }) + + it('defers transient merge-lock contention for an outbox retry', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + mockApplyEditToLiveFileDoc.mockResolvedValueOnce({ + applied: false, + status: 'merge-unavailable', + }) + + await expect(handler()(PAYLOAD, context())).resolves.toEqual( + expect.objectContaining({ outcome: 'deferred' }) + ) + }) + + it('rejects malformed payloads before touching durable state', async () => { + await expect(handler()({ ...PAYLOAD, version: 0 }, context())).rejects.toThrow( + 'invalid version' + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('does not materialize files beyond the collaborative editor boundary', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.md', + name: 'file.md', + type: 'text/markdown', + sizeBytes: 6 * 1024 * 1024, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + VERSION.getTime(), + expect.any(AbortSignal) + ) + }) + + it('invalidates a live markdown generation when the durable file changes type', async () => { + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.bin', + name: 'file.bin', + type: 'application/octet-stream', + sizeBytes: 100, + contentUpdatedAt: VERSION, + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + VERSION.getTime(), + expect.any(AbortSignal) + ) + }) + + it('still invalidates after a later binary write supersedes the type-changing event', async () => { + const latestVersion = VERSION.getTime() + 1 + dbChainMockFns.limit.mockResolvedValueOnce([ + { + key: 'workspace/workspace-1/file.bin', + name: 'file.bin', + type: 'application/octet-stream', + sizeBytes: 100, + contentUpdatedAt: new Date(latestVersion), + }, + ]) + + await handler()(PAYLOAD, context()) + + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockApplyEditToLiveFileDoc).not.toHaveBeenCalled() + expect(mockInvalidateLiveFileDoc).toHaveBeenCalledWith( + 'file-1', + latestVersion, + expect.any(AbortSignal) + ) + }) +}) diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts new file mode 100644 index 00000000000..b7dfe968f71 --- /dev/null +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox.ts @@ -0,0 +1,116 @@ +import { db } from '@sim/db' +import { workspaceFiles } from '@sim/db/schema' +import { PASTE_LIMITS } from '@sim/utils/paste' +import { and, eq, isNull } from 'drizzle-orm' +import { + deferOutboxHandler, + enqueueOutboxEvent, + type OutboxHandler, + type OutboxHandlerRegistry, + processOutboxEventById, +} from '@/lib/core/outbox/service' +import { applyEditToLiveFileDoc, invalidateLiveFileDoc } from '@/lib/realtime/notify' +import { downloadFile } from '@/lib/uploads/core/storage-service' +import { isMarkdownFile } from '@/lib/uploads/utils/file-utils' + +export const WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT = 'workspace-file.live-doc.reconcile' + +interface WorkspaceFileLiveDocPayload { + workspaceId: string + fileId: string + version: number +} + +function parsePayload(payload: unknown): WorkspaceFileLiveDocPayload { + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + throw new Error('Workspace file live-document outbox payload must be an object') + } + const candidate = payload as Partial + if (typeof candidate.workspaceId !== 'string' || candidate.workspaceId.length === 0) { + throw new Error('Workspace file live-document outbox payload is missing workspaceId') + } + if (typeof candidate.fileId !== 'string' || candidate.fileId.length === 0) { + throw new Error('Workspace file live-document outbox payload is missing fileId') + } + if ( + typeof candidate.version !== 'number' || + !Number.isSafeInteger(candidate.version) || + candidate.version <= 0 + ) { + throw new Error('Workspace file live-document outbox payload has an invalid version') + } + return candidate as WorkspaceFileLiveDocPayload +} + +const reconcileWorkspaceFileLiveDoc: OutboxHandler = async (rawPayload, context) => { + const payload = parsePayload(rawPayload) + context.signal.throwIfAborted() + const [file] = await db + .select({ + key: workspaceFiles.key, + name: workspaceFiles.originalName, + type: workspaceFiles.contentType, + sizeBytes: workspaceFiles.sizeBytes, + contentUpdatedAt: workspaceFiles.contentUpdatedAt, + }) + .from(workspaceFiles) + .where( + and( + eq(workspaceFiles.id, payload.fileId), + eq(workspaceFiles.workspaceId, payload.workspaceId), + eq(workspaceFiles.context, 'workspace'), + isNull(workspaceFiles.deletedAt) + ) + ) + .limit(1) + + if (!file) return + const currentVersion = file.contentUpdatedAt.getTime() + if (currentVersion < payload.version) { + throw new Error('Workspace file live-document reconciliation is ahead of durable content') + } + if ( + !isMarkdownFile(file) || + file.sizeBytes === null || + file.sizeBytes > PASTE_LIMITS.RICH_MARKDOWN_BYTES + ) { + /** Later binary writes do not enqueue reconciliation, so retire the latest unsupported version. */ + await invalidateLiveFileDoc(payload.fileId, currentVersion, context.signal) + return + } + if (currentVersion > payload.version) return + + const content = await downloadFile({ + key: file.key, + context: 'workspace', + maxBytes: PASTE_LIMITS.RICH_MARKDOWN_BYTES, + signal: context.signal, + }) + context.signal.throwIfAborted() + const result = await applyEditToLiveFileDoc( + payload.fileId, + content.toString('utf-8'), + { version: payload.version }, + context.signal + ) + if (result.status === 'merge-unavailable') { + return deferOutboxHandler('Live document merge slot is temporarily unavailable') + } +} + +export const workspaceFileLiveDocOutboxHandlers = { + [WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT]: reconcileWorkspaceFileLiveDoc, +} satisfies OutboxHandlerRegistry + +/** Enqueues live-document reconciliation in the same transaction as the durable file version. */ +export function enqueueWorkspaceFileLiveDocReconciliation( + executor: Pick, + payload: WorkspaceFileLiveDocPayload +): Promise { + return enqueueOutboxEvent(executor, WORKSPACE_FILE_LIVE_DOC_OUTBOX_EVENT, payload) +} + +/** Attempts a newly committed reconciliation immediately; the outbox worker owns retries. */ +export function processWorkspaceFileLiveDocReconciliationNow(eventId: string) { + return processOutboxEventById(eventId, workspaceFileLiveDocOutboxHandlers) +} diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 20a99b69163..3d79ea322c6 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -55,8 +55,13 @@ import { acquireFolderMutationLock } from '@/lib/folders/locks' import { parseFolderPath } from '@/lib/folders/paths' import { loadActiveFolderPathIndex, resolveFolderPathFromIndex } from '@/lib/folders/queries' import type { FolderIdScope } from '@/lib/folders/scope' -import { mergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' import { getServePathPrefix } from '@/lib/uploads' +import type { WorkspaceFileFolderRecord } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager' +import { + enqueueWorkspaceFileLiveDocReconciliation, + processWorkspaceFileLiveDocReconciliationNow, +} from '@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox' import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, initializeWorkspaceFileSecretProvenanceInTx, @@ -87,7 +92,6 @@ import { import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid } from '@/executor/constants' import type { UserFile } from '@/executor/types' -import type { WorkspaceFileFolderRecord } from './workspace-file-folder-manager' import { assertWorkspaceFileFolderTarget, buildWorkspaceFileFolderPathMap, @@ -1816,6 +1820,7 @@ export async function updateWorkspaceFileContent( oldKey: string sizeDiff: number updatedUsage: number | undefined + liveDocEventId: string | undefined } try { finalized = await db.transaction(async (tx) => { @@ -1926,11 +1931,23 @@ export async function updateWorkspaceFileContent( ) } + const liveDocEventId = + options?.syncLiveDoc !== false && + (isMarkdownFile({ type: currentFile.contentType, name: currentFile.originalName }) || + isMarkdownFile({ type: updatedFile.contentType, name: updatedFile.originalName })) + ? await enqueueWorkspaceFileLiveDocReconciliation(tx, { + workspaceId, + fileId, + version: updatedFile.contentUpdatedAt.getTime(), + }) + : undefined + return { file: updatedFile, oldKey: currentFile.key, sizeDiff, updatedUsage, + liveDocEventId, } }) } catch (finalizationError) { @@ -1949,22 +1966,25 @@ export async function updateWorkspaceFileContent( await cleanupWorkspaceStorageObject(finalized.oldKey, 'version replacement') } - // Stream this write into any open collaborative editor as a CRDT merge, so a copilot/tool edit - // shows up live instead of the file silently changing underneath the reader. Gated to markdown (the - // only format the collaborative editor renders) and best-effort (a no-op when nobody has the file - // open; never throws). This is the single chokepoint every external writer shares — the relay's own - // persist and empty-shell creates pass `syncLiveDoc: false` to stay out of it. - if ( - options?.syncLiveDoc !== false && - isMarkdownFile({ type: nextContentType, name: finalized.file.originalName }) - ) { - // Pass the new CONTENT version this write produced, so the relay records that its live doc now - // incorporates this durable version — the collab persist's optimistic-concurrency guard then won't - // treat this (already-merged) write as an out-of-band conflict. Must be the SAME field the CAS - // guards on (`contentUpdatedAt`), not `updatedAt`, or the relay's token wouldn't match the CAS. - await mergeEditIntoLiveFileDoc(fileId, content.toString('utf-8'), { - version: finalized.file.contentUpdatedAt.getTime(), - }) + if (finalized.liveDocEventId) { + try { + const result = await processWorkspaceFileLiveDocReconciliationNow(finalized.liveDocEventId) + if (result !== 'completed') { + logger.warn('Live document reconciliation deferred to outbox retry', { + workspaceId, + fileId, + eventId: finalized.liveDocEventId, + result, + }) + } + } catch (error) { + logger.warn('Live document reconciliation deferred after inline processing error', { + workspaceId, + fileId, + eventId: finalized.liveDocEventId, + error: getErrorMessage(error), + }) + } } const pathPrefix = getServePathPrefix() diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index e084217a435..d58fb0200f6 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -2,8 +2,9 @@ * @vitest-environment node */ import { workspaceFiles } from '@sim/db/schema' -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing' import { describeError } from '@sim/utils/errors' +import { PASTE_LIMITS } from '@sim/utils/paste' import { eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -11,6 +12,7 @@ const { mockDecrementStorageUsageForBillingContextInTx, mockDeleteFile, mockEnqueueWorkspaceFileStorageCleanup, + mockEnqueueWorkspaceFileLiveDocReconciliation, mockGetWorkspaceWithOwner, mockHasCloudStorage, mockHeadObject, @@ -20,9 +22,9 @@ const { mockLoadActiveFolderPathIndex, mockInitializeWorkspaceFileSecretProvenanceInTx, mockMaybeNotifyStorageLimitForBillingContext, - mockMergeEditIntoLiveFileDoc, mockNotifyWorkspaceFilesChanged, mockProcessWorkspaceFileStorageCleanupNow, + mockProcessWorkspaceFileLiveDocReconciliationNow, mockResolveStorageBillingContext, mockResolveFolderPathFromIndex, mockResolveWorkspaceFileFolderTarget, @@ -32,6 +34,7 @@ const { mockDecrementStorageUsageForBillingContextInTx: vi.fn(), mockDeleteFile: vi.fn(), mockEnqueueWorkspaceFileStorageCleanup: vi.fn(), + mockEnqueueWorkspaceFileLiveDocReconciliation: vi.fn(), mockGetWorkspaceWithOwner: vi.fn(), mockHasCloudStorage: vi.fn(), mockHeadObject: vi.fn(), @@ -41,9 +44,9 @@ const { mockLoadActiveFolderPathIndex: vi.fn(), mockInitializeWorkspaceFileSecretProvenanceInTx: vi.fn(), mockMaybeNotifyStorageLimitForBillingContext: vi.fn(), - mockMergeEditIntoLiveFileDoc: vi.fn(), mockNotifyWorkspaceFilesChanged: vi.fn(), mockProcessWorkspaceFileStorageCleanupNow: vi.fn(), + mockProcessWorkspaceFileLiveDocReconciliationNow: vi.fn(), mockResolveStorageBillingContext: vi.fn(), mockResolveFolderPathFromIndex: vi.fn(), mockResolveWorkspaceFileFolderTarget: vi.fn(), @@ -59,10 +62,14 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () })) vi.mock('@/lib/realtime/notify', () => ({ - mergeEditIntoLiveFileDoc: mockMergeEditIntoLiveFileDoc, notifyWorkspaceFilesChanged: mockNotifyWorkspaceFilesChanged, })) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-live-doc-outbox', () => ({ + enqueueWorkspaceFileLiveDocReconciliation: mockEnqueueWorkspaceFileLiveDocReconciliation, + processWorkspaceFileLiveDocReconciliationNow: mockProcessWorkspaceFileLiveDocReconciliationNow, +})) + vi.mock('@/lib/billing/storage', () => ({ decrementStorageUsageForBillingContextInTx: mockDecrementStorageUsageForBillingContextInTx, incrementStorageUsageForBillingContextInTx: mockIncrementStorageUsageForBillingContextInTx, @@ -166,9 +173,10 @@ describe('workspace file metadata and storage accounting', () => { mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined) mockDeleteFile.mockResolvedValue(undefined) mockEnqueueWorkspaceFileStorageCleanup.mockResolvedValue('cleanup-event-1') - mockMergeEditIntoLiveFileDoc.mockResolvedValue(undefined) + mockEnqueueWorkspaceFileLiveDocReconciliation.mockResolvedValue('live-doc-event-1') mockNotifyWorkspaceFilesChanged.mockResolvedValue(undefined) mockProcessWorkspaceFileStorageCleanupNow.mockResolvedValue('completed') + mockProcessWorkspaceFileLiveDocReconciliationNow.mockResolvedValue('completed') mockReplaceWorkspaceFileSecretProvenanceInTx.mockResolvedValue(undefined) }) @@ -740,7 +748,23 @@ describe('workspace file metadata and storage accounting', () => { const MD_ROW = { ...FILE_ROW, originalName: 'note.md', contentType: 'text/markdown' } - it('streams a markdown overwrite into any open collaborative editor (the shared merge chokepoint)', async () => { + it('transactionally enqueues a markdown overwrite for live-document reconciliation', async () => { + const transaction = { ...dbChainMock.db } + let committed = false + dbChainMockFns.transaction.mockImplementationOnce(async (callback) => { + const result = await callback(transaction) + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith( + transaction, + expect.objectContaining({ fileId: MD_ROW.id }) + ) + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).not.toHaveBeenCalled() + committed = true + return result + }) + mockProcessWorkspaceFileLiveDocReconciliationNow.mockImplementationOnce(async () => { + expect(committed).toBe(true) + return 'completed' + }) // Distinct updatedAt vs contentUpdatedAt so the assertion proves the merge carries the CONTENT // version (the persist If-Match token), not `updatedAt` — reverting that wiring would fail here. const updatedFile = { @@ -761,9 +785,14 @@ describe('workspace file metadata and storage accounting', () => { Buffer.from('# new content', 'utf-8') ) - expect(mockMergeEditIntoLiveFileDoc).toHaveBeenCalledWith(MD_ROW.id, '# new content', { + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(transaction, { + workspaceId: MD_ROW.workspaceId, + fileId: MD_ROW.id, version: updatedFile.contentUpdatedAt.getTime(), }) + expect(mockProcessWorkspaceFileLiveDocReconciliationNow).toHaveBeenCalledWith( + 'live-doc-event-1' + ) }) it('does NOT merge when syncLiveDoc is false (the relay persist / empty-shell opt-out)', async () => { @@ -781,7 +810,7 @@ describe('workspace file metadata and storage accounting', () => { { syncLiveDoc: false } ) - expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() }) it('does NOT merge a non-markdown write (the collaborative editor only renders markdown)', async () => { @@ -798,7 +827,57 @@ describe('workspace file metadata and storage accounting', () => { 'application/octet-stream' ) - expect(mockMergeEditIntoLiveFileDoc).not.toHaveBeenCalled() + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).not.toHaveBeenCalled() + }) + + it('enqueues an oversized markdown write so an older live generation is invalidated', async () => { + const size = PASTE_LIMITS.RICH_MARKDOWN_BYTES + 1 + const updatedFile = { ...MD_ROW, size, sizeBytes: size } + dbChainMockFns.limit.mockResolvedValueOnce([MD_ROW]).mockResolvedValueOnce([MD_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: MD_ROW.key }) + + await updateWorkspaceFileContent( + MD_ROW.workspaceId, + MD_ROW.id, + MD_ROW.userId, + Buffer.alloc(size) + ) + + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: MD_ROW.workspaceId, + fileId: MD_ROW.id, + version: updatedFile.contentUpdatedAt.getTime(), + }) + }) + + it('enqueues a markdown-to-binary replacement so an older live generation is invalidated', async () => { + const markdownByType = { ...MD_ROW, originalName: 'note.txt' } + const updatedFile = { + ...markdownByType, + contentType: 'application/octet-stream', + size: 12, + sizeBytes: 12, + } + dbChainMockFns.limit + .mockResolvedValueOnce([markdownByType]) + .mockResolvedValueOnce([markdownByType]) + dbChainMockFns.returning.mockResolvedValueOnce([updatedFile]) + mockUploadFile.mockResolvedValueOnce({ key: markdownByType.key }) + + await updateWorkspaceFileContent( + markdownByType.workspaceId, + markdownByType.id, + markdownByType.userId, + Buffer.alloc(12), + 'application/octet-stream' + ) + + expect(mockEnqueueWorkspaceFileLiveDocReconciliation).toHaveBeenCalledWith(expect.anything(), { + workspaceId: markdownByType.workspaceId, + fileId: markdownByType.id, + version: updatedFile.contentUpdatedAt.getTime(), + }) }) it('writes when the expectedUpdatedAt optimistic-concurrency guard matches', async () => { diff --git a/packages/realtime-protocol/src/file-doc.test.ts b/packages/realtime-protocol/src/file-doc.test.ts index 48fd7db5b8a..ca1d8a495d1 100644 --- a/packages/realtime-protocol/src/file-doc.test.ts +++ b/packages/realtime-protocol/src/file-doc.test.ts @@ -9,5 +9,7 @@ describe('FILE_DOC_TIMEOUTS ordering invariants', () => { // The relay's `/seed` fetch must finish before the client's readiness deadline lapses into its // read-only fallback, or a late-but-successful seed can never reach the client. expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) + expect(FILE_DOC_TIMEOUTS.seedRequestMs).toBeLessThan(FILE_DOC_TIMEOUTS.joinAckMs) + expect(FILE_DOC_TIMEOUTS.joinAckMs).toBeLessThan(FILE_DOC_TIMEOUTS.readinessDeadlineMs) }) }) diff --git a/packages/realtime-protocol/src/file-doc.ts b/packages/realtime-protocol/src/file-doc.ts index dde704bee73..c1cef7c4398 100644 --- a/packages/realtime-protocol/src/file-doc.ts +++ b/packages/realtime-protocol/src/file-doc.ts @@ -25,6 +25,13 @@ export const FILE_DOC_EVENTS = { LEAVE: 'leave-file-doc', /** Both directions: a framed Yjs message (binary), tagged by {@link FILE_DOC_MESSAGE_TYPE}. */ MESSAGE: 'file-doc-message', + /** + * Client → server: one idempotent batch of user-authored Yjs updates. Unlike the handshake and + * awareness channel, this event is acknowledged only after the shared stream accepts the batch. + */ + UPDATE: 'file-doc-update', + /** Server → client: the durable file was replaced outside this live document's generation. */ + INVALIDATED: 'file-doc-invalidated', /** * Server → client: the roster of collaborators currently in the document * ({@link FileDocPresence}), for the avatar stack. Identity is server-authenticated (from @@ -34,6 +41,11 @@ export const FILE_DOC_EVENTS = { PRESENCE: 'file-doc-presence', } as const +/** Schema assumed for peers from before schema negotiation was added. */ +export const FILE_DOC_LEGACY_SCHEMA_VERSION = 1 + +export const FILE_DOC_SCHEMA_VERSION = 1 + /** * The tag carried in the first varUint of a {@link FILE_DOC_EVENTS.MESSAGE} * payload — the standard Yjs websocket framing distinguishing a document-sync @@ -67,8 +79,8 @@ export const FILE_DOC_MESSAGE_TYPE = { * 1. **Never overwrite content with an unseeded doc.** The markdown-mirror autosave MUST be gated on * the document being both synced AND seeded — otherwise an empty/still-syncing doc could be saved * over the real file (the one true data-loss path). - * 2. **One provider per socket.** Destroy the previous provider before creating the next (document - * switch), so a stale provider's binary-frame listener can't apply another document's updates. + * 2. **One active file per shared socket.** Multiple providers may show the same file, but opening a + * different file must make older providers terminal before its unscoped binary frames can arrive. * 3. **Treat a fatal (`retryable: false`) join error as terminal.** Latch it and fall back to a * read-only view of the file's stored content — do not keep rejoining. The server auto-reclaims a * same-user client-id collision silently (the reconnecting socket succeeds), so `CLIENT_ID_IN_USE` @@ -121,10 +133,17 @@ export const FILE_DOC_TIMEOUTS = { seedRequestMs: 8_000, mergeRequestMs: 3_000, applyEditMs: 6_000, + joinAckMs: 10_000, + updateAckMs: 6_000, readinessDeadlineMs: 12_000, persistRequestMs: 8_000, } as const +export const FILE_DOC_LIMITS = { + /** Leaves framing and acknowledgement headroom under Socket.IO's 8 MiB event ceiling. */ + updateBytes: 6 * 1024 * 1024, +} as const + /** Client → server join request. `fileId` is the `workspace_files.id`. */ export interface JoinFileDocPayload { fileId: string @@ -134,6 +153,8 @@ export interface JoinFileDocPayload { * client — an authenticated peer cannot forge or clear another's presence. */ clientId: number + /** Optional during rolling deploys; absent peers use the original version-1 schema. */ + schemaVersion?: number } /** Server → client acceptance of a {@link FILE_DOC_EVENTS.JOIN}. */ @@ -141,6 +162,10 @@ export interface JoinFileDocSuccess { fileId: string /** The provider whose join was accepted. Optional while older relays are still deployed. */ clientId?: number + /** Whether this relay durably acknowledges client updates. Absent on older relays. */ + acknowledgedUpdates?: true + /** Durable version incorporated by the admitted generation; absent on older relays. */ + version?: number /** * The identity of the document this room holds ({@link FILE_DOC_SEED.docIdKey}), so a client can tell * "the room I left" from "a document built in its place" BEFORE it syncs. Absent for a room whose doc @@ -148,6 +173,8 @@ export interface JoinFileDocSuccess { * exactly the case where there is nothing to compare and the client proceeds. */ docId?: string + /** Optional while older relays are still deployed. */ + schemaVersion?: number } /** Server → client rejection of a {@link FILE_DOC_EVENTS.JOIN}. */ @@ -165,6 +192,38 @@ export interface LeaveFileDocPayload { fileId: string } +/** Server → client invalidation after a durable replacement that cannot merge into the rich editor. */ +export interface FileDocInvalidated { + fileId: string + message: string + /** Generation atomically removed by this invalidation, when one existed. */ + docId?: string + /** Durable replacement version; absent only on older relays. */ + version?: number +} + +/** A bounded, retry-safe batch of user-authored changes. */ +export interface FileDocUpdatePayload { + fileId: string + docId: string + updateId: string + update: Uint8Array +} + +export type FileDocUpdateAck = + | { status: 'accepted'; updateId: string } + | { + status: 'rejected' + updateId?: string + code: + | 'ACCESS_REVOKED' + | 'DOCUMENT_REPLACED' + | 'INVALID_UPDATE' + | 'NOT_JOINED' + | 'TEMPORARY_FAILURE' + retryable: boolean + } + /** One collaborator session in a {@link FileDocPresence} roster — server-authenticated identity. * Keyed per socket (session), not per user: the client excludes its OWN `socketId` and then * dedupes the rest per user for the avatar stack, so a second tab of the same account still