diff --git a/apps/sim/lib/execution/payloads/limits.ts b/apps/sim/lib/execution/payloads/limits.ts index 4166e14d66c..e06cff1d4ea 100644 --- a/apps/sim/lib/execution/payloads/limits.ts +++ b/apps/sim/lib/execution/payloads/limits.ts @@ -1,4 +1,5 @@ export const MAX_DURABLE_LARGE_VALUE_BYTES = 64 * 1024 * 1024 +export const MAX_TRACE_ARCHIVE_BYTES = 512 * 1024 * 1024 export const MAX_INLINE_MATERIALIZATION_BYTES = 16 * 1024 * 1024 export const MAX_FUNCTION_FILE_BYTES = 64 * 1024 * 1024 export const MAX_FUNCTION_INLINE_BYTES = 10 * 1024 * 1024 diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index 8e8035a90f7..46ce92ecdfc 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -75,12 +75,15 @@ function getLogger(options: ExecutionMaterializationContext): Logger { return options.logger ?? logger } -export function assertDurableLargeValueSize(size: number): void { - if (size > MAX_DURABLE_LARGE_VALUE_BYTES) { +export function assertDurableLargeValueSize( + size: number, + limitBytes = MAX_DURABLE_LARGE_VALUE_BYTES +): void { + if (size > limitBytes) { throw new ExecutionResourceLimitError({ resource: 'execution_payload_bytes', attemptedBytes: size, - limitBytes: MAX_DURABLE_LARGE_VALUE_BYTES, + limitBytes, }) } } diff --git a/apps/sim/lib/execution/payloads/store.test.ts b/apps/sim/lib/execution/payloads/store.test.ts index 60bcdcf05d2..fa830cc438c 100644 --- a/apps/sim/lib/execution/payloads/store.test.ts +++ b/apps/sim/lib/execution/payloads/store.test.ts @@ -8,12 +8,19 @@ import { clearLargeValueCacheForTests, materializeLargeValueRefSync, } from '@/lib/execution/payloads/cache' -import { MAX_DURABLE_LARGE_VALUE_BYTES } from '@/lib/execution/payloads/limits' +import { + MAX_DURABLE_LARGE_VALUE_BYTES, + MAX_TRACE_ARCHIVE_BYTES, +} from '@/lib/execution/payloads/limits' import { readLargeValueRefFromStorage, readUserFileContent, } from '@/lib/execution/payloads/materialization.server' -import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store' +import { + materializeLargeValueRef, + storeExecutionTraceArchive, + storeLargeValue, +} from '@/lib/execution/payloads/store' import { EXECUTION_RESOURCE_LIMIT_CODE } from '@/lib/execution/resource-errors' const { @@ -350,6 +357,51 @@ describe('large execution payload store', () => { requireDurable: true, }) ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE }) + expect(mockUploadFile).not.toHaveBeenCalled() + }) + + it('admits a trace archive at its separate size cap with durable ownership', async () => { + const ref = await storeExecutionTraceArchive({}, '{}', MAX_TRACE_ARCHIVE_BYTES, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + }) + + expect(mockUploadFile).toHaveBeenCalledOnce() + expect(mockRegisterLargeValueOwner).toHaveBeenCalledWith( + expect.objectContaining({ key: ref.key, size: MAX_TRACE_ARCHIVE_BYTES }), + [] + ) + expect(materializeLargeValueRefSync(ref, { executionId: 'execution-1' })).toBeUndefined() + }) + + it('rejects archives above the trace cap before upload or metadata writes', async () => { + await expect( + storeExecutionTraceArchive({}, '{}', MAX_TRACE_ARCHIVE_BYTES + 1, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + }) + ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE }) + expect(mockUploadFile).not.toHaveBeenCalled() + expect(mockRegisterLargeValueOwner).not.toHaveBeenCalled() + }) + + it('requires durable storage for trace archives even if the caller disables it', async () => { + mockUploadFile.mockRejectedValueOnce(new Error('storage unavailable')) + + await expect( + storeExecutionTraceArchive({}, '{}', 2, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + requireDurable: false, + }) + ).rejects.toThrow('storage unavailable') + expect(mockRegisterLargeValueOwner).not.toHaveBeenCalled() }) it('bounds explicit server-side materialization', async () => { diff --git a/apps/sim/lib/execution/payloads/store.ts b/apps/sim/lib/execution/payloads/store.ts index 0cb1e14385e..7b67fa8094d 100644 --- a/apps/sim/lib/execution/payloads/store.ts +++ b/apps/sim/lib/execution/payloads/store.ts @@ -9,6 +9,10 @@ import { type LargeValueKind, type LargeValueRef, } from '@/lib/execution/payloads/large-value-ref' +import { + MAX_DURABLE_LARGE_VALUE_BYTES, + MAX_TRACE_ARCHIVE_BYTES, +} from '@/lib/execution/payloads/limits' import { assertDurableLargeValueSize, assertInlineMaterializationSize, @@ -164,7 +168,33 @@ export async function storeLargeValue( size: number, context: LargeValueStoreContext ): Promise { - assertDurableLargeValueSize(size) + return persistLargeValue(value, json, size, context, MAX_DURABLE_LARGE_VALUE_BYTES) +} + +/** Stores a completed execution archive with a larger cap than individual workflow values. */ +export async function storeExecutionTraceArchive( + value: Record, + json: string, + size: number, + context: LargeValueStoreContext +): Promise { + return persistLargeValue( + value, + json, + size, + { ...context, requireDurable: true }, + MAX_TRACE_ARCHIVE_BYTES + ) +} + +async function persistLargeValue( + value: unknown, + json: string, + size: number, + context: LargeValueStoreContext, + limitBytes: number +): Promise { + assertDurableLargeValueSize(size, limitBytes) const referencedKeys = collectLargeValueKeys(value) const id = `lv_${generateShortId(12)}` let key = await persistValue(id, json, context) diff --git a/apps/sim/lib/logs/execution/trace-store-storage.test.ts b/apps/sim/lib/logs/execution/trace-store-storage.test.ts new file mode 100644 index 00000000000..00ebdc1cef7 --- /dev/null +++ b/apps/sim/lib/logs/execution/trace-store-storage.test.ts @@ -0,0 +1,134 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' +import { + MAX_DURABLE_LARGE_VALUE_BYTES, + MAX_TRACE_ARCHIVE_BYTES, +} from '@/lib/execution/payloads/limits' +import { storeLargeValue } from '@/lib/execution/payloads/store' +import { EXECUTION_RESOURCE_LIMIT_CODE } from '@/lib/execution/resource-errors' +import { + externalizeExecutionData, + materializeExecutionData, + TRACE_STORE_REF_KEY, +} from '@/lib/logs/execution/trace-store' + +const { mockUploadFile, mockDownloadFile, mockRegisterOwner, mockAddReference } = vi.hoisted( + () => ({ + mockUploadFile: vi.fn(), + mockDownloadFile: vi.fn(), + mockRegisterOwner: vi.fn(), + mockAddReference: vi.fn(), + }) +) + +/** Scale the two caps down to exercise real serialization and storage reads with small fixtures. */ +vi.mock('@/lib/execution/payloads/limits', async (importOriginal) => ({ + ...(await importOriginal()), + MAX_DURABLE_LARGE_VALUE_BYTES: 1024, + MAX_TRACE_ARCHIVE_BYTES: 4096, +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { uploadFile: mockUploadFile, downloadFile: mockDownloadFile }, +})) + +vi.mock('@/lib/execution/payloads/large-value-metadata', () => ({ + registerLargeValueOwner: mockRegisterOwner, + addLargeValueReference: mockAddReference, +})) + +const CONTEXT = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', +} + +beforeEach(() => { + vi.clearAllMocks() + clearLargeValueCacheForTests() + mockRegisterOwner.mockResolvedValue(true) + mockUploadFile.mockImplementation(async ({ customKey, file }) => { + mockDownloadFile.mockResolvedValue(file) + return { key: customKey } + }) +}) + +describe('trace archive storage round trip', () => { + it('uploads an archive above the ordinary value cap and reads it back after a cache miss', async () => { + const data = { + traceSpans: [{ id: 'span-1', output: 'é'.repeat(MAX_DURABLE_LARGE_VALUE_BYTES) }], + traceSpanCount: 1, + hasTraceSpans: true, + } + const json = JSON.stringify(data) + const size = Buffer.byteLength(json, 'utf8') + expect(size).toBeGreaterThan(MAX_DURABLE_LARGE_VALUE_BYTES) + expect(size).toBeLessThan(MAX_TRACE_ARCHIVE_BYTES) + + await expect(storeLargeValue(data, json, size, CONTEXT)).rejects.toMatchObject({ + code: EXECUTION_RESOURCE_LIMIT_CODE, + }) + expect(mockUploadFile).not.toHaveBeenCalled() + + const slim = await externalizeExecutionData(data, CONTEXT, { throwOnError: true }) + expect(slim).toEqual({ + [TRACE_STORE_REF_KEY]: expect.objectContaining({ size, key: expect.any(String) }), + traceSpanCount: 1, + hasTraceSpans: true, + }) + expect(mockRegisterOwner).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: CONTEXT.workspaceId, + workflowId: CONTEXT.workflowId, + executionId: CONTEXT.executionId, + size, + }), + [] + ) + clearLargeValueCacheForTests() + + await expect(materializeExecutionData(slim, CONTEXT)).resolves.toEqual(data) + expect(mockDownloadFile).toHaveBeenCalledExactlyOnceWith({ + key: expect.any(String), + context: 'execution', + maxBytes: size, + }) + expect(mockAddReference).not.toHaveBeenCalled() + }) + + it('rejects an over-limit archive before uploading it', async () => { + await expect( + externalizeExecutionData( + { traceSpans: [{ output: 'x'.repeat(MAX_TRACE_ARCHIVE_BYTES) }] }, + CONTEXT, + { throwOnError: true } + ) + ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE }) + expect(mockUploadFile).not.toHaveBeenCalled() + expect(mockRegisterOwner).not.toHaveBeenCalled() + }) + + it('bounds stored archive reads even if a reference declares a larger size', async () => { + const data = { traceSpans: [], hasTraceSpans: false } + const slim = await externalizeExecutionData(data, CONTEXT, { throwOnError: true }) + clearLargeValueCacheForTests() + + await expect( + materializeExecutionData( + { + ...slim, + [TRACE_STORE_REF_KEY]: { + ...(slim[TRACE_STORE_REF_KEY] as Record), + size: MAX_TRACE_ARCHIVE_BYTES + 1, + }, + }, + CONTEXT + ) + ).resolves.toEqual({ hasTraceSpans: false }) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/logs/execution/trace-store.test.ts b/apps/sim/lib/logs/execution/trace-store.test.ts index 5707e24be19..614147802ab 100644 --- a/apps/sim/lib/logs/execution/trace-store.test.ts +++ b/apps/sim/lib/logs/execution/trace-store.test.ts @@ -3,13 +3,17 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { decryptSecretMock, materializeLargeValueRefMock, storeLargeValueMock, mockLogger } = - vi.hoisted(() => ({ - decryptSecretMock: vi.fn(), - materializeLargeValueRefMock: vi.fn(), - storeLargeValueMock: vi.fn(), - mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, - })) +const { + decryptSecretMock, + materializeLargeValueRefMock, + storeExecutionTraceArchiveMock, + mockLogger, +} = vi.hoisted(() => ({ + decryptSecretMock: vi.fn(), + materializeLargeValueRefMock: vi.fn(), + storeExecutionTraceArchiveMock: vi.fn(), + mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})) vi.mock('@sim/logger', () => ({ createLogger: () => mockLogger, @@ -21,7 +25,7 @@ vi.mock('@/lib/core/security/encryption', () => ({ vi.mock('@/lib/execution/payloads/store', () => ({ materializeLargeValueRef: materializeLargeValueRefMock, - storeLargeValue: storeLargeValueMock, + storeExecutionTraceArchive: storeExecutionTraceArchiveMock, })) import { @@ -54,7 +58,7 @@ describe('execution data storage', () => { it('propagates the original storage failure for strict backfills', async () => { const cause = new Error('column "size_bytes" does not exist') const error = new Error('Failed query', { cause }) - storeLargeValueMock.mockRejectedValueOnce(error) + storeExecutionTraceArchiveMock.mockRejectedValueOnce(error) await expect( externalizeExecutionData({ traceSpans: [] }, CONTEXT, { throwOnError: true }) @@ -70,12 +74,12 @@ describe('execution data storage', () => { { throwOnError: true } ) ).rejects.toThrow('Trace storage requires workspaceId, workflowId, and userId') - expect(storeLargeValueMock).not.toHaveBeenCalled() + expect(storeExecutionTraceArchiveMock).not.toHaveBeenCalled() }) it('preserves inline completion data and logs the underlying database error', async () => { const data = { traceSpans: [] } - storeLargeValueMock.mockRejectedValueOnce( + storeExecutionTraceArchiveMock.mockRejectedValueOnce( new Error('Failed query\nparams: private-payload', { cause: new Error('permission denied for table workspace_files'), }) @@ -100,7 +104,7 @@ describe('execution data storage', () => { executionId: 'execution-1', preview: { unsafe: 'must-not-remain-inline' }, } as const - storeLargeValueMock.mockResolvedValue(ref) + storeExecutionTraceArchiveMock.mockResolvedValue(ref) materializeLargeValueRefMock.mockRejectedValue(new Error('object unavailable')) const slim = await externalizeExecutionData( diff --git a/apps/sim/lib/logs/execution/trace-store.ts b/apps/sim/lib/logs/execution/trace-store.ts index 996556ee7d7..7162ecc5456 100644 --- a/apps/sim/lib/logs/execution/trace-store.ts +++ b/apps/sim/lib/logs/execution/trace-store.ts @@ -2,7 +2,11 @@ import { createLogger } from '@sim/logger' import { describeError, toError } from '@sim/utils/errors' import { isRecordLike, omit } from '@sim/utils/object' import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' -import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store' +import { MAX_TRACE_ARCHIVE_BYTES } from '@/lib/execution/payloads/limits' +import { + materializeLargeValueRef, + storeExecutionTraceArchive, +} from '@/lib/execution/payloads/store' import { FunctionalOutputsUnavailableError } from '@/lib/logs/execution/functional-outputs' import { projectTraceSpansForSecrets } from '@/lib/logs/execution/trace-secret-projection' import type { TraceSpan } from '@/lib/logs/types' @@ -254,15 +258,12 @@ export async function externalizeExecutionData( const json = JSON.stringify(executionData) const size = Buffer.byteLength(json, 'utf8') - // storeLargeValue persists to the execution bucket with a conforming key and - // registers owner + dependency closure (trace -> nested span large values), - // so GC keeps nested children alive while this run's log row exists. - const ref = await storeLargeValue(executionData, json, size, { + /** Register the archive owner and dependencies so nested span values survive with the log. */ + const ref = await storeExecutionTraceArchive(executionData, json, size, { workspaceId, workflowId, executionId, userId, - requireDurable: true, }) const { preview: _preview, ...slimRef } = ref @@ -316,7 +317,7 @@ export async function materializeExecutionData( workspaceId: context.workspaceId, workflowId, executionId: context.executionId, - maxBytes: ref.size, + maxBytes: Math.min(ref.size, MAX_TRACE_ARCHIVE_BYTES), // Read-only: the value is already referenced by its own execution; don't // re-register (or fail) on every view/export. trackReference: false, diff --git a/apps/sim/scripts/backfill-trace-spans.test.ts b/apps/sim/scripts/backfill-trace-spans.test.ts index 57c0ae05979..c8fe20caaa4 100644 --- a/apps/sim/scripts/backfill-trace-spans.test.ts +++ b/apps/sim/scripts/backfill-trace-spans.test.ts @@ -2,7 +2,10 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { MAX_DURABLE_LARGE_VALUE_BYTES } from '@/lib/execution/payloads/limits' +import { + MAX_DURABLE_LARGE_VALUE_BYTES, + MAX_TRACE_ARCHIVE_BYTES, +} from '@/lib/execution/payloads/limits' const { mockPrimaryRead, @@ -362,15 +365,60 @@ describe('trace backfill', () => { { ...candidate, executionData: null, - payloadBytes: MAX_DURABLE_LARGE_VALUE_BYTES + 1, + payloadBytes: MAX_TRACE_ARCHIVE_BYTES + 1, }, ]) - await expect(backfillTraceStorage(options)).rejects.toThrow('backfill limit') + await expect(backfillTraceStorage(options)).rejects.toThrow('trace archive limit') expect(mockDataRead).toHaveBeenCalledTimes(2) expect(mockExternalize).not.toHaveBeenCalled() expect(mockTransaction).not.toHaveBeenCalled() }) + it.each([MAX_DURABLE_LARGE_VALUE_BYTES + 1, MAX_TRACE_ARCHIVE_BYTES])( + 'uploads and commits a %i-byte archive without skipping it', + async (payloadBytes) => { + const large = { ...candidate, payloadBytes } + mockDataRead + .mockResolvedValueOnce([candidateMetadata]) + .mockResolvedValueOnce([large]) + .mockResolvedValueOnce([large]) + + await expect(backfillTraceStorage(options)).resolves.toEqual({ migrated: 1 }) + expect(mockExternalize).toHaveBeenCalledOnce() + expect(mockUpdate).toHaveBeenCalledOnce() + expect(mockReplaceReferences).toHaveBeenCalledOnce() + expect(mockInfo).toHaveBeenCalledWith( + 'Backfill checkpoint', + expect.objectContaining(candidateMetadata) + ) + } + ) + + it('requires enough byte budget before fetching a larger archive and preserves the checkpoint', async () => { + const cursor = { + version: 1 as const, + order: options.order, + before: options.before, + startedAt: '2025-01-01T00:00:00.123456Z', + id: 'previous-log', + } + mockDataRead + .mockResolvedValueOnce([candidateMetadata]) + .mockResolvedValueOnce([ + { id: candidate.id, payloadBytes: MAX_DURABLE_LARGE_VALUE_BYTES + 1 }, + ]) + + await expect(backfillTraceStorage({ ...options, maxInFlightMiB: 64, cursor })).rejects.toThrow( + 'increase the byte budget' + ) + expect(mockDataRead).toHaveBeenCalledTimes(2) + expect(mockExternalize).not.toHaveBeenCalled() + expect(mockTransaction).not.toHaveBeenCalled() + const checkpoints = mockInfo.mock.calls.filter(([message]) => message === 'Backfill checkpoint') + expect(checkpoints.length).toBeGreaterThan(0) + expect(checkpoints.every(([, value]) => value.id === cursor.id)).toBe(true) + }) + it('fails before uploading when the payload grows past its reserved capacity', async () => { mockDataRead .mockResolvedValueOnce([candidateMetadata]) diff --git a/apps/sim/scripts/backfill-trace-spans.ts b/apps/sim/scripts/backfill-trace-spans.ts index 0862e0aee74..22e9b89b60e 100644 --- a/apps/sim/scripts/backfill-trace-spans.ts +++ b/apps/sim/scripts/backfill-trace-spans.ts @@ -27,7 +27,9 @@ * failure after draining active workers; reruns skip committed rows. * Reports migrated rows, throughput, and elapsed time every five seconds. * Concurrency accepts 1–512 workers; it does not set a rows-per-second target. - * Payload reads share a serialized-byte budget (512 MiB by default). Parsed + * Trace archives are capped at 512 MiB; individual workflow values keep their + * separate 64 MiB cap. Payload reads share a serialized-byte budget (512 MiB + * by default), so larger archives reduce effective concurrency. Parsed * objects, serialization copies, and the shared cache use additional memory. * Reports RSS and cumulative average timings per stage without counting rows. * SIGINT/SIGTERM stop scheduling and drain active writes. A partial page never @@ -56,7 +58,7 @@ import { collectLargeValueReferenceKeys, replaceLargeValueReferenceKeysWithClient, } from '@/lib/execution/payloads/large-value-metadata' -import { MAX_DURABLE_LARGE_VALUE_BYTES } from '@/lib/execution/payloads/limits' +import { MAX_TRACE_ARCHIVE_BYTES } from '@/lib/execution/payloads/limits' import { externalizeExecutionData, stripSpanCosts, @@ -165,10 +167,7 @@ export function parseArgs(argv: string[]): Options { if (options.concurrency > MAX_CONCURRENCY) { throw new Error(`--concurrency must be between 1 and ${MAX_CONCURRENCY}`) } - if ( - options.maxInFlightMiB < MAX_DURABLE_LARGE_VALUE_BYTES / MIB || - options.maxInFlightMiB > 4096 - ) { + if (options.maxInFlightMiB < 64 || options.maxInFlightMiB > 4096) { throw new Error('--max-in-flight-mib must be between 64 and 4096') } if (options.cursor) { @@ -385,9 +384,14 @@ export async function backfillTraceStorage( .limit(rows.length) ) for (const { id, payloadBytes } of sizes) { - if (payloadBytes > MAX_DURABLE_LARGE_VALUE_BYTES) { + if (payloadBytes > MAX_TRACE_ARCHIVE_BYTES) { throw new Error( - `Execution log ${id} exceeds the ${MAX_DURABLE_LARGE_VALUE_BYTES}-byte backfill limit` + `Execution log ${id} is ${payloadBytes} bytes, exceeding the ${MAX_TRACE_ARCHIVE_BYTES}-byte trace archive limit` + ) + } + if (payloadBytes > options.maxInFlightMiB * MIB) { + throw new Error( + `Execution log ${id} is ${payloadBytes} bytes, exceeding --max-in-flight-mib=${options.maxInFlightMiB}; increase the byte budget to migrate it` ) } }