fix(file-search): back off indexing retries after database timeouts

apps/sim/background/workspace-file-search-index.test.ts

@@ -5,17 +5,20 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const mocks = vi.hoisted(() => ({
   indexWorkspaceFile: vi.fn(),
+  retry: vi.fn(),
   markFailed: vi.fn(),
   task: vi.fn((config: unknown) => config),
 }))
 
 vi.mock('@trigger.dev/sdk', () => ({ task: mocks.task }))
 vi.mock('@/lib/workspace-files/search/indexing', () => ({
   indexWorkspaceFileForSearch: mocks.indexWorkspaceFile,
+  getWorkspaceFileSearchRetry: mocks.retry,
   markWorkspaceFileSearchIndexFailed: mocks.markFailed,
 }))
 
 import {
+  FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS,
   FILE_SEARCH_INDEX_GLOBAL_CONCURRENCY,
   FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
 } from '@/lib/workspace-files/search/constants'
@@ -37,7 +40,7 @@ describe('workspace file search index task', () => {
       id: 'workspace-file-search-index',
       machine: 'medium-2x',
       maxDuration: FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
-      retry: { maxAttempts: 3 },
+      retry: { maxAttempts: FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS },
       queue: {
         name: 'workspace-file-search-index',
         concurrencyLimit: FILE_SEARCH_INDEX_GLOBAL_CONCURRENCY,
@@ -65,4 +68,15 @@ describe('workspace file search index task', () => {
     await workspaceFileSearchIndexTask.onFailure({ payload })
     expect(mocks.markFailed).toHaveBeenCalledWith(payload)
   })
+
+  it('lets the indexing retry policy schedule each failed attempt', async () => {
+    const error = new Error('statement timeout')
+    const decision = { retryAt: new Date('2026-08-29T12:02:00.000Z') }
+    mocks.retry.mockReturnValue(decision)
+
+    await expect(
+      workspaceFileSearchIndexTask.catchError({ error, ctx: { attempt: { number: 2 } } })
+    ).resolves.toBe(decision)
+    expect(mocks.retry).toHaveBeenCalledWith(error, 2)
+  })
 })

apps/sim/background/workspace-file-search-index.ts

@@ -1,9 +1,11 @@
 import { task } from '@trigger.dev/sdk'
 import {
+  FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS,
   FILE_SEARCH_INDEX_GLOBAL_CONCURRENCY,
   FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
 } from '@/lib/workspace-files/search/constants'
 import {
+  getWorkspaceFileSearchRetry,
   indexWorkspaceFileForSearch,
   markWorkspaceFileSearchIndexFailed,
   type WorkspaceFileSearchIndexPayload,
@@ -17,13 +19,15 @@ export const workspaceFileSearchIndexTask = task({
   id: 'workspace-file-search-index',
   machine: 'medium-2x',
   maxDuration: FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
-  retry: { maxAttempts: 3 },
+  /** The ceiling for capacity retries; `catchError` stops other failures sooner. */
+  retry: { maxAttempts: FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS },
   queue: {
     name: 'workspace-file-search-index',
     concurrencyLimit: FILE_SEARCH_INDEX_GLOBAL_CONCURRENCY,
   },
   run: (payload: WorkspaceFileSearchIndexPayload, { signal }) =>
     indexWorkspaceFileForSearch(payload, signal),
+  catchError: async ({ error, ctx }) => getWorkspaceFileSearchRetry(error, ctx.attempt.number),
   onFailure: async ({ payload }) => {
     await markWorkspaceFileSearchIndexFailed(payload)
   },

apps/sim/lib/workspace-files/search/README.md

@@ -14,7 +14,7 @@ Workers download and extract outside database transactions, then insert batches
 
 The chunk GIN index uses `fastupdate = off`. Each bounded insert updates the main index directly instead of appending to a shared pending list. With deferred updates enabled, even a small insert can cross the pending-list threshold and synchronously merge accumulated work from other files. Direct updates trade some bulk-write throughput for avoiding that foreground cleanup cliff. They do not eliminate normal index I/O, vacuum, or storage contention; the row and worker limits still apply. Bytes alone do not bound GIN posting updates: dense text can generate thousands of distinct keys in each chunk. The batch planner sums each chunk's distinct trigram count, including repeated keys across rows, and flushes before the next chunk exceeds the key target. A single 8 KiB chunk is always allowed to make progress even if word padding pushes its estimate slightly above the target; it is written alone. Storage validates the same budget before opening the transaction. These are write scheduling bounds, not file exclusions: chunk boundaries, complete-file publication, and the 25 MiB file coverage limit are unchanged. The estimate mirrors pg_trgm under `en_US.UTF-8`; it is a work estimate, not a latency guarantee. Capacity validation must still include dense text, concurrent writers, and an index working set larger than available cache. A batch that takes at least two seconds, including a canceled statement, is logged with its rows, bytes, and estimated key count. A failed query is reported to the task runner by error code only; Drizzle's message carries the bound file text.
 
-Indexing transactions have separate limits from search: ten seconds per statement, five seconds waiting for a lock, and thirty seconds total on PostgreSQL 17. The outer limit leaves time for ordinary statement cancellation and rollback instead of terminating the connection at the same ten-second deadline. PostgreSQL 16 uses the compatible idle-transaction guard. A canceled batch remains unpublished; the existing task retry starts a fresh fenced build, and cleanup retires the previous attempt. This does not automatically retry revisions already marked failed.
+Indexing transactions have separate limits from search: ten seconds per statement, five seconds waiting for a lock, and thirty seconds total on PostgreSQL 17. The outer limit leaves time for ordinary statement cancellation and rollback instead of terminating the connection at the same ten-second deadline. PostgreSQL 16 uses the compatible idle-transaction guard. A canceled batch remains unpublished; the task retry starts a fresh fenced build, and cleanup retires the previous attempt. One row's direct GIN insert is not interruptible, so when storage is saturated a single ordinary chunk can run well past the statement deadline and the cancellation lands only after it; smaller batches cannot prevent that. A statement, lock, or transaction timeout is therefore treated as missing database capacity rather than a bad file: the task retries it after about 2, 4, 8, 16, and 30 minutes (with jitter), six attempts in all, so the retries outlast a slow window instead of landing inside it. Other failures keep three attempts with the runner's short default delays. A run waiting to retry still holds one of its workspace's two outstanding dispatch slots. Only a revision that exhausts its attempts is marked failed; this does not automatically retry revisions already marked failed.
 
 The indexing task uses an isolated `medium-2x` Trigger worker (4 GB RAM). Document parsers can materialize expanded content before chunking, so source and extracted-text byte limits do not bound parser memory. Parser complexity guards and the worker's memory budget remain separate protections.
 

apps/sim/lib/workspace-files/search/constants.ts

@@ -78,6 +78,17 @@ export const FILE_SEARCH_INDEX_TRANSACTION_LIMITS = {
   transactionTimeout: 30 * 1000,
 } as const
 
+/** Attempts for failures other than database capacity cancellations. */
+export const FILE_SEARCH_INDEX_MAX_ATTEMPTS = 3
+/**
+ * Attempts when PostgreSQL cancels an indexing statement on a timeout. One row's direct GIN insert
+ * is not interruptible, so under storage saturation even a single ordinary chunk can outlive the
+ * statement deadline; smaller batches cannot help, only waiting out the slow window can.
+ */
+export const FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS = 6
+/** First capacity retry delay; later ones double up to the ceiling, about an hour in total. */
+export const FILE_SEARCH_INDEX_CAPACITY_RETRY_BASE_MS = 2 * 60 * 1000
+export const FILE_SEARCH_INDEX_CAPACITY_RETRY_MAX_MS = 30 * 60 * 1000
 export const FILE_SEARCH_INDEX_GLOBAL_CONCURRENCY = 10
 export const FILE_SEARCH_INDEX_WORKSPACE_OUTSTANDING = 2
 export const FILE_SEARCH_INDEX_MAX_OUTSTANDING = 100

apps/sim/lib/workspace-files/search/indexing.test.ts

@@ -24,24 +24,32 @@ vi.mock('@/lib/workspace-files/search/extract', () => ({
 }))
 
 import {
+  FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS,
+  FILE_SEARCH_INDEX_CAPACITY_RETRY_BASE_MS,
+  FILE_SEARCH_INDEX_CAPACITY_RETRY_MAX_MS,
+  FILE_SEARCH_INDEX_MAX_ATTEMPTS,
   FILE_SEARCH_INSERT_BATCH_BYTES,
   FILE_SEARCH_INSERT_BATCH_ROWS,
   FILE_SEARCH_MAX_SOURCE_BYTES,
   FILE_SEARCH_SLOW_INSERT_BATCH_MS,
 } from '@/lib/workspace-files/search/constants'
 import type { FileSearchChunk } from '@/lib/workspace-files/search/index-plan'
-import { indexWorkspaceFileForSearch } from '@/lib/workspace-files/search/indexing'
+import {
+  getWorkspaceFileSearchRetry,
+  indexWorkspaceFileForSearch,
+} from '@/lib/workspace-files/search/indexing'
 
 const logger = vi.mocked(createLogger).mock.results[
   vi.mocked(createLogger).mock.calls.findIndex(([name]) => name === 'WorkspaceFileSearchIndexer')
 ].value as { warn: ReturnType<typeof vi.fn> }
 
 const FILE_TEXT = 'confidential customer text'
 
-function statementTimeout(): DrizzleQueryError {
-  const driverError = Object.assign(new Error('canceling statement due to statement timeout'), {
-    code: '57014',
-  })
+function statementTimeout(
+  message = 'canceling statement due to statement timeout',
+  code = '57014'
+): DrizzleQueryError {
+  const driverError = Object.assign(new Error(message), { code })
   return new DrizzleQueryError(
     'insert into "workspace_file_search_chunk" values ($1)',
     [FILE_TEXT],
@@ -182,3 +190,70 @@ describe('complete-file indexing worker', () => {
     expect(mocks.publish).not.toHaveBeenCalled()
   })
 })
+
+describe('indexing retry policy', () => {
+  const now = Date.parse('2026-01-01T00:00:00.000Z')
+
+  /** The error the task runner receives: the redacted wrapper the worker throws. */
+  async function thrownBy(error: unknown): Promise<unknown> {
+    vi.clearAllMocks()
+    mocks.begin.mockResolvedValue({ id: 'build', ...payload })
+    mocks.file.mockResolvedValue({
+      name: 'notes.txt',
+      size: 100,
+      contentUpdatedAt: new Date(payload.sourceContentUpdatedAt),
+    })
+    mocks.load.mockResolvedValue({ buffer: Buffer.from('a\n') })
+    mocks.extract.mockResolvedValue({ text: 'a\n', lineCount: 1 })
+    mocks.append.mockRejectedValue(error)
+    return indexWorkspaceFileForSearch(payload, signal).catch((thrown) => thrown)
+  }
+
+  function delayOf(decision: ReturnType<typeof getWorkspaceFileSearchRetry>): number {
+    if (!decision || !('retryAt' in decision)) throw new Error('expected a scheduled retry')
+    return decision.retryAt.getTime() - now
+  }
+
+  it.each([
+    ['statement timeout', 'canceling statement due to statement timeout', '57014'],
+    ['lock timeout', 'canceling statement due to lock timeout', '55P03'],
+  ])('waits minutes, not seconds, after a %s', async (_label, message, code) => {
+    const thrown = await thrownBy(statementTimeout(message, code))
+    const first = delayOf(getWorkspaceFileSearchRetry(thrown, 1, now))
+    expect(first).toBeGreaterThanOrEqual(FILE_SEARCH_INDEX_CAPACITY_RETRY_BASE_MS * 0.8)
+    expect(first).toBeLessThanOrEqual(FILE_SEARCH_INDEX_CAPACITY_RETRY_BASE_MS * 1.2)
+  })
+
+  it('backs capacity retries off to a ceiling and spans a slow window', async () => {
+    const thrown = await thrownBy(statementTimeout())
+    let total = 0
+    for (let attempt = 1; attempt < FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS; attempt++) {
+      const delay = delayOf(getWorkspaceFileSearchRetry(thrown, attempt, now))
+      expect(delay).toBeLessThanOrEqual(FILE_SEARCH_INDEX_CAPACITY_RETRY_MAX_MS * 1.2)
+      total += delay
+    }
+    expect(total).toBeGreaterThanOrEqual(45 * 60 * 1000)
+  })
+
+  it('stops capacity retries at their attempt ceiling', async () => {
+    const thrown = await thrownBy(statementTimeout())
+    expect(
+      getWorkspaceFileSearchRetry(thrown, FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS, now)
+    ).toEqual({ skipRetrying: true })
+  })
+
+  it('keeps the short default retries and attempt count for other failures', () => {
+    const parserFailure = new Error('parser failed')
+    for (let attempt = 1; attempt < FILE_SEARCH_INDEX_MAX_ATTEMPTS; attempt++) {
+      expect(getWorkspaceFileSearchRetry(parserFailure, attempt, now)).toBeUndefined()
+    }
+    expect(getWorkspaceFileSearchRetry(parserFailure, FILE_SEARCH_INDEX_MAX_ATTEMPTS, now)).toEqual(
+      { skipRetrying: true }
+    )
+  })
+
+  it('treats a user cancellation as an ordinary failure', async () => {
+    const thrown = await thrownBy(statementTimeout('canceling statement due to user request'))
+    expect(getWorkspaceFileSearchRetry(thrown, 1, now)).toBeUndefined()
+  })
+})

apps/sim/lib/workspace-files/search/indexing.ts

@@ -1,10 +1,15 @@
 import { Buffer } from 'node:buffer'
 import { createLogger } from '@sim/logger'
-import { describeError } from '@sim/utils/errors'
+import { describeError, getPostgresCancellationReason } from '@sim/utils/errors'
+import { backoffWithJitter } from '@sim/utils/retry'
 import { redactDatabaseQueryError } from '@/lib/core/errors/database-query-error'
 import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace'
 import {
+  FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS,
+  FILE_SEARCH_INDEX_CAPACITY_RETRY_BASE_MS,
+  FILE_SEARCH_INDEX_CAPACITY_RETRY_MAX_MS,
+  FILE_SEARCH_INDEX_MAX_ATTEMPTS,
   FILE_SEARCH_MAX_SOURCE_BYTES,
   FILE_SEARCH_SLOW_INSERT_BATCH_MS,
 } from '@/lib/workspace-files/search/constants'
@@ -154,3 +159,33 @@ export async function markWorkspaceFileSearchIndexFailed(
     return
   await failFileSearchRevision(parseRevision(payload), payload.dispatchToken)
 }
+
+const CAPACITY_CANCELLATIONS = new Set(['statement_timeout', 'lock_timeout', 'transaction_timeout'])
+
+export type WorkspaceFileSearchRetryDecision =
+  | { retryAt: Date }
+  | { skipRetrying: true }
+  | undefined
+
+/**
+ * Chooses the next attempt after `attempt` (1-based) failed. A statement, lock, or transaction
+ * timeout means the database had no capacity for this build right now, not that the file is bad:
+ * those back off for minutes so the retries outlast a slow window instead of all landing inside
+ * it. Anything else keeps the ordinary short retries. `undefined` keeps the runner's default delay.
+ */
+export function getWorkspaceFileSearchRetry(
+  error: unknown,
+  attempt: number,
+  now = Date.now()
+): WorkspaceFileSearchRetryDecision {
+  const reason = getPostgresCancellationReason(error)
+  if (reason && CAPACITY_CANCELLATIONS.has(reason)) {
+    if (attempt >= FILE_SEARCH_INDEX_CAPACITY_MAX_ATTEMPTS) return { skipRetrying: true }
+    const delayMs = backoffWithJitter(attempt, null, {
+      baseMs: FILE_SEARCH_INDEX_CAPACITY_RETRY_BASE_MS,
+      maxMs: FILE_SEARCH_INDEX_CAPACITY_RETRY_MAX_MS,
+    })
+    return { retryAt: new Date(now + delayMs) }
+  }
+  return attempt >= FILE_SEARCH_INDEX_MAX_ATTEMPTS ? { skipRetrying: true } : undefined
+}