v0.8.45: tune autovacuum, increase indexing worker memory

.agents/skills/ship/SKILL.md

@@ -112,12 +112,19 @@ The repo is public. **Everything you publish — title, description, commit mess
 
 Describe the bug by its mechanism, not by how you found it. "Expired OAuth credentials fail to refresh in the worker" — not "the Sheets canary failed at 16:31Z for workspace abc-123". Aggregate counts are fine once detached from the tenant ("1,379 PDFs failed"); the same number attributed to a named customer is not. Replace real examples with placeholders (`<real sheet name>`) rather than cutting them — the illustration is usually the useful part.
 
-**Scrub before publishing, not after** — a leak is public the instant it posts, and editing later does not unsend the notification email. This applies to every PR you open, including ones created directly with `gh pr create` rather than through this skill. Grep the title, body, and `git log origin/staging..HEAD` before publishing:
+**Measurements are not the problem; absolute production scale is.** Keep the numbers that justify a change — durations, ratios, before/after timings, test and audit counts. They are the evidence a reviewer needs, and stripping them makes the rationale unfalsifiable. What does not belong is anything that sizes production or a tenant: table and index byte sizes, row/chunk/document totals, dead-tuple counts, buffer and heap-fetch counts, worker or instance counts. "Visiting four times as many tuples took 5.1s and 9.7s on consecutive runs" is fine; "on a 132k-chunk index" or "reclaims ~19 GB" is not. The same rule applies to code comments and migration comments, which are published exactly like a PR body — this is the most commonly missed case, because they do not feel like publishing.
+
+**Scrub before publishing, not after** — a leak is public the instant it posts, and editing later does not unsend the notification email. This applies to every PR you open, including ones created directly with `gh pr create` rather than through this skill. Grep the title, body, `git log origin/staging..HEAD`, AND the diff itself before publishing:
 
 ```bash
+# identities, IDs, infrastructure
 grep -niE 'customer-or-company-name|@[a-z0-9.-]+\.(com|io|ai)|[0-9a-f]{8}-[0-9a-f]{4}-|\.sharepoint\.com|arn:aws|https?://[a-z0-9.-]*\.internal'
+# absolute production scale — byte sizes, k/M-scale entity counts, 7-figure totals
+grep -niE '[0-9][0-9.,]* ?(TB|GB)\b|[0-9]+(\.[0-9]+)?[kKmM][- ](row|chunk|document|vector|tuple|doc)|[0-9]{1,3}(,[0-9]{3}){2,}'
 ```
 
+The second pattern deliberately allows ordinary engineering numbers (`5.1s`, `46 audits`, `2,921 tests`) and flags only production sizing.
+
 ## PR Description Format
 
 Use this exact template in the user's voice (concise, bullet points):

.github/workflows/publish-sim-cli.yml

@@ -20,13 +20,6 @@ concurrency:
 
 jobs:
   publish-npm:
-    # Job-level, not on the build step: `bun publish` runs `prepublishOnly`,
-    # which rebuilds `dist` a second time, and that second build is the one
-    # that ships. A build without the token reports nothing. See
-    # docs/cli/usage-data.
-    env:
-      SIM_CLI_TELEMETRY_KEY: ${{ vars.SIM_CLI_TELEMETRY_KEY }}
-      SIM_CLI_TELEMETRY_HOST: ${{ vars.SIM_CLI_TELEMETRY_HOST }}
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
     steps:
@@ -73,8 +66,15 @@ jobs:
         working-directory: packages/sim-cli
         run: bun run type-check
 
+      # The usage-reporting destination is baked into the bundle at build time
+      # (docs/cli/usage-data), so only the two steps that build it get the
+      # token. Every other step, the tests above in particular, runs without
+      # it: a test that runs a real command must never report to production.
       - name: Build package
         working-directory: packages/sim-cli
+        env:
+          SIM_CLI_TELEMETRY_KEY: ${{ vars.SIM_CLI_TELEMETRY_KEY }}
+          SIM_CLI_TELEMETRY_HOST: ${{ vars.SIM_CLI_TELEMETRY_HOST }}
         run: bun run build
 
       - name: Resolve release channel
@@ -140,11 +140,15 @@ jobs:
             exit 1
           fi
 
+      # `bun publish` runs `prepublishOnly`, which rebuilds `dist`, and that
+      # rebuild is what ships, so it needs the token as well.
       - name: Publish to npm
         working-directory: packages/sim-cli
         env:
           NPM_CONFIG_TOKEN: ${{ secrets.NPM_TOKEN }}
           NPM_TAG: ${{ steps.release.outputs.tag }}
+          SIM_CLI_TELEMETRY_KEY: ${{ vars.SIM_CLI_TELEMETRY_KEY }}
+          SIM_CLI_TELEMETRY_HOST: ${{ vars.SIM_CLI_TELEMETRY_HOST }}
         run: bun publish --access public --tag "$NPM_TAG" --no-save
 
       - name: Summarize release

.github/workflows/test-build.yml

@@ -94,11 +94,14 @@ jobs:
         working-directory: packages/db
         run: bun run db:migrate
 
-      - name: Verify retired-column contract migration in PostgreSQL
+      - name: Verify schema contract migrations in PostgreSQL
         working-directory: packages/db
         env:
-          RETIRED_COLUMNS_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
-        run: bunx vitest run scripts/retired-columns.postgres.test.ts
+          MIGRATION_CONTRACT_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
+        run: >-
+          bunx vitest run
+          scripts/retired-columns.postgres.test.ts
+          scripts/connector-sync-schedule-precision.postgres.test.ts
 
       - name: Verify OAuth lifecycle and SCIM membership guards in PostgreSQL
         working-directory: apps/sim

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -212,11 +212,14 @@ describe('Knowledge Search Utils', () => {
   describe('handleTagAndVectorSearch', () => {
     it('returns only bounded ranked rows without first materializing every matching tag ID', async () => {
       resetDbChainMock()
-      queueTableRows(
-        schemaMock.embedding,
-        Array.from({ length: 201 }, (_, index) => ({ id: `candidate-${index}` }))
-      )
-      queueTableRows(schemaMock.embedding, [makeResult('second', 0.2), makeResult('first', 0.1)])
+      dbChainMockFns.execute.mockImplementation(async (query) => {
+        const statement = (query as { toSQL: () => { sql: string } }).toSQL().sql
+        if (statement.includes('AS visible')) return []
+        if (statement.includes(') + 0 LIMIT')) return [{ id: 'first' }, { id: 'second' }]
+        if (statement.includes('WITH scored_search_candidates'))
+          return [makeResult('second', 0.2), makeResult('first', 0.1)]
+        return [{ id: 'doc-first' }, { id: 'doc-second' }]
+      })
       queueTableRows(schemaMock.embedding, [makeResult('second', 0.2), makeResult('first', 0.1)])
 
       const results = await handleTagAndVectorSearch({
@@ -231,11 +234,13 @@ describe('Knowledge Search Utils', () => {
       })
 
       expect(results.map((row) => row.id)).toEqual(['first', 'second'])
-      expect(dbChainMockFns.select).toHaveBeenCalledTimes(3)
-      expect(Object.keys(dbChainMockFns.select.mock.calls[0][0])).toEqual(['id'])
-      expect(dbChainMockFns.limit).toHaveBeenNthCalledWith(1, 400)
-      expect(dbChainMockFns.select.mock.calls[1][0]).toHaveProperty('distance')
-      expect(dbChainMockFns.limit).toHaveBeenCalledWith(20)
+      /** Only hydration reads through the query builder; ranking never materializes tag IDs. */
+      expect(dbChainMockFns.select).toHaveBeenCalledTimes(1)
+      expect(dbChainMockFns.select.mock.calls[0][0]).toHaveProperty('distance')
+      const exact = dbChainMockFns.execute.mock.calls
+        .map(([query]) => (query as { toSQL: () => { sql: string; params: unknown[] } }).toSQL())
+        .find((statement) => statement.sql.includes(') + 0 LIMIT'))!
+      expect(exact.params).toContain(400)
     })
 
     it('should throw error when no filters provided', async () => {
@@ -552,13 +557,13 @@ describe('Knowledge Search Utils', () => {
       })
 
       expect(results.map((r) => r.id)).toEqual(['vector-hit'])
-      expect(dbChainMockFns.select).toHaveBeenCalledTimes(2)
+      expect(dbChainMockFns.select).toHaveBeenCalledTimes(1)
     })
 
     it('runs both legs and fuses them in hybrid mode', async () => {
       /**
-       * The raw vector probe does not consume a table chain. Keyword ranking and
-       * hydration complete before vector exact ranking and content hydration.
+       * Vector ranking is raw SQL throughout and consumes no table chain. Keyword ranking and
+       * hydration complete before vector content hydration.
        */
       dbChainMockFns.execute.mockResolvedValue([{ id: 'vector-hit' }])
       queueTableRows(schemaMock.embedding, [{ id: 'keyword-hit', keywordRank: 0.9 }])
@@ -576,7 +581,7 @@ describe('Knowledge Search Utils', () => {
       })
 
       expect(results.map((r) => r.id).sort()).toEqual(['keyword-hit', 'vector-hit'])
-      expect(dbChainMockFns.select).toHaveBeenCalledTimes(4)
+      expect(dbChainMockFns.select).toHaveBeenCalledTimes(3)
     })
 
     it('propagates unexpected keyword errors after the vector leg finishes', async () => {
@@ -601,7 +606,7 @@ describe('Knowledge Search Utils', () => {
           queryVector: { vector: JSON.stringify(TEST_EMBEDDING), dimensions: 1536 },
         })
       ).rejects.toBe(failure)
-      expect(dbChainMockFns.select).toHaveBeenCalledTimes(3)
+      expect(dbChainMockFns.select).toHaveBeenCalledTimes(2)
     })
 
     it('skips both query legs when only tag filters are provided', async () => {

apps/sim/background/workspace-file-search-index.test.ts

@@ -35,7 +35,7 @@ describe('workspace file search index task', () => {
   it('uses isolated medium workers with a hard global concurrency and duration cap', () => {
     expect(workspaceFileSearchIndexTask).toMatchObject({
       id: 'workspace-file-search-index',
-      machine: 'medium-1x',
+      machine: 'medium-2x',
       maxDuration: FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
       retry: { maxAttempts: 3 },
       queue: {

apps/sim/background/workspace-file-search-index.ts

@@ -15,7 +15,7 @@ import {
  */
 export const workspaceFileSearchIndexTask = task({
   id: 'workspace-file-search-index',
-  machine: 'medium-1x',
+  machine: 'medium-2x',
   maxDuration: FILE_SEARCH_INDEX_MAX_DURATION_SECONDS,
   retry: { maxAttempts: 3 },
   queue: {

apps/sim/connectors/google-workspace/company-crawl.test.ts

@@ -502,21 +502,24 @@ describe('Google Workspace per-user central crawl', () => {
     expect((await list(context(), undefined, CONFIG, 'google_calendar')).documents).toHaveLength(1)
   })
 
-  it('isolates explicit Calendar list access failures without claiming a disabled service', async () => {
-    listUserDocuments.mockRejectedValueOnce(
-      new GoogleApiError('calendar.events.list', 403, ['forbidden'])
-    )
-    const first = await list(context(), undefined, CONFIG, 'google_calendar')
-    expect(first.listingFailures?.samples[0]).toEqual({
-      scope: 'alice@corp.com',
-      operation: 'calendar.events.list',
-      status: 403,
-      reasons: ['forbidden'],
-    })
-    const second = await list(context(), first.nextCursor, CONFIG, 'google_calendar')
-    expect(second.documents[0].acl).toEqual(['u:bob@corp.com'])
-    expect(second.reconciliationSafe).toBe(false)
-  })
+  it.each(['forbidden', 'notACalendarUser'])(
+    'isolates explicit Calendar list access failures (%s) without claiming a disabled service',
+    async (reason) => {
+      listUserDocuments.mockRejectedValueOnce(
+        new GoogleApiError('calendar.events.list', 403, [reason])
+      )
+      const first = await list(context(), undefined, CONFIG, 'google_calendar')
+      expect(first.listingFailures?.samples[0]).toEqual({
+        scope: 'alice@corp.com',
+        operation: 'calendar.events.list',
+        status: 403,
+        reasons: [reason],
+      })
+      const second = await list(context(), first.nextCursor, CONFIG, 'google_calendar')
+      expect(second.documents[0].acl).toEqual(['u:bob@corp.com'])
+      expect(second.reconciliationSafe).toBe(false)
+    }
+  )
 
   it.each([{ error: { code: 403 } }, { error: { code: 403, errors: [], details: [] } }])(
     'propagates a Calendar 403 without reason codes: %j',
@@ -542,6 +545,9 @@ describe('Google Workspace per-user central crawl', () => {
     [403, ['domainPolicy']],
     [403, ['unrecognized-provider-code']],
     [403, ['forbidden', 'unrecognized-provider-code']],
+    [403, ['notACalendarUser', 'unrecognized-provider-code']],
+    [403, ['notACalendarUser', 'insufficientPermissions']],
+    [403, ['notACalendarUser', 'rateLimitExceeded']],
     [401, ['authError']],
     [429, []],
     [500, ['backendError']],
@@ -571,6 +577,15 @@ describe('Google Workspace per-user central crawl', () => {
     await expect(list(context(), undefined, CONFIG, 'google_calendar')).rejects.toBe(error)
   })
 
+  it.each([
+    new GoogleApiError('calendar.events.list', 403, ['notACalendarUser'], false),
+    new GoogleApiError('calendar.calendarList.list', 403, ['notACalendarUser']),
+    new GoogleApiError('calendar.events.list', 401, ['notACalendarUser']),
+  ])('does not isolate Calendar unavailability outside a complete list 403: %s', async (error) => {
+    listUserDocuments.mockRejectedValueOnce(error)
+    await expect(list(context(), undefined, CONFIG, 'google_calendar')).rejects.toBe(error)
+  })
+
   it('does not suppress delegation failures that resemble provider list failures', async () => {
     const ctx = context()
     const error = new GoogleApiError('gmail.threads.list', 400, ['failedPrecondition'])

apps/sim/connectors/google-workspace/company-crawl.ts

@@ -177,7 +177,7 @@ function userListingFailure(
       : error.diagnostic.operation === 'calendar.events.list' &&
         error.status === 403 &&
         reasons.length > 0 &&
-        reasons.every((reason) => reason === 'forbidden')
+        reasons.every((reason) => reason === 'forbidden' || reason === 'notACalendarUser')
   return isolated
     ? { operation: error.diagnostic.operation, status: error.status, reasons: [...reasons] }
     : null

apps/sim/lib/knowledge/__integration__/kb-block-search.integration.ts

@@ -91,7 +91,7 @@ describe('API-key KB block fan-out', () => {
       const previousDebug = db.$client.options.debug
       const statements: string[] = []
       db.$client.options.debug = (_connection, query) => {
-        if (statements.length < 250) statements.push(query)
+        if (statements.length < 1000) statements.push(query)
       }
       try {
         const results = await Promise.all(
@@ -124,9 +124,26 @@ describe('API-key KB block fan-out', () => {
           expect(result.rows[0].knowledgeBaseId).toBe(bases[index].id)
           expect(result.rows[0].distance).toBeCloseTo(0)
         }
-        expect(statements.filter((query) => query.includes('statement_timeout'))).toHaveLength(54)
-        expect(statements.filter((query) => query.includes('+ 0'))).toHaveLength(18)
-        expect(statements.some((query) => query.includes('hnsw.iterative_scan'))).toBe(false)
+        const matching = (fragment: string) =>
+          statements.filter((query) => query.includes(fragment))
+        /**
+         * Every statement runs under the leg's deadline: the candidate search reinstates it after
+         * tuning the scan, and the probe, the exact ranking, the rerank and hydration each open
+         * with one of their own.
+         */
+        expect(matching('statement_timeout')).toHaveLength(bases.length * 6)
+        /**
+         * A scope this small leaves the bounded traversal short of its candidate limit, so every
+         * search probes once and rescues once — never a widening retry loop.
+         */
+        expect(matching('hnsw.iterative_scan')).toHaveLength(bases.length)
+        expect(matching('AS visible')).toHaveLength(bases.length)
+        expect(matching(') + 0 LIMIT')).toHaveLength(bases.length)
+        expect(matching('scored_search_candidates')).toHaveLength(bases.length)
+        /** The probe enumerates visible documents; it never ranks them. */
+        expect(
+          statements.filter((query) => query.includes('AS id FROM') && !query.includes('ORDER BY'))
+        ).toHaveLength(bases.length)
       } finally {
         db.$client.options.debug = previousDebug
       }

apps/sim/lib/knowledge/application/search-source-overview.test.ts

@@ -24,6 +24,7 @@ vi.mock('@/lib/knowledge/read-access', () => ({
 }))
 
 import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview'
+import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants'
 
 const principal = { kind: 'session', userId: 'reader', sessionId: 'session' } as const
 const input = { organizationId: 'org-1', workspaceId: null }
@@ -36,10 +37,24 @@ const searchableProbeCount = () =>
   dbChainMockFns.limit.mock.calls.filter(([rows]) => rows === 1).length -
   AUTHORIZATION_SINGLE_ROW_READS
 
+/** The configured-provider list is read once, before the batches, under the same bound. */
+const CONFIGURED_PROVIDER_READS = 1
+
+/** Every other provider-bounded read in this use case is the indexing probe. */
+const indexingProbeCount = () =>
+  dbChainMockFns.limit.mock.calls.filter(([rows]) => rows === MAX_SEARCH_SOURCE_PROVIDER_TYPES)
+    .length - CONFIGURED_PROVIDER_READS
+
+/** Counted at the yield, so batches the use case never asks for stay uncounted. */
 function yieldBatches(count: number) {
+  const consumed = { batches: 0 }
   mocks.batches.mockImplementation(async function* () {
-    for (let index = 0; index < count; index += 1) yield sql`batch-${sql.raw(String(index))}`
+    for (let index = 0; index < count; index += 1) {
+      consumed.batches += 1
+      yield sql`batch-${sql.raw(String(index))}`
+    }
   })
+  return consumed
 }
 
 beforeEach(() => {
@@ -73,4 +88,71 @@ describe('readSearchSourceOverview', () => {
     expect(result.hasSearchableDocuments).toBe(false)
     expect(searchableProbeCount()).toBe(3)
   })
+
+  it('stops probing for indexing once every configured provider type is known', async () => {
+    const consumed = yieldBatches(3)
+    queueTableRows(member, [{ role: 'owner' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+
+    const result = await readSearchSourceOverview.execute({ principal, input })
+
+    expect(result).toEqual({
+      providers: [{ connectorType: 'gmail', isSyncing: true }],
+      hasSearchableDocuments: false,
+    })
+    expect(indexingProbeCount()).toBe(1)
+    /** The searchable probe is still unsatisfied, so the batches keep being consumed. */
+    expect(consumed.batches).toBe(3)
+  })
+
+  it('keeps probing every batch while a configured provider type is still unaccounted for', async () => {
+    yieldBatches(3)
+    queueTableRows(member, [{ role: 'owner' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }, { connectorType: 'notion' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+
+    const result = await readSearchSourceOverview.execute({ principal, input })
+
+    expect(result.providers).toEqual([
+      { connectorType: 'gmail', isSyncing: true },
+      { connectorType: 'notion', isSyncing: false },
+    ])
+    expect(indexingProbeCount()).toBe(3)
+  })
+
+  it('stops consuming access batches once neither probe can change the result', async () => {
+    const consumed = yieldBatches(3)
+    queueTableRows(member, [{ role: 'owner' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+    queueTableRows(document, [{ id: 'doc-1' }])
+
+    const result = await readSearchSourceOverview.execute({ principal, input })
+
+    expect(result).toEqual({
+      providers: [{ connectorType: 'gmail', isSyncing: true }],
+      hasSearchableDocuments: true,
+    })
+    expect(consumed.batches).toBe(1)
+  })
+
+  it('keeps consuming access batches for a provider type still unaccounted for', async () => {
+    const consumed = yieldBatches(3)
+    queueTableRows(member, [{ role: 'owner' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }, { connectorType: 'notion' }])
+    queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }])
+    queueTableRows(document, [{ id: 'doc-1' }])
+
+    const result = await readSearchSourceOverview.execute({ principal, input })
+
+    expect(result).toEqual({
+      providers: [
+        { connectorType: 'gmail', isSyncing: true },
+        { connectorType: 'notion', isSyncing: false },
+      ],
+      hasSearchableDocuments: true,
+    })
+    expect(consumed.batches).toBe(3)
+  })
 })