From ba6f412a7e0da1d0749834e71b3d38ebcc5745d7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 21:59:46 -0700 Subject: [PATCH 1/2] improvement(knowledge): read the source card with the result and hold Tin readiness longer --- .../lib/knowledge/application/search.test.ts | 53 ++++++------- apps/sim/lib/knowledge/application/search.ts | 76 ++++++++----------- apps/sim/lib/knowledge/search/diagnostics.ts | 1 - apps/sim/lib/knowledge/search/queries.test.ts | 4 + apps/sim/lib/knowledge/search/queries.ts | 21 ++++- apps/sim/lib/knowledge/search/tin-keyword.ts | 5 +- 6 files changed, 80 insertions(+), 80 deletions(-) diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 2defebd93e1..715254c332f 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -21,7 +21,6 @@ const mocks = vi.hoisted(() => ({ generateEmbedding: vi.fn(), executeSearch: vi.fn(), retrieval: vi.fn(), - getDocumentMetadata: vi.fn(), getTagDefinitions: vi.fn(), getTagDefinitionsBatch: vi.fn(), recordEmbeddingUsage: vi.fn(), @@ -100,7 +99,6 @@ vi.mock('@/lib/knowledge/search/queries', () => ({ retrieval: mocks.retrieval(), readAccess: (args[0] as { access: unknown }).access, }), - getDocumentMetadataByIds: mocks.getDocumentMetadata, })) vi.mock('@/lib/knowledge/tags/service', () => ({ @@ -170,6 +168,9 @@ describe('knowledge search application use case', () => { content: 'answer', chunkIndex: 0, distance: 0.2, + filename: 'guide.pdf', + sourceUrl: null, + connectorType: null, tag1: null, tag2: null, tag3: null, @@ -189,29 +190,11 @@ describe('knowledge search application use case', () => { boolean3: null, }, ]) - mocks.getDocumentMetadata.mockResolvedValue({ - 'document-1': { filename: 'guide.pdf', sourceUrl: null }, - }) mocks.getTagDefinitions.mockResolvedValue([]) mocks.recordEmbeddingUsage.mockResolvedValue(undefined) mocks.importProvenance.mockResolvedValue({ imported: true, documentMetadata: {} }) }) - it('drops passages whose access was revoked before the final document metadata read', async () => { - mocks.getDocumentMetadata.mockResolvedValue({}) - const result = await searchKnowledge.execute({ - principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, - input: { - workspaceId: 'workspace-1', - knowledgeBaseIds: ['knowledge-1'], - query: 'orion', - topK: 20, - }, - }) - expect(result.results).toEqual([]) - expect(result.totalResults).toBe(0) - }) - it.each([false, true])( 'requires explicit partial-result support for empty incomplete searches (allowPartialResults=%s)', async (allowPartialResults) => { @@ -512,7 +495,6 @@ describe('knowledge search application use case', () => { }) ).rejects.toThrow('Search superseded during embedding') expect(mocks.executeSearch).not.toHaveBeenCalled() - expect(mocks.getDocumentMetadata).not.toHaveBeenCalled() }) it('does not start reranking or metadata reads after retrieval is cancelled', async () => { @@ -536,7 +518,6 @@ describe('knowledge search application use case', () => { }) ).rejects.toThrow('Search superseded during retrieval') expect(mocks.rerank).not.toHaveBeenCalled() - expect(mocks.getDocumentMetadata).not.toHaveBeenCalled() expect(mocks.searched).not.toHaveBeenCalled() }) @@ -550,6 +531,9 @@ describe('knowledge search application use case', () => { content: 'first', chunkIndex: 0, distance: 0.1, + filename: 'thread', + sourceUrl: null, + connectorType: 'slack', }, { id: 'chunk-2', @@ -558,6 +542,9 @@ describe('knowledge search application use case', () => { content: 'second', chunkIndex: 1, distance: 0.2, + filename: 'thread', + sourceUrl: null, + connectorType: 'slack', }, { id: 'chunk-3', @@ -566,12 +553,11 @@ describe('knowledge search application use case', () => { content: 'third', chunkIndex: 0, distance: 0.3, + filename: 'page', + sourceUrl: null, + connectorType: 'gitlab', }, ]) - mocks.getDocumentMetadata.mockResolvedValue({ - 'document-1': { filename: 'thread', connectorType: 'slack' }, - 'document-2': { filename: 'page', connectorType: 'gitlab' }, - }) await searchKnowledge.execute({ principal: { kind: 'personal_api_key', userId: 'user-1', keyId: 'key-1' }, input: { @@ -808,14 +794,20 @@ describe('knowledge search application use case', () => { it('keeps the source card metadata when a provenance registry is present', async () => { const registry = { markIncomplete: vi.fn() } const sourceModifiedAt = new Date('2026-08-20T12:00:00Z') - mocks.getDocumentMetadata.mockResolvedValueOnce({ - 'document-1': { + mocks.executeSearch.mockResolvedValueOnce([ + { + id: 'embedding-1', + documentId: 'document-1', + knowledgeBaseId: 'knowledge-1', + content: 'answer', + chunkIndex: 0, + distance: 0.2, + sourceModifiedAt, filename: 'guide.pdf', sourceUrl: 'https://example.com/guide', - sourceModifiedAt, connectorType: 'google_drive', }, - }) + ]) const result = await searchKnowledge.execute({ principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, @@ -828,7 +820,6 @@ describe('knowledge search application use case', () => { }, }) - expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(['document-1'], expect.anything()) expect(result.results[0]).toMatchObject({ documentName: 'guide.pdf', sourceUrl: 'https://example.com/guide', diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 6530c6bb64a..43cc7ad22d9 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -42,7 +42,6 @@ import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' import { annotateSearchDiagnostics, measureSearchStage } from '@/lib/knowledge/search/diagnostics' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' import { - getDocumentMetadataByIds, type RetrievalStatus, retrieveKnowledgeSearch, type SearchResult, @@ -634,49 +633,40 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ ]) ) /** - * Always read: the provenance snapshot vouches for the name, URL, and tags - * a model may see, but the source card's modified time and connector type - * are only carried here, under the same access predicate as the search. + * The provenance snapshot vouches for the name, URL, and tags a model may see; the source + * card's modified time and connector type ride on the hydrated row, read under the same + * predicate as the content. */ - const basicDocumentMetadata = await measureSearchStage('metadata', () => - getDocumentMetadataByIds( - rows.map((row) => row.documentId), - retrieved.readAccess - ) - ) - const results = rows - .filter((row) => basicDocumentMetadata[row.documentId]) - .map((row): KnowledgeSearchItem => { - const metadata: Record = {} - const tagMap = tagMaps.get(row.knowledgeBaseId) - const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId] - const basicDocument = basicDocumentMetadata[row.documentId] - const document = provenanceDocument ?? basicDocument - for (const slot of ALL_TAG_SLOTS) { - const value = - provenanceDocument && slot.startsWith('tag') - ? provenanceDocument[ - slot as 'tag1' | 'tag2' | 'tag3' | 'tag4' | 'tag5' | 'tag6' | 'tag7' - ] - : row[slot] - if (value !== null && value !== undefined) metadata[tagMap?.get(slot) ?? slot] = value - } - const rerankerScore = rerankerScores.get(row.id) - return { - embeddingId: row.id, - knowledgeBaseId: row.knowledgeBaseId, - documentId: row.documentId, - documentName: document?.filename ?? null, - sourceUrl: document?.sourceUrl ?? null, - sourceModifiedAt: basicDocument?.sourceModifiedAt ?? null, - connectorType: basicDocument?.connectorType ?? null, - content: row.content, - chunkIndex: row.chunkIndex, - metadata, - similarity: hasQuery ? 1 - row.distance : 1, - ...(rerankerScore !== undefined ? { rerankerScore } : {}), - } - }) + const results = rows.map((row): KnowledgeSearchItem => { + const metadata: Record = {} + const tagMap = tagMaps.get(row.knowledgeBaseId) + const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId] + const document = provenanceDocument ?? row + for (const slot of ALL_TAG_SLOTS) { + const value = + provenanceDocument && slot.startsWith('tag') + ? provenanceDocument[ + slot as 'tag1' | 'tag2' | 'tag3' | 'tag4' | 'tag5' | 'tag6' | 'tag7' + ] + : row[slot] + if (value !== null && value !== undefined) metadata[tagMap?.get(slot) ?? slot] = value + } + const rerankerScore = rerankerScores.get(row.id) + return { + embeddingId: row.id, + knowledgeBaseId: row.knowledgeBaseId, + documentId: row.documentId, + documentName: document?.filename ?? null, + sourceUrl: document?.sourceUrl ?? null, + sourceModifiedAt: row.sourceModifiedAt ?? null, + connectorType: row.connectorType ?? null, + content: row.content, + chunkIndex: row.chunkIndex, + metadata, + similarity: hasQuery ? 1 - row.distance : 1, + ...(rerankerScore !== undefined ? { rerankerScore } : {}), + } + }) if (registry && provenanceSnapshot) { for (const [documentId, document] of Object.entries(provenanceSnapshot.documentMetadata)) { const renderedMetadata = results diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index e8c753941b6..57db3d6b7c9 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -38,7 +38,6 @@ export type SearchStage = | 'usage_recording' | 'overage_billing' | 'tag_definitions' - | 'metadata' | 'metadata.sql' | 'metadata_provenance' | 'activity_recording' diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index 134ad41fb9e..6619a842727 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -557,6 +557,10 @@ describe('workspace-scoped vector retrieval', () => { expect(fields).toContain(String(schemaMock.embeddingSearch.vector512)) expect(fields).not.toContain(String(schemaMock.embedding.embedding)) expect(JSON.stringify(dbChainMockFns.leftJoin.mock.calls)).toContain('embeddingSearch') + /** The source card's name, URL and connector type ride on the same read; no second pass. */ + expect(fields).toContain(String(schemaMock.document.filename)) + expect(fields).toContain(String(schemaMock.knowledgeConnector.connectorType)) + expect(JSON.stringify(dbChainMockFns.leftJoin.mock.calls)).toContain('knowledgeConnector') }) it('passes over a slice whose documents went away instead of ending the pool there', async () => { diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index f7080770bb2..2c6b84804f1 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -65,7 +65,6 @@ import type { StructuredFilter } from '@/lib/knowledge/types' import { embeddingCandidateDimensions, embeddingCandidateDistance, - embeddingDistance, } from '@/lib/knowledge/vector-columns' const logger = createLogger('KnowledgeSearchQueries') @@ -345,6 +344,10 @@ export interface SearchResult { knowledgeBaseId: string /** When the source last changed the document; NULL for uploads and sources that do not say. */ sourceModifiedAt: Date | null + filename: string + sourceUrl: string | null + /** The connector type behind the document; NULL for an upload. */ + connectorType: string | null } /** @@ -440,6 +443,9 @@ const getSearchResultFields = (distanceExpr: SQL | SQL.Aliased) distance: distanceExpr, knowledgeBaseId: embedding.knowledgeBaseId, sourceModifiedAt: document.sourceModifiedAt, + filename: document.filename, + sourceUrl: document.sourceUrl, + connectorType: knowledgeConnector.connectorType, }) /** @@ -921,6 +927,7 @@ function hydrateSearchCandidates( .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) .leftJoin(embeddingSearch, eq(embeddingSearch.id, embedding.id)) + .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) .where( and( inArray(embedding.id, ids), @@ -1018,6 +1025,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise`0`.as('distance'))) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) + .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) .where( and( eq(embedding.knowledgeBaseId, kbId), @@ -1036,6 +1044,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise`0`.as('distance'))) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) + .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) .where( and( inArray(embedding.knowledgeBaseId, knowledgeBaseIds), @@ -2320,15 +2329,21 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise return [] } - /** Hydration pass: full rows plus the cosine distance, bounded to the survivors. */ + /** Hydration pass: full rows plus the projection's distance, bounded to the survivors. */ const hydrated = await db .select( getSearchResultFields( - embeddingDistance(queryVector.dimensions, queryVector.vector).as('distance') + embeddingCandidateDistance( + queryVector.dimensions, + queryVector.vector, + queryVector.model + ).as('distance') ) ) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) + .leftJoin(embeddingSearch, eq(embeddingSearch.id, embedding.id)) + .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) .where(and(inArray(embedding.id, topIds), ...getVisibilityConditions(access, params.filters))) const rowById = new Map(hydrated.map((row) => [row.id, row])) diff --git a/apps/sim/lib/knowledge/search/tin-keyword.ts b/apps/sim/lib/knowledge/search/tin-keyword.ts index fe1e95d06cf..0a3e705d89f 100644 --- a/apps/sim/lib/knowledge/search/tin-keyword.ts +++ b/apps/sim/lib/knowledge/search/tin-keyword.ts @@ -11,9 +11,10 @@ const logger = createLogger('TinKeywordSearch') /** * How long a readiness answer holds. The index only becomes valid once the projection is fully - * backfilled, and flipping the flag off takes effect within this window. + * backfilled, and never invalid again; the flag is checked on every search regardless, so a + * long hold costs nothing but the one read it saves each search. */ -const READINESS_TTL_MS = 60 * 1000 +const READINESS_TTL_MS = 10 * 60 * 1000 const SEARCH_INDEX_TTL_MS = 10 * 60 * 1000 /** From 2afd6ece3417e3103fe68119c7acc2c8b19a58ec Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 22:08:05 -0700 Subject: [PATCH 2/2] improvement(knowledge): keep the Tin readiness hold at a minute --- apps/sim/lib/knowledge/search/tin-keyword.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/knowledge/search/tin-keyword.ts b/apps/sim/lib/knowledge/search/tin-keyword.ts index 0a3e705d89f..fe1e95d06cf 100644 --- a/apps/sim/lib/knowledge/search/tin-keyword.ts +++ b/apps/sim/lib/knowledge/search/tin-keyword.ts @@ -11,10 +11,9 @@ const logger = createLogger('TinKeywordSearch') /** * How long a readiness answer holds. The index only becomes valid once the projection is fully - * backfilled, and never invalid again; the flag is checked on every search regardless, so a - * long hold costs nothing but the one read it saves each search. + * backfilled, and flipping the flag off takes effect within this window. */ -const READINESS_TTL_MS = 10 * 60 * 1000 +const READINESS_TTL_MS = 60 * 1000 const SEARCH_INDEX_TTL_MS = 10 * 60 * 1000 /**