diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index b1869c719f9..560dd1d269f 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -277,6 +277,7 @@ jobs: lib/knowledge/__integration__/member-document-lifecycle.integration.ts lib/knowledge/__integration__/slack-empty-threads.integration.ts lib/knowledge/__integration__/kb-block-search.integration.ts + lib/knowledge/__integration__/gitlab-workspace.integration.ts lib/knowledge/__integration__/unfilled-projection-source.integration.ts lib/core/outbox/service.integration.ts lib/knowledge/__integration__/connector-upload.integration.ts diff --git a/apps/docs/content/docs/knowledgebase/connectors.mdx b/apps/docs/content/docs/knowledgebase/connectors.mdx index dd74389e69e..60deb304e64 100644 --- a/apps/docs/content/docs/knowledgebase/connectors.mdx +++ b/apps/docs/content/docs/knowledgebase/connectors.mdx @@ -56,6 +56,7 @@ Other connectors use **API keys** or **personal access tokens** instead. The set | **Fireflies** | Generate from the Integrations page in your Fireflies account | | **Typeform** | Personal access token from your Typeform account settings | | **Azure DevOps** | Personal access token with Wiki (Read), Work Items (Read), and Code (Read) scopes | +| **GitLab** | Personal access token with `read_api` scope and access to the selected project | | **YouTube** | YouTube Data API key from the Google Cloud Console | | **Amazon S3** | Secret Access Key (the Access Key ID, region, and bucket are entered as config fields) | | **Sentry** | Auth token with `project:read` and `event:read` scopes | @@ -65,10 +66,14 @@ Other connectors use **API keys** or **personal access tokens** instead. The set | **Databricks** | Personal access token from your workspace's user settings (the workspace host is entered as a config field) | | **Workday Help** | Register an API client for integrations in your tenant, then enter the client secret and refresh token together as `clientSecret:refreshToken` (the client ID, tenant host, and tenant name are entered as config fields) | +Enter an API key directly, or type `{{` to select an accessible personal or workspace secret. Sim resolves the secret when you connect and stores an encrypted copy of the token. Later changes to the secret do not automatically update the connector. + If you rotate an API key in the external service, update it in Sim as well — OAuth tokens refresh automatically, but API keys do not. +For GitLab in a regular knowledge base, a project-readable PAT is enough. Imported content uses the knowledge base's access rules; it does not mirror each person's GitLab permissions. The administrator-token and non-admin CSV setup paths apply when using [GitLab source permissions](/search/gitlab). + @@ -78,6 +83,7 @@ Each connector has source-specific fields that control what gets synced. Example - **Notion** — sync an entire workspace, a specific database, or a single page tree - **GitHub** — specify a repository, branch, and optional file extension filter +- **GitLab** — specify a project path or ID and your instance host (leave blank for GitLab.com), then choose repository files, wiki pages, issues, or merge requests. Each connector syncs one project; submodules need their own connectors. - **Confluence** — enter your Atlassian domain and choose spaces, or **All** for all spaces accessible at each sync. Optionally filter by content type or label. PDF, Word (`.docx`, Word 97–2003 `.doc`), Excel (`.xlsx`), and PowerPoint (`.pptx`) attachments on matching pages and blog posts are included as separate documents. - **Azure DevOps** — choose what to sync (wiki pages, work items, repository files, or all), with optional work item type/state filters, a custom WIQL query, and repository/branch/path filters - **Amazon S3** — point at a bucket with an optional key prefix and a customizable file extension allowlist; S3-compatible stores (Cloudflare R2, MinIO) are supported via a custom endpoint diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx index 07ec5c2c42d..db8044595c9 100644 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx @@ -6,6 +6,11 @@ import { NuqsTestingAdapter } from 'nuqs/adapters/testing' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/hooks/queries/environment', () => ({ + usePersonalEnvironment: () => ({ data: {} }), + useWorkspaceEnvironment: () => ({ data: { workspace: {}, personal: {} } }), +})) + const mocks = vi.hoisted(() => ({ canAdmin: true, hasMaxAccess: true, @@ -269,6 +274,7 @@ async function chooseSyncFrequency(label: string) { async function fill(placeholder: string, value: string) { const input = document.querySelector(`input[placeholder="${placeholder}"]`) expect(input, `Input ${placeholder}`).not.toBeNull() + await act(async () => input?.focus()) await act(async () => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call(input, value) input?.dispatchEvent(new Event('input', { bubbles: true })) @@ -897,8 +903,8 @@ describe('member content credentials in real add and edit dialogs', () => { ) await click(card!) expect(document.body.textContent).not.toContain('Connected members') - expect(button('Administrator token')).toHaveAttribute('aria-checked', 'true') - expect(document.body.textContent).not.toContain('Connection method') + expect(document.body.textContent).not.toContain('Administrator token') + expect(document.body.textContent).toContain('Everyone in this workspace') await fill('Enter your GitLab PAT', 'new-pat') await fill('gitlab.example.com', 'gitlab.example.test') await fill('group/project or numeric ID', '1') @@ -906,7 +912,7 @@ describe('member content credentials in real add and edit dialogs', () => { await click(button('Connect & Sync')) expect(mocks.create.mock.calls[1][0]).toMatchObject({ connectorType: 'gitlab', - accessMode: 'admin', + accessMode: 'workspace', apiKey: 'new-pat', }) expect(mocks.create.mock.calls[1][0].sourceConfig).not.toHaveProperty('excludeChannels') diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx index e9b3be36d43..846b202f7a8 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx @@ -43,6 +43,14 @@ const mocks = vi.hoisted(() => ({ | 'ready', })) +vi.mock('@/hooks/queries/environment', () => ({ + usePersonalEnvironment: () => ({ data: {} }), + useWorkspaceEnvironment: () => ({ data: { workspace: { GITLAB_PAT: '***' }, personal: {} } }), +})) +vi.mock('@/hooks/use-settings-navigation', () => ({ + useSettingsNavigation: () => ({ navigateToSettings: vi.fn() }), +})) + vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }), usePathname: () => '/o/org-1/settings/integrations', @@ -790,6 +798,7 @@ describe('Search setup options', () => { initialAccessMode: connectorType === 'gitlab' ? 'admin' : 'members', }) + if (connectorType === 'gitlab') expect(document.body.textContent).not.toContain('Sync using') const primaryFields = configFieldsProps() for (const fieldId of primary) expect(fieldVisible(primaryFields, fieldId)).toBe(true) for (const fieldId of optional) expect(fieldVisible(primaryFields, fieldId)).toBe(false) @@ -834,21 +843,122 @@ describe('Search setup options', () => { } ) - it('uses GitLab service-account access and token tabs without an access selector in regular KBs', async () => { + it.each(['fixture-pat', '{{GITLAB_PAT}}'])( + 'connects a regular GitLab KB with %s and workspace access', + async (apiKey) => { + mocks.memberAccess = false + mocks.mirroredAccess = false + const sourceConfig = { host: 'gitlab.example.com', project: 'group/project' } + mocks.resolveSourceConfig.mockReturnValue(sourceConfig) + await render({ + initialConnectorType: 'gitlab', + initialAccessMode: 'workspace', + isSearchIndex: false, + }) + expect(document.body.textContent).not.toContain('Administrator token') + expect(document.body.textContent).not.toContain('Non-admin token') + expect(document.body.textContent).not.toContain('User mapping') + expect(document.body.textContent).not.toContain('Project permissions') + expect(document.body.textContent).not.toContain('Connection method') + expect(document.body.textContent).toContain('Sync Frequency') + const input = document.querySelector( + 'input[placeholder="Enter your GitLab PAT"]' + )! + await act(async () => input.focus()) + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call( + input, + apiKey + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + await act(async () => button('Connect & Sync').click()) + expect(mocks.create).toHaveBeenCalledWith( + { + knowledgeBaseId: 'kb-search', + connectorType: 'gitlab', + apiKey, + sourceConfig, + syncIntervalMinutes: 1440, + accessMode: 'workspace', + }, + expect.any(Object) + ) + } + ) + + it('selects a saved secret with the shared picker without submitting on Enter', async () => { await render({ initialConnectorType: 'gitlab', initialAccessMode: 'workspace', isSearchIndex: false, }) - expect(document.body.textContent).toContain('Administrator token') - expect(document.body.textContent).toContain('Non-admin token') - expect(document.body.textContent).not.toContain('Connection method') - expect(button('More options')).toHaveAttribute('aria-expanded', 'false') - expect(document.body.textContent).not.toContain('Sync Frequency') - await act(async () => button('More options').click()) - expect(document.body.textContent).toContain('Sync Frequency') + const input = document.querySelector( + 'input[placeholder="Enter your GitLab PAT"]' + )! + await act(async () => input.focus()) + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call( + input, + '{{GIT' + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + expect(document.body.textContent).toContain('GITLAB_PAT') + expect(document.querySelector('span[class="text-[var(--brand-secondary)]"]')).toBeNull() + await act(async () => { + input.dispatchEvent(new MouseEvent('mousedown', { bubbles: true })) + input.setSelectionRange(0, 0) + input.dispatchEvent(new MouseEvent('mouseup', { bubbles: true })) + }) + expect(document.querySelector('[role="menuitem"]')).toBeNull() + expect(input.value).toBe('{{GIT') + await act(async () => { + input.setSelectionRange(input.value.length, input.value.length) + input.dispatchEvent(new KeyboardEvent('keyup', { key: 'End', bubbles: true })) + }) + expect(document.body.textContent).toContain('GITLAB_PAT') + await act(async () => + input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) + ) + expect(input.value).toBe('{{GITLAB_PAT}}') + expect(document.querySelector('span[class="text-[var(--brand-secondary)]"]')).toHaveTextContent( + '{{GITLAB_PAT}}' + ) + expect(mocks.create).not.toHaveBeenCalled() + await act(async () => button('Connect & Sync').click()) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: '{{GITLAB_PAT}}', accessMode: 'workspace' }), + expect.any(Object) + ) }) + it.each(['{{', '{{MISSING_SECRET}}', 'literal-pat'])( + 'does not highlight unresolved or literal API-key text: %s', + async (value) => { + await render({ + initialConnectorType: 'gitlab', + initialAccessMode: 'workspace', + isSearchIndex: false, + }) + const input = document.querySelector( + 'input[placeholder="Enter your GitLab PAT"]' + )! + await act(async () => input.focus()) + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call( + input, + value + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) + expect(document.querySelector('span[class="text-[var(--brand-secondary)]"]')).toBeNull() + await act(async () => input.blur()) + expect(input.value).toBe('•'.repeat(value.length)) + expect(document.body.textContent).not.toContain(value) + } + ) + it('keeps administrator-required fields in the primary form even if metadata marks them optional', async () => { mocks.credentials = [{ id: 'service', name: 'Indexing account', type: 'service_account' }] await render({ initialConnectorType: 'google_drive', initialAccessMode: 'admin' }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index b8b1a7c334f..ebde7ec2b90 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -38,6 +38,7 @@ import { useServiceAccountConnectTarget, } from '@/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' +import { ConnectorApiKeyInput } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/connector-api-key-input' import { derivedAclCapFieldIds, isConnectorFieldRequired, @@ -167,7 +168,6 @@ export function AddConnectorModal({ const gitlabPermissions = useGitLabPermissionForm() const [apiKeyValue, setApiKeyValue] = useState('') const [useApiKey, setUseApiKey] = useState(!isSearchIndex) - const [apiKeyFocused, setApiKeyFocused] = useState(false) const [searchTerm, setSearchTerm] = useState('') useOAuthReturnForKBConnectors( @@ -502,7 +502,6 @@ export function AddConnectorModal({ }) setApiKeyValue('') setUseApiKey(!isSearchIndex) - setApiKeyFocused(false) setDisabledTagIds(new Set()) setShowMetadata(false) setCanonicalModes({}) @@ -734,13 +733,10 @@ export function AddConnectorModal({ )} {isApiKeyMode ? ( - setApiKeyValue(e.target.value)} - onFocus={() => setApiKeyFocused(true)} - onBlur={() => setApiKeyFocused(false)} + onChange={setApiKeyValue} + workspaceId={owner.workspaceId} placeholder={apiKeyConfig?.placeholder || 'Enter API key'} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/connector-api-key-input.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/connector-api-key-input.tsx new file mode 100644 index 00000000000..57f521ddbb7 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/connector-api-key-input.tsx @@ -0,0 +1,99 @@ +'use client' + +import { useRef, useState } from 'react' +import { SecretInput } from '@sim/emcn' +import { + checkEnvVarTrigger, + EnvVarDropdown, +} from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/env-var-dropdown' +import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text' +import { useAvailableEnvVarKeys } from '@/hooks/use-available-env-vars' + +const NO_ENV_VARS = new Set() + +interface ConnectorApiKeyInputProps { + value: string + onChange: (value: string) => void + placeholder?: string + workspaceId?: string +} + +export function ConnectorApiKeyInput({ + value, + onChange, + placeholder, + workspaceId, +}: ConnectorApiKeyInputProps) { + const inputRef = useRef(null) + const overlayRef = useRef(null) + const [isFocused, setIsFocused] = useState(false) + const [cursorPosition, setCursorPosition] = useState(0) + const [showSecrets, setShowSecrets] = useState(false) + const availableEnvVars = useAvailableEnvVarKeys(workspaceId, { enabled: isFocused }) + const trigger = checkEnvVarTrigger(value, cursorPosition) + const visible = showSecrets && trigger.show + + return ( +
+ { + onChange(next) + setCursorPosition(inputRef.current?.selectionStart ?? next.length) + setShowSecrets(true) + }} + onSelect={(event) => { + setCursorPosition(event.currentTarget.selectionStart ?? value.length) + }} + onFocus={() => setIsFocused(true)} + onBlur={() => setIsFocused(false)} + onScroll={(event) => { + if (overlayRef.current) { + overlayRef.current.style.transform = `translateX(-${event.currentTarget.scrollLeft}px)` + } + }} + inputClassName={isFocused ? 'text-transparent caret-[var(--text-primary)]' : undefined} + placeholder={placeholder} + /> + {isFocused && ( +
+
{ + overlayRef.current = element + if (element) { + element.style.transform = `translateX(-${inputRef.current?.scrollLeft ?? 0}px)` + } + }} + className='whitespace-pre' + > + {formatDisplayText(value, { availableEnvVars: availableEnvVars ?? NO_ENV_VARS })} +
+
+ )} + {visible && ( + setShowSecrets(false)} + onSelect={(next, cursor) => { + onChange(next) + setCursorPosition(cursor) + setShowSecrets(false) + requestAnimationFrame(() => { + inputRef.current?.focus() + inputRef.current?.setSelectionRange(cursor, cursor) + }) + }} + /> + )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx index bd2cc556aa0..fb1d1dda457 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx @@ -140,7 +140,8 @@ export function ConnectorAccessField({ for (const entry of modes) entry.allowed &&= supportsConnectorAccessMode(connectorConfig, entry.mode) if ( - connectorConfig.supportedAccessModes?.length === 1 && + connectorConfig.supportedAccessModes?.filter((mode) => allowWorkspace || mode !== 'workspace') + .length === 1 && modes.some((entry) => entry.mode === value.accessMode && entry.allowed) ) return canAdmin && footer ?
{footer}
: null diff --git a/apps/sim/connectors/gitlab/meta.ts b/apps/sim/connectors/gitlab/meta.ts index 586089d9735..6a8ded013d2 100644 --- a/apps/sim/connectors/gitlab/meta.ts +++ b/apps/sim/connectors/gitlab/meta.ts @@ -10,7 +10,7 @@ export const gitlabConnectorMeta: ConnectorMeta = { 'Sync repository files, wiki pages, issues, merge requests, and their non-internal comments from a GitLab project', version: '1.3.0', mirrorsSourceAcls: true, - supportedAccessModes: ['admin'], + supportedAccessModes: ['admin', 'workspace'], adminSetupHint: 'Use an administrator token, or a non-admin token with CSV permissions. Both require read_api access and a self-managed GitLab host.', icon: GitLabIcon, diff --git a/apps/sim/lib/knowledge/__integration__/gitlab-workspace.integration.ts b/apps/sim/lib/knowledge/__integration__/gitlab-workspace.integration.ts new file mode 100644 index 00000000000..5d2824a4c9b --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/gitlab-workspace.integration.ts @@ -0,0 +1,242 @@ +/** Real PAT storage, creation, sync, indexing, and workspace authorization with fixture GitLab replies. */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + environment, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorPermissionSnapshot, + organization, + permissions, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull } from 'drizzle-orm' +import { afterAll, beforeAll, expect, it, vi } from 'vitest' + +const fixture = vi.hoisted(() => ({ root: '', fetch: vi.fn() })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixture.root + }, +})) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ + secureFetchWithRetry: fixture.fetch, +})) +vi.mock('@/lib/embeddings', async () => ({ + ...(await import('@/lib/embeddings/client')), + assertKnowledgeEmbeddingCapacity: async () => {}, + embedKnowledge: async (texts: string[]) => ({ + embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), + totalTokens: texts.length, + billableTokens: 0, + isBYOK: true, + modelName: 'text-embedding-3-small', + pricingId: 'text-embedding-3-small', + }), +})) + +import { decryptApiKey } from '@/lib/api-key/crypto' +import { encryptSecret } from '@/lib/core/security/encryption' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { + createKnowledgeConnector, + updateKnowledgeConnector, +} from '@/lib/knowledge/application/connectors' +import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' +import { searchKnowledge } from '@/lib/knowledge/application/search' + +const ids = createKnowledgeAclFixtureIds() +const sourceConfig = { + host: 'gitlab.example.com', + project: 'group/project', + contentTypes: 'repo', +} +const principal = { kind: 'session' as const, userId: ids.aliceId, sessionId: generateId() } +const input = { + knowledgeBaseId: ids.knowledgeBaseId, + assertedWorkspaceId: ids.workspaceId, + connectorType: 'gitlab', + apiKey: 'fixture-read-only-pat', + sourceConfig, + syncIntervalMinutes: 1440, +} + +beforeAll(async () => { + fixture.root = mkdtempSync(path.join(tmpdir(), 'sim-gitlab-workspace-')) + await seedKnowledgeAclFixture(ids) + await db.insert(environment).values({ + id: ids.aliceId, + userId: ids.aliceId, + variables: { GITLAB_PAT: (await encryptSecret(input.apiKey)).encrypted }, + }) + await db + .update(permissions) + .set({ permissionType: 'write' }) + .where(and(eq(permissions.entityId, ids.workspaceId), eq(permissions.userId, ids.aliceId))) + fixture.fetch.mockImplementation(async (raw: string, init: RequestInit) => { + const url = new URL(raw) + expect(url.origin).toBe('https://gitlab.example.com') + expect(init.method ?? 'GET').toBe('GET') + expect(new Headers(init.headers).get('PRIVATE-TOKEN')).toBe(input.apiKey) + const project = '/api/v4/projects/group%2Fproject' + if (url.pathname === project) + return Response.json({ + id: 42, + path_with_namespace: 'group/project', + default_branch: 'master', + }) + if (url.pathname === `${project}/repository/commits/master`) + return Response.json({ id: 'commit-v1' }) + expect(url.searchParams.get('ref')).toBe('master') + if (url.pathname === `${project}/repository/tree`) + return Response.json([{ id: 'blob-v1', name: 'orion.md', path: 'orion.md', type: 'blob' }]) + if (url.pathname === `${project}/repository/files/orion.md`) + return Response.json({ + file_path: 'orion.md', + blob_id: 'blob-v1', + encoding: 'base64', + content: Buffer.from('Orion firmware connector fixture.').toString('base64'), + }) + throw new Error(`Unexpected GitLab API endpoint: ${url.pathname}`) + }) +}) + +afterAll(async () => { + vi.restoreAllMocks() + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await rm(fixture.root, { recursive: true, force: true }) + await db.$client.end() +}) + +it.each([input.apiKey, '{{GITLAB_PAT}}'])( + 'creates, syncs, edits, and searches a workspace GitLab source using %s', + async (apiKey) => { + const { connector } = await createKnowledgeConnector.execute({ + principal, + input: { ...input, apiKey }, + }) + expect(connector.accessMode).toBe('workspace') + expect(JSON.stringify(connector)).not.toContain(input.apiKey) + await expect + .poll( + async () => { + const [row] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connector.id)) + return { status: row.status, error: row.lastSyncError, synced: Boolean(row.lastSyncAt) } + }, + { timeout: 15000 } + ) + .toEqual({ status: 'active', error: null, synced: true }) + const [stored] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connector.id)) + expect(stored.syncIntervalMinutes).toBe(1440) + expect(stored.encryptedApiKey).not.toBe(input.apiKey) + expect((await decryptApiKey(stored.encryptedApiKey!)).decrypted).toBe(input.apiKey) + expect( + await db + .select() + .from(knowledgeConnectorPermissionSnapshot) + .where(eq(knowledgeConnectorPermissionSnapshot.connectorId, connector.id)) + ).toEqual([]) + const docs = await db + .select() + .from(document) + .where(and(eq(document.connectorId, connector.id), isNull(document.deletedAt))) + expect(docs).toHaveLength(1) + expect(docs[0].externalId).toBe('file:orion.md') + expect(docs[0].acl).toEqual(['ws']) + await expect + .poll( + async () => { + const [doc] = await db.select().from(document).where(eq(document.id, docs[0].id)) + return doc.processingStatus + }, + { timeout: 15000 } + ) + .toBe('completed') + + const reader = { ...principal, userId: ids.bobId } + const results = await searchKnowledge.execute({ + principal: reader, + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + searchMode: 'hybrid', + topK: 5, + }, + }) + expect(results.results.map((result) => result.documentId)).toContain(docs[0].id) + await expect( + readKnowledgeDocument.execute({ + principal: reader, + input: { knowledgeBaseId: ids.knowledgeBaseId, documentId: docs[0].id }, + }) + ).resolves.toBeDefined() + await expect( + readKnowledgeDocument.execute({ + principal: { ...principal, userId: generateId() }, + input: { knowledgeBaseId: ids.knowledgeBaseId, documentId: docs[0].id }, + }) + ).rejects.toBeDefined() + + await updateKnowledgeConnector.execute({ + principal, + input: { + connectorId: connector.id, + updates: { sourceConfig: { ...sourceConfig, ref: 'master' } }, + }, + }) + await expect + .poll( + async () => { + const [row] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connector.id)) + return { + status: row.status, + error: row.lastSyncError, + synced: Boolean(row.lastSyncAt && row.lastSyncAt > stored.lastSyncAt!), + } + }, + { timeout: 15000 } + ) + .toEqual({ status: 'active', error: null, synced: true }) + }, + 30000 +) + +it('still refuses workspace access on a Search index before contacting GitLab', async () => { + const searchId = generateId() + await db.insert(knowledgeBase).values({ + id: searchId, + userId: ids.aliceId, + workspaceId: ids.workspaceId, + name: 'Search access regression', + isSearchIndex: true, + }) + const requestCount = fixture.fetch.mock.calls.length + await expect( + createKnowledgeConnector.execute({ + principal, + input: { ...input, knowledgeBaseId: searchId }, + }) + ).rejects.toThrow('Search sources must support per-person access or source permissions') + expect(fixture.fetch.mock.calls).toHaveLength(requestCount) +}) diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 918a76321fe..18783a33c64 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -8,6 +8,7 @@ import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vites const mocks = vi.hoisted(() => ({ resolveKnowledgeBase: vi.fn(), + resolveEnvironment: vi.fn(), resolveConnector: vi.fn(), resolvePermission: vi.fn(), createConnector: vi.fn(), @@ -33,6 +34,10 @@ const mocks = vi.hoisted(() => ({ getForConnectors: vi.fn(), })) +vi.mock('@/lib/environment/utils', () => ({ + resolveEffectiveEnvironmentVariables: mocks.resolveEnvironment, +})) + vi.mock('@sim/audit', () => ({ AuditAction: { CONNECTOR_CREATED: 'connector.created', @@ -229,6 +234,66 @@ const delegatedPrincipal = { const BILLING = { actorUserId: 'shared-user', workspaceId: 'workspace-a' } as never describe('knowledge connector application use cases', () => { + const patInput = { + knowledgeBaseId: 'knowledge-b', + connectorType: 'gitlab', + apiKey: '{{GITLAB_PAT}}', + sourceConfig: { project: 'group/project' }, + syncIntervalMinutes: 1440, + } + const patPrincipal = { kind: 'session' as const, userId: 'writer', sessionId: 'session' } + + it('resolves an API-key reference using the caller and canonical workspace before persistence', async () => { + mocks.resolveEnvironment.mockResolvedValue({ GITLAB_PAT: { value: 'resolved-pat' } }) + mocks.createConnector.mockResolvedValueOnce({ + success: true, + connector: { id: 'new-connector', connectorType: 'gitlab', accessMode: 'workspace' }, + }) + await createKnowledgeConnector.execute({ principal: patPrincipal, input: patInput }) + expect(mocks.resolveEnvironment).toHaveBeenCalledWith('writer', 'workspace-b', ['GITLAB_PAT']) + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'resolved-pat' }) + ) + }) + + it.each([{}, { GITLAB_PAT: { value: '' } }])( + 'rejects missing or empty secrets before contacting the provider', + async (variables) => { + mocks.resolveEnvironment.mockResolvedValue(variables) + await expect( + createKnowledgeConnector.execute({ principal: patPrincipal, input: patInput }) + ).rejects.toMatchObject({ + code: 'validation', + message: 'Secret "GITLAB_PAT" is unavailable or empty', + }) + expect(mocks.createConnector).not.toHaveBeenCalled() + } + ) + + it('checks workspace write permission before resolving a secret', async () => { + mocks.resolvePermission.mockResolvedValue('read') + await expect( + createKnowledgeConnector.execute({ principal: patPrincipal, input: patInput }) + ).rejects.toMatchObject({ name: 'InsufficientWorkspacePermissionsError' }) + expect(mocks.resolveEnvironment).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + }) + + it('passes literal PATs through without reading secrets', async () => { + mocks.createConnector.mockResolvedValueOnce({ + success: true, + connector: { id: 'new-connector', connectorType: 'gitlab', accessMode: 'workspace' }, + }) + await createKnowledgeConnector.execute({ + principal: patPrincipal, + input: { ...patInput, apiKey: 'literal-pat' }, + }) + expect(mocks.resolveEnvironment).not.toHaveBeenCalled() + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'literal-pat' }) + ) + }) + it('refuses workspace-wide or unreviewed source ingestion into the canonical search index', async () => { mocks.resolveKnowledgeBase.mockResolvedValue({ ...crossWorkspaceContext, @@ -237,6 +302,7 @@ describe('knowledge connector application use cases', () => { mocks.resolvePermission.mockResolvedValue('admin') for (const [connectorType, accessMode] of [ ['confluence', 'workspace'], + ['gitlab', 'workspace'], ['notion', 'members'], ] as const) { await expect( @@ -558,6 +624,45 @@ describe('knowledge connector application use cases', () => { ) }) + it.each(['result', 'exception'] as const)( + 'redacts stored tokens from provider validation %s when editing a connector', + async (failure) => { + const message = 'Provider rejected existing-pat' + if (failure === 'exception') { + mocks.validateConnectorConfig.mockRejectedValueOnce( + new OrchestrationError('validation', message) + ) + } else { + mocks.validateConnectorConfig.mockResolvedValueOnce({ valid: false, error: message }) + } + const result = validateConnectorSourceConfig({ + principal: patPrincipal, + requestId: 'request', + workspaceId: 'workspace-b', + actingUserId: 'writer', + sourceConfig: { owner: 'acme', repo: 'handbook' }, + connector: { + ...connectorContext.connector, + connectorType: 'github', + credentialId: null, + encryptedApiKey: 'persisted-cipher', + accessMode: 'workspace', + } as Parameters[0]['connector'], + }) + if (failure === 'exception') { + await expect(result).rejects.toMatchObject({ + code: 'validation', + message: 'Provider rejected [REDACTED]', + }) + } else { + await expect(result).resolves.toEqual({ + errorCode: 'validation', + message: 'Provider rejected [REDACTED]', + }) + } + } + ) + it.each([ [ 'create', diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index e6f317fd4a2..b347c5d8465 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -11,6 +11,7 @@ import { knowledgeConnectorMemberSyncLog, knowledgeConnectorSyncLog, } from '@sim/db/schema' +import { toError } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' import { and, asc, desc, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm' import type { ConnectorDocumentFilter } from '@/lib/api/contracts/knowledge/connectors' @@ -26,8 +27,10 @@ import { resourceScopeFields, resourceScopeFromOwner, } from '@/lib/core/resource-scope' +import { redactKnownSensitiveValues } from '@/lib/core/security/redaction' import { generateRequestId } from '@/lib/core/utils/request' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' +import { resolveEffectiveEnvironmentVariables } from '@/lib/environment/utils' import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' @@ -496,14 +499,21 @@ export async function validateConnectorSourceConfig(input: { ) } } - const validation = await connectorConfig.validateConfig( - resolved.accessToken, - input.sourceConfig, - validationContext - ) + const validation = await connectorConfig + .validateConfig(resolved.accessToken, input.sourceConfig, validationContext) + .catch((error: unknown) => { + const sanitized = toError(error) + sanitized.message = redactKnownSensitiveValues(sanitized.message, [resolved.accessToken]) + throw sanitized + }) return validation.valid ? null - : { message: validation.error || 'Invalid source configuration', errorCode: 'validation' } + : { + message: redactKnownSensitiveValues(validation.error || 'Invalid source configuration', [ + resolved.accessToken, + ]), + errorCode: 'validation', + } } export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ @@ -768,6 +778,26 @@ async function summarizeConnectorMembers( return { active: row?.active ?? 0, suspended: row?.suspended ?? 0, stale: row?.stale ?? 0 } } +/** Resolves a secret reference at setup time; the connector stores an encrypted token snapshot. */ +async function resolveConnectorApiKey( + apiKey: string | undefined, + principal: Principal, + workspaceId: string | undefined +): Promise { + const name = apiKey?.trim().match(/^\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}$/)?.[1] + if (!name) return apiKey + const userId = resolvePrincipalSubjectUserId(principal) + if (!userId) { + throw new OrchestrationError('forbidden', 'Secret references require a user identity') + } + const variables = await resolveEffectiveEnvironmentVariables(userId, workspaceId, [name]) + const value = Object.hasOwn(variables, name) ? variables[name].value : undefined + if (!value) { + throw new OrchestrationError('validation', `Secret "${name}" is unavailable or empty`) + } + return value +} + async function executeCreateKnowledgeConnector( { principal, @@ -882,19 +912,20 @@ async function executeCreateKnowledgeConnector( sourceConfig: membersBinding?.sourceConfig ?? input.sourceConfig, }) if (membersBinding) membersBinding = { ...membersBinding, sourceConfig } + const apiKey = await resolveConnectorApiKey(input.apiKey, principal, workspaceId) const permissionChange = input.permissionConfig ? await prepareConnectorPermissions(input.connectorType, { accessMode: input.accessMode ?? 'workspace', sourceConfig, permissionConfig: input.permissionConfig, - apiKey: input.apiKey, + apiKey, }) : undefined const outcome = await performCreateKnowledgeConnector({ knowledgeBase: connectorTarget(context), connectorType: input.connectorType, credentialId: input.credentialId, - apiKey: input.apiKey, + apiKey, permissionChange, /** Members mode stores the config with its listing caps cleared. */ sourceConfig, @@ -1089,7 +1120,7 @@ export const updateKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ accessMode: connector.accessMode, sourceConfig: updates.sourceConfig ?? (connector.sourceConfig as Record), permissionConfig, - apiKey, + apiKey: await resolveConnectorApiKey(apiKey, principal, context.workspaceId), existing: connector, }) if (permissionChange && !permissionConfig && apiKey === undefined) { diff --git a/apps/sim/lib/knowledge/connectors/permission-config.server.ts b/apps/sim/lib/knowledge/connectors/permission-config.server.ts index 67f769b17bb..85e5f212b9f 100644 --- a/apps/sim/lib/knowledge/connectors/permission-config.server.ts +++ b/apps/sim/lib/knowledge/connectors/permission-config.server.ts @@ -1,4 +1,6 @@ +import { toError } from '@sim/utils/errors' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { redactKnownSensitiveValues } from '@/lib/core/security/redaction' import type { ConnectorPermissionSummary, PrepareConnectorPermissionsInput, @@ -30,7 +32,11 @@ export async function prepareConnectorPermissions( } return undefined } - return capability.prepare(input) + return capability.prepare(input).catch((error: unknown) => { + const sanitized = toError(error) + sanitized.message = redactKnownSensitiveValues(sanitized.message, [input.apiKey ?? '']) + throw sanitized + }) } export async function readConnectorPermissionSummary(connectorType: string, connectorId: string) { diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 13356fcc73b..d3dea29604b 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -201,6 +201,33 @@ describe('performCreateKnowledgeConnector', () => { } ) + it.each(['result', 'exception'])( + 'redacts credentials from provider validation %s errors', + async (failure) => { + const token = 'private/value' + const message = `Invalid credential ${token} (${encodeURIComponent(token)})` + if (failure === 'result') + mockValidateGitHub.mockResolvedValueOnce({ valid: false, error: message }) + else mockValidateGitHub.mockRejectedValueOnce(new OrchestrationError('validation', message)) + const request = performCreateKnowledgeConnector({ + ...createParams, + connectorType: 'github', + apiKey: token, + }) + const expected = { + errorCode: 'validation', + error: 'Invalid credential [REDACTED] ([REDACTED])', + } + if (failure === 'result') await expect(request).resolves.toMatchObject(expected) + else + await expect(request).rejects.toMatchObject({ + code: expected.errorCode, + message: expected.error, + }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + } + ) + it('validates and encrypts a GitHub PAT without resolving an OAuth account or returning the secret', async () => { dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'kb-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([ diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index cce1784e0b9..5004501463f 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -9,7 +9,7 @@ import { knowledgeConnectorMember, } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' +import { getErrorMessage, getPostgresErrorCode, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq, isNull, sql } from 'drizzle-orm' import { encryptApiKey } from '@/lib/api-key/crypto' @@ -32,6 +32,7 @@ import { resourceScopeFromOwner, } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' +import { redactKnownSensitiveValues } from '@/lib/core/security/redaction' import { generateRequestId } from '@/lib/core/utils/request' import type { DbOrTx } from '@/lib/db/types' import { @@ -342,14 +343,17 @@ export async function performCreateKnowledgeConnector( ...(accessMode === 'members' ? PER_MEMBER_LISTING_CONTEXT : {}), } params.permissionChange?.populateSyncContext(validationContext, 'setup') - const configValidation = await connectorConfig.validateConfig( - accessToken, - sourceConfig, - validationContext - ) + const configValidation = await connectorConfig + .validateConfig(accessToken, sourceConfig, validationContext) + .catch((error: unknown) => { + const sanitized = toError(error) + sanitized.message = redactKnownSensitiveValues(sanitized.message, [accessToken]) + throw sanitized + }) if (!configValidation.valid) { return fail( - configValidation.error || + (configValidation.error && + redactKnownSensitiveValues(configValidation.error, [accessToken])) || `The ${connectorType} connector rejected sourceConfig without a reason — re-check its required fields in knowledgebases/connectors/${connectorType}.json before retrying; the same config will fail again.`, 'validation' )