diff --git a/CHANGELOG.md b/CHANGELOG.md index 348a804c87..8ce722fd93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -189,6 +189,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- `codegraph_explore` finds code containing storage keys, command-line flags, and event names; rebuild existing indexes to enable these matches. + - **Codex and Astra read project guidance from `AGENTS.md`.** The canonical agent guide now lives in `AGENTS.md` (with a nested `docs/AGENTS.md` for long validation notes); `CLAUDE.md` is a thin `@AGENTS.md` wrapper for Claude Code. Codex/Astra no longer miss the old CLAUDE-only instructions. - **Next.js pages and their navigation are in the graph.** App Router pages (`app/(group)/blog/[slug]/page.tsx` → `/blog/:slug`) and Pages Router pages are routes bound to the component they export, and ``, an internal ``, `router.push` / `router.replace` (`next/navigation` and `next/router`), `redirect()` / `permanentRedirect()` and the middleware's `NextResponse.redirect(…)` are `navigates` edges between them. `app/api/**/route.ts` exports (`GET`, `POST`, …) are endpoints bound to their functions, and `pages/api/*` handlers are `ANY /api/…`. Re-index after upgrading. diff --git a/__tests__/kernel-retry-materialize.test.ts b/__tests__/kernel-retry-materialize.test.ts index 23965bd2dd..83e4db4239 100644 --- a/__tests__/kernel-retry-materialize.test.ts +++ b/__tests__/kernel-retry-materialize.test.ts @@ -56,7 +56,7 @@ describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { it('storeExtractionResult persists the decoded nodes of a raw kernel result', async () => { const source = 'def target_fn(root, mission_path):\n' + - ' return (root, mission_path)\n' + + ' return (root, mission_path, "adapter.mission")\n' + '\n' + 'class Adapter:\n' + ' def adapt(self):\n' + @@ -93,6 +93,7 @@ describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { expect(nodes.length).toBe(raw!.counts.nodes); expect(nodes.map((n) => n.name)).toContain('target_fn'); expect(nodes.map((n) => n.name)).toContain('Adapter'); + expect(cg.findLiteralSeedIds('"adapter.mission"').map(id => cg.getNode(id)?.name)).toEqual(['target_fn']); }); }); diff --git a/__tests__/literal-capture.test.ts b/__tests__/literal-capture.test.ts new file mode 100644 index 0000000000..a27406a51c --- /dev/null +++ b/__tests__/literal-capture.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest'; +import type { Node } from '../src/types'; +import { captureLiterals, isSeedLiteral, seedLiteralsInQuery } from '../src/extraction/literal-capture'; + +function node(id: string, kind: Node['kind'], startLine: number, endLine: number): Node { + return { + id, kind, name: id, qualifiedName: id, filePath: 'src/a.ts', language: 'typescript', + startLine, endLine, startColumn: 0, endColumn: Number.MAX_SAFE_INTEGER, updatedAt: 0, + }; +} + +describe('isSeedLiteral', () => { + it('keeps storage keys, flags, dotted names and paths', () => { + for (const v of ['bompus_custom_ds_players', '--start', '-v', 'draft.pick', 'api/v1/users', 'ns:event']) + expect(isSeedLiteral(v), v).toBe(v !== '-v'); + }); + it('drops plain words, prose, and values that start with a separator', () => { + for (const v of ['ready', 'Error', 'not found', './utils', '../x', '', 'a_b']) + expect(isSeedLiteral(v), v).toBe(false); + }); +}); + +describe('seedLiteralsInQuery', () => { + it('finds quoted spans and bare runs, stripping surrounding punctuation', () => { + expect(seedLiteralsInQuery('who writes "bompus_custom_ds_players" (via --start)?')) + .toEqual(['bompus_custom_ds_players', '--start']); + }); + it('returns nothing for a symbol-anchored question', () => { + expect(seedLiteralsInQuery('callers of espnPlayerKey in shared')).toEqual([]); + }); +}); + +describe('captureLiterals', () => { + const source = [ + `import { x } from './utils/helpers';`, // 1: path starts with '.', never qualifies + `const KEY = 'bompus_custom_ds_players';`, // 2: top level → file node + `export function save() {`, // 3 + ` storage.set('bompus_custom_ds_players', 1);`, // 4 + ` log('ready');`, // 5: plain word + ` emit(\`draft.pick\`); emit(\`draft.\${n}\`);`, // 6: second one interpolates + `}`, // 7 + `export class Boot { start() { run('--start'); } }`,// 8 + ].join('\n'); + + it('attributes each literal to the innermost enclosing symbol, else the file', () => { + const file = node('file:src/a.ts', 'file', 1, 8); + const save = node('save', 'function', 3, 7); + const boot = node('Boot', 'class', 8, 8); + const start = node('start', 'method', 8, 8); + const imp = node('./utils/helpers', 'import', 1, 1); + const nodes = [file, imp, save, boot, start]; + captureLiterals(source, nodes); + expect(file.literals).toEqual(['bompus_custom_ds_players']); + expect(save.literals).toEqual(['bompus_custom_ds_players', 'draft.pick']); + expect(start.literals).toEqual(['--start']); + expect(boot.literals).toBeUndefined(); + expect(imp.literals).toBeUndefined(); + }); + + it('dedupes per node and caps at 32', () => { + const many = Array.from({ length: 40 }, (_, i) => `k('key_${i}'); k('key_${i}');`).join('\n'); + const fn = node('f', 'function', 1, 40); + captureLiterals(many, [node('file:src/a.ts', 'file', 1, 40), fn]); + expect(fn.literals).toHaveLength(32); + expect(new Set(fn.literals).size).toBe(32); + }); + + it('uses UTF-16 columns to distinguish same-line siblings after non-ASCII source', () => { + const prefix = '/* café 😀 */ '; + const first = "function writer(){return 'cache.write';}"; + const second = "function reader(){return 'cache.read';}"; + const writer = { ...node('writer', 'function', 1, 1), startColumn: prefix.length, endColumn: (prefix + first).length }; + const reader = { ...node('reader', 'function', 1, 1), startColumn: (prefix + first).length, endColumn: (prefix + first + second).length }; + captureLiterals(prefix + first + second, [writer, reader]); + expect(writer.literals).toEqual(['cache.write']); + expect(reader.literals).toEqual(['cache.read']); + }); +}); diff --git a/__tests__/literal-compiled-index.test.ts b/__tests__/literal-compiled-index.test.ts new file mode 100644 index 0000000000..ceb39d9588 --- /dev/null +++ b/__tests__/literal-compiled-index.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import type CodeGraph from '../src'; +import type { QueryBuilder } from '../src/db/queries'; + +const built = path.join(__dirname, '..', 'dist', 'index.js'); +const kernel = path.join(__dirname, '..', 'codegraph-kernel', 'prebuilds', `${process.platform}-${process.arch}`, 'codegraph-kernel.node'); + +// Source-mode suites cannot exercise the compiled parse/store worker boundary. +describe.runIf(fs.existsSync(built))('literal persistence through compiled indexing', () => { + let dir: string | undefined; + let cg: CodeGraph | undefined; + afterEach(() => { + cg?.destroy(); + cg = undefined; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + dir = undefined; + vi.unstubAllEnvs(); + }); + + for (const mode of ['native-worker', 'native-main', 'wasm-worker']) { + it.runIf(mode === 'wasm-worker' || fs.existsSync(kernel))(`${mode}: fresh, unchanged, and migrated indexes retain exact owners`, async () => { + vi.stubEnv('CODEGRAPH_PARSE_WORKERS', '1'); + vi.stubEnv('CODEGRAPH_KERNEL', mode === 'wasm-worker' ? '0' : '1'); + vi.stubEnv('CODEGRAPH_NO_STORE_WORKER', mode === 'native-main' ? '1' : '0'); + const BuiltCodeGraph: typeof CodeGraph = require(built).default; + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-literal-compiled-')); + fs.writeFileSync(path.join(dir, 'cache.py'), "def persist():\n return 'cache.write'\n"); + fs.writeFileSync(path.join(dir, 'siblings.ts'), "/* café 😀 */ export function writer(){return 'sibling.write';} export function reader(){return 'sibling.read';}\n"); + cg = await BuiltCodeGraph.init(dir, { silent: true }); + const names = (key: string) => cg!.findLiteralSeedIds(key).map(id => cg!.getNode(id)?.name); + const assertOwners = () => { + expect(names('cache.write')).toEqual(['persist']); + expect(names('sibling.write')).toEqual(['writer']); + expect(names('sibling.read')).toEqual(['reader']); + }; + await cg.indexAll(); + assertOwners(); + await cg.indexAll(); + assertOwners(); + + // A schema-only migration has no literal data; unchanged source still needs backfill. + const queries = (cg as unknown as { queries: QueryBuilder }).queries; + queries.replaceLiteralsForFile('cache.py', []); + queries.replaceLiteralsForFile('siblings.ts', []); + queries.setMetadata('indexed_with_extraction_version', '26'); + expect(cg.isIndexStale()).toBe(true); + await cg.indexAll(); + assertOwners(); + expect(cg.isIndexStale()).toBe(false); + fs.rmSync(path.join(dir, 'cache.py')); + await cg.indexAll(); + expect(names('cache.write')).toEqual([]); + expect(names('sibling.write')).toEqual(['writer']); + }); + } +}); diff --git a/__tests__/literal-seeds.test.ts b/__tests__/literal-seeds.test.ts new file mode 100644 index 0000000000..35c871a5db --- /dev/null +++ b/__tests__/literal-seeds.test.ts @@ -0,0 +1,253 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import CodeGraph from '../src'; +import { QueryBuilder } from '../src/db/queries'; +import { createDatabase, type SqliteDatabase } from '../src/db/sqlite-adapter'; +import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from '../src/db/migrations'; +import { ToolHandler } from '../src/mcp/tools'; + +/** + * The string-anchored explore path: a query that names a storage key or a CLI + * flag reaches the symbols whose bodies hold it, through the `literals` side + * table (written on the node write path, deleted with the file, cleared by a + * full index), and those symbols lead the ranking. + */ +describe('literal seeds — index, query, and lifecycle', () => { + let dir: string; + let cg: CodeGraph; + const queries = () => (cg as unknown as { queries: QueryBuilder }).queries; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-literals-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'writer.ts'), + `export function persistRows(rows: unknown[]) { + chrome.storage.local.set({ 'bompus_custom_ds_players': rows }); +} +export function unrelatedHelper() { return 'ready'; } +`, + ); + fs.writeFileSync( + path.join(dir, 'src', 'reader.ts'), + `export async function readRows() { + const got = await chrome.storage.local.get('bompus_custom_ds_players'); + return got; +} +`, + ); + fs.writeFileSync( + path.join(dir, 'src', 'cli.ts'), + `export function main(argv: string[]) { + if (argv.includes('--start')) return start(); + return 0; +} +function start() { return 1; } +`, + ); + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('resolves a storage key to every symbol whose body holds it', () => { + const ids = queries().findNodeIdsByLiteral(['bompus_custom_ds_players']); + const names = ids.map((id) => cg.getNode(id)?.name).sort(); + expect(names).toEqual(['persistRows', 'readRows']); + }); + + it('recommends rebuilding an index from before literal capture', () => { + const current = queries().getMetadata('indexed_with_extraction_version'); + try { + queries().setMetadata('indexed_with_extraction_version', '26'); + expect(cg.isIndexStale()).toBe(true); + } finally { + queries().setMetadata('indexed_with_extraction_version', current!); + } + expect(cg.isIndexStale()).toBe(false); + }); + + it('a quoted key in a prose query puts the writers at the top of the subgraph', async () => { + const sub = await cg.findRelevantContext('which modules write "bompus_custom_ds_players" to storage'); + const rootNames = sub.roots.map((id) => sub.nodes.get(id)?.name); + expect(rootNames.slice(0, 2).sort()).toEqual(['persistRows', 'readRows']); + expect(rootNames).not.toContain('unrelatedHelper'); + }); + + it('a bare CLI flag seeds too', () => { + const ids = queries().findNodeIdsByLiteral(['--start']); + expect(ids.map((id) => cg.getNode(id)?.name)).toEqual(['main']); + }); + + it('CODEGRAPH_LITERAL_SEEDS=0 turns the seed off (the ablation switch)', async () => { + process.env.CODEGRAPH_LITERAL_SEEDS = '0'; + try { + const sub = await cg.findRelevantContext('bompus_custom_ds_players'); + const names = [...sub.nodes.values()].map((n) => n.name); + expect(names).not.toContain('persistRows'); + } finally { + delete process.env.CODEGRAPH_LITERAL_SEEDS; + } + }); + + it("a file's rows leave with its nodes on re-index, and a full index clears the table", async () => { + fs.writeFileSync(path.join(dir, 'src', 'reader.ts'), `export function readRows() { return null; }\n`); + await cg.indexFiles(['src/reader.ts']); + let names = queries().findNodeIdsByLiteral(['bompus_custom_ds_players']).map((id) => cg.getNode(id)?.name); + expect(names).toEqual(['persistRows']); + + fs.rmSync(path.join(dir, 'src', 'writer.ts')); + await cg.indexAll(); + expect(queries().findNodeIdsByLiteral(['bompus_custom_ds_players'])).toEqual([]); + }); +}); + +/** + * A literal held through a constant lives in a small file with no callers. In explore's file + * sort that file must take a source slot the way a file defining a named symbol does; on graph + * centrality alone it loses the slot to a hub file that never mentions the literal. + */ +describe('literal seeds — explore renders the holder file', () => { + let dir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litrender-')); + fs.mkdirSync(path.join(dir, 'shared'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'callers'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'shared', 'bag.ts'), + `const BAG_KEY = 'bompus_custom_ds_players'; +export function persistBag(rows: unknown[]) { + chrome.storage.local.set({ [BAG_KEY]: rows }); +} +`, + ); + // The hub: matches the query word "storage" by name and is called from many files. + fs.writeFileSync( + path.join(dir, 'shared', 'storage.ts'), + `export function storageSet(key: string, value: unknown) { + return chrome.storage.local.set({ [key]: value }); +} +export function storageGet(key: string) { + return chrome.storage.local.get(key); +} +`, + ); + for (let i = 0; i < 8; i++) { + fs.writeFileSync( + path.join(dir, 'callers', `writer${i}.ts`), + `import { storageSet, storageGet } from '../shared/storage'; +export function storageWriter${i}() { + storageGet('k${i}'); + return storageSet('k${i}', ${i}); +} +`, + ); + } + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + handler = new ToolHandler(cg); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('the file holding the quoted key takes the first source slot', async () => { + const res = await handler.execute('codegraph_explore', { + query: 'which modules write "bompus_custom_ds_players" to storage', + maxFiles: 1, + }); + const text = res.content[0].text as string; + const sourced = [...text.matchAll(/^\*\*`(.+?)`\*\* —/gm)].map((m) => m[1]); + expect(sourced).toEqual(['shared/bag.ts']); + }); +}); + +/** + * A key held in more files than the entry-point cap (`searchLimit`, 3 by default and 8 in + * explore) must reach every holder: the cap is for text matches, and the literal lookup is + * already bounded. Before this, holders competed for the cap by path order and `shared/` + * lost to `scripts/`. + */ +describe('literal seeds — every holder reaches the subgraph past the entry cap', () => { + let dir: string; + let cg: CodeGraph; + const N = 10; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litcap-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + for (let i = 0; i < N; i++) { + fs.writeFileSync( + path.join(dir, 'src', `holder${i}.ts`), + `export function write${i}(rows: unknown[]) { + chrome.storage.local.set({ 'bompus_custom_ds_players': rows }); +} +`, + ); + } + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('all holders are roots with the default searchLimit of 3', async () => { + const sub = await cg.findRelevantContext('which modules write "bompus_custom_ds_players" to storage'); + const rootFiles = new Set(sub.roots.map((id) => sub.nodes.get(id)?.filePath)); + expect(rootFiles.size).toBe(N); + }); + + it('explore renders every holder with the default file cap', async () => { + const res = await new ToolHandler(cg).execute('codegraph_explore', { + query: 'which modules write "bompus_custom_ds_players" to storage', + }); + const text = res.content[0].text as string; + const sourced = [...text.matchAll(/^\*\*`(.+?)`\*\* —/gm)].map((m) => m[1]); + expect(sourced.length).toBe(N); + }); +}); + +describe('literals — v12 migration', () => { + let dir: string; + let db: SqliteDatabase | null = null; + + afterEach(() => { + db?.close(); + db = null; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function makeV11Db(): SqliteDatabase { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litmigrate-')); + const conn = createDatabase(path.join(dir, 'legacy.db')).db; + conn.exec(` + CREATE TABLE schema_versions (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL, description TEXT); + INSERT INTO schema_versions VALUES (11, 0, 'legacy'); + `); + db = conn; + return conn; + } + + it('creates the empty table and is idempotent on replay', () => { + const conn = makeV11Db(); + runMigrations(conn, 11); + expect(getCurrentVersion(conn)).toBe(CURRENT_SCHEMA_VERSION); + expect((conn.prepare('SELECT COUNT(*) AS n FROM literals').get() as { n: number }).n).toBe(0); + conn.prepare('DELETE FROM schema_versions WHERE version >= 12').run(); + expect(() => runMigrations(conn, 11)).not.toThrow(); + }); +}); diff --git a/src/context/index.ts b/src/context/index.ts index 6cea28c2d1..4006988f8e 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -172,8 +172,16 @@ const DEFAULT_FIND_OPTIONS: Required = { edgeKinds: [], nodeKinds: HIGH_VALUE_NODE_KINDS, // Filter out imports/exports by default seedNames: [], // Segment-vocab supplement — filled by the facade + seedNodeIds: [], // Literal-table seeds — filled by the facade }; +/** + * Score for a literal seed. Above the BM25 magnitudes the FTS channel returns + * (tens to hundreds) and every additive boost below, so a symbol holding the + * quoted literal leads the ranking whatever else the query matched. + */ +const LITERAL_SEED_SCORE = 1000; + // Re-export the low-confidence sentinel (defined in a dependency-free leaf so // the MCP layer can import it without pulling this module's deps onto the // cold-start path). Builder code below uses the imported binding directly. @@ -483,7 +491,17 @@ export class ContextBuilder { // Step 2: Look up exact matches for extracted symbols let exactMatches: SearchResult[] = []; - if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0) { + // Literal seeds bypass every size trim below: the lookup already bounds + // them, and a storage key held in eleven files must reach all eleven, not + // the first `searchLimit` by path order. + const literalSeedIds = new Set(opts.seedNodeIds); + const keepSeedsThenTop = (results: SearchResult[], n: number): SearchResult[] => { + if (literalSeedIds.size === 0) return results.slice(0, n); + const seeds = results.filter((r) => literalSeedIds.has(r.node.id)); + const rest = results.filter((r) => !literalSeedIds.has(r.node.id)).slice(0, n); + return [...seeds, ...rest]; + }; + if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0 || opts.seedNodeIds.length > 0) { try { if (symbolsFromQuery.length > 0) { // Get more results so we can apply co-location boosting before trimming @@ -493,6 +511,20 @@ export class ContextBuilder { }); } + // Literal seeds: the query quoted a string that lives in these + // symbols' bodies. No name matches it, so they enter here, first and + // at a fixed top score; the co-location pass below still compounds + // several hits in one file. + if (opts.seedNodeIds.length > 0) { + const byId = this.queries.getNodesByIds(opts.seedNodeIds); + for (const id of opts.seedNodeIds) { + const node = byId.get(id); + if (!node || exactMatches.some((r) => r.node.id === id)) continue; + exactMatches.push({ node, score: LITERAL_SEED_SCORE }); + } + logDebug('Literal seed matches', { seedNodeIds: opts.seedNodeIds, added: byId.size }); + } + // Step 2a: segment-vocabulary seeds. Word-level query terms cannot // reach camelCase names through FTS (one token per name), so the // caller resolves query words → names via the segment vocab and hands @@ -538,7 +570,7 @@ export class ContextBuilder { } // Trim back to reasonable size - exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 2)); + exactMatches = keepSeedsThenTop(exactMatches, Math.ceil(opts.searchLimit * 2)); logDebug('Exact symbol matches', { count: exactMatches.length }); } catch (error) { logDebug('Exact symbol lookup failed', { error: String(error) }); @@ -587,7 +619,7 @@ export class ContextBuilder { } } exactMatches.sort((a, b) => b.score - a.score); - exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3)); + exactMatches = keepSeedsThenTop(exactMatches, Math.ceil(opts.searchLimit * 3)); } // Step 2c: Match an exact Han filename (with or without its extension). @@ -1006,7 +1038,7 @@ export class ContextBuilder { // compound) have now contributed. Sort by score so multi-term matches from // later steps can outrank dampened single-term matches from earlier steps. searchResults.sort((a, b) => b.score - a.score); - searchResults = searchResults.slice(0, opts.searchLimit * 3); + searchResults = keepSeedsThenTop(searchResults, opts.searchLimit * 3); // Filter by minimum score let filteredResults = searchResults.filter((r) => r.score >= opts.minScore); @@ -1030,7 +1062,7 @@ export class ContextBuilder { // With 36 entry points and maxNodes=120, each gets only 3 nodes — useless. // Cap to searchLimit so each entry point gets a meaningful traversal budget. if (filteredResults.length > opts.searchLimit) { - filteredResults = filteredResults.slice(0, opts.searchLimit); + filteredResults = keepSeedsThenTop(filteredResults, opts.searchLimit); } // Confidence signal for the honest-handoff footer (consumed in buildContext). diff --git a/src/db/migrations.ts b/src/db/migrations.ts index bb7b84bf53..9d7c9d5940 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 11; +export const CURRENT_SCHEMA_VERSION = 12; /** * Migration definition @@ -216,6 +216,26 @@ const migrations: Migration[] = [ `); }, }, + { + version: 12, + description: + 'Add literals — identifier-like string literal → enclosing symbol, so explore seeds on storage keys and flags', + up: (db) => { + // DDL only. No backfill: the values come from file CONTENT the migration + // cannot see, so the table stays empty until the next full index and + // explore behaves exactly as before for those queries. Keep in lockstep + // with schema.sql. + db.exec(` + CREATE TABLE IF NOT EXISTS literals ( + value TEXT NOT NULL, + node_id TEXT NOT NULL, + file_path TEXT NOT NULL, + PRIMARY KEY (value, node_id) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_literals_file ON literals(file_path); + `); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index a12e7c2643..b1112d8ddb 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -273,6 +273,7 @@ export class QueryBuilder { updateNode?: SqliteStatement; deleteNode?: SqliteStatement; deleteNodesByFile?: SqliteStatement; + deleteLiteralsByFile?: SqliteStatement; getNodeById?: SqliteStatement; getNodesByFile?: SqliteStatement; getNodesByKind?: SqliteStatement; @@ -520,6 +521,43 @@ export class QueryBuilder { // import-only name can never be surfaced (getSegmentMatches requires a // real definition), so its rows would only inflate the rarity statistics. if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); + const literalRows: unknown[][] = []; + this.collectLiteralRows(node, literalRows); + this.insertLiteralRows(literalRows); + } + + /** Rows for the `literals` side table (see schema.sql) — one per captured literal. */ + private collectLiteralRows(node: Node, rows: unknown[][]): void { + for (const value of node.literals ?? []) rows.push([value, node.id, node.filePath]); + } + + private insertLiteralRows(rows: unknown[][]): void { + this.runBatched( + 'insertLiterals', + 'INSERT OR IGNORE INTO literals (value, node_id, file_path) VALUES ', + '(?,?,?)', + rows + ); + } + + /** + * Node ids whose body holds one of `values` verbatim, joined to nodes so a + * row left behind by a stale index is never surfaced. Explore seeds on these + * ahead of every name-derived candidate. + */ + findNodeIdsByLiteral(values: string[], limit = 24): string[] { + if (values.length === 0) return []; + const placeholders = values.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT DISTINCT l.node_id AS id FROM literals l + JOIN nodes n ON n.id = l.node_id + WHERE l.value IN (${placeholders}) + ORDER BY n.kind = 'file', n.file_path, n.start_line + LIMIT ?` + ) + .all(...values, limit) as Array<{ id: string }>; + return rows.map((r) => r.id); } /** Which node kinds contribute their name to the segment vocabulary — the @@ -551,6 +589,7 @@ export class QueryBuilder { // per-.run() call overhead dominates the store phase on full indexes. const rows: unknown[][] = []; const segmentRows: unknown[][] = []; + const literalRows: unknown[][] = []; for (const node of nodes) { if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { console.error('[CodeGraph] Skipping node with missing required fields:', { @@ -587,6 +626,7 @@ export class QueryBuilder { node.updatedAt ?? Date.now(), ]); if (this.isSegmentableKind(node.kind)) this.collectNameSegmentRows(node.name, segmentRows); + this.collectLiteralRows(node, literalRows); } this.runBatched( 'insertNodes', @@ -606,6 +646,7 @@ export class QueryBuilder { '(?,?)', segmentRows ); + this.insertLiteralRows(literalRows); })(); } @@ -773,9 +814,28 @@ export class QueryBuilder { this.nodeCache.delete(id); } } + if (!this.stmts.deleteLiteralsByFile) { + this.stmts.deleteLiteralsByFile = this.db.prepare('DELETE FROM literals WHERE file_path = ?'); + } + this.stmts.deleteLiteralsByFile.run(filePath); this.stmts.deleteNodesByFile.run(filePath); } + /** Refresh extracted literals even when unchanged source keeps its existing nodes. */ + replaceLiteralsForFile(filePath: string, nodes: Node[]): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM literals WHERE file_path = ?').run(filePath); + const rows: unknown[][] = []; + for (const node of nodes) this.collectLiteralRows(node, rows); + this.insertLiteralRows(rows); + })(); + } + + /** Full indexing repopulates present files and removes literals from deleted files. */ + clearLiterals(): void { + this.db.exec('DELETE FROM literals'); + } + // =========================================================================== // Name-segment vocabulary (prompt-hook graph-derived gate) // =========================================================================== @@ -4023,6 +4083,7 @@ export class QueryBuilder { this.db.transaction(() => { this.db.exec('DELETE FROM unresolved_refs'); this.db.exec('DELETE FROM edges'); + this.db.exec('DELETE FROM literals'); this.db.exec('DELETE FROM nodes'); this.db.exec('DELETE FROM files'); })(); diff --git a/src/db/schema.sql b/src/db/schema.sql index 91ae4805ba..12795a23bc 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -161,6 +161,22 @@ CREATE TABLE IF NOT EXISTS name_segment_vocab ( PRIMARY KEY (segment, name) ) WITHOUT ROWID; +-- Identifier-like string literal → the symbol whose body holds it +-- (extraction/literal-capture.ts). Lets an explore query that names a storage +-- key, CLI flag, or event name seed on its readers and writers: nodes_fts +-- covers names, docstrings and signatures, never string contents. Written on +-- the node write path; a file's rows are deleted with its nodes and a full +-- index clears the table. Reads still join nodes, so a stale row is never +-- surfaced. Empty on migrated databases until the next full index (the +-- source strings are not recoverable from the graph). +CREATE TABLE IF NOT EXISTS literals ( + value TEXT NOT NULL, + node_id TEXT NOT NULL, + file_path TEXT NOT NULL, + PRIMARY KEY (value, node_id) +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS idx_literals_file ON literals(file_path); + -- Edge indexes. -- idx_edges_source / idx_edges_target are intentionally omitted — -- the (source, kind) and (target, kind) composites below cover the diff --git a/src/extraction/extraction-version.ts b/src/extraction/extraction-version.ts index 3691e95f0a..e41b5f6149 100644 --- a/src/extraction/extraction-version.ts +++ b/src/extraction/extraction-version.ts @@ -21,4 +21,4 @@ * turns the re-index hint into noise — keep it honest (see CLAUDE.md, "Honesty * in the product is load-bearing"). */ -export const EXTRACTION_VERSION = 27; +export const EXTRACTION_VERSION = 28; diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 0ca3145be2..0366cc8bbd 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -2945,6 +2945,7 @@ export class ExtractionOrchestrator { existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0; const incomingHasContent = result.nodes.length > 0; if (!existingIsMarker || !incomingHasContent) { + this.queries.replaceLiteralsForFile(filePath, result.nodes); return; // No changes } } diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index 1e71d170c3..cb8c3fe959 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -19,6 +19,7 @@ import { commonJsRequireRefs } from '../commonjs-requires'; import { EXTRACTORS } from '../languages'; import { getKernel, kernelSupports } from './loader'; import { decodeExtractBuffers } from './decode'; +import { captureLiterals } from '../literal-capture'; import { KERNEL_ABI_VERSION as LAYOUT_ABI, META as LAYOUT_META, @@ -234,7 +235,7 @@ export function tryKernelExtractRaw( buffers.arena.toString('utf8', errorsOff, errorsOff + errorsLen) ) as ExtractionResult['errors']; } - return { buffers, counts, errors }; + return { buffers: { ...buffers, literalSource: pre }, counts, errors }; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (message.includes('defer:')) { @@ -269,6 +270,7 @@ export function materializeKernelResult( filePath, language ); + if (b.literalSource !== undefined) captureLiterals(b.literalSource, decoded.nodes); decoded.durationMs = result.durationMs; // References read beside the tables (a CommonJS `require`) ride on the transport. if (result.unresolvedReferences.length > 0) decoded.unresolvedReferences.push(...result.unresolvedReferences); @@ -299,6 +301,16 @@ export function tryKernelExtract( POST_PASSES[language]?.(result, source); // Read beside the tables, as TreeSitterExtractor does (a CommonJS `require`). result.unresolvedReferences.push(...commonJsRequireRefs(filePath, source, language)); + // Literal seeds are a pass over source text and the node list, never over + // the tree (literal-capture.ts), so they need no Rust mirror — the same + // pass the wasm extractor runs at the end of extract() applies to the + // kernel's nodes here, and the two paths stay node-for-node identical. + // `pre`, not `source`: the wasm extractor captures from its own preParsed + // text, and a preParse blanks bytes a literal could otherwise be read from. + // Without this every routed language loses its seeds while markdown and the + // unrouted ones keep theirs, and a quoted-key explore query silently stops + // finding holders. + captureLiterals(pre, result.nodes); result.durationMs = Date.now() - t0; return result; } catch (err) { diff --git a/src/extraction/literal-capture.ts b/src/extraction/literal-capture.ts new file mode 100644 index 0000000000..3144a1d13b --- /dev/null +++ b/src/extraction/literal-capture.ts @@ -0,0 +1,99 @@ +/** + * Identifier-like string literals — storage keys, CLI flags, event names, + * config paths — attributed to the symbol whose body contains them, so an + * explore query naming the literal seeds on its readers and writers instead + * of degrading to bag-of-words FTS (the literal is never a symbol name). + * + * The capture is a regex over the file's source text, not a tree walk: the + * per-node JS↔WASM crossing is the parse floor (docs/design/native-extraction-kernel.md), + * and a second walk would pay it again for strings alone. Quotes inside a + * comment can produce a spurious literal; the predicate below keeps only + * identifier-shaped values, so prose never qualifies. + */ +import type { Node } from '../types'; + +/** Distinct literals kept per node; a table of keys beyond this is data, not a seam. */ +const MAX_LITERALS_PER_NODE = 32; + +/** + * A value qualifies when it looks like an identifier an agent would quote + * back verbatim: leading letter or underscore (or a `-`/`--` flag prefix), + * only identifier, path, and namespace characters, and at least one + * separator — `bompus_draft_state`, `--start`, `draft.pick`, `api/v1/users`. + * A plain word (`ready`, `Error`) has no separator and stays out: it would + * seed on every function that logs it. + */ +export function isSeedLiteral(value: string): boolean { + return value.length <= 200 && /^-{0,2}[A-Za-z_][A-Za-z0-9_.:/-]{3,}$/.test(value) && /[_.:/-]/.test(value); +} + +/** + * Quoted spans in the query text, plus any bare whitespace-delimited run that + * qualifies. Deduplicated, query order. + */ +export function seedLiteralsInQuery(query: string): string[] { + const out = new Set(); + for (const m of query.matchAll(/["'`]([^"'`\s]+)["'`]/g)) { + const quoted = m[1] ?? ''; + if (isSeedLiteral(quoted)) out.add(quoted); + } + for (const run of query.split(/\s+/)) { + const bare = run.replace(/^[("'`[]+|[)"'`\],.;:?!]+$/g, ''); + if (isSeedLiteral(bare)) out.add(bare); + } + return [...out]; +} + +/** Single-line quoted strings; template literals with `${` interpolation are skipped. */ +const STRING_RE = /(["'`])((?:\\.|(?!\1)[^\\\n])*)\1/g; + +/** + * Attach qualifying literals in `source` to `nodes` (mutated): each literal + * goes to the innermost non-file node whose source range contains it, else + * the file node. Both extraction paths expose UTF-16 columns (the native + * kernel converts its tree-sitter byte columns before emitting nodes). + */ +export function captureLiterals(source: string, nodes: Node[]): void { + if (nodes.length === 0 || source.length === 0) return; + const symbols = nodes.filter((n) => n.kind !== 'file' && n.kind !== 'import'); + const fileNode = nodes.find((n) => n.kind === 'file'); + const lineStarts = [0]; + for (let i = 0; i < source.length; i++) if (source.charCodeAt(i) === 10) lineStarts.push(i + 1); + const lineOf = (offset: number): number => { + let lo = 0; + let hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if ((lineStarts[mid] ?? 0) <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; + }; + const seen = new Map>(); + for (const m of source.matchAll(STRING_RE)) { + const value = m[2] ?? ''; + if (m[1] === '`' && value.includes('${')) continue; + if (!isSeedLiteral(value)) continue; + const line = lineOf(m.index ?? 0); + const startColumn = (m.index ?? 0) - (lineStarts[line - 1] ?? 0); + const endColumn = startColumn + m[0].length; + let owner: Node | undefined; + for (const n of symbols) { + if (n.startLine > line || n.endLine < line) continue; + if (n.startLine === line && n.startColumn > startColumn) continue; + if (n.endLine === line && n.endColumn < endColumn) continue; + // A contained range is deeper; on identical ranges the later walker node wins. + if (!owner || (n.startLine > owner.startLine || + (n.startLine === owner.startLine && n.startColumn >= owner.startColumn)) && + (n.endLine < owner.endLine || + (n.endLine === owner.endLine && n.endColumn <= owner.endColumn))) owner = n; + } + owner ??= fileNode; + if (!owner) continue; + let set = seen.get(owner.id); + if (!set) seen.set(owner.id, (set = new Set())); + if (set.size >= MAX_LITERALS_PER_NODE || set.has(value)) continue; + set.add(value); + (owner.literals ??= []).push(value); + } +} diff --git a/src/extraction/store-worker.ts b/src/extraction/store-worker.ts index a562f557af..fcf2893657 100644 --- a/src/extraction/store-worker.ts +++ b/src/extraction/store-worker.ts @@ -32,6 +32,7 @@ import { QueryBuilder } from '../db/queries'; import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter'; import { finalizeStoreBundle, type KernelStoreBundle, type StoreBundle } from './store-writer'; import { decodeExtractBuffers } from './kernel/decode'; +import { captureLiterals } from './literal-capture'; if (!parentPort) { throw new Error('store-worker must be run as a worker thread'); @@ -71,6 +72,9 @@ function decodeKernelBundle(bundle: KernelStoreBundle): StoreBundle { bundle.language ); if (bundle.extraRefs?.length) decoded.unresolvedReferences.push(...bundle.extraRefs); + if (bundle.buffers.literalSource !== undefined) { + captureLiterals(bundle.buffers.literalSource, decoded.nodes); + } return finalizeStoreBundle(decoded, bundle.filePath, bundle.language, bundle.file); } diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index 5c27523acc..ce0f61065a 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -33,6 +33,7 @@ import { MyBatisExtractor } from './mybatis-extractor'; import { CfmlExtractor } from './cfml-extractor'; import { tryKernelExtract, takeDeferredPreParse } from './kernel'; import { commonJsRequireRefs } from './commonjs-requires'; +import { captureLiterals } from './literal-capture'; import { getAllFrameworkResolvers, getApplicableFrameworks, @@ -654,6 +655,7 @@ export class TreeSitterExtractor { // nodes and import refs are complete and the file node is still pushed. this.flushFnRefCandidates(); this.flushValueRefs(); + captureLiterals(this.source, this.nodes); if (packageNodeId) this.nodeStack.pop(); this.nodeStack.pop(); diff --git a/src/index.ts b/src/index.ts index 47330503aa..2319b8e202 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,6 +60,7 @@ import ignore from 'ignore'; import { loadDeprioritizePatterns } from './project-config'; import { CodeGraphPackageVersion } from './mcp/version'; import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; +import { seedLiteralsInQuery } from './extraction/literal-capture'; import { createYielder } from './resolution/cooperative-yield'; import { minRefsForPool } from './resolution/resolver-pool'; @@ -568,6 +569,7 @@ export class CodeGraph { // path as every file (re-)indexes below — so a full index is also the // orphan-cleanup pass for names deleted since the last one. try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ } + try { this.queries.clearLiterals(); } catch { /* literals are repopulated even for unchanged files */ } // Bulk FTS mode for the mass-insert phase: drop the per-row FTS sync // triggers, rebuild nodes_fts once from the nodes table afterwards. // Crash inside the window is healed on the next DatabaseConnection.open. @@ -2320,7 +2322,24 @@ export class CodeGraph { seedNames = []; } } - return this.contextBuilder.findRelevantContext(query, { ...options, seedNames }); + const seedNodeIds = options?.seedNodeIds ?? this.findLiteralSeedIds(query); + return this.contextBuilder.findRelevantContext(query, { ...options, seedNames, seedNodeIds }); + } + + /** + * Literal seeds: a storage key, flag, or event name quoted in the query is + * never a symbol name, so resolve it through the literals table to the + * symbols whose bodies hold it. `CODEGRAPH_LITERAL_SEEDS=0` is the ablation + * switch; failures (pre-v12 database) degrade to no seeds. Explore's file + * sort calls this too, so a holder file ranks as a named file. + */ + findLiteralSeedIds(query: string): string[] { + if (process.env.CODEGRAPH_LITERAL_SEEDS === '0') return []; + try { + return this.queries.findNodeIdsByLiteral(seedLiteralsInQuery(query)); + } catch { + return []; + } } /** diff --git a/src/mcp/server-instructions.ts b/src/mcp/server-instructions.ts index 80a6247a80..b75ac03501 100644 --- a/src/mcp/server-instructions.ts +++ b/src/mcp/server-instructions.ts @@ -51,6 +51,8 @@ calls; a grep/read exploration is dozens. ## How to query +Storage keys, command-line flags, and event names in a query also match symbols containing those exact string literals. Existing indexes need a full rebuild to capture them. + - **Almost any question — "how does X work", architecture, a bug, "what/where is X", or surveying an area** → \`codegraph_explore\` with a natural-language question or the relevant names. ONE capped call returns the verbatim source grouped by file; most often the ONLY call you need. - **"How does X reach/become Y? / the flow / the path from X to Y"** → \`codegraph_explore\`, naming the symbols that span the flow (e.g. \`mutateElement renderScene\`) — it surfaces the call path among them, riding dynamic-dispatch hops, and returns their source. - **Reading or editing a file/symbol you can name** → put its name or file path in the \`codegraph_explore\` query — it returns that current line-numbered source (safe to \`Edit\` from) with the call path and blast radius attached, so you don't Read it separately. For an overloaded name it returns every matching definition's body in one call. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index c8fec8d3e5..19e3168eab 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -4085,7 +4085,7 @@ export class ToolHandler { } catch { budget = getExploreOutputBudget(Infinity); } - const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + let maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); // File paths named in the query become PINNED files: guaranteed admission, // top of the rank order, funded first — and their span is REMOVED from the @@ -4129,6 +4129,19 @@ export class ToolHandler { return ` Not pinned: ${which}, which define${s.files.length === 1 ? 's' : ''} none of the named symbols.`; }).join(''); const pinnedSet = new Set(pinnedFiles); + // A literal quoted in the query names every file holding it, so the + // default file cap (sized for ranked padding) rises to the holder count; + // the character budget still bounds the answer, and an explicit maxFiles + // stands. + const literalSeedIds = cg.findLiteralSeedIds(matchQuery); + if (!args.maxFiles && literalSeedIds.length > 0) { + const holderFiles = new Set(); + for (const id of literalSeedIds) { + const n = cg.getNode(id); + if (n) holderFiles.add(n.filePath); + } + maxFiles = clamp(Math.max(maxFiles, holderFiles.size), 1, 12); + } const pinnedOrder = new Map(pinnedFiles.map((p, i) => [p, i])); // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG @@ -4556,6 +4569,13 @@ export class ToolHandler { } } } + // Exact literal holders deserve the same source priority as named symbols. + for (const id of literalSeedIds) { + if (subgraph.nodes.has(id)) { + namedSeedIds.add(id); + tierSeedIds.add(id); + } + } // Step 2: Group nodes by file, score by relevance // `peripheral` accumulates separately so it can be capped — see diff --git a/src/types.ts b/src/types.ts index 44ffaf4e43..e9a4127756 100644 --- a/src/types.ts +++ b/src/types.ts @@ -188,6 +188,13 @@ export interface Node { /** Generic type parameters */ typeParameters?: string[]; + /** + * Identifier-like string literals inside this symbol's body (storage keys, + * CLI flags, event names — see extraction/literal-capture.ts). Stored in the + * `literals` side table, not a nodes column; seeds explore on an exact hit. + */ + literals?: string[]; + /** * Normalized return/result type name for a function/method (the bare class * name, smart-pointer pointee unwrapped). Captured for C/C++ so resolution @@ -302,6 +309,8 @@ export interface ExtractionResult { edges: Uint8Array; refs: Uint8Array; arena: Uint8Array; + /** Preparsed source for literal attribution after deferred node decoding. */ + literalSource?: string; }; kernelCounts?: { nodes: number; edges: number; refs: number }; } @@ -698,4 +707,12 @@ export interface FindRelevantContextOptions { * SEGMENTS the query's words name are seeded here instead. */ seedNames?: string[]; + + /** + * Node ids whose body holds a string literal the query quoted verbatim + * (`literals` table, CodeGraph.findRelevantContext). Ranked above every + * name-derived candidate: an exact literal is the strongest evidence a + * query carries, and the symbol is never named after it. + */ + seedNodeIds?: string[]; }