diff --git a/CHANGELOG.md b/CHANGELOG.md index 348a804c87..91e7899f9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -189,6 +189,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- `codegraph_explore` finds code containing storage keys, command-line flags, and event names; rebuild existing indexes to enable these matches. + - **Codex and Astra read project guidance from `AGENTS.md`.** The canonical agent guide now lives in `AGENTS.md` (with a nested `docs/AGENTS.md` for long validation notes); `CLAUDE.md` is a thin `@AGENTS.md` wrapper for Claude Code. Codex/Astra no longer miss the old CLAUDE-only instructions. - **Next.js pages and their navigation are in the graph.** App Router pages (`app/(group)/blog/[slug]/page.tsx` → `/blog/:slug`) and Pages Router pages are routes bound to the component they export, and ``, an internal ``, `router.push` / `router.replace` (`next/navigation` and `next/router`), `redirect()` / `permanentRedirect()` and the middleware's `NextResponse.redirect(…)` are `navigates` edges between them. `app/api/**/route.ts` exports (`GET`, `POST`, …) are endpoints bound to their functions, and `pages/api/*` handlers are `ANY /api/…`. Re-index after upgrading. @@ -221,6 +223,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Dart extension-type getters remain searchable when using the WebAssembly parser. - Calling a built-in method on an awaited value no longer records a call into an unrelated class that happens to declare a method of the same name, and a variable bound to an awaited call now resolves methods on the type that call returns. Thanks @maxmilian. (#1840) +- Questions about selectors and cache invalidation now retain matching declarations alongside named source bodies within the existing response limit. + +- Questions naming source files, types or tests now preserve more of the requested definitions and assertions, and Vue layout questions include matching template and style code. +- Broad code questions now retain matching readers and writers, complete test assertions, and more of the requested Vue column and cell source. - Spring mappings now include every declared path combination and resolve constants declared in the same file, while unresolved paths no longer appear as false root routes. (#1461) - `codegraph node` now accepts a file reference that carries a line number — `src/app.ts:42`, `src/app.ts:42-80`, `src/app.ts#L42`, `src/app.ts#L42-L80` — instead of reporting the file as not indexed; the line range becomes the window that is read, and an `--offset`/`--limit` you pass yourself still wins. A path that really is named that way is still looked up as written. (#1831) - `codegraph callers`, `codegraph callees` and `codegraph impact` now resolve qualified names, group results and JSON edges by definition, and accept `--file` to narrow ambiguous names; thanks @ferrine. (#1512, #1656) diff --git a/__tests__/explore-pinned-allocation.test.ts b/__tests__/explore-pinned-allocation.test.ts index e093c8ad20..b462fbd9a7 100644 --- a/__tests__/explore-pinned-allocation.test.ts +++ b/__tests__/explore-pinned-allocation.test.ts @@ -19,6 +19,29 @@ const cand = ( const budget = getExploreOutputBudget(1000); describe('allocateExploreBudget — pinned files', () => { + it('funds a named body and its supporting source without overspending the pool', () => { + const { allowances, pool } = allocateExploreBudget([ + cand('pipeline.ts', 100, { pinned: true, minChars: 6500 }), + ...Array.from({ length: 5 }, (_, i) => cand(`caller${i}.ts`, 100)), + ], budget, 8); + expect(allowances.get('pipeline.ts')!).toBeGreaterThanOrEqual(6500); + expect([...allowances.values()].reduce((sum, n) => sum + n, 0)).toBeLessThanOrEqual(pool); + for (const [file, bytes] of allowances) { + if (file !== 'pipeline.ts') expect(bytes).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + } + }); + + it('shares unaffordable source minimums and ignores nonfinite demands', () => { + const { allowances, pool } = allocateExploreBudget([ + cand('a.ts', 100, { minChars: 100000 }), + cand('b.ts', 100, { minChars: 100000 }), + cand('c.ts', 100, { minChars: NaN }), + ], budget, 8); + expect(allowances.get('a.ts')).toBe(allowances.get('b.ts')); + expect(allowances.get('c.ts')!).toBeGreaterThanOrEqual(EXPLORE_ALLOCATION.MIN_CHARS); + expect([...allowances.values()].reduce((sum, n) => sum + n, 0)).toBeLessThanOrEqual(pool); + }); + it('never cliffs a pinned file, however low it scores', () => { const { allowances, cliffed } = allocateExploreBudget( [ diff --git a/__tests__/explore-requested-source.test.ts b/__tests__/explore-requested-source.test.ts new file mode 100644 index 0000000000..6afaa63963 --- /dev/null +++ b/__tests__/explore-requested-source.test.ts @@ -0,0 +1,200 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +let dir: string; +let cg: CodeGraph; + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-requested-source-')); + fs.mkdirSync(path.join(dir, 'test')); + fs.writeFileSync(path.join(dir, 'package.json'), '{"name":"requested-source-fixture"}'); + fs.writeFileSync(path.join(dir, 'snapshot.ts'), [ + ...Array.from({ length: 150 }, (_, i) => `export function helper${i}() { return ${i}; }`), + 'export function recommendedPickSnapshot(picks: unknown[]) {', + ' return picks.map(player => ({ player, marker: "snapshot-body-end" }));', + '}', + ].join('\n')); + fs.writeFileSync(path.join(dir, 'test', 'snapshot.test.ts'), [ + 'import { recommendedPickSnapshot } from "../snapshot";', + ...Array.from({ length: 35 }, (_, i) => [ + `it("unrelated sorting ${i}", () => {`, + ' const rows = recommendedPickSnapshot([]);', + ...Array.from({ length: 8 }, () => ' expect(rows).toEqual([]);'), + '});', + ].join('\n')), + 'it("recommendedPickSnapshot preserves recentPicks identity", () => {', + ' const recentPicks = recommendedPickSnapshot([{ id: "4430871" }]);', + ' expect(recentPicks).toMatchObject([', + ' {', + ' player: {', + ' id: "4430871",', + ' },', + ' marker: "snapshot-body-end",', + ' },', + ' ]);', + ' expect(recentPicks[0].player.id).toBe("4430871");', + '});', + ].join('\n')); + fs.writeFileSync(path.join(dir, 'LayoutBoard.vue'), [ + '', + '', + '', + ].join('\n')); + fs.writeFileSync(path.join(dir, 'models.ts'), [ + 'export interface ConsensusPlayer {', + ...Array.from({ length: 120 }, (_, i) => ` providerField${i}?: number | null;`), + ' rawProviderEvidence: number | null;', + '}', + 'export function calculateConsensusPlayer() {', + ...Array.from({ length: 90 }, (_, i) => ` const decoy${i} = "${'incidental calculation '.repeat(8)}";`), + ' return "incidental-name-infix";', + '}', + ...Array.from({ length: 250 }, (_, i) => `export function observe${i}() { return ${i}; }`), + ].join('\n')); + fs.writeFileSync(path.join(dir, 'pipeline.ts'), [ + 'const state = { isMockActive: false, draftedPlayerNames: [] as string[] };', + 'export const pipeline = {', + ...Array.from({ length: 160 }, (_, i) => ` helper${i}() { return "${'padding '.repeat(20)}"; },`), + ' consensusCalcKey() {', + ' return [state.isMockActive, state.draftedPlayerNames.length].join("|");', + ' },', + ' evaluateConsensus() { return this.consensusCalcKey(); },', + '};', + ].join('\n')); + fs.writeFileSync(path.join(dir, 'receiver.ts'), [ + ...Array.from({ length: 100 }, (_, i) => `function filler${i}() { return "${'unrelated padding '.repeat(10)}"; }`), + 'function persistQueuePriority(value: number) {', + ...Array.from({ length: 12 }, (_, i) => ` const check${i} = "${'validate the input '.repeat(4)}";`), + ' return value;', + '}', + 'function receiveQueueSnapshot(value: number) {', + ...Array.from({ length: 12 }, (_, i) => ` const context${i} = "${'validate the context '.repeat(4)}";`), + ' return persistQueuePriority(value);', + '}', + 'export function listenForQueueMessage(value: number) {', + ...Array.from({ length: 12 }, (_, i) => ` const channel${i} = "${'validate the sender '.repeat(4)}";`), + ' return receiveQueueSnapshot(value);', + '}', + ].join('\n')); + fs.writeFileSync(path.join(dir, 'identity.ts'), [ + 'import { unrelatedHub } from "./hub";', + 'export function playerIdentity(id: number) { unrelatedHub(); return id; }', + ].join('\n')); + fs.writeFileSync(path.join(dir, 'hub.ts'), [ + ...Array.from({ length: 30 }, (_, i) => `function hubStep${i}() { return ${i}; }`), + `export function unrelatedHub() { ${Array.from({ length: 30 }, (_, i) => `hubStep${i}();`).join(' ')} }`, + ].join('\n')); + fs.writeFileSync(path.join(dir, 'receipt.ts'), [ + 'import { playerIdentity } from "./identity";', + 'export function receipt0() { return playerIdentity(0); }', + 'export function receipt1() { return playerIdentity(1); }', + 'export function receipt2() { return playerIdentity(2); }', + ].join('\n')); + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 60_000); + +afterAll(() => { + cg?.destroy(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); +}); + +async function explore(query: string): Promise { + const result = await new ToolHandler(cg).execute('codegraph_explore', { query }); + expect(result.isError).not.toBe(true); + return result.content?.[0]?.text ?? ''; +} + +function sourceIn(text: string, file: string): string { + const start = text.indexOf('**`' + file + '`**'); + expect(start, file).toBeGreaterThan(-1); + const block = text.slice(start).match(/```[^\n]*\n([\s\S]*?)```/); + expect(block, file).not.toBeNull(); + return block![1]!; +} + +describe('requested evidence in large selected files', () => { + it('prioritizes direct callers over a dense callee for a single named function', async () => { + const result = await new ToolHandler(cg).execute('codegraph_explore', { query: 'playerIdentity', maxFiles: 2 }); + const out = result.content?.[0]?.text ?? ''; + expect(sourceIn(out, 'identity.ts')).toContain('function playerIdentity'); + expect(sourceIn(out, 'receipt.ts')).toContain('return playerIdentity(0)'); + }); + it('keeps a compound-concept writer and its local receiver chain complete', async () => { + const out = await explore('How does queue_priority reach storage?'); + const source = sourceIn(out, 'receiver.ts'); + expect(source).toContain('return value;'); + expect(source).toContain('return persistQueuePriority(value);'); + expect(source).toContain('return receiveQueueSnapshot(value);'); + expect(source).toContain('const context11 ='); + expect(source).toContain('const channel11 ='); + expect(out.length).toBeLessThanOrEqual(25_000); + }); + it('returns the full named test when the query does not name its assertion', async () => { + const out = await explore('test/snapshot.test.ts recommendedPickSnapshot'); + expect(sourceIn(out, 'test/snapshot.test.ts')).toContain('expect(recentPicks[0].player.id).toBe("4430871")'); + }); + + it('retains bounded vocabulary seeds when other names rank above them', async () => { + const graph = await cg.findRelevantContext('snapshot_helper storage', { + searchLimit: 1, traversalDepth: 0, seedNames: ['helper149', 'recommendedPickSnapshot'], + }); + expect(graph.roots.map(id => graph.nodes.get(id)?.name)).toEqual( + expect.arrayContaining(['helper149', 'recommendedPickSnapshot']), + ); + }); + it('keeps the declaration using the requested invalidation inputs', async () => { + const out = await explore('evaluateConsensus in pipeline.ts: how do isMockActive and draftedPlayerNames invalidate cached players?'); + expect(sourceIn(out, 'pipeline.ts')).toContain('return [state.isMockActive, state.draftedPlayerNames.length].join("|")'); + }); + it('returns the complete identity assertion alongside the named implementation', async () => { + const out = await explore('recommendedPickSnapshot in snapshot.ts: locate recentPicks identity assertions in test/snapshot.test.ts:'); + const test = sourceIn(out, 'test/snapshot.test.ts'); + expect(test).toContain('expect(recentPicks[0].player.id).toBe("4430871")'); + expect(test).toContain('marker: "snapshot-body-end"'); + expect(sourceIn(out, 'snapshot.ts')).toContain('return picks.map(player => ({ player, marker: "snapshot-body-end" }))'); + }); + + it.each([ + 'LayoutBoard.vue "col-w-survival"', + 'How does LayoutBoard format survival and set column widths?', + ])('returns template and CSS evidence for %s', async query => { + const out = await explore(query); + const source = sourceIn(out, 'LayoutBoard.vue'); + expect(source).toContain(''); + expect(source).toContain('.col-w-survival {'); + expect(source).toContain('min-width: 176px;'); + expect(out.length).toBeLessThanOrEqual(25_000); + }); + + it('returns a named interface body instead of unrelated declarations', async () => { + const out = await explore('ConsensusPlayer fields in models.ts: locate raw provider evidence'); + const source = sourceIn(out, 'models.ts'); + expect(source).toContain('providerField0?: number | null;'); + expect(source).toContain('providerField60?: number | null;'); + expect(source).toContain('rawProviderEvidence: number | null;'); + }); + + it('recognizes punctuation after a requested symbol', async () => { + const out = await explore('recommendedPickSnapshot: preserve the returned snapshot'); + expect(sourceIn(out, 'snapshot.ts')).toContain('return picks.map(player => ({ player, marker: "snapshot-body-end" }))'); + }); +}); diff --git a/__tests__/kernel-retry-materialize.test.ts b/__tests__/kernel-retry-materialize.test.ts index 23965bd2dd..83e4db4239 100644 --- a/__tests__/kernel-retry-materialize.test.ts +++ b/__tests__/kernel-retry-materialize.test.ts @@ -56,7 +56,7 @@ describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { it('storeExtractionResult persists the decoded nodes of a raw kernel result', async () => { const source = 'def target_fn(root, mission_path):\n' + - ' return (root, mission_path)\n' + + ' return (root, mission_path, "adapter.mission")\n' + '\n' + 'class Adapter:\n' + ' def adapt(self):\n' + @@ -93,6 +93,7 @@ describe.skipIf(!kernelBuilt)('kernel buffer-transport storage (#1541)', () => { expect(nodes.length).toBe(raw!.counts.nodes); expect(nodes.map((n) => n.name)).toContain('target_fn'); expect(nodes.map((n) => n.name)).toContain('Adapter'); + expect(cg.findLiteralSeedIds('"adapter.mission"').map(id => cg.getNode(id)?.name)).toEqual(['target_fn']); }); }); diff --git a/__tests__/literal-capture.test.ts b/__tests__/literal-capture.test.ts new file mode 100644 index 0000000000..a27406a51c --- /dev/null +++ b/__tests__/literal-capture.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest'; +import type { Node } from '../src/types'; +import { captureLiterals, isSeedLiteral, seedLiteralsInQuery } from '../src/extraction/literal-capture'; + +function node(id: string, kind: Node['kind'], startLine: number, endLine: number): Node { + return { + id, kind, name: id, qualifiedName: id, filePath: 'src/a.ts', language: 'typescript', + startLine, endLine, startColumn: 0, endColumn: Number.MAX_SAFE_INTEGER, updatedAt: 0, + }; +} + +describe('isSeedLiteral', () => { + it('keeps storage keys, flags, dotted names and paths', () => { + for (const v of ['bompus_custom_ds_players', '--start', '-v', 'draft.pick', 'api/v1/users', 'ns:event']) + expect(isSeedLiteral(v), v).toBe(v !== '-v'); + }); + it('drops plain words, prose, and values that start with a separator', () => { + for (const v of ['ready', 'Error', 'not found', './utils', '../x', '', 'a_b']) + expect(isSeedLiteral(v), v).toBe(false); + }); +}); + +describe('seedLiteralsInQuery', () => { + it('finds quoted spans and bare runs, stripping surrounding punctuation', () => { + expect(seedLiteralsInQuery('who writes "bompus_custom_ds_players" (via --start)?')) + .toEqual(['bompus_custom_ds_players', '--start']); + }); + it('returns nothing for a symbol-anchored question', () => { + expect(seedLiteralsInQuery('callers of espnPlayerKey in shared')).toEqual([]); + }); +}); + +describe('captureLiterals', () => { + const source = [ + `import { x } from './utils/helpers';`, // 1: path starts with '.', never qualifies + `const KEY = 'bompus_custom_ds_players';`, // 2: top level → file node + `export function save() {`, // 3 + ` storage.set('bompus_custom_ds_players', 1);`, // 4 + ` log('ready');`, // 5: plain word + ` emit(\`draft.pick\`); emit(\`draft.\${n}\`);`, // 6: second one interpolates + `}`, // 7 + `export class Boot { start() { run('--start'); } }`,// 8 + ].join('\n'); + + it('attributes each literal to the innermost enclosing symbol, else the file', () => { + const file = node('file:src/a.ts', 'file', 1, 8); + const save = node('save', 'function', 3, 7); + const boot = node('Boot', 'class', 8, 8); + const start = node('start', 'method', 8, 8); + const imp = node('./utils/helpers', 'import', 1, 1); + const nodes = [file, imp, save, boot, start]; + captureLiterals(source, nodes); + expect(file.literals).toEqual(['bompus_custom_ds_players']); + expect(save.literals).toEqual(['bompus_custom_ds_players', 'draft.pick']); + expect(start.literals).toEqual(['--start']); + expect(boot.literals).toBeUndefined(); + expect(imp.literals).toBeUndefined(); + }); + + it('dedupes per node and caps at 32', () => { + const many = Array.from({ length: 40 }, (_, i) => `k('key_${i}'); k('key_${i}');`).join('\n'); + const fn = node('f', 'function', 1, 40); + captureLiterals(many, [node('file:src/a.ts', 'file', 1, 40), fn]); + expect(fn.literals).toHaveLength(32); + expect(new Set(fn.literals).size).toBe(32); + }); + + it('uses UTF-16 columns to distinguish same-line siblings after non-ASCII source', () => { + const prefix = '/* café 😀 */ '; + const first = "function writer(){return 'cache.write';}"; + const second = "function reader(){return 'cache.read';}"; + const writer = { ...node('writer', 'function', 1, 1), startColumn: prefix.length, endColumn: (prefix + first).length }; + const reader = { ...node('reader', 'function', 1, 1), startColumn: (prefix + first).length, endColumn: (prefix + first + second).length }; + captureLiterals(prefix + first + second, [writer, reader]); + expect(writer.literals).toEqual(['cache.write']); + expect(reader.literals).toEqual(['cache.read']); + }); +}); diff --git a/__tests__/literal-compiled-index.test.ts b/__tests__/literal-compiled-index.test.ts new file mode 100644 index 0000000000..ceb39d9588 --- /dev/null +++ b/__tests__/literal-compiled-index.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import type CodeGraph from '../src'; +import type { QueryBuilder } from '../src/db/queries'; + +const built = path.join(__dirname, '..', 'dist', 'index.js'); +const kernel = path.join(__dirname, '..', 'codegraph-kernel', 'prebuilds', `${process.platform}-${process.arch}`, 'codegraph-kernel.node'); + +// Source-mode suites cannot exercise the compiled parse/store worker boundary. +describe.runIf(fs.existsSync(built))('literal persistence through compiled indexing', () => { + let dir: string | undefined; + let cg: CodeGraph | undefined; + afterEach(() => { + cg?.destroy(); + cg = undefined; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + dir = undefined; + vi.unstubAllEnvs(); + }); + + for (const mode of ['native-worker', 'native-main', 'wasm-worker']) { + it.runIf(mode === 'wasm-worker' || fs.existsSync(kernel))(`${mode}: fresh, unchanged, and migrated indexes retain exact owners`, async () => { + vi.stubEnv('CODEGRAPH_PARSE_WORKERS', '1'); + vi.stubEnv('CODEGRAPH_KERNEL', mode === 'wasm-worker' ? '0' : '1'); + vi.stubEnv('CODEGRAPH_NO_STORE_WORKER', mode === 'native-main' ? '1' : '0'); + const BuiltCodeGraph: typeof CodeGraph = require(built).default; + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-literal-compiled-')); + fs.writeFileSync(path.join(dir, 'cache.py'), "def persist():\n return 'cache.write'\n"); + fs.writeFileSync(path.join(dir, 'siblings.ts'), "/* café 😀 */ export function writer(){return 'sibling.write';} export function reader(){return 'sibling.read';}\n"); + cg = await BuiltCodeGraph.init(dir, { silent: true }); + const names = (key: string) => cg!.findLiteralSeedIds(key).map(id => cg!.getNode(id)?.name); + const assertOwners = () => { + expect(names('cache.write')).toEqual(['persist']); + expect(names('sibling.write')).toEqual(['writer']); + expect(names('sibling.read')).toEqual(['reader']); + }; + await cg.indexAll(); + assertOwners(); + await cg.indexAll(); + assertOwners(); + + // A schema-only migration has no literal data; unchanged source still needs backfill. + const queries = (cg as unknown as { queries: QueryBuilder }).queries; + queries.replaceLiteralsForFile('cache.py', []); + queries.replaceLiteralsForFile('siblings.ts', []); + queries.setMetadata('indexed_with_extraction_version', '26'); + expect(cg.isIndexStale()).toBe(true); + await cg.indexAll(); + assertOwners(); + expect(cg.isIndexStale()).toBe(false); + fs.rmSync(path.join(dir, 'cache.py')); + await cg.indexAll(); + expect(names('cache.write')).toEqual([]); + expect(names('sibling.write')).toEqual(['writer']); + }); + } +}); diff --git a/__tests__/literal-seeds.test.ts b/__tests__/literal-seeds.test.ts new file mode 100644 index 0000000000..35c871a5db --- /dev/null +++ b/__tests__/literal-seeds.test.ts @@ -0,0 +1,253 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import CodeGraph from '../src'; +import { QueryBuilder } from '../src/db/queries'; +import { createDatabase, type SqliteDatabase } from '../src/db/sqlite-adapter'; +import { runMigrations, getCurrentVersion, CURRENT_SCHEMA_VERSION } from '../src/db/migrations'; +import { ToolHandler } from '../src/mcp/tools'; + +/** + * The string-anchored explore path: a query that names a storage key or a CLI + * flag reaches the symbols whose bodies hold it, through the `literals` side + * table (written on the node write path, deleted with the file, cleared by a + * full index), and those symbols lead the ranking. + */ +describe('literal seeds — index, query, and lifecycle', () => { + let dir: string; + let cg: CodeGraph; + const queries = () => (cg as unknown as { queries: QueryBuilder }).queries; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-literals-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'writer.ts'), + `export function persistRows(rows: unknown[]) { + chrome.storage.local.set({ 'bompus_custom_ds_players': rows }); +} +export function unrelatedHelper() { return 'ready'; } +`, + ); + fs.writeFileSync( + path.join(dir, 'src', 'reader.ts'), + `export async function readRows() { + const got = await chrome.storage.local.get('bompus_custom_ds_players'); + return got; +} +`, + ); + fs.writeFileSync( + path.join(dir, 'src', 'cli.ts'), + `export function main(argv: string[]) { + if (argv.includes('--start')) return start(); + return 0; +} +function start() { return 1; } +`, + ); + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('resolves a storage key to every symbol whose body holds it', () => { + const ids = queries().findNodeIdsByLiteral(['bompus_custom_ds_players']); + const names = ids.map((id) => cg.getNode(id)?.name).sort(); + expect(names).toEqual(['persistRows', 'readRows']); + }); + + it('recommends rebuilding an index from before literal capture', () => { + const current = queries().getMetadata('indexed_with_extraction_version'); + try { + queries().setMetadata('indexed_with_extraction_version', '26'); + expect(cg.isIndexStale()).toBe(true); + } finally { + queries().setMetadata('indexed_with_extraction_version', current!); + } + expect(cg.isIndexStale()).toBe(false); + }); + + it('a quoted key in a prose query puts the writers at the top of the subgraph', async () => { + const sub = await cg.findRelevantContext('which modules write "bompus_custom_ds_players" to storage'); + const rootNames = sub.roots.map((id) => sub.nodes.get(id)?.name); + expect(rootNames.slice(0, 2).sort()).toEqual(['persistRows', 'readRows']); + expect(rootNames).not.toContain('unrelatedHelper'); + }); + + it('a bare CLI flag seeds too', () => { + const ids = queries().findNodeIdsByLiteral(['--start']); + expect(ids.map((id) => cg.getNode(id)?.name)).toEqual(['main']); + }); + + it('CODEGRAPH_LITERAL_SEEDS=0 turns the seed off (the ablation switch)', async () => { + process.env.CODEGRAPH_LITERAL_SEEDS = '0'; + try { + const sub = await cg.findRelevantContext('bompus_custom_ds_players'); + const names = [...sub.nodes.values()].map((n) => n.name); + expect(names).not.toContain('persistRows'); + } finally { + delete process.env.CODEGRAPH_LITERAL_SEEDS; + } + }); + + it("a file's rows leave with its nodes on re-index, and a full index clears the table", async () => { + fs.writeFileSync(path.join(dir, 'src', 'reader.ts'), `export function readRows() { return null; }\n`); + await cg.indexFiles(['src/reader.ts']); + let names = queries().findNodeIdsByLiteral(['bompus_custom_ds_players']).map((id) => cg.getNode(id)?.name); + expect(names).toEqual(['persistRows']); + + fs.rmSync(path.join(dir, 'src', 'writer.ts')); + await cg.indexAll(); + expect(queries().findNodeIdsByLiteral(['bompus_custom_ds_players'])).toEqual([]); + }); +}); + +/** + * A literal held through a constant lives in a small file with no callers. In explore's file + * sort that file must take a source slot the way a file defining a named symbol does; on graph + * centrality alone it loses the slot to a hub file that never mentions the literal. + */ +describe('literal seeds — explore renders the holder file', () => { + let dir: string; + let cg: CodeGraph; + let handler: ToolHandler; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litrender-')); + fs.mkdirSync(path.join(dir, 'shared'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'callers'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'shared', 'bag.ts'), + `const BAG_KEY = 'bompus_custom_ds_players'; +export function persistBag(rows: unknown[]) { + chrome.storage.local.set({ [BAG_KEY]: rows }); +} +`, + ); + // The hub: matches the query word "storage" by name and is called from many files. + fs.writeFileSync( + path.join(dir, 'shared', 'storage.ts'), + `export function storageSet(key: string, value: unknown) { + return chrome.storage.local.set({ [key]: value }); +} +export function storageGet(key: string) { + return chrome.storage.local.get(key); +} +`, + ); + for (let i = 0; i < 8; i++) { + fs.writeFileSync( + path.join(dir, 'callers', `writer${i}.ts`), + `import { storageSet, storageGet } from '../shared/storage'; +export function storageWriter${i}() { + storageGet('k${i}'); + return storageSet('k${i}', ${i}); +} +`, + ); + } + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + handler = new ToolHandler(cg); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('the file holding the quoted key takes the first source slot', async () => { + const res = await handler.execute('codegraph_explore', { + query: 'which modules write "bompus_custom_ds_players" to storage', + maxFiles: 1, + }); + const text = res.content[0].text as string; + const sourced = [...text.matchAll(/^\*\*`(.+?)`\*\* —/gm)].map((m) => m[1]); + expect(sourced).toEqual(['shared/bag.ts']); + }); +}); + +/** + * A key held in more files than the entry-point cap (`searchLimit`, 3 by default and 8 in + * explore) must reach every holder: the cap is for text matches, and the literal lookup is + * already bounded. Before this, holders competed for the cap by path order and `shared/` + * lost to `scripts/`. + */ +describe('literal seeds — every holder reaches the subgraph past the entry cap', () => { + let dir: string; + let cg: CodeGraph; + const N = 10; + + beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litcap-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + for (let i = 0; i < N; i++) { + fs.writeFileSync( + path.join(dir, 'src', `holder${i}.ts`), + `export function write${i}(rows: unknown[]) { + chrome.storage.local.set({ 'bompus_custom_ds_players': rows }); +} +`, + ); + } + cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + }); + + afterAll(() => { + cg.destroy(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('all holders are roots with the default searchLimit of 3', async () => { + const sub = await cg.findRelevantContext('which modules write "bompus_custom_ds_players" to storage'); + const rootFiles = new Set(sub.roots.map((id) => sub.nodes.get(id)?.filePath)); + expect(rootFiles.size).toBe(N); + }); + + it('explore renders every holder with the default file cap', async () => { + const res = await new ToolHandler(cg).execute('codegraph_explore', { + query: 'which modules write "bompus_custom_ds_players" to storage', + }); + const text = res.content[0].text as string; + const sourced = [...text.matchAll(/^\*\*`(.+?)`\*\* —/gm)].map((m) => m[1]); + expect(sourced.length).toBe(N); + }); +}); + +describe('literals — v12 migration', () => { + let dir: string; + let db: SqliteDatabase | null = null; + + afterEach(() => { + db?.close(); + db = null; + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function makeV11Db(): SqliteDatabase { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-litmigrate-')); + const conn = createDatabase(path.join(dir, 'legacy.db')).db; + conn.exec(` + CREATE TABLE schema_versions (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL, description TEXT); + INSERT INTO schema_versions VALUES (11, 0, 'legacy'); + `); + db = conn; + return conn; + } + + it('creates the empty table and is idempotent on replay', () => { + const conn = makeV11Db(); + runMigrations(conn, 11); + expect(getCurrentVersion(conn)).toBe(CURRENT_SCHEMA_VERSION); + expect((conn.prepare('SELECT COUNT(*) AS n FROM literals').get() as { n: number }).n).toBe(0); + conn.prepare('DELETE FROM schema_versions WHERE version >= 12').run(); + expect(() => runMigrations(conn, 11)).not.toThrow(); + }); +}); diff --git a/__tests__/query-paths.test.ts b/__tests__/query-paths.test.ts index 7c23a5dcce..ed3640e7bc 100644 --- a/__tests__/query-paths.test.ts +++ b/__tests__/query-paths.test.ts @@ -65,6 +65,15 @@ describe('queryMightContainPaths — the cheap pre-gate', () => { }); describe('extractQueryPaths — resolution and stripping', () => { + it('pins prose paths ending in a colon while preserving line references', () => { + for (const suffix of [':', ':123', ':12-40', '#L88']) { + const out = extractQueryPaths(`src/lib/chat-manager.ts${suffix} locate the handler`, INDEX); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + expect(out.strippedQuery).toBe('locate the handler'); + expect(out.unresolvedPathSpans).toEqual([]); + } + }); + it('resolves a bracketed SvelteKit path and strips it from the query', () => { const q = 'auto-scroll logic in src/routes/m/projects/[id]/runs/[runId]/+page.svelte — atBottom tracking'; const out = extractQueryPaths(q, INDEX); diff --git a/docs/benchmarks/upstream-requested-source-evidence.md b/docs/benchmarks/upstream-requested-source-evidence.md new file mode 100644 index 0000000000..7a26b436f8 --- /dev/null +++ b/docs/benchmarks/upstream-requested-source-evidence.md @@ -0,0 +1,48 @@ +# Requested-source contribution validation + +Measured 2026-09-11 UTC. These measurements validate the upstream-based contribution, not the consolidated fork. + +The literal-seeding foundation `248493bf` is based on upstream `3ed73bc1`. The requested-source implementation `0bf5b98d` adds bounded source selection and allocation on top of that foundation. A later follow-up on this branch ports the consolidated broad-receiver/source reservations (`245d00d4`) without fork-only Markdown/session or deployment configuration. + +## Validation + +- Foundation: application and native-kernel builds passed; 100 tests across six focused native suites passed, including deferred-buffer literal persistence and the rebuild recommendation for pre-literal indexes. The four relevant WASM suites passed all 42 tests. +- Requested source: application build passed; 345 tests across 28 retrieval suites passed with the native kernel, and the same 345 passed with `CODEGRAPH_KERNEL=0`. The unchanged upstream Rust sources use the kernel built for the foundation. +- Builds and tests ran on WSL with Node 24 at low priority, at most three test/compiler workers, sequentially. This is not a full-suite or cross-platform validation claim. + +## Fixed-index comparison + +### Review follow-up + +Foundation revision `3b95088b` fixes three subsequently reproduced gaps: compiled native store-worker decoding omitted literal capture, unchanged-file re-indexing erased literal entries, and same-line functions received incorrect ownership. Its TypeScript compilation, 101 focused native tests, 43 WASM tests, and three compiled-index regressions passed. The compiled regressions cover fresh/repeated indexing, migration backfill, deleted files, and UTF-16 ownership after non-ASCII text across native-worker, native-main-thread, and WASM-worker modes. + +The dependent branch incorporates that foundation update. All 346 retrieval tests passed on each backend, and the three compiled-index regressions passed. The fixed-index measurements below remain measurements of the original revisions; they were not repeated after the extraction/lifecycle fixes and do not establish fresh-index behavior of the updated branch. + +The [16-query fixture](https://github.com/bompus/chrome-ext-bompus-espn-draft/blob/74fcd4e147020d834a3110ac161cb1f6d10cf4b2/docs/agents/codegraph-retrieval-repro.json) pins source from ESPN commit `c570e32b`. Each referenced source file was verified against its recorded SHA-256 before its query. Both engines used the same existing checkout-local index, a fresh process and ToolHandler per query, and an explicit projectPath, with no maxFiles override. This isolates retrieval against fixed extracted data; it does not compare separately rebuilt real-project indexes. Foundation extraction and persistence were tested independently above. + +Cells count returned / expected nonblank source lines in the fixture's expected ranges. A plus separates ranges, not calls. Each row required one call per engine. + +| Case | Foundation | Requested source | +| --- | --- | --- | +| 1 | 29/29 + 4/23 | 29/29 + 23/23 | +| 2 | 9/9 | 9/9 | +| 3 | 0/2 + 0/22 | 2/2 + 22/22 | +| 4 | 2/50 + 50/73 | 50/50 + 73/73 | +| 5 | 0/31 | 30/31 | +| 6 | 0/132 | 132/132 | +| 7 | 0/33 | 33/33 | +| 8 | 36/36 + 61/61 + 0/20 | 36/36 + 61/61 + 0/20 | +| 9 | 0/78 | 0/78 | +| 10 | 0/78 | 0/78 | +| 11 | 0/50 | 50/50 | +| 12 | 0/1 + 0/6 + 0/2 + 0/1 + 0/12 | 0/1 + 6/6 + 0/2 + 0/1 + 12/12 | +| 13 | 0/31 | 23/31 | +| 14 | 33/33 | 33/33 | +| 15 | 0/1 + 0/3 | 1/1 + 3/3 | +| 16 | 35/50 + 78/78 | 50/50 + 78/78 | + +The combined contribution restores the full snapshot assertion, Vue selectors, cache-key body, named interface, and specific shared-key sender/receiver evidence. No measured range loses coverage relative to the foundation. Maximum response length was 24,989 JavaScript characters. + +A later follow-up on this branch retains compound-concept readers/writers, whole matching tests, and additional Vue regions. Synthetic regressions cover those paths. The optional mapper test (8) remains a documented non-goal. Returning sender and receiver source in case 16 does not create a publisher/listener graph edge. + +These are deterministic source-completeness checks, not agent A/B runs, productivity measurements, or evidence of a latency improvement. diff --git a/src/context/index.ts b/src/context/index.ts index 6cea28c2d1..fb4732e02d 100644 --- a/src/context/index.ts +++ b/src/context/index.ts @@ -172,8 +172,16 @@ const DEFAULT_FIND_OPTIONS: Required = { edgeKinds: [], nodeKinds: HIGH_VALUE_NODE_KINDS, // Filter out imports/exports by default seedNames: [], // Segment-vocab supplement — filled by the facade + seedNodeIds: [], // Literal-table seeds — filled by the facade }; +/** + * Score for a literal seed. Above the BM25 magnitudes the FTS channel returns + * (tens to hundreds) and every additive boost below, so a symbol holding the + * quoted literal leads the ranking whatever else the query matched. + */ +const LITERAL_SEED_SCORE = 1000; + // Re-export the low-confidence sentinel (defined in a dependency-free leaf so // the MCP layer can import it without pulling this module's deps onto the // cold-start path). Builder code below uses the imported binding directly. @@ -483,7 +491,16 @@ export class ContextBuilder { // Step 2: Look up exact matches for extracted symbols let exactMatches: SearchResult[] = []; - if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0) { + // Explicit seed lookups already bound their results. Retain those results + // through ranking: a later reader/writer must not disappear at a size trim. + const retainedSeedIds = new Set(opts.seedNodeIds); + const keepSeedsThenTop = (results: SearchResult[], n: number): SearchResult[] => { + if (retainedSeedIds.size === 0) return results.slice(0, n); + const seeds = results.filter((r) => retainedSeedIds.has(r.node.id)); + const rest = results.filter((r) => !retainedSeedIds.has(r.node.id)).slice(0, n); + return [...seeds, ...rest]; + }; + if (symbolsFromQuery.length > 0 || opts.seedNames.length > 0 || opts.seedNodeIds.length > 0) { try { if (symbolsFromQuery.length > 0) { // Get more results so we can apply co-location boosting before trimming @@ -493,6 +510,20 @@ export class ContextBuilder { }); } + // Literal seeds: the query quoted a string that lives in these + // symbols' bodies. No name matches it, so they enter here, first and + // at a fixed top score; the co-location pass below still compounds + // several hits in one file. + if (opts.seedNodeIds.length > 0) { + const byId = this.queries.getNodesByIds(opts.seedNodeIds); + for (const id of opts.seedNodeIds) { + const node = byId.get(id); + if (!node || exactMatches.some((r) => r.node.id === id)) continue; + exactMatches.push({ node, score: LITERAL_SEED_SCORE }); + } + logDebug('Literal seed matches', { seedNodeIds: opts.seedNodeIds, added: byId.size }); + } + // Step 2a: segment-vocabulary seeds. Word-level query terms cannot // reach camelCase names through FTS (one token per name), so the // caller resolves query words → names via the segment vocab and hands @@ -502,12 +533,18 @@ export class ContextBuilder { // in one file (pinFeedIfNearBottom + feedAtBottom + handleFeedScroll) // is exactly the evidence that file is the answer. if (opts.seedNames.length > 0) { + const compoundQuery = symbolsFromQuery.some(s => /[a-z][A-Z]|_/.test(s)); + const namedCallable = exactMatches.some(r => ['function', 'method', 'component', 'class'].includes(r.node.kind) + && symbolsFromQuery.some(s => isDistinctiveIdentifier(s) && s.toLowerCase() === r.node.name.toLowerCase())); const seedResults = this.queries.findNodesByExactName(opts.seedNames, { limit: Math.ceil(opts.searchLimit * 3), kinds: opts.nodeKinds && opts.nodeKinds.length > 0 ? opts.nodeKinds : undefined, }); const known = new Set(exactMatches.map((r) => r.node.id)); for (const r of seedResults) { + // A concept-only query needs these bounded matches to survive; + // explicit callable queries retain their existing focused ranking. + if (compoundQuery && !namedCallable) retainedSeedIds.add(r.node.id); if (known.has(r.node.id)) continue; known.add(r.node.id); exactMatches.push({ ...r, score: r.score * 0.6 }); @@ -538,7 +575,7 @@ export class ContextBuilder { } // Trim back to reasonable size - exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 2)); + exactMatches = keepSeedsThenTop(exactMatches, Math.ceil(opts.searchLimit * 2)); logDebug('Exact symbol matches', { count: exactMatches.length }); } catch (error) { logDebug('Exact symbol lookup failed', { error: String(error) }); @@ -587,7 +624,7 @@ export class ContextBuilder { } } exactMatches.sort((a, b) => b.score - a.score); - exactMatches = exactMatches.slice(0, Math.ceil(opts.searchLimit * 3)); + exactMatches = keepSeedsThenTop(exactMatches, Math.ceil(opts.searchLimit * 3)); } // Step 2c: Match an exact Han filename (with or without its extension). @@ -1006,7 +1043,7 @@ export class ContextBuilder { // compound) have now contributed. Sort by score so multi-term matches from // later steps can outrank dampened single-term matches from earlier steps. searchResults.sort((a, b) => b.score - a.score); - searchResults = searchResults.slice(0, opts.searchLimit * 3); + searchResults = keepSeedsThenTop(searchResults, opts.searchLimit * 3); // Filter by minimum score let filteredResults = searchResults.filter((r) => r.score >= opts.minScore); @@ -1030,7 +1067,7 @@ export class ContextBuilder { // With 36 entry points and maxNodes=120, each gets only 3 nodes — useless. // Cap to searchLimit so each entry point gets a meaningful traversal budget. if (filteredResults.length > opts.searchLimit) { - filteredResults = filteredResults.slice(0, opts.searchLimit); + filteredResults = keepSeedsThenTop(filteredResults, opts.searchLimit); } // Confidence signal for the honest-handoff footer (consumed in buildContext). diff --git a/src/db/migrations.ts b/src/db/migrations.ts index bb7b84bf53..9d7c9d5940 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 11; +export const CURRENT_SCHEMA_VERSION = 12; /** * Migration definition @@ -216,6 +216,26 @@ const migrations: Migration[] = [ `); }, }, + { + version: 12, + description: + 'Add literals — identifier-like string literal → enclosing symbol, so explore seeds on storage keys and flags', + up: (db) => { + // DDL only. No backfill: the values come from file CONTENT the migration + // cannot see, so the table stays empty until the next full index and + // explore behaves exactly as before for those queries. Keep in lockstep + // with schema.sql. + db.exec(` + CREATE TABLE IF NOT EXISTS literals ( + value TEXT NOT NULL, + node_id TEXT NOT NULL, + file_path TEXT NOT NULL, + PRIMARY KEY (value, node_id) + ) WITHOUT ROWID; + CREATE INDEX IF NOT EXISTS idx_literals_file ON literals(file_path); + `); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index a12e7c2643..b1112d8ddb 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -273,6 +273,7 @@ export class QueryBuilder { updateNode?: SqliteStatement; deleteNode?: SqliteStatement; deleteNodesByFile?: SqliteStatement; + deleteLiteralsByFile?: SqliteStatement; getNodeById?: SqliteStatement; getNodesByFile?: SqliteStatement; getNodesByKind?: SqliteStatement; @@ -520,6 +521,43 @@ export class QueryBuilder { // import-only name can never be surfaced (getSegmentMatches requires a // real definition), so its rows would only inflate the rarity statistics. if (this.isSegmentableKind(node.kind)) this.insertNameSegments(node.name); + const literalRows: unknown[][] = []; + this.collectLiteralRows(node, literalRows); + this.insertLiteralRows(literalRows); + } + + /** Rows for the `literals` side table (see schema.sql) — one per captured literal. */ + private collectLiteralRows(node: Node, rows: unknown[][]): void { + for (const value of node.literals ?? []) rows.push([value, node.id, node.filePath]); + } + + private insertLiteralRows(rows: unknown[][]): void { + this.runBatched( + 'insertLiterals', + 'INSERT OR IGNORE INTO literals (value, node_id, file_path) VALUES ', + '(?,?,?)', + rows + ); + } + + /** + * Node ids whose body holds one of `values` verbatim, joined to nodes so a + * row left behind by a stale index is never surfaced. Explore seeds on these + * ahead of every name-derived candidate. + */ + findNodeIdsByLiteral(values: string[], limit = 24): string[] { + if (values.length === 0) return []; + const placeholders = values.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT DISTINCT l.node_id AS id FROM literals l + JOIN nodes n ON n.id = l.node_id + WHERE l.value IN (${placeholders}) + ORDER BY n.kind = 'file', n.file_path, n.start_line + LIMIT ?` + ) + .all(...values, limit) as Array<{ id: string }>; + return rows.map((r) => r.id); } /** Which node kinds contribute their name to the segment vocabulary — the @@ -551,6 +589,7 @@ export class QueryBuilder { // per-.run() call overhead dominates the store phase on full indexes. const rows: unknown[][] = []; const segmentRows: unknown[][] = []; + const literalRows: unknown[][] = []; for (const node of nodes) { if (!node.id || !node.kind || !node.name || !node.filePath || !node.language) { console.error('[CodeGraph] Skipping node with missing required fields:', { @@ -587,6 +626,7 @@ export class QueryBuilder { node.updatedAt ?? Date.now(), ]); if (this.isSegmentableKind(node.kind)) this.collectNameSegmentRows(node.name, segmentRows); + this.collectLiteralRows(node, literalRows); } this.runBatched( 'insertNodes', @@ -606,6 +646,7 @@ export class QueryBuilder { '(?,?)', segmentRows ); + this.insertLiteralRows(literalRows); })(); } @@ -773,9 +814,28 @@ export class QueryBuilder { this.nodeCache.delete(id); } } + if (!this.stmts.deleteLiteralsByFile) { + this.stmts.deleteLiteralsByFile = this.db.prepare('DELETE FROM literals WHERE file_path = ?'); + } + this.stmts.deleteLiteralsByFile.run(filePath); this.stmts.deleteNodesByFile.run(filePath); } + /** Refresh extracted literals even when unchanged source keeps its existing nodes. */ + replaceLiteralsForFile(filePath: string, nodes: Node[]): void { + this.db.transaction(() => { + this.db.prepare('DELETE FROM literals WHERE file_path = ?').run(filePath); + const rows: unknown[][] = []; + for (const node of nodes) this.collectLiteralRows(node, rows); + this.insertLiteralRows(rows); + })(); + } + + /** Full indexing repopulates present files and removes literals from deleted files. */ + clearLiterals(): void { + this.db.exec('DELETE FROM literals'); + } + // =========================================================================== // Name-segment vocabulary (prompt-hook graph-derived gate) // =========================================================================== @@ -4023,6 +4083,7 @@ export class QueryBuilder { this.db.transaction(() => { this.db.exec('DELETE FROM unresolved_refs'); this.db.exec('DELETE FROM edges'); + this.db.exec('DELETE FROM literals'); this.db.exec('DELETE FROM nodes'); this.db.exec('DELETE FROM files'); })(); diff --git a/src/db/schema.sql b/src/db/schema.sql index 91ae4805ba..12795a23bc 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -161,6 +161,22 @@ CREATE TABLE IF NOT EXISTS name_segment_vocab ( PRIMARY KEY (segment, name) ) WITHOUT ROWID; +-- Identifier-like string literal → the symbol whose body holds it +-- (extraction/literal-capture.ts). Lets an explore query that names a storage +-- key, CLI flag, or event name seed on its readers and writers: nodes_fts +-- covers names, docstrings and signatures, never string contents. Written on +-- the node write path; a file's rows are deleted with its nodes and a full +-- index clears the table. Reads still join nodes, so a stale row is never +-- surfaced. Empty on migrated databases until the next full index (the +-- source strings are not recoverable from the graph). +CREATE TABLE IF NOT EXISTS literals ( + value TEXT NOT NULL, + node_id TEXT NOT NULL, + file_path TEXT NOT NULL, + PRIMARY KEY (value, node_id) +) WITHOUT ROWID; +CREATE INDEX IF NOT EXISTS idx_literals_file ON literals(file_path); + -- Edge indexes. -- idx_edges_source / idx_edges_target are intentionally omitted — -- the (source, kind) and (target, kind) composites below cover the diff --git a/src/extraction/extraction-version.ts b/src/extraction/extraction-version.ts index 3691e95f0a..e41b5f6149 100644 --- a/src/extraction/extraction-version.ts +++ b/src/extraction/extraction-version.ts @@ -21,4 +21,4 @@ * turns the re-index hint into noise — keep it honest (see CLAUDE.md, "Honesty * in the product is load-bearing"). */ -export const EXTRACTION_VERSION = 27; +export const EXTRACTION_VERSION = 28; diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 0ca3145be2..0366cc8bbd 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -2945,6 +2945,7 @@ export class ExtractionOrchestrator { existingFile.nodeCount === 0 && (existingFile.errors?.length ?? 0) > 0; const incomingHasContent = result.nodes.length > 0; if (!existingIsMarker || !incomingHasContent) { + this.queries.replaceLiteralsForFile(filePath, result.nodes); return; // No changes } } diff --git a/src/extraction/kernel/index.ts b/src/extraction/kernel/index.ts index 1e71d170c3..cb8c3fe959 100644 --- a/src/extraction/kernel/index.ts +++ b/src/extraction/kernel/index.ts @@ -19,6 +19,7 @@ import { commonJsRequireRefs } from '../commonjs-requires'; import { EXTRACTORS } from '../languages'; import { getKernel, kernelSupports } from './loader'; import { decodeExtractBuffers } from './decode'; +import { captureLiterals } from '../literal-capture'; import { KERNEL_ABI_VERSION as LAYOUT_ABI, META as LAYOUT_META, @@ -234,7 +235,7 @@ export function tryKernelExtractRaw( buffers.arena.toString('utf8', errorsOff, errorsOff + errorsLen) ) as ExtractionResult['errors']; } - return { buffers, counts, errors }; + return { buffers: { ...buffers, literalSource: pre }, counts, errors }; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (message.includes('defer:')) { @@ -269,6 +270,7 @@ export function materializeKernelResult( filePath, language ); + if (b.literalSource !== undefined) captureLiterals(b.literalSource, decoded.nodes); decoded.durationMs = result.durationMs; // References read beside the tables (a CommonJS `require`) ride on the transport. if (result.unresolvedReferences.length > 0) decoded.unresolvedReferences.push(...result.unresolvedReferences); @@ -299,6 +301,16 @@ export function tryKernelExtract( POST_PASSES[language]?.(result, source); // Read beside the tables, as TreeSitterExtractor does (a CommonJS `require`). result.unresolvedReferences.push(...commonJsRequireRefs(filePath, source, language)); + // Literal seeds are a pass over source text and the node list, never over + // the tree (literal-capture.ts), so they need no Rust mirror — the same + // pass the wasm extractor runs at the end of extract() applies to the + // kernel's nodes here, and the two paths stay node-for-node identical. + // `pre`, not `source`: the wasm extractor captures from its own preParsed + // text, and a preParse blanks bytes a literal could otherwise be read from. + // Without this every routed language loses its seeds while markdown and the + // unrouted ones keep theirs, and a quoted-key explore query silently stops + // finding holders. + captureLiterals(pre, result.nodes); result.durationMs = Date.now() - t0; return result; } catch (err) { diff --git a/src/extraction/literal-capture.ts b/src/extraction/literal-capture.ts new file mode 100644 index 0000000000..3144a1d13b --- /dev/null +++ b/src/extraction/literal-capture.ts @@ -0,0 +1,99 @@ +/** + * Identifier-like string literals — storage keys, CLI flags, event names, + * config paths — attributed to the symbol whose body contains them, so an + * explore query naming the literal seeds on its readers and writers instead + * of degrading to bag-of-words FTS (the literal is never a symbol name). + * + * The capture is a regex over the file's source text, not a tree walk: the + * per-node JS↔WASM crossing is the parse floor (docs/design/native-extraction-kernel.md), + * and a second walk would pay it again for strings alone. Quotes inside a + * comment can produce a spurious literal; the predicate below keeps only + * identifier-shaped values, so prose never qualifies. + */ +import type { Node } from '../types'; + +/** Distinct literals kept per node; a table of keys beyond this is data, not a seam. */ +const MAX_LITERALS_PER_NODE = 32; + +/** + * A value qualifies when it looks like an identifier an agent would quote + * back verbatim: leading letter or underscore (or a `-`/`--` flag prefix), + * only identifier, path, and namespace characters, and at least one + * separator — `bompus_draft_state`, `--start`, `draft.pick`, `api/v1/users`. + * A plain word (`ready`, `Error`) has no separator and stays out: it would + * seed on every function that logs it. + */ +export function isSeedLiteral(value: string): boolean { + return value.length <= 200 && /^-{0,2}[A-Za-z_][A-Za-z0-9_.:/-]{3,}$/.test(value) && /[_.:/-]/.test(value); +} + +/** + * Quoted spans in the query text, plus any bare whitespace-delimited run that + * qualifies. Deduplicated, query order. + */ +export function seedLiteralsInQuery(query: string): string[] { + const out = new Set(); + for (const m of query.matchAll(/["'`]([^"'`\s]+)["'`]/g)) { + const quoted = m[1] ?? ''; + if (isSeedLiteral(quoted)) out.add(quoted); + } + for (const run of query.split(/\s+/)) { + const bare = run.replace(/^[("'`[]+|[)"'`\],.;:?!]+$/g, ''); + if (isSeedLiteral(bare)) out.add(bare); + } + return [...out]; +} + +/** Single-line quoted strings; template literals with `${` interpolation are skipped. */ +const STRING_RE = /(["'`])((?:\\.|(?!\1)[^\\\n])*)\1/g; + +/** + * Attach qualifying literals in `source` to `nodes` (mutated): each literal + * goes to the innermost non-file node whose source range contains it, else + * the file node. Both extraction paths expose UTF-16 columns (the native + * kernel converts its tree-sitter byte columns before emitting nodes). + */ +export function captureLiterals(source: string, nodes: Node[]): void { + if (nodes.length === 0 || source.length === 0) return; + const symbols = nodes.filter((n) => n.kind !== 'file' && n.kind !== 'import'); + const fileNode = nodes.find((n) => n.kind === 'file'); + const lineStarts = [0]; + for (let i = 0; i < source.length; i++) if (source.charCodeAt(i) === 10) lineStarts.push(i + 1); + const lineOf = (offset: number): number => { + let lo = 0; + let hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if ((lineStarts[mid] ?? 0) <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; + }; + const seen = new Map>(); + for (const m of source.matchAll(STRING_RE)) { + const value = m[2] ?? ''; + if (m[1] === '`' && value.includes('${')) continue; + if (!isSeedLiteral(value)) continue; + const line = lineOf(m.index ?? 0); + const startColumn = (m.index ?? 0) - (lineStarts[line - 1] ?? 0); + const endColumn = startColumn + m[0].length; + let owner: Node | undefined; + for (const n of symbols) { + if (n.startLine > line || n.endLine < line) continue; + if (n.startLine === line && n.startColumn > startColumn) continue; + if (n.endLine === line && n.endColumn < endColumn) continue; + // A contained range is deeper; on identical ranges the later walker node wins. + if (!owner || (n.startLine > owner.startLine || + (n.startLine === owner.startLine && n.startColumn >= owner.startColumn)) && + (n.endLine < owner.endLine || + (n.endLine === owner.endLine && n.endColumn <= owner.endColumn))) owner = n; + } + owner ??= fileNode; + if (!owner) continue; + let set = seen.get(owner.id); + if (!set) seen.set(owner.id, (set = new Set())); + if (set.size >= MAX_LITERALS_PER_NODE || set.has(value)) continue; + set.add(value); + (owner.literals ??= []).push(value); + } +} diff --git a/src/extraction/store-worker.ts b/src/extraction/store-worker.ts index a562f557af..fcf2893657 100644 --- a/src/extraction/store-worker.ts +++ b/src/extraction/store-worker.ts @@ -32,6 +32,7 @@ import { QueryBuilder } from '../db/queries'; import { createDatabase, SqliteDatabase } from '../db/sqlite-adapter'; import { finalizeStoreBundle, type KernelStoreBundle, type StoreBundle } from './store-writer'; import { decodeExtractBuffers } from './kernel/decode'; +import { captureLiterals } from './literal-capture'; if (!parentPort) { throw new Error('store-worker must be run as a worker thread'); @@ -71,6 +72,9 @@ function decodeKernelBundle(bundle: KernelStoreBundle): StoreBundle { bundle.language ); if (bundle.extraRefs?.length) decoded.unresolvedReferences.push(...bundle.extraRefs); + if (bundle.buffers.literalSource !== undefined) { + captureLiterals(bundle.buffers.literalSource, decoded.nodes); + } return finalizeStoreBundle(decoded, bundle.filePath, bundle.language, bundle.file); } diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index 5c27523acc..ce0f61065a 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -33,6 +33,7 @@ import { MyBatisExtractor } from './mybatis-extractor'; import { CfmlExtractor } from './cfml-extractor'; import { tryKernelExtract, takeDeferredPreParse } from './kernel'; import { commonJsRequireRefs } from './commonjs-requires'; +import { captureLiterals } from './literal-capture'; import { getAllFrameworkResolvers, getApplicableFrameworks, @@ -654,6 +655,7 @@ export class TreeSitterExtractor { // nodes and import refs are complete and the file node is still pushed. this.flushFnRefCandidates(); this.flushValueRefs(); + captureLiterals(this.source, this.nodes); if (packageNodeId) this.nodeStack.pop(); this.nodeStack.pop(); diff --git a/src/index.ts b/src/index.ts index 47330503aa..2319b8e202 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,6 +60,7 @@ import ignore from 'ignore'; import { loadDeprioritizePatterns } from './project-config'; import { CodeGraphPackageVersion } from './mcp/version'; import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; +import { seedLiteralsInQuery } from './extraction/literal-capture'; import { createYielder } from './resolution/cooperative-yield'; import { minRefsForPool } from './resolution/resolver-pool'; @@ -568,6 +569,7 @@ export class CodeGraph { // path as every file (re-)indexes below — so a full index is also the // orphan-cleanup pass for names deleted since the last one. try { this.queries.clearNameSegmentVocab(); } catch { /* vocab is advisory — never fail an index over it */ } + try { this.queries.clearLiterals(); } catch { /* literals are repopulated even for unchanged files */ } // Bulk FTS mode for the mass-insert phase: drop the per-row FTS sync // triggers, rebuild nodes_fts once from the nodes table afterwards. // Crash inside the window is healed on the next DatabaseConnection.open. @@ -2320,7 +2322,24 @@ export class CodeGraph { seedNames = []; } } - return this.contextBuilder.findRelevantContext(query, { ...options, seedNames }); + const seedNodeIds = options?.seedNodeIds ?? this.findLiteralSeedIds(query); + return this.contextBuilder.findRelevantContext(query, { ...options, seedNames, seedNodeIds }); + } + + /** + * Literal seeds: a storage key, flag, or event name quoted in the query is + * never a symbol name, so resolve it through the literals table to the + * symbols whose bodies hold it. `CODEGRAPH_LITERAL_SEEDS=0` is the ablation + * switch; failures (pre-v12 database) degrade to no seeds. Explore's file + * sort calls this too, so a holder file ranks as a named file. + */ + findLiteralSeedIds(query: string): string[] { + if (process.env.CODEGRAPH_LITERAL_SEEDS === '0') return []; + try { + return this.queries.findNodeIdsByLiteral(seedLiteralsInQuery(query)); + } catch { + return []; + } } /** diff --git a/src/mcp/explore-source-ranges.ts b/src/mcp/explore-source-ranges.ts new file mode 100644 index 0000000000..e4c9109730 --- /dev/null +++ b/src/mcp/explore-source-ranges.ts @@ -0,0 +1,138 @@ +import type { Node as SyntaxNode } from 'web-tree-sitter'; +import type { Language, Node } from '../types'; +import { getParser, loadGrammarsForLanguages } from '../extraction/grammars'; +import { seedLiteralsInQuery } from '../extraction/literal-capture'; +import { extractSearchTerms, isTestPath } from '../search/query-utils'; + +export interface RequestedSourceRange { + start: number; + end: number; + name: string; + score: number; + nodeId?: string; +} + +/** Source evidence inside an already selected file; never adds graph nodes or edges. */ +export async function requestedSourceRanges( + filePath: string, source: string, language: Language, query: string, nodes: readonly Node[] = [], +): Promise { + const test = isTestPath(filePath) && ['javascript', 'typescript', 'jsx', 'tsx'].includes(language); + const vue = language === 'vue'; + + // A component's name identifies its file, not every line containing "table" + // or "board". Match the remaining question against its template and styles. + const basename = filePath.split('/').pop()!.replace(/\.[^.]+$/, ''); + const escapedBasename = basename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const question = query.replace(new RegExp(`\\b${escapedBasename}\\b`, 'gi'), ''); + const terms = extractSearchTerms(question); + const literals = seedLiteralsInQuery(question); + const identifiers = [...new Set((question.match(/[A-Za-z_$][\w$]*/g) ?? []) + .filter(word => /[a-z][A-Z]|_/.test(word)).map(word => word.toLowerCase()))]; + if (terms.length === 0 && literals.length === 0) return []; + const scoreCeiling = terms.length + (identifiers.length + literals.length) * (terms.length + 1) + 1; + const score = (text: string): number => { + const words = new Set(extractSearchTerms(text, { stems: false })); + return terms.filter(t => words.has(t)).length + + identifiers.filter(t => words.has(t)).length * (terms.length + 1) + + literals.filter(l => text.includes(l)).length * (terms.length + 1); + }; + const ranges: RequestedSourceRange[] = []; + const declarations: RequestedSourceRange[] = []; + const sourceLines = source.split('\n'); + if (!test) { + for (const node of nodes) { + if (!['function', 'method', 'constant', 'variable', 'property'].includes(node.kind) + || node.startLine < 1 || node.endLine < node.startLine + || node.endLine - node.startLine + 1 > sourceLines.length / 2) continue; + const hit = score(sourceLines.slice(node.startLine - 1, node.endLine).join('\n')); + if (hit > 0) declarations.push({ + start: node.startLine, end: node.endLine, name: node.name, score: hit, nodeId: node.id, + }); + } + } + if (vue) { + // Script definitions already have indexed ranges. Only supplement the + // unmodelled template/style text, with bounded windows around actual hits. + let inScript = false; + const lines = source.split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + if (/)/i.test(line)) inScript = true; + if (!inScript) { + const hit = score(line); + if (hit > 0) ranges.push({ + start: Math.max(1, i - 3), end: Math.min(lines.length, i + 5), name: 'query match', score: hit, + }); + } + if (/<\/script\s*>/i.test(line)) inScript = false; + } + // Small style blocks are a useful unit: the selector alone does not show + // its widths. Large blocks still use the matching-line windows above. + for (const match of source.matchAll(/]*)?>[\s\S]*?<\/style\s*>/gi)) { + const start = source.slice(0, match.index).split('\n').length; + const end = start + match[0].split('\n').length - 1; + const hit = score(match[0]); + if (hit > 0 && end - start < 200) ranges.push({ start, end, name: 'style', score: hit }); + } + } else if (test) { + await loadGrammarsForLanguages([language]); + const tree = getParser(language)?.parse(source); + if (!tree) return []; + try { + const visit = (node: SyntaxNode): void => { + if (node.type === 'call_expression') { + const callee = node.childForFieldName('function')?.text ?? ''; + const args = node.childForFieldName('arguments')?.namedChildren ?? []; + if (/^(?:it|test)(?:\.(?:only|skip|concurrent|serial|failing))*$/.test(callee) + && args.some(a => ['arrow_function', 'function_expression'].includes(a.type))) { + const hit = score(node.text) + score(args[0]?.text ?? ''); + if (hit > 0) { + ranges.push({ + start: node.startPosition.row + 1, end: node.endPosition.row + 1, name: 'test', score: hit + 1, + }); + // A long test may not fit whole. Its matching setup/assertion + // statements remain complete units, including multiline arrays. + const statements = (child: SyntaxNode): void => { + if (['expression_statement', 'lexical_declaration', 'variable_declaration'].includes(child.type)) { + const match = score(child.text); + if (match > 0) ranges.push({ + start: child.startPosition.row + 1, end: child.endPosition.row + 1, + name: 'test statement', score: hit + match / scoreCeiling, + }); + return; + } + for (const nested of child.namedChildren) statements(nested); + }; + for (const arg of args) { + if (['arrow_function', 'function_expression'].includes(arg.type)) statements(arg); + } + } + return; + } + } + for (const child of node.namedChildren) visit(child); + }; + visit(tree.rootNode); + } finally { + tree.delete(); + } + } + // Adjacent template hits describe one region; they must not consume every + // candidate slot and exclude a later cell or style block for the same query. + if (vue) { + // An exact identifier/literal on a template line must not lose to a + // larger declaration accumulating incidental prose matches across its body. + const declarationScore = Math.max(0, ...declarations.map(r => r.score)); + for (const r of ranges) if (r.score > terms.length) r.score += declarationScore; + const merged: RequestedSourceRange[] = []; + for (const r of ranges.sort((a, b) => a.start - b.start)) { + const last = merged[merged.length - 1]; + if (last && r.start <= last.end + 1) { + last.end = Math.max(last.end, r.end); + last.score = Math.max(last.score, r.score); + } else merged.push({ ...r }); + } + return [...declarations, ...merged].sort((a, b) => b.score - a.score || a.start - b.start).slice(0, 12); + } + return [...declarations, ...ranges].sort((a, b) => b.score - a.score || a.start - b.start).slice(0, 12); +} diff --git a/src/mcp/server-instructions.ts b/src/mcp/server-instructions.ts index 80a6247a80..3f45a9e8f7 100644 --- a/src/mcp/server-instructions.ts +++ b/src/mcp/server-instructions.ts @@ -51,9 +51,12 @@ calls; a grep/read exploration is dozens. ## How to query +Storage keys, command-line flags, and event names in a query also match symbols containing those exact string literals. Existing indexes need a full rebuild to capture them. + - **Almost any question — "how does X work", architecture, a bug, "what/where is X", or surveying an area** → \`codegraph_explore\` with a natural-language question or the relevant names. ONE capped call returns the verbatim source grouped by file; most often the ONLY call you need. - **"How does X reach/become Y? / the flow / the path from X to Y"** → \`codegraph_explore\`, naming the symbols that span the flow (e.g. \`mutateElement renderScene\`) — it surfaces the call path among them, riding dynamic-dispatch hops, and returns their source. - **Reading or editing a file/symbol you can name** → put its name or file path in the \`codegraph_explore\` query — it returns that current line-numbered source (safe to \`Edit\` from) with the call path and blast radius attached, so you don't Read it separately. For an overloaded name it returns every matching definition's body in one call. +- Requested files include declarations matching the question alongside named bodies, including selectors and cache keys. Requested test files prioritize matching test blocks with their assertions; Vue layout questions reserve space for matching template and style regions. Compound concepts can also return a matching reader/writer with its existing local caller chain; a single named function prioritizes direct callers over unrelated files. These excerpts share the output budget and do not imply additional graph edges. - **Need more?** Call \`codegraph_explore\` again with more specific names — treat the source it returns as already Read. Suggested call counts are advisory only, NOT a quota; extra calls are never rejected or rate-limited. - Qualified symbol names accept dots, \`::\`, or slashes, including containers whose names contain dots (for example, \`AppWeb.Format.group\`). - Named-symbol call paths require exact matches; partial or mistyped names are never silently substituted as flow endpoints. If a graph query reports a missing symbol with did-you-mean suggestions, query the suggested name explicitly. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index c8fec8d3e5..0e6ce36dd3 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -34,7 +34,7 @@ import { } from '../sync/worktree'; import type { PendingFile } from '../sync'; import type { Node, Edge, SearchResult, Subgraph, NodeKind } from '../types'; -import { isTestFile, normalizeNameToken } from '../search/query-utils'; +import { isDistinctiveIdentifier, isTestFile, normalizeNameToken } from '../search/query-utils'; import { groupDefinitions, isQualifiedSymbol, lastQualifierPart, matchesSymbol } from '../graph/symbol-lookup'; import { extractQueryPaths, @@ -61,6 +61,8 @@ import { getUpdateNotice } from '../upgrade/update-check'; import { measurePendingChanges } from './index-freshness'; import { validateAnswerFiles, type AnswerFile } from './answer-freshness'; import { ExploreDiagnostics } from './explore-diagnostics'; +import { requestedSourceRanges } from './explore-source-ranges'; +import { extractSegmentSearchWords, splitIdentifierSegments } from '../search/identifier-segments'; import { EXPLORE_EMISSION_KEY, EXPLORE_SESSION_VIEW_ARG, @@ -737,6 +739,8 @@ export interface ExploreAllocationCandidate { * share stands, bounded only by the pool. */ named?: boolean; + /** Source needed for named bodies plus the strongest supporting declaration/region. */ + minChars?: number; } export interface ExploreAllocation { @@ -839,7 +843,13 @@ export function allocateExploreBudget( // remainder is what concentrates a precise one — the top file's slice grows // with its weight share, uncapped by any flat per-file limit. const ceiling = Math.round(budget.maxOutputChars * A.MAX_SHARE); - const floors = Math.min(pool, A.MIN_CHARS * admitted.length); + const baseFloor = Math.min(A.MIN_CHARS, Math.floor(pool / admitted.length)); + const requestedFloors = admitted.map(c => Math.max(baseFloor, + Math.min(ceiling, Number.isFinite(c.minChars) ? c.minChars! : baseFloor))); + const extraFloor = requestedFloors.reduce((s, n) => s + n - baseFloor, 0); + const floorScale = extraFloor > 0 ? Math.min(1, Math.max(0, pool - baseFloor * admitted.length) / extraFloor) : 0; + const floorByPath = new Map(admitted.map((c, i) => [c.path, baseFloor + Math.floor((requestedFloors[i]! - baseFloor) * floorScale)])); + const floors = [...floorByPath.values()].reduce((s, n) => s + n, 0); const remainder = Math.max(0, pool - floors); // Both parts FLOOR: a sum of rounded shares can exceed the remainder that fed // it (by up to half a char per file), and the reservations must fit the pool @@ -849,7 +859,7 @@ export function allocateExploreBudget( // The valve spares a file the query named (see MAX_SHARE): its share is already // a slice of `pool`, so the reservations still fit the pool exactly. for (const c of admitted) { - const share = Math.floor(floors / admitted.length) + const share = floorByPath.get(c.path)! + Math.floor((remainder * (weights.get(c.path) ?? 0)) / total); allowances.set(c.path, isNamedCandidate(c) ? share : Math.min(share, ceiling)); } @@ -4085,7 +4095,7 @@ export class ToolHandler { } catch { budget = getExploreOutputBudget(Infinity); } - const maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); + let maxFiles = clamp((args.maxFiles as number) || budget.defaultMaxFiles, 1, 20); // File paths named in the query become PINNED files: guaranteed admission, // top of the rank order, funded first — and their span is REMOVED from the @@ -4129,6 +4139,19 @@ export class ToolHandler { return ` Not pinned: ${which}, which define${s.files.length === 1 ? 's' : ''} none of the named symbols.`; }).join(''); const pinnedSet = new Set(pinnedFiles); + // A literal quoted in the query names every file holding it, so the + // default file cap (sized for ranked padding) rises to the holder count; + // the character budget still bounds the answer, and an explicit maxFiles + // stands. + const literalSeedIds = cg.findLiteralSeedIds(matchQuery); + if (!args.maxFiles && literalSeedIds.length > 0) { + const holderFiles = new Set(); + for (const id of literalSeedIds) { + const n = cg.getNode(id); + if (n) holderFiles.add(n.filePath); + } + maxFiles = clamp(Math.max(maxFiles, holderFiles.size), 1, 12); + } const pinnedOrder = new Map(pinnedFiles.map((p, i) => [p, i])); // Per-file allocation diagnostic (CG-4). `null` unless CODEGRAPH_EXPLORE_DEBUG @@ -4202,8 +4225,14 @@ export class ToolHandler { for (const fp of pinnedFiles) { let fileNodes: Node[] = []; try { fileNodes = cg.getNodesInFile(fp); } catch { continue; } - fileNodes - .filter((n) => n.kind !== 'file' && n.kind !== 'import' && n.kind !== 'export') + const seeds = fileNodes.filter((n) => n.kind !== 'file' && n.kind !== 'import' && n.kind !== 'export'); + // A callback-only test file may have no named definitions. Its indexed + // file node still admits the requested source, subject to config guards. + if (seeds.length === 0) { + const file = fileNodes.find(n => n.kind === 'file' && !CONFIG_LEAF_LANGUAGES.has(n.language)); + if (file) seeds.push(file); + } + seeds .sort((a, b) => a.startLine - b.startLine) .slice(0, PINNED_FILE_NODE_CAP) .forEach((n) => { if (!subgraph.nodes.has(n.id)) subgraph.nodes.set(n.id, n); }); @@ -4380,7 +4409,7 @@ export class ToolHandler { const callerCount = (n: Node) => { try { return cg.getCallers(n.id).length; } catch { return 0; } }; const tokens = [...new Set( matchQuery.split(/[\s,()[\]]+/) - .map((t) => t.replace(FILE_EXT, '').trim()) + .map((t) => t.replace(/(? t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) )].slice(0, 16); // PascalCase tokens in the query are type/file disambiguators — when the @@ -4453,13 +4482,16 @@ export class ToolHandler { // files those declarations live in and exempt them from the // declaration-only penalty below (CG-28). Only PRECISE tokens count, by // the same NL-stopword reasoning as the seeding above: "…the file body…" - // must not exempt a `Body` interface it never meant to name. Kept - // separate from `namedSeedIds`, which is callable-only by construction — - // a type never becomes a named seed, so it cannot be the guard here. + // must not exempt a `Body` interface it never meant to name. + // Types also need their bodies prioritized, not just their files + // exempted from the declaration penalty. if (isPreciseToken(t)) { for (const n of raw) { if (DECLARATION_KINDS.has(n.kind) && n.name.toLowerCase() === t.toLowerCase()) { namedTypeFiles.add(n.filePath); + subgraph.nodes.set(n.id, n); + namedSeedIds.add(n.id); + tierSeedIds.add(n.id); } } } @@ -4476,7 +4508,8 @@ export class ToolHandler { // token at a hump boundary or as a prefix. // Exact-empty + camel-shaped only (bare words keep the NL-stopword // guard below), shortest-first, capped so a hot infix can't flood. - if (cands.length === 0 && !isQual && /[a-z][A-Z]/.test(t)) { + const namesDeclaration = raw.some(n => DECLARATION_KINDS.has(n.kind) && n.name.toLowerCase() === t.toLowerCase()); + if (cands.length === 0 && !namesDeclaration && !isQual && /[a-z][A-Z]/.test(t)) { const lcToken = t.toLowerCase(); cands = cg .getNodesByNameSubstring(t, { @@ -4556,6 +4589,13 @@ export class ToolHandler { } } } + // Exact literal holders deserve the same source priority as named symbols. + for (const id of literalSeedIds) { + if (subgraph.nodes.has(id)) { + namedSeedIds.add(id); + tierSeedIds.add(id); + } + } // Step 2: Group nodes by file, score by relevance // `peripheral` accumulates separately so it can be capped — see @@ -4983,6 +5023,19 @@ export class ToolHandler { // tier so it isn't buried under files that merely share surface words (#1064). for (const fp of changeSurfaceFiles) namedSeedFiles.add(fp); + // For one exact callable, direct users are stronger supporting evidence + // than unrelated search roots. Keep the named definition ahead of both. + const directCallerFiles = new Set(); + if (/^[A-Za-z_$][\w$]*$/.test(matchQuery.trim()) && namedSeedIds.size === 1) { + const id = [...namedSeedIds][0]!; + const node = subgraph.nodes.get(id); + if (node && ['function', 'method'].includes(node.kind)) { + for (const caller of cg.getCallers(id)) { + if (!isTestFile(caller.node.filePath)) directCallerFiles.add(caller.node.filePath); + } + } + } + // Multi-term corroboration tier: a file that is BOTH (a) an entry/central file // (a search root, named seed, or graph-central hub — i.e. structurally part of // the answer) AND (b) matched by ≥2 DISTINCT query terms must not be buried by @@ -5020,6 +5073,8 @@ export class ToolHandler { const aNamed = namedSeedFiles.has(a[0]) ? 1 : 0; const bNamed = namedSeedFiles.has(b[0]) ? 1 : 0; if (aNamed !== bNamed) return bNamed - aNamed; + const callerOrder = Number(directCallerFiles.has(b[0])) - Number(directCallerFiles.has(a[0])); + if (callerOrder) return callerOrder; // Corroborated (entry/central + ≥2 terms) tier, above the graph signal. const aCorr = isCorroborated(a[0]) ? 1 : 0; @@ -5186,6 +5241,61 @@ export class ToolHandler { // Score-proportional byte allocation (CG-12). Every file's share of the // envelope is reserved HERE, before a single byte renders, so the render loop // spends a reservation instead of racing for whatever the files above it left. + const sourceMinimums = new Map(); + const exactQueryNames = new Set((matchQuery.match(/[A-Za-z_$][\w$]*/g) ?? []).map(name => name.toLowerCase())); + const querySegments = new Set(extractSegmentSearchWords(matchQuery)); + const conceptOnly = /[a-z][A-Z]|_/.test(matchQuery) && ![...namedSeedIds].some(id => { + const n = subgraph.nodes.get(id); + return n && ['function', 'method', 'component', 'class'].includes(n.kind) + && exactQueryNames.has(n.name.toLowerCase()) && isDistinctiveIdentifier(n.name); + }); + const callerSourceIds = new Set(); + for (const [fp, group] of sortedFiles.slice(0, maxFiles)) { + const stem = fp.split('/').pop()!.replace(/\.[^.]+$/, ''); + const fileNamed = pinnedSet.has(fp) || matchQuery.split(/[^\w$]+/).includes(stem); + // A compound concept can identify a reader/writer without spelling its + // full name. Keep its already-gathered local callers with the body. + const conceptRoots = group.nodes.filter(n => conceptOnly && subgraph.roots.includes(n.id) + && ['function', 'method'].includes(n.kind) && !exactQueryNames.has(n.name.toLowerCase()) + && splitIdentifierSegments(n.name).filter(s => querySegments.has(s)).length >= 2).slice(0, 8); + const localCallers = new Set(conceptRoots.map(n => n.id)); + let frontier = conceptRoots; + for (let depth = 0; depth < 2; depth++) { + const next = frontier.flatMap(n => cg.getCallers(n.id).map(c => c.node)) + .filter(n => n.filePath === fp && group.nodes.some(g => g.id === n.id) && !localCallers.has(n.id)); + frontier = next.slice(0, Math.max(0, 8 - localCallers.size)); + for (const n of frontier) localCallers.add(n.id); + } + if (!fileNamed && localCallers.size === 0) continue; + try { + const absolute = validatePathWithinRoot(projectRoot, fp); + if (!absolute) continue; + const source = readFileSync(absolute, 'utf8'); + const sourceLines = source.split('\n'); + const nodes = cg.getNodesInFile(fp); + const requested = fileNamed ? await requestedSourceRanges(fp, source, group.nodes[0]?.language || 'unknown', matchQuery, nodes) : []; + const supporting = requested.find(r => r.nodeId && !namedSeedIds.has(r.nodeId)); + const region = requested.find(r => !r.nodeId); + if (!supporting && !region && localCallers.size === 0) continue; + const needed = nodes.filter(n => namedSeedIds.has(n.id) && exactQueryNames.has(n.name.toLowerCase()) + && !['file', 'component', 'module'].includes(n.kind) + && n.endLine - n.startLine + 1 <= sourceLines.length / 2) + .map(n => ({ start: n.startLine, end: n.endLine })); + if (supporting) needed.push(...requested.filter(r => r.nodeId && r.score === supporting.score)); + if (region) needed.push(...(fp.endsWith('.vue') && !supporting ? requested.filter(r => !r.nodeId) : [region])); + for (const n of group.nodes) { + if (localCallers.has(n.id) && n.endLine - n.startLine < 200) { + needed.push({ start: n.startLine, end: n.endLine }); + callerSourceIds.add(n.id); + } + } + const chars = mergeRanges(needed).reduce((sum, r) => sum + + sourceLines.slice(Math.max(0, r.start - 4), r.end + 3) + .reduce((bytes, line, i) => bytes + line.length + String(Math.max(1, r.start - 3) + i).length + 2, 0) + + 100, 0); + sourceMinimums.set(fp, chars); + } catch { /* unreadable source must not fail allocation */ } + } const allocation = allocateExploreBudget( sortedFiles.map(([fp, group]) => ({ path: fp, @@ -5200,6 +5310,7 @@ export class ToolHandler { || anchorSpans.has(fp), pinned: pinnedSet.has(fp), named: namedSeedFiles.has(fp), + minChars: sourceMinimums.get(fp), })), budget, maxFiles, @@ -5878,7 +5989,47 @@ export class ToolHandler { parts: ReadonlyArray, ceiling: number, focusLines: ReadonlyArray = [], + requested: ReadonlyArray = [], ): SectionPart[] => { + // Fund complete requested bodies before source-order filler. A method's + // opening line being visible does not mean its return or assertion is. + const protectedRanges: ExploreLineRange[] = []; + let protectedChars = 0; + for (const r of requested) { + if (!parts.some(p => p.range.start <= r.start && p.range.end >= r.end)) continue; + if (protectedRanges.some(p => p.start <= r.start && p.end >= r.end)) continue; + const cost = renderSpan(r).length + (protectedRanges.length ? GAP_MARKER.length : 0); + if (protectedChars + cost > ceiling) continue; + protectedRanges.push(r); + protectedChars += cost; + } + if (protectedRanges.length > 0) { + const remaining: SectionPart[] = []; + for (const p of parts) { + let start = p.range.start; + for (const kept of mergeRanges(protectedRanges)) { + if (kept.end < start || kept.start > p.range.end) continue; + if (kept.start > start) { + const range = { start, end: kept.start - 1 }; + remaining.push({ range, text: renderSpan(range) }); + } + start = Math.max(start, kept.end + 1); + } + if (start <= p.range.end) { + const range = { start, end: p.range.end }; + remaining.push({ range, text: renderSpan(range) }); + } + } + const kept = [...protectedRanges]; + let room = ceiling - protectedChars; + for (const p of remaining) { + const win = headWindowOf(p.range, room - GAP_MARKER.length); + if (!win || win.end - win.start + 1 < MIN_WINDOW_LINES) continue; + kept.push(win); + room -= GAP_MARKER.length + renderSpan(win).length; + } + return mergeRanges(kept).map(range => ({ range, text: renderSpan(range) })); + } const inParts = (line: number) => parts.some((p) => line >= p.range.start && line <= p.range.end); const focus = [...new Set(focusLines)] @@ -6422,13 +6573,17 @@ export class ToolHandler { // clusters rank first and get the spine's bounded overshoot (below), so a // qualified `SQLCompiler.as_sql` is no longer outranked by a denser cluster // around `SQLInsertCompiler.as_sql` that merely matched the bare name. - const EXACT_IMPORTANCE = 11; + // Sits above the named-seed and requested-source tiers (12 and up, scored + // below) so those rank over entries without inheriting exact handling. + const EXACT_IMPORTANCE = 1000; const ranges: Array<{ start: number; end: number; name: string; kind: string; importance: number; spine: boolean; spineCallLine?: number; qualifiedName?: string }> = [...rangeNodes.values()] // Drop whole-file envelope nodes (containers covering >50% of the file). .filter(n => !(ENVELOPE_KINDS.has(n.kind) && (n.endLine - n.startLine + 1) > fileLines.length * 0.5)) .map(n => { let importance = 1; if (exactNodeIds.has(n.id)) importance = EXACT_IMPORTANCE; + else if (namedSeedIds.has(n.id)) importance = 12; + else if (callerSourceIds.has(n.id)) importance = 11; else if (entryNodeIds.has(n.id)) importance = 10; else if (flow.namedNodeIds.has(n.id)) importance = 9; // agent named it → keep its cluster else if (glueNodeIds.has(n.id)) importance = 6; // bridging caller/callee of an entry @@ -6441,6 +6596,32 @@ export class ToolHandler { return { start: n.startLine, end: n.endLine, name: n.name, kind: n.kind, importance, spine: flow.pathNodeIds.has(n.id), spineCallLine: flow.spineCallSites.get(n.id), qualifiedName: n.qualifiedName }; }); + // A path or named component can ask for evidence that has no standalone + // symbol: a test callback's assertions, a template cell or a CSS rule. + // Supplement only requested files, after the drift guard, and spend the + // same cluster budget as indexed source. This never synthesizes an edge. + const stem = filePath.split('/').pop()!.replace(/\.[^.]+$/, ''); + const fileNamed = pinnedSet.has(filePath) || matchQuery.split(/[^\w$]+/).includes(stem); + if (fileNamed || group.nodes.some(n => namedSeedIds.has(n.id))) { + try { + const requested = await requestedSourceRanges(filePath, fileContent, lang || 'unknown', matchQuery, fileNamed ? fileIndexNodes : []); + const declarationScore = Math.max(0, ...requested.filter(r => r.nodeId).map(r => r.score)); + if (declarationScore > 0) { + for (const r of ranges) { + if (r.importance === 12 && exactQueryNames.has(r.name.toLowerCase())) { + r.importance = Math.min(EXACT_IMPORTANCE - 1, 14 + declarationScore); + } + } + } + for (const r of requested) { + const importance = Math.min(EXACT_IMPORTANCE - 1, 13 + r.score); + const existing = r.nodeId && ranges.find(n => n.start === r.start && n.end === r.end); + if (existing) existing.importance = Math.max(existing.importance, importance); + else ranges.push({ ...r, kind: 'source', spine: false, importance }); + } + } catch { /* an unavailable parser must not fail source retrieval */ } + } + // Add edge source locations in this file — captures template references // (component usages, event handlers) that aren't nodes themselves. // Query edges directly from the DB (not just the subgraph) because BFS @@ -6862,7 +7043,10 @@ export class ToolHandler { if (!Number.isFinite(ceiling) || sectionText(r.parts).length <= ceiling) return r; // Windows are subsets of spans dedupeSpans already cleared, so the record // still only ever claims source that was actually sent. - const parts = windowToCeiling(r.parts, ceiling, focusLinesOf(c)); + const requested = c.hasSpine ? [] : c.members + .filter(m => m.importance >= 12) + .sort((a, b) => b.importance - a.importance || a.start - b.start); + const parts = windowToCeiling(r.parts, ceiling, focusLinesOf(c), requested); return { parts, covered: r.covered, shrunk: true }; }; if (sectionText(base.parts).length <= Math.min(cap, guard ?? Infinity)) { diff --git a/src/search/query-paths.ts b/src/search/query-paths.ts index 072e133cb2..17a96b16d5 100644 --- a/src/search/query-paths.ts +++ b/src/search/query-paths.ts @@ -227,7 +227,7 @@ function stripWrapping(token: string): { path: string; lines: { start: number; e for (;;) { const last = s[s.length - 1]; if (!last) break; - if ('\'"`>.,;!?'.includes(last)) { s = s.slice(0, -1); continue; } + if ('\'"`>.,;!?:'.includes(last)) { s = s.slice(0, -1); continue; } if (last === ')' && !s.includes('(')) { s = s.slice(0, -1); continue; } if (last === ']' && !s.includes('[')) { s = s.slice(0, -1); continue; } if (last === '}' && !s.includes('{')) { s = s.slice(0, -1); continue; } diff --git a/src/types.ts b/src/types.ts index 44ffaf4e43..e9a4127756 100644 --- a/src/types.ts +++ b/src/types.ts @@ -188,6 +188,13 @@ export interface Node { /** Generic type parameters */ typeParameters?: string[]; + /** + * Identifier-like string literals inside this symbol's body (storage keys, + * CLI flags, event names — see extraction/literal-capture.ts). Stored in the + * `literals` side table, not a nodes column; seeds explore on an exact hit. + */ + literals?: string[]; + /** * Normalized return/result type name for a function/method (the bare class * name, smart-pointer pointee unwrapped). Captured for C/C++ so resolution @@ -302,6 +309,8 @@ export interface ExtractionResult { edges: Uint8Array; refs: Uint8Array; arena: Uint8Array; + /** Preparsed source for literal attribution after deferred node decoding. */ + literalSource?: string; }; kernelCounts?: { nodes: number; edges: number; refs: number }; } @@ -698,4 +707,12 @@ export interface FindRelevantContextOptions { * SEGMENTS the query's words name are seeded here instead. */ seedNames?: string[]; + + /** + * Node ids whose body holds a string literal the query quoted verbatim + * (`literals` table, CodeGraph.findRelevantContext). Ranked above every + * name-derived candidate: an exact literal is the strongest evidence a + * query carries, and the symbol is never named after it. + */ + seedNodeIds?: string[]; }