fix(sync): recover index freshness after lock contention (#1959)

__tests__/extraction.test.ts

@@ -8593,6 +8593,29 @@ describe('Nested non-submodule git repos', () => {
     expect(ig.ignores('scratch/tmp.ts')).toBe(true);
   });
 
+  it('filesystem fallback retains git info/exclude and core.excludesFile when ls-files fails (#1959)', async () => {
+    const { execFileSync } = await import('child_process');
+    const root = path.join(tempDir, 'fallback-excludes-root');
+    fs.mkdirSync(root, { recursive: true });
+    execFileSync('git', ['init', '-q'], { cwd: root, stdio: 'pipe' });
+    const globalExcludes = path.join(tempDir, 'fallback-global-excludes');
+    fs.writeFileSync(globalExcludes, 'scratch/\n');
+    execFileSync('git', ['config', 'core.excludesFile', globalExcludes], { cwd: root, stdio: 'pipe' });
+    fs.writeFileSync(path.join(root, '.git', 'info', 'exclude'), 'worktrees/\n');
+    fs.mkdirSync(path.join(root, 'scratch'));
+    fs.mkdirSync(path.join(root, 'worktrees'));
+    fs.writeFileSync(path.join(root, 'app.ts'), 'export const app = 1;\n');
+    fs.writeFileSync(path.join(root, 'scratch', 'hidden.ts'), 'export const hidden = 1;\n');
+    fs.writeFileSync(path.join(root, 'worktrees', 'hidden.ts'), 'export const hidden = 2;\n');
+
+    // rev-parse/config still work, but both ls-files and status fail as they
+    // would under a Git timeout. This exercises the real filesystem walk.
+    fs.writeFileSync(path.join(root, '.git', 'index'), 'not a git index');
+    expect(() => execFileSync('git', ['ls-files'], { cwd: root, stdio: 'pipe' })).toThrow();
+    expect(scanDirectory(root)).toEqual(['app.ts']);
+    expect(await scanDirectoryAsync(root)).toEqual(['app.ts']);
+  });
+
   it('buildScopeIgnore prunes dirs ignored only by a nested .gitignore (#1728)', async () => {
     const { execFileSync } = await import('child_process');
     const git = (cwd: string, ...args: string[]) =>

__tests__/mcp-freshness-bounds.test.ts

@@ -0,0 +1,91 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { createHash } from 'crypto';
+import { Readable } from 'stream';
+import { validateAnswerFiles } from '../src/mcp/answer-freshness';
+import { measurePendingChanges } from '../src/mcp/index-freshness';
+
+vi.mock('fs', async importOriginal => {
+  const actual = await importOriginal<typeof import('fs')>();
+  return { ...actual, createReadStream: vi.fn(actual.createReadStream) };
+});
+
+const workers = vi.hoisted(() => [] as any[]);
+vi.mock('worker_threads', async () => {
+  const { EventEmitter } = await import('events');
+  return { Worker: class extends EventEmitter {
+    terminate = vi.fn(async () => 0);
+    constructor() { super(); workers.push(this); }
+  } };
+});
+
+describe('bounded freshness validation (#1959)', () => {
+  let root: string | undefined;
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.restoreAllMocks();
+    workers.length = 0;
+    if (root) fs.rmSync(root, { recursive: true, force: true });
+    root = undefined;
+  });
+
+  it('marks a file beyond the byte budget unchecked instead of fresh', async () => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-freshness-bounds-'));
+    const content = 'x'.repeat(9 * 1024 * 1024);
+    fs.writeFileSync(path.join(root, 'large.ts'), content);
+    const result = await validateAnswerFiles(root, [{
+      path: 'large.ts', contentHash: createHash('sha256').update(content).digest('hex'),
+    }]);
+    expect(result).toEqual({ stale: [], unchecked: ['large.ts'] });
+  });
+
+  it('bounds a stuck measurement, shares concurrent probes, and terminates its worker', async () => {
+    vi.useFakeTimers();
+    const pending = measurePendingChanges('/freshness-timeout');
+    expect(measurePendingChanges('/freshness-timeout')).toBe(pending);
+    expect(workers).toHaveLength(1);
+    await vi.advanceTimersByTimeAsync(8000);
+    expect(await pending).toBeNull();
+    expect(workers[0].terminate).toHaveBeenCalledOnce();
+  });
+
+  it('returns unknown on deadline even if worker termination is delayed', async () => {
+    vi.useFakeTimers();
+    const pending = measurePendingChanges('/freshness-delayed-exit');
+    let release!: () => void;
+    workers[0].terminate.mockImplementation(() => new Promise<void>(resolve => { release = resolve; }));
+    await vi.advanceTimersByTimeAsync(8000);
+    expect(await pending).toBeNull();
+    release();
+    await Promise.resolve();
+  });
+
+  it('aborts stalled file reads and reports all remaining paths as unchecked', async () => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-freshness-deadline-'));
+    fs.writeFileSync(path.join(root, 'stalled.ts'), 'export const x = 1;');
+    vi.mocked(fs.createReadStream).mockImplementationOnce((_file, options) => new Readable({
+      read() {}, signal: (options as { signal: AbortSignal }).signal,
+    }) as fs.ReadStream);
+    const result = await validateAnswerFiles(root, [
+      { path: 'stalled.ts', contentHash: 'hash' },
+      { path: 'next.ts', contentHash: 'hash' },
+    ]);
+    expect(result).toEqual({ stale: [], unchecked: ['stalled.ts', 'next.ts'] });
+  });
+
+  it('caps measurements across projects and releases the slots on failure', async () => {
+    const first = measurePendingChanges('/freshness-one');
+    const second = measurePendingChanges('/freshness-two');
+    expect(await measurePendingChanges('/freshness-three')).toBeNull();
+    expect(workers).toHaveLength(2);
+    workers[0].emit('error', new Error('worker failed'));
+    workers[1].emit('message', { added: -1, modified: 0, removed: 0 });
+    expect(await first).toBeNull();
+    expect(await second).toBeNull();
+    const retry = measurePendingChanges('/freshness-three');
+    workers[2].emit('message', { added: 0, modified: 1, removed: 0 });
+    expect(await retry).toEqual({ added: 0, modified: 1, removed: 0 });
+  });
+});

__tests__/mcp-stale-refusal.test.ts

@@ -0,0 +1,154 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import CodeGraph from '../src/index';
+import { ExploreSessionState } from '../src/mcp/explore-session-state';
+import { ToolHandler } from '../src/mcp/tools';
+import { QueryPool } from '../src/mcp/query-pool';
+import { Worker } from 'worker_threads';
+import { __setFsWatchForTests } from '../src/sync/watcher';
+
+describe('a degraded index refuses answers from changed files (#1959)', () => {
+  let root: string;
+  let cg: CodeGraph;
+  let handler: ToolHandler;
+  let pool: QueryPool | undefined;
+
+  beforeEach(async () => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-stale-refusal-'));
+    fs.mkdirSync(path.join(root, 'folder with spaces'), { recursive: true });
+    fs.writeFileSync(path.join(root, 'folder with spaces/目录 🦀.ts'), 'export function unicodeSymbol() { return 1; }\n');
+    fs.writeFileSync(path.join(root, 'alpha.ts'), 'export function alphaOnly() { return 1; }\n');
+    fs.writeFileSync(path.join(root, 'beta.ts'), 'export function betaOnly() { return 2; }\n');
+    fs.writeFileSync(
+      path.join(root, 'gamma.ts'),
+      "import { alphaOnly } from './alpha';\nexport function gammaUses() { return alphaOnly(); }\n"
+    );
+    cg = CodeGraph.initSync(root);
+    await cg.indexAll();
+    handler = new ToolHandler(cg);
+
+    __setFsWatchForTests(() => {
+      const err = new Error('too many open files') as NodeJS.ErrnoException;
+      err.code = 'EMFILE';
+      throw err;
+    });
+    expect(cg.watch()).toBe(false);
+    expect(cg.isWatcherDegraded()).toBe(true);
+    __setFsWatchForTests(null);
+  });
+
+  afterEach(async () => {
+    await pool?.destroy();
+    pool = undefined;
+    await handler.closeAll();
+    __setFsWatchForTests(null);
+    try { cg.unwatch(); } catch { /* ignore */ }
+    try { cg.close(); } catch { /* ignore */ }
+    fs.rmSync(root, { recursive: true, force: true });
+  });
+
+  it('names a changed file without serving its result, but keeps unaffected source available', async () => {
+    const alphaPath = path.join(root, 'alpha.ts');
+    const primed = await handler.execute('codegraph_explore', { query: 'alphaOnly' });
+    expect(primed.content[0].text).toContain('export function alphaOnly');
+    const before = fs.statSync(alphaPath);
+    fs.writeFileSync(alphaPath, 'export function alphaOnly() { return 9; }\n');
+    // Identical size and indexed mtime: the last-mile guard must hash bytes,
+    // not trust metadata or a prior two-second drift-cache verdict.
+    fs.utimesSync(alphaPath, before.atime, before.mtime);
+
+    const session = new ExploreSessionState();
+    const refused = await handler.execute('codegraph_explore', { query: 'alphaOnly' }, session);
+    expect(refused.isError).toBeFalsy();
+    expect(refused.content[0].text).toContain('alpha.ts');
+    expect(refused.content[0].text).toContain('cannot answer from this index');
+    expect(refused.content[0].text).not.toContain('export function alphaOnly');
+    expect(session.view().projects).toEqual([]);
+
+    const unaffected = await handler.execute('codegraph_explore', { query: 'betaOnly' }, session);
+    expect(unaffected.content[0].text).toContain('export function betaOnly');
+    expect(unaffected.content[0].text).toContain('auto-sync is DISABLED');
+
+    // Let the ordinary sync see a definite metadata change, then ensure the
+    // same session receives the source it was not shown before.
+    fs.utimesSync(alphaPath, before.atime, new Date(before.mtimeMs + 2000));
+    await cg.sync();
+    const refreshed = await handler.execute('codegraph_explore', { query: 'alphaOnly' }, session);
+    expect(refreshed.content[0].text).toContain('export function alphaOnly');
+    expect(refreshed.content[0].text).toContain('return 9');
+    expect(refreshed.content[0].text).not.toContain('cannot answer from this index');
+  });
+
+  it('refuses a graph answer that names a changed file, and serves one that does not', async () => {
+    const callers = await handler.execute('codegraph_callers', { symbol: 'alphaOnly' });
+    expect(callers.content[0].text).toContain('gammaUses');
+
+    fs.writeFileSync(
+      path.join(root, 'gamma.ts'),
+      "import { alphaOnly } from './alpha';\nexport function gammaUses() { return 0; }\n"
+    );
+
+    const refused = await handler.execute('codegraph_callers', { symbol: 'alphaOnly' });
+    expect(refused.isError).toBeFalsy();
+    expect(refused.content[0].text).toContain('cannot answer from this index');
+    expect(refused.content[0].text).toContain('- gamma.ts');
+    expect(refused.content[0].text).not.toContain('gammaUses');
+
+    const search = await handler.execute('codegraph_search', { query: 'gammaUses' });
+    expect(search.content[0].text).toContain('cannot answer from this index');
+
+    const unaffected = await handler.execute('codegraph_search', { query: 'betaOnly' });
+    expect(unaffected.content[0].text).toContain('beta.ts');
+    expect(unaffected.content[0].text).not.toContain('cannot answer from this index');
+  });
+  it.each([false, true])('preserves spaces and Unicode through provenance (pooled=%s)', async pooled => {
+    if (pooled) {
+      pool = new QueryPool({ root, size: 1, createWorker: () => new Worker(
+        path.resolve(__dirname, '../dist/mcp/query-worker.js'), { workerData: { root } },
+      ) });
+      handler.setQueryPool(pool);
+      await vi.waitFor(() => expect(pool!.ready).toBe(true), { timeout: 15000 });
+    }
+    const relative = 'folder with spaces/目录 🦀.ts';
+    const initial = await handler.execute('codegraph_search', { query: 'unicodeSymbol' });
+    expect(initial.content[0].text).toContain(relative);
+    expect(initial).not.toHaveProperty('_cgAnswerFiles');
+    const before = fs.statSync(path.join(root, relative));
+    fs.writeFileSync(path.join(root, relative), 'export function unicodeSymbol() { return 9; }\n');
+    fs.utimesSync(path.join(root, relative), before.atime, before.mtime);
+    const result = await handler.execute('codegraph_search', { query: 'unicodeSymbol' });
+    expect(result.isError).toBeFalsy();
+    expect(result.content[0].text).toContain('cannot answer from this index');
+    expect(result.structuredContent).toEqual({ freshness: { stale: [relative], unchecked: [] } });
+    expect(result).not.toHaveProperty('_cgAnswerFiles');
+  });
+
+  it.each(['codegraph_search', 'codegraph_explore', 'codegraph_callers', 'codegraph_callees', 'codegraph_impact'])(
+    'refuses deleted contributing files in %s', async tool => {
+      fs.unlinkSync(path.join(root, 'alpha.ts'));
+      const result = await handler.execute(tool, { query: 'alphaOnly', symbol: 'alphaOnly' });
+      expect(result.isError).toBeFalsy();
+      expect(result.content[0].text).toContain('cannot answer from this index');
+      expect(result.content[0].text).toContain('alpha.ts');
+      expect(result).not.toHaveProperty('_cgExploreEmission');
+    },
+  );
+
+  it('refuses rather than silently validating only the first 200 contributing files', async () => {
+    for (let i = 0; i < 201; i++) {
+      fs.writeFileSync(path.join(root, `caller${i}.ts`),
+        `import { alphaOnly } from './alpha'; export function caller${i}() { return alphaOnly(); }\n`);
+    }
+    await cg.indexAll();
+    const raw = await handler.executeReadTool('codegraph_impact', { symbol: 'alphaOnly' });
+    expect(raw._cgAnswerFiles!.length).toBeGreaterThan(200);
+    const result = await handler.execute('codegraph_impact', { symbol: 'alphaOnly' });
+    expect(result.isError).toBeFalsy();
+    expect(result.content[0].text).toContain('validation budget');
+    expect((result.structuredContent!.freshness as { unchecked: string[] }).unchecked.length).toBeGreaterThan(0);
+    expect(result.content[0].text).not.toContain('**Impact');
+  });
+
+});

__tests__/mcp-staleness-banner.test.ts

@@ -20,7 +20,7 @@
  * left untouched.
  */
 
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
@@ -72,6 +72,7 @@ describe('MCP staleness banner', () => {
 
   afterEach(() => {
     __setFsWatchForTests(null); // reset the injected fs.watch seam
+    vi.restoreAllMocks();
     try { cg.unwatch(); } catch { /* ignore */ }
     try { cg.close(); } catch { /* ignore */ }
     if (fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true });
@@ -209,6 +210,26 @@ describe('MCP staleness banner', () => {
     // status renders the notice inline, so the auto-banner is not also prepended.
     expect(text.startsWith('⚠️')).toBe(false);
   });
+
+  it('distinguishes a re-armed but not-yet-caught-up watcher from a disabled one (#1959)', async () => {
+    vi.spyOn(cg, 'isWatcherDegraded').mockReturnValue(true);
+    vi.spyOn(cg, 'isWatcherRecovering').mockReturnValue(true);
+
+    const search = await handler.execute('codegraph_search', { query: 'alphaOnly' });
+    expect(search.content[0].text).toMatch(/auto-sync is RECOVERING/);
+    expect(search.content[0].text).not.toMatch(/auto-sync is DISABLED/);
+
+    const status = await handler.execute('codegraph_status', {});
+    expect(status.content[0].text).toContain('**Auto-sync recovering:**');
+    expect(status.content[0].text).not.toContain('**Auto-sync disabled:**');
+  });
+
+  it('asks the owned watcher to re-arm on the next MCP tool call (#1959)', async () => {
+    const rearm = vi.spyOn(cg, 'rearmWatcherAfterLockContention').mockReturnValue(false);
+
+    await handler.execute('codegraph_status', {});
+    expect(rearm).toHaveBeenCalledTimes(1);
+  });
 });
 
 describe('MCP staleness banner — matching whole paths (#1968)', () => {

__tests__/mcp-status-freshness.test.ts

@@ -0,0 +1,64 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { execFileSync } from 'child_process';
+import CodeGraph from '../src/index';
+import { measurePendingChanges } from '../src/mcp/index-freshness';
+import { ToolHandler } from '../src/mcp/tools';
+
+describe('MCP status freshness (#1959)', () => {
+  let root: string;
+  let cg: CodeGraph;
+  let handler: ToolHandler;
+
+  beforeEach(async () => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-status-freshness-'));
+    fs.writeFileSync(path.join(root, 'modify.ts'), 'export const modify = 1;\n');
+    fs.writeFileSync(path.join(root, 'remove.ts'), 'export const remove = 1;\n');
+    const git = (...args: string[]) => execFileSync('git', args, { cwd: root, stdio: 'pipe' });
+    git('init', '-q');
+    git('config', 'user.name', 'CodeGraph Test');
+    git('config', 'user.email', 'codegraph-test@example.invalid');
+    git('add', 'modify.ts', 'remove.ts');
+    git('commit', '-qm', 'baseline');
+    cg = CodeGraph.initSync(root);
+    await cg.indexAll();
+    handler = new ToolHandler(cg);
+  });
+
+  afterEach(() => {
+    try { cg.close(); } catch { /* ignore */ }
+    fs.rmSync(root, { recursive: true, force: true });
+  });
+
+  it('reports the latest indexed file and exact change counts', async () => {
+    const initial = (await handler.execute('codegraph_status', {})).content[0].text;
+    expect(initial).toMatch(/\*\*Latest file indexed:\*\* \d{4}-\d\d-\d\dT/);
+    expect(initial).toContain('**Changes since index:** 0 added, 0 modified, 0 removed');
+
+    fs.writeFileSync(path.join(root, 'modify.ts'), 'export const modify = 42;\n');
+    fs.unlinkSync(path.join(root, 'remove.ts'));
+    fs.writeFileSync(path.join(root, 'add.ts'), 'export const added = 1;\n');
+
+    const result = await handler.execute('codegraph_status', {});
+    expect(result.structuredContent).toEqual({ freshness: {
+      lastIndexedAt: cg.getLastIndexedAt(), changes: { added: 1, modified: 1, removed: 1 }, complete: true,
+    } });
+    const changed = result.content[0].text;
+    expect(changed).toContain('**Changes since index:** 1 added, 1 modified, 1 removed');
+  });
+
+  it('returns unknown rather than a false zero when the measurement cannot open an index', async () => {
+    expect(await measurePendingChanges(path.join(root, 'missing'))).toBeNull();
+  });
+
+  it('counts edits committed after the index even when the working tree is clean', async () => {
+    fs.writeFileSync(path.join(root, 'modify.ts'), 'export const modify = 99;\n');
+    execFileSync('git', ['add', 'modify.ts'], { cwd: root, stdio: 'pipe' });
+    execFileSync('git', ['commit', '-qm', 'changed'], { cwd: root, stdio: 'pipe' });
+
+    const status = (await handler.execute('codegraph_status', {})).content[0].text;
+    expect(status).toContain('**Changes since index:** 0 added, 1 modified, 0 removed');
+  });
+});