From 516ebe4196f723a3745c6e0a07297f2be212e2b1 Mon Sep 17 00:00:00 2001 From: Aaron Queen Date: Sat, 10 Oct 2026 14:52:27 -0600 Subject: [PATCH] fix: read directories in name order so Bun indexes like Node Node's readdirSync returns names sorted; Bun returns the filesystem's order. Files were therefore indexed in a different order under Bun, and equal-score search and explore results differed from Node's. Read directories through a name-sorted helper in the scanner and resolution walks. Also guard channel.unref in the highlight worker: Bun's subprocess.channel has no ref/unref, so highlighting fell back to plain text under Bun. --- .../codegraph-build-validation/SKILL.md | 2 +- CHANGELOG.md | 1 + README.md | 2 +- __tests__/index-order-ties.test.ts | 32 +++++++++++++++++++ docs/viewer-launch-changelog.md | 2 ++ src/directory.ts | 17 ++++++++-- src/extraction/index.ts | 11 +++---- src/resolution/import-resolver.ts | 3 +- src/resolution/index.ts | 4 +-- src/resolution/workspace-packages.ts | 5 +-- src/resolution/workspace-source-entries.ts | 3 +- src/ui-server/highlight/bounded-tokenize.ts | 3 +- 12 files changed, 67 insertions(+), 18 deletions(-) create mode 100644 __tests__/index-order-ties.test.ts diff --git a/.claude/skills/codegraph-build-validation/SKILL.md b/.claude/skills/codegraph-build-validation/SKILL.md index 5c1f8494f..2d9bdde5f 100644 --- a/.claude/skills/codegraph-build-validation/SKILL.md +++ b/.claude/skills/codegraph-build-validation/SKILL.md @@ -25,7 +25,7 @@ npx vitest run __tests__/installer-targets.test.ts npx vitest run __tests__/extraction.test.ts -t "TypeScript" ``` -Under Bun the suite runs green via `__tests__/bun-homedir.setup.ts` (an `os.homedir` shim for oven-sh/bun#29244) plus three test-file adaptations linked to their upstream issues (#42891, #42893, #25498). +Under Bun the suite runs via `__tests__/bun-homedir.setup.ts` (an `os.homedir` shim for oven-sh/bun#29244) plus three test-file adaptations linked to their upstream issues (#42891, #42893, #25498). `npm run test:bun` still fails 19 tests that pass on Node: tests that replace modules or `fs` functions (Bun's `syncBuiltinESMExports` is a no-op and a `vi.mock` of a module loaded through the lazy `require('./extraction')` appears not to apply), one that empties `PATH` (Bun still finds `git`), and a V8-only `gc` test. Do not read those as regressions; a new Bun-only failure is one outside that list. `copy-assets` (called from `build`) copies `src/db/schema.sql` into `dist/`. **Any new SQL asset must be copied or it won't ship.** diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d0b18a0..393c145fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- **Indexing no longer depends on the runtime's directory order.** Node lists a directory's files alphabetically while Bun lists them in the filesystem's own order, so under Bun files were indexed in a different order and equal-score search and explore results came out differently from Node's. Directories are now read in name order on both. - The first session search over a large Claude Code history is faster: transcripts of 256 KB and up are scanned for the entries that can hold prose instead of every line being decoded and parsed, since about 95% of a transcript is tool traffic. The indexed passages and titles are identical (checked on 536 real transcripts); spaced JSON or a large file with no match falls back to the whole-file parse. In-process cold build of those 536 transcripts: about 1.0 s before, 0.7 s after. - The first session search over a large Codex history is faster the same way: rollouts of 256 KB and up are scanned for the message lines and the `session_meta` line instead of every line being decoded and parsed, since about 95% of a rollout is tool traffic. The indexed passages and session ids are identical (checked on 297 real rollouts, and the Claude Code reader's output on 1,331 transcripts after the scan moved to a shared helper); spaced JSON or a large file with no match falls back to the whole-file parse. Parsing those 297 rollouts took about 3.6 s before and 1.6 s after. diff --git a/README.md b/README.md index f4649a78f..66e0c5cda 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Both builds rebuild the links inferred from events, callbacks and function point | `codegraph status` | 173 ms | 117 ms | 86 ms | | `codegraph explore ""` | 224 ms | 121 ms | 99 ms | -**Node or Bun.** The fork produces the same node and edge counts on both, and the full test suite passed on Node.js 24.21.0, Node.js 26.10.0 and Bun 1.4.2 at `48903f5` (`npm run test:bun`; one test is skipped under Bun for [oven-sh/bun#42891](https://github.com/oven-sh/bun/issues/42891)). On Bun, peak index memory is 8–27% lower than on Node, the idle MCP server holds 37–41% less memory, and commands start faster. Index and sync times range from 6% slower to 15% faster than Node's. Bun has two costs. Eight concurrent explores on n8n take 12% longer; contention between worker threads reading through Bun's SQLite is the suspected cause ([oven-sh/bun#44084](https://github.com/oven-sh/bun/issues/44084), [#44187](https://github.com/oven-sh/bun/issues/44187)). Idle CPU is two to four times Node's, though still under 1% of one core. +**Node or Bun.** The fork produces the same node and edge counts on both, and the full test suite passed on Node.js 24.21.0, Node.js 26.10.0 and Bun 1.4.2 at `48903f5` (`npm run test:bun`; one test is skipped under Bun for [oven-sh/bun#42891](https://github.com/oven-sh/bun/issues/42891)). Since then the suite has grown: at `2bf24ef4` plus the directory-order fix, `npm run test:bun` fails 19 of 8,780 tests that pass on Node (writer-lock, watcher-startup and grammar-load-failure tests that replace modules or `fs` functions, a test that empties `PATH`, and a V8-only garbage-collection test); none is a known product defect. On Bun, peak index memory is 8–27% lower than on Node, the idle MCP server holds 37–41% less memory, and commands start faster. Index and sync times range from 6% slower to 15% faster than Node's. Bun has two costs. Eight concurrent explores on n8n take 12% longer; contention between worker threads reading through Bun's SQLite is the suspected cause ([oven-sh/bun#44084](https://github.com/oven-sh/bun/issues/44084), [#44187](https://github.com/oven-sh/bun/issues/44187)). Idle CPU is two to four times Node's, though still under 1% of one core. Measured separately: diff --git a/__tests__/index-order-ties.test.ts b/__tests__/index-order-ties.test.ts new file mode 100644 index 000000000..d72801bcc --- /dev/null +++ b/__tests__/index-order-ties.test.ts @@ -0,0 +1,32 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +let dir: string; +let cg: CodeGraph; + +// Node's readdirSync returns names sorted; Bun's returns the filesystem's own +// order. Equal-score search hits fall back to indexing order, so the walk must +// read directories in name order on both runtimes. +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-index-order-')); + for (const name of ['large-doc2.md', 'large-doc0.md', 'large-doc1.md']) { + fs.writeFileSync(path.join(dir, name), '## Section300\n\nSame text in every file.\n'); + } + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 60_000); + +afterAll(() => { + cg?.destroy(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('indexing order', () => { + it('breaks equal-score search ties in file-name order', () => { + const files = cg.searchNodes('Section300', { limit: 10 }).map((r) => r.node.filePath); + expect(files).toEqual(['large-doc0.md', 'large-doc1.md', 'large-doc2.md']); + }); +}); diff --git a/docs/viewer-launch-changelog.md b/docs/viewer-launch-changelog.md index 9bafed8d6..2d8636908 100644 --- a/docs/viewer-launch-changelog.md +++ b/docs/viewer-launch-changelog.md @@ -121,6 +121,8 @@ These describe `codegraph ui` and its screens. They were taken out of `## [Unrel ## Fixes +- **Syntax highlighting works when the server runs on Bun.** The highlighting worker's setup called a method Bun's subprocess channel does not have, so every slice fell back to plain text. + - Steps diagrams retain database operations made through external client chains without inventing internal dependencies. ## Fixes — Screens, links and navigation diff --git a/src/directory.ts b/src/directory.ts index a90e0d72b..b5ac8b745 100644 --- a/src/directory.ts +++ b/src/directory.ts @@ -21,6 +21,17 @@ export const WSL_CODEGRAPH_DIR = '.codegraph-wsl'; let warnedBadDirName = false; +/** + * A directory's entries in name order. Node's `readdirSync` returns them sorted + * (libuv sorts); Bun returns the filesystem's own order. Indexing, resolution + * and equal-score tie-breaks follow this order, so every walk that feeds them + * reads through here and gets the same order on both runtimes. + */ +export function readDirEntries(dir: string): fs.Dirent[] { + return fs.readdirSync(dir, { withFileTypes: true }) + .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); +} + /** * Resolve the per-project data directory name, honoring the `CODEGRAPH_DIR` * environment override (default `.codegraph`). The override is a single path @@ -442,7 +453,7 @@ export function findIndexedSubprojectRoots( const walk = (dir: string, depth: number): void => { if (out.length >= max || depth > maxDepth) return; let entries: fs.Dirent[]; - try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; } + try { entries = readDirEntries(dir); } catch { return; } for (const e of entries) { if (out.length >= max) return; if (!e.isDirectory()) continue; @@ -1019,7 +1030,7 @@ export function listDirectoryContents(projectRoot: string): string[] { const files: string[] = []; function walkDir(dir: string, prefix: string = ''): void { - const entries = fs.readdirSync(dir, { withFileTypes: true }); + const entries = readDirEntries(dir); for (const entry of entries) { const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; @@ -1054,7 +1065,7 @@ export function getDirectorySize(projectRoot: string): number { let totalSize = 0; function walkDir(dir: string): void { - const entries = fs.readdirSync(dir, { withFileTypes: true }); + const entries = readDirEntries(dir); for (const entry of entries) { // Skip symlinks to prevent following links outside .codegraph diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 32fb55a03..544454d68 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -26,7 +26,7 @@ import { materializeKernelResult } from './kernel'; import { detectGeneratedFile } from './generated-detection'; import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, isMpegTransportStream, hasMpegTsExtension, MPEG_TS_SNIFF_BYTES, hasGrammarLoadFailure, isShopifyThemeMarker, shopifyThemeRoot } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns, PROJECT_CONFIG_FILENAME } from '../project-config'; -import { isCodeGraphDataDir } from '../directory'; +import { isCodeGraphDataDir, readDirEntries } from '../directory'; import { logDebug, logWarn } from '../errors'; import { validatePathWithinRoot, normalizePath } from '../utils'; import ignore, { Ignore } from 'ignore'; @@ -622,7 +622,7 @@ function collectIncludedFiles( let entries: fs.Dirent[]; try { - entries = fs.readdirSync(absDir, { withFileTypes: true }); + entries = readDirEntries(absDir); } catch { return; } @@ -772,7 +772,7 @@ function findNestedGitRepos(absDir: string, relPrefix: string): string[] { if (depth >= EMBEDDED_REPO_SEARCH_DEPTH) continue; let entries: fs.Dirent[]; try { - entries = fs.readdirSync(abs, { withFileTypes: true }); + entries = readDirEntries(abs); } catch { continue; } @@ -1692,7 +1692,7 @@ function scanDirectoryWalk( let entries: fs.Dirent[]; try { - entries = fs.readdirSync(dir, { withFileTypes: true }); + entries = readDirEntries(dir); } catch (error) { logDebug('Skipping unreadable directory', { dir, error: String(error) }); return; @@ -1954,8 +1954,7 @@ export class ExtractionOrchestrator { ? rootDir : path.join(rootDir, relativePath); try { - return fs - .readdirSync(target, { withFileTypes: true }) + return readDirEntries(target) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name); } catch { diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index d3281028b..ef22ecd6d 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -11,6 +11,7 @@ import { Binding, Language } from '../types'; import { UnresolvedRef, ResolutionContext, ImportMapping } from './types'; import { applyAliases, type AliasMap } from './path-aliases'; import { resolveWorkspaceImport } from './workspace-packages'; +import { readDirEntries } from '../directory'; /** * Extension resolution order by language @@ -650,7 +651,7 @@ function loadCppIncludeDirsHeuristic(projectRoot: string): string[] { const conventionDirs = ['include', 'src', 'lib', 'api', 'inc']; try { - const entries = fs.readdirSync(projectRoot, { withFileTypes: true }); + const entries = readDirEntries(projectRoot); for (const entry of entries) { if (!entry.isDirectory()) continue; const name = entry.name; diff --git a/src/resolution/index.ts b/src/resolution/index.ts index f937ecd68..a758ae509 100644 --- a/src/resolution/index.ts +++ b/src/resolution/index.ts @@ -47,6 +47,7 @@ const NODE_BUILTINS = new Set(builtinModules); import { getKernel, type KernelResolverLike, type ResolveRefIn, type ResolveOutcome } from '../extraction/kernel/loader'; import { LRUCache } from './lru-cache'; import { SynthSkips, SYNTH_SKIPS_VERSION } from './synth-skips'; +import { readDirEntries } from '../directory'; // SUPERTYPE_TARGET_KINDS (the kinds an extends/implements edge may TARGET) // lives in ./types — the name-matcher needs the same set to restrict its @@ -709,8 +710,7 @@ export class ReferenceResolver { ? this.projectRoot : path.join(this.projectRoot, relativePath); try { - return fs - .readdirSync(target, { withFileTypes: true }) + return readDirEntries(target) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name); } catch (error) { diff --git a/src/resolution/workspace-packages.ts b/src/resolution/workspace-packages.ts index 81533ff4c..3fe9e5b44 100644 --- a/src/resolution/workspace-packages.ts +++ b/src/resolution/workspace-packages.ts @@ -29,6 +29,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { logDebug } from '../errors'; import { loadWorkspaceSourceEntries } from './workspace-source-entries'; +import { readDirEntries } from '../directory'; export interface WorkspacePackages { /** Exact public exports mapped through static Rollup/Rolldown bundle entries. */ @@ -178,7 +179,7 @@ function collectOhpmFileDeps(projectRoot: string): Map { let entries: fs.Dirent[]; try { - entries = fs.readdirSync(abs, { withFileTypes: true }); + entries = readDirEntries(abs); } catch { continue; } @@ -355,7 +356,7 @@ function expandWorkspaceGlob(projectRoot: string, pattern: string): string[] { const base = norm.slice(0, star).replace(/\/+$/, ''); let entries: fs.Dirent[]; try { - entries = fs.readdirSync(path.join(projectRoot, base), { withFileTypes: true }); + entries = readDirEntries(path.join(projectRoot, base)); } catch { return []; } diff --git a/src/resolution/workspace-source-entries.ts b/src/resolution/workspace-source-entries.ts index a188fa312..459ee75a1 100644 --- a/src/resolution/workspace-source-entries.ts +++ b/src/resolution/workspace-source-entries.ts @@ -5,6 +5,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { parseSourceTreeSync, TreeNode } from '../extraction/parse-tree'; +import { readDirEntries } from '../directory'; type Value = string | Value[] | { [key: string]: Value | undefined } | undefined; type ObjectValue = { [key: string]: Value | undefined }; @@ -284,7 +285,7 @@ function builtOutputSourceEntries(projectRoot: string, member: string, name: str let seen = 0; const walk = (rel: string): void => { let entries: fs.Dirent[]; - try { entries = fs.readdirSync(path.join(srcRoot, rel), { withFileTypes: true }); } catch { return; } + try { entries = readDirEntries(path.join(srcRoot, rel)); } catch { return; } for (const e of entries) { if (++seen > WILDCARD_FILE_BUDGET) return; const child = rel ? `${rel}/${e.name}` : e.name; diff --git a/src/ui-server/highlight/bounded-tokenize.ts b/src/ui-server/highlight/bounded-tokenize.ts index 1e1493d80..ed855a353 100644 --- a/src/ui-server/highlight/bounded-tokenize.ts +++ b/src/ui-server/highlight/bounded-tokenize.ts @@ -37,7 +37,8 @@ function ensureChild(file: string): ChildProcess { // just doesn't declare it, hence the spread. const c = fork(file, [], { stdio: ['ignore', 'ignore', 'ignore', 'ipc'], execArgv: [], ...{ windowsHide: true } }); c.unref(); - c.channel?.unref(); + // Bun's subprocess.channel has no ref/unref (oven-sh/bun#33593). + c.channel?.unref?.(); c.on('message', (msg: { id: number; result: TokenizeResult | null }) => { if (child !== c) return; const p = pending.get(msg.id);