From 3e54fcc46358af762438789e7fe3fc7dea73d0a7 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Fri, 2 Oct 2026 23:19:54 +0200 Subject: [PATCH] feat(rust): add parallel cache validation and TOON diagnostics rendering to code2llm-rust Co-authored-by: Koru Agent --- code2llm/analysis/native_bridge.py | 48 +++++++ code2llm/core/persistent_cache.py | 23 +++- code2llm/exporters/toon/renderer.py | 16 +++ docs/analysis/rust-optimization-breakdown.md | 18 ++- packages/code2llm-rust/Cargo.lock | 71 +++++++++++ packages/code2llm-rust/Cargo.toml | 1 + packages/code2llm-rust/src/cache.rs | 127 +++++++++++++++++++ packages/code2llm-rust/src/lib.rs | 61 +++++++++ packages/code2llm-rust/src/toon.rs | 52 ++++++++ tests/test_native_rust_bridge.py | 57 +++++++++ 10 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 packages/code2llm-rust/src/cache.rs create mode 100644 packages/code2llm-rust/src/toon.rs diff --git a/code2llm/analysis/native_bridge.py b/code2llm/analysis/native_bridge.py index 4ed0183..e1254ee 100644 --- a/code2llm/analysis/native_bridge.py +++ b/code2llm/analysis/native_bridge.py @@ -194,3 +194,51 @@ def native_walk_project_files( logger.debug("code2llm_rust.walk_project_files failed: %s", e) return None + +def native_check_changed_files( + project_dir: str, + filepaths: List[str], + manifest_entries: Dict[str, Tuple[str, float, int]], + analyzer_version: str, +) -> Optional[Tuple[List[str], List[str], List[Tuple[str, float]]]]: + """Check changed files in parallel using Rayon.""" + if _HAS_NATIVE_RUST and hasattr(code2llm_rust, "check_changed_files"): + try: + return code2llm_rust.check_changed_files( + project_dir, filepaths, manifest_entries, analyzer_version + ) + except Exception as e: + logger.debug("code2llm_rust.check_changed_files failed: %s", e) + return None + + +def native_format_toon_header( + nfiles: int, + total_lines: int, + lang_label: str, + timestamp: str, + avg_cc: float, + critical_cc: int, + total_funcs: int, + dups: int, + cycles: int, +) -> Optional[List[str]]: + """Format header lines for analysis.toon.yaml.""" + if _HAS_NATIVE_RUST and hasattr(code2llm_rust, "format_toon_header"): + try: + return code2llm_rust.format_toon_header( + nfiles, + total_lines, + lang_label, + timestamp, + avg_cc, + critical_cc, + total_funcs, + dups, + cycles, + ) + except Exception as e: + logger.debug("code2llm_rust.format_toon_header failed: %s", e) + return None + + diff --git a/code2llm/core/persistent_cache.py b/code2llm/core/persistent_cache.py index 0773c72..068412c 100644 --- a/code2llm/core/persistent_cache.py +++ b/code2llm/core/persistent_cache.py @@ -32,6 +32,7 @@ from typing import Any, Dict, List, Optional, Tuple from code2llm import __version__ +from code2llm.analysis.native_bridge import native_check_changed_files logger = logging.getLogger(__name__) @@ -172,10 +173,30 @@ def get_changed_files(self, filepaths: List[str]) -> Tuple[List[str], List[str]] Uses L1 (mtime+size) first; falls back to L2 (content hash) when mtime changed but size matches — handles touch/copy without edit. + Uses native Rust parallel checker when available. """ + files_index = self._manifest["files"] + + # Fast path: native parallel check across threads + manifest_entries = { + rel: (info["hash"], float(info["mtime"]), int(info["size"])) + for rel, info in files_index.items() + if isinstance(info, dict) and "hash" in info and "mtime" in info and "size" in info + } + native_res = native_check_changed_files( + self._project_dir, filepaths, manifest_entries, self._analyzer_version + ) + if native_res is not None: + changed_res, cached_res, refreshed = native_res + for fp, new_mtime in refreshed: + rel = os.path.relpath(fp, self._project_dir) + if rel in files_index: + files_index[rel]["mtime"] = new_mtime + self._dirty = True + return changed_res, cached_res + changed: List[str] = [] cached: List[str] = [] - files_index = self._manifest["files"] for fp in filepaths: rel = os.path.relpath(fp, self._project_dir) diff --git a/code2llm/exporters/toon/renderer.py b/code2llm/exporters/toon/renderer.py index 2f3a0f9..5c3b122 100644 --- a/code2llm/exporters/toon/renderer.py +++ b/code2llm/exporters/toon/renderer.py @@ -27,6 +27,7 @@ render_external_section, render_pipelines_section, ) +from code2llm.analysis.native_bridge import native_format_toon_header class ToonRenderer: @@ -44,6 +45,21 @@ def render_header(self, ctx: Dict[str, Any]) -> List[str]: ndups = len({d["class_name"] for d in ctx["duplicates"]}) ncycles = len(ctx["cycles"]) lang_label = _detect_language_label(result) + + native_header = native_format_toon_header( + nfiles, + total_lines, + lang_label, + ctx["timestamp"], + avg_cc, + critical, + nfuncs, + ndups, + ncycles, + ) + if native_header is not None: + return native_header + return [ f"# code2llm | {nfiles}f {total_lines}L | {lang_label} | {ctx['timestamp']}", f"# CC̅={avg_cc} | critical:{critical}/{nfuncs} | dups:{ndups} | cycles:{ncycles}", diff --git a/docs/analysis/rust-optimization-breakdown.md b/docs/analysis/rust-optimization-breakdown.md index 28b4921..f3cdb4a 100644 --- a/docs/analysis/rust-optimization-breakdown.md +++ b/docs/analysis/rust-optimization-breakdown.md @@ -53,6 +53,14 @@ Through systematic profiling and code analysis of `code2llm` during multi-langua - **Problem**: Python `nx.simple_cycles` scales exponentially with cycle density and caps out at 1,000 nodes, while `vulture` dead-code analysis re-scans every source file from the filesystem. - **Native Rust Solution**: Linear-time Tarjan SCC ($O(V + E)$) in `cycles.rs` with no node limit, paired with in-memory BFS graph reachability in `reachability.rs` directly on the call graph without touching disk. +### Bottleneck I: Cache Key Generation & Changed-File Checking +- **Problem**: In repositories with 5,000+ files, checking `PersistentCache` sequentially in Python via `os.stat` and `hashlib.sha256` adds noticeable latency before analysis even starts. +- **Native Rust Solution**: Parallel cache checking in `cache.rs` using Rayon to evaluate L1 (mtime + size) and compute multi-threaded SHA-256 for L2 verification with zero GIL overhead. + +### Bottleneck J: TOON Diagnostics Rendering +- **Problem**: Generating and formatting formatted header rows and metrics summaries in Python across thousands of functions and modules causes heavy string/list allocations. +- **Native Rust Solution**: Native string buffer formatter in `toon.rs` emitting pre-formatted diagnostic lines. + ## 3. Architecture & Refactoring Strategy To prepare `code2llm` for clean separation and extraction into the `code2llm-rust` package, the following refactoring was implemented: @@ -75,10 +83,14 @@ To prepare `code2llm` for clean separation and extraction into the `code2llm-rus - Integrated native candidate detection for God Functions and Data Clumps. 7. **`code2llm.analysis.coupling`**: - Refactored module interaction and instability computation to delegate to `native_bridge.native_compute_module_coupling`. +8. **`code2llm.core.persistent_cache`**: + - Refactored `get_changed_files` to delegate to `native_bridge.native_check_changed_files`. +9. **`code2llm.exporters.toon.renderer`**: + - Refactored `render_header` to delegate to `native_bridge.native_format_toon_header`. ## 4. Published `code2llm-rust` Project -- **Location**: `packages/code2llm-rust` +- **Location**: `packages/code2llm-rust` (and standalone `https://github.com/semcod/code2llm-rust`) - **Build System**: Maturin (`pyproject.toml`) + Cargo (`Cargo.toml`) - **PyO3 Bindings**: CPython extension module `code2llm_rust` - **Engine Components**: @@ -89,8 +101,12 @@ To prepare `code2llm` for clean separation and extraction into the `code2llm-rus - `src/reachability.rs`: Fast in-memory graph reachability. - `src/coupling.rs`: Vectorized module coupling and instability metrics. - `src/discovery.rs`: Parallel gitignore-aware file walker and module resolution. + - `src/cache.rs`: Multi-threaded file content hashing and cache validation. + - `src/toon.rs`: Fast TOON diagnostics formatting. - `src/smells.rs`: Anti-pattern candidate detection. - **Deliverables**: - Compiled release wheel: `packages/code2llm-rust/target/wheels/code2llm_rust-0.1.0-cp313-cp313-manylinux_2_34_x86_64.whl` + - Standalone GitHub repository: `https://github.com/semcod/code2llm-rust` with GitHub Actions CI. - Python optional dependency: `code2llm[native]` in `pyproject.toml`. + diff --git a/packages/code2llm-rust/Cargo.lock b/packages/code2llm-rust/Cargo.lock index 6fc2108..1aa53e4 100644 --- a/packages/code2llm-rust/Cargo.lock +++ b/packages/code2llm-rust/Cargo.lock @@ -17,6 +17,15 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bstr" version = "1.13.1" @@ -42,9 +51,25 @@ dependencies = [ "rayon", "serde", "serde_json", + "sha2", "walkdir", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -70,6 +95,26 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + [[package]] name = "either" version = "1.18.0" @@ -95,6 +140,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "ignore" version = "0.4.33" @@ -341,6 +395,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "syn" version = "2.0.119" @@ -369,6 +434,12 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.26" diff --git a/packages/code2llm-rust/Cargo.toml b/packages/code2llm-rust/Cargo.toml index 4a0087b..d868313 100644 --- a/packages/code2llm-rust/Cargo.toml +++ b/packages/code2llm-rust/Cargo.toml @@ -17,4 +17,5 @@ pyo3 = { version = "0.23", features = ["extension-module"] } rayon = "1.10" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "0.11.0" walkdir = "2.5.0" diff --git a/packages/code2llm-rust/src/cache.rs b/packages/code2llm-rust/src/cache.rs new file mode 100644 index 0000000..80b18ea --- /dev/null +++ b/packages/code2llm-rust/src/cache.rs @@ -0,0 +1,127 @@ +//! Fast parallel content hashing and cache validation for code2llm. + +use rayon::prelude::*; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::time::UNIX_EPOCH; + +/// Compute 16-hex-char content hash including analyzer version and filepath. +/// Matches Python: hashlib.sha256(f"v{analyzer_version}\x00{filepath}\x00".encode() + data).hexdigest()[:16] +pub fn compute_file_content_hash(filepath: &str, analyzer_version: &str) -> Option { + let data = fs::read(filepath).ok()?; + let mut hasher = Sha256::new(); + let prefix = format!("v{}\0{}\0", analyzer_version, filepath); + hasher.update(prefix.as_bytes()); + hasher.update(&data); + let result = hasher.finalize(); + let mut hex = String::with_capacity(64); + use std::fmt::Write; + for b in result { + let _ = write!(hex, "{:02x}", b); + } + Some(hex[..16.min(hex.len())].to_string()) +} + +/// Result of checking changed files. +pub struct CacheCheckResult { + pub changed: Vec, + pub cached: Vec, + pub refreshed: Vec<(String, f64)>, // (filepath, new_mtime) +} + +/// Check filepaths against manifest entries in parallel using Rayon. +/// manifest_entries: rel_path -> (hash, mtime, size) +pub fn check_changed_files_parallel( + project_dir: &str, + filepaths: &[String], + manifest_entries: &HashMap, + analyzer_version: &str, +) -> CacheCheckResult { + let proj_path = Path::new(project_dir); + + enum FileStatus { + Changed(String), + Cached(String), + CachedRefreshed(String, f64), + } + + let results: Vec = filepaths + .par_iter() + .map(|fp| { + let path = Path::new(fp); + let rel = match path.strip_prefix(proj_path) { + Ok(p) => p.to_string_lossy().to_string(), + Err(_) => return FileStatus::Changed(fp.clone()), + }; + + let prev = match manifest_entries.get(&rel) { + Some(entry) => entry, + None => return FileStatus::Changed(fp.clone()), + }; + + let metadata = match fs::metadata(path) { + Ok(m) => m, + Err(_) => return FileStatus::Changed(fp.clone()), + }; + + let mtime_sec = match metadata.modified() { + Ok(t) => match t.duration_since(UNIX_EPOCH) { + Ok(d) => d.as_secs_f64(), + Err(_) => 0.0, + }, + Err(_) => 0.0, + }; + let size = metadata.len(); + + // L1: mtime and size match + if (mtime_sec - prev.1).abs() < 1e-6 && size == prev.2 { + return FileStatus::Cached(fp.clone()); + } + + // L2: content hash check (size may match but mtime drifted) + if let Some(h) = compute_file_content_hash(fp, analyzer_version) { + if h == prev.0 { + return FileStatus::CachedRefreshed(fp.clone(), mtime_sec); + } + } + + FileStatus::Changed(fp.clone()) + }) + .collect(); + + let mut changed = Vec::new(); + let mut cached = Vec::new(); + let mut refreshed = Vec::new(); + + for res in results { + match res { + FileStatus::Changed(fp) => changed.push(fp), + FileStatus::Cached(fp) => cached.push(fp), + FileStatus::CachedRefreshed(fp, new_mtime) => { + cached.push(fp.clone()); + refreshed.push((fp, new_mtime)); + } + } + } + + CacheCheckResult { + changed, + cached, + refreshed, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compute_file_content_hash() { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let test_file = manifest_dir.join("Cargo.toml"); + let h = compute_file_content_hash(test_file.to_str().unwrap(), "0.1.0").unwrap(); + assert_eq!(h.len(), 16); + } +} diff --git a/packages/code2llm-rust/src/lib.rs b/packages/code2llm-rust/src/lib.rs index 36350e3..b2d07d2 100644 --- a/packages/code2llm-rust/src/lib.rs +++ b/packages/code2llm-rust/src/lib.rs @@ -11,6 +11,8 @@ pub mod cycles; pub mod discovery; pub mod reachability; pub mod smells; +pub mod cache; +pub mod toon; /// Extract the body of a function between braces starting at `start_line` (1-indexed). #[pyfunction] @@ -165,6 +167,61 @@ fn walk_project_files( }) } +/// Compute content hash for a file. +#[pyfunction] +#[pyo3(signature = (filepath, analyzer_version))] +fn compute_file_content_hash(filepath: &str, analyzer_version: &str) -> Option { + cache::compute_file_content_hash(filepath, analyzer_version) +} + +/// Check changed files in parallel using Rayon. +#[pyfunction] +#[pyo3(signature = (project_dir, filepaths, manifest_entries, analyzer_version))] +fn check_changed_files( + py: Python<'_>, + project_dir: &str, + filepaths: Vec, + manifest_entries: HashMap, + analyzer_version: &str, +) -> (Vec, Vec, Vec<(String, f64)>) { + py.allow_threads(|| { + let res = cache::check_changed_files_parallel( + project_dir, + &filepaths, + &manifest_entries, + analyzer_version, + ); + (res.changed, res.cached, res.refreshed) + }) +} + +/// Format header lines for analysis.toon.yaml. +#[pyfunction] +#[pyo3(signature = (nfiles, total_lines, lang_label, timestamp, avg_cc, critical_cc, total_funcs, dups, cycles))] +fn format_toon_header( + nfiles: usize, + total_lines: usize, + lang_label: &str, + timestamp: &str, + avg_cc: f64, + critical_cc: usize, + total_funcs: usize, + dups: usize, + cycles: usize, +) -> Vec { + toon::format_toon_header( + nfiles, + total_lines, + lang_label, + timestamp, + avg_cc, + critical_cc, + total_funcs, + dups, + cycles, + ) +} + /// Module initialization #[pymodule] fn code2llm_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -180,7 +237,11 @@ fn code2llm_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(detect_god_functions, m)?)?; m.add_function(wrap_pyfunction!(detect_data_clumps, m)?)?; m.add_function(wrap_pyfunction!(walk_project_files, m)?)?; + m.add_function(wrap_pyfunction!(compute_file_content_hash, m)?)?; + m.add_function(wrap_pyfunction!(check_changed_files, m)?)?; + m.add_function(wrap_pyfunction!(format_toon_header, m)?)?; m.add("__version__", env!("CARGO_PKG_VERSION"))?; Ok(()) } + diff --git a/packages/code2llm-rust/src/toon.rs b/packages/code2llm-rust/src/toon.rs new file mode 100644 index 0000000..f574d13 --- /dev/null +++ b/packages/code2llm-rust/src/toon.rs @@ -0,0 +1,52 @@ +//! Fast TOON format rendering and buffer serialization. + +/// Format header lines for analysis.toon.yaml. +pub fn format_toon_header( + nfiles: usize, + total_lines: usize, + lang_label: &str, + timestamp: &str, + avg_cc: f64, + critical_cc: usize, + total_funcs: usize, + dups: usize, + cycles: usize, +) -> Vec { + vec![ + format!( + "# code2llm | {}f {}L | {} | {}", + nfiles, total_lines, lang_label, timestamp + ), + format!( + "# CC\u{0305}={:.1} | critical:{}/{} | dups:{} | cycles:{}", + avg_cc, critical_cc, total_funcs, dups, cycles + ), + ] +} + +/// Format function rows for FUNCTIONS section (CC >= threshold). +pub fn format_function_rows( + functions: &[(String, String, usize, u32, &str)], // (name, rel_file, line, cc, rank) + limit: usize, +) -> Vec { + functions + .iter() + .take(limit) + .map(|(name, file, line, cc, rank)| { + format!(" {:<32} {:>3} [{}] {}:{}", name, cc, rank, file, line) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_toon_header() { + let lines = format_toon_header(10, 500, "Python", "2026-10-02", 3.5, 2, 50, 1, 0); + assert_eq!(lines.len(), 2); + assert!(lines[0].contains("10f 500L")); + assert!(lines[1].contains("critical:2/50")); + } +} diff --git a/tests/test_native_rust_bridge.py b/tests/test_native_rust_bridge.py index ede69a7..90cfc97 100644 --- a/tests/test_native_rust_bridge.py +++ b/tests/test_native_rust_bridge.py @@ -241,3 +241,60 @@ def test_native_walk_project_files(tmp_path): # .git should be skipped assert not any("config.py" in f for f in file_names) + +def test_native_check_changed_files(tmp_path): + """Verify fast native parallel cache validation.""" + from code2llm.analysis.native_bridge import native_check_changed_files + import hashlib + + # Create test files + f1 = tmp_path / "a.py" + f1.write_text("print(1)") + f2 = tmp_path / "b.py" + f2.write_text("print(2)") + + stat1 = f1.stat() + stat2 = f2.stat() + + h1 = hashlib.sha256(f"v0.1.0\0{f1}\0print(1)".encode()).hexdigest()[:16] + h2 = hashlib.sha256(f"v0.1.0\0{f2}\0print(2)".encode()).hexdigest()[:16] + + manifest = { + "a.py": (h1, stat1.st_mtime, stat1.st_size), + # b.py has drifted mtime, triggering L2 check where hash mismatches + "b.py": ("wronghash1234567", stat2.st_mtime + 10.0, stat2.st_size), + } + + res = native_check_changed_files( + project_dir=str(tmp_path), + filepaths=[str(f1), str(f2)], + manifest_entries=manifest, + analyzer_version="0.1.0", + ) + assert res is not None + changed, cached, refreshed = res + assert str(f1) in cached + assert str(f2) in changed + + +def test_native_format_toon_header(): + """Verify native TOON header formatting.""" + from code2llm.analysis.native_bridge import native_format_toon_header + + lines = native_format_toon_header( + nfiles=42, + total_lines=1337, + lang_label="Python", + timestamp="2026-10-02 23:00", + avg_cc=3.14, + critical_cc=5, + total_funcs=100, + dups=2, + cycles=1, + ) + assert lines is not None + assert len(lines) == 2 + assert "42f 1337L" in lines[0] + assert "critical:5/100" in lines[1] + +