feat(rust): parallel cache validation and TOON diagnostics rendering

code2llm/analysis/native_bridge.py

@@ -194,3 +194,51 @@ def native_walk_project_files(
             logger.debug("code2llm_rust.walk_project_files failed: %s", e)
     return None
 
+
+def native_check_changed_files(
+    project_dir: str,
+    filepaths: List[str],
+    manifest_entries: Dict[str, Tuple[str, float, int]],
+    analyzer_version: str,
+) -> Optional[Tuple[List[str], List[str], List[Tuple[str, float]]]]:
+    """Check changed files in parallel using Rayon."""
+    if _HAS_NATIVE_RUST and hasattr(code2llm_rust, "check_changed_files"):
+        try:
+            return code2llm_rust.check_changed_files(
+                project_dir, filepaths, manifest_entries, analyzer_version
+            )
+        except Exception as e:
+            logger.debug("code2llm_rust.check_changed_files failed: %s", e)
+    return None
+
+
+def native_format_toon_header(
+    nfiles: int,
+    total_lines: int,
+    lang_label: str,
+    timestamp: str,
+    avg_cc: float,
+    critical_cc: int,
+    total_funcs: int,
+    dups: int,
+    cycles: int,
+) -> Optional[List[str]]:
+    """Format header lines for analysis.toon.yaml."""
+    if _HAS_NATIVE_RUST and hasattr(code2llm_rust, "format_toon_header"):
+        try:
+            return code2llm_rust.format_toon_header(
+                nfiles,
+                total_lines,
+                lang_label,
+                timestamp,
+                avg_cc,
+                critical_cc,
+                total_funcs,
+                dups,
+                cycles,
+            )
+        except Exception as e:
+            logger.debug("code2llm_rust.format_toon_header failed: %s", e)
+    return None
+
+

code2llm/core/persistent_cache.py

@@ -32,6 +32,7 @@
 from typing import Any, Dict, List, Optional, Tuple
 
 from code2llm import __version__
+from code2llm.analysis.native_bridge import native_check_changed_files
 
 logger = logging.getLogger(__name__)
 
@@ -172,10 +173,30 @@ def get_changed_files(self, filepaths: List[str]) -> Tuple[List[str], List[str]]
 
         Uses L1 (mtime+size) first; falls back to L2 (content hash) when
         mtime changed but size matches — handles touch/copy without edit.
+        Uses native Rust parallel checker when available.
         """
+        files_index = self._manifest["files"]
+
+        # Fast path: native parallel check across threads
+        manifest_entries = {
+            rel: (info["hash"], float(info["mtime"]), int(info["size"]))
+            for rel, info in files_index.items()
+            if isinstance(info, dict) and "hash" in info and "mtime" in info and "size" in info
+        }
+        native_res = native_check_changed_files(
+            self._project_dir, filepaths, manifest_entries, self._analyzer_version
+        )
+        if native_res is not None:
+            changed_res, cached_res, refreshed = native_res
+            for fp, new_mtime in refreshed:
+                rel = os.path.relpath(fp, self._project_dir)
+                if rel in files_index:
+                    files_index[rel]["mtime"] = new_mtime
+                    self._dirty = True
+            return changed_res, cached_res
+
         changed: List[str] = []
         cached: List[str] = []
-        files_index = self._manifest["files"]
 
         for fp in filepaths:
             rel = os.path.relpath(fp, self._project_dir)

code2llm/exporters/toon/renderer.py

@@ -27,6 +27,7 @@
     render_external_section,
     render_pipelines_section,
 )
+from code2llm.analysis.native_bridge import native_format_toon_header
 
 
 class ToonRenderer:
@@ -44,6 +45,21 @@ def render_header(self, ctx: Dict[str, Any]) -> List[str]:
         ndups = len({d["class_name"] for d in ctx["duplicates"]})
         ncycles = len(ctx["cycles"])
         lang_label = _detect_language_label(result)
+
+        native_header = native_format_toon_header(
+            nfiles,
+            total_lines,
+            lang_label,
+            ctx["timestamp"],
+            avg_cc,
+            critical,
+            nfuncs,
+            ndups,
+            ncycles,
+        )
+        if native_header is not None:
+            return native_header
+
         return [
             f"# code2llm | {nfiles}f {total_lines}L | {lang_label} | {ctx['timestamp']}",
             f"# CC̅={avg_cc} | critical:{critical}/{nfuncs} | dups:{ndups} | cycles:{ncycles}",

docs/analysis/rust-optimization-breakdown.md

@@ -53,6 +53,14 @@ Through systematic profiling and code analysis of `code2llm` during multi-langua
 - **Problem**: Python `nx.simple_cycles` scales exponentially with cycle density and caps out at 1,000 nodes, while `vulture` dead-code analysis re-scans every source file from the filesystem.
 - **Native Rust Solution**: Linear-time Tarjan SCC ($O(V + E)$) in `cycles.rs` with no node limit, paired with in-memory BFS graph reachability in `reachability.rs` directly on the call graph without touching disk.
 
+### Bottleneck I: Cache Key Generation & Changed-File Checking
+- **Problem**: In repositories with 5,000+ files, checking `PersistentCache` sequentially in Python via `os.stat` and `hashlib.sha256` adds noticeable latency before analysis even starts.
+- **Native Rust Solution**: Parallel cache checking in `cache.rs` using Rayon to evaluate L1 (mtime + size) and compute multi-threaded SHA-256 for L2 verification with zero GIL overhead.
+
+### Bottleneck J: TOON Diagnostics Rendering
+- **Problem**: Generating and formatting formatted header rows and metrics summaries in Python across thousands of functions and modules causes heavy string/list allocations.
+- **Native Rust Solution**: Native string buffer formatter in `toon.rs` emitting pre-formatted diagnostic lines.
+
 ## 3. Architecture & Refactoring Strategy
 
 To prepare `code2llm` for clean separation and extraction into the `code2llm-rust` package, the following refactoring was implemented:
@@ -75,10 +83,14 @@ To prepare `code2llm` for clean separation and extraction into the `code2llm-rus
    - Integrated native candidate detection for God Functions and Data Clumps.
 7. **`code2llm.analysis.coupling`**:
    - Refactored module interaction and instability computation to delegate to `native_bridge.native_compute_module_coupling`.
+8. **`code2llm.core.persistent_cache`**:
+   - Refactored `get_changed_files` to delegate to `native_bridge.native_check_changed_files`.
+9. **`code2llm.exporters.toon.renderer`**:
+   - Refactored `render_header` to delegate to `native_bridge.native_format_toon_header`.
 
 ## 4. Published `code2llm-rust` Project
 
-- **Location**: `packages/code2llm-rust`
+- **Location**: `packages/code2llm-rust` (and standalone `https://github.com/semcod/code2llm-rust`)
 - **Build System**: Maturin (`pyproject.toml`) + Cargo (`Cargo.toml`)
 - **PyO3 Bindings**: CPython extension module `code2llm_rust`
 - **Engine Components**:
@@ -89,8 +101,12 @@ To prepare `code2llm` for clean separation and extraction into the `code2llm-rus
   - `src/reachability.rs`: Fast in-memory graph reachability.
   - `src/coupling.rs`: Vectorized module coupling and instability metrics.
   - `src/discovery.rs`: Parallel gitignore-aware file walker and module resolution.
+  - `src/cache.rs`: Multi-threaded file content hashing and cache validation.
+  - `src/toon.rs`: Fast TOON diagnostics formatting.
   - `src/smells.rs`: Anti-pattern candidate detection.
 - **Deliverables**:
   - Compiled release wheel: `packages/code2llm-rust/target/wheels/code2llm_rust-0.1.0-cp313-cp313-manylinux_2_34_x86_64.whl`
+  - Standalone GitHub repository: `https://github.com/semcod/code2llm-rust` with GitHub Actions CI.
   - Python optional dependency: `code2llm[native]` in `pyproject.toml`.
 
+

packages/code2llm-rust/Cargo.toml

@@ -17,4 +17,5 @@ pyo3 = { version = "0.23", features = ["extension-module"] }
 rayon = "1.10"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+sha2 = "0.11.0"
 walkdir = "2.5.0"