Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 39 additions & 4 deletions code2llm/core/analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@
# Config is serialised to a plain dict before pickling (dataclasses are not picklable).
# Results from worker processes are plain dicts; _merge_results rebuilds dataclasses.

import hashlib
import logging
import multiprocessing as mp
import os
import time
from collections import Counter
from concurrent.futures import ProcessPoolExecutor, as_completed
from pathlib import Path
from typing import Dict, List, Optional, Tuple
Expand Down Expand Up @@ -172,7 +174,7 @@ def _load_from_persistent_cache(
cached_results: List[Dict] = []
for fp in cached_paths:
r = pcache.get_file_result(fp)
if r is not None:
if isinstance(r, dict) and getattr(r.get("module"), "name", None) == path_to_module[fp]:
cached_results.append(r)
else:
changed_paths.append(fp)
Expand Down Expand Up @@ -332,7 +334,7 @@ def _collect_files(self, project_path: Path) -> List[Tuple[str, str]]:
for f_path, mod_name in native_files
if self.file_filter.should_process(f_path)
]
return filtered
return self._source_module_names(filtered, project_path)

files = []
ext_set = set(ALL_EXTENSIONS)
Expand Down Expand Up @@ -363,7 +365,32 @@ def _collect_files(self, project_path: Path) -> List[Tuple[str, str]]:
module_name = self._compute_module_name(rel, filename, project_name)
files.append((file_str, module_name))

return files
return self._source_module_names(files, project_path)

@staticmethod
def _source_module_names(files, project_path):
"""Give colliding logical names distinct, deterministic static identities.

File/package aliases can coexist on disk. Keeping either under their
shared import name would silently overwrite symbols and may bind calls
to the wrong file. Unique names retain compatibility; ambiguous names
receive source-path identities before analysis, on both file walkers.
"""
counts = Counter(name for _, name in files)
reserved = set(counts)
selected = {}
for file_path, name in sorted(files):
if counts[name] == 1:
selected[file_path] = name
continue
relative = Path(file_path).relative_to(project_path).as_posix()
digest = hashlib.sha256(relative.encode("utf-8")).hexdigest()
candidate = name + ".__source_" + digest
while candidate in reserved:
candidate += "_"
reserved.add(candidate)
selected[file_path] = candidate
return [(path, selected[path]) for path, _ in files]

def _wrap_tqdm(self, iterator, total: int, desc: str = "Analyzing"):
"""Wrap iterator with tqdm when verbose=False and tqdm is available."""
Expand Down Expand Up @@ -433,8 +460,16 @@ def _analyze_sequential(self, files: List[Tuple[str, str]]) -> List[Dict]:
file_iterator = self._wrap_tqdm(list(enumerate(files, 1)), total=total)
for i, (file_path, module_name) in file_iterator:
try:
result = analyzer.analyze_file(file_path, module_name)
cached = self.cache.get_fast(file_path) if self.cache else None
stale_identity = cached is not None and (
not isinstance(cached, dict)
or getattr(cached.get("module"), "name", None) != module_name
)
selected = FileAnalyzer(self.config, None) if stale_identity else analyzer
result = selected.analyze_file(file_path, module_name)
if result:
if stale_identity:
self.cache.put_fast(file_path, result)
results.append(result)
except Exception as e:
if self.config.verbose:
Expand Down
6 changes: 6 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,9 @@ automation = {requires-python = ">=3.12"}

[dependency-groups]
automation = ["goal>=2.2.0"]

[tool.uv.sources]
code2graph = { path = "packages/code2graph", editable = true }
code2toon = { path = "packages/code2toon", editable = true }
code2flow = { path = "packages/code2flow", editable = true }
code2llm-rust = { path = "packages/code2llm-rust", editable = true }
69 changes: 69 additions & 0 deletions tests/test_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,3 +256,72 @@ def test_mermaid_export_sanitizes_unsafe_identifiers(self, tmp_path):

if __name__ == "__main__":
pytest.main([__file__, "-v"])


@pytest.mark.parametrize("native", [False, True])
def test_colliding_package_and_module_keep_every_source_identity(tmp_path, monkeypatch, native):
"""Logical import aliases must not discard different static source files."""
import copy
import hashlib

import code2llm.core.analyzer as implementation
reserved = "__source_" + hashlib.sha256(b"pkg/widget.py").hexdigest()
files = ["pkg/widget.py", "pkg/widget/__init__.py", "pkg/unique.py", "pkg/widget/" + reserved + ".py"]
content = "class Shared:\n def same(self):\n return 1\n"
for relative in files:
path = tmp_path / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
records = [(str(tmp_path / relative), implementation.ProjectAnalyzer._compute_module_name(
relative, Path(relative).name, tmp_path.name)) for relative in files]
monkeypatch.setattr(implementation, "native_walk_project_files", lambda *a, **kw: list(reversed(records)) if native else None)
config = copy.deepcopy(FAST_CONFIG)
config.no_cache = True
config.performance.enable_cache = False
config.performance.parallel_enabled = False
config.filters.min_function_lines = 1
analyzer = implementation.ProjectAnalyzer(config, tmp_path)
result = analyzer.analyze_project(str(tmp_path))
assert {m.file for m in result.modules.values()} == {str(tmp_path / f) for f in files}
assert len(result.modules) == len(files)
assert "pkg.widget." + reserved in result.modules
assert "pkg.unique" in result.modules
conflicting = [m.name for m in result.modules.values() if m.file.endswith(("widget.py", "widget/__init__.py"))]
assert len(set(conflicting)) == 2 and "pkg.widget" not in conflicting
assert {f.file for f in result.functions.values()} == {str(tmp_path / f) for f in files}
assert len(result.functions) == len(files)
assert len(result.classes) == len(files)
assert all(n.function in result.functions for n in result.nodes.values())
first = dict(analyzer._collect_files(tmp_path))
monkeypatch.setattr(implementation, "native_walk_project_files", lambda *a, **kw: records if native else None)
assert dict(analyzer._collect_files(tmp_path)) == first


def test_collision_identity_refreshes_warm_caches_when_sibling_changes(tmp_path, monkeypatch):
import copy

import code2llm.core.analyzer as implementation
monkeypatch.setattr(implementation, "native_walk_project_files", lambda *a, **kw: None)
config = copy.deepcopy(FAST_CONFIG)
config.performance.parallel_enabled = False
config.performance.enable_cache = True
config.performance.cache_dir = str(tmp_path / ".file-cache")
config.filters.min_function_lines = 1
source = tmp_path / "pkg/widget.py"
source.parent.mkdir()
source.write_text("def same():\n return 1\n")
analyzer = implementation.ProjectAnalyzer(config, tmp_path)
assert set(analyzer.analyze_project(str(tmp_path)).modules) == {"pkg.widget"}
sibling = tmp_path / "pkg/widget/__init__.py"
sibling.parent.mkdir()
sibling.write_text("def same():\n return 2\n")
for _ in range(2):
result = analyzer.analyze_project(str(tmp_path))
assert {m.file for m in result.modules.values()} == {str(source), str(sibling)}
assert len(result.functions) == 2
assert "pkg.widget" not in result.modules
assert {m.file: m.name for m in result.modules.values()} == dict(analyzer._collect_files(tmp_path))
sibling.unlink()
result = analyzer.analyze_project(str(tmp_path))
assert set(result.modules) == {"pkg.widget"}
assert len(result.functions) == 1
60 changes: 58 additions & 2 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading