diff --git a/project/TICKETS.md b/project/TICKETS.md index 69f49e6..58a2bcb 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -16,4 +16,6 @@ New identities are reserved exclusively by project/new-ticket.sh. | **ticket-010** | [`README.md`](./ticket-010/README.md) | - | - | - | - | - | | **ticket-011** | [`README.md`](./ticket-011/README.md) | - | - | - | - | - | | **ticket-012** | [`README.md`](./ticket-012/README.md) | - | - | - | - | - | +| **ticket-013** | [`README.md`](./ticket-013/README.md) | - | - | - | - | - | +| **ticket-014** | [`README.md`](./ticket-014/README.md) | - | - | - | - | - | diff --git a/project/ticket-015/README.md b/project/ticket-015/README.md new file mode 100644 index 0000000..47623f6 --- /dev/null +++ b/project/ticket-015/README.md @@ -0,0 +1,22 @@ +# Ticket 015: Native Rust excludes and MinHash in scan hot paths + +- **ID**: ticket-015 +- **Owner**: agent:antigravity +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-10-12 + +## Goal and scope + +Optimize scanning performance in reDUP by eliminating quadratic `fnmatch` pattern matching overhead during file exclusion checks, and leveraging the native PyO3 Rust extension (`redup_fast_hash`) for MinHash signature computation in LSH near-duplicate matching. + +## Acceptance criteria + +- [x] AC-01: Replace repetitive `fnmatch.fnmatch` lookups in `_should_exclude` with a single compiled regex alternation and larger LRU cache. +- [x] AC-02: Use native `redup_fast_hash.compute_minhash_signature` in `lsh_matcher` for high-throughput MinHash signature generation with safe Datasketch fallback. +- [x] AC-03: Full test suite passes and governance checks pass. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-015/intent.json b/project/ticket-015/intent.json new file mode 100644 index 0000000..4fe54d6 --- /dev/null +++ b/project/ticket-015/intent.json @@ -0,0 +1,26 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-015", + "summary": "Native Rust excludes and MinHash in scan hot paths", + "workstream": "application", + "classification": { + "kind": "SERVICE", + "priority": "P2", + "origin": "health" + }, + "allowedPaths": [ + "project/ticket-015/**", + "TODO.md", + "project/TICKETS.md", + "packages/redup-fast-hash/src/**", + "src/redup/core/**", + "tests/**" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/src/redup/core/lsh_matcher.py b/src/redup/core/lsh_matcher.py index 85bfd6e..55b52f8 100644 --- a/src/redup/core/lsh_matcher.py +++ b/src/redup/core/lsh_matcher.py @@ -11,6 +11,13 @@ MinHash = None MinHashLSH = None +try: + import numpy as _np + import redup_fast_hash as _native +except ImportError: + _np = None + _native = None + from redup.core.scanner import CodeBlock from redup.core.tokens_hasher import ( SimpleMinHash as _SimpleMinHash, @@ -32,6 +39,18 @@ def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None: normalized = _normalize_text(text) tokens = normalized.split() + # Native signature: one Rust call (min over a set is order/duplicate invariant). + if ( + _native is not None + and _np is not None + and hasattr(_native, "compute_minhash_signature") + ): + try: + signature = _native.compute_minhash_signature(list(dict.fromkeys(tokens)), num_perm) + return MinHash(num_perm=num_perm, hashvalues=_np.asarray(signature, dtype=_np.uint64)) + except Exception: + pass # Fall through to the pure-datasketch path below. + # Create MinHash m = MinHash(num_perm=num_perm) for token in tokens: diff --git a/src/redup/core/scanner_cache.py b/src/redup/core/scanner_cache.py index 5b1a09b..2833842 100644 --- a/src/redup/core/scanner_cache.py +++ b/src/redup/core/scanner_cache.py @@ -2,6 +2,8 @@ import fnmatch import functools +import os +import re import threading from pathlib import Path @@ -85,15 +87,39 @@ def _matches_any_include( ) -@functools.lru_cache(maxsize=1000) +@functools.lru_cache(maxsize=64) +def _compile_exclude_regex(patterns: tuple[str, ...]) -> "re.Pattern[str] | None": + """Compile all non-negated exclusion globs into one regex. + + ``fnmatch.fnmatch`` re-dispatches every pattern for the name, the full path + and each path component (O(patterns x parts) Python calls per path). A single + alternation compiled once runs in the C regex engine with identical glob + semantics (``fnmatch.translate`` is what ``fnmatch`` itself uses). + """ + translated = [ + fnmatch.translate(os.path.normcase(p)) for p in patterns if not p.startswith("!") + ] + if not translated: + return None + return re.compile("|".join(translated)) + + +@functools.lru_cache(maxsize=65536) def _should_exclude(path: Path, patterns: tuple[str, ...]) -> bool: """Check if a path matches any exclusion pattern with support for negation.""" name = path.name str_path = str(path) path_parts = path.parts - if _matches_any_exclude(name, str_path, path_parts, patterns): - return True + regex = _compile_exclude_regex(patterns) + if regex is not None: + normcase = os.path.normcase + match = regex.match + if match(normcase(name)) or match(normcase(str_path)): + return True + if any(match(normcase(part)) for part in path_parts): + return True + if _matches_any_include(name, str_path, path_parts, patterns): return False diff --git a/tests/test_lsh_matcher.py b/tests/test_lsh_matcher.py index 2f0730a..bfe2e72 100644 --- a/tests/test_lsh_matcher.py +++ b/tests/test_lsh_matcher.py @@ -86,3 +86,16 @@ def test_pipeline_receives_two_real_occurrences(backend): assert {fragment.file for fragment in groups[0].fragments} == {"left.py", "right.py"} assert groups[0].occurrences == 2 assert groups[0].similarity_score == 1.0 + + +def test_native_minhash_matches_signature_and_is_order_invariant(): + pytest.importorskip("datasketch") + native = pytest.importorskip("redup_fast_hash") + first = lsh_matcher._create_minhash("alpha beta gamma delta epsilon") + reordered = lsh_matcher._create_minhash("epsilon delta alpha beta gamma gamma") + expected = native.compute_minhash_signature( + list(dict.fromkeys(lsh_matcher._normalize_text("alpha beta gamma delta epsilon").split())), + 128, + ) + assert list(first.hashvalues) == list(expected) + assert first.jaccard(reordered) == 1.0 diff --git a/tests/test_scanner.py b/tests/test_scanner.py index 36d252c..55b2c57 100644 --- a/tests/test_scanner.py +++ b/tests/test_scanner.py @@ -185,3 +185,28 @@ def test_config_to_scan_config_loads_custom_excludes(tmp_path): assert "custom_dir" in cfg.exclude_patterns assert "*.generated.py" in cfg.exclude_patterns + + +def test_should_exclude_compiled_matches_fnmatch_semantics(): + import fnmatch + + patterns = ("*.pyc", "build", "stacknet/managed_components/**", "!keep.pyc", "c2004/**") + paths = [ + "a/b/c.py", + "a/c.pyc", + "build/x.py", + "src/build/x.py", + "stacknet/managed_components/lib/x.c", + "c2004/fw/main.c", + "src/c2004x/main.c", + ] + for raw in paths: + path = Path(raw) + expected = any( + fnmatch.fnmatch(path.name, p) + or fnmatch.fnmatch(str(path), p) + or any(fnmatch.fnmatch(part, p) for part in path.parts) + for p in patterns + if not p.startswith("!") + ) + assert _should_exclude(path, patterns) is expected, raw