perf(scan): compiled exclude regex and native Rust MinHash signatures (ticket-015)

project/TICKETS.md

@@ -16,4 +16,6 @@ New identities are reserved exclusively by project/new-ticket.sh.
 | **ticket-010** | [`README.md`](./ticket-010/README.md) | - | - | - | - | - |
 | **ticket-011** | [`README.md`](./ticket-011/README.md) | - | - | - | - | - |
 | **ticket-012** | [`README.md`](./ticket-012/README.md) | - | - | - | - | - |
+| **ticket-013** | [`README.md`](./ticket-013/README.md) | - | - | - | - | - |
+| **ticket-014** | [`README.md`](./ticket-014/README.md) | - | - | - | - | - |
 <!-- AUTO:TICKET_INDEX:END -->

project/ticket-015/README.md

@@ -0,0 +1,22 @@
+# Ticket 015: Native Rust excludes and MinHash in scan hot paths
+
+- **ID**: ticket-015
+- **Owner**: agent:antigravity
+- **Status**: IN_PROGRESS
+- **Workflow state**: EDIT
+- **Created**: 2026-10-12
+
+## Goal and scope
+
+Optimize scanning performance in reDUP by eliminating quadratic `fnmatch` pattern matching overhead during file exclusion checks, and leveraging the native PyO3 Rust extension (`redup_fast_hash`) for MinHash signature computation in LSH near-duplicate matching.
+
+## Acceptance criteria
+
+- [x] AC-01: Replace repetitive `fnmatch.fnmatch` lookups in `_should_exclude` with a single compiled regex alternation and larger LRU cache.
+- [x] AC-02: Use native `redup_fast_hash.compute_minhash_signature` in `lsh_matcher` for high-throughput MinHash signature generation with safe Datasketch fallback.
+- [x] AC-03: Full test suite passes and governance checks pass.
+
+## Tracking boundary
+
+This directory contains the minimal reviewed intent. Optional participant prose
+and raw command logs are not required delivery output.

project/ticket-015/intent.json

@@ -0,0 +1,26 @@
+{
+  "schema": "new-project.intent/v3",
+  "ticket": "ticket-015",
+  "summary": "Native Rust excludes and MinHash in scan hot paths",
+  "workstream": "application",
+  "classification": {
+    "kind": "SERVICE",
+    "priority": "P2",
+    "origin": "health"
+  },
+  "allowedPaths": [
+    "project/ticket-015/**",
+    "TODO.md",
+    "project/TICKETS.md",
+    "packages/redup-fast-hash/src/**",
+    "src/redup/core/**",
+    "tests/**"
+  ],
+  "forbiddenPaths": [
+    "project/ticket-*/user-*.md"
+  ],
+  "stacks": [],
+  "dependsOn": [],
+  "conflictsWith": [],
+  "integrationTicket": null
+}

src/redup/core/lsh_matcher.py

@@ -11,6 +11,13 @@
     MinHash = None
     MinHashLSH = None
 
+try:
+    import numpy as _np
+    import redup_fast_hash as _native
+except ImportError:
+    _np = None
+    _native = None
+
 from redup.core.scanner import CodeBlock
 from redup.core.tokens_hasher import (
     SimpleMinHash as _SimpleMinHash,
@@ -32,6 +39,18 @@ def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None:
     normalized = _normalize_text(text)
     tokens = normalized.split()
 
+    # Native signature: one Rust call (min over a set is order/duplicate invariant).
+    if (
+        _native is not None
+        and _np is not None
+        and hasattr(_native, "compute_minhash_signature")
+    ):
+        try:
+            signature = _native.compute_minhash_signature(list(dict.fromkeys(tokens)), num_perm)
+            return MinHash(num_perm=num_perm, hashvalues=_np.asarray(signature, dtype=_np.uint64))
+        except Exception:
+            pass  # Fall through to the pure-datasketch path below.
+
     # Create MinHash
     m = MinHash(num_perm=num_perm)
     for token in tokens:

src/redup/core/scanner_cache.py

@@ -2,6 +2,8 @@
 
 import fnmatch
 import functools
+import os
+import re
 import threading
 from pathlib import Path
 
@@ -85,15 +87,39 @@ def _matches_any_include(
     )
 
 
-@functools.lru_cache(maxsize=1000)
+@functools.lru_cache(maxsize=64)
+def _compile_exclude_regex(patterns: tuple[str, ...]) -> "re.Pattern[str] | None":
+    """Compile all non-negated exclusion globs into one regex.
+
+    ``fnmatch.fnmatch`` re-dispatches every pattern for the name, the full path
+    and each path component (O(patterns x parts) Python calls per path). A single
+    alternation compiled once runs in the C regex engine with identical glob
+    semantics (``fnmatch.translate`` is what ``fnmatch`` itself uses).
+    """
+    translated = [
+        fnmatch.translate(os.path.normcase(p)) for p in patterns if not p.startswith("!")
+    ]
+    if not translated:
+        return None
+    return re.compile("|".join(translated))
+
+
+@functools.lru_cache(maxsize=65536)
 def _should_exclude(path: Path, patterns: tuple[str, ...]) -> bool:
     """Check if a path matches any exclusion pattern with support for negation."""
     name = path.name
     str_path = str(path)
     path_parts = path.parts
 
-    if _matches_any_exclude(name, str_path, path_parts, patterns):
-        return True
+    regex = _compile_exclude_regex(patterns)
+    if regex is not None:
+        normcase = os.path.normcase
+        match = regex.match
+        if match(normcase(name)) or match(normcase(str_path)):
+            return True
+        if any(match(normcase(part)) for part in path_parts):
+            return True
+
 
     if _matches_any_include(name, str_path, path_parts, patterns):
         return False

tests/test_lsh_matcher.py

@@ -86,3 +86,16 @@ def test_pipeline_receives_two_real_occurrences(backend):
     assert {fragment.file for fragment in groups[0].fragments} == {"left.py", "right.py"}
     assert groups[0].occurrences == 2
     assert groups[0].similarity_score == 1.0
+
+
+def test_native_minhash_matches_signature_and_is_order_invariant():
+    pytest.importorskip("datasketch")
+    native = pytest.importorskip("redup_fast_hash")
+    first = lsh_matcher._create_minhash("alpha beta gamma delta epsilon")
+    reordered = lsh_matcher._create_minhash("epsilon delta alpha beta gamma gamma")
+    expected = native.compute_minhash_signature(
+        list(dict.fromkeys(lsh_matcher._normalize_text("alpha beta gamma delta epsilon").split())),
+        128,
+    )
+    assert list(first.hashvalues) == list(expected)
+    assert first.jaccard(reordered) == 1.0

tests/test_scanner.py

@@ -185,3 +185,28 @@ def test_config_to_scan_config_loads_custom_excludes(tmp_path):
     assert "custom_dir" in cfg.exclude_patterns
     assert "*.generated.py" in cfg.exclude_patterns
 
+
+
+def test_should_exclude_compiled_matches_fnmatch_semantics():
+    import fnmatch
+
+    patterns = ("*.pyc", "build", "stacknet/managed_components/**", "!keep.pyc", "c2004/**")
+    paths = [
+        "a/b/c.py",
+        "a/c.pyc",
+        "build/x.py",
+        "src/build/x.py",
+        "stacknet/managed_components/lib/x.c",
+        "c2004/fw/main.c",
+        "src/c2004x/main.c",
+    ]
+    for raw in paths:
+        path = Path(raw)
+        expected = any(
+            fnmatch.fnmatch(path.name, p)
+            or fnmatch.fnmatch(str(path), p)
+            or any(fnmatch.fnmatch(part, p) for part in path.parts)
+            for p in patterns
+            if not p.startswith("!")
+        )
+        assert _should_exclude(path, patterns) is expected, raw