diff --git a/packages/redup-fast-hash/src/lib.rs b/packages/redup-fast-hash/src/lib.rs index 30ff2bd..de6ec3f 100644 --- a/packages/redup-fast-hash/src/lib.rs +++ b/packages/redup-fast-hash/src/lib.rs @@ -73,6 +73,23 @@ fn py_compute_minhash_signature(features: Vec, num_perm: usize) -> Vec, + items: Vec>, + num_perm: usize, +) -> Vec> { + use rayon::prelude::*; + py.allow_threads(|| { + items + .par_iter() + .map(|feats| lsh::compute_minhash_signature(feats, num_perm)) + .collect() + }) +} + /// Python wrapper for MinHash Jaccard similarity. #[pyfunction] #[pyo3(signature = (a, b))] @@ -89,6 +106,7 @@ fn redup_fast_hash(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(batch_simhash, m)?)?; m.add_function(wrap_pyfunction!(py_find_simhash_candidates, m)?)?; m.add_function(wrap_pyfunction!(py_compute_minhash_signature, m)?)?; + m.add_function(wrap_pyfunction!(batch_minhash_signature, m)?)?; m.add_function(wrap_pyfunction!(py_minhash_jaccard, m)?)?; // Add aliases matching standard function names diff --git a/project/TICKETS.md b/project/TICKETS.md index 58a2bcb..5f58e77 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -18,4 +18,5 @@ New identities are reserved exclusively by project/new-ticket.sh. | **ticket-012** | [`README.md`](./ticket-012/README.md) | - | - | - | - | - | | **ticket-013** | [`README.md`](./ticket-013/README.md) | - | - | - | - | - | | **ticket-014** | [`README.md`](./ticket-014/README.md) | - | - | - | - | - | +| **ticket-015** | [`README.md`](./ticket-015/README.md) | - | - | - | - | - | diff --git a/project/ticket-016/README.md b/project/ticket-016/README.md new file mode 100644 index 0000000..dd946a5 --- /dev/null +++ b/project/ticket-016/README.md @@ -0,0 +1,24 @@ +# Ticket 016: Native Rust MinHash scheme compatibility and batch Rayon signatures + +- **ID**: ticket-016 +- **Owner**: agent:antigravity +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-10-13 + +## Goal and scope + +Resolve Datasketch >=2.0 scheme compatibility for native Rust MinHash signature instantiation, introduce multi-threaded batch MinHash signatures via Rayon in `redup_fast_hash`, vectorize LSH block indexing in `LSHIndex.add_blocks`, and eliminate redundant pairwise similarity re-computations in `find_fuzzy_groups`. + +## Acceptance criteria + +- [x] AC-01: Support datasketch >=2.0 scheme specification (`scheme='legacy'`) and fallback assignment for native Rust MinHash signature instantiation. +- [x] AC-02: Implement `batch_minhash_signature` in `redup-fast-hash` with Rayon parallelization and GIL release. +- [x] AC-03: Implement batch indexing in `LSHIndex.add_blocks` for high-throughput parallel signature generation. +- [x] AC-04: Cache pairwise similarity in `find_fuzzy_groups` to eliminate duplicate `sequence_similarity` re-computation. +- [x] AC-05: Pass test suite and governance checks. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-016/intent.json b/project/ticket-016/intent.json new file mode 100644 index 0000000..43b0919 --- /dev/null +++ b/project/ticket-016/intent.json @@ -0,0 +1,26 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-016", + "summary": "Native Rust MinHash scheme compatibility and batch Rayon signatures", + "workstream": "application", + "classification": { + "kind": "SERVICE", + "priority": "P2", + "origin": "health" + }, + "allowedPaths": [ + "project/ticket-016/**", + "TODO.md", + "project/TICKETS.md", + "packages/**", + "src/**", + "tests/**" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/src/redup/core/lsh_matcher.py b/src/redup/core/lsh_matcher.py index 55b52f8..5a3137e 100644 --- a/src/redup/core/lsh_matcher.py +++ b/src/redup/core/lsh_matcher.py @@ -30,6 +30,22 @@ ) +def _make_minhash_from_signature(signature: Any, num_perm: int) -> MinHash | None: + """Instantiate a datasketch MinHash from raw permutation hashvalues safely.""" + if MinHash is None or _np is None: + return None + arr = _np.asarray(signature, dtype=_np.uint64) + try: + return MinHash(num_perm=num_perm, hashvalues=arr, scheme="legacy") + except (TypeError, ValueError): + try: + m = MinHash(num_perm=num_perm) + m.hashvalues = arr + return m + except Exception: + return None + + def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None: """Create MinHash from normalized text.""" if MinHash is None: @@ -47,7 +63,9 @@ def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None: ): try: signature = _native.compute_minhash_signature(list(dict.fromkeys(tokens)), num_perm) - return MinHash(num_perm=num_perm, hashvalues=_np.asarray(signature, dtype=_np.uint64)) + mh = _make_minhash_from_signature(signature, num_perm) + if mh is not None: + return mh except Exception: pass # Fall through to the pure-datasketch path below. @@ -87,6 +105,36 @@ def add(self, block: CodeBlock) -> None: minhash = _create_simple_minhash(block.text, self.num_perm) self.minhashes.append(minhash) + def add_blocks(self, blocks: list[CodeBlock]) -> None: + """Add multiple code blocks to the LSH index in a vectorized/parallel batch.""" + if not blocks: + return + + if ( + _native is not None + and _np is not None + and hasattr(_native, "batch_minhash_signature") + and MinHash is not None + ): + try: + feature_lists = [ + list(dict.fromkeys(_normalize_text(b.text).split())) + for b in blocks + ] + signatures = _native.batch_minhash_signature(feature_lists, self.num_perm) + for block, sig in zip(blocks, signatures): + self.blocks.append(block) + minhash = _make_minhash_from_signature(sig, self.num_perm) + if minhash and self.lsh: + self.lsh.insert(str(len(self.blocks) - 1), minhash) + self.minhashes.append(minhash) + return + except Exception: + pass + + for block in blocks: + self.add(block) + def find_near_duplicates(self, block: CodeBlock) -> list[tuple[CodeBlock, float]]: """Find near-duplicates of the given block.""" if MinHash is None or not self.minhashes: @@ -175,11 +223,8 @@ def build_lsh_index( ) -> LSHIndex: """Build LSH index from code blocks.""" index = LSHIndex(threshold=threshold) - - for block in blocks: - if block.line_count >= min_lines: - index.add(block) - + eligible = [b for b in blocks if b.line_count >= min_lines] + index.add_blocks(eligible) return index diff --git a/src/redup/core/pipeline/duplicate_finder.py b/src/redup/core/pipeline/duplicate_finder.py index 4f24050..9880325 100644 --- a/src/redup/core/pipeline/duplicate_finder.py +++ b/src/redup/core/pipeline/duplicate_finder.py @@ -108,6 +108,7 @@ def find_fuzzy_groups( continue cluster = [anchor] + cluster_sims: list[float] = [] for other_index in sorted(candidate_indices.get(index, ())): other = candidates[other_index] other_key = (other.file, other.line_start) @@ -123,6 +124,7 @@ def find_fuzzy_groups( if similarity < min_similarity: continue cluster.append(other) + cluster_sims.append(similarity) if len(cluster) < 2: continue @@ -138,9 +140,7 @@ def find_fuzzy_groups( ) for block in cluster ] - avg_similarity = sum( - sequence_similarity(cluster[0].text, block.text) for block in cluster[1:] - ) / (len(cluster) - 1) + avg_similarity = sum(cluster_sims) / len(cluster_sims) groups.append( DuplicateGroup( diff --git a/tests/test_lsh_matcher.py b/tests/test_lsh_matcher.py index bfe2e72..10515a4 100644 --- a/tests/test_lsh_matcher.py +++ b/tests/test_lsh_matcher.py @@ -99,3 +99,30 @@ def test_native_minhash_matches_signature_and_is_order_invariant(): ) assert list(first.hashvalues) == list(expected) assert first.jaccard(reordered) == 1.0 + + +def test_batch_minhash_signature(): + pytest.importorskip("datasketch") + native = pytest.importorskip("redup_fast_hash") + assert hasattr(native, "batch_minhash_signature") + items = [ + ["alpha", "beta"], + ["gamma", "delta"], + ] + sigs = native.batch_minhash_signature(items, 128) + assert len(sigs) == 2 + assert sigs[0] == native.compute_minhash_signature(items[0], 128) + assert sigs[1] == native.compute_minhash_signature(items[1], 128) + + +def test_index_add_blocks_batch(backend): + blocks = [ + block("b1.py", "alpha beta gamma delta epsilon"), + block("b2.py", "alpha beta gamma delta epsilon"), + block("b3.py", "one two three four five"), + ] + index = lsh_matcher.LSHIndex(threshold=0.8) + index.add_blocks(blocks) + assert len(index.blocks) == 3 + matches = index.find_near_duplicates(block("query.py", "alpha beta gamma delta epsilon")) + assert len(matches) == 2