diff --git a/differential/CRASH_FINDINGS.md b/differential/CRASH_FINDINGS.md new file mode 100644 index 00000000..fa1a56ba --- /dev/null +++ b/differential/CRASH_FINDINGS.md @@ -0,0 +1,381 @@ +# Crash-only fuzzing findings + +Failures `crash_fuzz` found in `pathmap` on 2026-09-16, at commit `7c61024` on +`fuzz-fixes-v2`. The op table is `differential/src/crash.rs`; see +`differential/src/bin/crash_fuzz.rs` for the runner. The sites below are as +found; see "Fixed on `fuzz-fixes-v3`" for where they stand now. + +Each finding is an input that makes the crate panic, fail a debug assertion, +abort, segfault or hang **through its public API**, with every documented +precondition met. Calls the crate documents as panicking, stubs, and failures +already known from the differential work are steered around unless +`--include-known` is passed; they are listed at the end. + +## Fixed on `fuzz-fixes-v3` + +No site outside "Known failures" reproduces on `fuzz-fixes-v3` (fixes on +`fuzz-fixes-v2` and `v3`, each with a test that fails without it). These +turned up only once the earlier ones were gone: + +- A `ZipperHead` reader cloned an ancestor node, so the next exclusive writer + copied that node and left live writers pointing into the old copy + (use-after-free). Readers now own a private root holding only their entry. + This was behind findings 3, 4 and 8 and the "Lock is missing" panic in + `zipper_tracking.rs`. +- `PrefixZipper::fork_read_zipper` always forked from the prefix start. +- `ProductZipper` sibling steps that fail at a factor root lost the factor. +- `TrieRef::is_shared` read the refcount of the empty sentinel. +- Joins of two empty child nodes under one key: a dense-node debug assertion + and a list node left with two onward children. + +After the fixes, with the debug-assertion build and known failures steered +around: 0 failures in 4M inputs (seed 36) and in 8M inputs with `--maxlen +2000` (seed 39). With `--include-known`, only the known failures remain. + +## How the surveys were run + +| Survey | Build | Inputs | Seed | Time | Failures | Sites | +|---|---|---|---|---|---|---| +| A | debug assertions + overflow checks | 50,000 | 12 | 222 s | 7,165 | 27 | +| B | release | 50,000 | 11 | 134 s | 2,043 | 17 | +| C | release, `--features pathmap/all_dense_nodes` | 10,000 | 13 | 62 s | 845 | 11 | + +```sh +# A +CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=true CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS=true \ + CARGO_TARGET_DIR=target/dbgassert cargo build --release -p differential --bin crash_fuzz +target/dbgassert/release/crash_fuzz --random 50000 --seed 12 -j 56 --timeout 8 \ + --keep-going --max-failures 20000 --save runs/crash-a +# B +cargo build --release -p differential --bin crash_fuzz +target/release/crash_fuzz --random 50000 --seed 11 -j 56 --timeout 5 --keep-going --save runs/crash-b +# C +CARGO_TARGET_DIR=target/alldense cargo build --release -p differential --bin crash_fuzz \ + --features pathmap/all_dense_nodes +target/alldense/release/crash_fuzz --random 10000 --seed 13 -j 56 --timeout 5 --keep-going +``` + +Survey B's release build was taken before the last `KNOWN_FAILURES` entry +(stand-alone `to_next_k_path`) existed. The site that entry covers is listed +under "Known failures" below. + +**Reproducing a fuzz input.** Inputs are deterministic in (seed, index), so +"A #3620" means: + +```sh +target/dbgassert/release/crash_fuzz --random 50000 --seed 12 --dump 3620 > in.bin +CRASH_TRACE=1 CRASH_BACKTRACE=1 target/dbgassert/release/crash_fuzz in.bin +``` + +`CRASH_TRACE=1` prints every op as it starts, so the last lines before the +failure name the call. Indices are only valid for the op table at `7c61024`. +Replay a survey A input with the debug build and a B input with the release +build; the other build may fail at a different site or not at all. + +**Minimal reproducers** for eight of the findings are compiled in +`differential/src/bin/crash_repros.rs`: + +```sh +cargo run -p differential --bin crash_repros -- --list +cargo run -p differential --bin crash_repros -- +``` + +## Summary + +Counts are hits in survey A (debug) / survey B (release); `-` means none in +that survey. Hangs are counted separately at the end. + +| # | Site | A | B | Reached through | Repro | +|---|---|---|---|---|---| +| 1 | `zipper.rs:3616` unreachable_unchecked / SIGSEGV | 33 | 23 | `ZipperHeadOwned::write_zipper_at_exclusive_path` | `owned_head_second_exclusive_path` | +| 2 | `trie_node.rs:1021` unreachable_unchecked / SIGILL | 7 | 4 | `PathMap::join`, `WriteZipper::join_map_into` | A #3620, B #6905 | +| 3 | `malloc(): unaligned tcache chunk detected` (SIGABRT) | 1 | - | `ZipperHead`, then a read zipper | A #8814 | +| 4 | `trie_node.rs:3213` misaligned pointer dereference | 1 (earlier run) | - | `ZipperHead::write_zipper_at_exclusive_path` | - | +| 5 | `write_zipper.rs:2869` slice out of range | 176 | 128 | `ZipperHeadOwned::write_zipper_at_exclusive_path` | A #464 | +| 6 | `zipper_head.rs:347` unwrap on `None` | 24 | 27 | `write_zipper_at_exclusive_path` on a head from `WriteZipper::zipper_head` or `into_zipper_head` | `write_zipper_head_second_exclusive_path` | +| 7 | `write_zipper.rs:1337` assertion `origin_path` not a slice | 1,593 | - | `WriteZipper::zipper_head` on a zipper made by `write_zipper_at_path` | `write_zipper_head_second_exclusive_path` (debug) | +| 8 | `trie_node.rs:3209` make_unique on an empty sentinel | 1 | 16 | `ZipperHead::write_zipper_at_exclusive_path` | A #36325 | +| 9 | `write_zipper.rs:1284` assertion `at_root` | 4 | - | `ZipperHead::write_zipper_at_exclusive_path` | A #69 | +| 10 | `write_zipper.rs:1435` unwrap on `None` | - | 2 | `set_val` on a zipper from a nested `ZipperHead` | B #8504 | +| 11 | `write_zipper.rs:1453` unwrap on `None` | - | 7 | `remove_val` (via `deserialize_paths`) on a zipper from a nested `ZipperHead` | B #5364 | +| 12 | `zipper.rs:1871` explicit panic | 617 | 341 | `get_trie_ref` on a read zipper from a `ZipperHead` | `head_read_zipper_get_trie_ref` | +| 13 | `zipper.rs:2644` unwrap on `None` | 126 | 57 | `ProductZipper::is_shared` | `product_zipper_is_shared` | +| 14 | `zipper.rs:3048` unwrap on `None` | 3 | 3 | `ProductZipper::val_count` | `product_zipper_val_count` | +| 15 | `product_zipper.rs:178` assertion `focus_factor() == factor_count() - 1` | 77 | - | `ProductZipper::val_count` | `product_zipper_val_count` (debug) | +| 16 | `product_zipper.rs:502` assertion "must ascend" | 1,148 | - | `ProductZipperG` sibling moves, `to_next_val`, k-path walks | A #31, A #8704 | +| 17 | `dependent_zipper.rs:166` assertion "must ascend" | 1,163 | - | `DependentProductZipperG` sibling moves, `to_next_val`, k-path walks | A #16 | +| 18 | `overlay_zipper.rs:163` assertion (`focus_byte` of the two sides) | 884 | - | `OverlayZipper::to_next_step`, `to_next_step_observed` | A #17 | +| 19 | `overlay_zipper.rs:152` assertion (`depth` of the two sides) | 2 | - | `OverlayZipper` | A #9317 | +| 20 | `overlay_zipper.rs:330` assertion (`path` of the two sides) | 1 | - | `OverlayZipper::ascend_until` | A #21316 | +| 21 | `zipper.rs:557` subtract with overflow | 12 | - | `OverlayZipper::to_next_step_observed` | A #269 | +| 22 | `prefix_zipper.rs:448` assertion `path_exists()` | 64 | - | `PrefixZipper::descend_indexed_byte`, `descend_last_byte`, `to_next_step_observed` | A #400 | +| 23 | `prefix_zipper.rs:355` index out of bounds | 1 | 3 | `PrefixZipper::to_next_step`, `descend_indexed_byte` | A #46237, B #3769 | +| 24 | `write_zipper.rs:2557` / `2560` slice out of range | 32 | 20 | `graft_child_maps` under a root path of 48+ bytes | `graft_child_maps_long_root` | +| 25 | `zipper.rs:3328` assertion / `zipper.rs:3332` / `3330` slice out of range | 12 | 10 | `get_val_with_witness` on a `ReadZipperOwned` | `owned_read_zipper_witness` | +| 26 | `write_zipper.rs:1304` assertion `at_root` | 1 | - | `join_k_path_into` | A #22746 | +| 27 | `line_list_node.rs:1818` unwrap on `None` | 2 | - | `PathMap::merkleize` | A #4250, A #45491 | +| 28 | `trie_ref.rs:171` subtract with overflow | 2 | - | `trie_ref_at_path`, `get_focus_at` | A #17556, A #49829 | +| 29 | `write_zipper.rs:1132` slice out of range | - | - | `WriteZipperUntracked::path`, only with `all_dense_nodes` (survey C #3135) | C #3135 | + +## Undefined behaviour from safe code + +### 1. `ZipperHeadOwned` rooted below the map root, second exclusive path + +```rust +let zh = sample().into_zipper_head(&[1u8]); +drop(zh.write_zipper_at_exclusive_path(&[])); +let _ = zh.write_zipper_at_exclusive_path(&[9u8]); +``` + +Release: SIGSEGV. Debug: `unsafe precondition(s) violated: +hint::unreachable_unchecked must never be reached` at `zipper.rs:3616`. The +first exclusive zipper at the head's own root, once dropped, leaves the head in +a state the next request walks off. Sites 5 and 6 are panics from the same +function (`prepare_exclusive_write_path`), reached from differing states; +`write_zipper.rs:2869` is `KeyFields::root_prefix_path` slicing past the end of +the prefix buffer. (`sample()` is the map `{[0], [1,2,1], [1,2,1,0], +[1,2,1,3,3], [2,2]}`, all values 7.) + +### 2. `as_dense_unchecked` on a node that is not dense + +Reached from `PathMap::join` and from `WriteZipper::join_map_into` (A #3620, +A #17025, B #6905). `TaggedNodeRef::as_dense_unchecked` hits +`unreachable_unchecked` (`trie_node.rs:1021`); a release build executes it as +an illegal instruction (SIGILL). In A #3620 the destination zipper is at a +path below a map built by earlier writes; no minimal repro yet. The most likely +candidate is a node type the join dispatch assumes cannot occur there (e.g. a +`CellByteNode`, which `ZipperHead` creates), but that is not confirmed. + +### 3. Heap corruption after `ZipperHead` writes + +A #8814: a `ZipperHead` over a map, an exclusive write zipper doing +`join_k_path_into` and `subtract_into`, then a read zipper from the same head +calling `descend_last_path`. The process aborts with `malloc(): unaligned +tcache chunk detected`. Reproduces alone, in both builds (release: SIGABRT with the same message). + +### 4. Misaligned pointer dereference + +`trie_node.rs:3213`: `misaligned pointer dereference: address must be a +multiple of 0x4 but is 0xff19cac196ee4b9`, one hit in an earlier debug survey +(seed 11, #8513) against an older revision of the op table, in the same code +as site 8 (`make_unique`), from `ZipperHead::write_zipper_at_exclusive_path`. +That index no longer reproduces at `7c61024`, and survey A did not hit it. + +## Panics with a minimal reproducer + +### 6, 7. `zipper_head()` on a write zipper made with a borrowed path + +```rust +let mut map = sample(); +let mut wz = map.write_zipper_at_path(&[1u8]); +let zh = wz.zipper_head(); +drop(zh.write_zipper_at_exclusive_path(&[])); +let _ = zh.write_zipper_at_exclusive_path(&[]); +``` + +Debug fails at once, in `zipper_head()`: +`!self.key.origin_path.is_slice() || self.key.origin_path.len() == 0` +(`write_zipper.rs:1337`, `as_static_path_zipper`). `write_zipper_at_path` takes +the path by reference, so any non-empty root trips it. Release carries on +and panics on the second exclusive path at `zipper_head.rs:347` (`root_val` +unwrap in `prepare_exclusive_write_path`). Site 7 is the most frequent +failure in survey A. + +### 12. `get_trie_ref` on a read zipper from a `ZipperHead` + +```rust +let mut map = sample(); +let zh = map.zipper_head(); +let rz = zh.read_zipper_at_path(&[1u8]).unwrap(); +let _ = rz.get_trie_ref(); +``` + +`focus_parent_borrowed` calls `OwnedOrBorrowed::as_borrowed_ref`, which panics +on the owned root node a head's read zipper holds (`zipper.rs:1871`). The same +path is reached from `get_focus_at` on those zippers. Most frequent panic in +survey B. + +### 13. `ProductZipper::is_shared` + +```rust +let (a, b) = (sample(), sample()); +let mut z = ProductZipper::new(a.read_zipper_at_path(&[1u8, 2, 1]), [b.read_zipper()]); +while z.to_next_step() { + let _ = z.is_shared(); +} +``` + +Once the focus is in the second factor, `ReadZipperCore::is_shared` looks the +focus up in the parent node of the *primary* trie and unwraps `None` +(`zipper.rs:2644`). + +### 14, 15. `ProductZipper::val_count` outside the last factor + +```rust +let mut a = PathMap::::new(); +a.set_val_at([1u8, 2], 1); +let mut b = PathMap::::new(); +b.set_val_at([3u8, 4], 1); +let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); +loop { + let _ = z.val_count(); + if !z.to_next_step() { break } +} +``` + +Debug: `focus_factor() == factor_count() - 1` (`product_zipper.rs:178`), i.e. +`val_count` is only implemented for the last factor. Release: +`get_focus_at` unwraps `None` (`zipper.rs:3048`). + +### 24. `graft_child_maps` under a long root path + +```rust +let mut map = PathMap::::new(); +let mut wz = map.write_zipper_at_path(&[0u8; 48]); +wz.graft_child_maps(ByteMask::from_iter([1u8]), [PathMap::single([2u8], 5)], false); +``` + +`range end index 49 out of range for slice of length 48` +(`write_zipper.rs:2560`; with a longer root, `2557`). Roots of 47 bytes or +fewer work. A fixed 48-byte buffer is being indexed by the full path length. + +### 25. `get_val_with_witness` on an owned read zipper + +```rust +let mut map = PathMap::::new(); +map.set_val_at([1u8], 1); +let z = map.into_read_zipper(&[]); +let w = z.witness(); +let _ = z.get_val_with_witness(&w); +``` + +Debug: `root_parent_key_start < usize::MAX` (`zipper.rs:3328`). Release: +`range start index 18446744073709551615 out of range` (`zipper.rs:3332`, and +`3330` in another state). The root path here is empty; a 3-byte root path +returns `None` without panicking. + +### 23 (and a k = 0 case). `PrefixZipper` + +```rust +let mut map = PathMap::::new(); +map.set_val_at([0x22u8], 1); +let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); +z.descend_to_byte(2); +z.descend_last_path(); +z.descend_last_path(); +z.descend_first_k_path(0); +``` + +`prefix_zipper.rs:571`, slice out of range: `descend_first_k_path` computes the +untaken part of the prefix as if the focus were still inside it, although the +zipper has moved into the source. +`k = 0` is in `KNOWN_FAILURES` (degenerate) and is skipped by default, but a +slice panic is not a degenerate answer. Related, without minimal repros: +site 22, `descend_indexed_byte` / `descend_last_byte` landing on a path that +does not exist (`prefix_zipper.rs:448`, A #400); site 23, index out of bounds in +`to_next_step` / `descend_indexed_byte` (`prefix_zipper.rs:355`, B #3769: "the +len is 0 but the index is 0"). + +## Findings with fuzz inputs only + +### 16, 17. `ProductZipperG` and `DependentProductZipperG`: "must ascend" + +`to_sibling_byte` gets `Some` from `focus_byte()` and then `ascend(1)` returns +0 (`product_zipper.rs:502`, `dependent_zipper.rs:166`). Reached from +`to_next_sibling_byte`, `to_prev_sibling_byte`, `to_next_step`, `to_next_val` +and k-path walks (A #31, #8704; A #16). In release the assertion is gone, and +these zippers hang instead: most hangs in survey B are in these two types. + +### 18–21. `OverlayZipper`: the two sides drift apart + +The overlay moves both source zippers in step and asserts they agree. They +stop agreeing on `focus_byte` (site 18, A #17), `depth` (19, A #9317) and +`path` (20, A #21316: `[0, 0, 2, 0, 1]` against `[2, 0, 1]`). From there, +`to_next_step_observed` reports more ascent to the observer than it descended +(21, A #269: `Vec` observer underflow at `zipper.rs:557`, with the observer +started from the zipper's current path), and in release `to_next_step` hangs. +Not diagnosed; a move that succeeds on one source and not the other is the +obvious suspect. + +### 8, 9. `ZipperHead` exclusive paths + +- Site 8, `Attempted to make_unique on an empty sentinel node` + (`trie_node.rs:3209`, from `make_cell_node` in `prepare_exclusive_write_path`): + A #36325, B #1730. Seen after an exclusive zipper did `take_map`, + `graft_child_maps` or `set_val` at a path and was dropped. A direct attempt + (`take_map` then a new exclusive zipper at the same path) did not reproduce. +- Site 9, assertion `self.at_root()` (`write_zipper.rs:1284`): A #69. + +### 10, 11. Writes through a zipper from a nested `ZipperHead` + +A head made by `WriteZipper::zipper_head()` on a write zipper at a long root, +then `set_val` (`write_zipper.rs:1435`, B #8504) or `remove_val` inside +`deserialize_paths` (`write_zipper.rs:1453`, B #5364) on a zipper it hands out. +Both unwrap `None`. Release only: in a debug build these inputs stop earlier +at site 7. + +### 26. `join_k_path_into`: assertion `at_root` + +`write_zipper.rs:1304`, A #22746. + +### 27. `PathMap::merkleize` + +`line_list_node.rs:1818`, `node_replace_child` unwraps `None` from +`get_child_mut`: A #4250 (`PathMap`), A #45491 (`PathMap`). +The list node cannot find the child `merkleize` asks it to replace. + +### 28. `trie_ref_at_path` / `get_focus_at`: subtract with overflow + +`trie_ref.rs:171`, A #17556, A #49829. Release wraps silently. + +### 29. `all_dense_nodes`: `WriteZipperUntracked::path` + +`write_zipper.rs:1132`, `range start index 3 out of range for slice of length +2`, survey C #3135 only. The default build passes this input. + +## Hangs + +| Survey | Hangs | Where (last op before the timeout) | +|---|---|---| +| A (debug) | 1,178 | as below; many B hangs fail an assertion (16–21) in A instead | +| B (release) | 1,392 | classified on a 3,000-input sample: `ProductZipperG` / `DependentProductZipperG` sibling moves and k-path walks; `OverlayZipper::to_next_step`; `ProductZipper` k-path walks (k ≥ 1) | +| C (all_dense) | 581 | same kinds | + +Hangs are infinite rather than slow: ten sampled inputs were all still running +after 40 s single-threaded, and one after 5 minutes, where a `to_next_step` or a `k ≤ 5` k-path step +should take microseconds. The simple cases (two small maps, a whole +`to_next_step` or k-path walk over a product or overlay zipper) do not hang; +the hanging inputs move the zipper first. Shrunk inputs are in +`.fuzzcorpus/crash/shrunk-h*.min.bin` locally; they were not minimised to Rust. + +## Not reproduced + +- A #30871: SIGABRT in the survey, runs clean alone (3 tries). Possibly + caused by another thread's failure in the same process. + +## Known failures (steered around by default) + +These are `KNOWN_FAILURES` in `differential/src/crash.rs`. `--include-known` +runs them. + +- `val_count` is `todo!()` on `OverlayZipper` (`overlay_zipper.rs:173`) and + `unimplemented!()` on `ProductZipperG` (`product_zipper.rs:699`) and + `DependentProductZipperG` (`dependent_zipper.rs:360`). +- `TrieRef::fork_read_zipper` at a missing path unwraps `None` + (`trie_ref.rs:322`; commented upstream as issue #96). +- `k == 0` for k-path iteration, `join_k_path_into` and `drop_head`: spins on + several zipper kinds, trips `debug_assert!(byte_cnt > 0)` in + `LineListNode::drop_head_dyn`, and panics in `PrefixZipper` (above). +- `to_next_k_path` without a preceding `descend_first_k_path`: underflows + `path_len` (`zipper.rs:2841`, overflow check) and reaches + `unreachable!()` in `EmptyNode` from `k_path_internal` on a fork of a write + zipper (`empty_node.rs:70`, survey B, 9 hits). +- `meet_k_path_into` spins when the focus has no children, and escapes it + when `k == 0`. + +## Not covered by the crash table + +`viz`, `old_cursor`, `bridge_nodes`, ACT file operations +(`dump_from_zipper`, `open_mmap`, `merge_zipper_into_file`), a `ZipperHead` +shared between threads, custom allocators, `PolyZipper`, `SplitCata` and +`counters`. diff --git a/differential/Cargo.toml b/differential/Cargo.toml index 4218b74a..23fefb0e 100644 --- a/differential/Cargo.toml +++ b/differential/Cargo.toml @@ -7,3 +7,26 @@ description = "The Rust side of the differential fuzzing harness for pathmap's z [dependencies] pathmap = { path = "..", features = ["arena_compact"] } +# `crash_fuzz` attributes segfaults and aborts to an input from a signal handler. +libc = "0.2" +# Only for `bin/afl_differential.rs`, behind the `afl` feature, so a plain +# `cargo build -p differential` neither needs it nor builds it. See that file. +afl = { version = "0.18", optional = true } + +[features] +afl = ["dep:afl"] + +# Coverage-guided front end for the in-process comparator. Declared explicitly +# rather than left to src/bin autodiscovery so it can carry `required-features`: +# without it, `cargo build -p differential` would try to compile a file that +# needs the `afl` crate. Build it with `cargo afl build`, not `cargo build` -- +# the instrumentation is a rustc flag `cargo afl` sets. +[[bin]] +name = "afl_differential" +path = "src/bin/afl_differential.rs" +required-features = ["afl"] + +[[bin]] +name = "afl_crash" +path = "src/bin/afl_crash.rs" +required-features = ["afl"] diff --git a/differential/crash_shrink.py b/differential/crash_shrink.py new file mode 100755 index 00000000..f4e7b7d8 --- /dev/null +++ b/differential/crash_shrink.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Shrink an input that makes `crash_fuzz` fail, keeping the failure the same. + + differential/crash_shrink.py [-o out.bin] [--timeout SECS] [--bin PATH] + +The failure is identified by its kind and, for a panic, its site (file:line and +the message with numbers blanked), as `crash_fuzz --keep-going` groups them. A +hang is identified by the kind alone and tested with a short timeout, so shrink +hangs with a timeout comfortably above how long the input takes to get stuck. + +Greedy: delete chunks of halving size from the end towards the start, then try +lowering each byte, keeping any change that preserves the failure. +""" +import argparse, os, re, subprocess, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def signature(binary, blob, timeout, tmp): + with open(tmp, "wb") as f: + f.write(blob) + try: + p = subprocess.run([binary, tmp, "-j", "1", "--timeout", str(timeout)], + capture_output=True, timeout=timeout + 20) + except subprocess.TimeoutExpired: + return "hang" + err = p.stderr.decode(errors="replace") + m = re.search(r"CRASH .*?kind=(\w+).*?(?:msg=(.*))?$", err, re.M) + if not m: + return None if p.returncode in (0,) else "exit %d" % p.returncode + kind, msg = m.group(1), m.group(2) or "" + if kind == "panic": + site = re.sub(r"\b\d+\b", "N", msg.split(" | ")[0]) + site = re.sub(r"(src/[\w/]+\.rs):N:N", lambda s: s.group(0), site) + loc = re.search(r"panicked at (\S+?):(\d+):(\d+)", msg) + return "panic %s:%s %s" % (loc.group(1), loc.group(2), site.split(": ", 1)[-1]) if loc else "panic " + site + if kind == "signal": + return "signal " + msg.split(" ")[1] if msg else "signal" + return kind + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("input") + ap.add_argument("-o", "--out") + ap.add_argument("--timeout", type=int, default=3) + ap.add_argument("--bin", default=os.path.join(ROOT, "target", "release", "crash_fuzz")) + a = ap.parse_args() + blob = open(a.input, "rb").read() + tmp = os.path.join(tempfile.gettempdir(), "crash-shrink-%d.bin" % os.getpid()) + want = signature(a.bin, blob, a.timeout, tmp) + if want is None: + sys.exit("input does not fail") + print("signature:", want, file=sys.stderr) + + def ok(b): + return signature(a.bin, b, a.timeout, tmp) == want + + chunk = max(1, len(blob) // 2) + while chunk >= 1: + i = len(blob) - chunk + changed = False + while i >= 0: + cand = blob[:i] + blob[i + chunk:] + if ok(cand): + blob = cand + changed = True + i -= chunk + print("chunk %d -> %d bytes" % (chunk, len(blob)), file=sys.stderr) + if not changed: + chunk //= 2 + for i in range(len(blob)): + for v in (0, 1, blob[i] // 2): + if v < blob[i]: + cand = blob[:i] + bytes([v]) + blob[i + 1:] + if ok(cand): + blob = cand + break + out = a.out or re.sub(r"(\.bin)?$", ".min.bin", a.input, count=1) + open(out, "wb").write(blob) + try: + os.unlink(tmp) + except OSError: + pass + print("%s: %d bytes, %s" % (out, len(blob), want)) + + +if __name__ == "__main__": + main() diff --git a/differential/src/bin/afl_crash.rs b/differential/src/bin/afl_crash.rs new file mode 100644 index 00000000..7699fa1a --- /dev/null +++ b/differential/src/bin/afl_crash.rs @@ -0,0 +1,26 @@ +//! Coverage-guided front end for the crash-only table in [`differential::crash`]. +//! +//! ```text +//! cargo afl build --release -p differential --features afl --bin afl_crash +//! mkdir -p out/afl-crash-in && target/release/crash_fuzz --random 64 --dump 0 > out/afl-crash-in/0 +//! cargo afl fuzz -i out/afl-crash-in -o out/afl-crash-out -t 5000 target/release/afl_crash +//! ``` +//! +//! `cargo afl build` turns on debug assertions and overflow checks, so replay +//! findings with a `crash_fuzz` built the same way (see its docs); a plain +//! release `crash_fuzz` runs most of them clean. +//! +//! AFL runs each input in a forked child, so a panic, an abort and a hang each +//! cost one child and are saved (`crashes/`, `hangs/`). Replay them with +//! `crash_fuzz `, which names the panic site, and group them with +//! `crash_fuzz --keep-going `. As with `afl_differential`, check +//! `hangs/` as well as `crashes/`: with a piped `core_pattern`, AFL can file +//! a crash as a hang. + +use differential::crash::run; + +fn main() { + afl::fuzz!(|data: &[u8]| { + run(data); + }); +} diff --git a/differential/src/bin/afl_differential.rs b/differential/src/bin/afl_differential.rs new file mode 100644 index 00000000..33561a3b --- /dev/null +++ b/differential/src/bin/afl_differential.rs @@ -0,0 +1,110 @@ +//! Coverage-guided differential fuzzing: the Rust reference model against the +//! real crate, driven by AFL++ instead of by a random byte generator. +//! +//! ```text +//! cargo install cargo-afl # once +//! cargo afl build --release -p differential --features afl +//! ./lean/afl-seed.sh out/afl-in # or any corpus +//! cargo afl fuzz -i out/afl-in -o out/afl-out \ +//! target/release/afl_differential +//! ``` +//! +//! # Why a second front end at all +//! +//! `in_process` generates uniformly random bytes. That is the right shape for +//! *measuring* — a rate over a known distribution, comparable with +//! `differential.py`'s — and the wrong shape for *finding*: the divergence +//! classes it turns up sit between 1 in 30,000 and 1 in 2,250,000 inputs, which +//! is what a blind sampler costs when the interesting programs are a thin set. +//! AFL keeps the inputs that reached new edges and mutates those, so a 40-op +//! program that got a write zipper into an unusual node representation becomes +//! the stem for the next thousand. Same comparison, same two op tables, better +//! search. +//! +//! The wire format suits it: every operand is a byte, reduced mod a small number +//! at the point of use, so AFL's byte flips, arithmetic and splices all land on +//! op selectors and path bytes rather than being rejected by a parser. There is +//! no checksum, no length prefix over the whole input, and a truncated input is +//! a valid shorter program. +//! +//! # Why this is safe where `in_process` has to be careful +//! +//! `in_process` runs every input in one long-lived process, so a panic out of +//! the middle of a trie mutation cannot be caught and recovered from — +//! `pathmap` is not unwind-safe, and dropping half-updated refcounted nodes +//! while the stack unwinds corrupts the heap. It therefore reports from a panic +//! *hook* and exits. +//! +//! AFL removes the problem rather than working around it. Each input runs in a +//! child forked from the fork server, and `afl::fuzz!` installs a hook that +//! **aborts** rather than unwinds. So a panicking input kills one child, is +//! written to `out/afl-out/default/crashes/`, and fuzzing continues from the +//! next one. Corruption cannot outlive the input that caused it. That is the +//! same property the subprocess design in `differential.py` bought, at a +//! fraction of the cost, and it is why this file does not reproduce +//! `in_process`'s panic-hook dance. +//! +//! A *divergence* is reported the same way a panic is — by panicking — so AFL +//! saves the input. Replay one with the plain comparator, which prints the +//! differing line rather than a backtrace: +//! +//! ```text +//! target/release/in_process out/afl-out/default/crashes/id:* out/afl-out/default/hangs/id:* +//! ./lean/differential.py out/afl-out/default/crashes/* # KNOWN-table breakdown +//! ``` +//! +//! # Look in `hangs/` as well as `crashes/` +//! +//! Which of the two a divergence lands in is a property of the *machine*, not of +//! the finding. AFL decides "crashed" by reaping the child and reading its +//! signal, and when `/proc/sys/kernel/core_pattern` is a pipe — apport, systemd +//! -coredump, any distro default — the kernel hands the corpse to that helper +//! first, so AFL's wait races the helper and times out instead. It says so at +//! startup ("To avoid having crashes misinterpreted as timeouts...") and +//! `AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1` only silences the refusal to start. +//! +//! Measured here: a 180s run saved **0 crashes and 18 hangs**, and all 18 hangs +//! replay through `in_process` as real divergences. So always sweep both +//! directories; a genuine timeout (an infinite loop in the crate, itself a +//! finding) is then the input in `hangs/` that `in_process` does *not* flag. +//! +//! `echo core | sudo tee /proc/sys/kernel/core_pattern` (or `cargo afl +//! system-config`) puts them back in `crashes/`, and needs root. +//! +//! # `crashes/` is not a list of new bugs +//! +//! It fills up with the *known* residual defects (`meet_keeps_dangling` and +//! friends) within the first minute, because to this target they are +//! indistinguishable from a new finding. Triage is `differential.py`'s job: it +//! owns the one `KNOWN` table. + +use differential::harness::run as crate_run; +use differential::reference::fuzz::run as model_run; + +fn main() { + afl::fuzz!(|data: &[u8]| { + // An empty or near-empty input decodes to `EMPTY` on both sides; let AFL + // keep it as a seed anyway, it costs one comparison. + let model = model_run(data, false); + let real = crate_run(data, false); + if model == real { + return; + } + // Panicking is the reporting channel: `afl::fuzz!`'s hook turns it into + // an abort, which AFL records as a crash and saves the input for. + let first = model + .lines() + .zip(real.lines()) + .enumerate() + .find(|(_, (a, b))| a != b) + .map(|(i, (a, b))| format!("line {i}\n model: {a}\n crate: {b}")) + .unwrap_or_else(|| { + format!( + "length {} (model) vs {} (crate) lines", + model.lines().count(), + real.lines().count() + ) + }); + panic!("model/crate divergence:\n{first}"); + }); +} diff --git a/differential/src/bin/crash_fuzz.rs b/differential/src/bin/crash_fuzz.rs new file mode 100644 index 00000000..6dee5e02 --- /dev/null +++ b/differential/src/bin/crash_fuzz.rs @@ -0,0 +1,466 @@ +//! Crash-only fuzzer over [`differential::crash`]: no model, no trace. An input +//! fails by panicking (including a failed `debug_assert!`), aborting, or +//! running longer than `--timeout` seconds. +//! +//! crash_fuzz --random 1000000 -j 56 +//! crash_fuzz --random 1000000 -j 56 --keep-going --save runs/crashes +//! crash_fuzz runs/crashes/*.bin +//! +//! Debug assertions are the point of half of this, so build it with them, and +//! overflow checks, on. That is also what `cargo afl build` does, so it is the +//! build to replay `afl_crash` findings with: +//! +//! CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=true CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS=true \ +//! CARGO_TARGET_DIR=target/dbgassert cargo build --release -p differential --bin crash_fuzz +//! +//! `CRASH_TRACE=1` prints each operation as it starts and `CRASH_BACKTRACE=1` +//! a backtrace on panic; both are for replaying one saved input. +//! +//! `--include-known` also makes the calls in `differential::crash::KNOWN_FAILURES`, +//! which are otherwise steered around. +//! +//! # Failure handling +//! +//! As in `in_process`, a panic is never caught: `pathmap` is not unwind-safe. +//! The panic hook names the input, saves it, and exits. An abort or a +//! segfault gets no hook at all. +//! +//! A segfault or an abort cannot run a panic hook, but a signal handler can +//! still say which input was in flight on the faulting thread (exit 103); the +//! supervisor saves that input. +//! +//! A hang cannot be interrupted either, but it does not have to end the run: +//! the watchdog reports the input, the stuck thread is abandoned to spin, and +//! the others carry on. The process gives up (exit 102) once half its threads +//! are stuck. +//! +//! Without `--keep-going` the first panic ends the run. With it, this process +//! becomes a supervisor: it runs the fuzzer as a child, collects every failure +//! the child reports, and starts a new child past them, skipping inputs already +//! reported. A death nothing attributed is located by re-running from the +//! child's last low-water mark one input at a time. At the end the +//! failures are grouped by where they panicked. + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Write}; +use std::process::{Command, Stdio}; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use differential::crash::{run, set_include_known}; +use differential::source::Source; + +static IN_FLIGHT: [AtomicUsize; 256] = [const { AtomicUsize::new(usize::MAX) }; 256]; +/// Milliseconds since `EPOCH` at which each slot started its current input. +static STARTED: [AtomicU64; 256] = [const { AtomicU64::new(0) }; 256]; +static EPOCH: OnceLock = OnceLock::new(); +static SOURCE: OnceLock = OnceLock::new(); +static SAVE: OnceLock> = OnceLock::new(); + +thread_local! { + static SLOT: std::cell::Cell = const { std::cell::Cell::new(usize::MAX) }; +} + +fn now_ms() -> u64 { + EPOCH.get().unwrap().elapsed().as_millis() as u64 +} + +/// Report one failure on stderr in the line format the supervisor parses, and +/// save the input. +fn report(idx: usize, kind: &str, msg: &str) { + let one_line = msg.replace('\n', " | "); + let mut saved = String::new(); + if let (Some(Some(dir)), Some(src)) = (SAVE.get(), SOURCE.get()) { + let _ = std::fs::create_dir_all(dir); + let path = std::path::Path::new(dir).join(format!("{idx:08}.bin")); + if std::fs::write(&path, src.get(idx)).is_ok() { + saved = format!(" saved={}", path.display()); + } + } + let name = SOURCE.get().map(|s| s.name(idx)).unwrap_or_default(); + eprintln!("CRASH idx={idx} kind={kind} input={name}{saved} msg={one_line}"); + let _ = std::io::stderr().flush(); +} + +struct Args { + all: Vec, +} + +impl Args { + fn flag(&self, name: &str, default: usize) -> usize { + self.str_flag(name).and_then(|v| v.parse().ok()).unwrap_or(default) + } + fn str_flag(&self, name: &str) -> Option { + self.all.iter().position(|a| a == name).and_then(|i| self.all.get(i + 1)).cloned() + } + fn has(&self, name: &str) -> bool { + self.all.iter().any(|a| a == name) + } +} + +const VALUE_FLAGS: &[&str] = &["--max-failures", "--random", "--from", "--seed", "--maxlen", "-j", "--save", "--dump", "--timeout", "--skip", "--to"]; + +fn main() { + EPOCH.get_or_init(Instant::now); + let args = Args { all: std::env::args().skip(1).collect() }; + let files: Vec = { + let mut v = Vec::new(); + let mut it = args.all.iter(); + while let Some(a) = it.next() { + if a.starts_with('-') { + if VALUE_FLAGS.contains(&a.as_str()) { it.next(); } + } else { + v.push(a.clone()); + } + } + v + }; + let count = args.flag("--random", 0); + if count == 0 && files.is_empty() { + eprintln!("usage: crash_fuzz [--random N] [--seed S] [--maxlen L] [--from I] [--to I] [-j N] \ + [--timeout SECS] [--save DIR] [--keep-going [--max-failures N]] [--include-known] [--dump IDX] [FILES...]"); + std::process::exit(2); + } + let source = if !files.is_empty() { + Source::Files(files) + } else { + Source::Random { seed: args.flag("--seed", 1) as u64, count, maxlen: args.flag("--maxlen", 600) } + }; + + if let Some(i) = args.str_flag("--dump") { + let idx: usize = i.parse().expect("--dump IDX"); + std::io::stdout().write_all(&source.get(idx)).unwrap(); + return; + } + + if args.has("--keep-going") { + supervise(&args, source); + return; + } + worker(&args, source); +} + +// --------------------------------------------------------------------------- +// Worker: runs inputs until the first failure +// --------------------------------------------------------------------------- + +/// Say which input the faulting thread was running, with only async-signal-safe +/// calls, and exit. Runs on the alternate signal stack std sets up for every +/// thread, so a stack overflow reaches it too. +extern "C" fn on_fatal_signal(sig: libc::c_int) { + // One report per process: a second faulting thread waits to be killed by the exit. + static REPORTING: AtomicBool = AtomicBool::new(false); + if REPORTING.swap(true, Ordering::SeqCst) { + loop { unsafe { libc::pause(); } } + } + let slot = SLOT.with(|s| s.get()); + let idx = if slot < 256 { IN_FLIGHT[slot].load(Ordering::Relaxed) } else { usize::MAX }; + let mut buf = [0u8; 128]; + let mut n = 0; + let mut put = |bytes: &[u8]| { + for &b in bytes { + if n < buf.len() { buf[n] = b; n += 1; } + } + }; + fn digits(mut v: u64, out: &mut [u8; 20]) -> usize { + let mut i = out.len(); + loop { + i -= 1; + out[i] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { return i } + } + } + let mut d = [0u8; 20]; + put(b"\nCRASH idx="); + if idx == usize::MAX { put(b"none") } else { let i = digits(idx as u64, &mut d); put(&d[i..]); } + put(b" kind=signal msg=signal "); + let i = digits(sig as u64, &mut d); + put(&d[i..]); + put(match sig { libc::SIGSEGV => b" (SIGSEGV)\n" as &[u8], libc::SIGBUS => b" (SIGBUS)\n", libc::SIGABRT => b" (SIGABRT)\n", _ => b"\n" }); + unsafe { + libc::write(2, buf.as_ptr() as *const libc::c_void, n); + libc::_exit(103); + } +} + +fn install_signal_handlers() { + for sig in [libc::SIGSEGV, libc::SIGBUS, libc::SIGABRT, libc::SIGILL] { + unsafe { + let mut sa: libc::sigaction = core::mem::zeroed(); + sa.sa_sigaction = on_fatal_signal as extern "C" fn(libc::c_int) as usize; + sa.sa_flags = libc::SA_ONSTACK; + libc::sigemptyset(&mut sa.sa_mask); + libc::sigaction(sig, &sa, core::ptr::null_mut()); + } + } +} + +fn worker(args: &Args, source: Source) { + install_signal_handlers(); + set_include_known(args.has("--include-known")); + let from = args.flag("--from", 0); + let to = args.flag("--to", usize::MAX).min(source.len()); + let jobs = args.flag("-j", 1).clamp(1, 256); + let timeout_ms = args.flag("--timeout", 10) as u64 * 1000; + let trace_starts = args.has("--trace-starts"); + let skip: std::collections::HashSet = args + .str_flag("--skip") + .map(|s| s.split(',').filter_map(|x| x.parse().ok()).collect()) + .unwrap_or_default(); + SAVE.get_or_init(|| args.str_flag("--save")); + SOURCE.get_or_init(|| source); + let src = SOURCE.get().unwrap(); + + std::panic::set_hook(Box::new(|info| { + // One report per process: a second panicking thread waits for the exit. + static REPORTING: AtomicBool = AtomicBool::new(false); + if REPORTING.swap(true, Ordering::SeqCst) { + loop { std::thread::park(); } + } + let slot = SLOT.with(|s| s.get()); + let idx = if slot < 256 { IN_FLIGHT[slot].load(Ordering::Relaxed) } else { usize::MAX }; + let loc = info.location().map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column())).unwrap_or_default(); + let payload = info + .payload() + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| info.payload().downcast_ref::().cloned()) + .unwrap_or_default(); + if std::env::var_os("CRASH_BACKTRACE").is_some() { + eprintln!("{}", std::backtrace::Backtrace::force_capture()); + } + report(idx, "panic", &format!("panicked at {loc}: {payload}")); + std::process::exit(101); + })); + + // Threads are detached, not scoped: a thread stuck in a hang never returns, + // and a scope would wait for it forever. + static NEXT: AtomicUsize = AtomicUsize::new(0); + static DONE: AtomicUsize = AtomicUsize::new(0); + static RUNNING: AtomicUsize = AtomicUsize::new(0); + static HUNG: [AtomicBool; 256] = [const { AtomicBool::new(false) }; 256]; + NEXT.store(from, Ordering::Relaxed); + RUNNING.store(jobs, Ordering::Relaxed); + let skip: &'static std::collections::HashSet = Box::leak(Box::new(skip)); + let start = Instant::now(); + for slot in 0..jobs { + std::thread::spawn(move || { + SLOT.with(|s| s.set(slot)); + loop { + let idx = NEXT.fetch_add(1, Ordering::Relaxed); + if idx >= to { + IN_FLIGHT[slot].store(usize::MAX, Ordering::Relaxed); + RUNNING.fetch_sub(1, Ordering::Relaxed); + return; + } + if skip.contains(&idx) { continue } + STARTED[slot].store(now_ms(), Ordering::Relaxed); + IN_FLIGHT[slot].store(idx, Ordering::Relaxed); + if trace_starts { + eprintln!("START {idx}"); + } + run(&src.get(idx)); + DONE.fetch_add(1, Ordering::Relaxed); + } + }); + } + + // Watchdog and progress: hangs are reported and their threads written off, + // and the low-water mark lets a supervisor locate an abort. + let mut hung = 0usize; + let mut last_report = Instant::now(); + loop { + std::thread::sleep(Duration::from_millis(100)); + let now = now_ms(); + let mut low = NEXT.load(Ordering::Relaxed); + for slot in 0..jobs { + if HUNG[slot].load(Ordering::Relaxed) { continue } + let idx = IN_FLIGHT[slot].load(Ordering::Relaxed); + if idx == usize::MAX { continue } + low = low.min(idx); + if now.saturating_sub(STARTED[slot].load(Ordering::Relaxed)) > timeout_ms { + HUNG[slot].store(true, Ordering::Relaxed); + hung += 1; + report(idx, "hang", &format!("still running after {}s", timeout_ms / 1000)); + } + } + if last_report.elapsed() > Duration::from_secs(1) { + eprintln!("LOW {low}"); + last_report = Instant::now(); + } + if hung * 2 >= jobs { + eprintln!("{hung} of {jobs} threads hung; giving up at {low}"); + eprintln!("LOW {low}"); + std::process::exit(102); + } + if RUNNING.load(Ordering::Relaxed) <= hung { + break; + } + } + let n = DONE.load(Ordering::Relaxed); + let secs = start.elapsed().as_secs_f64(); + println!("{n} inputs ran clean in {secs:.2}s -> {:.0} inputs/s ({hung} hung)", n as f64 / secs); + // Not a return: hung threads would keep the process alive. + std::process::exit(if hung > 0 { 3 } else { 0 }); +} + +// --------------------------------------------------------------------------- +// Supervisor: restarts workers past failures and collects them +// --------------------------------------------------------------------------- + +struct Crash { + idx: usize, + kind: String, + msg: String, +} + +/// Group key: the panic site, or the kind for hangs and aborts. +fn site(c: &Crash) -> String { + match c.msg.strip_prefix("panicked at ") { + Some(rest) => { + let (loc, what) = rest.split_once(": ").unwrap_or((rest, "")); + // Assertion messages carry values; keep the text before the first value. + let what: String = what.split(" | ").next().unwrap_or("").chars().take(90).collect(); + // Index and length values differ per input; the site does not. + let what: String = what.split(' ').map(|w| if w.chars().all(|c| c.is_ascii_digit()) && !w.is_empty() { "N" } else { w }).collect::>().join(" "); + format!("{loc}: {what}") + } + None if c.kind == "signal" || c.kind == "abort" => format!("{}: {}", c.kind, c.msg.chars().take(60).collect::()), + None => c.kind.clone(), + } +} + +struct ChildResult { + crashes: Vec, + low: usize, + /// The child ran its whole range (exit 0, or 3 when some inputs hung). + completed: bool, + status: String, + last_start: Option, +} + +/// Run one child over `[from, to)` and collect what it reports. +fn run_child(args: &Args, from: usize, to: usize, jobs: usize, skip: &[usize], trace_starts: bool) -> ChildResult { + let exe = std::env::current_exe().unwrap(); + let mut cmd = Command::new(exe); + let mut it = args.all.iter(); + while let Some(a) = it.next() { + match a.as_str() { + "--keep-going" => {} + "--from" | "--to" | "-j" | "--skip" | "--max-failures" => { it.next(); } + _ => { cmd.arg(a); } + } + } + cmd.args(["--from", &from.to_string(), "--to", &to.to_string(), "-j", &jobs.to_string()]); + if !skip.is_empty() { + cmd.args(["--skip", &skip.iter().map(|i| i.to_string()).collect::>().join(",")]); + } + if trace_starts { + cmd.arg("--trace-starts"); + } + cmd.stderr(Stdio::piped()).stdout(Stdio::null()); + let mut child = cmd.spawn().expect("cannot start worker"); + let stderr = child.stderr.take().unwrap(); + let mut r = ChildResult { crashes: Vec::new(), low: from, completed: false, status: String::new(), last_start: None }; + let mut abort_msg = String::new(); + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + if let Some(rest) = line.strip_prefix("CRASH ") { + let field = |k: &str| rest.split(' ').find_map(|f| f.strip_prefix(k)).unwrap_or("").to_string(); + let msg = rest.split_once("msg=").map(|(_, m)| m.to_string()).unwrap_or_default(); + eprintln!("{line}"); + r.crashes.push(Crash { idx: field("idx=").parse().unwrap_or(usize::MAX), kind: field("kind="), msg }); + } else if let Some(l) = line.strip_prefix("LOW ") { + r.low = l.parse().unwrap_or(r.low); + } else if let Some(s) = line.strip_prefix("START ") { + r.last_start = s.parse().ok(); + } else if !line.is_empty() && abort_msg.len() < 300 { + // An abort's own words: a stack overflow, the allocator, a UB check. + abort_msg.push_str(&line); + abort_msg.push(' '); + } + } + let status = child.wait().unwrap(); + r.completed = matches!(status.code(), Some(0) | Some(3)); + let panicked = matches!(status.code(), Some(101) | Some(102) | Some(103)); + if !r.completed && !panicked { + // No hook ran. `idx` is filled in by locating, or from `--trace-starts`. + let msg = format!("{status} {}", abort_msg.trim()); + eprintln!("CRASH kind=abort msg={msg}"); + r.crashes.push(Crash { idx: r.last_start.unwrap_or(usize::MAX), kind: "abort".into(), msg }); + } + r.status = format!("{status}"); + r +} + +fn supervise(args: &Args, source: Source) { + let n = source.len(); + let save = args.str_flag("--save"); + let max_failures = args.flag("--max-failures", 2000); + let jobs = args.flag("-j", 1).clamp(1, 256); + let to = args.flag("--to", usize::MAX).min(n); + let mut from = args.flag("--from", 0); + let mut skip: Vec = Vec::new(); + let mut crashes: Vec = Vec::new(); + let start = Instant::now(); + loop { + let mut r = run_child(args, from, to, jobs, &skip, false); + for c in r.crashes.iter_mut() { + if c.idx != usize::MAX { continue } + // No hook ran: find the input one at a time from the low-water mark. + eprintln!("unattributed failure ({}); locating from {}", c.msg, r.low); + let l = run_child(args, r.low, to, 1, &skip, true); + match l.crashes.into_iter().find(|x| x.idx != usize::MAX) { + Some(found) => { eprintln!(" located at {}", found.idx); c.idx = found.idx; if found.kind == "abort" { c.msg = found.msg; } else { *c = found; } } + None => eprintln!(" could not locate"), + } + } + for c in r.crashes.iter_mut().filter(|c| (c.kind == "signal" || c.kind == "abort") && c.idx != usize::MAX) { + //Threads share one heap, so memory corrupted by one input can abort another. Only + // blame an input that fails on its own. + let alone = run_child(args, c.idx, c.idx + 1, 1, &[], false); + if alone.completed { + eprintln!(" input {} runs clean alone; not attributing the {}", c.idx, c.kind); + c.kind = format!("{} (not reproduced alone)", c.kind); + } + } + let progressed = !r.crashes.is_empty() || r.completed; + for c in r.crashes.iter().filter(|c| (c.kind == "signal" || c.kind == "abort") && c.idx != usize::MAX) { + // The child saves what its hooks report; a signal or an abort leaves it to us. + if let Some(dir) = &save { + let _ = std::fs::create_dir_all(dir); + let _ = std::fs::write(std::path::Path::new(dir).join(format!("{:08}.bin", c.idx)), source.get(c.idx)); + } + } + for c in r.crashes { + if c.idx != usize::MAX { skip.push(c.idx); } + crashes.push(c); + } + if r.completed { + break; + } + if !progressed { + eprintln!("child died ({}) with nothing to show; stopping", r.status); + break; + } + from = r.low; + if crashes.len() >= max_failures { + eprintln!("{max_failures} failures; stopping (--max-failures)"); + break; + } + } + let mut by_site: BTreeMap> = BTreeMap::new(); + for c in &crashes { + by_site.entry(site(c)).or_default().push(c.idx); + } + println!("--- failures by site ---"); + for (s, idxs) in &by_site { + let shown: Vec = idxs.iter().take(5).map(|i| i.to_string()).collect(); + println!("{:6} {s}\n inputs: {}{}", idxs.len(), shown.join(" "), if idxs.len() > 5 { " ..." } else { "" }); + } + println!("{} failures ({} sites) over inputs {}..{} in {:.1}s", crashes.len(), by_site.len(), args.flag("--from", 0), to, start.elapsed().as_secs_f64()); + if !crashes.is_empty() { + std::process::exit(1); + } +} diff --git a/differential/src/bin/crash_repros.rs b/differential/src/bin/crash_repros.rs new file mode 100644 index 00000000..3e12b82d --- /dev/null +++ b/differential/src/bin/crash_repros.rs @@ -0,0 +1,109 @@ +//! Minimal reproducers for failures `crash_fuzz` turned up. See +//! `differential/CRASH_FINDINGS.md` for the write-up. +//! +//! cargo run -p differential --bin crash_repros -- --list +//! cargo run -p differential --bin crash_repros -- +//! +//! Each case ends the process (panic, abort or segfault), so run them one at a +//! time. Where a debug build fails earlier on an assertion, the case says so. + +use pathmap::PathMap; +use pathmap::utils::ByteMask; +use pathmap::zipper::*; + +fn sample() -> PathMap { + let mut m = PathMap::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 2, 1, 3, 3], &[0], &[2, 2]] { + m.set_val_at(p, 7); + } + m +} + +fn owned_head_second_exclusive_path() { + let zh = sample().into_zipper_head(&[1u8]); + drop(zh.write_zipper_at_exclusive_path(&[])); + let _ = zh.write_zipper_at_exclusive_path(&[9u8]); // UB: segfault / unreachable_unchecked +} +fn write_zipper_head_second_exclusive_path() { + let mut map = sample(); + let mut wz = map.write_zipper_at_path(&[1u8]); + let zh = wz.zipper_head(); + drop(zh.write_zipper_at_exclusive_path(&[])); + let _ = zh.write_zipper_at_exclusive_path(&[]); // zipper_head.rs:347 +} +fn head_read_zipper_get_trie_ref() { + let mut map = sample(); + let zh = map.zipper_head(); + let rz = zh.read_zipper_at_path(&[1u8]).unwrap(); + let _ = rz.get_trie_ref(); // zipper.rs:1871 +} +fn product_zipper_is_shared() { + let (a, b) = (sample(), sample()); + let mut z = ProductZipper::new(a.read_zipper_at_path(&[1u8, 2, 1]), [b.read_zipper()]); + while z.to_next_step() { + let _ = z.is_shared(); // zipper.rs:2644 once the focus is in the second factor + } +} +fn product_zipper_val_count() { + let mut a = PathMap::::new(); + a.set_val_at([1u8, 2], 1); + let mut b = PathMap::::new(); + b.set_val_at([3u8, 4], 1); + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); + loop { + let _ = z.val_count(); // zipper.rs:3048 (debug: product_zipper.rs:178) + if !z.to_next_step() { break } + } +} +fn graft_child_maps_long_root() { + let mut map = PathMap::::new(); + let mut wz = map.write_zipper_at_path(&[0u8; 48]); + wz.graft_child_maps(ByteMask::from_iter([1u8]), [PathMap::single([2u8], 5)], false); // write_zipper.rs:2557/2560 +} +fn owned_read_zipper_witness() { + let mut map = PathMap::::new(); + map.set_val_at([1u8], 1); + let z = map.into_read_zipper(&[]); + let w = z.witness(); + let _ = z.get_val_with_witness(&w); // zipper.rs:3332 (debug: zipper.rs:3328) +} +fn prefix_zipper_k0_after_last_path() { + let mut map = PathMap::::new(); + map.set_val_at([0x22u8], 1); + let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); + z.descend_to_byte(2); + z.descend_last_path(); + z.descend_last_path(); + z.descend_first_k_path(0); // prefix_zipper.rs:571 +} + +const CASES: &[(&str, fn(), &str)] = &[ + ("owned_head_second_exclusive_path", owned_head_second_exclusive_path, "undefined behaviour: segfault in release, unreachable_unchecked in debug"), + ("write_zipper_head_second_exclusive_path", write_zipper_head_second_exclusive_path, "zipper_head.rs:347 unwrap (debug: write_zipper.rs:1337 assertion)"), + ("head_read_zipper_get_trie_ref", head_read_zipper_get_trie_ref, "zipper.rs:1871 explicit panic"), + ("product_zipper_is_shared", product_zipper_is_shared, "zipper.rs:2644 unwrap"), + ("product_zipper_val_count", product_zipper_val_count, "zipper.rs:3048 unwrap (debug: product_zipper.rs:178 assertion)"), + ("graft_child_maps_long_root", graft_child_maps_long_root, "write_zipper.rs:2560 slice out of range"), + ("owned_read_zipper_witness", owned_read_zipper_witness, "zipper.rs:3332 slice out of range (debug: zipper.rs:3328 assertion)"), + ("prefix_zipper_k0_after_last_path", prefix_zipper_k0_after_last_path, "prefix_zipper.rs:571 slice out of range"), +]; + +fn main() { + let arg = std::env::args().nth(1).unwrap_or_else(|| "--list".into()); + if arg == "--list" { + for (name, _, what) in CASES { + println!("{name:40} {what}"); + } + return; + } + match CASES.iter().find(|(name, _, _)| *name == arg) { + Some((_, case, _)) => { + case(); + println!("{arg}: no failure (fixed?)"); + } + None => { + eprintln!("unknown case {arg}; see --list"); + std::process::exit(2); + } + } +} diff --git a/differential/src/bin/in_process.rs b/differential/src/bin/in_process.rs new file mode 100644 index 00000000..742733bf --- /dev/null +++ b/differential/src/bin/in_process.rs @@ -0,0 +1,297 @@ +//! In-process differential fuzzer: the Rust reference model against the real crate. +//! +//! in_process --random 100000 +//! in_process --random 10000000 -j 56 --max-fails 0 --save runs/diverged +//! in_process --random 2000000 -j 56 --act +//! in_process corpus/*.bin +//! +//! `lean/differential.py` compares the *Lean* model against something else, and +//! pays for it: two child processes, a hex-encoded input over a pipe, a rendered +//! trace back, and a Python driver in the middle. That buys language +//! independence — two transcriptions of one specification, written in different +//! languages — and it is the right tool for validating the port +//! (`differential.py --model`). +//! +//! It is the wrong tool for *volume*. Once the Rust model is known to agree with +//! the Lean one, the crate can be checked against the Rust model with no +//! processes, no pipes and no serialisation at all: both run in this binary, on +//! the same input, and the results are compared in memory. Nothing is rendered +//! unless something diverges. +//! +//! # What is being compared +//! +//! Exactly what `differential.py` compares, through exactly the same two op +//! tables — there is no fourth transcription here: +//! +//! * the model side is [`differential::reference::fuzz`], the port of `Fuzz.lean`; +//! * the crate side is [`differential::harness`], shared with `pathmap_trace`, +//! `act_trace` and the repro generator. +//! +//! Both are driven from the same bytes and must produce the same trace. Sharing +//! the op tables rather than copying them is the point: a fourth copy would drift. +//! +//! # Why this could not run before, and can now +//! +//! It was written against a crate that was **not unwind-safe** and panicked on +//! roughly one random input in eight. Catching such a panic with `catch_unwind` +//! and carrying on corrupts the heap — `malloc(): unaligned tcache chunk +//! detected` — because the half-updated refcounted nodes are dropped while the +//! stack unwinds. So this does **not** catch panics, and never should: a panic +//! *hook* runs before unwinding starts, which makes it the last safe moment to +//! say which input was responsible; it reports and `exit`s, and nothing unwinds +//! through the crate's internals. With a 1-in-8 panic rate that ended a run +//! within a handful of inputs, which is why the subprocess design existed at all: +//! it is not overhead, it is panic tolerance. A dead child costs one input. +//! +//! The panicking findings have since been fixed. The design here is unchanged — +//! there are simply no panics left to end the run. If one reappears, this will +//! stop on it and name it, which is the correct outcome and a finding in its own +//! right. +//! +//! # Reporting +//! +//! A divergence is printed as the first differing trace line, and `--save DIR` +//! writes the input beside it. Classification into the known-defect buckets is +//! deliberately *not* reimplemented here: the `KNOWN` table lives in +//! `lean/differential.py` and there is exactly one of it. Run +//! `./lean/differential.py DIR/*` over the saved inputs to get the breakdown — +//! which also re-checks each one against the Lean oracle rather than against this +//! binary's own model. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use differential::act::run_act as crate_run_act; +use differential::harness::run as crate_run; +use differential::reference::fuzz::run as model_run; +use differential::source::Source; + +/// The index each thread is currently working on, so a panic or an abort can be +/// attributed. +/// +/// A panic ends the run by design (see the module docs); an *abort* is not +/// catchable in-process at all. Either way the fuzzer prints what every thread +/// had in flight on the way down, and `--from` resumes past it. +static IN_FLIGHT: [AtomicUsize; 256] = [const { AtomicUsize::new(usize::MAX) }; 256]; + +/// Compare one input. Returns the rendered report on divergence, `None` on +/// agreement. +/// +/// The traces are only *rendered* because both op tables render them today; the +/// comparison itself is a string equality, and neither side leaves this process. +fn compare(blob: &[u8], act: bool) -> Option { + let model = model_run(blob, act); + // NOT wrapped in `catch_unwind`, and that is deliberate. See the module + // docs: catching a panic out of the middle of a trie mutation and carrying + // on corrupts the heap. The panic hook installed in `main` reports and + // exits instead of letting the stack unwind at all. + let real = if act { crate_run_act(blob, false) } else { crate_run(blob, false) }; + // The fast path is one `memcmp` over two buffers. Individual lines are only + // needed to *report* a divergence, so they are only split out then. + if model == real { + return None; + } + for (i, (a, b)) in model.lines().zip(real.lines()).enumerate() { + if a != b { + return Some(format!("line {i}\n model: {a}\n crate: {b}")); + } + } + Some(format!( + "length {} (model) vs {} (crate) lines", + model.lines().count(), + real.lines().count() + )) +} + +/// The operation name of the first differing line, used only for the run's own +/// summary. Buckets by *op*, not by defect: the defect taxonomy is the `KNOWN` +/// table in `lean/differential.py`, and there is one of it. +fn first_diff_op(msg: &str) -> String { + let line = msg + .lines() + .find_map(|l| l.trim_start().strip_prefix("model: ")) + .unwrap_or("?"); + let mut it = line.split_whitespace(); + match it.next() { + // An operation line is ` ret=...`; the trailer lines are + // `MAP0 ` and `ROOT0 `, whose own first token is the name. + Some(tok) if tok.parse::().is_ok() => it.next().unwrap_or("?").to_string(), + Some(tok) => tok.to_string(), + None => "?".to_string(), + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let flag = |name: &str, default: usize| -> usize { + args.iter() + .position(|a| a == name) + .and_then(|i| args.get(i + 1)) + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + let str_flag = |name: &str| -> Option { + args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned() + }; + let count = flag("--random", 0); + // Index to start at. Inputs are deterministic in their index, so a run that + // dies can be resumed past the offending one. + let from = flag("--from", 0); + let seed = flag("--seed", 1) as u64; + let maxlen = flag("--maxlen", 300); + let jobs = flag("-j", 1).max(1); + // 0 means "never stop": a long survey wants every divergence, not the first ten. + let max_fails = flag("--max-fails", 10); + let save_dir = str_flag("--save"); + // Read source is an `ArenaCompactTree` built from map1 rather than the + // `PathMap` itself, exactly as `act_trace` does it. The model takes the + // same flag and skips the operations ACT cannot serve. + let act = args.iter().any(|a| a == "--act"); + let value_flags = [ + "--random", "--from", "--seed", "--maxlen", "-j", "--max-fails", "--save", "--dump", + "--emit-corpus", + ]; + let files: Vec = { + let mut v = Vec::new(); + let mut it = args.iter().peekable(); + while let Some(a) = it.next() { + if a.starts_with('-') { + if value_flags.contains(&a.as_str()) { + it.next(); // its value + } + } else { + v.push(a.clone()); + } + } + v + }; + let source = if !files.is_empty() { + Source::Files(files) + } else if count > 0 { + Source::Random { seed, count, maxlen } + } else { + eprintln!( + "usage: in_process [--random N] [--seed S] [--maxlen L] [--from I] \ + [-j N] [--max-fails N] [--save DIR] [--emit-corpus DIR] [--act] [FILES...]" + ); + std::process::exit(2); + }; + + // `--dump IDX` writes one input to stdout and exits, so an input that ends a + // run can still be extracted and replayed against the crate alone. + if let Some(i) = args.iter().position(|a| a == "--dump") { + let idx: usize = args[i + 1].parse().expect("--dump IDX"); + use std::io::Write; + std::io::stdout().write_all(&source.get(idx)).unwrap(); + return; + } + + // `--emit-corpus DIR` writes the whole source out as one file per input, and + // runs nothing. That is the seed corpus for `afl_differential`: AFL needs + // starting points that already reach interesting states, and the same + // generator that feeds this binary is the obvious source of them. + if let Some(d) = str_flag("--emit-corpus") { + std::fs::create_dir_all(&d).expect("cannot create --emit-corpus directory"); + for idx in 0..source.len() { + let path = std::path::Path::new(&d).join(format!("{idx:06}.bin")); + std::fs::write(&path, source.get(idx)).expect("cannot write seed"); + } + println!("wrote {} seeds to {d}", source.len()); + return; + } + + if let Some(d) = &save_dir { + std::fs::create_dir_all(d).expect("cannot create --save directory"); + } + + let n = source.len(); + let next = AtomicUsize::new(from); + let agreed = AtomicUsize::new(0); + let stop = AtomicBool::new(false); + let reports: Mutex> = Mutex::new(Vec::new()); + // A crate panic ends the run, by design. The hook runs *before* the stack + // unwinds, so it is the last safe moment to say which input did it -- and + // exiting from here means nothing unwinds through `pathmap`'s internals. + std::panic::set_hook(Box::new(|info| { + let live: Vec = IN_FLIGHT + .iter() + .map(|a| a.load(Ordering::Relaxed)) + .filter(|&i| i != usize::MAX) + .collect(); + eprintln!("CRATE PANIC on input index {live:?}: {info}"); + eprintln!(" the crate panics on this input; the model is total and cannot."); + eprintln!(" extract it with --dump IDX, resume past it with --from {}", + live.iter().max().map_or(0, |m| m + 1)); + // Not `abort`: exit runs no destructors and unwinds nothing. + std::process::exit(101); + })); + + let start = std::time::Instant::now(); + let (src, next_r, stop_r, agreed_r, reports_r) = (&source, &next, &stop, &agreed, &reports); + std::thread::scope(|scope| { + for slot in 0..jobs { + scope.spawn(move || { + loop { + if stop_r.load(Ordering::Relaxed) { + return; + } + let idx = next_r.fetch_add(1, Ordering::Relaxed); + if idx >= n { + IN_FLIGHT[slot.min(255)].store(usize::MAX, Ordering::Relaxed); + return; + } + IN_FLIGHT[slot.min(255)].store(idx, Ordering::Relaxed); + match compare(&src.get(idx), act) { + None => { + agreed_r.fetch_add(1, Ordering::Relaxed); + } + Some(msg) => { + let mut r = reports_r.lock().unwrap(); + r.push((idx, src.name(idx), msg)); + if max_fails != 0 && r.len() >= max_fails { + stop_r.store(true, Ordering::Relaxed); + } + } + } + } + }); + } + }); + let elapsed = start.elapsed().as_secs_f64(); + + // Sorted by input index, so a -j run reports in the same order a -j1 run does. + let mut reports = reports.into_inner().unwrap(); + reports.sort_by_key(|(idx, _, _)| *idx); + let mut by_op: std::collections::BTreeMap = Default::default(); + for (idx, name, msg) in &reports { + *by_op.entry(first_diff_op(msg)).or_default() += 1; + match &save_dir { + Some(d) => { + let path = std::path::Path::new(d).join(format!("{idx:08}.bin")); + let _ = std::fs::write(&path, source.get(*idx)); + println!("FAIL {name} [saved {}]: {msg}", path.display()); + } + None => println!("FAIL {name}: {msg}"), + } + } + let ok = agreed.load(Ordering::Relaxed); + let done = ok + reports.len(); + if !by_op.is_empty() { + println!("--- first differing op ---"); + for (op, k) in &by_op { + println!("{k:8} {op} (1 in {})", done / k.max(&1)); + } + println!( + "run `./lean/differential.py {}/*` for the KNOWN-table breakdown", + save_dir.as_deref().unwrap_or("DIR") + ); + } + println!( + "{ok}/{done} inputs agree ({} divergences) in {elapsed:.2}s -> {:.0} inputs/s", + reports.len(), + done as f64 / elapsed + ); + if !reports.is_empty() { + std::process::exit(1); + } +} diff --git a/differential/src/bin/reference.rs b/differential/src/bin/reference.rs new file mode 100644 index 00000000..05a456d7 --- /dev/null +++ b/differential/src/bin/reference.rs @@ -0,0 +1,47 @@ +//! Prints the differential trace for a fuzzer input, from the **Rust reference +//! model** — the port of `lean/PathMapModel/` in `differential/src/reference/`. +//! +//! reference # or read the bytes from stdin +//! reference --act # ACT-mode skips +//! +//! There are three front ends over one wire format: +//! +//! | binary | drives | +//! |---|---| +//! | `lean/.lake/build/bin/pathmap-oracle` | the Lean model (`lean/PathMapModel/Fuzz.lean`) | +//! | `pathmap_trace` | the real crate (`differential/src/harness.rs`) | +//! | this | the Rust model (`differential/src/reference/`) | +//! +//! `lean/differential.py` diffs any two of them. Model-against-model — +//! `differential.py --model` — is the acceptance test for the port: the two are +//! independent transcriptions of the same specification in different languages, +//! so a diff means one of them is wrong and nothing about the crate is in +//! question. The `KNOWN` table of tolerated crate defects therefore does not +//! apply in that mode, and `differential.py` does not consult it. +//! +//! Once the two models are known to agree, `in_process` drops the pipes +//! entirely and compares the Rust model against the crate in memory. + +use differential::reference::fuzz::run; +use differential::server::serve; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let act = args.iter().any(|a| a == "--act"); + // Resident mode: one process, many inputs over stdin. See `serve`. + if args.iter().any(|a| a == "--server") { + serve(act, run); + return; + } + let file = args.into_iter().find(|a| a != "--act" && a != "--server"); + let bytes: Vec = match file { + Some(p) => std::fs::read(p).expect("cannot read input"), + None => { + use std::io::Read; + let mut v = Vec::new(); + std::io::stdin().read_to_end(&mut v).unwrap(); + v + } + }; + print!("{}", run(&bytes, act)); +} diff --git a/differential/src/bin/zipper_bug_repros.rs b/differential/src/bin/zipper_bug_repros.rs index c7a0dbe4..6a517ffb 100644 --- a/differential/src/bin/zipper_bug_repros.rs +++ b/differential/src/bin/zipper_bug_repros.rs @@ -92,7 +92,18 @@ fn run(name: &str) { let mut empty = PathMap::::new(); let st = { let mut w = empty.write_zipper(); w.join_into(&rz) }; - println!(" into EMPTY dst -> {st:?}; dst = {} <-- source lost", vals(&empty)); + println!(" into EMPTY dst -> {st:?}; dst = {}", vals(&empty)); + + // The form that outlived the empty-destination fix: a dense destination that already + // holds everything under the source's mid-key focus was *replaced* by the source. + let mut dense = PathMap::::new(); + for i in 0..4u8 { dense.insert(&[i], i as u64); } + let mut src2 = PathMap::::new(); + src2.insert(&[0u8, 0, 0], 0); + let mut rz2 = src2.read_zipper(); + rz2.descend_to(&[0u8, 0]); + let st = { let mut w = dense.write_zipper(); w.join_into(&rz2) }; + println!(" DENSE dst u mid-key src -> {st:?}; dst = {} (expect Identity, 4 values)", vals(&dense)); let mut nonempty = PathMap::::new(); nonempty.insert(&[9u8], 5); diff --git a/differential/src/crash.rs b/differential/src/crash.rs new file mode 100644 index 00000000..6652f80c --- /dev/null +++ b/differential/src/crash.rs @@ -0,0 +1,1119 @@ +//! A second op table for `pathmap`, with no model behind it: an input only has +//! to run to the end without a panic, a failed debug assertion, an abort or a +//! hang. +//! +//! [`crate::harness`] is differential, so it can only reach what the Lean model +//! specifies, and that leaves a lot of the crate alone. This table covers that +//! surface. It has no trace, no lockstep partner and nothing to agree with, so +//! it can grow freely: +//! +//! * **paths** outside the harness's 4-letter alphabet and 5-byte limit: bytes +//! spread over all four `ByteMask` words, full bytes, and keys up to 67 bytes, +//! longer than a list node holds inline; +//! * **value types** other than `u64`: `()`, `bool` and `u16`, whose lattice +//! impls report identities differently; +//! * **`prune = true`** on every operation that takes it, and `prune_path` / +//! `prune_ascend` off the map root; +//! * **`PathMap` methods** the harness never calls: `insert`/`remove`/`get_mut`, +//! `remove_branches_at`, map-level `join`/`meet`/`subtract`/`restrict` and the +//! `Lattice` impls, clones and copy-on-write, `iter`/`from_iter`, `merkleize`, +//! `new_from_ana`; +//! * **zipper kinds**: owned read and write zippers, `read_zipper_at_borrowed_path`, +//! forks, `TrieRef`, `ZipperHead` and `ZipperHeadOwned` with several zippers +//! live at once, `ProductZipper`, `ProductZipperG`, +//! `DependentProductZipperG`, `OverlayZipper`, `PrefixZipper`, `EmptyZipper` +//! and the ACT zipper; +//! * **operations** absent from the harness: `join_into_take`, `drop_head`, +//! `graft_map` on arbitrary maps, `meet_2` over two independent sources, +//! `take_map` without restoring, `meet_k_path_into` on any focus, `k = 0`, +//! witnesses, the `_observed` variants, `descend_indexed_branch`, +//! `reserve_buffers`, `get_focus`, `try_borrow_focus`; +//! * **catamorphisms** in all four flavours, fallible ones stopping early, and +//! `hash`; +//! * **`.paths` serialization**, round trips and decoding of corrupted streams. +//! +//! Calls with a *documented* panic are made to satisfy the documented +//! precondition (see [`KNOWN_PRECONDITIONS`]); anything else that panics is a +//! finding. +//! +//! # Wire format +//! +//! Byte-oriented like the harness's, so AFL mutations land on operands: every +//! operand is one or a few bytes reduced at the point of use, and a truncated +//! input is a shorter valid program. The first byte picks the value type, then +//! come three seeded maps, then the op stream. + +use std::io::Cursor; + +use pathmap::PathMap; +use pathmap::arena_compact::ArenaCompactTree; +use pathmap::morphisms::Catamorphism; +use pathmap::paths_serialization::{deserialize_paths, for_each_deserialized_path, serialize_paths}; +use pathmap::ring::{AlgebraicResult, DistributiveLattice, Lattice, SELF_IDENT}; +use pathmap::utils::{BitMask, ByteMask}; +use pathmap::zipper::*; + +use crate::harness::{Dec, hex_path}; + +/// `CRASH_TRACE=1` prints every operation to stderr as it starts, so the last +/// line before a hang or a panic names the call that did it. +static TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); + +macro_rules! note { + ($($arg:tt)*) => { + if *TRACE.get_or_init(|| std::env::var_os("CRASH_TRACE").is_some()) { + eprintln!($($arg)*); + } + }; +} + +/// Total work per input, counted in operations including those inside episodes. +pub const MAX_STEPS: usize = 512; +/// How many steps one episode (one zipper's lifetime) may take. +const EPISODE_STEPS: usize = 48; +/// Maps in play. +const NMAPS: usize = 3; +/// Bound on any loop this file drives itself (iteration walks and the like). +const WALK: usize = 64; +/// A map that grows past this many values stops accepting growth ops, so a run +/// of `insert_prefix` and joins cannot turn one input into a memory benchmark. +const MAX_VALS: usize = 4096; + +/// Known failures the table steers around unless [`set_include_known`] says +/// otherwise, so a survey reports what is new. Each is either a documented +/// stub or a documented open issue. +pub const KNOWN_FAILURES: &[&str] = &[ + "val_count is todo!/unimplemented! on OverlayZipper, ProductZipperG and DependentProductZipperG", + "TrieRef::fork_read_zipper panics on a TrieRef at a non-existent path (GOAT, issue #96)", + "k == 0 is degenerate for k-path iteration, join_k_path_into and drop_head (the harness skips it as \ + skip:k0): it spins on several zipper kinds and trips `debug_assert!(byte_cnt > 0)` in drop_head", + "to_next_k_path without a preceding descend_first_k_path continues state nothing set up; the harness \ + only runs whole walks, and alone it underflows path_len (and read_zipper_at_borrowed_path's)", + "meet_k_path_into spins forever when the focus has no children, and escapes it when k == 0 \ + (see Zip.meetKPathUnspecified in the Lean model)", +]; + +static INCLUDE_KNOWN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Run the calls listed in [`KNOWN_FAILURES`] as well. +pub fn set_include_known(on: bool) { + INCLUDE_KNOWN.store(on, std::sync::atomic::Ordering::Relaxed); +} + +fn include_known() -> bool { + INCLUDE_KNOWN.load(std::sync::atomic::Ordering::Relaxed) +} + +thread_local! { + /// Set while driving a zipper type whose `val_count` is a stub. + static VAL_COUNT_STUB: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Marks the current episode as driving a `val_count` stub, until dropped. +struct ValCountStub; + +impl ValCountStub { + fn new() -> Self { + VAL_COUNT_STUB.with(|c| c.set(true)); + ValCountStub + } +} + +impl Drop for ValCountStub { + fn drop(&mut self) { + VAL_COUNT_STUB.with(|c| c.set(false)); + } +} + +fn val_count_ok() -> bool { + include_known() || !VAL_COUNT_STUB.with(|c| c.get()) +} + +/// Preconditions this table deliberately satisfies, because the crate documents +/// a panic when they are broken. Anything else that panics is a finding. +pub const KNOWN_PRECONDITIONS: &[&str] = &[ + "ProductZipper::new: secondary factors must be at node roots (factors are map-root zippers)", + "TrieBuilder::push / push_byte / graft_at_byte: first bytes strictly increasing", + "TrieBuilder::graft_at_byte: the source focus must hold a node (called only with children below)", + "graft_child_maps: at least one map per set bit of the mask", +]; + +/// A value type the table can run over. +pub trait CrashValue: + Clone + Send + Sync + Unpin + Lattice + DistributiveLattice + core::hash::Hash + core::fmt::Debug + Default + 'static +{ + fn from_byte(b: u8) -> Self; + fn to_u64(&self) -> u64; +} + +impl CrashValue for u64 { + fn from_byte(b: u8) -> Self { b as u64 } + fn to_u64(&self) -> u64 { *self } +} +impl CrashValue for u16 { + fn from_byte(b: u8) -> Self { (b as u16) << (b % 9) } + fn to_u64(&self) -> u64 { *self as u64 } +} +impl CrashValue for bool { + fn from_byte(b: u8) -> Self { b & 1 == 1 } + fn to_u64(&self) -> u64 { *self as u64 } +} +impl CrashValue for () { + fn from_byte(_: u8) -> Self {} + fn to_u64(&self) -> u64 { 0 } +} + +// --------------------------------------------------------------------------- +// Operand decoding +// --------------------------------------------------------------------------- + +/// A path over one of three alphabets, chosen by the low two bits of its header +/// byte: `0..4` (so paths share prefixes, as in the harness), sixteen bytes +/// spread over the whole range (so all four mask words are used), or any byte. +/// The rest of the header picks the length: mostly `0..7`, sometimes 7 to 67. +fn path(d: &mut Dec) -> Option> { + let h = d.u8()?; + let s = (h >> 2) as usize; + let len = if s < 48 { s % 7 } else { 7 + (s - 48) * 4 }; + let mut v = Vec::with_capacity(len); + for _ in 0..len { + let b = d.u8()?; + v.push(match h & 3 { + 0 | 1 => b % 4, + 2 => (b % 16) * 17, + _ => b, + }); + } + note!(" path {}", hex_path(&v)); + Some(v) +} + +/// A short path, for places where a long one only costs time. +fn short_path(d: &mut Dec) -> Option> { + let mut p = path(d)?; + p.truncate(8); + note!(" short_path {}", hex_path(&p)); + Some(p) +} + +/// One byte, mostly from the small alphabet so it hits existing children. +fn byte(d: &mut Dec) -> Option { + let b = d.u8()?; + if b < 160 { Some(b % 4) } else { d.u8() } +} + +fn mask(d: &mut Dec) -> Option { + let n = d.modn(6)?; + let mut m = ByteMask::EMPTY; + for _ in 0..n { + m.set_bit(byte(d)?); + } + note!(" mask {:?}", m.iter().collect::>()); + Some(m) +} + +fn val(d: &mut Dec) -> Option { + let b = d.u8()?; + note!(" val {b}"); + Some(V::from_byte(b)) +} + +/// Result of a map-level lattice operation, as a map. +fn resolve(r: AlgebraicResult>, a: &PathMap, b: &PathMap) -> PathMap { + match r { + AlgebraicResult::None => PathMap::new(), + AlgebraicResult::Identity(m) => if m & SELF_IDENT != 0 { a.clone() } else { b.clone() }, + AlgebraicResult::Element(x) => x, + } +} + +/// Count one step against the input's budget; `None` ends the input. +fn tick(steps: &mut usize) -> Option<()> { + *steps += 1; + if *steps > MAX_STEPS { None } else { Some(()) } +} + +/// The per-input state: the maps, and the step budget shared by everything. +pub struct State { + pub maps: [PathMap; NMAPS], + steps: usize, +} + +impl State { + fn small(&self, m: usize) -> bool { + self.maps[m].val_count() < MAX_VALS + } +} + +// --------------------------------------------------------------------------- +// Steps shared by every zipper kind +// --------------------------------------------------------------------------- + +// Observers are started from the zipper's current path: a `Vec` or `usize` +// observer mirrors the path, and an ascent above where it started would underflow it. + +/// One movement or query. Everything a `ZipperMoving + ZipperPath` offers. +fn move_step(d: &mut Dec, z: &mut Z) -> Option<()> { + let __op = d.modn(34)?; + note!("move {__op} at {}", hex_path(z.path())); + match __op { + 0 => { let p = path(d)?; note!(" p={}", hex_path(&p)); z.descend_to(&p); } + 1 => { let b = byte(d)?; z.descend_to_byte(b); } + 2 => { let p = path(d)?; let _ = z.descend_to_check(&p); } + 3 => { let p = path(d)?; let _ = z.descend_to_existing(&p); } + 4 => { let p = path(d)?; let _ = z.descend_to_val(&p); } + 5 => { let b = byte(d)?; let _ = z.descend_to_existing_byte(b); } + 6 => { let i = d.modn(8)?; let _ = z.descend_indexed_byte(i); } + #[allow(deprecated)] + 7 => { let i = d.modn(8)?; let _ = z.descend_indexed_branch(i); } + 8 => { let _ = z.descend_first_byte(); } + 9 => { let _ = z.descend_last_byte(); } + 10 => { let _ = z.descend_until(); } + 11 => { let mut o = z.path().to_vec(); let _ = z.descend_until_observed(&mut o); } + 12 => { let n = d.modn(12)?; let _ = z.descend_until_max_bytes(n); } + 13 => { let n = d.modn(12)?; let mut o = z.path().len(); let _ = z.descend_until_max_bytes_observed(n, &mut o); } + 14 => { let n = d.modn(12)?; let _ = z.ascend(n); } + 15 => { let _ = z.ascend_byte(); } + 16 => { let _ = z.ascend_until(); } + 17 => { let _ = z.ascend_until_branch(); } + 18 => { let _ = z.to_next_sibling_byte(); } + 19 => { let _ = z.to_prev_sibling_byte(); } + 20 => { let _ = z.to_next_step(); } + 21 => { let mut o = z.path().to_vec(); let _ = z.to_next_step_observed(&mut o); } + 22 => { let p = path(d)?; let _ = z.move_to_path(&p); } + 23 => z.reset(), + 24 => { let _ = (z.path_exists(), z.is_val(), z.child_count(), z.child_mask()); } + 25 => { let _ = (z.depth(), z.at_root(), z.focus_byte(), z.path().len()); } + 26 => { if val_count_ok() { let _ = z.val_count(); } } + 27 => { + // A sibling walk, the way callers enumerate children. + if z.descend_first_byte().is_some() { + let mut n = 0; + while n < WALK && z.to_next_sibling_byte().is_some() { n += 1; } + let _ = z.ascend_byte(); + } + } + 28 => { + if z.descend_last_byte().is_some() { + let mut n = 0; + while n < WALK && z.to_prev_sibling_byte().is_some() { n += 1; } + let _ = z.ascend_byte(); + } + } + 29 => { let mut n = 0; while n < WALK && z.to_next_step() { n += 1; } } + #[allow(deprecated)] + 30 => { let _ = z.is_value(); } + #[allow(deprecated)] + 31 => { let p = path(d)?; let _ = z.descend_to_value(&p); } + _ => { let b = byte(d)?; z.descend_to_byte(b); let _ = z.ascend_byte(); } + } + Some(()) +} + +/// A k for k-path iteration: `0..n`, or `1..n` unless known failures are included. +fn kpath_k(d: &mut Dec, n: usize) -> Option { + let k = d.modn(n)?; + Some(if k == 0 && !include_known() { 1 } else { k }) +} + +/// One iteration step, or a movement. +fn iter_step(d: &mut Dec, z: &mut Z) -> Option<()> { + let __op = d.modn(14)?; + note!("iter {__op} at {}", hex_path(z.path())); + match __op { + 0 => { let _ = z.to_next_val(); } + 1 => { let mut o = z.path().to_vec(); let _ = z.to_next_val_observed(&mut o); } + 2 => { let _ = z.descend_last_path(); } + 3 => { let mut o = z.path().len(); let _ = z.descend_last_path_observed(&mut o); } + 4 => { let k = kpath_k(d, 6)?; note!(" k={k}"); let _ = z.descend_first_k_path(k); } + 5 => { + let k = kpath_k(d, 6)?; + note!(" k={k}"); + if include_known() { let _ = z.to_next_k_path(k); } else if z.descend_first_k_path(k) { let _ = z.to_next_k_path(k); } + } + 6 => { + // A whole k-path walk. + let k = kpath_k(d, 5)?; + note!(" k={k}"); + if z.descend_first_k_path(k) { + let mut n = 0; + while n < WALK && z.to_next_k_path(k) { n += 1; } + } + } + 7 => { + let k = kpath_k(d, 5)?; + note!(" k={k}"); + let mut o = z.path().to_vec(); + if z.descend_first_k_path_observed(k, &mut o) { + let mut n = 0; + while n < WALK && z.to_next_k_path_observed(k, &mut o) { n += 1; } + } + } + 8 => { let mut n = 0; while n < WALK && z.to_next_val() { n += 1; } } + _ => move_step(d, z)?, + } + Some(()) +} + +/// A read program over a zipper with iteration and values. +fn read_program(d: &mut Dec, steps: &mut usize, z: &mut Z) -> Option<()> +where + V: Clone, + Z: ZipperMoving + ZipperPath + ZipperIteration + ZipperValues, +{ + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + match d.modn(8)? { + 0 => { let _ = z.val().cloned(); } + #[allow(deprecated)] + 1 => { let _ = z.value().cloned(); } + _ => iter_step(d, z)?, + } + } + Some(()) +} + +/// A read program over a zipper that moves and has values, but cannot iterate. +fn move_program(d: &mut Dec, steps: &mut usize, z: &mut Z) -> Option<()> +where + V: Clone, + Z: ZipperMoving + ZipperPath + ZipperValues, +{ + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + match d.modn(8)? { + 0 => { let _ = z.val().cloned(); } + _ => move_step(d, z)?, + } + } + Some(()) +} + +/// The read-only value accessors: references that outlive the borrow of the +/// zipper, and iteration that hands them out. +macro_rules! ro_extras { + ($d:expr, $z:expr) => {{ + let __e = $d.modn(4)?; + note!("ro_extras {__e}"); + match __e { + 0 => { let _ = $z.get_val().cloned(); } + 1 => { let p = path($d)?; let _ = $z.get_val_at(&p).cloned(); } + 2 => { let mut n = 0; while n < WALK && $z.to_next_get_val().is_some() { n += 1; } } + _ => { let mut o = $z.path().to_vec(); let _ = $z.to_next_get_val_observed(&mut o).cloned(); } + } + }}; +} + +/// Everything a subtrie-capable read zipper offers beyond moving: witnesses, +/// focus borrowing, forks, trie refs, buffer management. +macro_rules! sub_extras { + ($d:expr, $steps:expr, $z:expr) => {{ + let __e = $d.modn(12)?; + note!("sub_extras {__e}"); + match __e { + 0 => { let w = $z.witness(); let _ = $z.get_val_with_witness(&w).cloned(); } + 1 => { + let w = $z.witness(); + let mut n = 0; + while n < WALK && $z.to_next_get_val_with_witness(&w).is_some() { n += 1; } + } + 2 => { let _ = $z.make_map().val_count(); } + 3 => { let _ = $z.try_make_map().map(|m| m.val_count()); } + 4 => { let p = path($d)?; let t = $z.trie_ref_at_path(&p); trie_ref_ops($d, $steps, &t)?; } + 5 => { let _ = ($z.is_shared(), $z.shared_node_id()); } + 6 => { let _ = $z.get_focus(); let p = path($d)?; let _ = $z.get_focus_at(&p); let _ = $z.try_borrow_focus().is_some(); } + 7 => { let (a, b) = ($d.modn(80)?, $d.modn(24)?); $z.reserve_buffers(a, b); $z.prepare_buffers(); } + 8 => { let mut f = $z.fork_read_zipper(); read_program($d, $steps, &mut f)?; } + 9 => { let _ = ($z.origin_path().len(), $z.root_prefix_path().len()); } + 10 => { let _ = $z.get_trie_ref().child_count(); } + _ => { let _ = $z.native_subtries(); let _ = $z.trie_ref().is_some(); let p = path($d)?; let _ = $z.val_at(&p).cloned(); } + } + }}; +} + +/// Operations on a `TrieRef`, which is a zipper that cannot move. +fn trie_ref_ops(d: &mut Dec, steps: &mut usize, t: &T) -> Option<()> +where + V: Clone + Send + Sync + Unpin, + T: ZipperValuesAt + Zipper + ZipperInfallibleSubtries + ZipperConcrete, +{ + let n = d.modn(8)?; + for _ in 0..n { + tick(steps)?; + match d.modn(6)? { + 0 => { let _ = (t.val().cloned(), t.path_exists(), t.is_val(), t.child_count(), t.child_mask()); } + 1 => { let p = path(d)?; let _ = t.val_at(&p).cloned(); } + 2 => { let _ = t.make_map().val_count(); } + 3 => { let _ = (t.is_shared(), t.shared_node_id()); } + 4 => { let p = path(d)?; let _ = t.get_focus_at(&p); let _ = t.get_trie_ref().child_count(); } + _ => { let _ = t.try_borrow_focus().is_some(); } + } + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Writing +// --------------------------------------------------------------------------- + +/// One write, or a movement. `srcs` are clones of the maps taken when the +/// episode began: sources that share nodes with the destination, which is what +/// most callers of the algebraic operations hand in. +fn write_step(d: &mut Dec, z: &mut W, srcs: &[PathMap; NMAPS]) -> Option<()> +where + V: CrashValue, + W: ZipperWriting + ZipperMoving + ZipperPath + ZipperValues, +{ + let src = |d: &mut Dec| -> Option<&PathMap> { let i = d.modn(NMAPS)?; note!(" src map {i}"); Some(&srcs[i]) }; + let op = d.modn(40)?; + note!("write {op} at {}", hex_path(z.path())); + // Growth is refused below an oversized focus; everything else still runs. + let grows = matches!(op, 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 23 | 26 | 28 | 29 | 30); + if grows && z.val_count() > MAX_VALS { + return Some(()); + } + match op { + 0 => { let v = val(d)?; let _ = z.set_val(v); } + 1 => { let pr = d.boolean()?; note!(" pr={pr}"); let _ = z.remove_val(pr); } + 2 => { let v = val(d)?; if let Some(slot) = z.get_val_mut() { *slot = v; } } + 3 => { let v = val(d)?; let _ = z.get_val_or_set_mut(v).clone(); } + 4 => { let v = val(d)?; let _ = z.get_val_or_set_mut_with(|| v).clone(); } + 5 => { let _ = z.create_path(); } + 6 => { let _ = z.prune_path(); } + 7 => { let _ = z.prune_ascend(); } + 8 => { let pr = d.boolean()?; let _ = z.remove_branches(pr); } + 9 => { let s = src(d)?; let p = short_path(d)?; z.graft(&s.read_zipper_at_path(&p)); } + 10 => { let s = src(d)?; let p = short_path(d)?; z.graft_map(s.read_zipper_at_path(&p).make_map()); } + 11 => { let s = src(d)?; let p = short_path(d)?; let q = path(d)?; z.graft_src_at(&s.read_zipper_at_path(&p), &q); } + 12 => { let s = src(d)?; let p = short_path(d)?; let _ = z.join_into(&s.read_zipper_at_path(&p)); } + 13 => { let s = src(d)?; let p = short_path(d)?; let _ = z.join_map_into(s.read_zipper_at_path(&p).make_map()); } + 14 => { let s = src(d)?; let p = short_path(d)?; let pr = d.boolean()?; let _ = z.meet_into(&s.read_zipper_at_path(&p), pr); } + 15 => { let s = src(d)?; let p = short_path(d)?; let pr = d.boolean()?; let _ = z.subtract_into(&s.read_zipper_at_path(&p), pr); } + 16 => { let s = src(d)?; let p = short_path(d)?; let _ = z.restrict(&s.read_zipper_at_path(&p)); } + 17 => { let s = src(d)?; let p = short_path(d)?; let _ = z.restricting(&s.read_zipper_at_path(&p)); } + 18 => { + // Two independent sources, possibly different maps. + let (a, pa) = (src(d)?, short_path(d)?); + let (b, pb) = (src(d)?, short_path(d)?); + let _ = z.meet_2(&a.read_zipper_at_path(&pa), &b.read_zipper_at_path(&pb)); + } + 19 => { + // The source is a write zipper on a private copy, emptied by the join. + let s = src(d)?.clone(); + let p = short_path(d)?; + let pr = d.boolean()?; + note!(" pr={pr}"); + let mut sw = s.into_write_zipper(&p); + let _ = z.join_into_take(&mut sw, pr); + let _ = sw.into_map().val_count(); + } + 20 => { let k = kpath_k(d, 6)?; let pr = d.boolean()?; note!(" k={k} pr={pr}"); let _ = z.join_k_path_into(k, pr); } + 21 => { + let k = d.modn(6)?; + let pr = d.boolean()?; + if include_known() || (k > 0 && z.child_count() > 0) { + let _ = z.meet_k_path_into(k, pr); + } + } + #[allow(deprecated)] + 22 => { let k = kpath_k(d, 6)?; let _ = z.drop_head(k); } + 23 => { let p = path(d)?; let _ = z.insert_prefix(&p); } + 24 => { let n = d.modn(10)?; let _ = z.remove_prefix(n); } + 25 => { let pr = d.boolean()?; let _ = z.take_map(pr).map(|m| m.val_count()); } + 26 => { + let pr = d.boolean()?; + if let Some(m) = z.take_map(pr) { + let p = short_path(d)?; + let _ = z.descend_to_existing(&p); + z.graft_map(m); + } + } + 27 => { let m = mask(d)?; let pr = d.boolean()?; z.remove_unmasked_branches(m, pr); } + 28 => { let s = src(d)?; let p = short_path(d)?; let m = mask(d)?; let ru = d.boolean()?; note!(" ru={ru}"); z.graft_masked_branches(&s.read_zipper_at_path(&p), m, ru); } + 29 => { + let s = src(d)?; + let m = mask(d)?; + let ru = d.boolean()?; + let extra = d.modn(3)?; + // One map per set bit, as documented, plus possibly a few spare ones. + let maps: Vec> = m.iter().map(|b| s.read_zipper_at_path([b]).make_map()) + .chain((0..extra).map(|_| s.clone())).collect(); + z.graft_child_maps(m, maps, ru); + } + 30 => { + // `.paths` into the focus, from a serialized copy of a source. + let s = src(d)?; + let mut buf = Vec::new(); + if serialize_paths(s.read_zipper(), &mut buf).is_ok() { + let v = val(d)?; + let _ = deserialize_paths(&mut *z, Cursor::new(&buf[..]), v); + } + } + _ => move_step(d, z)?, + } + Some(()) +} + +/// A write program over one write zipper. +fn write_program(d: &mut Dec, steps: &mut usize, z: &mut W, srcs: &[PathMap; NMAPS]) -> Option<()> +where + V: CrashValue, + W: ZipperWriting + ZipperMoving + ZipperPath + ZipperValues, +{ + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + write_step(d, z, srcs)?; + } + Some(()) +} + +fn snapshot(maps: &[PathMap; NMAPS]) -> [PathMap; NMAPS] { + [maps[0].clone(), maps[1].clone(), maps[2].clone()] +} + +// --------------------------------------------------------------------------- +// Episodes: one zipper kind, created, driven, dropped +// --------------------------------------------------------------------------- + +/// Traces the paths holding values in each map. +fn note_maps(maps: &[PathMap; NMAPS]) { + if *TRACE.get_or_init(|| std::env::var_os("CRASH_TRACE").is_some()) { + for (i, mp) in maps.iter().enumerate() { + let mut rz = mp.read_zipper(); + let mut ps = vec![]; + if rz.is_val() { ps.push("_".to_string()); } + //Values, and the ends of dangling paths marked `~` + while ps.len() < 64 && rz.to_next_step() { + if rz.is_val() { ps.push(hex_path(rz.path())) } else if rz.child_count() == 0 { ps.push(format!("{}~", hex_path(rz.path()))) } + } + note!(" map {i}: {}", ps.join(" ")); + } + } +} + +fn read_episode(d: &mut Dec, st: &mut State) -> Option<()> { + let m = d.modn(NMAPS)?; + let p = short_path(d)?; + let map = st.maps[m].clone(); + let steps = &mut st.steps; + let __kind = d.modn(12)?; + note!("read episode {__kind} map {m} at {}", hex_path(&p)); + note_maps(&st.maps); + match __kind { + 0 => { + let mut z = map.read_zipper_at_path(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => ro_extras!(d, z), 1 => sub_extras!(d, steps, z), _ => iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + } + 1 => { + let mut z = map.read_zipper_at_borrowed_path(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => ro_extras!(d, z), 1 => sub_extras!(d, steps, z), _ => iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + } + 2 => { + let mut z = map.into_read_zipper(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + if d.boolean()? { sub_extras!(d, steps, z) } else { iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + let mut c = z.clone(); + read_program(d, steps, &mut c)?; + } + 3 => { + let t = map.trie_ref_at_path(&p); + trie_ref_ops(d, steps, &t)?; + let q = path(d)?; + let t2 = t.trie_ref_at_path(&q); + trie_ref_ops(d, steps, &t2)?; + if include_known() || t2.path_exists() { + let mut f = t2.fork_read_zipper(); + read_program(d, steps, &mut f)?; + } + } + 4 => { + // A prefix in front of a zipper, possibly rooted part way into it. + let prefix = path(d)?; + let mut z = PrefixZipper::new(&prefix[..], map.read_zipper_at_path(&p)); + if d.boolean()? { + let cut = d.modn(prefix.len() + 1)?; + note!(" prefix={} cut={cut}", hex_path(&prefix)); + let _ = z.set_root_prefix_path(&prefix[..cut]); + } + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => ro_extras!(d, z), 1 => sub_extras!(d, steps, z), _ => iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + } + 5 => { + let other = st.maps[d.modn(NMAPS)?].clone(); + let q = short_path(d)?; + let _stub = ValCountStub::new(); + let mut z = OverlayZipper::new(map.read_zipper_at_path(&p), other.read_zipper_at_path(&q)); + move_program(d, &mut st.steps, &mut z)?; + } + 6 => { + // Secondary factors are map-root zippers: see KNOWN_PRECONDITIONS. + let pick: Vec = (0..d.modn(4)?).map(|_| d.modn(NMAPS)).collect::>()?; + let others: Vec> = pick.iter().map(|&i| st.maps[i].clone()).collect(); + let more_i = d.modn(NMAPS)?; + note!(" product factors {pick:?}, more {more_i}"); + let more = st.maps[more_i].clone(); + let steps = &mut st.steps; + let mut z = ProductZipper::new(map.read_zipper_at_path(&p), others.iter().map(|o| o.read_zipper())); + if d.boolean()? { + note!(" new_factors"); + z.new_factors([more.read_zipper()]); + } + for _ in 0..d.modn(8)? { + tick(steps)?; + let __c = d.modn(4)?; + note!(" product op {__c} at {}", hex_path(z.path())); + match __c { + 0 => { let _ = (z.focus_factor(), z.factor_count(), z.path_indices().len()); } + 1 => { let w = z.witness(); let _ = z.get_val_with_witness(&w).cloned(); } + 2 => { let _ = (z.is_shared(), z.shared_node_id(), z.origin_path().len()); } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + } + 7 => { + let others: Vec> = (0..d.modn(4)?).map(|_| d.modn(NMAPS).map(|i| st.maps[i].clone())).collect::>()?; + let steps = &mut st.steps; + let _stub = ValCountStub::new(); + let mut z = ProductZipperG::new(map.read_zipper_at_path(&p), others.iter().map(|o| o.read_zipper_at_path(&[]))); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { + 0 => { let _ = (z.focus_factor(), z.factor_count(), z.path_indices().len()); } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + } + 8 => { + // Factors enrolled as the zipper walks: a map-root zipper on one of + // the maps, chosen from the path and the depth, a bounded number of times. + let pool = snapshot(&st.maps); + let steps = &mut st.steps; + let sel = d.u8()?; + let budget = d.modn(4)?; + let _stub = ValCountStub::new(); + let mut z = DependentProductZipperG::new_enroll( + map.read_zipper_at_path(&p), + budget, + move |left: usize, path: &[u8], depth: usize| { + let pick = (path.iter().fold(sel as usize, |a, &b| a.wrapping_mul(31).wrapping_add(b as usize)) + depth) % (NMAPS + 1); + if left == 0 || pick == NMAPS { + (left, None) + } else { + (left - 1, Some(pool[pick].clone().into_read_zipper(&[]))) + } + }, + ); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { + 0 => { let _ = (z.focus_factor(), z.path_indices().len()); } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + } + 9 => { + let mut z = EmptyZipper::new_at_path(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => { let _: Option<&V> = z.get_val(); } _ => iter_step(d, &mut z)? } + } + read_program::(d, steps, &mut z)?; + } + 10 => { + // A fork taken part way through a walk, driven while the parent lives. + let mut z = map.read_zipper_at_path(&p); + move_program(d, steps, &mut z)?; + let mut f = z.fork_read_zipper(); + read_program(d, steps, &mut f)?; + drop(f); + read_program(d, steps, &mut z)?; + } + _ => { + // The arena-compact form of the map. + let act = ArenaCompactTree::from_zipper(map.read_zipper(), |v: &V| v.to_u64()); + let mut z = act.read_zipper_at_path_u64(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(4)? { + 0 => { let _ = act.get_val_at(&p); } + 1 => { let mut n = 0; for _ in act.iter() { n += 1; if n > WALK { break } } } + 2 => { let mut f = z.fork_read_zipper(); read_program(d, steps, &mut f)?; } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + let mut u = act.read_zipper_at_path(&p); + read_program(d, steps, &mut u)?; + } + } + Some(()) +} + +fn write_episode(d: &mut Dec, st: &mut State) -> Option<()> { + let m = d.modn(NMAPS)?; + let p = path(d)?; + let srcs = snapshot(&st.maps); + let State { maps, steps } = st; + let __kind = d.modn(7)?; + note!("write episode {__kind} map {m} at {}", hex_path(&p)); + note_maps(maps); + match __kind { + 0 => { + let mut z = maps[m].write_zipper_at_path(&p); + write_program(d, steps, &mut z, &srcs)?; + } + 1 => { + let mut z = maps[m].write_zipper(); + z.descend_to(&p); + write_program(d, steps, &mut z, &srcs)?; + // A fork of the write zipper, read while the writer is still live. + { + let mut f = z.fork_read_zipper(); + read_program(d, steps, &mut f)?; + } + write_program(d, steps, &mut z, &srcs)?; + } + 2 => { + // An owned write zipper, turned back into the map afterwards. + let map = std::mem::take(&mut maps[m]); + let mut z = map.into_write_zipper(&p); + let r = write_program(d, steps, &mut z, &srcs); + maps[m] = z.into_map(); + r?; + } + 3 => { + let zh = maps[m].zipper_head(); + zh_program(d, steps, &zh, &srcs)?; + } + 4 => { + // A ZipperHead handed out by a write zipper at its focus. + let mut z = maps[m].write_zipper_at_path(&p); + write_program(d, steps, &mut z, &srcs)?; + { + let zh = z.zipper_head(); + zh_program(d, steps, &zh, &srcs)?; + } + write_program(d, steps, &mut z, &srcs)?; + } + 5 => { + let map = std::mem::take(&mut maps[m]); + let zh = map.into_zipper_head(&p); + let r = zh_program(d, steps, &zh, &srcs); + maps[m] = zh.into_map(); + r?; + } + _ => { + // A write zipper and a read zipper on the same map at once, through a head. + let zh = maps[m].zipper_head(); + let q = short_path(d)?; + if let (Ok(mut w), Ok(mut r)) = (zh.write_zipper_at_exclusive_path(&p), zh.read_zipper_at_path(&q)) { + for _ in 0..d.modn(EPISODE_STEPS)? { + tick(steps)?; + if d.boolean()? { write_step(d, &mut w, &srcs)? } else { iter_step(d, &mut r)? } + } + } + } + } + Some(()) +} + +/// Several zippers from one head, live at once and driven in turn. Requests +/// that conflict are refused with `Err`, which is the documented outcome. +fn zh_program<'t, V, H>(d: &mut Dec, steps: &mut usize, zh: &H, srcs: &[PathMap; NMAPS]) -> Option<()> +where + V: CrashValue, + H: ZipperCreation<'t, V>, +{ + let paths: Vec> = (0..3).map(|_| short_path(d)).collect::>()?; + let mut w0 = zh.write_zipper_at_exclusive_path(&paths[0]).ok(); + let mut w1 = zh.write_zipper_at_exclusive_path(&paths[1]).ok(); + let mut r0 = zh.read_zipper_at_path(&paths[2]).ok(); + let mut r1 = zh.read_zipper_at_borrowed_path(&paths[2]).ok(); + note!(" head zippers: w0 {} {}, w1 {} {}, r0/r1 {} {} {}", hex_path(&paths[0]), w0.is_some(), hex_path(&paths[1]), w1.is_some(), hex_path(&paths[2]), r0.is_some(), r1.is_some()); + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + let __which = d.modn(7)?; + note!(" head step {__which}"); + match __which { + 0 => if let Some(z) = w0.as_mut() { write_step(d, z, srcs)? }, + 1 => if let Some(z) = w1.as_mut() { write_step(d, z, srcs)? }, + 2 => if let Some(z) = r0.as_mut() { iter_step(d, z)? }, + 3 => if let Some(z) = r1.as_mut() { if d.boolean()? { sub_extras!(d, steps, z) } else { iter_step(d, z)? } }, + 4 => { drop(w0.take()); let q = short_path(d)?; w0 = zh.write_zipper_at_exclusive_path(&q).ok(); note!(" w0 = {} {}", hex_path(&q), w0.is_some()); } + 5 => { drop(r0.take()); let q = short_path(d)?; r0 = zh.read_zipper_at_path(&q).ok(); note!(" r0 = {} {}", hex_path(&q), r0.is_some()); } + _ => { w1 = None; r1 = None; } + } + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Whole-map operations +// --------------------------------------------------------------------------- + +fn map_op(d: &mut Dec, st: &mut State) -> Option<()> { + let m = d.modn(NMAPS)?; + let __op = d.modn(20)?; + note!("map {__op}"); + match __op { + 0 => { let p = path(d)?; let v = val(d)?; if st.small(m) { let _ = st.maps[m].set_val_at(&p, v); } } + 1 => { let p = path(d)?; let v = val(d)?; if st.small(m) { let _ = st.maps[m].insert(&p, v); } } + 2 => { let p = path(d)?; let pr = d.boolean()?; let _ = st.maps[m].remove_val_at(&p, pr); } + 3 => { let p = path(d)?; let _ = st.maps[m].remove(&p); } + 4 => { + let p = path(d)?; + let v = val(d)?; + match d.modn(3)? { + 0 => { if let Some(slot) = st.maps[m].get_val_mut_at(&p) { *slot = v; } } + 1 => { let _ = st.maps[m].get_val_or_set_mut_at(&p, v).clone(); } + _ => { let _ = st.maps[m].get_val_or_set_mut_with_at(&p, || v).clone(); } + } + } + 5 => { + let p = path(d)?; + let map = &st.maps[m]; + let _ = (map.get(&p).cloned(), map.contains(&p), map.path_exists_at(&p), map.is_empty()); + #[allow(deprecated)] + let _ = map.contains_path(&p); + } + 6 => { let p = path(d)?; let _ = st.maps[m].create_path(&p); } + 7 => { let p = path(d)?; let _ = st.maps[m].prune_path(&p); } + 8 => { let p = path(d)?; let pr = d.boolean()?; let _ = st.maps[m].remove_branches_at(&p, pr); } + 9 => { let to = d.modn(NMAPS)?; st.maps[to] = st.maps[m].clone(); } + 10 => { + let (b, to) = (d.modn(NMAPS)?, d.modn(NMAPS)?); + let (x, y) = (&st.maps[m], &st.maps[b]); + let r = match d.modn(8)? { + 0 => x.join(y), + 1 => x.meet(y), + 2 => x.subtract(y), + 3 => x.restrict(y), + 4 => resolve(x.pjoin(y), x, y), + 5 => resolve(x.pmeet(y), x, y), + 6 => resolve(x.psubtract(y), x, y), + _ => y.restrict(x), + }; + if r.val_count() <= MAX_VALS { st.maps[to] = r; } + } + 11 => { + // A comb: many children under one prefix, over a chosen alphabet, so + // list nodes split and dense nodes form. + let prefix = path(d)?; + let n = d.modn(40)?; + let v: V = val(d)?; + if st.small(m) { + for _ in 0..n { + let mut k = prefix.clone(); + k.push(byte(d)?); + k.extend(short_path(d)?); + let _ = st.maps[m].set_val_at(&k, v.clone()); + } + } + } + 12 => { + let to = d.modn(NMAPS)?; + let rebuilt: PathMap = st.maps[m].iter().take(MAX_VALS).map(|(k, v)| (k, v.clone())).collect(); + let _ = st.maps[m].clone().into_iter().take(WALK).count(); + st.maps[to] = rebuilt; + } + 13 => { let _ = st.maps[m].merkleize(); } + 14 => { let to = d.modn(NMAPS)?; st.maps[to] = ana(d, &st.maps)?; } + 15 => cata(d, &st.maps[m])?, + 16 => { + // `.paths` round trip, then decoding a damaged copy, which must fail + // cleanly or decode something, not panic. + let p = short_path(d)?; + let mut buf = Vec::new(); + let _ = serialize_paths(st.maps[m].read_zipper_at_path(&p), &mut buf); + let to = d.modn(NMAPS)?; + let q = short_path(d)?; + let v = val(d)?; + if st.small(to) { + let _ = deserialize_paths(st.maps[to].write_zipper_at_path(&q), Cursor::new(&buf[..]), v); + } + let flips = d.modn(4)?; + let cut = d.u8()? as usize; + let mut bad = buf.clone(); + for _ in 0..flips { + if bad.is_empty() { break } + let i = d.u8()? as usize * 7 % bad.len(); + bad[i] ^= d.u8()? | 1; + } + if cut & 1 == 1 { bad.truncate(cut % (bad.len() + 1)); } + decode_bounded(&bad); + } + 17 => { + // Decoding arbitrary bytes from the input. + let len = d.modn(64)?; + let raw: Vec = (0..len).map(|_| d.u8()).collect::>()?; + decode_bounded(&raw); + } + 18 => { + let p = path(d)?; + let map = &st.maps[m]; + let _ = (map.is_shared(), map.shared_node_id(), map.val().cloned(), map.val_at(&p).cloned(), map.goat_val_count()); + } + _ => { + // A clone mutated while the original is read: copy-on-write. + let mut c = st.maps[m].clone(); + let p = path(d)?; + let v = val(d)?; + if st.small(m) { + let _ = c.set_val_at(&p, v); + let q = path(d)?; + let _ = c.remove_branches_at(&q, d.boolean()?); + } + let _ = (st.maps[m].val_count(), c.val_count()); + } + } + Some(()) +} + +fn decode_bounded(bytes: &[u8]) { + let mut n = 0usize; + let _ = for_each_deserialized_path(Cursor::new(bytes), |_, _| { + n += 1; + if n > MAX_VALS { Err(std::io::Error::other("enough")) } else { Ok(()) } + }); +} + +/// A trie built by an anamorphism. `W` is the remaining depth; the children +/// pushed at each step come from the input, sorted and de-duplicated by first +/// byte as `TrieBuilder` requires, and sometimes grafted from one of the maps. +fn ana(d: &mut Dec, maps: &[PathMap; NMAPS]) -> Option> { + let depth = d.modn(5)?; + let spec: Vec = (0..32).map(|_| d.u8()).collect::>()?; + let graft_from = maps[d.modn(NMAPS)?].clone(); + let mut calls = 0usize; + Some(PathMap::::new_from_ana(depth, |left: usize, v: &mut Option, children, path: &[u8]| { + calls += 1; + let h = spec[(calls + path.len()) % spec.len()]; + if h & 1 == 1 { + *v = Some(V::from_byte(h)); + } + if left == 0 || calls > 256 { + return; + } + let fan = (h >> 1) as usize % 4; + let mut firsts: Vec<(u8, Vec)> = (0..fan) + .map(|i| { + let s = spec[(calls * 7 + i * 3) % spec.len()]; + let first = if s & 0x80 == 0 { s % 4 } else { s }; + let tail = (0..(s as usize % 3)).map(|j| spec[(i + j + calls) % spec.len()] % 4).collect(); + (first, tail) + }) + .collect(); + firsts.sort_by_key(|(b, _)| *b); + firsts.dedup_by_key(|(b, _)| *b); + for (i, (first, tail)) in firsts.into_iter().enumerate() { + let rz = graft_from.read_zipper_at_path([first]); + if (h >> 4) as usize % 4 == i && rz.child_count() > 0 { + children.graft_at_byte(first, &rz); + } else { + let mut sub = vec![first]; + sub.extend(tail); + children.push(&sub, left - 1); + } + } + })) +} + +/// The four catamorphism flavours, some stopping early with an error, and `hash`. +fn cata(d: &mut Dec, map: &PathMap) -> Option<()> { + let p = short_path(d)?; + let stop = d.modn(32)?; + let z = map.read_zipper_at_path(&p); + match d.modn(9)? { + 0 => { let _: usize = z.into_cata_side_effect(|_m, ch: &mut [usize], v: Option<&V>, path: &[u8]| ch.iter().sum::() + v.is_some() as usize + path.len()); } + 1 => { let _: usize = z.into_cata_jumping_side_effect(|_m, ch: &mut [usize], jump, v: Option<&V>, _p: &[u8]| ch.iter().sum::() + jump + v.is_some() as usize); } + 2 => { let _: usize = z.into_cata_cached(|_m, ch: &mut [usize], v: Option<&V>| ch.iter().sum::() + v.map_or(0, |v| v.to_u64() as usize)); } + 3 => { let _: usize = z.into_cata_jumping_cached(|_m, ch: &mut [usize], v: Option<&V>, sub: &[u8]| ch.iter().sum::() + sub.len() + v.is_some() as usize); } + 4 => { + let mut n = 0; + let _: Result = z.into_cata_side_effect_fallible(|_m, ch: &mut [usize], _v: Option<&V>, _p: &[u8]| { n += 1; if n > stop { Err(()) } else { Ok(ch.len()) } }); + } + 5 => { + let mut n = 0; + let _: Result = z.into_cata_jumping_side_effect_fallible(|_m, ch: &mut [usize], j, _v: Option<&V>, _p: &[u8]| { n += 1; if n > stop { Err(()) } else { Ok(ch.len() + j) } }); + } + 6 => { let _: Result = z.into_cata_cached_fallible(|m: &ByteMask, ch: &mut [usize], _v: Option<&V>| if m.count_bits() > stop % 4 { Err(()) } else { Ok(ch.len()) }); } + 7 => { let _: Result = z.into_cata_jumping_cached_fallible(|_m, ch: &mut [usize], _v: Option<&V>, sub: &[u8]| if sub.len() > stop { Err(()) } else { Ok(ch.len()) }); } + _ => { let _ = z.hash(); } + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +fn seed(d: &mut Dec) -> Option> { + let mut st = State { maps: Default::default(), steps: 0 }; + for m in 0..NMAPS { + for _ in 0..d.modn(12)? { + let p = path(d)?; + let v = val(d)?; + note!("seed map {m}: {} = {v:?}", hex_path(&p)); + st.maps[m].set_val_at(&p, v); + } + } + Some(st) +} + +fn run_typed(d: &mut Dec) { + let Some(mut st) = seed::(d) else { return }; + let _ = (|| -> Option<()> { + loop { + tick(&mut st.steps)?; + match d.modn(8)? { + 0 | 1 | 2 => map_op(d, &mut st)?, + 3 | 4 => read_episode(d, &mut st)?, + _ => write_episode(d, &mut st)?, + } + } + })(); + // Tear down in a varied order: dropping shared nodes is part of the surface. + let order = d.u8().unwrap_or(0) as usize; + for i in 0..NMAPS { + let m = (order + i) % NMAPS; + let _ = st.maps[m].val_count(); + st.maps[m] = PathMap::new(); + } +} + +/// Run one input. Returns normally unless the crate panics, aborts or hangs. +/// +/// Anything this reaches with `debug_assertions` off is also reachable with +/// them on; build with them on to turn internal invariant checks into failures. +pub fn run(bytes: &[u8]) { + let mut d = Dec { bytes, pos: 0 }; + match d.u8().map(|b| b % 4) { + None => {} + Some(0) | Some(1) => run_typed::(&mut d), + Some(2) => run_typed::<()>(&mut d), + Some(_) => { + if d.boolean().unwrap_or(false) { run_typed::(&mut d) } else { run_typed::(&mut d) } + } + } +} diff --git a/differential/src/harness.rs b/differential/src/harness.rs index 0b0d72d9..ce3b0361 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -145,31 +145,24 @@ pub fn fingerprint + ZipperAbso /// * `skip:act` — the ACT read source cannot be a merge source /// (`ZipperInfallibleSubtries` is not implemented for it) or does not /// implement the trait the op needs. -/// * `skip:at-root` — `to_next`/`to_prev_sibling_byte` at the zipper root, -/// where the native read zipper escapes its own root. /// * `skip:k0` — a degenerate `k = 0`. /// * `skip:empty-focus` — the focus has nothing below it, where the op's /// behaviour is a function of node materialisation rather than trie state. -/// * `skip:empty-path` — `insert_prefix("")`, which destroys the subtrie. /// * `skip:off-root-prune` — a prune on a write zipper not rooted at the map /// root, where the depth pruned is a function of internal node layout. -/// * `skip:quarantined` — the op is disabled outright (op 54). /// /// Each is recorded in lean/FINDINGS.md and commented at its site. pub const SKIP_ACT: &str = "skip:act"; -pub const SKIP_AT_ROOT: &str = "skip:at-root"; pub const SKIP_K0: &str = "skip:k0"; pub const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; -pub const SKIP_EMPTY_PATH: &str = "skip:empty-path"; pub const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; -pub const SKIP_QUARANTINED: &str = "skip:quarantined"; /// Does the focus have no descendants at all? /// /// Several return values (`remove_branches`, `restricting`, `join_map_into`, -/// `take_map`, `restrict`) hinge on whether an *empty node* happens to be -/// materialised at the focus rather than on the logical state, so the harness -/// masks them here. See lean/README.md. +/// `take_map`) hinge on whether an *empty node* happens to be materialised at +/// the focus rather than on the logical state, so the harness masks them here. +/// `restrict` used to be in that list and no longer is. See lean/README.md. pub fn focus_node_empty(z: &Z) -> bool { z.child_count() == 0 } @@ -224,9 +217,8 @@ pub trait ReadSource: fn do_graft>(&self, _wz: &mut W) -> bool { false } fn do_graft_masked>(&self, _wz: &mut W, _m: ByteMask, _ru: bool) -> bool { false } - /// Currently unused: op 54 is quarantined (lean/FINDINGS.md #15). Kept so - /// the op can be re-enabled with one line once the method is fixed. - #[allow(dead_code)] + /// `graft_child_maps` fed this zipper's own child subtries (op 54), which must + /// agree with `graft_masked_branches` on the same mask. fn do_graft_child_maps>(&self, _wz: &mut W, _m: ByteMask, _ru: bool) -> bool { false } /// `meet_2` needs two sources; the second is this zipper moved to `path`. fn do_meet_2>(&self, _wz: &mut W, _path: &[u8]) -> Option { None } @@ -550,23 +542,16 @@ pub fn run_ops( } 11 => { let t = get!(d.modn(2)); - // Skipped at the zipper root: the native ReadZipper escapes - // its own root there. See `Zip.toNextSiblingByte`. - if tgt!(t, wz, *rz, z, z.at_root()) { - ("to_next_sibling_byte", SKIP_AT_ROOT.to_string()) - } else { - let r = tgt!(t, wz, *rz, z, z.to_next_sibling_byte()); - ("to_next_sibling_byte", show_byte_opt(r)) - } + // Formerly skipped at the zipper root, where the native + // ReadZipper escaped its own root (FINDINGS.md #3). Fixed: + // the root case is now compared like any other. + let r = tgt!(t, wz, *rz, z, z.to_next_sibling_byte()); + ("to_next_sibling_byte", show_byte_opt(r)) } 12 => { let t = get!(d.modn(2)); - if tgt!(t, wz, *rz, z, z.at_root()) { - ("to_prev_sibling_byte", SKIP_AT_ROOT.to_string()) - } else { - let r = tgt!(t, wz, *rz, z, z.to_prev_sibling_byte()); - ("to_prev_sibling_byte", show_byte_opt(r)) - } + let r = tgt!(t, wz, *rz, z, z.to_prev_sibling_byte()); + ("to_prev_sibling_byte", show_byte_opt(r)) } 13 => { let t = get!(d.modn(2)); @@ -747,7 +732,9 @@ pub fn run_ops( ("join_map_into", s) } 38 => { - let _pr = get!(d.boolean()); // decoded for stream alignment; see `no_prune` + // `prune = true` is best-effort (nodes shared with the source may be + // left unpruned; see `Zip.meetInto`), so only `prune = false` is compared. + let _pr = get!(d.boolean()); ("meet_into", show_status_opt((*rz).do_meet_into(&mut wz, no_prune))) } 39 => { @@ -758,14 +745,11 @@ pub fn run_ops( ) } 40 => { - let leaky = focus_node_empty(&wz); + // The status used to be masked to `?` at a focus with + // nothing below it (FINDINGS.md #8). No longer: it tracks + // the spec over the whole sweep. let st = (*rz).do_restrict(&mut wz); - let s = if leaky && st.is_some() { - "?".to_string() - } else { - show_status_opt(st) - }; - ("restrict", s) + ("restrict", show_status_opt(st)) } 41 => { // Skipped when either side has nothing below its focus; see @@ -786,24 +770,20 @@ pub fn run_ops( if k == 0 { ("join_k_path_into", SKIP_K0.to_string()) } else { - // The bool leaks node materialisation; see FINDINGS.md #8. + // The bool was masked to `?` on an empty resulting + // focus as a suspected materialisation leak + // (FINDINGS.md #8). It is not one: unmasked it tracks + // the spec over the whole sweep. let r = wz.join_k_path_into(k, no_prune); - let s = if focus_node_empty(&wz) { - "?".to_string() - } else { - show_bool(r).to_string() - }; - ("join_k_path_into", s) + ("join_k_path_into", show_bool(r).to_string()) } } 43 => { let p = get!(d.path(6)); - // `insert_prefix("")` destroys the subtrie in pathmap 0.3.1. - if p.is_empty() { - ("insert_prefix", SKIP_EMPTY_PATH.to_string()) - } else { - ("insert_prefix", show_bool(wz.insert_prefix(&p)).to_string()) - } + // The empty prefix was skipped while it destroyed the + // subtrie (FINDINGS.md #4); fixed upstream, so it is + // compared like any other prefix. + ("insert_prefix", show_bool(wz.insert_prefix(&p)).to_string()) } 44 => { let n = get!(d.modn(6)); @@ -823,7 +803,7 @@ pub fn run_ops( } 46 => { let k = get!(d.modn(4)); - let _pr = get!(d.boolean()); // decoded for stream alignment; see `no_prune` + let _pr = get!(d.boolean()); // decoded for stream alignment; see op 38 // `meet_k_path_into` spins forever when the focus has no // children, and escapes the focus subtree when k == 0. // See `Zip.meetKPathUnspecified`. @@ -926,12 +906,14 @@ pub fn run_ops( let mut canon: Vec = m.clone(); canon.sort_unstable(); canon.dedup(); - // Skipped outright: `graft_child_maps` is broken three ways - // (lean/FINDINGS.md #15) and the node representations it - // leaves behind degrade the AlgebraicStatus that *later* - // operations report, contaminating the rest of the run. - let _ = (mask, ru, &canon); - ("graft_child_maps", SKIP_QUARANTINED.to_string()) + // Fed the source's own child subtries, so it must agree with + // `graft_masked_branches` on the same mask (op 53). + let s = if (*rz).do_graft_child_maps(&mut wz, mask, ru) { + format!("{}:{}", hex_path(&canon), show_bool(ru)) + } else { + SKIP_ACT.to_string() + }; + ("graft_child_maps", s) } 55 => { let p = get!(d.path(6)); diff --git a/differential/src/lib.rs b/differential/src/lib.rs index 661cf803..72f3c0a7 100644 --- a/differential/src/lib.rs +++ b/differential/src/lib.rs @@ -7,14 +7,27 @@ //! table are a contract shared with `lean/PathMapModel/Fuzz.lean`. //! * [`server`] is the resident-process protocol the driver speaks. //! * [`repro`] turns an input back into standalone `pathmap` calls. +//! * [`source`] generates random inputs by index, for the in-process front ends. +//! * [`crash`] is a second op table that only has to not crash; see `bin/crash_fuzz.rs`. //! * [`act`] is the `ArenaCompactTree` read source behind `act_trace`. +//! * [`reference`] is a second executable model: a Rust transcription of the +//! same Lean specification, sharing no code with `pathmap`. It is what +//! `bin/reference.rs` and `bin/in_process.rs` drive, and what +//! `differential.py --model` validates against the Lean oracle. pub mod act; pub mod harness; +pub mod reference; pub mod repro; pub mod server; +pub mod source; +pub mod crash; pub use act::*; pub use harness::*; pub use repro::*; pub use server::*; + +// `reference` is deliberately *not* glob re-exported: it defines its own +// `PathMap`, `run`, `hex_path` and `show_val`, which are the model's and must +// never be confused with the crate's. Name it by path. diff --git a/differential/src/reference/basic.rs b/differential/src/reference/basic.rs new file mode 100644 index 00000000..2bb3ea87 --- /dev/null +++ b/differential/src/reference/basic.rs @@ -0,0 +1,245 @@ +//! `Basic.lean` — the vocabulary the rest of the model is written in. +//! +//! * [`path`] — keys, and the prefix order on them. +//! * [`ValOps`] / [`ValRes`] — the fragment of `Lattice` / `DistributiveLattice` +//! that the trie operations actually consume. +//! * [`ByteMask`] — the 256-bit child mask. +//! * [`AlgStatus`] — `pathmap::ring::AlgebraicStatus`. + +// =========================================================================== +// Basic.lean — paths +// =========================================================================== + +/// Path helpers. +/// +/// `Path.isPrefixOf`, `Path.stripPrefix` and `Path.lt` are all provided by +/// `[u8]` itself — `starts_with`, `strip_prefix` and `Ord` respectively — and +/// the slice order is exactly the model's `Path.lt`, so they are not restated. +pub mod path { + /// `Path.prefixes`: every prefix of `p`, shortest first: `[]`, `p[..1]`, ..., `p`. + pub fn prefixes(p: &[u8]) -> impl Iterator { + (0..=p.len()).map(move |n| &p[..n]) + } + + /// `Path.properPrefixes`: every *proper* prefix of `p`, shortest first. + /// + /// Used to phrase "the nearest ancestor such that ..." as a filter over + /// ancestors rather than as a loop that walks upward. + pub fn proper_prefixes(p: &[u8]) -> impl Iterator { + (0..p.len()).map(move |n| &p[..n]) + } + + /// The deepest proper prefix of `p` satisfying `pred`, if any. + /// + /// `proper_prefixes` is shortest-first and the Lean model takes the *last* + /// qualifying element; scanning from the deep end and stopping at the first + /// hit is the same answer without building the list. + pub fn deepest_proper_prefix(p: &[u8], mut pred: impl FnMut(&[u8]) -> bool) -> Option<&[u8]> { + (0..p.len()).rev().map(|n| &p[..n]).find(|a| pred(a)) + } + + /// `p ++ q`, as an owned path. + pub fn cat(p: &[u8], q: &[u8]) -> Vec { + let mut r = Vec::with_capacity(p.len() + q.len()); + r.extend_from_slice(p); + r.extend_from_slice(q); + r + } + + /// `p ++ [b]`, as an owned path. + pub fn push(p: &[u8], b: u8) -> Vec { + let mut r = Vec::with_capacity(p.len() + 1); + r.extend_from_slice(p); + r.push(b); + r + } +} + +// =========================================================================== +// Basic.lean — value operations +// =========================================================================== + +/// `pathmap::ring::AlgebraicResult` at the *value* level: either a freshly +/// computed element, an assertion that the result is already one of the operands +/// (`Identity`, with the `SELF_IDENT` / `COUNTER_IDENT` mask spelled out as two +/// flags), or annihilation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ValRes { + /// `AlgebraicResult::Element` + Elem(V), + /// `AlgebraicResult::Identity(mask)`; the flags are `SELF_IDENT`, `COUNTER_IDENT`. + Identity(bool, bool), + /// `AlgebraicResult::None` + None, +} + +impl ValRes { + /// Turn a `ValRes` into the value it denotes, given both operands. + pub fn resolve(self, a: &V, b: &V) -> Option + where + V: Clone, + { + match self { + ValRes::Elem(v) => Some(v), + ValRes::Identity(s, _) => Some(if s { a.clone() } else { b.clone() }), + ValRes::None => None, + } + } +} + +/// A value type together with the algebraic operations `pathmap` requires of it. +/// +/// These return [`ValRes`] rather than plain values because `pathmap` reports +/// `AlgebraicStatus` to the caller, and the status depends on *which variant* +/// the value operation returned, not on whether the stored value changed. +/// `u64`'s `pjoin`, for instance, returns `Identity(SELF_IDENT)` rather than +/// `Element` of the value it selected, so a join reports that nothing changed. +pub trait ValOps { + /// `Lattice::pjoin` + fn pjoin(&self, a: &V, b: &V) -> ValRes; + /// `Lattice::pmeet` + fn pmeet(&self, a: &V, b: &V) -> ValRes; + /// `DistributiveLattice::psubtract` + fn psub(&self, a: &V, b: &V) -> ValRes; + /// Decidable equality on values. + fn beq(&self, a: &V, b: &V) -> bool; +} + +/// The instance `pathmap` provides for `u64` (see `impl Lattice for u64` in +/// `src/ring.rs`). +/// +/// Both `pjoin` and `pmeet` return `Identity(SELF_IDENT)`: they are *left-biased +/// projections* that ignore the counterpart value entirely. `psubtract` +/// annihilates only when the two values are equal, and otherwise returns +/// `Identity(SELF_IDENT)`: subtracting a value that is not there leaves the +/// destination alone, and says so. This is the instance the differential fuzz +/// target uses, so the model reproduces it exactly rather than assuming a "real" +/// lattice. +#[derive(Clone, Copy, Debug, Default)] +pub struct U64Ops; + +impl ValOps for U64Ops { + fn pjoin(&self, _: &u64, _: &u64) -> ValRes { + ValRes::Identity(true, false) + } + fn pmeet(&self, _: &u64, _: &u64) -> ValRes { + ValRes::Identity(true, false) + } + fn psub(&self, a: &u64, b: &u64) -> ValRes { + if a == b { ValRes::None } else { ValRes::Identity(true, false) } + } + fn beq(&self, a: &u64, b: &u64) -> bool { + a == b + } +} + +// =========================================================================== +// Basic.lean — byte masks +// =========================================================================== + +/// `pathmap`'s 256-bit child mask, modelled as the ascending, duplicate-free +/// list of its set bytes — the only thing the API observes about it. +/// +/// Deliberately *not* `pathmap::utils::ByteMask`: a model that borrowed the +/// crate's mask could not detect a bug in it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ByteMask(Vec); + +impl ByteMask { + /// Sort/dedup a byte list into canonical form — `ByteMask.ofList`. + pub fn of_list(bs: impl IntoIterator) -> Self { + let mut v: Vec = bs.into_iter().collect(); + v.sort_unstable(); + v.dedup(); + ByteMask(v) + } + + /// Every byte set — the mask a `remove_unmasked_branches` no-op needs. + pub fn full() -> Self { + ByteMask((0..=255u8).collect()) + } + + /// The set bytes, ascending. + pub fn bytes(&self) -> &[u8] { + &self.0 + } + + /// `ByteMask::count_bits`. + pub fn count_bits(&self) -> usize { + self.0.len() + } + + /// Whether `b` is set. + pub fn contains(&self, b: u8) -> bool { + self.0.binary_search(&b).is_ok() + } + + /// `ByteMask::indexed_bit::` — the `idx`-th set byte in ascending order. + pub fn indexed_bit(&self, idx: usize) -> Option { + self.0.get(idx).copied() + } + + /// `ByteMask::next_bit` — the least set byte strictly greater than `b`. + pub fn next_bit(&self, b: u8) -> Option { + self.0.iter().copied().find(|&x| x > b) + } + + /// `ByteMask::prev_bit` — the greatest set byte strictly less than `b`. + pub fn prev_bit(&self, b: u8) -> Option { + self.0.iter().copied().rev().find(|&x| x < b) + } +} + +// =========================================================================== +// Basic.lean — algebraic status +// =========================================================================== + +/// `pathmap::ring::AlgebraicStatus`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AlgStatus { + /// `self` holds the operation's output. + Element, + /// `self` was not modified by the operation. + Identity, + /// `self` was annihilated and is now empty. + None, +} + +impl AlgStatus { + /// `AlgebraicStatus::merge`, transcribed from `src/ring.rs`. + pub fn merge(a: AlgStatus, b: AlgStatus, self_none: bool, b_none: bool) -> AlgStatus { + match (a, b) { + (AlgStatus::None, AlgStatus::None) => AlgStatus::None, + (AlgStatus::None, AlgStatus::Element) => AlgStatus::Element, + (AlgStatus::None, AlgStatus::Identity) => { + if self_none { AlgStatus::Identity } else { AlgStatus::Element } + } + (AlgStatus::Identity, AlgStatus::Element) => AlgStatus::Element, + (AlgStatus::Identity, AlgStatus::Identity) => AlgStatus::Identity, + (AlgStatus::Identity, AlgStatus::None) => { + if b_none { AlgStatus::Identity } else { AlgStatus::Element } + } + (AlgStatus::Element, _) => AlgStatus::Element, + } + } + + /// The `AlgebraicStatus` a [`ValRes`] induces. + pub fn of_val_res(r: &ValRes) -> AlgStatus { + match r { + ValRes::Elem(_) => AlgStatus::Element, + ValRes::Identity(_, _) => AlgStatus::Identity, + ValRes::None => AlgStatus::None, + } + } +} + +impl std::fmt::Display for AlgStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + AlgStatus::Element => "Element", + AlgStatus::Identity => "Identity", + AlgStatus::None => "None", + }) + } +} + diff --git a/differential/src/reference/check.rs b/differential/src/reference/check.rs new file mode 100644 index 00000000..c819eb08 --- /dev/null +++ b/differential/src/reference/check.rs @@ -0,0 +1,531 @@ +//! `Check.lean` — the checks, plus a randomised sweep with no Lean counterpart. +//! +//! `Check.lean`'s `#guard`s are evaluated by the Lean compiler on every build; +//! here they are `#[test]`s, which is why the example carries `test = true`. + +// =========================================================================== +// Check.lean — build-time checks +// =========================================================================== + +#[cfg(test)] +mod tests { + //! Two kinds of check, transcribed from `lean/PathMapModel/Check.lean`, where + //! they are `#guard`s evaluated by the Lean compiler on every build: + //! + //! * **Regression fixtures** transcribed from `pathmap`'s own unit tests + //! (`src/write_zipper.rs`). These pin the model to observed crate behaviour + //! for the operations whose semantics are hardest to read off the source — + //! pruning and `drop_head`. + //! * **Law checks**: the metamorphic properties from [`crate::reference::laws`], + //! evaluated over a battery of maps chosen to cover branch points, + //! single-child runs, values at interior nodes, dangling paths, and the + //! empty map. + + use crate::reference::basic::U64Ops; + use crate::reference::laws::*; + use crate::reference::map; + use crate::reference::pathmap::PathMap; + use crate::reference::zipper::Zip; + + type T = PathMap; + const OPS: U64Ops = U64Ops; + + /// Build a map from a list of path/value pairs. + fn mk(entries: &[(&[u8], u64)]) -> T { + let mut t = PathMap::empty(); + for (p, v) in entries { + t.set_val(p, *v); + } + t + } + + fn zip_at(t: &T, root: &[u8], path: &[u8]) -> Zip { + Zip::at_path(t.clone(), root, path) + } + + // -- fixtures ----------------------------------------------------------- + + /// Branching at the root and at depth 1, with a value at an interior node. + fn f_branch() -> T { + mk(&[(&[], 0), (&[0], 1), (&[0, 0], 2), (&[0, 1], 3), (&[1], 4)]) + } + /// A single-child run: the shape `descend_until` / `ascend_until` care about. + fn f_run() -> T { + mk(&[(&[0, 0, 0, 0], 7)]) + } + /// Overlapping prefixes without a root value. + fn f_side() -> T { + mk(&[(&[0, 0], 9), (&[0, 2], 8), (&[2], 7)]) + } + /// The empty map. + fn f_empty() -> T { + PathMap::empty() + } + /// A dangling path: `[0,1,2]` exists but carries no value. + fn f_dangle() -> T { + let mut t = mk(&[(&[0], 5)]); + t.add_path(&[0, 1, 2]); + t + } + /// Two values under a shared 2-byte prefix, plus a deeper third. + fn f_deep() -> T { + mk(&[(&[1, 1, 0], 1), (&[1, 1, 1], 2), (&[1, 1, 1, 3], 3)]) + } + + fn fixtures() -> Vec { + vec![f_branch(), f_run(), f_side(), f_empty(), f_dangle(), f_deep()] + } + + /// Focus positions worth probing in each fixture. + fn probes() -> Vec> { + vec![ + vec![], + vec![0], + vec![1], + vec![0, 0], + vec![0, 1], + vec![1, 1], + vec![0, 1, 2], + vec![3], + ] + } + + fn all_zips() -> Vec> { + let mut zs = Vec::new(); + for t in fixtures() { + for p in probes() { + zs.push(Zip::at_path(t.clone(), &[], &p)); + } + } + zs + } + + // -- regression fixtures from `src/write_zipper.rs` ---------------------- + + /// `write_zipper_prune_path_test2`, first phase: removing the value at + /// `[0,0,1,0,0]` and pruning removes exactly 3 bytes, back to the branch at + /// `[0,0]`. + #[test] + fn prune_path_test2() { + let t = mk(&[(&[0], 0), (&[0, 0, 0], 0), (&[0, 0, 1, 0, 0], 0)]); + let mut z = zip_at(&t, &[], &[0, 0, 1, 0, 0]); + z.remove_val(false); + assert!(z.path_exists(), "remove_val(false) leaves the location dangling"); + assert_eq!(z.prune_path(), 3); + assert!(!z.path_exists()); + assert_eq!(zip_at(&t, &[], &[0, 0]).child_count(), 2); + } + + /// Same test, later phase: a chain with every value removed prunes all the + /// way to the map root (7 bytes), and the root itself survives. + #[test] + fn prune_path_test2b() { + let mut t = mk(&[(&[0, 0, 0, 1, 2, 3, 4], 0)]); + t.remove_val(&[0, 0, 0, 1, 2, 3, 4]); + + let mut z = zip_at(&t, &[], &[0, 0, 0, 1, 2, 3, 4]); + assert_eq!(z.prune_path(), 7); + z.reset(); + assert!(z.path_exists(), "the map root always survives"); + + // Pruning is a no-op above the dangling tip (the location still has a + // child) and below it (the location does not exist). + assert_eq!(zip_at(&t, &[], &[0, 0, 0, 1, 2, 3]).prune_path(), 0); + assert_eq!(zip_at(&t, &[], &[0, 0, 0, 1, 2, 3, 4, 5]).prune_path(), 0); + } + + /// Pruning *does* rise above the zipper's root, contradicting the doc comment + /// on `ZipperWriting::prune_path`. A zipper rooted at `[0,0]` looking at the + /// dangling tip of the same chain prunes all 7 bytes, back to the map root — + /// not the 5 that lie below its own root. Verified against pathmap 0.3.1. + #[test] + fn prune_path_rises_above_zipper_root() { + let mut t = mk(&[(&[0, 0, 0, 1, 2, 3, 4], 0)]); + t.remove_val(&[0, 0, 0, 1, 2, 3, 4]); + let mut z = zip_at(&t, &[0, 0], &[0, 1, 2, 3, 4]); + assert_eq!(z.prune_path(), 7); + assert!(z.trie.is_empty_map()); + } + + /// `write_zipper_drop_head_test3`: `[[0,0],[0,1],[1,0],[1,1]]` with + /// `join_k_path_into(1)` collapses to 2 values. + #[test] + fn drop_head_test3() { + let t = mk(&[(&[0, 0], 0), (&[0, 1], 1), (&[1, 0], 2), (&[1, 1], 3)]); + let mut z = zip_at(&t, &[], &[]); + z.join_k_path_into(&OPS, 1, true); + assert_eq!(z.val_count(), 2); + } + + /// `write_zipper_drop_head_test6`: dropping 4 bytes from paths that are at + /// most 4 long annihilates everything, because values at depth exactly `k` + /// are lost. + #[test] + fn drop_head_test6() { + let t = mk(&[ + (&[193, 191, 193, 193, 191], 0), + (&[193, 191, 193, 194, 12, 28], 1), + (&[193, 191, 193, 194, 18, 9], 2), + (&[193, 191, 194, 193, 191], 3), + (&[193, 191, 194, 194, 12, 28], 4), + (&[193, 191, 194, 194, 15, 47], 5), + (&[193, 191, 194, 194, 18, 9], 6), + ]); + let mut z = zip_at(&t, &[], &[193, 191]); + assert!(!z.join_k_path_into(&OPS, 4, true)); + assert_eq!(z.val_count(), 0); + } + + /// `write_zipper_drop_head_test1`: under the root `123:`, dropping 4 bytes + /// rewrites `abc:Bob` to `Bob` and `dog:Bob:Fido` to `Bob:Fido`. + #[test] + fn drop_head_test1() { + let t = mk(&[(b"123:abc:Bob", 0), (b"123:dog:Bob:Fido", 1)]); + let mut z = zip_at(&t, b"123:", &[]); + z.join_k_path_into(&OPS, 4, true); + let r = z.trie; + assert_eq!(r.val_at(b"123:Bob"), Some(&0)); + assert_eq!(r.val_at(b"123:Bob:Fido"), Some(&1)); + assert_eq!(r.val_count(&[]), 2); + } + + // -- structural invariants over every fixture --------------------------- + + #[test] + fn structural_invariants() { + for t in fixtures() { + assert!(vals_exist(&t)); + assert!(prefix_closed(&t)); + assert!(val_count_agrees(&t)); + for p in probes() { + assert!(child_mask_agrees(&t, &p)); + } + } + } + + // -- zipper law checks -------------------------------------------------- + + #[test] + fn zipper_laws() { + for z in all_zips() { + assert!(remove_val_leaves_path(&z)); + assert!(create_path_idempotent(&z)); + assert!(create_then_prune(&OPS, &z)); + assert!(descend_indexed_lands(&z)); + assert!(sibling_round_trip(&z)); + assert!(descend_until_observed_exact(&z)); + assert!(ascend_until_accounts(&z)); + assert!(to_next_val_monotone(&z)); + assert!(set_val_then_val(&OPS, &z, 42)); + assert!(take_then_graft(&OPS, &z)); + assert!(join_empty_identity(&OPS, &z)); + assert!(join_self_identity(&OPS, &z)); + assert!(remove_branches_keeps_val(&OPS, &z)); + assert!(remove_unmasked_full_mask(&OPS, &z)); + assert!(remove_unmasked_empty_mask(&OPS, &z)); + for p in probes() { + assert!(descend_to_existing_lands(&z, &p)); + } + for pre in [&[0u8][..], &[0, 1][..], &[2, 2][..]] { + assert!(drop_head_undoes_insert_prefix(&OPS, &z, pre)); + } + } + } + + #[test] + fn graft_laws() { + let zs = all_zips(); + for z in &zs { + for w in &zs { + assert!(graft_then_make_map(&OPS, z, w)); + } + } + for s in fixtures() { + for (a, b) in [ + (&[0u8][..], &[9u8][..]), + (&[1][..], &[2][..]), + (&[0, 0][..], &[1, 1][..]), + ] { + assert!(grafted_copies_independent(&OPS, &s, a, b, &[3, 7], 999)); + } + } + } + + // -- algebraic law checks ----------------------------------------------- + + #[test] + fn algebraic_laws() { + let fs = fixtures(); + for a in &fs { + assert!(join_idem(&OPS, a)); + assert!(meet_idem_on_vals(&OPS, a)); + assert!(meet_idem(&OPS, a)); + assert!(meet_pruned_idem(&OPS, a)); + assert!(sub_self_empty_vals(&OPS, a)); + assert!(restrict_self(&OPS, a)); + for b in &fs { + assert!(join_comm_on_paths(&OPS, a, b)); + assert!(meet_comm_on_paths(&OPS, a, b)); + for c in &fs { + assert!(join_assoc(&OPS, a, b, c)); + } + } + } + } + + // -- naive oracles ------------------------------------------------------ + // + // `PathMap::join` / `meet` / `sub` are defined in terms of `ValOps`, which was + // transcribed from `src/ring.rs` — so a defect in the crate's `Option` + // lattice impls could have been copied into the model, after which the + // differential would agree and report nothing. + // + // These oracles are written from set theory instead: they say which *keys* + // survive without consulting `ValOps`, `PathMap`, or anything else the model + // shares with the crate. They are deliberately naive and quadratic. Where + // they and the real definitions agree, the shared-derivation risk is excluded + // for that operation. + // + // `U64Ops::psub` annihilates exactly when the two values are equal, which is + // the one fact about the value type these need. + + fn keys(t: &T) -> Vec> { + t.vals().map(|(k, _)| k.clone()).collect() + } + + fn sorted_dedup(mut v: Vec>) -> Vec> { + v.sort(); + v.dedup(); + v + } + + /// Keys of `a` and `b` together: what `join` must produce. + fn join_keys_oracle(a: &T, b: &T) -> Vec> { + let mut v = keys(a); + v.extend(keys(b)); + sorted_dedup(v) + } + + /// Keys in both: what `meet` must produce. + fn meet_keys_oracle(a: &T, b: &T) -> Vec> { + sorted_dedup(keys(a).into_iter().filter(|k| b.val_at(k).is_some()).collect()) + } + + /// Keys of `a` whose value `b` does not annihilate: what `sub` must produce. + /// For `u64`, `psubtract` annihilates exactly on equal values. + fn sub_keys_oracle(a: &T, b: &T) -> Vec> { + sorted_dedup( + a.vals() + .filter(|(k, av)| b.val_at(k) != Some(av)) + .map(|(k, _)| k.clone()) + .collect(), + ) + } + + /// `restrict` against the `BTreeSet` oracle from + /// `tests/pathmap_algebra_differential.rs`: a path of `a` survives when some + /// prefix of it — the empty prefix and the path itself both count — carries a + /// value in `b`. + fn restrict_oracle(a: &T, b: &T) -> Vec> { + a.vals() + .filter(|(p, _)| (0..=p.len()).any(|i| b.val_at(&p[..i]).is_some())) + .map(|(p, _)| p.clone()) + .collect() + } + + #[test] + fn naive_oracles_agree() { + let fs = fixtures(); + for a in &fs { + for b in &fs { + assert_eq!(keys(&PathMap::join(&OPS, a, b)), join_keys_oracle(a, b)); + assert_eq!(keys(&PathMap::meet(&OPS, a, b)), meet_keys_oracle(a, b)); + assert_eq!(keys(&PathMap::sub(&OPS, a, b)), sub_keys_oracle(a, b)); + assert_eq!(keys(&map::restrict(a, b)), restrict_oracle(a, b)); + } + } + } + + /// The minimal shapes from `restrict_matches_btreeset_oracle`, which caught a + /// real `prestrict` bug: a location that both carries a value and branches. + #[test] + fn restrict_minimal_shapes() { + let shapes: Vec> = vec![ + vec![(&[0, 1], 0), (&[0, 1, 2], 1)], + vec![(&[0, 1], 0), (&[0, 1, 2], 1), (&[0, 1, 3], 2)], + vec![(&[0], 0), (&[0, 1], 1), (&[0, 1, 2], 2)], + vec![(&[0], 0), (&[0, 1, 2], 1), (&[0, 1, 3], 2)], + vec![(&[0], 0), (&[0, 1], 1), (&[0, 1, 2], 2), (&[0, 1, 3], 3), (&[9, 9], 4)], + ]; + for es in shapes { + assert!(restrict_self(&OPS, &mk(&es))); + } + } +} + +#[cfg(test)] +mod fuzz_tests { + //! Randomised self-consistency, over far more states than the fixture + //! battery reaches. + //! + //! The fixtures in [`crate::reference::check::tests`] are the ones the Lean `#guard`s use: 6 + //! maps × 8 probes. They were chosen to cover the interesting *shapes*, but + //! they are still 48 states, and a range query with an off-by-one in it can + //! easily survive all 48. So this walks random operation sequences, and + //! after every step asserts + //! + //! * the map is still canonical — the one invariant `BTreeMap` does not + //! enforce for us, and the one every constructor here is responsible for; + //! * every law from [`crate::reference::laws`] that applies to a single state. + //! + //! What this cannot catch is a *faithful-looking but wrong* transcription + //! that is internally consistent — a mis-ordered `AlgebraicStatus::merge`, + //! say. Only a differential against the Lean model or the crate finds those. + + use crate::reference::basic::U64Ops; + use crate::reference::laws::*; + use crate::reference::pathmap::PathMap; + use crate::reference::zipper::Zip; + + const OPS: U64Ops = U64Ops; + + /// splitmix64 — a deterministic PRNG, so a failure is reproducible from the + /// seed alone and the test pulls in no dependency. + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + /// A short path over a 3-letter alphabet, so the generated maps share + /// prefixes heavily — that is where branch points, dangling chains and + /// single-child runs live. + fn path(&mut self, max: u64) -> Vec { + let n = self.below(max + 1); + (0..n).map(|_| self.below(3) as u8).collect() + } + } + + /// Every law that is a predicate on one zipper state. + fn check_state(z: &Zip) { + assert!(z.trie.is_canonical(), "not canonical: {:?}", z.trie.entries.keys()); + assert!(vals_exist(&z.trie)); + assert!(prefix_closed(&z.trie)); + assert!(val_count_agrees(&z.trie)); + assert!(child_mask_agrees(&z.trie, &z.focus())); + assert!(remove_val_leaves_path(z)); + assert!(create_path_idempotent(z)); + // Not universal: `prune_path` reaches above the zipper root, so a + // dangling ancestor is swept up along with the created chain. + if create_then_prune_applies(z) { + assert!(create_then_prune(&OPS, z)); + } + assert!(descend_indexed_lands(z)); + assert!(sibling_round_trip(z)); + assert!(descend_until_observed_exact(z)); + assert!(ascend_until_accounts(z)); + assert!(to_next_val_monotone(z)); + assert!(set_val_then_val(&OPS, z, 42)); + assert!(take_then_graft(&OPS, z)); + assert!(join_empty_identity(&OPS, z)); + assert!(join_self_identity(&OPS, z)); + assert!(remove_branches_keeps_val(&OPS, z)); + assert!(remove_unmasked_full_mask(&OPS, z)); + assert!(remove_unmasked_empty_mask(&OPS, z)); + assert!(drop_head_undoes_insert_prefix(&OPS, z, &[0, 1])); + assert!(join_idem(&OPS, &z.trie)); + assert!(meet_idem_on_vals(&OPS, &z.trie)); + assert!(sub_self_empty_vals(&OPS, &z.trie)); + // Not universal: a dangling path with no valued ancestor is dropped by + // `restrict`. See `restrict_self`. + if restrict_self_applies(&z.trie) { + assert!(restrict_self(&OPS, &z.trie)); + } + } + + #[test] + fn random_programs_keep_every_law() { + for seed in 0..256u64 { + let mut r = Rng(seed.wrapping_mul(0x2545F4914F6CDD1D) | 1); + let root = r.path(2); + let mut z = Zip::at(PathMap::empty(), &root); + // A source zipper over an independently seeded map, for the binary + // operations. + let mut src_map = PathMap::empty(); + for _ in 0..r.below(6) { + src_map.set_val(&r.path(4), r.below(4)); + } + let src = Zip::at_path(src_map, &[], &r.path(2)); + + for _ in 0..40 { + let k = r.path(3); + match r.below(18) { + 0 => { + z.set_val(r.below(4)); + } + 1 => { + z.remove_val(r.below(2) == 1); + } + 2 => { + z.create_path(); + } + 3 => { + z.prune_path(); + } + 4 => { + z.remove_branches(r.below(2) == 1); + } + 5 => { + z.descend_to(&k); + } + 6 => { + z.ascend(r.below(3) as usize); + } + 7 => { + z.ascend_until(); + } + 8 => { + z.descend_until(); + } + 9 => { + z.to_next_val(); + } + 10 => { + z.graft(&src); + } + 11 => { + z.join_into(&OPS, &src); + } + 12 => { + z.meet_into(&OPS, &src, r.below(2) == 1); + } + 13 => { + z.subtract_into(&OPS, &src, r.below(2) == 1); + } + 14 => { + z.restrict(&OPS, &src); + } + 15 => { + z.insert_prefix(&k); + } + 16 => { + z.join_k_path_into(&OPS, 1 + r.below(2) as usize, r.below(2) == 1); + } + _ => { + z.take_map(r.below(2) == 1); + } + } + check_state(&z); + // The zipper must never leave the subtree it was created in. + assert!(z.focus().starts_with(&root), "escaped its root"); + } + } + } +} diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs new file mode 100644 index 00000000..ab1c257f --- /dev/null +++ b/differential/src/reference/fuzz.rs @@ -0,0 +1,795 @@ +//! `Fuzz.lean` — the differential front end: decode a fuzzer input into a +//! program over two maps and two zippers, run it against the model, and render +//! the trace. +//! +//! A transcription of `lean/PathMapModel/Fuzz.lean`, which it must track byte +//! for byte: the same operand decoding in the same order (including bytes +//! consumed before an operation decides to `skip`), the same operation table, +//! and the same rendering. `NOPS` and `MAX_STEPS` are the same contract +//! `differential/src/harness.rs` names. +//! +//! Kept as its own module rather than inline in a binary so both front ends — +//! `bin/reference.rs` and the in-process comparator `bin/in_process.rs` — drive +//! the model through the same op table, instead of there being a fourth copy of +//! it. The three copies that must stay in lockstep are `Fuzz.lean`, +//! `differential/src/harness.rs` and this file; there is no fourth. + +// One growable buffer rather than a `Vec`; see the note in +// `differential/src/harness.rs`. +use core::fmt::Write as _; + +use crate::reference::basic::{AlgStatus, ByteMask, U64Ops, path}; +use crate::reference::pathmap::PathMap; +use crate::reference::zipper::Zip; + +/// Number of distinct operations. Must match `PathMapModel.Fuzz.nops` and +/// `NOPS` in `differential/src/harness.rs`. +const NOPS: usize = 56; +/// Maximum operations executed. Must match the `maxSteps` default in `Fuzz.run`. +const MAX_STEPS: usize = 256; +/// Maximum entries in a `dump`. Must match `Fuzz.dumpAt`. +const DUMP_CAP: usize = 64; + +const OPS: U64Ops = U64Ops; + +// --------------------------------------------------------------------------- +// Skip reasons — `Fuzz` §Skip reasons +// --------------------------------------------------------------------------- +// +// Why an operation was skipped. Every `skip` in the trace carries one of these, +// so a skipped op says which rule declined it rather than just that something +// declined it. `lean/PathMapModel/Fuzz.lean` and `differential/src/harness.rs` +// emit the same tokens; all three must agree exactly or every input with a skip +// diverges. +// +// Spelled out here rather than imported from `crate::harness`: those are the +// *crate side's* strings, and the point of a second transcription is that it +// agrees with the Lean one by having been written from it, not by sharing a +// constant with the thing it is checking. +// +// Each is recorded in lean/FINDINGS.md and commented at its site. + +/// The ACT read source cannot be a merge source (`ZipperInfallibleSubtries` is +/// not implemented for it) or does not implement the trait the op needs. +const SKIP_ACT: &str = "skip:act"; +/// A degenerate `k = 0`. +const SKIP_K0: &str = "skip:k0"; +/// The focus has nothing below it, where the op's behaviour is a function of +/// node materialisation rather than of trie state. +const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; +/// A prune on a write zipper not rooted at the map root, where the depth pruned +/// is a function of internal node layout. +const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; + +// --------------------------------------------------------------------------- +// Decoder — `Fuzz.Dec` +// --------------------------------------------------------------------------- + +struct Dec<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl Dec<'_> { + /// Read one byte; `None` once the input is exhausted, which ends the program. + fn u8(&mut self) -> Option { + let b = *self.bytes.get(self.pos)?; + self.pos += 1; + Some(b) + } + /// Read one byte reduced modulo `m`. + fn modn(&mut self, m: usize) -> Option { + let b = self.u8()?; + Some(if m == 0 { 0 } else { (b as usize) % m }) + } + /// Path bytes live in a 4-letter alphabet so generated tries share prefixes. + fn path_byte(&mut self) -> Option { + Some(self.u8()? % 4) + } + fn path_n(&mut self, n: usize) -> Option> { + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(self.path_byte()?); + } + Some(v) + } + /// Read a length-prefixed path (`len := u8 % lim`). + fn path(&mut self, lim: usize) -> Option> { + let n = self.modn(lim)?; + self.path_n(n) + } + fn boolean(&mut self) -> Option { + Some(self.u8()? % 2 == 1) + } +} + +// --------------------------------------------------------------------------- +// Rendering — `Fuzz` §Rendering +// --------------------------------------------------------------------------- + +fn hex_path(p: &[u8]) -> String { + if p.is_empty() { + "_".to_string() + } else { + p.iter().map(|b| format!("{b:02x}")).collect() + } +} + +fn show_val(v: Option<&u64>) -> String { + match v { + None => "-".to_string(), + Some(v) => format!("{v}"), + } +} + +/// Render the `Option` the movement operations return: the byte moved to, or +/// `-` for "did not move". +fn show_byte_opt(b: Option) -> String { + match b { + None => "-".to_string(), + Some(b) => format!("{b:02x}"), + } +} + +fn show_bool(b: bool) -> &'static str { + if b { "1" } else { "0" } +} + +fn show_status(s: AlgStatus) -> String { + s.to_string() +} + +/// All locations at and below `root`, depth-first, capped so a runaway trie +/// cannot make the trace unbounded. +fn dump_at(t: &PathMap, root: &[u8]) -> String { + t.subtrie(root) + .paths() + .take(DUMP_CAP) + .map(|q| format!("{}:{}", hex_path(q), show_val(t.val_at(&path::cat(root, q))))) + .collect::>() + .join(",") +} + +// --------------------------------------------------------------------------- +// Interpreter state — `Fuzz.St` +// --------------------------------------------------------------------------- + +/// Two maps, two zippers: `wz` writes into map0, `rz` reads map1. Keeping the +/// read source in a *separate* map is what lets the real crate hold both zippers +/// at once. +struct St { + wz: Zip, + rz: Zip, + out: String, + step: usize, + /// Read source is an `ArenaCompactTree` rather than a `PathMap`, which cannot + /// be the source of a graft or an algebraic merge. Those operations report + /// `skip` on both sides of the comparison. + act: bool, +} + +/// The per-step fingerprint of one zipper. +fn fingerprint(z: &Zip) -> String { + format!( + "{} o{} e{} v{} c{} n{} f{}", + hex_path(&z.path), + hex_path(&z.focus()), + show_bool(z.path_exists()), + show_val(z.val()), + z.child_count(), + z.val_count(), + // `focus_byte` is unspecified at the root, so it is only compared below it. + if z.at_root() { "?".to_string() } else { show_byte_opt(z.focus_byte()) } + ) +} + +impl St { + fn emit(&mut self, name: &str, ret: &str) { + let _ = writeln!(self.out, + "{} {} ret={} W={} R={}", + self.step, + name, + ret, + fingerprint(&self.wz), + fingerprint(&self.rz) + ); + self.step += 1; + } + + /// The zipper selected by a target byte (`0` = write zipper, `1` = read). + fn target(&mut self, t: usize) -> &mut Zip { + if t == 0 { &mut self.wz } else { &mut self.rz } + } + + fn target_ref(&self, t: usize) -> &Zip { + if t == 0 { &self.wz } else { &self.rz } + } + + /// Is the *explicit* `prune_path` / `prune_ascend` well-defined for this + /// state? Only for a write zipper rooted at the map root: with a non-empty + /// root the depth pruned becomes a function of node layout rather than of the + /// logical trie. See `Fuzz.pruneable` and lean/FINDINGS.md finding 7. + fn pruneable(&self) -> bool { + self.wz.root.is_empty() + } +} + +/// The `prune` flag the harness passes to operations that take one. +/// +/// Always `false`: the flag's effect is a function of internal node layout rather +/// than of the logical trie, so there is nothing for a model to agree with. +const NO_PRUNE: bool = false; + +/// A full `k`-path iteration: `descend_first_k_path` followed by `to_next_k_path` +/// until it runs out (capped at 32 stops). This is the only well-defined way to +/// use the k-path primitives. +fn k_walk(z: &mut Zip, k: usize) -> Vec> { + if !z.descend_first_k_path(k) { + return Vec::new(); + } + let mut acc = vec![z.path.clone()]; + for _ in 0..31 { + if z.to_next_k_path(k) { + acc.push(z.path.clone()); + } else { + break; + } + } + acc +} + +// --------------------------------------------------------------------------- +// The operation table — `Fuzz.step` +// --------------------------------------------------------------------------- +// +// `op % NOPS` selects the operation. Ops `0`–`26` act on a target zipper chosen +// by a following `u8 % 2` byte (`0` = write zipper, `1` = read zipper); the rest +// are write-zipper operations. +// +// Operand bytes are consumed *before* an operation decides to `skip`, exactly as +// in `Fuzz.lean`, because the byte stream must stay in step across all three +// front ends. `None` means the input ran out mid-operation, which ends the run. + +fn step(s: &mut St, d: &mut Dec) -> Option<()> { + let op = d.u8()? as usize % NOPS; + match op { + 0 => { + let t = d.modn(2)?; + let p = d.path(6)?; + s.target(t).descend_to(&p); + s.emit("descend_to", &hex_path(&p)); + } + 1 => { + let t = d.modn(2)?; + let b = d.path_byte()?; + s.target(t).descend_to_byte(b); + s.emit("descend_to_byte", &format!("{b:02x}")); + } + 2 => { + let t = d.modn(2)?; + let n = d.modn(8)?; + let r = s.target(t).ascend(n); + s.emit("ascend", &r.to_string()); + } + 3 => { + let t = d.modn(2)?; + let r = s.target(t).ascend_byte(); + s.emit("ascend_byte", show_bool(r)); + } + 4 => { + let t = d.modn(2)?; + s.target(t).reset(); + s.emit("reset", "-"); + } + 5 => { + let t = d.modn(2)?; + let r = s.target(t).descend_first_byte(); + s.emit("descend_first_byte", &show_byte_opt(r)); + } + 6 => { + let t = d.modn(2)?; + let r = s.target(t).descend_last_byte(); + s.emit("descend_last_byte", &show_byte_opt(r)); + } + 7 => { + let t = d.modn(2)?; + let i = d.modn(6)?; + let r = s.target(t).descend_indexed_byte(i); + s.emit("descend_indexed_byte", &show_byte_opt(r)); + } + 8 => { + let t = d.modn(2)?; + let r = s.target(t).descend_until(); + s.emit("descend_until", show_bool(r)); + } + 9 => { + let t = d.modn(2)?; + let r = s.target(t).ascend_until(); + s.emit("ascend_until", &r.to_string()); + } + 10 => { + let t = d.modn(2)?; + let r = s.target(t).ascend_until_branch(); + s.emit("ascend_until_branch", &r.to_string()); + } + 11 => { + // Formerly skipped at the zipper root, where + // `ReadZipper::to_next_sibling_byte` escaped its own root + // (FINDINGS.md #3). Fixed: the root case is compared like any other. + let t = d.modn(2)?; + let r = s.target(t).to_next_sibling_byte(); + s.emit("to_next_sibling_byte", &show_byte_opt(r)); + } + 12 => { + let t = d.modn(2)?; + let r = s.target(t).to_prev_sibling_byte(); + s.emit("to_prev_sibling_byte", &show_byte_opt(r)); + } + 13 => { + let t = d.modn(2)?; + let r = s.target(t).to_next_step(); + s.emit("to_next_step", show_bool(r)); + } + 14 => { + // `ZipperIteration` is read-only: the target byte is still consumed, + // but the operation always applies to the read zipper. + let _t = d.modn(2)?; + let r = s.rz.to_next_val(); + s.emit("to_next_val", show_bool(r)); + } + 15 => { + let _t = d.modn(2)?; + let k = d.modn(4)?; + // `k = 0` is degenerate: `k_path_internal` treats "already at depth + // base+0" as a hit and reports success without moving, then + // `to_next_k_path(0)` reports success forever. Skipped. + if k == 0 { + s.emit("descend_first_k_path", SKIP_K0); + } else { + let r = s.rz.descend_first_k_path(k); + s.emit("descend_first_k_path", show_bool(r)); + } + } + 16 => { + let _t = d.modn(2)?; + let k = d.modn(4)?; + // `to_next_k_path` is only meaningful as the continuation of a + // `descend_first_k_path` iteration, so the op is the whole walk. + if k == 0 { + s.emit("k_path_walk", SKIP_K0); + } else { + let ps = k_walk(&mut s.rz, k); + let ret = ps.iter().map(|p| hex_path(p)).collect::>().join(","); + s.emit("k_path_walk", &ret); + } + } + 17 => { + let _t = d.modn(2)?; + let r = s.rz.descend_last_path(); + s.emit("descend_last_path", show_bool(r)); + } + 18 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let n = s.target(t).move_to_path(&p); + s.emit("move_to_path", &n.to_string()); + } + 19 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let n = s.target(t).descend_to_existing(&p); + s.emit("descend_to_existing", &n.to_string()); + } + 20 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let n = s.target(t).descend_to_val(&p); + s.emit("descend_to_val", &n.to_string()); + } + 21 => { + let t = d.modn(2)?; + let b = d.path_byte()?; + let r = s.target(t).descend_to_existing_byte(b); + s.emit("descend_to_existing_byte", show_bool(r)); + } + 22 => { + let t = d.modn(2)?; + let n = d.modn(8)?; + let r = s.target(t).descend_until_max_bytes(n); + s.emit("descend_until_max_bytes", show_bool(r)); + } + 23 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let r = s.target(t).descend_to_check(&p); + s.emit("descend_to_check", show_bool(r)); + } + 24 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let ret = show_val(s.target_ref(t).val_at(&p)); + s.emit("val_at", &ret); + } + 25 => { + let t = d.modn(2)?; + if s.act && t == 1 { + s.emit("make_map_val_count", SKIP_ACT); + } else { + let ret = s.target_ref(t).make_map().val_count(&[]).to_string(); + s.emit("make_map_val_count", &ret); + } + } + 26 => { + let t = d.modn(2)?; + let z = s.target_ref(t); + let ret = dump_at(&z.trie, &z.focus()); + s.emit("dump", &ret); + } + 27 => { + let v = d.u8()?; + let old = s.wz.set_val(v as u64); + s.emit("set_val", &show_val(old.as_ref())); + } + 28 => { + let _pr = d.boolean()?; + let old = s.wz.remove_val(NO_PRUNE); + s.emit("remove_val", &show_val(old.as_ref())); + } + 29 => { + let r = s.wz.create_path(); + s.emit("create_path", show_bool(r)); + } + 30 => { + if s.pruneable() { + let n = s.wz.prune_path(); + s.emit("prune_path", &n.to_string()); + } else { + s.emit("prune_path", SKIP_OFF_ROOT_PRUNE); + } + } + 31 => { + if s.pruneable() { + let n = s.wz.prune_ascend(); + s.emit("prune_ascend", &n.to_string()); + } else { + s.emit("prune_ascend", SKIP_OFF_ROOT_PRUNE); + } + } + 32 => { + let _pr = d.boolean()?; + let leaky = s.wz.focus_node_is_empty(); + let r = s.wz.remove_branches(NO_PRUNE); + let ret = if leaky { "?".to_string() } else { show_bool(r).to_string() }; + s.emit("remove_branches", &ret); + } + 33 => { + let n = d.modn(4)?; + let m = d.path_n(n)?; + let _pr = d.boolean()?; + let mask = ByteMask::of_list(m); + s.wz.remove_unmasked_branches(&mask, NO_PRUNE); + let ret = hex_path(mask.bytes()); + s.emit("remove_unmasked_branches", &ret); + } + 34 => { + if s.act { + s.emit("graft", SKIP_ACT); + } else { + s.wz.graft(&s.rz); + s.emit("graft", "-"); + } + } + 35 => { + let p = d.path(6)?; + if s.act { + s.emit("graft_src_at", SKIP_ACT); + } else { + s.wz.graft_src_at(&s.rz, &p); + s.emit("graft_src_at", &hex_path(&p)); + } + } + 36 => { + if s.act { + s.emit("join_into", SKIP_ACT); + } else { + let st = s.wz.join_into(&OPS, &s.rz); + s.emit("join_into", &show_status(st)); + } + } + 37 => { + if s.act { + s.emit("join_map_into", SKIP_ACT); + } else { + let leaky = s.wz.focus_node_is_empty(); + let m = s.rz.make_map(); + let st = s.wz.join_map_into(&OPS, &m); + let ret = if leaky { "?".to_string() } else { show_status(st) }; + s.emit("join_map_into", &ret); + } + } + 38 => { + let _pr = d.boolean()?; + if s.act { + s.emit("meet_into", SKIP_ACT); + } else { + // `prune = true` is best-effort (nodes shared with the source may be + // left unpruned; see `Zip::meet_into`), so only `prune = false` is compared. + let st = s.wz.meet_into(&OPS, &s.rz, NO_PRUNE); + s.emit("meet_into", &show_status(st)); + } + } + 39 => { + let _pr = d.boolean()?; + if s.act { + s.emit("subtract_into", SKIP_ACT); + } else { + let st = s.wz.subtract_into(&OPS, &s.rz, NO_PRUNE); + s.emit("subtract_into", &show_status(st)); + } + } + 40 => { + if s.act { + s.emit("restrict", SKIP_ACT); + } else { + // Formerly masked to `?` at an empty focus (FINDINGS.md #8); + // the crate's status tracks the spec, so it is compared. + let st = s.wz.restrict(&OPS, &s.rz); + let ret = show_status(st); + s.emit("restrict", &ret); + } + } + 41 => { + // Skipped, not merely masked, when either side has nothing below its + // focus: there `restricting` branches on whether an empty node + // happens to be materialised, and the two branches differ in + // *effect*, not just in the reported bool. See FINDINGS.md #8. + // + // This guard is checked *before* the ACT one because the harness + // reaches the ACT skip only by calling `do_restricting`, which it + // does not do once this guard has fired; the two orders were + // indistinguishable while both reasons rendered as a bare `skip`. + if s.wz.focus_node_is_empty() || s.rz.focus_node_is_empty() { + s.emit("restricting", SKIP_EMPTY_FOCUS); + } else if s.act { + s.emit("restricting", SKIP_ACT); + } else { + let r = s.wz.restricting(&s.rz); + s.emit("restricting", show_bool(r)); + } + } + 42 => { + let k = d.modn(4)?; + let _pr = d.boolean()?; + // `join_k_path_into(0)` should be the identity but destroys the + // subtrie in pathmap 0.3.1. + if k == 0 { + s.emit("join_k_path_into", SKIP_K0); + } else { + // The bool was masked to `?` on an empty resulting focus as a + // suspected `AbstractNodeRef` leak (FINDINGS.md #8). It is not + // one: unmasked it tracks the spec over the whole sweep. + let r = s.wz.join_k_path_into(&OPS, k, NO_PRUNE); + let ret = show_bool(r).to_string(); + s.emit("join_k_path_into", &ret); + } + } + 43 => { + let p = d.path(6)?; + // The empty prefix was skipped while it destroyed the subtrie + // (FINDINGS.md #4); fixed upstream, so it is compared like any + // other prefix. + let r = s.wz.insert_prefix(&p); + s.emit("insert_prefix", show_bool(r)); + } + 44 => { + let n = d.modn(6)?; + let r = s.wz.remove_prefix(n); + s.emit("remove_prefix", show_bool(r)); + } + 45 => { + let _pr = d.boolean()?; + let leaky = s.wz.focus_node_is_empty() && s.wz.val().is_none(); + let m = s.wz.take_map(NO_PRUNE); + if leaky { + let mm = m.unwrap_or_else(PathMap::empty); + s.wz.graft_map(&mm); + s.emit("take_map_restore", "?"); + } else { + match m { + Some(mm) => { + s.wz.graft_map(&mm); + s.emit("take_map_restore", "1"); + } + None => s.emit("take_map_restore", "0"), + } + } + } + 46 => { + let k = d.modn(4)?; + let _pr = d.boolean()?; + // `meet_k_path_into` is not implementable for these arguments; see + // `Zip::meet_k_path_unspecified`, whose two disjuncts are split out + // here so the skip names which one fired. The crate side matches. + if k == 0 { + s.emit("meet_k_path_into", SKIP_K0); + } else if s.wz.focus_node_is_empty() { + s.emit("meet_k_path_into", SKIP_EMPTY_FOCUS); + } else { + let r = s.wz.meet_k_path_into(&OPS, k, NO_PRUNE); + s.emit("meet_k_path_into", show_bool(r)); + } + } + 47 => { + let t = d.modn(2)?; + // The blind-zipper addition: `descend_until` reporting the bytes it + // descended. The observer's output is a blind zipper's only account + // of where it went, so it is compared byte for byte. + let (r, obs) = s.target(t).descend_until_observed(); + let ret = format!("{}:{}", show_bool(r), hex_path(&obs)); + s.emit("descend_until_observed", &ret); + } + 48 => { + let v = d.u8()?; + // Writing through the reference `get_val_mut` hands back. It must + // behave like `set_val` where a value exists and do nothing — + // crucially, *not* create the path — where one does not. + let old = s.wz.get_val_mut_write(v as u64); + s.emit("get_val_mut_write", &show_val(old.as_ref())); + } + 49 => { + let v = d.u8()?; + let r = s.wz.get_val_or_set_mut(v as u64); + s.emit("get_val_or_set_mut", &show_val(Some(&r))); + } + 50 => { + let v = d.u8()?; + // `ran` records whether the closure was invoked. The contract says it + // supplies the value "if no value exists", so invoking it when a value + // is already present is observable to any caller whose closure has a + // side effect. + let (r, ran) = s.wz.get_val_or_set_mut_with(v as u64); + let ret = format!("{}:{}", show_val(Some(&r)), show_bool(ran)); + s.emit("get_val_or_set_mut_with", &ret); + } + 51 => { + let t = d.modn(2)?; + let p = d.path(6)?; + // `get_val`/`get_val_at` differ from `val`/`val_at` only in the + // lifetime of the reference they return, so they must give the same + // answer. `agree` is `1` in the model by construction; a `0` from the + // crate is the whole point of the op. + let z = s.target_ref(t); + let ret = format!("{}:{}:1", show_val(z.val()), show_val(z.val_at(&p))); + s.emit("get_val_agrees", &ret); + } + 52 => { + // `to_next_get_val` must advance exactly as `to_next_val` does and + // hand back the value at the new focus. + if s.act { + s.emit("to_next_get_val", SKIP_ACT); + } else { + let moved = s.rz.to_next_val(); + let v = if moved { s.rz.val().copied() } else { None }; + let ret = format!("{}:{}:1", show_bool(moved), show_val(v.as_ref())); + s.emit("to_next_get_val", &ret); + } + } + 53 => { + let n = d.modn(4)?; + let m = d.path_n(n)?; + let ru = d.boolean()?; + if s.act { + s.emit("graft_masked_branches", SKIP_ACT); + } else { + let mask = ByteMask::of_list(m); + s.wz.graft_masked_branches(&s.rz, &mask, ru); + let ret = format!("{}:{}", hex_path(mask.bytes()), show_bool(ru)); + s.emit("graft_masked_branches", &ret); + } + } + 54 => { + let n = d.modn(4)?; + let m = d.path_n(n)?; + let ru = d.boolean()?; + // Fed the source's own child subtries, this must agree with + // `graft_masked_branches` on the same mask. It used to be skipped + // outright (`skip:quarantined`) because `graft_child_maps` was broken + // three ways (FINDINGS.md #15); the Lean model has since let it back + // in, and this follows. + if s.act { + s.emit("graft_child_maps", SKIP_ACT); + } else { + let mask = ByteMask::of_list(m); + let sf = s.rz.focus(); + let maps: Vec<(ByteMask, PathMap)> = mask + .bytes() + .iter() + .map(|&b| { + (ByteMask::of_list([b]), s.rz.trie.subtrie(&path::push(&sf, b))) + }) + .collect(); + s.wz.graft_child_maps(&maps, ru); + let ret = format!("{}:{}", hex_path(mask.bytes()), show_bool(ru)); + s.emit("graft_child_maps", &ret); + } + } + 55 => { + let p = d.path(6)?; + // `meet_2` takes two sources; the second is the first moved to `p`. + if s.act { + s.emit("meet_2", SKIP_ACT); + } else { + let mut b = s.rz.clone(); + b.path.extend_from_slice(&p); + let st = s.wz.meet_2(&OPS, &s.rz, &b); + s.emit("meet_2", &show_status(st)); + } + } + _ => s.emit("nop", "-"), + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Header and entry point — `Fuzz.header` / `Fuzz.run` +// --------------------------------------------------------------------------- + +/// Decode `n` seed entries and insert them into `t`. +fn seed(t: &mut PathMap, d: &mut Dec, n: usize) -> Option<()> { + for _ in 0..n { + let p = d.path(6)?; + let v = d.u8()?; + t.set_val(&p, v as u64); + } + Some(()) +} + +/// Decode the header: two seeded maps and the two zipper roots. +fn header(d: &mut Dec, act: bool) -> Option { + let n0 = d.modn(8)?; + let mut m0 = PathMap::empty(); + seed(&mut m0, d, n0)?; + let n1 = d.modn(8)?; + let mut m1 = PathMap::empty(); + seed(&mut m1, d, n1)?; + let r0 = d.path(4)?; + let r1 = d.path(4)?; + // Both zipper roots are created if absent. A zipper whose *root* does not + // exist can escape it — `to_next_sibling_byte` and `to_next_step` fall back + // on the parent's child mask and walk out of the granted subtrie. Making the + // roots exist keeps that one bug from contaminating every other comparison. + if !r0.is_empty() { + m0.add_path(&r0); + } + if !r1.is_empty() { + m1.add_path(&r1); + } + Some(St { + wz: Zip::at(m0, &r0), + rz: Zip::at(m1, &r1), + out: String::new(), + step: 0, + act, + }) +} + +/// Decode and run a fuzzer input, returning the trace lines. +pub fn run(bytes: &[u8], act: bool) -> String { + let mut d = Dec { bytes, pos: 0 }; + let Some(mut s) = header(&mut d, act) else { + return "EMPTY\n".to_string(); + }; + for _ in 0..MAX_STEPS { + if step(&mut s, &mut d).is_none() { + break; + } + } + let mut out = s.out; + let _ = writeln!(out, "MAP0 {}", dump_at(&s.wz.trie, &[])); + let _ = writeln!(out, "MAP1 {}", dump_at(&s.rz.trie, &[])); + let _ = writeln!(out, "ROOT0 {}", hex_path(&s.wz.root)); + let _ = writeln!(out, "ROOT1 {}", hex_path(&s.rz.root)); + out +} + +// The resident-server protocol. Plumbing only — it knows nothing about tries, diff --git a/differential/src/reference/laws.rs b/differential/src/reference/laws.rs new file mode 100644 index 00000000..32e25b9f --- /dev/null +++ b/differential/src/reference/laws.rs @@ -0,0 +1,401 @@ +//! `Spec.lean` §2 — checkable laws. +//! +//! Metamorphic properties relating *different* API functions: "`join` is +//! commutative", "`take_map` then `graft_map` is the identity", "`drop_head` +//! undoes `insert_prefix`". +//! +//! Each returns `true` when it holds, so the same function can serve as a unit +//! test over the fixture battery *and* as a runtime assertion on every state a +//! fuzzer reaches. A property that holds in the model but fails in `pathmap` is +//! a crate bug; one that fails in both is a spec bug. Both are worth finding, +//! which is why the laws are kept separate from the definitions. +//! +//! The Lean model's §1 — the *proved* theorems about the cursor algebra — has no +//! counterpart here. Those are proofs, not tests; see +//! `lean/PathMapModel/Spec.lean`. + +use crate::reference::basic::{AlgStatus, ByteMask, ValOps, path}; +use crate::reference::map; +use crate::reference::pathmap::{Entry, PathMap}; +use crate::reference::zipper::Zip; + +/// Every location carrying a value exists. +/// +/// Structurally true of any [`PathMap`] — a value cannot be recorded at a +/// location that is not in the map — so a failure here means a map was built +/// by hand rather than through the constructors. +pub fn vals_exist(t: &PathMap) -> bool { + t.vals().all(|(k, _)| t.path_exists(k)) +} + +/// The set of existing locations is closed under taking prefixes. +pub fn prefix_closed(t: &PathMap) -> bool { + t.path_exists(&[]) && t.paths().all(|q| path::prefixes(q).all(|r| t.path_exists(r))) +} + +/// `child_mask` lists exactly the bytes whose child location exists. +pub fn child_mask_agrees(t: &PathMap, p: &[u8]) -> bool { + let m = t.child_mask(p); + m.bytes().iter().all(|&b| t.path_exists(&path::push(p, b))) + && t.paths().all(|q| { + if q.len() == p.len() + 1 && q.starts_with(p) { m.contains(q[p.len()]) } else { true } + }) +} + +/// `val_count` at the root counts exactly the entries of `iter`. +pub fn val_count_agrees(t: &PathMap) -> bool { + t.val_count(&[]) == t.vals().count() +} + +/// After `set_val` the focus is a location carrying exactly that value. +/// +/// One `Entry` case, rather than "the value is `v`" and "the path exists" +/// checked separately — the second was only there because the first could not +/// say it. +pub fn set_val_then_val(ops: &impl ValOps, z: &Zip, v: V) -> bool { + let mut z = z.clone(); + z.set_val(v.clone()); + match z.entry() { + Entry::Valued(w) => ops.beq(w, &v), + Entry::Bare | Entry::Absent => false, + } +} + +/// `remove_val` clears the value but leaves the location dangling — which is +/// to say the focus ends up exactly [`Entry::Bare`]. +pub fn remove_val_leaves_path(z: &Zip) -> bool { + if !z.path_exists() { + return true; + } + let mut z = z.clone(); + z.remove_val(false); + z.entry().is_bare() +} + +/// `create_path` makes the focus exist; a second call reports "already there". +pub fn create_path_idempotent(z: &Zip) -> bool { + if z.at_root() { + return true; + } + let mut z = z.clone(); + z.create_path(); + let existed = z.path_exists(); + let again = z.create_path(); + existed && !again && z.path_exists() +} + +/// `prune_path` undoes a `create_path` that dangled off an existing location. +/// +/// **Precondition: the location it dangles off must survive the prune.** +/// `create_path` adds one child to the deepest existing ancestor `a` of the +/// focus, and `prune_path` then walks back up to the first location that +/// carries a value, branches, or is the *map* root — it does not stop at the +/// zipper root (see [`Zip::prune_path`]). So when `a` is itself a dangling +/// tip, the prune sweeps `a` up as well and the map does not come back. With +/// `{[] , [2]}` and a zipper rooted at `[2]` focused at `[2,0]`, create then +/// prune leaves `{[]}`: `[2]` is gone. Verified to hold in the Lean model +/// too, so this is the law's scope, not a divergence. +/// +/// `Check.lean` never sees it because every zipper in its battery is rooted +/// at `[]`; check [`create_then_prune_applies`] before asserting this one. +pub fn create_then_prune(ops: &impl ValOps, z: &Zip) -> bool { + if z.at_root() || z.path_exists() { + return true; + } + let mut w = z.clone(); + w.create_path(); + w.prune_path(); + w.trie.beq_t(ops, &z.trie) +} + +/// Whether [`create_then_prune`]'s precondition holds: the deepest existing +/// ancestor of the focus is the map root, carries a value, or already has a +/// child — in each case it still qualifies as a stop once `create_path` has +/// hung one more child off it. +pub fn create_then_prune_applies(z: &Zip) -> bool { + let f = z.focus(); + match path::deepest_proper_prefix(&f, |a| z.trie.path_exists(a)) { + Some(a) => a.is_empty() || z.trie.val_at(a).is_some() || z.trie.child_count(a) >= 1, + None => true, + } +} + +/// `descend_to_existing` always lands on an existing location, provided the +/// focus existed when it was called. +pub fn descend_to_existing_lands(z: &Zip, k: &[u8]) -> bool { + if !z.path_exists() { + return true; + } + let mut z = z.clone(); + z.descend_to_existing(k); + z.path_exists() +} + +/// `descend_indexed_byte` lands on an existing location for every valid index, +/// and the byte it reports is the byte it landed on. +pub fn descend_indexed_lands(z: &Zip) -> bool { + (0..z.child_count()).all(|i| { + let mut w = z.clone(); + match w.descend_indexed_byte(i) { + Some(b) => w.path_exists() && w.focus_byte() == Some(b), + None => false, + } + }) +} + +/// `to_next_sibling_byte` and `to_prev_sibling_byte` are mutually inverse — +/// **but only from an existing location**. +/// +/// From an off-map focus the round trip fails, and correctly so: the sibling +/// moves are defined by the parent's `child_mask`, which does not contain the +/// current byte, so `next_bit` jumps to some larger set byte and `prev_bit` +/// comes back to a *different* one. A caller that steps sideways from a +/// non-existent path cannot expect to step back. +pub fn sibling_round_trip(z: &Zip) -> bool { + if !z.path_exists() { + return true; + } + let mut w = z.clone(); + match w.to_next_sibling_byte() { + None => true, + Some(b) => { + w.focus_byte() == Some(b) + && w.to_prev_sibling_byte().is_some() + && w.path == z.path + } + } +} + +/// `descend_until_observed` reports exactly the bytes it descended. +/// +/// A blind zipper has no `path()`, so this sequence is its only account of +/// where it went; it must equal the path delta. +#[allow(clippy::nonminimal_bool)] // stated as the Lean law is, not simplified +pub fn descend_until_observed_exact(z: &Zip) -> bool { + let mut w = z.clone(); + let (moved, obs) = w.descend_until_observed(); + w.path == path::cat(&z.path, &obs) && moved == !obs.is_empty() +} + +/// `ascend_until` never ascends past the zipper's root, and reports the exact +/// distance travelled. +pub fn ascend_until_accounts(z: &Zip) -> bool { + let mut w = z.clone(); + let n = w.ascend_until(); + w.path.len() + n == z.path.len() && z.path.len() >= n +} + +/// `to_next_val` enumerates values in strictly increasing depth-first order +/// and finishes at the root. +pub fn to_next_val_monotone(z: &Zip) -> bool { + let mut w = z.clone(); + if w.to_next_val() { + z.path < w.path && w.is_val() + } else { + w.at_root() + } +} + +/// `join` is commutative **on locations**, but not on values. +/// +/// `pathmap`'s `u64` (and `usize`, `u32`, `u16`, `u8`) `Lattice` instances +/// define `pjoin` as `Identity(SELF_IDENT)` — a left-biased projection. So +/// joining two maps that disagree at a key keeps whichever value belongs to +/// the receiver, and `a.join(b)` and `b.join(a)` differ. The *set of paths* +/// is still symmetric, and that is what this law asserts. Any test that +/// assumes value-level commutativity is asserting something the crate does not +/// promise for these value types. +pub fn join_comm_on_paths( + ops: &impl ValOps, + a: &PathMap, + b: &PathMap, +) -> bool { + let ab = PathMap::join(ops, a, b); + let ba = PathMap::join(ops, b, a); + ab.paths().collect::>() == ba.paths().collect::>() + && ab.vals().map(|(k, _)| k).collect::>() + == ba.vals().map(|(k, _)| k).collect::>() +} + +/// `join` is idempotent. +pub fn join_idem(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::join(ops, a, a).beq_t(ops, a) +} + +/// `join` is associative, values included: left-biasing is itself associative. +pub fn join_assoc( + ops: &impl ValOps, + a: &PathMap, + b: &PathMap, + c: &PathMap, +) -> bool { + let l = PathMap::join(ops, &PathMap::join(ops, a, b), c); + let r = PathMap::join(ops, a, &PathMap::join(ops, b, c)); + l.beq_t(ops, &r) +} + +/// `meet` is idempotent *on values*. +pub fn meet_idem_on_vals(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::meet(ops, a, a).vals().map(|(k, _)| k.clone()).collect::>() + == a.vals().map(|(k, _)| k.clone()).collect::>() +} + +/// `meet` is idempotent outright, dangling paths included: a location survives +/// exactly when both operands have it. +pub fn meet_idem(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::meet(ops, a, a).beq_t(ops, a) +} + +/// `meet_pruned(a, a)` is `a` with its dangling paths dropped. +pub fn meet_pruned_idem(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::meet_pruned(ops, a, a).beq_t(ops, &a.drop_dangling()) +} + +/// `meet` is commutative on locations. +pub fn meet_comm_on_paths(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> bool { + PathMap::meet(ops, a, b).paths().collect::>() == PathMap::meet(ops, b, a).paths().collect::>() +} + +/// Subtracting a map from itself leaves no values. +pub fn sub_self_empty_vals(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::sub(ops, a, a).vals().next().is_none() +} + +/// `restrict a a = a`. This is the law that caught a real `prestrict` bug — +/// see `tests/pathmap_algebra_differential.rs`. +/// +/// **Precondition: every location of `a` leads to a value.** The Lean +/// model's justification — "every path of `a` is validated by the value at +/// its own end" — assumes every path *ends* at a value, which is exactly what +/// `create_path` and `remove_val(false)` make false. On +/// `PathMap::empty().add_path(&[0])` the law fails, in this model and in the +/// Lean one alike: `[0]` has no valued prefix, so `restrict` drops it. +/// `Check.lean`'s six fixtures all happen to satisfy the precondition, which +/// is why the `#guard` holds there; a fuzzer reaching a dangling path does +/// not, so check [`restrict_self_applies`] before asserting this one. +pub fn restrict_self(ops: &impl ValOps, a: &PathMap) -> bool { + map::restrict(a, a).beq_t(ops, a) +} + +/// Whether [`restrict_self`]'s precondition holds: every non-root location of +/// `a` has a value at or above it. +pub fn restrict_self_applies(a: &PathMap) -> bool { + a.paths().all(|q| q.is_empty() || PathMap::validated_by(a, q)) +} + +/// `take_map` followed by `graft_map` restores the map exactly. +pub fn take_then_graft(ops: &impl ValOps, z: &Zip) -> bool { + let mut w = z.clone(); + let m = w.take_map(false).unwrap_or_else(PathMap::empty); + w.graft_map(&m); + w.trie.beq_t(ops, &z.trie) +} + +/// Grafting one submap into two places makes two *independent* copies: +/// writing under one leaves the other exactly as it was. +/// +/// In the model this cannot fail — a [`PathMap`] is a flat map of entries, so +/// the two copies are separate elements and there is no aliasing to leak +/// through. The law is stated anyway for two reasons: it is what the crate +/// must do (`graft` clones a refcounted pointer, so the copies really are +/// shared until copy-on-write separates them), and it would catch a model +/// that grew sharing of its own. +/// +/// See FINDINGS.md #16: the crate gets this right where it completes, but +/// aborts when the shared submap contains a dangling path. +pub fn grafted_copies_independent( + ops: &impl ValOps, + s: &PathMap, + a: &[u8], + b: &[u8], + k: &[u8], + v: V, +) -> bool { + let mut z = Zip::at_path(PathMap::empty(), &[], a); + z.graft_map(s); + let mut z = Zip::at_path(z.trie, &[], b); + z.graft_map(s); + let both = z.trie.clone(); + let mut z = Zip::at_path(z.trie, &[], &path::cat(a, k)); + z.set_val(v); + both.subtrie(b).beq_t(ops, &z.trie.subtrie(b)) +} + +/// `graft` copies the source submap: after grafting, `make_map` at the +/// destination equals `make_map` at the source. +pub fn graft_then_make_map( + ops: &impl ValOps, + dst: &Zip, + src: &Zip, +) -> bool { + let mut d = dst.clone(); + d.graft(src); + d.make_map().beq_t(ops, &src.make_map()) +} + +/// `drop_head k` undoes `insert_prefix` of a `k`-byte prefix, as documented +/// on `ZipperWriting::insert_prefix`. +/// +/// Only the *branches* are compared: `insert_prefix` does not move the focus +/// value, and `drop_head` discards values at depth exactly `k`, so the focus +/// value plays no part on either side. +pub fn drop_head_undoes_insert_prefix( + ops: &impl ValOps, + z: &Zip, + pre: &[u8], +) -> bool { + if z.focus_node_is_empty() || pre.is_empty() { + return true; + } + let mut w = z.clone(); + w.insert_prefix(pre); + w.join_k_path_into(ops, pre.len(), false); + w.focus_node().beq_t(ops, &z.focus_node()) +} + +/// `join_into` with an empty source is a no-op and reports `Identity` (or +/// `None` when the destination is empty too). +pub fn join_empty_identity(ops: &impl ValOps, z: &Zip) -> bool { + let src = Zip::new(PathMap::empty()); + let mut w = z.clone(); + let st = w.join_into(ops, &src); + let want = if z.focus_node_is_empty() { AlgStatus::None } else { AlgStatus::Identity }; + w.trie.beq_t(ops, &z.trie) && st == want +} + +/// Joining a zipper into itself is the identity, and reports it. +pub fn join_self_identity(ops: &impl ValOps, z: &Zip) -> bool { + let src = z.clone(); + let mut w = z.clone(); + let st = w.join_into(ops, &src); + let want = if z.focus_node_is_empty() { AlgStatus::None } else { AlgStatus::Identity }; + w.trie.beq_t(ops, &z.trie) && st == want +} + +/// `remove_branches` empties the focus but preserves its value. +pub fn remove_branches_keeps_val(ops: &impl ValOps, z: &Zip) -> bool { + let mut w = z.clone(); + w.remove_branches(false); + w.focus_node_is_empty() + && match (z.val(), w.val()) { + (None, None) => true, + (Some(a), Some(b)) => ops.beq(a, b), + _ => false, + } +} + +/// `remove_unmasked_branches` with a full mask changes nothing. +pub fn remove_unmasked_full_mask(ops: &impl ValOps, z: &Zip) -> bool { + let mut w = z.clone(); + w.remove_unmasked_branches(&ByteMask::full(), false); + w.trie.beq_t(ops, &z.trie) +} + +/// `remove_unmasked_branches` with an empty mask equals `remove_branches`. +pub fn remove_unmasked_empty_mask(ops: &impl ValOps, z: &Zip) -> bool { + let mut a = z.clone(); + a.remove_unmasked_branches(&ByteMask::of_list([]), false); + let mut b = z.clone(); + b.remove_branches(false); + a.trie.beq_t(ops, &b.trie) +} diff --git a/differential/src/reference/map.rs b/differential/src/reference/map.rs new file mode 100644 index 00000000..001488ad --- /dev/null +++ b/differential/src/reference/map.rs @@ -0,0 +1,153 @@ +//! `Map.lean` — the `PathMap` surface. +//! +//! A `pathmap::PathMap` *is* a map: the value at the empty path is the map's +//! root value. Almost every `PathMap` method is implemented in `pathmap` by +//! opening a temporary zipper at the root and delegating, and the model does the +//! same, so the two layers cannot drift apart. +//! +//! The one genuinely map-level operation is [`restrict`], which differs from +//! [`Zip::restrict`]: it consults the *root value* of the right-hand map, which a +//! node-level `prestrict` cannot see. + +use crate::reference::basic::{ByteMask, ValOps}; +use crate::reference::pathmap::{Entry, PathMap}; +use crate::reference::zipper::Zip; + +/// `PathMap::new`. +pub fn new() -> PathMap { + PathMap::empty() +} + +/// `PathMap::single`. +pub fn single(p: &[u8], v: V) -> PathMap { + let mut t = PathMap::empty(); + t.set_val(p, v); + t +} + +/// A read/write zipper at the root of the map. +pub fn zipper(t: PathMap) -> Zip { + Zip::new(t) +} + +/// A zipper rooted at `p` (`read_zipper_at_path` / `write_zipper_at_path`). +/// The zipper cannot ascend above `p`, and its `val()` at the root is the +/// map's value at `p`. +pub fn zipper_at(t: PathMap, p: &[u8]) -> Zip { + Zip::at(t, p) +} + +/// `PathMap::get_val_at` / `get`. +pub fn get_val_at<'a, V: Clone>(t: &'a PathMap, p: &[u8]) -> Option<&'a V> { + t.val_at(p) +} + +/// `PathMap::contains`: whether there is a **value** at `p`. +/// +/// Not the same question as `path_exists_at`, and the gap between them is the +/// bare case: a path created by `create_path`, or left behind by +/// `remove_val(false)`, exists but is not contained. +pub fn contains(t: &PathMap, p: &[u8]) -> bool { + match t.entry_at(p) { + Entry::Valued(_) => true, + Entry::Bare | Entry::Absent => false, + } +} + +/// `PathMap::path_exists_at`. +pub fn path_exists_at(t: &PathMap, p: &[u8]) -> bool { + t.path_exists(p) +} + +/// `PathMap::set_val_at` / `insert`. +pub fn set_val_at(t: &mut PathMap, p: &[u8], v: V) -> Option { + t.set_val(p, v) +} + +/// `PathMap::remove_val_at` / `remove` (`remove` passes `prune = true`). +pub fn remove_val_at(t: &mut PathMap, p: &[u8], prune: bool) -> Option { + with_zipper_at(t, p, |z| z.remove_val(prune)) +} + +/// `PathMap::val_count`. +pub fn val_count(t: &PathMap) -> usize { + t.val_count(&[]) +} + +/// `PathMap::is_empty`. +pub fn is_empty(t: &PathMap) -> bool { + t.is_empty_map() +} + +/// `PathMap::create_path`. +pub fn create_path(t: &mut PathMap, p: &[u8]) -> bool { + with_zipper_at(t, p, |z| z.create_path()) +} + +/// `PathMap::prune_path`. +pub fn prune_path(t: &mut PathMap, p: &[u8]) -> usize { + with_zipper_at(t, p, |z| z.prune_path()) +} + +/// `PathMap::remove_branches_at`. +pub fn remove_branches_at(t: &mut PathMap, p: &[u8], prune: bool) -> bool { + with_zipper_at(t, p, |z| z.remove_branches(prune)) +} + +/// All key/value pairs, in depth-first (lexicographic) key order — +/// `PathMap::iter`. +pub fn iter(t: &PathMap) -> Vec<(Vec, V)> { + t.vals().map(|(k, v)| (k.clone(), v.clone())).collect() +} + +/// `PathMap::join`: union, root values included. +pub fn join(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + PathMap::join(ops, a, b) +} + +/// `PathMap::meet`: intersection, root values included. +pub fn meet(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + PathMap::meet(ops, a, b) +} + +/// `PathMap::subtract`: difference, root values included. +pub fn subtract(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + PathMap::sub(ops, a, b) +} + +/// `PathMap::restrict`: keep the paths of `a` that have *some* prefix +/// carrying a value in `b`. +/// +/// The empty prefix counts here, so a root value in `b` validates everything +/// and the result is `a` unchanged (root value included). Otherwise the +/// result never has a root value, because a node-level `prestrict` has no +/// slot for one. This is the documented behaviour of `PathMap::restrict`, +/// and it is *not* what [`Zip::restrict`] does. +pub fn restrict(a: &PathMap, b: &PathMap) -> PathMap { + if b.val_at(&[]).is_some() { + a.clone() + } else { + PathMap::restrict_below_root(a, b) + } +} + +/// Every `PathMap` method above that mutates is `pathmap`'s "open a temporary +/// zipper at the root, descend, delegate" — spelled out once here so the two +/// layers cannot drift apart. +fn with_zipper_at( + t: &mut PathMap, + p: &[u8], + f: impl FnOnce(&mut Zip) -> R, +) -> R { + let mut z = Zip::new(std::mem::take(t)); + z.descend_to(p); + let r = f(&mut z); + *t = z.trie; + r +} + +/// Re-exported so callers of [`Zip::remove_unmasked_branches`] have a mask to +/// hand without reaching for the crate's own. +pub fn full_mask() -> ByteMask { + ByteMask::full() +} diff --git a/differential/src/reference/mod.rs b/differential/src/reference/mod.rs new file mode 100644 index 00000000..b65c5758 --- /dev/null +++ b/differential/src/reference/mod.rs @@ -0,0 +1,164 @@ +//! An executable reference model of the `pathmap` trie and zipper API: a Rust +//! transcription of the Lean 4 specification in `lean/PathMapModel/`. +//! +//! See `lean/README.md` for the reasoning behind the specification and +//! `lean/FINDINGS.md` for what it found. Every item names the `pathmap` item it +//! specifies, and the file layout mirrors the Lean one one-for-one, so drift +//! between the two is visible as a diff rather than hidden inside a module: +//! +//! | Lean | here | +//! |--------------------------|-----------------| +//! | `Basic.lean` | `basic.rs` | +//! | `PathMap.lean` | `pathmap.rs` | +//! | `Zipper.lean` | `zipper.rs` | +//! | `Write.lean` | `write.rs` | +//! | `Map.lean` | `map.rs` | +//! | `Spec.lean` §2 | `laws.rs` | +//! | `Check.lean` | `check.rs` | +//! | `Fuzz.lean` | `fuzz.rs` | +//! +//! It lives in the `differential` crate rather than in `pathmap/src/` because +//! nothing in the library may depend on it: it is a testing oracle, not a data +//! structure, and keeping it out of the crate proper is what makes "the model +//! shares no code with the implementation" a fact about the build rather than a +//! promise in a comment. +//! +//! # This is not a trie, and that is the point +//! +//! A trie is a prefix tree, which `pathmap` implements with four node types and +//! a great deal of care. The model is a flat `BTreeMap, Option>` of +//! whole paths. It is not a prefix tree and does not try to be. +//! +//! That is deliberate. This is a *specification*, and what it specifies is the +//! meaning a trie carries, not the trie. A model shaped like the implementation +//! would inherit the implementation's structure, and then a bug in how that +//! structure is handled — a node type promoted wrongly, a child index off by +//! one, an empty node where a real one was expected — could be present in both +//! and cancel out. Findings 14, 15 and 16 are bugs of exactly that kind, and +//! they are visible only because the model has no nodes to get wrong. +//! +//! # The model must never reach into the crate +//! +//! No file in this directory may `use pathmap::...`; not `utils::ByteMask`, not +//! `ring::AlgebraicStatus`, not a helper. Shared code is shared risk. In the +//! archive this was enforced by the build — the model was its own cargo example +//! target, which did not depend on `pathmap` at all — and it now lives beside +//! [`crate::harness`], which does. So the rule is checked instead, by the +//! `model_does_not_touch_the_crate` test below, which reads these files and +//! fails if any of them names the crate. +//! +//! The comparison between model and crate belongs in the harness and in +//! `bin/in_process.rs`, not here. +//! +//! # Why `BTreeMap, Option>` +//! +//! The Lean model keeps a list of `(Path, Option V)` pairs held canonical by +//! hand: sorted by the lexicographic path order, duplicate-free, prefix-closed, +//! and containing the empty path. Rust's `Ord for Vec` **is** that order — +//! `[] < [0] < [0,0] < [0,255] < [1]` — which is the order a depth-first +//! traversal of a radix trie visits paths in. So a `BTreeMap` discharges +//! "sorted" and "duplicate-free" structurally, its iteration order is depth-first +//! order for free, and every "the least existing path after the focus such +//! that ..." in the specification becomes a range query. Prefix-closure and the +//! presence of the root remain the model's own responsibility; +//! `PathMap::mk` establishes them and [`laws::prefix_closed`] checks +//! them. +//! +//! Structural equality of two canonical maps is therefore observational +//! equality, which is what lets the model decide `AlgebraicStatus::Identity` +//! against `Element` (see `PathMap::beq_t`). +//! +//! # Deviations from the Lean model +//! +//! * The Lean model is purely functional: every operation returns a new `Zip`. +//! Here the mutating operations take `&mut self` and return only what the +//! corresponding `pathmap` method returns, so the trace front end can call the +//! two side by side. Where a law needs the prior state, it clones. +//! * `Zip` owns its `PathMap` (as in Lean, where a read zipper holds a snapshot +//! and a write zipper holds the live map). Operations that read one zipper +//! and write another take the source by reference. +//! * The proved theorems of `Spec.lean` §1 have no counterpart: they are proofs, +//! not tests. The checkable laws of §2 are in [`laws`]. +//! * `V: Clone` is required throughout; the Lean model is generic over any `V`. +//! +//! # The three front ends +//! +//! | binary | drives | +//! |---|---| +//! | `lean/.lake/build/bin/pathmap-oracle` | the Lean model (`lean/PathMapModel/Fuzz.lean`) | +//! | `target/release/pathmap_trace` | the real crate ([`crate::harness`]) | +//! | `target/release/reference` | the Rust model in this directory | +//! +//! `lean/differential.py` diffs any two of them. Model-against-model — +//! `--model` — is the acceptance test for this port: the two are independent +//! transcriptions of the same specification in different languages, so a diff +//! means one of them is wrong and nothing about the crate is in question. +//! +//! Once that holds, `target/release/in_process` drops the pipes: it runs +//! [`fuzz::run`] and [`crate::harness::run`] on the same bytes in one process +//! and compares the traces in memory. + +// The model is a specification: it defines the whole API surface whether or not +// any particular front end happens to call each item. `laws` and `map` are +// reached only from `check.rs`, under `cfg(test)`. +#![allow(dead_code)] + +pub mod basic; +pub mod pathmap; +pub mod zipper; +pub mod write; +pub mod map; +pub mod laws; + +#[cfg(test)] +mod check; + +pub mod fuzz; + +#[cfg(test)] +mod tests { + /// The model may not name the crate it is a model of. + /// + /// In the archive this was a property of the build: `examples/reference/` + /// was its own cargo target and `pathmap` was not in scope for it at all. + /// Here the model sits in a crate that does depend on `pathmap`, so the + /// invariant has to be asserted rather than obtained for free — otherwise a + /// single `use pathmap::utils::ByteMask` would quietly make the model share + /// the implementation's bugs, and the differential would agree for the wrong + /// reason. + #[test] + fn model_does_not_touch_the_crate() { + // Spelled in halves so this file, which is itself part of the model + // directory and is scanned like the rest, does not match its own needle. + let crate_path = ["path", "map::"].concat(); + let extern_crate = ["extern crate ", "pathmap"].concat(); + let own_map = ["crate::reference::path", "map"].concat(); + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/reference"); + let mut offenders = Vec::new(); + for entry in std::fs::read_dir(&dir).expect("src/reference must exist") { + let path = entry.expect("readable dir entry").path(); + if path.extension().and_then(|e| e.to_str()) != Some("rs") { + continue; + } + let src = std::fs::read_to_string(&path).expect("readable model file"); + for (n, line) in src.lines().enumerate() { + // Doc comments talk *about* the crate; code may not name it. + let code = line.trim_start(); + if code.starts_with("//") { + continue; + } + // The model's *own* map module is also called `pathmap`, so + // strip its path before looking for the crate's. + let code = code.replace(&own_map, ""); + if code.contains(&crate_path) || code.contains(&extern_crate) { + offenders.push(format!("{}:{}: {}", path.display(), n + 1, line.trim())); + } + } + } + assert!( + offenders.is_empty(), + "the reference model must not reach into the crate it models:\n{}", + offenders.join("\n") + ); + } +} diff --git a/differential/src/reference/pathmap.rs b/differential/src/reference/pathmap.rs new file mode 100644 index 00000000..ffda0f9c --- /dev/null +++ b/differential/src/reference/pathmap.rs @@ -0,0 +1,598 @@ +//! `PathMap.lean` — what a `pathmap` trie means. +//! +//! See the module docs in `main.rs` for why the representation is a flat +//! `BTreeMap, Option>` and not a prefix tree. + +use std::collections::BTreeMap; +use std::collections::btree_map::Entry as BEntry; +use std::ops::Bound; + +use crate::reference::basic::{ByteMask, ValOps, path}; + +// =========================================================================== +// PathMap.lean +// =========================================================================== + +/// What a map holds at a path — the whole answer, in one value. +/// +/// Named for `HashMap`/`BTreeMap`'s `Entry`, and standing in the same relation +/// to the storage: an element of [`PathMap::entries`] is a location that is +/// *there*, while an `Entry` also answers for a path that is not — Rust's +/// `Occupied` and `Vacant`. The difference is that a `pathmap` location has +/// **three** states rather than two, and that third one is the whole reason this +/// type exists. +/// +/// `create_path` produces it, `remove_val(false)` leaves it behind, and findings +/// 7, 8 and 15 are all about operations that mishandle it. Asking through +/// `Entry` rather than through `path_exists` and `val` separately means a +/// definition cannot quietly forget that case: the `match` will not compile +/// until it says what happens. [`Entry::Valued`] implies the location exists, so +/// the impossible combination — a value at a path that is not there — is +/// unrepresentable rather than merely untrue. +/// +/// Generic over the value slot so it can be returned borrowed (`Entry<&V>`) or +/// owned (`Entry`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Entry { + /// The path is not in the map. Rust's `Vacant`. + Absent, + /// The path is in the map but carries no value — a location `path_exists` + /// reports `true` for and `val` reports `None` for. + Bare, + /// The path is in the map and carries this value. Rust's `Occupied`. + Valued(V), +} + +impl Entry { + /// The value, if any — what `ZipperValues::val` reports. + pub fn val(self) -> Option { + match self { + Entry::Valued(v) => Some(v), + _ => None, + } + } + + /// Whether the location is in the map — what `Zipper::path_exists` reports. + /// + /// Holding a value entails being there: there is no constructor for the + /// combination, so the Lean model's `present_of_val` needs no counterpart. + pub fn present(&self) -> bool { + !matches!(self, Entry::Absent) + } + + /// Whether the location is there but holds nothing. + pub fn is_bare(&self) -> bool { + matches!(self, Entry::Bare) + } +} + +/// A `pathmap` trie: a finite path→value map *plus* the prefix-closed set of +/// existing locations. +/// +/// Everything rests on one observation. A `pathmap` trie is not just a +/// path→value map: `create_path` makes a location that exists *without* carrying +/// a value, and `remove_val(false)` leaves one behind. So a location, and a +/// value at that location, are separate facts, and both are recorded — in one +/// map rather than a value map beside a path set, so that a value cannot be +/// recorded at a location that does not exist. A `None` here is exactly a +/// dangling path. +/// +/// # Canonical form +/// +/// * sorted and duplicate-free — structural, from `BTreeMap`; +/// * prefix-closed, and containing the empty path — maintained by the +/// constructors, checked by [`laws::prefix_closed`]. +/// +/// Canonical form makes structural equality observational equality, which is +/// what lets [`Zip`] decide `AlgebraicStatus::Identity` against `Element`. +#[derive(Clone, Debug)] +pub struct PathMap { + /// Every location that exists, in depth-first order, each carrying its value + /// if it has one. + pub entries: BTreeMap, Option>, +} + +impl Default for PathMap { + fn default() -> Self { + Self::empty() + } +} + +impl PathMap { + // -- construction ------------------------------------------------------- + + /// The empty trie: the root exists, nothing else does. + pub fn empty() -> Self { + let mut entries = BTreeMap::new(); + entries.insert(Vec::new(), None); + PathMap { entries } + } + + /// Build a canonical map from raw components — the Lean model's `mk'`. + /// + /// `paths` is completed with every prefix of every raw path and of every key + /// carrying a value, so callers never have to maintain closure by hand. + /// + /// **Left-biased**, matching the Lean `dedupVals` and `pathmap`'s + /// `Identity(SELF_IDENT)` value instances: the *first* binding offered for a + /// key wins. (`BTreeMap::insert` keeps the last, which is why this goes + /// through `or_insert`.) + pub fn mk( + vals: impl IntoIterator, V)>, + paths: impl IntoIterator>, + ) -> Self { + let mut vs: BTreeMap, V> = BTreeMap::new(); + for (k, v) in vals { + if let BEntry::Vacant(e) = vs.entry(k) { + e.insert(v); + } + } + let mut t = PathMap::empty(); + for p in paths { + t.ensure_path(&p); + } + for (k, v) in vs { + t.ensure_path(&k); + t.entries.insert(k, Some(v)); + } + t + } + + /// Make `p` and every prefix of it exist, without disturbing any value + /// already recorded along the way. + fn ensure_path(&mut self, p: &[u8]) { + for n in 0..=p.len() { + self.entries.entry(p[..n].to_vec()).or_insert(None); + } + } + + // -- ranges ------------------------------------------------------------- + // + // The three shapes of query the specification needs. Locations sharing a + // prefix are contiguous in `BTreeMap` order, so each is a range plus a + // `take_while`, and each stops as soon as it leaves the subtree. + + /// Every location at or below `p`, in depth-first order. + pub(crate) fn at_or_below<'a>(&'a self, p: &'a [u8]) -> impl Iterator, &'a Option)> { + self.entries + .range::<[u8], _>((Bound::Included(p), Bound::Unbounded)) + .take_while(move |(q, _)| q.starts_with(p)) + } + + /// Every location strictly below `p`, in depth-first order. + pub(crate) fn strictly_below<'a>( + &'a self, + p: &'a [u8], + ) -> impl Iterator, &'a Option)> { + self.entries + .range::<[u8], _>((Bound::Excluded(p), Bound::Unbounded)) + .take_while(move |(q, _)| q.starts_with(p)) + } + + /// Every location strictly after `from` and at or below `within`, in + /// depth-first order. `within` must be a prefix of `from`. + pub(crate) fn after_within<'a>( + &'a self, + from: &'a [u8], + within: &'a [u8], + ) -> impl Iterator, &'a Option)> { + debug_assert!(from.starts_with(within)); + self.entries + .range::<[u8], _>((Bound::Excluded(from), Bound::Unbounded)) + .take_while(move |(q, _)| q.starts_with(within)) + } + + // -- observations ------------------------------------------------------- + // + // These are the entire observable interface of a map; every law is phrased + // in terms of them. + + /// What the map holds at `p`. + pub fn entry_at(&self, p: &[u8]) -> Entry<&V> { + match self.entries.get(p) { + None => Entry::Absent, + Some(None) => Entry::Bare, + Some(Some(v)) => Entry::Valued(v), + } + } + + /// `Zipper::val` / `PathMap::get_val_at`: the value at `p`, if any. + pub fn val_at(&self, p: &[u8]) -> Option<&V> { + self.entry_at(p).val() + } + + /// `Zipper::path_exists`: whether `p` is a location in the map. True for + /// dangling paths (locations with no value and no children). + pub fn path_exists(&self, p: &[u8]) -> bool { + self.entries.contains_key(p) + } + + /// Every existing location, in depth-first order. + pub fn paths(&self) -> impl Iterator> { + self.entries.keys() + } + + /// Every location that carries a value, with it, in depth-first order — + /// `PathMap::iter`. + pub fn vals(&self) -> impl Iterator, &V)> { + self.entries + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))) + } + + /// `Zipper::child_mask`: the bytes `b` for which `p ++ [b]` exists. + pub fn child_mask(&self, p: &[u8]) -> ByteMask { + ByteMask::of_list( + self.strictly_below(p) + .filter(|(q, _)| q.len() == p.len() + 1) + .map(|(q, _)| q[p.len()]), + ) + } + + /// `Zipper::child_count`. + pub fn child_count(&self, p: &[u8]) -> usize { + self.child_mask(p).count_bits() + } + + /// `ZipperMoving::val_count`: values at and below `p`. + pub fn val_count(&self, p: &[u8]) -> usize { + self.at_or_below(p).filter(|(_, v)| v.is_some()).count() + } + + /// The existing locations at or below `p`, in depth-first order. + pub fn paths_below(&self, p: &[u8]) -> Vec> { + self.at_or_below(p).map(|(q, _)| q.clone()).collect() + } + + /// `TrieNode::node_is_empty` applied to the node *below* `p`: no descendants + /// at all. + pub fn below_is_empty(&self, p: &[u8]) -> bool { + self.strictly_below(p).next().is_none() + } + + /// `PathMap::is_empty`: no values and an empty root node. + pub fn is_empty_map(&self) -> bool { + self.vals().next().is_none() && self.below_is_empty(&[]) + } + + /// Whether the map is in canonical form. + /// + /// Sortedness and freedom from duplicates are structural — `BTreeMap` cannot + /// represent a violation — so what is left to check is that the root exists + /// and that the set of locations is prefix-closed. Every constructor here + /// establishes both; this is the assertion a harness can make after each + /// operation to catch one that does not. + pub fn is_canonical(&self) -> bool { + self.path_exists(&[]) + && self + .entries + .keys() + .all(|q| (0..q.len()).all(|n| self.entries.contains_key(&q[..n]))) + } + + /// Structural (hence observational) equality. Used to decide + /// `AlgebraicStatus::Identity` — `pathmap` reports `Identity(SELF_IDENT)` + /// exactly when the operation's output equals `self`. + /// + /// Values are compared with [`ValOps::beq`], not with `PartialEq`: the + /// notion of equality that decides the status is the value type's own. + pub fn beq_t(&self, ops: &impl ValOps, other: &PathMap) -> bool { + self.entries.len() == other.entries.len() + && self + .entries + .iter() + .zip(other.entries.iter()) + .all(|((ka, va), (kb, vb))| { + ka == kb + && match (va, vb) { + (None, None) => true, + (Some(x), Some(y)) => ops.beq(x, y), + _ => false, + } + }) + } + + // -- sub-maps and grafting --------------------------------------------- + + /// The submap rooted at `p`, **including** the value at `p` as its root + /// value. This is `make_map` / `take_map` under the default + /// `graft_root_vals` feature, and also what a zipper rooted at `p` sees. + /// Yields [`PathMap::empty`] when `p` does not exist. + pub fn subtrie(&self, p: &[u8]) -> PathMap { + let mut entries: BTreeMap, Option> = self + .at_or_below(p) + .map(|(q, v)| (q[p.len()..].to_vec(), v.clone())) + .collect(); + entries.entry(Vec::new()).or_insert(None); + PathMap { entries } + } + + /// Remove everything at and below `p`; `p` itself stops existing. + /// + /// The root always survives: a map with no locations at all is not + /// representable, and `mk'` in the Lean model re-adds it the same way. + pub fn remove_at(&mut self, p: &[u8]) { + let doomed: Vec> = self.at_or_below(p).map(|(q, _)| q.clone()).collect(); + for q in doomed { + self.entries.remove(&q); + } + self.entries.entry(Vec::new()).or_insert(None); + } + + /// Remove everything strictly below `p`; `p` itself, and its value, are + /// untouched. This is `ZipperWriting::remove_branches` without pruning. + pub fn remove_below(&mut self, p: &[u8]) { + let doomed: Vec> = self.strictly_below(p).map(|(q, _)| q.clone()).collect(); + for q in doomed { + self.entries.remove(&q); + } + } + + /// Replace everything strictly below `p` with the strictly-below part of `s`. + /// + /// The value at `p` is *not* touched (`graft_internal` only ever replaces a + /// node); `p` is created iff `s` has any non-root content, mirroring the fact + /// that grafting an empty node neither creates nor destroys the location. + pub fn graft_below(&mut self, p: &[u8], s: &PathMap) { + self.remove_below(p); + let mut any = false; + for (q, v) in s.entries.iter() { + if q.is_empty() { + continue; + } + if !any { + // The first non-root entry is what creates `p` (and its own + // ancestors); an empty source leaves the location alone. + self.ensure_path(p); + any = true; + } + self.entries.insert(path::cat(p, q), v.clone()); + } + } + + // -- point updates ------------------------------------------------------ + + /// `ZipperWriting::set_val` / `PathMap::set_val_at`. Creates `p` if needed. + /// Returns the replaced value. + pub fn set_val(&mut self, p: &[u8], v: V) -> Option { + self.ensure_path(p); + self.entries.insert(p.to_vec(), Some(v)).flatten() + } + + /// `ZipperWriting::remove_val` *without* pruning: the location survives as a + /// dangling path. Returns the removed value. Never creates anything. + pub fn remove_val(&mut self, p: &[u8]) -> Option { + self.entries.get_mut(p).and_then(|slot| slot.take()) + } + + /// `ZipperWriting::create_path`: make `p` exist as a dangling path. + pub fn add_path(&mut self, p: &[u8]) { + self.ensure_path(p); + } + + // -- pruning ------------------------------------------------------------ + + /// Is `p` a dangling tip — an existing location with neither value nor + /// children? + pub fn is_dangling_tip(&self, p: &[u8]) -> bool { + self.entry_at(p).is_bare() && self.below_is_empty(p) + } + + /// The number of bytes `prune_path` would remove at focus `p` for a zipper + /// whose root sits at depth `root_len`. `0` means "nothing to prune". + /// + /// The chain is removed back to the deepest strict ancestor that must be + /// kept: one carrying a value, one that branches, or the one at the stop + /// depth. The ancestor at `root_len` always qualifies, so there is always an + /// answer, and `max` makes "never prunes above the stop depth" syntactic. + pub fn prune_count(&self, root_len: usize, p: &[u8]) -> usize { + if p.len() <= root_len || !self.is_dangling_tip(p) { + return 0; + } + let a = path::deepest_proper_prefix(p, |a| { + a.len() <= root_len || self.val_at(a).is_some() || self.child_count(a) >= 2 + }) + .map(|a| a.len()) + .unwrap_or(0); + p.len() - root_len.max(a) + } + + /// `ZipperWriting::prune_path`: returns the number of bytes removed. + pub fn prune_path(&mut self, root_len: usize, p: &[u8]) -> usize { + let n = self.prune_count(root_len, p); + if n != 0 { + self.remove_at(&p[..p.len() - n + 1]); + } + n + } +} + +// --------------------------------------------------------------------------- +// PathMap.lean — algebraic operations +// --------------------------------------------------------------------------- +// +// These lift `pathmap`'s node-level `pjoin` / `pmeet` / `psubtract` / +// `prestrict` to whole maps. The interesting content is *which locations +// survive*, since `pathmap` drops any node that ends up empty. + +impl PathMap { + /// `Option::pjoin` from `src/ring.rs`. + fn join_val(ops: &impl ValOps, a: Option<&V>, b: Option<&V>) -> Option { + match (a, b) { + (None, b) => b.cloned(), + (Some(a), None) => Some(a.clone()), + (Some(a), Some(b)) => ops.pjoin(a, b).resolve(a, b), + } + } + + /// `Option::pmeet`: a value survives only where *both* sides have one. + fn meet_val(ops: &impl ValOps, a: Option<&V>, b: Option<&V>) -> Option { + match (a, b) { + (Some(a), Some(b)) => ops.pmeet(a, b).resolve(a, b), + _ => None, + } + } + + /// `Option::psubtract`. + fn sub_val(ops: &impl ValOps, a: Option<&V>, b: Option<&V>) -> Option { + match (a, b) { + (None, _) => None, + (Some(a), None) => Some(a.clone()), + (Some(a), Some(b)) => ops.psub(a, b).resolve(a, b), + } + } + + /// Join (union). Every location of either side survives; colliding values + /// are combined with [`ValOps::pjoin`]. + pub fn join(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + let mut keys: Vec<&Vec> = a.vals().map(|(k, _)| k).collect(); + keys.extend(b.vals().map(|(k, _)| k)); + keys.sort_unstable(); + keys.dedup(); + let vals = keys.into_iter().filter_map(|k| { + Self::join_val(ops, a.val_at(k), b.val_at(k)).map(|v| (k.clone(), v)) + }); + // Collected first: `mk` takes the value list by value, and the path + // iterators borrow `a` and `b` at the same time. + let vals: Vec<(Vec, V)> = vals.collect(); + PathMap::mk(vals, a.paths().chain(b.paths()).cloned()) + } + + /// Meet (intersection). A location exists in the result exactly when it + /// exists in both operands, dangling or not, and a value exists exactly where + /// both operands hold one, as the meet of the two. So meet is commutative, + /// associative and idempotent on locations, and a meet with an equal trie + /// changes nothing. + /// + /// This is the meet with `prune = false`; see [`PathMap::meet_pruned`]. + pub fn meet(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + let vals: Vec<(Vec, V)> = a + .vals() + .filter_map(|(k, av)| { + Self::meet_val(ops, Some(av), b.val_at(k)).map(|v| (k.clone(), v)) + }) + .collect(); + PathMap::mk(vals, a.paths().filter(|q| b.path_exists(q)).cloned()) + } + + /// The trie with every dangling path removed: only the root and the locations + /// on the way to a value remain. + pub fn drop_dangling(&self) -> PathMap { + let vals: Vec<(Vec, V)> = self.vals().map(|(k, v)| (k.clone(), v.clone())).collect(); + PathMap::mk(vals, std::iter::empty()) + } + + /// The meet with `prune = true`, at its most pruned: the same values as + /// [`PathMap::meet`], and only the locations on the way to one of them. + /// + /// `pathmap` may stop short of this: it can skip a node shared with the source + /// rather than walk it, so a dangling path inside a shared node may survive. + /// The result always lies between `meet_pruned` and `meet`, and is not + /// compared by the fuzzer. + pub fn meet_pruned(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + Self::meet(ops, a, b).drop_dangling() + } + + /// Subtract. + /// + /// Two rules interact here. Where `b` has no node at all, `a`'s subtree is + /// kept verbatim — *including its dangling paths*. Where `b` does have a + /// node, only locations leading to a surviving value are kept. `pathmap` + /// gets this from `psubtract_dyn` short-circuiting on absent children; the + /// model reproduces it by splitting on whether the location leaves `b`. + pub fn sub(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + let surviving: Vec<(Vec, V)> = a + .vals() + .filter_map(|(k, av)| { + Self::sub_val(ops, Some(av), b.val_at(k)).map(|v| (k.clone(), v)) + }) + .collect(); + // The locations at which `a` leaves `b` entirely: below one of these, + // `psubtract_dyn` never looks again and `a` is copied verbatim. + let untouched: Vec<&Vec> = a + .paths() + .filter(|q| !q.is_empty() && !b.path_exists(q) && b.path_exists(&q[..q.len() - 1])) + .collect(); + let kept: Vec> = a + .paths() + .filter(|q| { + surviving.iter().any(|(k, _)| k.starts_with(q)) + || untouched.iter().any(|u| q.starts_with(u.as_slice())) + }) + .cloned() + .collect(); + PathMap::mk(surviving, kept) + } + + /// Is `q` *validated* by `b` — does some non-empty prefix of `q` carry a + /// value in `b`? + /// + /// This is the node-level reading of `prestrict`: a node has no root value, + /// so the empty prefix never validates. `PathMap::restrict` adds the empty + /// prefix back in (see [`map::restrict`]), which is why the map-level and + /// zipper-level operations disagree when the source has a root value. + pub fn validated_by(b: &PathMap, q: &[u8]) -> bool { + (1..=q.len()).any(|i| b.val_at(&q[..i]).is_some()) + } + + /// `prestrict` at node level: keep the locations of `a` validated by `b`. + /// Once a location is validated, everything below it is kept verbatim. + pub fn restrict_below_root(a: &PathMap, b: &PathMap) -> PathMap { + let vals: Vec<(Vec, V)> = a + .vals() + .filter(|(k, _)| Self::validated_by(b, k)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let paths: Vec> = a + .paths() + .filter(|q| Self::validated_by(b, q)) + .cloned() + .collect(); + PathMap::mk(vals, paths) + } + + // -- path surgery ------------------------------------------------------- + + /// `ZipperWriting::insert_prefix`: put `k` in front of every path below the + /// root. The root value is dropped — a node has nowhere to keep one. + pub fn insert_prefix_below(&self, k: &[u8]) -> PathMap { + let vals: Vec<(Vec, V)> = self + .vals() + .filter(|(q, _)| !q.is_empty()) + .map(|(q, v)| (path::cat(k, q), v.clone())) + .collect(); + let paths: Vec> = self + .paths() + .filter(|q| !q.is_empty()) + .map(|q| path::cat(k, q)) + .collect(); + PathMap::mk(vals, paths) + } + + /// The existing locations exactly `k` bytes below the root, in depth-first + /// order. + pub fn k_paths(&self, k: usize) -> Vec> { + self.paths().filter(|q| q.len() == k).cloned().collect() + } + + /// `drop_head` / `ZipperWriting::join_k_path_into` at node level: strip the + /// first `k` bytes from every path and join the results. + /// + /// Values sitting at depth *exactly* `k` are **discarded** — the joined node + /// has nowhere to put a root value. (`meet_k_path_into` keeps them, because + /// it routes through `take_map`/`graft_map`, which do carry root values. The + /// asymmetry is real.) + pub fn drop_head(&self, ops: &impl ValOps, k: usize) -> PathMap { + if k == 0 { + return self.clone(); + } + let mut acc = PathMap::empty(); + for q in self.k_paths(k) { + let mut m = self.subtrie(&q); + m.remove_val(&[]); + acc = PathMap::join(ops, &acc, &m); + } + acc + } +} + diff --git a/differential/src/reference/write.rs b/differential/src/reference/write.rs new file mode 100644 index 00000000..f351ec19 --- /dev/null +++ b/differential/src/reference/write.rs @@ -0,0 +1,712 @@ +//! `Write.lean` — the write zipper: everything that mutates `Zip::trie` +//! through the focus `root ++ path`. + +use crate::reference::basic::{AlgStatus, ByteMask, ValOps, path}; +use crate::reference::pathmap::{Entry, PathMap}; +use crate::reference::zipper::Zip; + +// =========================================================================== +// Write.lean — the write zipper +// =========================================================================== +// +// Everything here mutates `Zip::trie` through the focus `root ++ path`. Two +// invariants shape the whole API and are worth stating up front: +// +// 1. **A node is what lies strictly below a location.** `get_focus`, +// `graft_internal`, and every `*_dyn` algebraic primitive operate on nodes, so +// they never see or touch the value *at* the focus. Operations that do affect +// the focus value (`graft`, `graft_map`, `make_map`, `take_map`, +// `join_map_into`, `meet_into`, `subtract_into`) do it in a separate step — +// this is the `graft_root_vals` cargo feature, which is on by default and +// which the model assumes throughout. Note the resulting asymmetry: `graft` +// adopts the source's focus value but `join_into` does **not** join focus +// values. +// +// 2. **Pruning is opt-in and local.** A write leaves dangling paths behind +// unless `prune` is passed, and even then `prune_path` only fires when the +// focus is a dangling tip. + +impl Zip { + // -- values at the focus ------------------------------------------------ + + /// `ZipperWriting::get_val_mut` — same observation as [`Zip::val`], mutably. + /// Returns `Some` exactly when `val` does; it never creates anything. + pub fn get_val_mut(&mut self) -> Option<&mut V> { + let f = self.focus(); + self.trie.entries.get_mut(&f).and_then(|slot| slot.as_mut()) + } + + /// `ZipperWriting::set_val`: sets the value at the focus, **creating the + /// path** if it did not exist. Returns the replaced value. + pub fn set_val(&mut self, v: V) -> Option { + let f = self.focus(); + self.trie.set_val(&f, v) + } + + /// Writing through the reference `get_val_mut` returns. + /// + /// Specified as: when there is a value, this is `set_val`; when there is not, + /// it is a no-op — in particular it must **not** create the path, which is + /// what separates it from `set_val`. + pub fn get_val_mut_write(&mut self, v: V) -> Option { + match self.get_val_mut() { + Some(slot) => Some(std::mem::replace(slot, v)), + None => None, + } + } + + /// `ZipperWriting::get_val_or_set_mut`: the value at the focus, inserting + /// `default` if there is none. Inserting creates the path, as `set_val` does. + pub fn get_val_or_set_mut(&mut self, d: V) -> V { + let (v, _) = self.get_val_or_set_mut_with(d); + v + } + + /// `ZipperWriting::get_val_or_set_mut_with`: as above, but the value is + /// produced by a closure. + /// + /// The documented contract is that the closure supplies the value "if no + /// value exists", so it must be called **exactly when the focus has no + /// value** — calling it otherwise would be observable to any caller whose + /// closure has a side effect (allocating, taking a lock, bumping a counter). + /// The second component of the result records whether it ran, so a harness + /// can compare that too. + pub fn get_val_or_set_mut_with(&mut self, d: V) -> (V, bool) { + match self.val() { + Some(v) => (v.clone(), false), + None => { + self.set_val(d.clone()); + (d, true) + } + } + } + + // -- pruning ------------------------------------------------------------ + + /// `ZipperWriting::prune_path`: delete the dangling chain ending at the + /// focus, stopping at the first location above it that carries a value or + /// branches. The focus does **not** move. + /// + /// Two things about this differ from the doc comment on + /// `ZipperWriting::prune_path`, both verified against `pathmap` 0.3.1: + /// + /// * **It prunes above the zipper's root.** The doc says "This method cannot + /// prune the trie above the zipper's root", but a write zipper rooted at + /// `ab` whose focus is a dangling tip deletes `a` and `ab` too, right up to + /// the nearest branch or value in the *whole map*. The model therefore + /// passes `0`, not `self.root.len()`, as the stop depth. + /// * **The returned count is not well-defined** when the zipper's root is + /// non-empty. `prune_path` returns `max(node_pruned_bytes, + /// trie_pruned_bytes)`, and `node_pruned_bytes` depends on where the + /// internal node holding the focus happens to begin. Empirically, a + /// 40-byte dangling chain under a zipper rooted at depth 5 reports 40 + /// (absolute) while a 100-byte one reports 95 (relative). The *effect* is + /// the same in both cases; only the number differs. The model reports the + /// absolute count, and a harness should compare the count only for zippers + /// rooted at the map root. + pub fn prune_path(&mut self) -> usize { + let f = self.focus(); + self.trie.prune_path(0, &f) + } + + /// `ZipperWriting::prune_ascend`: `prune_path` followed by ascending that far. + pub fn prune_ascend(&mut self) -> usize { + let n = self.prune_path(); + self.ascend(n); + n + } + + /// `ZipperWriting::remove_val`: removes the value, leaving the location as a + /// dangling path unless `prune` reclaims it. + /// + /// Pruning only happens when a value was actually removed: `remove_val` + /// returns early on the `None` branch, so `remove_val(true)` at a location + /// with no value leaves any dangling path in place. + pub fn remove_val(&mut self, prune: bool) -> Option { + let valued = match self.entry() { + // Nothing to remove. Note the `prune` flag does not fire here: + // `remove_val` returns early on this branch, so a dangling path is + // left in place. + Entry::Absent | Entry::Bare => false, + Entry::Valued(_) => true, + }; + if !valued { + return None; + } + let f = self.focus(); + let v = self.trie.remove_val(&f); + if prune { + self.trie.prune_path(0, &f); + } + v + } + + /// `ZipperWriting::create_path`: make the focus exist as a dangling path. + /// Returns whether new bytes were created. + /// + /// The guard is on the *absolute* focus, not on `at_root`: `create_path` + /// bails out when there is no key left to create, which is the map root, not + /// the zipper root. A zipper rooted at `ab` whose root does not yet exist + /// will happily create it. + pub fn create_path(&mut self) -> bool { + let f = self.focus(); + if f.is_empty() { + return false; + } + let create = match self.trie.entry_at(&f) { + Entry::Absent => true, + // Already there, with or without a value: nothing to create, and in + // particular an existing value is never disturbed. + Entry::Bare | Entry::Valued(_) => false, + }; + if create { + self.trie.add_path(&f); + } + create + } + + // -- removing subtries -------------------------------------------------- + + /// `ZipperWriting::remove_branches`: delete everything strictly below the + /// focus. The value at the focus survives. Returns whether anything was + /// removed. + pub fn remove_branches(&mut self, prune: bool) -> bool { + let f = self.focus(); + let removed = !self.trie.below_is_empty(&f); + self.trie.remove_below(&f); + if prune { + self.trie.prune_path(0, &f); + } + removed + } + + /// `ZipperWriting::remove_unmasked_branches`: keep only the child bytes set + /// in `mask`; delete the rest along with their subtries. + pub fn remove_unmasked_branches(&mut self, mask: &ByteMask, prune: bool) { + let f = self.focus(); + let doomed: Vec = self + .child_mask() + .bytes() + .iter() + .copied() + .filter(|&b| !mask.contains(b)) + .collect(); + for b in doomed { + self.trie.remove_at(&path::push(&f, b)); + } + if prune { + self.trie.prune_path(0, &f); + } + } + + // -- grafting ----------------------------------------------------------- + + /// Replace the submap at the focus with `m`, treating `m` as a whole + /// `PathMap`: its root value becomes the focus value (or clears it), and its + /// branches become the focus's branches. This is + /// `ZipperWriting::graft_map`. + pub fn graft_map(&mut self, m: &PathMap) { + let f = self.focus(); + graft_map_at(&mut self.trie, &f, m); + } + + /// `ZipperWriting::graft`: graft the submap at `src`'s focus, root value + /// included. + pub fn graft(&mut self, src: &Zip) { + self.graft_map(&src.make_map()); + } + + /// `ZipperWriting::graft_src_at`: graft the submap `k` bytes below `src`'s + /// focus. + pub fn graft_src_at(&mut self, src: &Zip, k: &[u8]) { + let p = path::cat(&src.focus(), k); + self.graft_map(&src.trie.subtrie(&p)); + } + + /// `ZipperWriting::graft_masked_branches`: graft the source's child branches + /// for each byte set in `mask`. + /// + /// Each set bit is a `graft_src_at` of the source's corresponding child, so + /// the child's *value* travels with it, and a set bit whose branch is absent + /// from the source leaves that branch absent here — grafting nothing removes. + /// With `remove_unset`, branches for clear bits are removed first, so + /// `child_mask` afterwards is a subset of `mask`; without it they are left + /// alone. + /// + /// `WriteZipperCore` overrides the trait's default implementation with a + /// native one, so this op is really comparing two implementations of the same + /// contract. + pub fn graft_masked_branches(&mut self, src: &Zip, mask: &ByteMask, remove_unset: bool) { + if remove_unset { + self.remove_branches(false); + } + let f = self.focus(); + let sf = src.focus(); + for &b in mask.bytes() { + let m = src.trie.subtrie(&path::push(&sf, b)); + graft_map_at(&mut self.trie, &path::push(&f, b), &m); + } + } + + /// `ZipperWriting::graft_child_maps`: as above, but the branches come from an + /// explicit list of maps rather than from a source zipper. + /// + /// Feeding it the source's own child submaps must therefore produce exactly + /// what [`Zip::graft_masked_branches`] produces from that source — a harness + /// should check the two against each other as well as against this + /// definition. + pub fn graft_child_maps(&mut self, maps: &[(ByteMask, PathMap)], remove_unset: bool) { + if remove_unset { + self.remove_branches(false); + } + let f = self.focus(); + for (mask, m) in maps { + if let [b] = mask.bytes() { + graft_map_at(&mut self.trie, &path::push(&f, *b), m); + } + } + } + + /// `ZipperWriting::take_map`: remove the submap at the focus (value included) + /// and return it as a `PathMap`. Returns `None` when there was nothing to + /// take. + pub fn take_map(&mut self, prune: bool) -> Option> { + let f = self.focus(); + let rv = self.trie.remove_val(&f); + if prune { + self.trie.prune_path(0, &f); + } + let below = self.focus_node(); + self.trie.remove_below(&f); + if prune { + self.trie.prune_path(0, &f); + } + let nothing = below.is_empty_map() && rv.is_none(); + let mut taken = below; + if let Some(v) = rv { + taken.set_val(&[], v); + } + if nothing { None } else { Some(taken) } + } + + // -- path surgery ------------------------------------------------------- + + /// `ZipperWriting::insert_prefix`: put `pre` in front of every path below the + /// focus. The focus value is untouched. Returns `false` at a location with + /// no descendants. + /// + /// BUG (`pathmap` 0.3.1): with an **empty** prefix this should be the + /// identity, but `make_parents_in(b"", node)` discards the node — the submap + /// below the focus is destroyed and `true` is still returned. The model + /// specifies the identity; a differential harness should skip + /// `insert_prefix("")` so the known divergence does not mask others. + pub fn insert_prefix(&mut self, pre: &[u8]) -> bool { + if self.focus_node_is_empty() { + return false; + } + let f = self.focus(); + let node = self.focus_node().insert_prefix_below(pre); + self.trie.graft_below(&f, &node); + true + } + + /// `ZipperWriting::remove_prefix`: lift the submap below the focus up by `n` + /// bytes, replacing whatever was below the new (ascended) focus. Returns + /// whether the full `n` bytes could be ascended. + /// + /// Note the value at the old focus is *not* carried up — it belonged to the + /// parent cell, not to the node that gets moved. + pub fn remove_prefix(&mut self, n: usize) -> bool { + let below = self.focus_node(); + // `ascend` reports how far it got, so "were all `n` bytes removed" is a + // comparison rather than the flag it used to return directly. + let ascended = self.ascend(n); + let f = self.focus(); + self.trie.graft_below(&f, &below); + ascended == n + } +} + +/// `graft_map` against a bare map, factored out so the three grafting entry +/// points cannot drift: replace everything below `p` with `m`'s branches, then +/// let `m`'s root value set or clear the value at `p`. +fn graft_map_at(t: &mut PathMap, p: &[u8], m: &PathMap) { + t.graft_below(p, m); + match m.val_at(&[]) { + Some(v) => { + t.set_val(p, v.clone()); + } + None => { + t.remove_val(p); + } + } +} + +// --------------------------------------------------------------------------- +// Write.lean — algebraic operations +// --------------------------------------------------------------------------- +// +// `AlgebraicStatus` is decided structurally: `Identity` exactly when the output +// equals the input (`Identity(SELF_IDENT)`), `None` when the output is empty, and +// `Element` otherwise. `Identity(COUNTER_IDENT)` — the output equals the +// *source* — is reported as `Element` by `pathmap`, and the model agrees because +// the output still differs from `self`. + +impl Zip { + /// The status of replacing `before` with `after`. + fn node_status(ops: &impl ValOps, before: &PathMap, after: &PathMap) -> AlgStatus { + if after.is_empty_map() { + AlgStatus::None + } else if after.beq_t(ops, before) { + AlgStatus::Identity + } else { + AlgStatus::Element + } + } + + /// Write `r` below the focus, pruning afterwards when the node annihilated + /// and the caller asked for it. The shape shared by `meet_into`, + /// `subtract_into` and their kin. + fn write_node(&mut self, st: AlgStatus, r: &PathMap, prune: bool) { + if st == AlgStatus::Identity { + return; + } + let f = self.focus(); + self.trie.graft_below(&f, r); + if st == AlgStatus::None && prune { + self.trie.prune_path(0, &f); + } + } + + /// `ZipperWriting::join_into`: union the source's submap into the focus's. + /// + /// The focus **values are not joined** — only the nodes below the focus are. + /// (The map-consuming variant [`Zip::join_map_into`] *does* join root values.) + pub fn join_into(&mut self, ops: &impl ValOps, src: &Zip) -> AlgStatus { + let self_b = self.focus_node(); + let src_b = src.focus_node(); + if src_b.is_empty_map() { + return if self_b.is_empty_map() { AlgStatus::None } else { AlgStatus::Identity }; + } + let r = PathMap::join(ops, &self_b, &src_b); + if r.beq_t(ops, &self_b) { + AlgStatus::Identity + } else { + let f = self.focus(); + self.trie.graft_below(&f, &r); + AlgStatus::Element + } + } + + /// `ZipperWriting::join_map_into`: union a consumed `PathMap` into the focus. + /// + /// Unlike [`Zip::join_into`] this *does* join the map's root value into the + /// focus value. It also short-circuits: when the map has no root node, the + /// node status is returned directly and the value status computed above is + /// discarded — even though the value has already been written. + pub fn join_map_into(&mut self, ops: &impl ValOps, m: &PathMap) -> AlgStatus { + let (val_status, val_was_none) = match (self.val().cloned(), m.val_at(&[]).cloned()) { + (Some(sv), Some(mv)) => { + let r = ops.pjoin(&sv, &mv); + let st = AlgStatus::of_val_res(&r); + match r.resolve(&sv, &mv) { + Some(v) => { + self.set_val(v); + } + None => { + self.remove_val(false); + } + } + (st, false) + } + (None, Some(mv)) => { + self.set_val(mv); + (AlgStatus::Element, true) + } + (Some(_), None) => (AlgStatus::Identity, false), + (None, None) => (AlgStatus::None, true), + }; + let mut src_b = m.clone(); + src_b.remove_val(&[]); + if src_b.is_empty_map() { + // Short-circuit, and note the asymmetry with `join_into`: this branch + // tests `self.get_focus().is_none()` (does a node exist at all?), not + // `node_is_empty()`. So a *bare* focus reports `Identity` here, where + // `join_into` on the same state reports `None`. + return match self.entry() { + Entry::Bare | Entry::Valued(_) => AlgStatus::Identity, + Entry::Absent => AlgStatus::None, + }; + } + let self_b = self.focus_node(); + let r = PathMap::join(ops, &self_b, &src_b); + let node_status = if r.beq_t(ops, &self_b) { AlgStatus::Identity } else { AlgStatus::Element }; + if node_status != AlgStatus::Identity { + let f = self.focus(); + self.trie.graft_below(&f, &r); + } + AlgStatus::merge(node_status, val_status, true, val_was_none) + } + + /// `ZipperWriting::join_into_take`: like [`Zip::join_into`], but the source + /// submap is removed from the source zipper's map. + pub fn join_into_take( + &mut self, + ops: &impl ValOps, + src: &mut Zip, + prune: bool, + ) -> AlgStatus { + let src_b = src.focus_node(); + let sf = src.focus(); + src.trie.remove_below(&sf); + if prune { + src.trie.prune_path(0, &sf); + } + let self_b = self.focus_node(); + if src_b.is_empty_map() { + return if self_b.is_empty_map() { AlgStatus::None } else { AlgStatus::Identity }; + } + let r = PathMap::join(ops, &self_b, &src_b); + let st = if r.beq_t(ops, &self_b) { AlgStatus::Identity } else { AlgStatus::Element }; + let f = self.focus(); + self.trie.graft_below(&f, &r); + st + } + + /// `ZipperWriting::meet_into`: intersect the focus's submap with the + /// source's. + /// + /// Below the focus the result is [`PathMap::meet`] with `prune = false` -- + /// every location both sides have survives, dangling ones included. With + /// `prune = true` the model gives [`PathMap::meet_pruned`]; `pathmap` may leave + /// dangling paths inside nodes shared with the source, so that case is + /// best-effort and not compared. The focus itself is never removed: pruning + /// stops at it, so a focus left without a value or anything below it is still + /// there. + pub fn meet_into(&mut self, ops: &impl ValOps, src: &Zip, prune: bool) -> AlgStatus { + let (val_status, val_was_none) = match (self.val().cloned(), src.val().cloned()) { + (Some(sv), Some(ov)) => { + let r = ops.pmeet(&sv, &ov); + let st = AlgStatus::of_val_res(&r); + match r.resolve(&sv, &ov) { + Some(v) => { + self.set_val(v); + } + None => { + self.remove_val(false); + } + } + (st, false) + } + (None, Some(_)) => (AlgStatus::None, true), + (Some(_), None) => { + self.remove_val(false); + (AlgStatus::None, false) + } + (None, None) => (AlgStatus::None, true), + }; + let self_b = self.focus_node(); + let src_b = src.focus_node(); + if self_b.is_empty_map() { + return AlgStatus::merge(AlgStatus::None, val_status, true, val_was_none); + } + let f = self.focus(); + if src_b.is_empty_map() { + self.trie.remove_below(&f); + return AlgStatus::merge(AlgStatus::None, val_status, false, val_was_none); + } + let r = if prune { PathMap::meet_pruned(ops, &self_b, &src_b) } else { PathMap::meet(ops, &self_b, &src_b) }; + let st = Self::node_status(ops, &self_b, &r); + if st != AlgStatus::Identity { + self.trie.graft_below(&f, &r); + } + AlgStatus::merge(st, val_status, false, val_was_none) + } + + /// `ZipperWriting::subtract_into`: remove the source's submap from the + /// focus's. + /// + /// Where the source has no node at all, `self`'s subtree survives untouched — + /// dangling paths included. Where it does, only locations leading to a + /// surviving value are kept. + pub fn subtract_into(&mut self, ops: &impl ValOps, src: &Zip, prune: bool) -> AlgStatus { + let (val_status, val_was_none) = match (self.val().cloned(), src.val().cloned()) { + (Some(sv), Some(ov)) => { + let r = ops.psub(&sv, &ov); + let st = AlgStatus::of_val_res(&r); + match r.resolve(&sv, &ov) { + Some(v) => { + self.set_val(v); + } + None => { + self.remove_val(prune); + } + } + (st, false) + } + (None, Some(_)) => (AlgStatus::None, true), + (Some(_), None) => (AlgStatus::Identity, false), + (None, None) => (AlgStatus::None, true), + }; + let self_b = self.focus_node(); + let src_b = src.focus_node(); + if src_b.is_empty_map() { + let node = if self_b.is_empty_map() { AlgStatus::None } else { AlgStatus::Identity }; + return AlgStatus::merge(node, val_status, self_b.is_empty_map(), val_was_none); + } + if self_b.is_empty_map() { + return AlgStatus::merge(AlgStatus::None, val_status, true, val_was_none); + } + let r = PathMap::sub(ops, &self_b, &src_b); + let st = Self::node_status(ops, &self_b, &r); + self.write_node(st, &r, prune); + AlgStatus::merge(st, val_status, false, val_was_none) + } + + /// `ZipperWriting::meet_2`: meet two *source* submaps and write the result at + /// the focus. There is no `prune` argument, so this is [`PathMap::meet`]: + /// every location both sources have survives, dangling ones included. + /// + /// Two things separate this from [`Zip::meet_into`]. It does not consult + /// what is already at the focus, so — as the implementation notes — it never + /// reports `Identity`, only `Element` or `None`. And it works on nodes, so + /// neither source's focus value is consulted and the focus value here is left + /// untouched. + pub fn meet_2(&mut self, ops: &impl ValOps, a: &Zip, b: &Zip) -> AlgStatus { + let an = a.focus_node(); + let bn = b.focus_node(); + let f = self.focus(); + if an.is_empty_map() || bn.is_empty_map() { + self.trie.remove_below(&f); + return AlgStatus::None; + } + let r = PathMap::meet(ops, &an, &bn); + if r.is_empty_map() { + self.trie.remove_below(&f); + AlgStatus::None + } else { + self.trie.graft_below(&f, &r); + AlgStatus::Element + } + } + + /// `ZipperWriting::restrict`: keep only the paths below the focus that are + /// prefixed by a path to a value in the source's submap. + /// + /// The empty prefix does **not** validate here: the source's *focus value* is + /// invisible to a node-level `prestrict`. [`map::restrict`] does consult the + /// root value, so the two disagree exactly when the source has a value at its + /// focus. The focus value of `self` is never touched. + pub fn restrict(&mut self, ops: &impl ValOps, src: &Zip) -> AlgStatus { + let src_b = src.focus_node(); + let self_b = self.focus_node(); + if src_b.is_empty_map() { + let f = self.focus(); + self.trie.remove_below(&f); + return AlgStatus::None; + } + if self_b.is_empty_map() { + return AlgStatus::None; + } + let r = PathMap::restrict_below_root(&self_b, &src_b); + let st = Self::node_status(ops, &self_b, &r); + if st == AlgStatus::Identity { + return AlgStatus::Identity; + } + let f = self.focus(); + self.trie.graft_below(&f, &r); + st + } + + /// `ZipperWriting::restricting`: the mirror image — fill in `self`'s "stem" + /// paths with the source's submaps. `self`'s submap is replaced by the + /// source's, restricted by the paths to values in `self`. + /// + /// Returns `false`, leaving `self` untouched, when either side has nothing + /// below its focus. Note this is decided by `get_focus().is_none()`, which + /// is true when there is no node below the focus but *false* when an empty + /// node happens to have been materialised there by `create_path` or + /// `remove_val` — see FINDINGS.md #8. The model specifies the common case. + pub fn restricting(&mut self, src: &Zip) -> bool { + if src.focus_node_is_empty() || self.focus_node_is_empty() { + return false; + } + let r = PathMap::restrict_below_root(&src.focus_node(), &self.focus_node()); + let f = self.focus(); + self.trie.graft_below(&f, &r); + true + } + + // -- collapsing path segments ------------------------------------------- + + /// `ZipperWriting::join_k_path_into` (a.k.a. `drop_head`): strip the leading + /// `k` bytes from every path below the focus and join the results. + /// + /// Values sitting at depth exactly `k` are **lost**: the joined node has no + /// root value slot. Returns whether anything survives below the focus. + /// + /// BUG (`pathmap` 0.3.1): `k = 0` should be the identity — dropping no bytes + /// — but `drop_head_dyn(0)` collapses the submap instead. On + /// `{[] ↦ 0, [0] ↦ 0, [0,0] ↦ 0, [1,0] ↦ 0}` it leaves `{[] ↦ 0, [0] ↦ 0}`. + /// The model specifies the identity; a harness should skip `k = 0`. + pub fn join_k_path_into(&mut self, ops: &impl ValOps, k: usize, prune: bool) -> bool { + let below = self.focus_node(); + let res = if below.is_empty_map() { + false + } else { + let r = below.drop_head(ops, k); + let survives = !r.is_empty_map(); + let f = self.focus(); + self.trie.graft_below(&f, &r); + survives + }; + if prune && !res { + self.prune_path(); + } + res + } + + /// `meet_k_path_into` is **not implementable** for these arguments: its + /// provisional implementation drives `descend_first_k_path` through the + /// `ZipperIteration` *default* loop, which spins forever when the focus has + /// no children, and which escapes the focus's subtree entirely when `k = 0`. + /// Verified against `pathmap` 0.3.1: `meet_k_path_into(1, false)` on a leaf + /// hangs. + pub fn meet_k_path_unspecified(&self, k: usize) -> bool { + k == 0 || self.child_count() == 0 + } + + /// `ZipperWriting::meet_k_path_into`: strip the leading `k` bytes from every + /// path below the focus and meet the results. + /// + /// Unlike [`Zip::join_k_path_into`], this routes through + /// `take_map`/`graft_map`, so values at depth exactly `k` *are* carried — they + /// become the focus value. Only meaningful when + /// [`Zip::meet_k_path_unspecified`] is `false`. + pub fn meet_k_path_into(&mut self, ops: &impl ValOps, k: usize, prune: bool) -> bool { + let f = self.focus(); + let kps = self.trie.subtrie(&f).k_paths(k); + let mut result: Option> = None; + for q in kps { + let m = self.trie.subtrie(&path::cat(&f, &q)); + result = Some(match result { + None => m, + Some(a) => PathMap::meet(ops, &a, &m), + }); + } + // `prune` drops every dangling path from the meet, including one that a + // single k-path's subtrie brings along unmet. The focus itself is never + // removed. + match result.map(|m| if prune { m.drop_dangling() } else { m }) { + Some(m) if !m.is_empty_map() => { + self.graft_map(&m); + true + } + _ => { + self.remove_branches(false); + false + } + } + } +} + diff --git a/differential/src/reference/zipper.rs b/differential/src/reference/zipper.rs new file mode 100644 index 00000000..152c66df --- /dev/null +++ b/differential/src/reference/zipper.rs @@ -0,0 +1,578 @@ +//! `Zipper.lean` — the zipper, and its read API. + +use crate::reference::basic::{ByteMask, path}; +use crate::reference::pathmap::{Entry, PathMap}; + +// =========================================================================== +// Zipper.lean — the read API +// =========================================================================== + +/// A cursor into a map. Three things determine everything it can observe or do: +/// +/// * the **map** it is looking at, +/// * its **root** — the absolute path at which it was created +/// (`root_prefix_path`); the zipper can never ascend above it, and can never +/// see anything outside the submap hanging below it, and +/// * its **path** — the relative path from the root to the current **focus** +/// (`path()`); `origin_path() = root ++ path`. +/// +/// Read zippers and write zippers share this state and differ only in which +/// operations are offered, so `Zip` models both. For a read zipper `trie` is a +/// snapshot taken when the zipper was created (`fork_read_zipper` etc.); for a +/// write zipper it is the live map, so mutations write back through +/// `root ++ path`. +/// +/// # The focus may not exist +/// +/// `descend_to` moves the focus anywhere, including off the map. `path_exists` +/// then reports `false` while `path()` still reports the full path and `ascend` +/// still walks back up. `path` is therefore an unconstrained byte string. +/// +/// # The blind-zipper contract +/// +/// `ZipperMoving` no longer provides `path()`: a zipper that does not track its +/// own path is a *blind* zipper, and `path()` / `move_to_path()` live in the +/// separate `ZipperPath: ZipperMoving` trait. The model keeps `path` as a field +/// because it has to represent the location somehow, but it mirrors the split in +/// what each operation is allowed to *observe*: [`Zip::focus_byte`] is the only +/// positional information a blind zipper can read, and its value at the root is +/// deliberately unspecified. +/// +/// The migration also changed what the movement operations report: `ascend`, +/// `ascend_until` and `ascend_until_branch` return the **number of bytes +/// ascended**, and `descend_indexed_byte`, `descend_first_byte`, +/// `descend_last_byte`, `to_next_sibling_byte` and `to_prev_sibling_byte` return +/// `Option` — the byte moved to — rather than a `bool`. +/// +/// # Depth-first order is lexicographic order +/// +/// Every iteration primitive is specified as "the least existing location +/// strictly after the focus, subject to ...", ordered by `Ord for [u8]`. That +/// is equivalent to the implementation's node-by-node walk, and far easier to +/// state and to check. +#[derive(Clone, Debug)] +pub struct Zip { + /// The map: a snapshot, for a read zipper; the live map, for a write zipper. + pub trie: PathMap, + /// `root_prefix_path()`: where the zipper was created. + pub root: Vec, + /// `path()`: the relative path from the root to the focus. + pub path: Vec, +} + +impl Zip { + /// A zipper at the root of `trie`. + pub fn new(trie: PathMap) -> Self { + Zip { trie, root: Vec::new(), path: Vec::new() } + } + + /// A zipper rooted at `root`, focused at its root. + pub fn at(trie: PathMap, root: &[u8]) -> Self { + Zip { trie, root: root.to_vec(), path: Vec::new() } + } + + /// A zipper rooted at `root` with its focus already at `path`. + pub fn at_path(trie: PathMap, root: &[u8], path: &[u8]) -> Self { + Zip { trie, root: root.to_vec(), path: path.to_vec() } + } + + /// `ZipperAbsolutePath::origin_path`: the absolute path of the focus. + pub fn focus(&self) -> Vec { + path::cat(&self.root, &self.path) + } + + /// `ZipperAbsolutePath::root_prefix_path`. + pub fn root_prefix_path(&self) -> &[u8] { + &self.root + } + + // -- trait Zipper ------------------------------------------------------- + + /// What the map holds at the focus: the whole answer to `path_exists` and + /// `val` at once. Definitions that must handle a location existing *without* + /// a value are written against this, so the `match` will not compile until + /// they say what happens to it. + pub fn entry(&self) -> Entry<&V> { + self.trie.entry_at(&self.focus()) + } + + /// `Zipper::path_exists`. + pub fn path_exists(&self) -> bool { + self.trie.path_exists(&self.focus()) + } + + /// `Zipper::is_val`. + pub fn is_val(&self) -> bool { + self.trie.val_at(&self.focus()).is_some() + } + + /// `Zipper::child_mask`. Empty on a leaf or a non-existent path. + pub fn child_mask(&self) -> ByteMask { + self.trie.child_mask(&self.focus()) + } + + /// `Zipper::child_count`. + pub fn child_count(&self) -> usize { + self.child_mask().count_bits() + } + + /// `ZipperMoving::focus_byte`: the byte last descended to reach the focus. + /// + /// **Unspecified at the root.** A zipper that retains knowledge of the map + /// above its root may return the byte leading to that root; one that does + /// not, or one rooted at the map root, returns `None`. So a `Some` here does + /// not mean the zipper has descended, and callers needing that distinction + /// must ask [`Zip::at_root`]. The model returns the last byte of the + /// relative path; a harness should mask the value at the root rather than + /// comparing it. + pub fn focus_byte(&self) -> Option { + self.path.last().copied() + } + + // -- trait ZipperValues / ZipperReadOnlyValues -------------------------- + + /// `ZipperValues::val` (and `ZipperReadOnlyValues::get_val`, which differs + /// only in the lifetime of the returned reference). + pub fn val(&self) -> Option<&V> { + self.trie.val_at(&self.focus()) + } + + /// `ZipperValues::val_at`: the value at `k`, relative to the focus. + pub fn val_at(&self, k: &[u8]) -> Option<&V> { + self.trie.val_at(&path::cat(&self.focus(), k)) + } + + // -- trait ZipperSubtries / ZipperInfallibleSubtries -------------------- + + /// `ZipperInfallibleSubtries::make_map`. Under the default `graft_root_vals` + /// feature the value at the focus becomes the new map's root value. + pub fn make_map(&self) -> PathMap { + self.trie.subtrie(&self.focus()) + } + + /// The node below the focus — what `get_focus` returns, i.e. `make_map` with + /// the focus value stripped. This, not `make_map`, is what the algebraic + /// operations and `graft_internal` consume. + pub fn focus_node(&self) -> PathMap { + let mut m = self.make_map(); + m.remove_val(&[]); + m + } + + /// `get_focus().is_none()`: the focus has no descendants. + pub fn focus_node_is_empty(&self) -> bool { + self.trie.below_is_empty(&self.focus()) + } + + // -- trait ZipperMoving — position -------------------------------------- + + /// `ZipperMoving::at_root`. + pub fn at_root(&self) -> bool { + self.path.is_empty() + } + + /// `ZipperMoving::reset`. + pub fn reset(&mut self) { + self.path.clear(); + } + + /// `ZipperMoving::val_count`: values at and below the focus. + pub fn val_count(&self) -> usize { + self.trie.val_count(&self.focus()) + } + + /// The absolute path of the relative location `q` — the model's `atPath`, + /// which lets an ancestor or descendant be *named* and then asked about, so + /// the specifications can say "the deepest ancestor such that ..." instead of + /// walking there step by step. + fn abs(&self, q: &[u8]) -> Vec { + path::cat(&self.root, q) + } + + // -- trait ZipperMoving — descent --------------------------------------- + + /// `ZipperMoving::descend_to`. Never fails; the focus may end up off-map. + pub fn descend_to(&mut self, k: &[u8]) { + self.path.extend_from_slice(k); + } + + /// `ZipperMoving::descend_to_byte`. + pub fn descend_to_byte(&mut self, b: u8) { + self.path.push(b); + } + + /// `ZipperMoving::descend_to_check`: descend, then report existence. + pub fn descend_to_check(&mut self, k: &[u8]) -> bool { + self.descend_to(k); + self.path_exists() + } + + /// How far along `k` the path still exists, starting from the focus. + /// + /// Existence is prefix-closed, so the prefixes of `k` that still exist form + /// an initial segment: the answer is the longest prefix of `k` that exists. + fn reach(&self, k: &[u8]) -> usize { + let f = self.focus(); + let mut probe = f.clone(); + let mut n = 0; + if !self.trie.path_exists(&probe) { + return 0; + } + for &b in k { + probe.push(b); + if !self.trie.path_exists(&probe) { + break; + } + n += 1; + } + n + } + + /// `ZipperMoving::descend_to_existing`: descend byte by byte, stopping where + /// the path stops existing. Returns the number of bytes actually descended. + pub fn descend_to_existing(&mut self, k: &[u8]) -> usize { + let n = self.reach(k); + self.descend_to(&k[..n]); + n + } + + /// `ZipperMoving::descend_to_val`: descend byte by byte, stopping at the + /// first value encountered *below* the starting focus, or where the path + /// stops existing. + pub fn descend_to_val(&mut self, k: &[u8]) -> usize { + let reach = self.reach(k); + let f = self.focus(); + // A value already at the focus does not stop it, so the scan starts at 1. + let stop = (1..=reach) + .find(|&j| self.trie.val_at(&path::cat(&f, &k[..j])).is_some()) + .unwrap_or(reach); + self.descend_to(&k[..stop]); + stop + } + + /// `ZipperMoving::descend_to_existing_byte`. + pub fn descend_to_existing_byte(&mut self, b: u8) -> bool { + self.path.push(b); + if self.path_exists() { + true + } else { + self.path.pop(); + false + } + } + + /// `ZipperMoving::descend_indexed_byte`: descend into the `idx`-th child in + /// ascending byte order, returning the byte moved to. Out-of-range indices + /// do nothing and return `None`. + pub fn descend_indexed_byte(&mut self, idx: usize) -> Option { + let b = self.child_mask().indexed_bit(idx)?; + self.path.push(b); + Some(b) + } + + /// `ZipperMoving::descend_first_byte`. + pub fn descend_first_byte(&mut self) -> Option { + self.descend_indexed_byte(0) + } + + /// `ZipperMoving::descend_last_byte`. + pub fn descend_last_byte(&mut self) -> Option { + let c = self.child_count(); + if c == 0 { None } else { self.descend_indexed_byte(c - 1) } + } + + /// `ZipperMoving::descend_until`: descend while there is exactly one child, + /// stopping on a value. A no-op on a branch, a leaf, or a non-existent path. + /// + /// Nothing happens unless the focus has exactly one child. When it does, the + /// locations below it form a chain until the first one that branches, ends, + /// or carries a value — so the destination is simply the *nearest* descendant + /// that is a value or is not single-childed. Depth-first order along a chain + /// is order of increasing depth, so the first hit is the nearest one. + pub fn descend_until(&mut self) -> bool { + if self.child_count() != 1 { + return false; + } + let f = self.focus(); + let hit = self + .trie + .strictly_below(&f) + .find(|(q, v)| v.is_some() || self.trie.child_count(q) != 1) + .map(|(q, _)| q.clone()); + match hit { + Some(q) => { + self.path = q[self.root.len()..].to_vec(); + true + } + None => false, + } + } + + /// `ZipperMoving::descend_until_observed`: `descend_until`, reporting each + /// byte it descends to a `PathObserver`. + /// + /// For the `Vec` observer — the one the harness uses — the reported + /// sequence is exactly the path delta, which is the only way a blind zipper + /// can learn where it ended up. That equality is the property worth checking. + pub fn descend_until_observed(&mut self) -> (bool, Vec) { + let before = self.path.len(); + let moved = self.descend_until(); + (moved, self.path[before..].to_vec()) + } + + /// `ZipperMoving::descend_until_max_bytes`: `descend_until`, then ascend back + /// to at most `max_bytes` below the starting depth. + pub fn descend_until_max_bytes(&mut self, max_bytes: usize) -> bool { + if max_bytes == 0 { + return false; + } + let target = self.path.len() + max_bytes; + let moved = self.descend_until(); + if self.path.len() > target { + self.path.truncate(target); + } + moved + } + + // -- trait ZipperMoving — ascent ---------------------------------------- + + /// `ZipperMoving::ascend`: ascend `steps` bytes, clamping at the zipper root. + /// Returns the **number of bytes actually ascended**, which is smaller than + /// `steps` when the root was closer than that. + pub fn ascend(&mut self, steps: usize) -> usize { + let n = steps.min(self.path.len()); + self.path.truncate(self.path.len() - n); + n + } + + /// `ZipperMoving::ascend_byte`: still a `bool`, defined as `ascend(1) == 1`. + pub fn ascend_byte(&mut self) -> bool { + self.ascend(1) == 1 + } + + /// `ZipperMoving::ascend_until`: ascend to the nearest strict ancestor that + /// carries a value or branches, or to the root. Returns the number of bytes + /// ascended; `0` means the zipper was already at its root. + pub fn ascend_until(&mut self) -> usize { + self.ascend_until_with(|z, a| { + z.trie.val_at(a).is_some() || z.trie.child_count(a) > 1 + }) + } + + /// `ZipperMoving::ascend_until_branch`: like `ascend_until`, but values do + /// not stop the ascent. Returns the number of bytes ascended. + pub fn ascend_until_branch(&mut self) -> usize { + self.ascend_until_with(|z, a| z.trie.child_count(a) > 1) + } + + /// The shared core: the destination is the deepest strict ancestor + /// satisfying `stop`, or the zipper root, which always qualifies — so there + /// is always an answer. + fn ascend_until_with(&mut self, stop: impl Fn(&Self, &[u8]) -> bool) -> usize { + if self.at_root() { + return 0; + } + let mut dest = 0; + for n in (1..self.path.len()).rev() { + let anc = self.abs(&self.path[..n]); + if stop(self, &anc) { + dest = n; + break; + } + } + let moved = self.path.len() - dest; + self.path.truncate(dest); + moved + } + + // -- trait ZipperMoving — lateral movement ------------------------------ + + /// `ZipperMoving::to_next_sibling_byte`. + /// + /// At the zipper root there is no last byte, so the documented answer — and + /// the `ZipperMoving` default implementation's answer — is "did not move". + /// + /// BUG (`pathmap` 0.3.1): the native `ReadZipper` implementation instead + /// consults the last byte of the *absolute* origin path, so a read zipper + /// whose root does not exist but has a sibling **leaves its own root**. With + /// map `{[0,0,3] ↦ v}` and a zipper rooted at `[0,0,1]`, + /// `to_next_sibling_byte()` returns `true`, `path()` still reports `[]` and + /// `at_root()` still reports `true`, but `origin_path()` is now `[0,0,3]` and + /// the zipper reads `v`. That breaks the containment a `ZipperHead` relies + /// on to hand out non-overlapping zippers. The model specifies the + /// documented behaviour; the differential harness skips the operation at the + /// root so the known bug does not mask others. + /// + /// Transcribed from the *current* `Zipper.lean`. The archive this file came + /// from described the bug as fixed and ran the op at the root; `Fuzz.lean` + /// and `harness.rs` both skip it today, so the model follows them. See the + /// report accompanying this port for whether the skip is still earned. + pub fn to_next_sibling_byte(&mut self) -> Option { + self.to_sibling_byte(true) + } + + /// `ZipperMoving::to_prev_sibling_byte`. + pub fn to_prev_sibling_byte(&mut self) -> Option { + self.to_sibling_byte(false) + } + + /// Both sibling moves are keyed on `focus_byte`, whose value at the root is + /// unspecified, so the `at_root` guard is what keeps the zipper inside its + /// own subtree. + // `to_*` here is `pathmap`'s "move the cursor to", not a conversion, so the + // `&mut self` receiver is right and clippy's naming convention does not apply. + #[allow(clippy::wrong_self_convention)] + fn to_sibling_byte(&mut self, forward: bool) -> Option { + let cur = self.focus_byte()?; + if self.at_root() { + return None; + } + self.path.pop(); + let mask = self.child_mask(); + let hit = if forward { mask.next_bit(cur) } else { mask.prev_bit(cur) }; + match hit { + Some(b) => { + self.path.push(b); + Some(b) + } + None => { + self.path.push(cur); + None + } + } + } + + /// `ZipperMoving::move_to_path`: jump to `p` relative to the zipper root. + /// Returns the number of bytes shared between the old and the new location. + pub fn move_to_path(&mut self, p: &[u8]) -> usize { + let overlap = p + .iter() + .zip(self.path.iter()) + .take_while(|(a, b)| a == b) + .count(); + self.path = p.to_vec(); + overlap + } + + // -- trait ZipperMoving — depth-first stepping -------------------------- + + /// `ZipperMoving::to_next_step`: the next existing location in depth-first + /// order. On exhaustion the focus returns to the root and the result is + /// `false`. + pub fn to_next_step(&mut self) -> bool { + self.step_to(|_, _| true) + } + + // -- trait ZipperIteration ---------------------------------------------- + + /// `ZipperIteration::to_next_val`: the next existing location carrying a + /// value, in depth-first order. Never reports the value at the starting + /// focus. On exhaustion the focus returns to the root and the result is + /// `false`. + pub fn to_next_val(&mut self) -> bool { + self.step_to(|_, v| v.is_some()) + } + + /// The shared core of the two: the least existing location strictly after the + /// focus and within the zipper's own subtree that satisfies `pred`. + fn step_to(&mut self, pred: impl Fn(&[u8], &Option) -> bool) -> bool { + let f = self.focus(); + let root = self.root.clone(); + let hit = self + .trie + .after_within(&f, &root) + .find(|(q, v)| pred(q, v)) + .map(|(q, _)| q.clone()); + match hit { + Some(q) => { + self.path = q[self.root.len()..].to_vec(); + true + } + None => { + self.reset(); + false + } + } + } + + /// `ZipperReadOnlyIteration::to_next_get_val`. + pub fn to_next_get_val(&mut self) -> Option<&V> { + if self.to_next_val() { self.val() } else { None } + } + + /// `ZipperIteration::descend_last_path`: follow the last child to the end of + /// the depth-first-greatest path below the focus. + pub fn descend_last_path(&mut self) -> bool { + let f = self.focus(); + match self.trie.at_or_below(&f).last().map(|(q, _)| q.clone()) { + Some(q) if q.len() > f.len() => { + self.path = q[self.root.len()..].to_vec(); + true + } + _ => false, + } + } + + /// The shared core of `descend_first_k_path` and `to_next_k_path` + /// (`k_path_internal`): the depth-first-least existing location that is + /// exactly `k` bytes below the common ancestor at depth `base`, and that + /// comes strictly after the current focus. On failure the focus moves to + /// that ancestor. + fn k_path_from(&mut self, base: usize, k: usize) -> bool { + let anc_rel = self.path[..base].to_vec(); + let anc = self.abs(&anc_rel); + let f = self.focus(); + let want = anc.len() + k; + let hit = self + .trie + .after_within(&f, &anc) + .find(|(q, _)| q.len() == want) + .map(|(q, _)| q.clone()); + match hit { + Some(q) => { + self.path = q[self.root.len()..].to_vec(); + true + } + None => { + self.path = anc_rel; + false + } + } + } + + /// `ZipperIteration::descend_first_k_path`: descend to the depth-first-first + /// existing location exactly `k` bytes below the focus. Leaves the focus + /// untouched and returns `false` when there is none. + pub fn descend_first_k_path(&mut self, k: usize) -> bool { + let base = self.path.len(); + self.k_path_from(base, k) + } + + /// `ZipperIteration::to_next_k_path`: the next existing location at the same + /// depth, under the common ancestor `k` bytes above the focus. On exhaustion + /// the focus moves to that ancestor and the result is `false`. + /// + /// NOTE: when the focus is shallower than `k`, the *native* `ReadZipper` falls + /// back to the **zipper root** as the common ancestor — so the call behaves + /// like `descend_first_k_path(k)` from the root and can succeed. The + /// `ZipperIteration` default implementation instead returns `false` without + /// moving. The model follows the native `ReadZipper`, which is what the + /// public API reaches. + pub fn to_next_k_path(&mut self, k: usize) -> bool { + if k <= self.path.len() { + let base = self.path.len() - k; + self.k_path_from(base, k) + } else { + self.k_path_from(0, k) + } + } + + // -- trait ZipperForking ------------------------------------------------ + + /// `ZipperForking::fork_read_zipper`: a new zipper rooted at the current + /// focus, over a snapshot of the same map. + pub fn fork_read_zipper(&self) -> Zip { + Zip { trie: self.trie.clone(), root: self.focus(), path: Vec::new() } + } +} + diff --git a/differential/src/repro.rs b/differential/src/repro.rs index 6d2c06f4..6c1373ec 100644 --- a/differential/src/repro.rs +++ b/differential/src/repro.rs @@ -172,8 +172,8 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 27 => { let v = g!(d.u8()) as u64; format!("wz.set_val({v});") } 28 => { let _pr = g!(d.boolean()); "wz.remove_val(false);".to_string() } 29 => "wz.create_path();".to_string(), - 30 => "wz.prune_path();".to_string(), - 31 => "wz.prune_ascend();".to_string(), + 30 => if r0.is_empty() { "wz.prune_path();".to_string() } else { "// prune_path off the map root: skipped by the harness".to_string() }, + 31 => if r0.is_empty() { "wz.prune_ascend();".to_string() } else { "// prune_ascend off the map root: skipped by the harness".to_string() }, 32 => { let _pr = g!(d.boolean()); "wz.remove_branches(false);".to_string() } 33 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let _pr = g!(d.boolean()); format!("wz.remove_unmasked_branches(ByteMask::from_iter({}.iter().copied()), false);", rs_mask(&m)) } @@ -185,12 +185,13 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 38 => { let _pr = g!(d.boolean()); "wz.meet_into(&rz, false);".to_string() } 39 => { let _pr = g!(d.boolean()); "wz.subtract_into(&rz, false);".to_string() } 40 => "wz.restrict(&rz);".to_string(), - 41 => "wz.restricting(&rz);".to_string(), + 41 => "if wz.child_count() != 0 && rz.child_count() != 0 { wz.restricting(&rz); }".to_string(), 42 => { let k = g!(d.modn(4)); let _pr = g!(d.boolean()); if k == 0 { "// join_k_path_into(0): skipped by the harness".to_string() } else { format!("wz.join_k_path_into({k}, false);") } } 43 => { let p = g!(d.path(6)); - if p.is_empty() { "// insert_prefix(\"\"): skipped by the harness".to_string() } + // `insert_prefix` is generic over the prefix, so a bare `&[]` has no element type. + if p.is_empty() { "wz.insert_prefix(&[0u8; 0]);".to_string() } else { format!("wz.insert_prefix({});", rs_bytes(&p)) } } 44 => { let n = g!(d.modn(6)); format!("wz.remove_prefix({n});") } 45 => { let _pr = g!(d.boolean()); @@ -205,14 +206,16 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 50 => { let v = g!(d.u8()) as u64; format!("wz.get_val_or_set_mut_with(|| {v});") } 51 => { let t = g!(d.modn(2)); let p = g!(d.path(6)); format!("{{ {z}.val(); {z}.val_at({p}); }}", z = z!(t), p = rs_bytes(&p)) } - 52 => "rz.to_next_val(); // to_next_get_val".to_string(), + 52 => "rz.to_next_get_val();".to_string(), 53 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let ru = g!(d.boolean()); format!("wz.graft_masked_branches(&rz, ByteMask::from_iter({}.iter().copied()), {ru});", rs_mask(&m)) } - 54 => { let n = g!(d.modn(4)); let _m = g!(d.path_n(n)); let _ru = g!(d.boolean()); - "// graft_child_maps: quarantined by the harness (FINDINGS #15)".to_string() } + // As `do_graft_child_maps`: fed the source's own child subtries under the mask. + 54 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let ru = g!(d.boolean()); + format!("{{ let m = ByteMask::from_iter({}.iter().copied()); \ + let maps: Vec> = m.iter().map(|b| {{ let mut c = rz.clone(); c.descend_to_byte(b); c.make_map() }}).collect(); \ + wz.graft_child_maps(m, maps, {ru}); }}", rs_mask(&m)) } 55 => { let p = g!(d.path(6)); - format!("{{ let mut b = map1.read_zipper_at_path({}); b.descend_to({}); wz.meet_2(&rz, &b); }}", - rs_bytes(&r1), rs_bytes(&p)) } + format!("{{ let mut b = rz.clone(); b.descend_to({}); wz.meet_2(&rz, &b); }}", rs_bytes(&p)) } _ => "// nop".to_string(), }; o.push_str(&format!(" /* {step:3} */ {line}\n")); diff --git a/differential/src/source.rs b/differential/src/source.rs new file mode 100644 index 00000000..ab0a16ab --- /dev/null +++ b/differential/src/source.rs @@ -0,0 +1,56 @@ +//! Where fuzzer inputs come from, shared by `in_process` and `crash_fuzz`. + +/// Where inputs come from. +/// +/// `get` is deterministic in `idx` and holds no state between calls, so which +/// thread runs an input cannot change it and a failing input can be re-derived +/// from its index alone. Mirrors `InputSource` in `lean/differential.py`; a +/// queue-backed source drops in here the same way. +pub enum Source { + Random { seed: u64, count: usize, maxlen: usize }, + Files(Vec), +} + +impl Source { + pub fn len(&self) -> usize { + match self { + Source::Random { count, .. } => *count, + Source::Files(v) => v.len(), + } + } + + pub fn name(&self, idx: usize) -> String { + match self { + Source::Random { .. } => format!("random#{idx:06}"), + Source::Files(v) => v[idx].clone(), + } + } + + pub fn get(&self, idx: usize) -> Vec { + match *self { + Source::Random { seed, maxlen, .. } => { + // splitmix64, seeded per index so generation parallelises without + // changing what gets tested. + let mut s = seed.wrapping_mul(0x9E3779B97F4A7C15) + ^ (idx as u64).wrapping_mul(0xBF58476D1CE4E5B9); + let mut next = || { + s = s.wrapping_add(0x9E3779B97F4A7C15); + let mut z = s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + }; + // The same *distribution* as `RandomInputs.get` in + // `lean/differential.py` -- `randrange(8, maxlen)`, i.e. uniform + // on `[8, maxlen)` -- so a divergence rate measured here is + // directly comparable to one measured there. The bit stream + // differs (splitmix64 against Python's Mersenne Twister) and is + // meant to: two independent samples of the same population. + let span = maxlen.saturating_sub(8).max(1); + let n = 8 + (next() as usize) % span; + (0..n).map(|_| next() as u8).collect() + } + Source::Files(ref v) => std::fs::read(&v[idx]).expect("cannot read input"), + } + } +} diff --git a/lean/FINDINGS.md b/lean/FINDINGS.md index 4d98c548..ea43df57 100644 --- a/lean/FINDINGS.md +++ b/lean/FINDINGS.md @@ -42,6 +42,30 @@ same `join_into` into a *non-empty* destination produces the right answer. Only the empty-destination case loses the data, and it reports `Identity` — "self was not modified" — rather than failing. +**Addendum (2026-09-11).** The empty-destination case was fixed in PR #70, but the +same key kept firing (99 of 100k seed-11 inputs). Two causes, both in the node +join, neither specific to an empty destination: + +* A source focus partway into a line node is a `TinyRefNode`, and the + `TINY_REF_NODE_TAG` arm of `pjoin_dyn` (dense and list nodes) evaluated the join + with the operands swapped and returned the identity mask un-inverted. A + destination that already contained the source reported `COUNTER_IDENT`, which + `join_into` takes as "the result is the source": a *dense* destination was + replaced by the source (data loss), a list destination merely reported + `Element`. The swapped operands also made the left-biased value join take the + source's values. The arm now expands the tiny node and keeps `self` on the + left. +* `merge_guts`, pairing a slot that holds an onward child with a slot whose key + is longer, always reported the pair as `Element`, even when the child already + held the other payload. It now reports the child slot as the identity. +* The integer `pjoin` placeholders returned `SELF_IDENT` alone even for equal + values; with equal values the result is also `other`, and without that bit a + join seen from the other side can never be a self-identity. (This also + removed two thirds of the finding-8 `join_map_into`/`restrict` hits.) + +Test: `write_zipper_join_into_mid_key_source_keeps_destination`, +`write_zipper_join_into_contained_under_child_is_identity`. + ## 2. A token-maintaining move leaves the zipper navigating wrongly `case: to_next_val_after_step` -- **silent wrong answer** @@ -469,6 +493,14 @@ it. `case: graft_child_maps_dense` -- **abort, or a silently created path** +*Fixed* (`bugfix/graft-child-maps`): the root-focus lookup no longer hands an +empty key to the dense node, and grafting nothing onto a branch now follows +`graft`: an existing branch is emptied but survives as a dangling path, an +absent one is not created. A third symptom found while fixing -- an empty map, +or one without a root value, left the destination's old branch or old value in +place -- is fixed the same way. Op 54 is no longer skipped by the harness, and +the regression test is `write_zipper::tests::graft_child_maps_dense`. + Two symptoms, one method. **On any dense destination it aborts.** diff --git a/lean/PathMapModel/Basic.lean b/lean/PathMapModel/Basic.lean index ff22667f..d3264a83 100644 --- a/lean/PathMapModel/Basic.lean +++ b/lean/PathMapModel/Basic.lean @@ -40,8 +40,8 @@ inductive ValRes (V : Type) where These return `ValRes` rather than plain values because `pathmap` reports `AlgebraicStatus` to the caller, and the status depends on *which constructor* the value operation returned, not on whether the value changed. `u64`'s -`psubtract`, for instance, returns `Element(*self)` — an `Element` status even -though the stored value is unchanged. -/ +`pjoin`, for instance, returns `Identity(SELF_IDENT)` rather than `Element` of +the value it selected, so a join reports that nothing changed. -/ structure ValOps (V : Type) where /-- `Lattice::pjoin` -/ pjoin : V → V → ValRes V @@ -63,12 +63,14 @@ def ValRes.resolve {V : Type} : ValRes V → V → V → Option V Both `pjoin` and `pmeet` return `Identity(SELF_IDENT)`: they are *left-biased projections* that ignore the counterpart value entirely. `psubtract` annihilates only when the two values are equal, and otherwise returns -`Element(*self)`. This is the instance the differential fuzz target uses, so -the model reproduces it exactly rather than assuming a "real" lattice. -/ +`Identity(SELF_IDENT)`: subtracting a value that is not there leaves the +destination alone, and says so. This is the instance the differential fuzz +target uses, so the model reproduces it exactly rather than assuming a "real" +lattice. -/ def u64Ops : ValOps UInt64 where pjoin _ _ := .identity true false pmeet _ _ := .identity true false - psub a b := if a == b then .none else .elem a + psub a b := if a == b then .none else .identity true false beq a b := a == b /-! ## Prefix order -/ diff --git a/lean/PathMapModel/Check.lean b/lean/PathMapModel/Check.lean index ea252df0..076107d2 100644 --- a/lean/PathMapModel/Check.lean +++ b/lean/PathMapModel/Check.lean @@ -147,6 +147,9 @@ def dropT1Result : T := ((zipAt dropT1 [0x31,0x32,0x33,0x3a] []).joinKPathInto o #guard fixtures.all (joinIdem ops) #guard fixtures.all (fun a => fixtures.all (fun b => fixtures.all (joinAssoc ops a b))) #guard fixtures.all (meetIdemOnVals ops) +#guard fixtures.all (meetIdem ops) +#guard fixtures.all (meetPrunedIdem ops) +#guard fixtures.all (fun a => fixtures.all (meetCommOnPaths ops a)) #guard fixtures.all (subSelfEmptyVals ops) #guard fixtures.all (restrictSelf ops) diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index de6c88e7..ab86db81 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -60,25 +60,18 @@ agree exactly or every input with a skip diverges. * `skip:act` — the ACT read source cannot be a merge source (`ZipperInfallibleSubtries` is not implemented for it) or does not implement the trait the op needs. -* `skip:at-root` — `to_next`/`to_prev_sibling_byte` at the zipper root, where - the native read zipper escapes its own root. * `skip:k0` — a degenerate `k = 0`. * `skip:empty-focus` — the focus has nothing below it, where the op's behaviour is a function of node materialisation rather than trie state. -* `skip:empty-path` — `insert_prefix("")`, which destroys the subtrie. * `skip:off-root-prune` — a prune on a write zipper not rooted at the map root, where the depth pruned is a function of internal node layout. -* `skip:quarantined` — the op is disabled outright (op 54). Each is recorded in FINDINGS.md and commented at its site. -/ def skipAct : String := "skip:act" -def skipAtRoot : String := "skip:at-root" def skipK0 : String := "skip:k0" def skipEmptyFocus : String := "skip:empty-focus" -def skipEmptyPath : String := "skip:empty-path" def skipOffRootPrune : String := "skip:off-root-prune" -def skipQuarantined : String := "skip:quarantined" /-! ## Rendering -/ @@ -290,17 +283,16 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do let (r, s) := onTarget s t (fun z => z.ascendUntilBranch) some (emit s "ascend_until_branch" (toString r), d) | 11 => do let (t, d) ← d.mod 2 - -- Skipped at the zipper root: `ReadZipper::to_next_sibling_byte` - -- escapes its own root there (see the notes in `Zip.toNextSiblingByte`). - if (getTarget s t).atRoot then some (emit s "to_next_sibling_byte" skipAtRoot, d) - else - let (r, s) := onTarget s t (fun z => z.toNextSiblingByte) - some (emit s "to_next_sibling_byte" (showByteOpt r), d) + -- Formerly skipped at the zipper root, where `ReadZipper:: + -- to_next_sibling_byte` used to escape its own root + -- (FINDINGS.md #3). That is fixed: the override now guards on + -- `at_root()`, returns `None` there and leaves `origin_path()` + -- alone, so the root case is compared like any other. + let (r, s) := onTarget s t (fun z => z.toNextSiblingByte) + some (emit s "to_next_sibling_byte" (showByteOpt r), d) | 12 => do let (t, d) ← d.mod 2 - if (getTarget s t).atRoot then some (emit s "to_prev_sibling_byte" skipAtRoot, d) - else - let (r, s) := onTarget s t (fun z => z.toPrevSiblingByte) - some (emit s "to_prev_sibling_byte" (showByteOpt r), d) + let (r, s) := onTarget s t (fun z => z.toPrevSiblingByte) + some (emit s "to_prev_sibling_byte" (showByteOpt r), d) | 13 => do let (t, d) ← d.mod 2 let (r, s) := onTarget s t (fun z => z.toNextStep) some (emit s "to_next_step" (showBool r), d) @@ -407,6 +399,8 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do | 38 => do let (_pr, d) ← d.bool if s.act then some (emit s "meet_into" skipAct, d) else + -- `prune = true` is best-effort (nodes shared with the source may be + -- left unpruned; see `Zip.meetInto`), so only `prune = false` is compared. let (st, z) := s.wz.meetInto ops s.rz noPrune some (emit { s with wz := z } "meet_into" (toString st), d) | 39 => do let (_pr, d) ← d.bool @@ -416,10 +410,13 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do some (emit { s with wz := z } "subtract_into" (toString st), d) | 40 => do if s.act then some (emit s "restrict" skipAct, d) else do - let leaky := s.wz.focusNodeIsEmpty + -- The status used to be masked to `?` at a focus with nothing + -- below it, as one of the node-materialisation leaks in + -- FINDINGS.md #8. It is no longer: over 48M inputs, of which + -- roughly seven in eight reach this op with an empty focus, the + -- crate's status matches the spec every time. let (st, z) := s.wz.restrict ops s.rz - some (emit { s with wz := z } "restrict" - (if leaky then "?" else toString st), d) + some (emit { s with wz := z } "restrict" (toString st), d) -- Skipped, not merely masked, when either side has nothing below -- its focus: there `restricting` branches on whether an empty node -- happens to be materialised, and the two branches differ in @@ -439,22 +436,20 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do -- subtrie in pathmap 0.3.1; see `Zip.joinKPathInto`. if k == 0 then some (emit s "join_k_path_into" skipK0, d) else - -- The bool is another `AbstractNodeRef` leak: an empty node still - -- comes back as `Some(...)` from `into_option()` for some - -- representations, so `true` gets reported for a collapse that - -- produced nothing. Compared only when something survived. - -- See FINDINGS.md #8. + -- The bool used to be masked to `?` when the collapse left an + -- empty focus, on the theory that it was another + -- `AbstractNodeRef` leak (FINDINGS.md #8). Unmasked it tracks + -- the spec: over 64M inputs, with the mask firing on about two + -- calls in three, no `join_k_path_into` line differs. let (r, z) := s.wz.joinKPathInto ops k noPrune - some (emit { s with wz := z } "join_k_path_into" - (if z.focusNodeIsEmpty then "?" else showBool r), d) + some (emit { s with wz := z } "join_k_path_into" (showBool r), d) | 43 => do let (p, d) ← d.path - -- `insert_prefix("")` destroys the subtrie in pathmap 0.3.1; see - -- `Zip.insertPrefix`. Skipped so the known bug does not mask others. - if p.isEmpty then - some (emit s "insert_prefix" skipEmptyPath, d) - else - let (r, z) := s.wz.insertPrefix p - some (emit { s with wz := z } "insert_prefix" (showBool r), d) + -- The empty prefix was skipped while `make_parents_in(b"", node)` + -- discarded the node instead of doing nothing (FINDINGS.md #4). + -- That is fixed upstream, with a regression test of its own, so + -- the empty prefix is compared like any other. + let (r, z) := s.wz.insertPrefix p + some (emit { s with wz := z } "insert_prefix" (showBool r), d) | 44 => do let (n, d) ← d.mod 6 let (r, z) := s.wz.removePrefix n some (emit { s with wz := z } "remove_prefix" (showBool r), d) @@ -527,11 +522,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do | 54 => do let (n, d) ← d.mod 4; let (m, d) ← d.pathN n; let (ru, d) ← d.bool -- Fed the source's own child subtries, this must agree with -- `graft_masked_branches` on the same mask. - -- Skipped outright, not just in ACT mode: `graft_child_maps` is - -- broken three ways (FINDINGS.md #15) and the node representations - -- it leaves behind degrade the `AlgebraicStatus` that *later* - -- operations report, which would contaminate the whole run. - if true then some (emit s "graft_child_maps" skipQuarantined, d) else + if s.act then some (emit s "graft_child_maps" skipAct, d) else do let mask := ByteMask.ofList m let maps := mask.map (fun b => ([b], s.rz.trie.subtrie (s.rz.focus ++ [b]))) diff --git a/lean/PathMapModel/PathMap.lean b/lean/PathMapModel/PathMap.lean index 2054dfd9..81fa43b4 100644 --- a/lean/PathMapModel/PathMap.lean +++ b/lean/PathMapModel/PathMap.lean @@ -338,11 +338,30 @@ def join (a b : PathMap V) : PathMap V := mk' (keys.filterMap fun k => (joinVal ops (a.valAt k) (b.valAt k)).map (k, ·)) (a.paths ++ b.paths) -/-- Meet (intersection). A location survives only if it lies on the way to a -surviving value, so dangling paths never survive a meet. -/ +/-- Meet (intersection). A location exists in the result exactly when it exists +in both operands, dangling or not, and a value exists exactly where both operands +hold one, as the meet of the two. So meet is commutative, associative and +idempotent on locations, and a meet with an equal trie changes nothing. + +This is the meet with `prune = false`; see `meetPruned`. -/ def meet (a b : PathMap V) : PathMap V := let keys := Path.sortDedup (a.vals.map (·.1)) - mk' (keys.filterMap fun k => (meetVal ops (a.valAt k) (b.valAt k)).map (k, ·)) [] + mk' (keys.filterMap fun k => (meetVal ops (a.valAt k) (b.valAt k)).map (k, ·)) + (a.paths.filter b.pathExists) + +/-- The trie with every dangling path removed: only the root and the locations on +the way to a value remain. -/ +def dropDangling (t : PathMap V) : PathMap V := mk' t.vals [] + +/-- The meet with `prune = true`, at its most pruned: the same values as `meet`, +and only the locations on the way to one of them. + +`pathmap` may stop short of this: it can skip a node shared with the source +rather than walk it, so a dangling path inside a shared node may survive. The +result always lies between `meetPruned` and `meet` -- same values, locations a +subset of `meet`'s and a superset of `meetPruned`'s -- and is not compared by +the fuzzer. -/ +def meetPruned (a b : PathMap V) : PathMap V := (meet ops a b).dropDangling /-- Subtract. diff --git a/lean/PathMapModel/Spec.lean b/lean/PathMapModel/Spec.lean index 44158a55..c930f056 100644 --- a/lean/PathMapModel/Spec.lean +++ b/lean/PathMapModel/Spec.lean @@ -314,11 +314,23 @@ def joinIdem (ops : ValOps V) (a : PathMap V) : Bool := def joinAssoc (ops : ValOps V) (a b c : PathMap V) : Bool := PathMap.beqT ops (PathMap.join ops (PathMap.join ops a b) c) (PathMap.join ops a (PathMap.join ops b c)) -/-- `meet` is idempotent *on values*. It is not idempotent on locations: a meet -discards dangling paths, so `meet a a` keeps only the value-bearing skeleton. -/ +/-- `meet` is idempotent *on values*. -/ def meetIdemOnVals (ops : ValOps V) (a : PathMap V) : Bool := (PathMap.meet ops a a).vals.map (·.1) == a.vals.map (·.1) +/-- `meet` is idempotent outright, dangling paths included: a location survives +exactly when both operands have it. -/ +def meetIdem (ops : ValOps V) (a : PathMap V) : Bool := + PathMap.beqT ops (PathMap.meet ops a a) a + +/-- `meetPruned a a` is `a` with its dangling paths dropped. -/ +def meetPrunedIdem (ops : ValOps V) (a : PathMap V) : Bool := + PathMap.beqT ops (PathMap.meetPruned ops a a) a.dropDangling + +/-- `meet` is commutative on locations. -/ +def meetCommOnPaths (ops : ValOps V) (a b : PathMap V) : Bool := + (PathMap.meet ops a b).paths == (PathMap.meet ops b a).paths + /-- Subtracting a map from itself leaves no values. -/ def subSelfEmptyVals (ops : ValOps V) (a : PathMap V) : Bool := (PathMap.sub ops a a).vals.isEmpty diff --git a/lean/PathMapModel/Write.lean b/lean/PathMapModel/Write.lean index 6ee3a84e..f086652a 100644 --- a/lean/PathMapModel/Write.lean +++ b/lean/PathMapModel/Write.lean @@ -328,9 +328,13 @@ def joinIntoTake (src : Zip V) (prune : Bool) : AlgStatus × Zip V × Zip V := /-- `ZipperWriting::meet_into`: intersect the focus's subtrie with the source's. -The value step runs first and can prune the focus out from under the node step. -A meet drops every dangling path, since a location only survives if it leads to -a surviving value. -/ +Below the focus the result is `PathMap.meet` with `prune = false` -- every +location both sides have survives, dangling ones included. With `prune = true` +the model gives `PathMap.meetPruned`, which keeps only the locations leading to a +surviving value; `pathmap` may leave dangling paths inside nodes shared with the +source, so that case is best-effort and not compared. The focus itself is never +removed: pruning stops at it, so a focus left without a value or anything below +it is still there. -/ def meetInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := let (valStatus, valWasNone, z1) := match z.val, src.val with @@ -339,26 +343,20 @@ def meetInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := (AlgStatus.ofValRes r, false, match r.resolve sv ov with | some v => (z.setVal v).2 - | none => (z.removeVal prune).2) + | none => (z.removeVal false).2) | none, some _ => (AlgStatus.none, true, z) - | some _, none => (AlgStatus.none, false, (z.removeVal prune).2) + | some _, none => (AlgStatus.none, false, (z.removeVal false).2) | none, none => (AlgStatus.none, true, z) let selfB := z1.focusNode let srcB := src.focusNode if selfB.isEmptyMap then (AlgStatus.merge .none valStatus true valWasNone, z1) else if srcB.isEmptyMap then - let z2 := z1.withTrie (z1.trie.removeBelow z1.focus) - let z3 := if prune then (z2.prunePath).2 else z2 - (AlgStatus.merge .none valStatus false valWasNone, z3) + (AlgStatus.merge .none valStatus false valWasNone, z1.withTrie (z1.trie.removeBelow z1.focus)) else - let r := PathMap.meet ops selfB srcB + let r := if prune then PathMap.meetPruned ops selfB srcB else PathMap.meet ops selfB srcB let st := nodeStatus ops selfB r - let z2 := - if st == .identity then z1 - else - let zg := z1.withTrie (z1.trie.graftBelow z1.focus r) - if st == .none && prune then (zg.prunePath).2 else zg + let z2 := if st == .identity then z1 else z1.withTrie (z1.trie.graftBelow z1.focus r) (AlgStatus.merge st valStatus false valWasNone, z2) /-- `ZipperWriting::subtract_into`: remove the source's subtrie from the focus's. @@ -396,7 +394,8 @@ def subtractInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := (AlgStatus.merge st valStatus false valWasNone, z2) /-- `ZipperWriting::meet_2`: meet two *source* subtries and write the result at -the focus. +the focus. There is no `prune` argument, so this is `PathMap.meet`: every location +both sources have survives, dangling ones included. Two things separate this from `meet_into`. It does not consult what is already at the focus, so — as the implementation notes — it never reports `Identity`, @@ -486,9 +485,11 @@ def meetKPathInto (k : Nat) (prune : Bool) : Bool × Zip V := match acc with | none => some m | some a => some (PathMap.meet ops a m)) none - match result with - | some m => if m.isEmptyMap then (false, (z.removeBranches prune).2) else (true, z.graftMap m) - | none => (false, (z.removeBranches prune).2) + -- `prune` drops every dangling path from the meet, including one that a single + -- k-path's subtrie brings along unmet. The focus itself is never removed. + match result.map fun m => if prune then m.dropDangling else m with + | some m => if m.isEmptyMap then (false, (z.removeBranches false).2) else (true, z.graftMap m) + | none => (false, (z.removeBranches false).2) end Zip end PathMapModel diff --git a/lean/README.md b/lean/README.md index f3a0cb43..e93231b5 100644 --- a/lean/README.md +++ b/lean/README.md @@ -331,12 +331,23 @@ skips diverges: |---|---| | `skip:k0` | `meet_k_path_into(0)`, `join_k_path_into(0)`, `descend_first_k_path(0)` / `to_next_k_path(0)` — degenerate; the first two should be the identity and destroy the subtrie, the last reports success without moving, forever. | | `skip:empty-focus` | `meet_k_path_into` with no children (it does not terminate), and `restricting` when either side has nothing below its focus (the two branches differ in *effect*, not just in the reported bool). | -| `skip:empty-path` | `insert_prefix("")` — should be the identity, destroys the subtrie. | -| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root — the native read zipper leaves its own root there. | | `skip:off-root-prune` | `prune_path` / `prune_ascend`, and the `prune` flag on every other operation, for a write zipper not rooted at the map root — the depth pruned is a function of internal node layout, so there is nothing to specify. | -| `skip:quarantined` | `graft_child_maps` (op 54), disabled outright: it is broken three ways (FINDINGS.md #15) and the node representations it leaves behind degrade the `AlgebraicStatus` that *later* operations report. | | `skip:act` | ACT mode only — the read source cannot be a merge source (`ZipperInfallibleSubtries` is not implemented for it) or does not implement the trait the op needs. | +#### Suppressions that have been lifted + +A skip is only worth keeping while the defect behind it is live. These were +re-tested against the current crate, found stale, and removed — the operation is +now compared like any other. Each was A/B'd by lifting the guard on all three +sides at once and re-running the same sweeps: the divergence report has to come +back unchanged, since a fixed operation changes no state. + +| token | why it is gone | +|---|---| +| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 64M inputs (20M at maxlen 120, 20M at 300 and 8M at 600 in crate mode, 8M each at 300/600 in ACT mode) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | +| `skip:empty-path` | `insert_prefix("")` (FINDINGS.md #4). Fixed upstream, with the regression test `write_zipper_insert_prefix_empty_is_identity`. Directly: over seven trie shapes (empty, root value only, one line, branchy, wide, dangling, grafted), at the map root and at a non-root focus, `insert_prefix(&[])` leaves the trie untouched and returns exactly what the model's `focusNodeIsEmpty` rule predicts. Same 64M-input A/B, again byte for byte identical. | +| `skip:quarantined` | `graft_child_maps` (op 54), the only op it ever named, was un-quarantined when its bug was fixed. The token outlived it as a definition with no call site on any of the three sides — the Rust one needed `#[allow(dead_code)]` to compile. Removed rather than kept: a vocabulary entry nothing can emit invites the next reader to keep a live suppression alive by analogy with it. | + Naming these turned up an ordering bug the bare token had hidden: for `restricting` the model tested ACT mode first and the harness tested the empty focus first, so in ACT mode with an empty focus the two took different branches @@ -348,11 +359,24 @@ the harness's order. `k_path_internal` carries iteration state and `pathmap`'s own debug assertions flag calling it cold. -Five return values are compared as `?` when the focus has no descendants -(`remove_branches`, `join_map_into`, `restrict`, `restricting`, `take_map`): -they report on whether an empty node happens to be materialised at the focus, -which is representation state rather than trie state. The *effects* are still -compared in full. +Three return values are compared as `?` when the focus has no descendants +(`remove_branches`, `join_map_into`, `take_map`): they report on whether an +empty node happens to be materialised at the focus, which is representation +state rather than trie state. The *effects* are still compared in full. Each +was re-tested by unmasking it alone and re-running the sweeps, and each diverges +on roughly one input in eight the moment the mask comes off, which is what keeps +it. + +`restrict` and `join_k_path_into` were on that list and are not any more. +Unmasked, both match the specification over the whole sweep — 20M inputs at +maxlen 120, 20M at 300, 8M at 600, and for `join_k_path_into` 8M each at +300/600 in ACT mode as well; `restrict` is `skip:act`, so ACT says nothing +about it. Neither is a branch that never runs: tracing 1500 inputs, the mask +fired on 824 of 947 `restrict` calls and 678 of 1003 `join_k_path_into` calls, +which puts the sweeps at roughly 26M and 29M formerly-masked returns actually +compared. That is evidence the materialisation leak does not reach these two, +not a proof that it cannot; each was lifted in its own commit so it can be put +back alone. ## The blind-zipper contract diff --git a/lean/corpus/README.md b/lean/corpus/README.md index 5862cec2..49a083f5 100644 --- a/lean/corpus/README.md +++ b/lean/corpus/README.md @@ -9,10 +9,16 @@ of them reduce to a snippet small enough to write out as a Rust example. ./lean/shrink.py lean/corpus/ # minimise one further ``` +That exits non-zero, and is meant to: `dangling-residue-only-in-map0.bin` is here +precisely because `classify()` cannot recognise it, so it is reported as a new +divergence every time. The exit code says "one entry is unclassified", not "the +crate regressed". + | file | what it shows | | --- | --- | | `status-imprecise-join_map_into.bin` | `join_map_into` reports `Element` where the trie is provably unchanged (FINDINGS.md #8) | | `status-imprecise-restrict.bin` | `restrict` reports `Element` where the trie is provably unchanged (FINDINGS.md #8) | +| `dangling-residue-only-in-map0.bin` | a kept dangling child that no operation's fingerprint reveals: every one of the run's steps agrees, and the extra valueless path shows up only in the final `MAP0` dump. Same defect as `meet_keeps_dangling`, but `classify()` keys that one on `child_count`/`val_count` moving on the trace line, and a `MAP` line has neither — so this manifestation is reported as a new divergence | Everything else in FINDINGS.md has a standalone reproducer instead; see `cargo run -p differential --bin zipper_bug_repros -- --list`. diff --git a/lean/corpus/dangling-residue-only-in-map0.bin b/lean/corpus/dangling-residue-only-in-map0.bin new file mode 100644 index 00000000..25d5d5b9 Binary files /dev/null and b/lean/corpus/dangling-residue-only-in-map0.bin differ diff --git a/lean/differential.py b/lean/differential.py index 47b593dd..b43c71a8 100755 --- a/lean/differential.py +++ b/lean/differential.py @@ -7,6 +7,7 @@ lean/.lake/build/bin/pathmap-oracle the Lean model (always) target/*/pathmap_trace the real crate (default) + target/*/reference the Rust model (--model) target/*/act_trace ACT read source (--act) Each child is spawned once with `--server` and stays resident, taking inputs as @@ -17,6 +18,14 @@ ./lean/differential.py corpus/* # check a corpus ./lean/differential.py --random 500 # generate and check + ./lean/differential.py --model --random 500 # check the Rust port + +`--model` is the acceptance test for `differential/src/reference/`: it compares +two independent transcriptions of the same specification, in different +languages, with the crate not involved at all. So the KNOWN table below does not +apply -- every divergence is a bug in one of the two models, and none may be +tolerated. Once it is clean, `target/release/in_process` compares the Rust model +against the crate in one process, with no pipes, about 4x faster. """ import argparse import multiprocessing @@ -41,6 +50,12 @@ os.path.join(ROOT, "target", "release", "act_trace"), os.path.join(ROOT, "target", "debug", "act_trace"), ] +# The Rust port of the Lean model (differential/src/reference/), driven by +# `--model`. Its trace front end is `differential/src/bin/reference.rs`. +MODEL_CANDIDATES = [os.environ.get("PATHMAP_REFERENCE", "")] + [ + os.path.join(ROOT, "target", "release", "reference"), + os.path.join(ROOT, "target", "debug", "reference"), +] # Seconds a single input may take. Measured over 2000 random programs against # the real crate, the non-hanging ones run in p50 0.19ms / p100 1.21ms, so this # is ~1600x the worst legitimate case and still cuts the cost of a hang by 15x @@ -48,10 +63,18 @@ TIMEOUT = 2.0 -def find_trace_bin(act): - for c in (ACT_CANDIDATES if act else TRACE_CANDIDATES): +def find_trace_bin(act, model=False): + if model: + candidates = MODEL_CANDIDATES + elif act: + candidates = ACT_CANDIDATES + else: + candidates = TRACE_CANDIDATES + for c in candidates: if c and os.path.exists(c): return c + if model: + sys.exit("build the Rust model first: cargo build --release -p differential") if act: sys.exit("build the ACT side first: " "cargo build --release -p differential") @@ -215,6 +238,12 @@ def run(self, blob): (["join_into"], "join_into() drops the source subtrie when the destination map is empty " "[join_into_empty_dst]"), + # Tested before the write-zipper entry below, which keys on the bare op name + # and was claiming these: here the write zipper is identical on both sides + # and only the ACT read zipper moved, so finding 9 cannot be the cause. + (["ascend_until", "ACT-READ-ZIPPER-ONLY"], + "ACTZipper::ascend_until()/ascend_until_branch() report a different ascent " + "than the model, usually one byte short [act: ascend_until_short]"), (["ascend_until"], "ascend_until()/ascend_until_branch() corrupt a write zipper rooted at a " "node boundary [ascend_until_wz]"), @@ -391,6 +420,20 @@ def divergence_shape(a, b): return None +def read_zipper_only(a, b): + """Do these differ only in the read zipper, leaving the write zipper equal? + + In `--act` mode the read zipper is the ArenaCompactTree one and the write + zipper is still a `PathMap` one, so this separates an ACT read-side defect + from a write-side defect that the same operation would also report. + """ + if any(" W=" not in t or " R=" not in t for t in (a, b)): + return False + w = lambda t: t.split(" W=", 1)[1].split(" R=", 1)[0] + r = lambda t: t.split(" R=", 1)[1] + return w(a) == w(b) and r(a) != r(b) + + def act_valcount_only(a, b): """Do these two trace lines differ *only* in the read zipper's val_count? @@ -430,6 +473,8 @@ def compare(blob, oracle, other, other_label, act=False): for i, (a, b) in enumerate(zip(lean, real)): if a != b: tags = ["ACT-VALCOUNT-ONLY"] if act and act_valcount_only(a, b) else [] + if act and read_zipper_only(a, b): + tags.append("ACT-READ-ZIPPER-ONLY") shape = divergence_shape(a, b) if shape: tags.append(shape) @@ -582,6 +627,9 @@ def main(): ap.add_argument("-v", "--verbose", action="store_true") ap.add_argument("--act", action="store_true", help="use an ArenaCompactTree as the read source") + ap.add_argument("--model", action="store_true", + help="compare the Lean model against the Rust model " + "(differential/src/reference/) instead of against the crate") ap.add_argument("-j", "--jobs", type=int, default=1, help="run this many worker processes in parallel " "(each owns its own pair of children)") @@ -592,8 +640,8 @@ def main(): args = ap.parse_args() TIMEOUT = args.timeout - trace_bin = find_trace_bin(args.act) - other_label = "crate" + trace_bin = find_trace_bin(args.act, args.model) + other_label = "rust" if args.model else "crate" # Inputs are produced by an `InputSource`, in whichever worker picks the # index up -- see the class docs. Nothing is written to disk and no blob is @@ -610,7 +658,7 @@ def main(): n_inputs = len(source) oracle_argv = [ORACLE] + (["--act"] if args.act else []) - other_argv = [trace_bin] + other_argv = [trace_bin] + (["--act"] if (args.act and args.model) else []) faildir = [] # created on first failure only def save(idx): @@ -637,7 +685,9 @@ def record(idx, msg): if args.verbose: print("ok %s" % name) return False - note = classify(msg) + # Model against model: the KNOWN table is a list of *crate* defects, and + # the crate is not involved. Every divergence is new. + note = None if args.model else classify(msg) if note: known[note] = known.get(note, 0) + 1 if args.verbose: diff --git a/lean/shrink.py b/lean/shrink.py index 65f23f64..2d94ae35 100755 --- a/lean/shrink.py +++ b/lean/shrink.py @@ -6,7 +6,7 @@ ./lean/shrink.py [-o out.bin] """ -import argparse, os, re, subprocess, sys +import argparse, os, re, subprocess, sys, tempfile ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # PATHMAP_ORACLE / PATHMAP_TRACE / PATHMAP_ACT_TRACE override the defaults, for @@ -21,7 +21,9 @@ ACT_DEBUG = os.path.join(ROOT, "target", "debug", "act_trace") TRACE = TRACE_RELEASE ORACLE_ARGS = [] -TMP = os.path.join(ROOT, "lean", ".shrink.bin") +# Per process, in $TMPDIR: a shared fixed path let concurrent shrinks overwrite each +# other's candidate inputs, so they shrank toward each other's bugs or none at all. +TMP = os.path.join(tempfile.gettempdir(), "pathmap-shrink-%d.bin" % os.getpid()) def signature(blob): diff --git a/src/arena_compact.rs b/src/arena_compact.rs index 3914ba8b..fe216598 100644 --- a/src/arena_compact.rs +++ b/src/arena_compact.rs @@ -2668,17 +2668,14 @@ where Storage: AsRef<[u8]> return start_len - self.path.len(); } - match &self.cur_node { - Node::Line(line) => { - if need_value && line.value.is_some() { - return start_len - self.path.len(); - } - } - Node::Branch(node) => { - if need_value && node.value.is_some() { - return start_len - self.path.len(); - } - } + //The focus is now the deepest real ancestor of where we started, which is a strict + //ancestor, so it is a candidate stop in its own right. It stops the ascent under + //exactly the conditions the rest of this method uses: a value here when `need_value`, + //or a branch. Both have to be read off the *focus*, not off the node: a line node + //carries its value at its end, so `line.value` says nothing about a focus that sits + //partway into the line, and a line's interior always has exactly one child. + if (need_value && self.is_val()) || self.child_count() > 1 { + return start_len - self.path.len(); } } while let Some(top_frame) = self.stack.last_mut() { @@ -2772,6 +2769,39 @@ where Storage: AsRef<[u8]> } fn to_sibling(&mut self, next: bool) -> Option { + //An off-trie focus has no stack frame -- the stack holds real nodes, and a byte that is + //not in the trie has no node -- so the index-based path below cannot serve it and used to + //answer `None`. That starves `to_next_step`, which is `ZipperIteration`'s default and + //moves by this method: from a non-existent focus that sorts before an existing sibling it + //would give up and reset rather than step to it. + // + //The sibling of a phantom byte is still well defined, because it is defined by the + //*parent's* children rather than by the focus: the next child byte strictly greater than + //the phantom one. That only makes sense while the parent itself is real, so a focus more + //than one byte off the trie has no siblings -- its parent has no children at all. + if self.invalid > 0 { + if self.invalid > 1 { + return None; + } + let byte = *self.path.last()?; + if self.ascend_invalid(Some(1)) != 1 { + //The phantom byte is the zipper's root, so there is nothing to be a sibling of. + return None; + } + let mask = self.child_mask(); + let target = if next { mask.next_bit(byte) } else { mask.prev_bit(byte) }; + match target { + Some(t) => { + self.descend_to_byte(t); + return Some(t); + } + None => { + //Documented to leave the zipper where it was when it does not move. + self.descend_to_byte(byte); + return None; + } + } + } let top_frame = self.stack.last().unwrap(); if self.stack.len() <= 1 || top_frame.node_depth > 0 { // can't move to sibling at root, or along the path @@ -2938,13 +2968,24 @@ where Storage: AsRef<[u8]> /// WARNING: This is not a cheap method. It may have an order-N cost fn val_count(&self) -> usize { timed_span!(ValueCount, COUNTERS); + //`ZipperMoving::val_count` counts the values at and below the *focus*. This used to + // `reset()` first, so it counted the whole subtrie below the zipper's root and returned + // the same number wherever the focus was -- right only when the focus was at the root. + // + //`to_next_val` walks the zipper's entire subtrie, not just the part below the focus, so + // the walk has to stop when it leaves: depth-first order visits everything below the + // focus before anything outside it, so the first path that no longer starts with the + // focus ends the count. let mut zipper = self.clone(); - zipper.reset(); + let focus: Vec = zipper.path().to_vec(); let mut count = 0; if zipper.is_val() { count += 1; } while zipper.to_next_val() { + if !zipper.path().starts_with(&focus) { + break; + } count += 1; } count @@ -4191,4 +4232,133 @@ mod tests { assert_eq!(az.val(), None); assert!(!az.path_exists()); } + /// `ACTZipper::val_count` opened with `reset()`, so it counted the values below + /// the zipper's *root* whatever the focus was, and was right only at the root. + /// `ZipperValues::val_count` counts at and below the focus. + #[test] + fn act_zipper_val_count_counts_from_the_focus() { + use crate::zipper::*; + let mut m = PathMap::::new(); + m.insert(b"aa", 1); + m.insert(b"ab", 2); + m.insert(b"b", 3); + let t = ArenaCompactTree::from_zipper(m.read_zipper(), |&v| v); + for path in [&b""[..], b"a", b"aa", b"ab", b"b", b"zz"] { + let mut az = t.read_zipper_u64(); + let mut pz = m.read_zipper(); + az.descend_to(path); + pz.descend_to(path); + assert_eq!(az.val_count(), pz.val_count(), "focus {path:?}"); + } + //And from a zipper rooted below the map root + for path in [&b""[..], b"a", b"b"] { + let mut az = t.read_zipper_at_path_u64(b"a"); + let mut pz = m.read_zipper_at_path(b"a"); + az.descend_to(path); + pz.descend_to(path); + assert_eq!(az.val_count(), pz.val_count(), "root a, focus {path:?}"); + } + } + + /// `ACTZipper::to_sibling` worked entirely off the node stack, which holds real + /// nodes, so a focus one byte off the trie had no frame and the method answered + /// `None`. That starved `to_next_step`, which moves by it: from a non-existent + /// focus sorting before an existing sibling it gave up instead of stepping to it, + /// and whole subtrees went unvisited. The sibling of a phantom byte is defined by + /// the parent's children, so it exists while the parent is real. + #[test] + fn act_zipper_sibling_step_from_an_off_trie_focus() { + use crate::zipper::*; + let mut m = PathMap::::new(); + { let mut w = m.write_zipper(); w.set_val(38); } + m.insert(&[1u8], 5); + m.insert(&[1u8, 0, 2], 22); + m.insert(&[3u8], 7); + let t = ArenaCompactTree::from_zipper(m.read_zipper(), |&v| v); + + //One byte off the trie, with a sibling on either side + let mut az = t.read_zipper_u64(); + az.descend_to(&[2u8]); + assert!(!az.path_exists()); + assert_eq!(az.to_next_sibling_byte(), Some(3)); + assert_eq!(az.path(), &[3u8]); + assert_eq!(az.val(), Some(&7)); + az.ascend(1); + az.descend_to(&[2u8]); + assert_eq!(az.to_prev_sibling_byte(), Some(1)); + assert_eq!(az.path(), &[1u8]); + assert_eq!(az.val(), Some(&5)); + + //No sibling on that side: the zipper stays where it was + let mut az = t.read_zipper_u64(); + az.descend_to(&[0u8]); + assert_eq!(az.to_prev_sibling_byte(), None); + assert_eq!(az.path(), &[0u8]); + assert!(!az.path_exists()); + assert_eq!(az.to_next_sibling_byte(), Some(1)); + + //Two bytes off the trie: the parent is not real, so there is no sibling + let mut az = t.read_zipper_u64(); + az.descend_to(&[2u8, 0]); + assert_eq!(az.to_next_sibling_byte(), None); + assert_eq!(az.path(), &[2u8, 0]); + + //`to_next_step` from an off-trie focus visits what follows it + let mut az = t.read_zipper_u64(); + az.descend_to(&[0u8]); + let mut seen = Vec::new(); + while az.to_next_step() { seen.push(az.path().to_vec()); } + assert_eq!(seen, vec![vec![1u8], vec![1, 0], vec![1, 0, 2], vec![3]]); + } + + /// `ascend_to_branch`, behind `ascend_until` and `ascend_until_branch`, ascends the + /// non-existent tail of the path first and then has to decide whether the real ancestor it + /// lands on already stops the ascent. It read that off the *node* rather than off the + /// focus: it took a line node's `value`, which sits at the line's end, for a value at a + /// focus partway into the line, and it never considered branching at all. So from an + /// off-trie focus the ascent stopped short of the first real value or branch whenever the + /// deepest real ancestor sat mid-line in a line that ends in a value, and ran a byte past + /// that ancestor whenever it was a branch. + #[test] + fn act_zipper_ascend_until_from_an_off_trie_focus() { + use crate::zipper::*; + let mut m = PathMap::::new(); + m.insert(&[1u8, 2, 3, 4], 11); //a line under 01, its value at the line's end + m.insert(&[5u8, 2], 22); //a branch at 05, two children, no value + m.insert(&[5u8, 6], 33); + m.insert(&[7u8], 44); //a value at 07, which branches below it as well + m.insert(&[7u8, 8], 55); + m.insert(&[7u8, 9], 66); + let t = ArenaCompactTree::from_zipper(m.read_zipper(), |&v| v); + + let off_trie: [&[u8]; 9] = [&[1, 2, 9], &[1, 2, 3, 9], &[1, 2, 3, 4, 9], &[1, 9, 9], + &[5, 9], &[5, 2, 9], &[7, 9, 9], &[7, 8, 9, 9], &[9]]; + for root in [&[][..], &[1u8], &[1, 2], &[7]] { + for focus in off_trie { + if !focus.starts_with(root) { continue } + let focus = &focus[root.len()..]; + for need_value in [false, true] { + let mut az = t.read_zipper_at_path_u64(root); + let mut pz = m.read_zipper_at_path(root); + assert_eq!(az.descend_to(focus), pz.descend_to(focus)); + assert!(!az.path_exists() && !pz.path_exists(), "{root:?} {focus:?}"); + let (a, p) = if need_value { + (az.ascend_until(), pz.ascend_until()) + } else { + (az.ascend_until_branch(), pz.ascend_until_branch()) + }; + assert_eq!(a, p, "root {root:?} focus {focus:?} need_value {need_value}"); + assert_eq!(az.path(), pz.path(), "root {root:?} focus {focus:?}"); + //Whether the frame still agrees with the path the ascent left the zipper + //at only shows on the next move, so look at the focus and then move + assert_eq!(az.path_exists(), pz.path_exists(), "{root:?} {focus:?}"); + assert_eq!(az.val(), pz.val(), "{root:?} {focus:?}"); + assert_eq!(az.child_count(), pz.child_count(), "{root:?} {focus:?}"); + assert_eq!(az.descend_first_byte(), pz.descend_first_byte(), "{root:?} {focus:?}"); + assert_eq!(az.path(), pz.path(), "{root:?} {focus:?} after descend"); + assert_eq!(az.val(), pz.val(), "{root:?} {focus:?} after descend"); + } + } + } + } } diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 65568d9c..cafd5d55 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -260,6 +260,94 @@ impl> ByteNode } } + /// [Self::join_child_into] with `node` as the *left* operand of the join, so on a collision + /// `node`'s values take precedence over `self`'s. The status is still relative to `self`. + pub(crate) fn join_child_into_left(&mut self, k: u8, node: TrieNodeODRc) -> AlgebraicStatus where V: Clone + Lattice { + let ix = self.mask.index_of(k) as usize; + if self.mask.test_bit(k) { + let cf = unsafe { self.values.get_unchecked_mut(ix) }; + match cf.rec_mut() { + Some(existing_node) => { + match node.pjoin(existing_node) { + //`COUNTER_IDENT` means the result is `existing_node`: nothing to do + AlgebraicResult::Identity(mask) if mask & COUNTER_IDENT > 0 => AlgebraicStatus::Identity, + AlgebraicResult::Identity(_) => { + *existing_node = node; + AlgebraicStatus::Element + }, + AlgebraicResult::Element(joined) => { + *existing_node = joined; + AlgebraicStatus::Element + }, + //Only two empty nodes join to nothing, and then there is nothing to change + AlgebraicResult::None => AlgebraicStatus::Identity, + } + }, + None => { + cf.set_rec(node); + AlgebraicStatus::Element + } + } + } else { + self.mask.set_bit(k); + let new_cf = CoFree::new(Some(node), None); + self.values.insert(ix, new_cf); + AlgebraicStatus::Element + } + } + + /// [Self::join_val_into] with `val` as the *left* operand of the join; see [Self::join_child_into_left] + pub(crate) fn join_val_into_left(&mut self, k: u8, val: V) -> AlgebraicStatus where V: Lattice { + let ix = self.mask.index_of(k) as usize; + if self.mask.test_bit(k) { + let cf = unsafe { self.values.get_unchecked_mut(ix) }; + match cf.val_mut() { + Some(existing_val) => { + match val.pjoin(existing_val) { + AlgebraicResult::Identity(mask) if mask & COUNTER_IDENT > 0 => AlgebraicStatus::Identity, + AlgebraicResult::Identity(_) => { + *existing_val = val; + AlgebraicStatus::Element + }, + AlgebraicResult::Element(joined) => { + *existing_val = joined; + AlgebraicStatus::Element + }, + //A join of two present values never has an empty result; see `Lattice::join_into` + AlgebraicResult::None => AlgebraicStatus::Identity, + } + } + None => { + cf.set_val(val); + AlgebraicStatus::Element + } + } + } else { + self.mask.set_bit(k); + let new_cf = CoFree::new(None, Some(val)); + self.values.insert(ix, new_cf); + AlgebraicStatus::Element + } + } + + /// Dispatches to [Self::join_child_into] or [Self::join_child_into_left] + #[inline] + pub(crate) fn join_child_into_oriented(&mut self, k: u8, node: TrieNodeODRc, incoming_is_left: bool) -> AlgebraicStatus where V: Clone + Lattice { + if incoming_is_left { self.join_child_into_left(k, node) } else { self.join_child_into(k, node) } + } + + /// Dispatches to [Self::join_payload_into] or its left-biased counterpart + #[inline] + pub(crate) fn join_payload_into_oriented(&mut self, k: u8, payload: ValOrChild, incoming_is_left: bool) -> AlgebraicStatus where V: Clone + Lattice { + if !incoming_is_left { + return self.join_payload_into(k, payload) + } + match payload { + ValOrChild::Child(child) => self.join_child_into_left(k, child), + ValOrChild::Val(val) => self.join_val_into_left(k, val), + } + } + /// Internal method to remove a CoFree from the node #[inline] fn remove(&mut self, k: u8) -> Option { @@ -418,10 +506,12 @@ impl> ByteNode let cf = unsafe{ self_node.values.get_unchecked(cf_idx) }; let mut new_cf = Cf::new(None, None); - //If there is a value at this key_byte, and the other node contains a value, subtract them + //If there is a value at this key_byte, and the other node contains a value, subtract them. + // If the other node only passes through this byte on the way to something deeper, it holds + // nothing at this exact location, so the value survives untouched. if let Some(self_val) = cf.val() { - if let Some(other_val) = other.node_get_val(&[key_byte]) { - match self_val.psubtract(other_val) { + match other.node_get_val(&[key_byte]) { + Some(other_val) => match self_val.psubtract(other_val) { AlgebraicResult::None => { is_identity = false; }, AlgebraicResult::Identity(mask) => { debug_assert_eq!(mask, SELF_IDENT); //subtract is not commutative @@ -431,7 +521,8 @@ impl> ByteNode is_identity = false; new_cf.set_val(e); }, - } + }, + None => new_cf.set_val(self_val.clone()), } } @@ -461,10 +552,17 @@ impl> ByteNode } } - //If we ended up with a value or a link in the CF, insert it into a new node + //If we ended up with a value or a link in the CF, insert it into a new node. + // Otherwise the location is gone from the result -- including when `self` held + // only a dangling path here (an empty link, or a CoFree with neither), which the + // other node reaches and so does not survive the subtraction. Either way the + // result is no longer `self`; leaving `is_identity` set here reported `Identity` + // for a node that had just lost a branch, and the caller kept the dangling path. if new_cf.has_rec() || new_cf.has_val() { new_node.mask.set_bit(key_byte); new_node.values.push(new_cf); + } else { + is_identity = false; } } else { new_node.mask.set_bit(key_byte); @@ -501,6 +599,14 @@ impl> ByteNode new_node.values.push(cf.clone()); } else { + //Without a value in `other`, the empty path doesn't validate, so a value of + // `self`'s at this byte is dropped. That is a modification, even when the + // onward link below it restricts to an identity, and it is also the whole of + // the change when there is no onward link at all. + if cf.val().is_some() { + is_identity = false; + } + //If there is an onward link in the CF and other node, continue the restriction recursively if let Some(self_child) = cf.rec() { let other_child = other.get_node_at_key(&[key_byte]); @@ -527,6 +633,14 @@ impl> ByteNode is_identity = false; } } + } else { + //No onward link either, so nothing of this byte reaches the result. A + // co-free with neither a value nor a child is the dense representation of a + // dangling path, and an unvalidated dangling path is dropped just like an + // unvalidated value -- so this branch is a modification too. Leaving the + // flag set reported `Identity`, and the caller then kept `self` with the + // dangling path still in it. + is_identity = false; } } } else { @@ -545,7 +659,13 @@ impl> ByteNode } /// Merges the entries in the ListNode into the ByteNode - pub fn merge_from_list_node(&mut self, list_node: &LineListNode) -> AlgebraicStatus where V: Clone + Lattice { + /// Joins the contents of `list_node` into `self`. + /// + /// The join is left-biased, so `list_is_left` says which operand `list_node` is: `false` when + /// the caller is computing `self ∪ list_node` (collisions keep `self`'s values), `true` when it + /// is computing `list_node ∪ self` into a clone of the byte node (collisions keep the list + /// node's values). The returned status is always relative to `self`. + pub fn merge_from_list_node(&mut self, list_node: &LineListNode, list_is_left: bool) -> AlgebraicStatus where V: Clone + Lattice { let self_was_empty = self.is_empty(); self.reserve_capacity(2); @@ -555,9 +675,9 @@ impl> ByteNode if key.len() > 1 { let mut child_node = LineListNode::::new_in(self.alloc.clone()); unsafe{ child_node.set_payload_owned::<0>(&key[1..], payload); } - self.join_child_into(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone())) + self.join_child_into_oriented(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone()), list_is_left) } else { - self.join_payload_into(key[0], payload) + self.join_payload_into_oriented(key[0], payload, list_is_left) } } else { if self_was_empty { @@ -573,9 +693,9 @@ impl> ByteNode if key.len() > 1 { let mut child_node = LineListNode::::new_in(self.alloc.clone()); unsafe{ child_node.set_payload_owned::<0>(&key[1..], payload); } - self.join_child_into(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone())) + self.join_child_into_oriented(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone()), list_is_left) } else { - self.join_payload_into(key[0], payload) + self.join_payload_into_oriented(key[0], payload, list_is_left) } } else { if self_was_empty { @@ -614,14 +734,11 @@ impl CellByteNode { } let cf = self.get_mut(k).unwrap(); let (rec, val) = cf.both_mut_refs(); - let rec = match rec { - Some(rec) => rec, - None => { - *rec = Some(TrieNodeODRc::new_allocated_in(0, 0, alloc)); - rec.as_mut().unwrap() - } - }; - (rec, val) + //A zipper can't be rooted at the empty sentinel, so an emptied link gets a real node too + if rec.as_ref().map_or(true, |rec| rec.is_empty()) { + *rec = Some(TrieNodeODRc::new_allocated_in(0, 0, alloc)); + } + (rec.as_mut().unwrap(), val) } } @@ -720,102 +837,6 @@ impl> TrieNode let cf = self.get_mut(key[0]).unwrap(); *cf.rec_mut().unwrap() = new_node; } - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - //DISCUSSION: This function appears overly complicated primarily because it needs to track - // whether or not a both the val and the rec each cofree are requested, but we don't have a bitmask - // in advance that records vals and rec links separately. Since we don't want nested loops, we leverage - // the fact that a rec must be requested before a val, to stash the val for the next trip through the - // loop. The loop body therefore is an annoying state-machine. But at least it's not that much code. - - //Becomes true if only half of `(Some, Some)` CoFree is requested, without requesting the other half - // This flag never gets unset once it gets set - let mut unrequested_cofree_half = false; - //Temporary state that bridges across multiple requests into a `(Some, Some)` CoFree, by holding the - // val until it's requested, leveraging the fact that values are requested after rec links - let mut stashed_val: Option<&V> = None; - //Tracks whether the current CoFree's val has been taken. So, `last_byte` toggles to `Some` and stays - // at `Some` until we move onto a different CF, while `stashed_val` toggles to `Some`, and toggles back - // as soon as the value is requested. - let mut last_byte: Option = None; - //Tracks which CoFrees have yet to be requested from the node - let mut requested_mask = ByteMask::from(self.mask); - - debug_assert!(results.len() >= keys.len()); - for ((key, expect_val), (result_key_len, payload_ref)) in keys.into_iter().zip(results.iter_mut()) { - if key.len() > 0 { - let byte = key[0]; - - //Check to see if we had a Val from the CoFree that we aren't going to request - match &last_byte { - Some(prev_byte) => { - if byte != *prev_byte { - if stashed_val.is_some() { - unrequested_cofree_half = true; - } - stashed_val = None; - last_byte = None; - } - }, - None => {} - } - - //Check to see if this trip through the loop is the request for the stashed val - match stashed_val { - Some(val) => { - if key.len() == 1 && *expect_val { - *result_key_len = 1; - *payload_ref = PayloadRef::Val(val); - stashed_val = None; - continue; - } - }, - None => {} - } - - requested_mask.clear_bit(byte); - match self.get(byte) { - Some(cf) => { - // An exact value-only request does not enumerate an onward link stored in the - // same CoFree. The preceding stashed-value fast path means this branch is - // reached only when that link was not requested separately. - if key.len() == 1 && *expect_val && cf.has_rec() { - unrequested_cofree_half = true; - } - - //A key longer than 1 byte or an explicit request for a rec link can be answered with a Child - if key.len() > 1 || !*expect_val { - match cf.rec() { - Some(rec) => { - *result_key_len = 1; - *payload_ref = PayloadRef::Child(rec); - }, - None => {} - } - } - match cf.val() { - Some(val) => { - //Answer an explicit request for this val, or stash the val for - if key.len() == 1 && *expect_val { - debug_assert!(stashed_val.is_none()); - *result_key_len = 1; - *payload_ref = PayloadRef::Val(val); - } else { - if last_byte.is_none() { - stashed_val = Some(val); - last_byte = Some(byte); - } - } - }, - None => {} - } - }, - None => {} - } - } - } - - !unrequested_cofree_half && stashed_val.is_none() && requested_mask.is_empty_mask() - } fn node_contains_val(&self, key: &[u8]) -> bool { if key.len() == 1 { match self.get(key[0]) { @@ -1308,7 +1329,7 @@ impl> TrieNode LINE_LIST_NODE_TAG => { let other_list_node = unsafe{ other.as_list_unchecked() }; let mut new_node = self.clone(); - let status = new_node.merge_from_list_node(other_list_node); + let status = new_node.merge_from_list_node(other_list_node, false); AlgebraicResult::from_status(status, || TrieNodeODRc::new_in(new_node, self.alloc.clone())) }, #[cfg(feature = "bridge_nodes")] @@ -1323,8 +1344,16 @@ impl> TrieNode self.pjoin(other_byte_node).map(|new_node| TrieNodeODRc::new_in(new_node, self.alloc.clone())) }, TINY_REF_NODE_TAG => { + //Expand the tiny node and keep `self` on the left. Delegating to the tiny node with + // the operands swapped returned its identity mask un-inverted -- a join that left + // `self` unchanged reported COUNTER_IDENT, and `join_into` then replaced the + // destination with the source -- and made the left-biased value join take the + // source's values let tiny_node = unsafe{ other.as_tiny_unchecked() }; - tiny_node.pjoin_dyn(self.as_tagged()) + match tiny_node.into_full() { + Some(full_node) => self.pjoin_dyn(full_node.as_tagged()), + None => AlgebraicResult::Identity(SELF_IDENT), + } } EMPTY_NODE_TAG => { AlgebraicResult::Identity(SELF_IDENT) @@ -1349,7 +1378,7 @@ impl> TrieNode let other_list_node = unsafe{ other_node.into_list_unchecked() }; //GOAT, optimization opportunity to take the contents from the list, rather than cloning // them, to turn around and drop the ListNode and free them / decrement the refcounts - self.merge_from_list_node(other_list_node) + self.merge_from_list_node(other_list_node, false) }, #[cfg(feature = "bridge_nodes")] TaggedNodeRefMut::BridgeNode(_other_bridge_node) => { @@ -1387,7 +1416,9 @@ impl> TrieNode }, _ => { let mut new_node = Self::new_in(self.alloc.clone()); - while let Some(cf) = self.values.pop() { + //Ascending key order, with the accumulated node as the left operand of every join, + // so a collision keeps the value from the lexicographically first path + for cf in self.values.drain(..) { let child = cf.into_rec().filter(|child| !child.is_empty()); let child = if byte_cnt > 1 { child.and_then(|mut child| child.make_mut().drop_head_dyn(byte_cnt-1)) @@ -1420,7 +1451,9 @@ impl> TrieNode }, LINE_LIST_NODE_TAG => { let other_list_node = unsafe { other.as_list_unchecked() }; - other_list_node.pmeet_dyn(self.as_tagged()).invert_identity() + //`self` is the left operand; the list node enumerates its payloads but must resolve + // every collision as `self op list`, hence `swapped`. + other_list_node.pmeet_dyn_oriented(self.as_tagged(), true).invert_identity() }, #[cfg(feature = "bridge_nodes")] TaggedNodeRef::BridgeNode(other_bridge_node) => { @@ -1432,7 +1465,8 @@ impl> TrieNode }, TINY_REF_NODE_TAG => { let tiny_node = unsafe { other.as_tiny_unchecked() }; - tiny_node.pmeet_dyn(self.as_tagged()).invert_identity() + let full_node = tiny_node.into_full().unwrap(); + self.pmeet_dyn(full_node.as_tagged()) }, EMPTY_NODE_TAG => AlgebraicResult::None, _ => unsafe{ unreachable_unchecked() } @@ -1575,14 +1609,30 @@ where } } - let src_range_mask = src_node.mask & ByteMask::from_range(range_start..=range_end); - let mut src_ix = src_node.mask.index_of(range_start) as usize; - for child_byte in src_range_mask.iter() { - let cf = unsafe { src_node.values.get_unchecked(src_ix) }; - src_ix += 1; - - if cf.has_rec() || cf.has_val() { - new_values.v.push(CfDst::from_cf(cf.clone())); + let range_mask = ByteMask::from_range(range_start..=range_end); + let src_range_mask = src_node.mask & range_mask; + let dst_range_mask = old_mask & range_mask; + for child_byte in (src_range_mask | dst_range_mask).iter() { + // A source branch counts only if it leads somewhere: a value, or a node with + // contents. A dangling branch in the source is nothing to graft, exactly like an + // absent one, so it must not create a location here. + let mut grafted = false; + if src_range_mask.test_bit(child_byte) { + let cf = unsafe { src_node.values.get_unchecked(src_node.mask.index_of(child_byte) as usize) }; + let rec_has_contents = cf.rec().map_or(false, |rec| !rec.as_tagged().node_is_empty()); + if rec_has_contents || cf.has_val() { + new_values.v.push(CfDst::from_cf(cf.clone())); + new_mask.set_bit(child_byte); + grafted = true; + } + } + if !grafted && !REMOVE_UNSET && dst_range_mask.test_bit(child_byte) { + // Grafting nothing over an existing branch removes its contents and its value, + // but the location itself survives as a dangling path -- the same thing + // `graft` of an empty source does at the focus, and what the model's + // `graftBelow` + `removeVal` specify. With `remove_unset` the branch was + // removed outright instead. + new_values.v.push(CfDst::new(Some(TrieNodeODRc::new_empty()), None)); new_mask.set_bit(child_byte); } } @@ -1875,18 +1925,56 @@ impl, Other rec_status.merge(val_status, true, true) } fn pmeet(&self, other: &OtherCf) -> AlgebraicResult { - //If one or the other cofree is dangling, it's an identity result for the dangling cofree - let mut identity_flag = 0; - if !self.has_rec() && !self.has_val() {identity_flag = SELF_IDENT;} - if !other.has_rec() && !other.has_val() {identity_flag |= COUNTER_IDENT;} - if identity_flag > 0 { - return AlgebraicResult::Identity(identity_flag) + //Both co-frees exist, so the location they stand at exists in both operands, and it survives + // the meet even when nothing at or below it does: the result is never `None`. An onward link + // to an empty node carries nothing, so it counts as no link at all. + let self_rec = self.rec().filter(|node| !node.as_tagged().node_is_empty()); + let other_rec = other.rec().filter(|node| !node.as_tagged().node_is_empty()); + let self_dangling = self_rec.is_none() && !self.has_val(); + let other_dangling = other_rec.is_none() && !other.has_val(); + if self_dangling || other_dangling { + //The meet is the bare location, which is exactly what a dangling side holds + let mut mask = 0; + if self_dangling { mask |= SELF_IDENT; } + if other_dangling { mask |= COUNTER_IDENT; } + return AlgebraicResult::Identity(mask) + } + + let rec = match (self_rec, other_rec) { + (Some(l), Some(r)) => l.pmeet(r), + _ => AlgebraicResult::None, + }; + let val = self.val().pmeet(&other.val()); + + //A part that meets to nothing equals the side that had nothing there + let (rec_self, rec_counter) = match &rec { + AlgebraicResult::Identity(mask) => (mask & SELF_IDENT > 0, mask & COUNTER_IDENT > 0), + AlgebraicResult::None => (self_rec.is_none(), other_rec.is_none()), + AlgebraicResult::Element(_) => (false, false), + }; + let (val_self, val_counter) = match &val { + AlgebraicResult::Identity(mask) => (mask & SELF_IDENT > 0, mask & COUNTER_IDENT > 0), + AlgebraicResult::None => (!self.has_val(), !other.has_val()), + AlgebraicResult::Element(_) => (false, false), + }; + let mut mask = 0; + if rec_self && val_self { mask |= SELF_IDENT; } + if rec_counter && val_counter { mask |= COUNTER_IDENT; } + if mask > 0 { + return AlgebraicResult::Identity(mask) } - //Otherwise actually work with what the cofrees contain - let rec = self.rec().pmeet(&other.rec()); - let val = self.val().pmeet(&other.val()); - self.combine_algebraic_results(other, rec, val) + let new_rec = match rec { + AlgebraicResult::Element(node) => Some(node), + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { self_rec.cloned() } else { other_rec.cloned() }, + AlgebraicResult::None => None, + }; + let new_val = match val { + AlgebraicResult::Element(val) => val, + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { self.val().cloned() } else { other.val().cloned() }, + AlgebraicResult::None => None, + }; + AlgebraicResult::Element(Self::new(new_rec, new_val)) } //GOAT, HeteroLattice will totally disappear when we do the policy refactor // fn join_all(_xs: &[&Self]) -> Self where Self: Sized { @@ -1902,6 +1990,16 @@ impl 0 { + return AlgebraicResult::Identity(SELF_IDENT) + } + } + } self.combine_algebraic_results(other, rec, val) } } @@ -2050,8 +2148,9 @@ impl, Other AlgebraicResult::None => { //Both nodes hold a dangling path (no value, no onward node) at this byte, e.g. after // `remove_branches(prune = false)`; it stays dangling in the join and is an identity for both - debug_assert!(!lv.has_rec() && !lv.has_val()); - debug_assert!(!rv.has_rec() && !rv.has_val()); + // An empty onward node counts as dangling too + debug_assert!(!lv.has_val() && lv.rec().map_or(true, |n| n.as_tagged().node_is_empty())); + debug_assert!(!rv.has_val() && rv.rec().map_or(true, |n| n.as_tagged().node_is_empty())); unsafe { new_v.get_unchecked_mut(c).write(Cf::new(None, None)) }; }, AlgebraicResult::Identity(mask) => { @@ -2350,12 +2449,66 @@ impl> ByteNode where Self: TrieNodeDowncast { + /// See [node_drop_dangling] + pub(crate) fn drop_dangling(&self, src: Option>) -> DropDangling { + let mut new_node: Option = None; + for (idx, byte) in self.mask.iter().enumerate() { + let cf = unsafe{ self.values.get_unchecked(idx) }; + let rec = match cf.rec() { + Some(child) => node_drop_dangling(child, meet_src_child(src, &[byte])), + None => DropDangling::Empty, + }; + let unchanged = match (&rec, cf.rec()) { + (DropDangling::Unchanged, _) => true, + (DropDangling::Empty, None) => cf.has_val(), + _ => false, + }; + if unchanged && new_node.is_none() { + continue + } + let new_node = new_node.get_or_insert_with(|| { + let mut node = Self::with_capacity_in(self.values.len(), self.alloc.clone()); + for (prev_idx, prev_byte) in self.mask.iter().enumerate().take(idx) { + node.set_cf(prev_byte, unsafe{ self.values.get_unchecked(prev_idx) }.rec().cloned(), unsafe{ self.values.get_unchecked(prev_idx) }.val().cloned()); + } + node + }); + let new_rec = match rec { + DropDangling::Unchanged => cf.rec().cloned(), + DropDangling::Empty => None, + DropDangling::New(node) => Some(node), + }; + if new_rec.is_some() || cf.has_val() { + new_node.set_cf(byte, new_rec, cf.val().cloned()); + } + } + match new_node { + None => DropDangling::Unchanged, + Some(node) if node.values.len() == 0 => DropDangling::Empty, + Some(node) => DropDangling::New(TrieNodeODRc::new_in(node, self.alloc.clone())), + } + } + fn set_cf(&mut self, byte: u8, rec: Option>, val: Option) { + if let Some(rec) = rec { + self.set_child(byte, rec); + } + if let Some(val) = val { + self.set_val(byte, val); + } + } +} + impl> ByteNode { fn prestrict>(&self, other: &ByteNode) -> AlgebraicResult where Self: Sized { // Iterate the overlap mask directly. Slot indexes are recovered with // prefix popcounts in each dense-mask word. let mut mm: ByteMask = self.mask & other.mask; - let mut is_identity = self.mask == mm && other.mask == mm; + //NOTE: restrict is non-commutative. The result is an identity of `self` when every one of + // `self`'s branches survives, i.e. `self.mask == mm`. Branches that exist only in `other` + // never contribute to the result, so requiring `other.mask == mm` as well only threw away + // identities that were there. + let mut is_identity = self.mask == mm; let mmc = [mm.0[0].count_ones(), mm.0[1].count_ones(), mm.0[2].count_ones(), mm.0[3].count_ones()]; diff --git a/src/dependent_zipper.rs b/src/dependent_zipper.rs index 5620e2a4..2df291f8 100644 --- a/src/dependent_zipper.rs +++ b/src/dependent_zipper.rs @@ -161,6 +161,10 @@ impl<'trie, PrimaryZ, SecondaryZ, V, C, F : Clone + for <'a> FnOnce(C, &'a [u8], /// a combination between `to_next_sibling` and `to_prev_sibling` fn to_sibling_byte(&mut self, next: bool) -> Option { + //`focus_byte` is unspecified at the root, and the root has no siblings + if self.depth() == 0 { + return None; + } let byte = self.focus_byte()?; let ascended = self.ascend(1); debug_assert_eq!(ascended, 1, "must ascend"); @@ -568,4 +572,20 @@ rubiconrubicon rubicundusrubicundus ") } + + /// Sibling steps at the root when the primary is rooted below the map root + #[test] + fn dependent_product_zipper_sibling_step_at_root() { + let mut a = PathMap::::new(); + for p in [&[1u8, 0, 2, 7][..], &[1, 0, 2, 8], &[9]] { a.set_val_at(p, 1); } + let mut z = DependentProductZipperG::new_enroll(a.read_zipper_at_path(&[1u8, 0, 2]), (), |_, _, _| ((), None::>)); + assert_eq!(z.to_next_sibling_byte(), None); + assert_eq!(z.to_prev_sibling_byte(), None); + assert_eq!(z.descend_first_byte(), Some(7)); + assert_eq!(z.to_next_sibling_byte(), Some(8)); + z.reset(); + let mut n = 0; + while z.to_next_val() { n += 1; assert!(n < 100); } + assert_eq!(n, 2); + } } diff --git a/src/empty_node.rs b/src/empty_node.rs index ce336dd1..0cf79d06 100644 --- a/src/empty_node.rs +++ b/src/empty_node.rs @@ -26,9 +26,6 @@ impl TrieNode for EmptyNode { fn node_replace_child(&mut self, _key: &[u8], _new_node: TrieNodeODRc) { unreachable!() //Should not be called unless it's known that the node being replaced exists } - fn node_get_payloads<'node, 'res>(&'node self, _keys: &[(&[u8], bool)], _results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - true - } fn node_contains_val(&self, _key: &[u8]) -> bool { false } diff --git a/src/lib.rs b/src/lib.rs index 6d38dc03..25d6c2f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -593,8 +593,11 @@ mod tests { assert_eq!(met, l); } + //`l` and `r` hold no value in common, but they share the first bytes of their paths, and + // those locations survive the meet as dangling paths let met = met.meet(&r); - assert!(met.is_empty()); + assert_eq!(met.val_count(), 0); + assert!(!met.is_empty()); } #[test] diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 6013ef25..3b309611 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1074,7 +1074,11 @@ impl LineListNode { /// Ensures that a node is valid by combining an illegal shared prefix between the keys if there is one /// This is currently used by drop_head, because dropping a disjoint prefix may cause downstream paths /// to collide, and thus require merging - fn factor_prefix(&mut self) where V: Clone + Lattice { + /// + /// `slot1_first` makes slot 1 the left operand of the merge, so a colliding value is taken from + /// slot 1. `drop_head_dyn` needs that when shortening the keys reversed their order: slot 1 then + /// holds the payload from the lexicographically first k-path, which must win the collision. + fn factor_prefix(&mut self, slot1_first: bool) where V: Clone + Lattice { let (key0, key1) = self.get_both_keys(); let overlap = find_prefix_overlap(key0, key1); //Overlap of 1 is legal if and only if ONE OF the following two conditions are true: @@ -1086,16 +1090,27 @@ impl LineListNode { //If the overlap is illegal, split the prefix if overlap > 0 && !legal_overlap { - match merge_guts::(overlap, key0, self, key1, self) { + let merged = if slot1_first { + merge_guts::(overlap, key1, self, key0, self) + } else { + merge_guts::(overlap, key0, self, key1, self) + }; + match merged { AlgebraicResult::Element((shared_key, merged_payload)) => { let mut new_node = Self::new_in(self.alloc.clone()); unsafe{ new_node.set_payload_owned::<0>(shared_key, merged_payload) }; *self = new_node; }, AlgebraicResult::Identity(mask) => { - debug_assert!(mask & SELF_IDENT > 0); + //SELF_IDENT names the left operand's slot, COUNTER_IDENT the right one's + let left_slot_wins = mask & SELF_IDENT > 0; + debug_assert!(left_slot_wins || mask & COUNTER_IDENT > 0); let mut new_node = Self::new_in(self.alloc.clone()); - unsafe{ new_node.set_payload_owned::<0>(key0, self.clone_payload::<0>().unwrap()) }; + if left_slot_wins != slot1_first { + unsafe{ new_node.set_payload_owned::<0>(key0, self.clone_payload::<0>().unwrap()) }; + } else { + unsafe{ new_node.set_payload_owned::<0>(key1, self.clone_payload::<1>().unwrap()) }; + } *self = new_node; }, AlgebraicResult::None => {} @@ -1192,6 +1207,31 @@ impl LineListNode { AlgebraicResult::Identity(SELF_IDENT) } } + /// `true` when the slot holds an *empty* onward link under the same key as the node's other + /// slot. + /// + /// Two slots may share a key: that is how a value and the onward child at the same path are + /// stored (see [validate_node]). When the onward child of such a pair is empty, the slot + /// contributes nothing at all -- there is nothing below an empty link, and the existence of + /// the path itself is already carried by the sibling standing at the same key. So the node + /// holds exactly the same trie with the slot as without it. + /// + /// Pairs like this are ordinary: `remove_subtries` leaves an empty link behind whenever it + /// clears a subtrie without pruning, and `graft_child_maps` with `remove_unmasked` does the + /// same. + fn slot_is_shadowed_dangling(&self, slot: usize) -> bool { + let is_used_child = if slot == 0 { self.is_used_child_0() } else { self.is_used_child_1() }; + if !is_used_child || !self.is_used::<1>() { + return false + } + let (key0, key1) = self.get_both_keys(); + if key0 != key1 { + return false + } + let child = unsafe{ if slot == 0 { self.child_in_slot::<0>() } else { self.child_in_slot::<1>() } }; + child.as_tagged().node_is_empty() + } + /// Internal method to restrict the contents of `SLOT` with the contents of the `other` node fn restrict_slot_contents(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Clone { if self.is_used::() { @@ -1304,6 +1344,35 @@ fn try_merge<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: us } /// The part of `try_merge` that we probably shouldn't inline +/// The subtrie left of one list-node slot after `byte_cnt` bytes are dropped from every path +/// below it, or `None` if nothing is left. Part of [TrieNode::drop_head_dyn]. +/// +/// A value sitting at depth `<= byte_cnt` is discarded (the joined node has nowhere to put a root +/// value), a dangling child has nothing below the dropped bytes, and a key longer than `byte_cnt` +/// just loses its first `byte_cnt` bytes. +fn drop_head_from_payload(key: &[u8], payload: ValOrChild, byte_cnt: usize, alloc: &A) -> Option> { + let key_len = key.len(); + if byte_cnt < key_len { + let mut new_node = LineListNode::new_in(alloc.clone()); + unsafe { new_node.set_payload_owned::<0>(&key[byte_cnt..], payload); } + debug_assert!(validate_node(&new_node)); + return Some(TrieNodeODRc::new_in(new_node, alloc.clone())) + } + match payload { + ValOrChild::Val(_) => None, + ValOrChild::Child(mut child) => { + if child.is_empty() { + return None + } + if byte_cnt == key_len { + Some(child) + } else { + child.make_mut().drop_head_dyn(byte_cnt - key_len) + } + } + } +} + fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: usize, const BSLOT: usize>(mut overlap: usize, a_key: &'a[u8], a: &LineListNode, b_key: &'a[u8], b: &LineListNode) -> AlgebraicResult<(&'a[u8], ValOrChild)> { debug_assert!(overlap > 0); let a_key_len = a_key.len(); @@ -1325,7 +1394,11 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u (true, true) => { //both are child nodes, so join them let a_child = unsafe{ a.child_in_slot::() }; let b_child = unsafe{ b.child_in_slot::() }; - return a_child.pjoin(b_child).map(|new_child| (a_key, ValOrChild::Child(new_child))) + return match a_child.pjoin(b_child) { + //Two empty children are both just the dangling path + AlgebraicResult::None => AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT), + joined => joined.map(|new_child| (a_key, ValOrChild::Child(new_child))), + } }, (false, false) => { //both are values, so join them let a_val = unsafe{ a.val_in_slot::() }; @@ -1345,14 +1418,21 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u unsafe{ intermediate_node.set_payload_owned::<0>(&a_key[overlap..], a_payload); } debug_assert!(validate_node(&intermediate_node)); let intermediate_node = TrieNodeODRc::new_in(intermediate_node, a.alloc.clone()); - let joined = b_child.pjoin(&intermediate_node).unwrap_or_else(|which_arg| { - match which_arg { - 0 => b_child.clone(), - 1 => intermediate_node, - _ => unreachable!() - } - }, || panic!()); - return AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))) + //`a` is the left operand of this merge, so its payload must be the left operand + // of the join (the join is left-biased on colliding values). + return match intermediate_node.pjoin(b_child) { + AlgebraicResult::Element(joined) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))), + //`b`'s child already held `a`'s payload, so `b`'s slot *is* the result -- COUNTER_IDENT + // from the caller's point of view. Reporting it as `Element` made a join whose + // destination was unchanged report `Element`. (The other identity, where the child + // held nothing beyond `a`'s payload, is still built as an `Element`: `a`'s slot has + // the same contents but not the `(prefix, child)` shape callers such as + // `drop_head` rely on.) The bit read here is COUNTER_IDENT rather than SELF_IDENT + // because `b_child` is now the right operand. + AlgebraicResult::Identity(mask) if mask & COUNTER_IDENT > 0 => AlgebraicResult::Identity(COUNTER_IDENT), + AlgebraicResult::Identity(_) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(intermediate_node))), + AlgebraicResult::None => unreachable!(), //`intermediate_node` is never empty + } } if a_key_len == overlap && a.is_child_ptr::() && b_key_len > overlap { let a_child = unsafe{ a.child_in_slot::() }; @@ -1361,14 +1441,13 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u unsafe{ intermediate_node.set_payload_owned::<0>(&b_key[overlap..], b_payload); } debug_assert!(validate_node(&intermediate_node)); let intermediate_node = TrieNodeODRc::new_in(intermediate_node, a.alloc.clone()); - let joined = a_child.pjoin(&intermediate_node).unwrap_or_else(|which_arg| { - match which_arg { - 0 => a_child.clone(), - 1 => intermediate_node, - _ => unreachable!() - } - }, || panic!()); - return AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))) + return match a_child.pjoin(&intermediate_node) { + AlgebraicResult::Element(joined) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))), + //Mirror of the case above: `a`'s slot is the result + AlgebraicResult::Identity(mask) if mask & SELF_IDENT > 0 => AlgebraicResult::Identity(SELF_IDENT), + AlgebraicResult::Identity(_) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(intermediate_node))), + AlgebraicResult::None => unreachable!(), //`intermediate_node` is never empty + } } //If we have overlapping initial bytes that can be joined together, make a new prefix node @@ -1744,52 +1823,6 @@ impl TrieNode for LineListNode debug_assert!(consumed_bytes == key.len()); *child_node = new_node; } - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - let mut slot_0_requested = !self.is_used::<0>(); - let mut slot_1_requested = !self.is_used::<1>(); - let (node_key_0, node_key_1) = self.get_both_keys(); - - debug_assert!(results.len() >= keys.len()); - for ((key, expect_val), (result_key_len, payload_ref)) in keys.into_iter().zip(results.iter_mut()) { - if self.is_used::<0>() { - if starts_with(key, node_key_0) { - let node_key_len = node_key_0.len(); - if self.is_child_ptr::<0>() { - if !*expect_val || node_key_len < key.len() { - slot_0_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Child(unsafe{ &*self.val_or_child0.child }); - } - } else { - if *expect_val && node_key_len == key.len() { - slot_0_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Val(unsafe{ &**self.val_or_child0.val }); - } - } - } - } - if self.is_used::<1>() { - if starts_with(key, node_key_1) { - let node_key_len = node_key_1.len(); - if self.is_child_ptr::<1>() { - if !*expect_val || node_key_len < key.len() { - slot_1_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Child(unsafe{ &*self.val_or_child1.child }); - } - } else { - if *expect_val && node_key_len == key.len() { - slot_1_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Val(unsafe{ &**self.val_or_child1.val }); - } - } - } - } - } - slot_0_requested && slot_1_requested - } fn node_contains_val(&self, key: &[u8]) -> bool { self.contains_val(key) } @@ -2482,21 +2515,25 @@ impl TrieNode for LineListNode && *byte < key[last_key_byte_idx] }) }; - let (sibling_byte, slot) = match key_byte(key1) { - Some(byte) => (byte, 1), - None => match key_byte(key0) { - Some(byte) => (byte, 0), - None => return (None, None), - }, + let sibling_byte = match key_byte(key1).or_else(|| key_byte(key0)) { + Some(byte) => byte, + None => return (None, None), }; - let sib_node = match slot { - 0 if key0.len() == key.len() && self.is_child_ptr::<0>() => { - Some(unsafe { self.child_in_slot::<0>().as_tagged() }) - }, - 1 if key1.len() == key.len() && self.is_child_ptr::<1>() => { - Some(unsafe { self.child_in_slot::<1>().as_tagged() }) - }, - _ => None, + //The sibling's onward child may sit in either slot: a location holding both a value + // and a child is stored as two slots with the same key, so the slot whose byte was + // found first may be the value one. Look for the child in both. + let holds_child = |candidate: &[u8], slot: usize| { + candidate.len() == key.len() + && candidate[last_key_byte_idx] == sibling_byte + && candidate[..last_key_byte_idx] == common_key[..] + && if slot == 0 { self.is_child_ptr::<0>() } else { self.is_child_ptr::<1>() } + }; + let sib_node = if holds_child(key1, 1) { + Some(unsafe { self.child_in_slot::<1>().as_tagged() }) + } else if holds_child(key0, 0) { + Some(unsafe { self.child_in_slot::<0>().as_tagged() }) + } else { + None }; (Some(sibling_byte), sib_node) } @@ -2621,7 +2658,7 @@ impl TrieNode for LineListNode DENSE_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other.as_dense_unchecked() }; let mut new_node = other_dense_node.clone(); - match new_node.merge_from_list_node(self) { + match new_node.merge_from_list_node(self, true) { //Both nodes were empty so the join is empty too AlgebraicStatus::None => { debug_assert!(self.node_is_empty() && other_dense_node.node_is_empty()); @@ -2636,9 +2673,9 @@ impl TrieNode for LineListNode unimplemented!() }, CELL_BYTE_NODE_TAG => { - let other_dense_node = unsafe{ other.as_dense_unchecked() }; + let other_dense_node = unsafe{ other.as_cell_unchecked() }; let mut new_node = other_dense_node.clone(); - match new_node.merge_from_list_node(self) { + match new_node.merge_from_list_node(self, true) { //See the DENSE_BYTE_NODE_TAG arm: two empty nodes join to an empty result AlgebraicStatus::None => { debug_assert!(self.node_is_empty() && other_dense_node.node_is_empty()); @@ -2649,8 +2686,12 @@ impl TrieNode for LineListNode } }, TINY_REF_NODE_TAG => { + //Expand the tiny node and keep `self` on the left (see DenseByteNode::pjoin_dyn) let tiny_node = unsafe{ other.as_tiny_unchecked() }; - tiny_node.pjoin_dyn(self.as_tagged()) + match tiny_node.into_full() { + Some(full_node) => self.pjoin_dyn(full_node.as_tagged()), + None => AlgebraicResult::Identity(SELF_IDENT), + } } EMPTY_NODE_TAG => { AlgebraicResult::Identity(SELF_IDENT) @@ -2670,7 +2711,7 @@ impl TrieNode for LineListNode DENSE_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other_node.as_dense_unchecked() }; let mut new_node = other_dense_node.clone(); - let status = new_node.merge_from_list_node(self); + let status = new_node.merge_from_list_node(self, true); debug_assert!(!status.is_none()); (AlgebraicStatus::Element, Err(TrieNodeODRc::new_in(new_node, self.alloc.clone()))) }, @@ -2681,7 +2722,7 @@ impl TrieNode for LineListNode CELL_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other_node.as_cell_unchecked() }; let mut new_node = other_dense_node.clone(); - let status = new_node.merge_from_list_node(self); + let status = new_node.merge_from_list_node(self, true); debug_assert!(!status.is_none()); (AlgebraicStatus::Element, Err(TrieNodeODRc::new_in(new_node, self.alloc.clone()))) }, @@ -2802,7 +2843,8 @@ impl TrieNode for LineListNode let mut new_key0_len = key0_len-byte_cnt; let mut new_key1_len = key1_len-byte_cnt; //Make sure the new keys are in the correctly sorted order - if &key0[byte_cnt..] <= &key1[byte_cnt..] { + let reordered = &key0[byte_cnt..] > &key1[byte_cnt..]; + if !reordered { unsafe { //Shorten key0 let base_ptr = temp_node.key_bytes.as_mut_ptr().cast::(); @@ -2836,86 +2878,69 @@ impl TrieNode for LineListNode core::mem::swap(&mut temp_node.val_or_child0, &mut temp_node.val_or_child1); } temp_node.header = Self::header0(slot0_child, new_key0_len) | Self::header1(slot1_child, new_key1_len); - temp_node.factor_prefix(); + //If the keys were reordered, slot 1 now holds the original slot 0: the lexicographically + // first k-path, whose value must win any collision the prefix merge finds + temp_node.factor_prefix(reordered); debug_assert!(validate_node(&temp_node)); return Some(TrieNodeODRc::new_in(temp_node, self.alloc.clone())) } - //The final case is to construct a brand new node from the remaining parts of the key after we have - // discarded what we can discard and then merged together what's left. And then call this function - // recursively on the newly merged nodes - let chop_bytes = key0_len.min(key1_len); - debug_assert!(chop_bytes <= byte_cnt); - debug_assert!(chop_bytes > 0); - let new_key0 = &key0[chop_bytes-1..]; - let new_key1 = &key1[chop_bytes-1..]; - - let overlap = find_prefix_overlap(&key0[chop_bytes..], &key1[chop_bytes..]); - let merged_payload = match merge_guts::(overlap+1, new_key0, &temp_node, new_key1, &temp_node) { - AlgebraicResult::Element((_shared_key, merged_payload)) => merged_payload, - AlgebraicResult::Identity(mask) => { - if mask & SELF_IDENT > 0 { - temp_node.clone_payload::<0>().unwrap() - } else { - debug_assert_eq!(mask, COUNTER_IDENT); - temp_node.clone_payload::<1>().unwrap() - } - }, - AlgebraicResult::None => unreachable!() //`merge_guts` shouldn't return AlgebraicResult::None because that should have been caught by an earlier case + //The final case: at least one key is no longer than `byte_cnt`. Drop the bytes from each + // slot on its own and join the two results, slot 0 (the lexicographically smaller key) on + // the left. That is the order `PathMap::drop_head` is specified in -- a fold over the + // k-paths in sorted order, keeping the first value on a collision -- and it is what the + // byte node does one level up. Merging the two slots at an intermediate depth and then + // dropping the remaining bytes, as this used to, joins the subtries in a different order + // and keeps different values. + let mut key0_buf: [MaybeUninit; KEY_BYTES_CNT] = [MaybeUninit::new(0); KEY_BYTES_CNT]; + let mut key1_buf: [MaybeUninit; KEY_BYTES_CNT] = [MaybeUninit::new(0); KEY_BYTES_CNT]; + let (key0, key1) = unsafe { + core::ptr::copy_nonoverlapping(key0.as_ptr(), key0_buf.as_mut_ptr().cast::(), key0_len); + core::ptr::copy_nonoverlapping(key1.as_ptr(), key1_buf.as_mut_ptr().cast::(), key1_len); + (core::slice::from_raw_parts(key0_buf.as_ptr().cast::(), key0_len), + core::slice::from_raw_parts(key1_buf.as_ptr().cast::(), key1_len)) }; - - if let ValOrChild::Child(mut child_node) = merged_payload { - //A dangling child (the empty sentinel) has nothing below the dropped bytes and can't be made mutable - if child_node.is_empty() { - return None - } - if chop_bytes == byte_cnt { - return Some(child_node) - } else { - return child_node.make_mut().drop_head_dyn(byte_cnt-chop_bytes) + //Take slot 1 first: taking slot 0 would shift slot 1 into its place. + let payload1 = temp_node.take_payload::<1>().unwrap(); + let payload0 = temp_node.take_payload::<0>().unwrap(); + let dropped0 = drop_head_from_payload(key0, payload0, byte_cnt, &self.alloc); + let dropped1 = drop_head_from_payload(key1, payload1, byte_cnt, &self.alloc); + match (dropped0, dropped1) { + (None, None) => None, + (Some(node), None) | (None, Some(node)) => Some(node), + (Some(node0), Some(node1)) => match node0.pjoin(&node1) { + AlgebraicResult::Element(joined) => Some(joined), + AlgebraicResult::Identity(mask) => Some(if mask & SELF_IDENT > 0 { node0 } else { node1 }), + AlgebraicResult::None => None, } } - - unreachable!() } fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { - debug_assert!(validate_node(self)); - - let mut self_payloads_buf: [(&[u8], PayloadRef); 2] = [(&[], PayloadRef::None); 2]; - - let self_slot_count = self.used_slot_count(); - let self_payloads = match self_slot_count { - 0 => return AlgebraicResult::None, - 1 => { - let key = unsafe{ self.key_unchecked::<0>() }; - let payload = unsafe{ self.payload_in_slot::<0>() }; - self_payloads_buf[0] = (key, payload); - &self_payloads_buf[..1] - }, - 2 => { - let (key0, key1) = self.get_both_keys(); - let payload0 = unsafe{ self.payload_in_slot::<0>() }; - let payload1 = unsafe{ self.payload_in_slot::<1>() }; - self_payloads_buf[0] = (key0, payload0); - self_payloads_buf[1] = (key1, payload1); - &self_payloads_buf[..2] - }, - _ => unsafe{ unreachable_unchecked() } - }; - - pmeet_generic::<2, V, A, _>(self_payloads, other, |payloads| { - debug_assert_eq!(payloads.len(), self_payloads.len()); - let slot0_payload = payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()); - let slot1_payload = payloads.get_mut(1).and_then(|p| core::mem::take(p)).map(|p| p.into()); - let new_node = self.clone_with_updated_payloads(slot0_payload, slot1_payload).unwrap(); - TrieNodeODRc::new_in(new_node, self.alloc.clone()) - }) + self.pmeet_dyn_oriented(other, false) } fn psubtract_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: DistributiveLattice { debug_assert!(validate_node(self)); let slot0_result = self.subtract_from_slot_contents::<0>(other); let slot1_result = self.subtract_from_slot_contents::<1>(other); + + //A dangling path the source reaches does not survive a subtraction, so + // `subtract_from_slot_contents` answers `None` for a slot holding an empty onward link, + // and dropping it is right. But when that slot is *shadowed* -- an empty link sharing its + // key with the sibling slot -- it was carrying nothing to begin with, so dropping it is + // not a change, and the node that is left holds exactly the trie it held before. Letting + // the `None` through rebuilt the node around the surviving slot and reported `Element` for + // a destination that `subtract_into` had not touched. + match (&slot0_result, &slot1_result) { + (AlgebraicResult::None, AlgebraicResult::Identity(_)) if self.slot_is_shadowed_dangling(0) => { + return AlgebraicResult::Identity(SELF_IDENT) + }, + (AlgebraicResult::Identity(_), AlgebraicResult::None) if self.slot_is_shadowed_dangling(1) => { + return AlgebraicResult::Identity(SELF_IDENT) + }, + _ => {} + } + self.combine_slot_results_into_node_result(slot0_result, slot1_result) } fn prestrict_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> { @@ -2930,6 +2955,152 @@ impl TrieNode for LineListNode } impl LineListNode { + /// The body of [TrieNode::pmeet_dyn]. `swapped` means `self` is really the *right* operand of + /// the meet and `other` the left one. A node type that cannot enumerate its own payloads cheaply + /// (a `ByteNode`) meets a list node by calling this with `swapped = true` and inverting the + /// identity mask of the result. + /// + /// The meet is left-biased (a `Lattice` impl resolves a collision as `left.pmeet(right)`), so + /// with `swapped` every value and every recursive node meet is computed as `other op self` and + /// its identity mask re-expressed relative to `self`. + /// + /// A location survives the meet exactly when both operands have it, dangling or not. So each + /// slot contributes the deepest prefix of its key that `other` also has, even when nothing at or + /// below that prefix survives. + /// + /// `SELF_IDENT` in the result is exact. `COUNTER_IDENT` is only ever claimed with `swapped`, + /// where the caller inverts it into its own `SELF_IDENT`, and there it is exact too. + pub(crate) fn pmeet_dyn_oriented(&self, other: TaggedNodeRef, swapped: bool) -> AlgebraicResult> where V: Lattice { + debug_assert!(validate_node(self)); + + //A shadowed dangling slot carries nothing (see `slot_is_shadowed_dangling`): the location it + // names is already carried by its sibling. Leaving it out of the meet gives the same trie. + let (use0, use1) = match self.used_slot_count() { + 0 => return AlgebraicResult::None, + 1 => (true, false), + _ => { + if self.slot_is_shadowed_dangling(0) { + (false, true) + } else if self.slot_is_shadowed_dangling(1) { + (true, false) + } else { + (true, true) + } + } + }; + let (key0, key1) = self.get_both_keys(); + + let out0 = if use0 { + meet_list_slot(key0, unsafe{ self.payload_in_slot::<0>() }, other, swapped) + } else { + (true, true, SlotMeet::Skipped) + }; + let out1 = if use1 { + meet_list_slot(key1, unsafe{ self.payload_in_slot::<1>() }, other, swapped) + } else { + (true, true, SlotMeet::Skipped) + }; + if out0.2.is_nothing() && out1.2.is_nothing() { + return AlgebraicResult::None + } + + let mut mask = 0; + if out0.0 && out1.0 { + mask |= SELF_IDENT; + } + if swapped && out0.1 && out1.1 { + let slots = [ + if use0 { Some((key0, self.is_child_ptr::<0>(), out0.2.reach(key0.len()))) } else { None }, + if use1 { Some((key1, self.is_child_ptr::<1>(), out1.2.reach(key1.len()))) } else { None }, + ]; + if meet_other_within_slots(other, &slots) { + mask |= COUNTER_IDENT; + } + } + if mask > 0 { + return AlgebraicResult::Identity(mask) + } + + //Build the result from what each slot contributes + let mut items: [Option<(&[u8], ValOrChild)>; 2] = [ + out0.2.into_item(key0, || self.clone_payload::<0>().unwrap()), + out1.2.into_item(key1, || self.clone_payload::<1>().unwrap()), + ]; + //A dangling path is redundant where the other item's path runs through it, or stands at it + // with a value or an onward node. Dropping it keeps the node valid (no onward link in slot_0 + // that slot_1 extends, no two onward links under one key). + for i in 0..2 { + let j = 1 - i; + let redundant = match (&items[i], &items[j]) { + (Some((key_i, ValOrChild::Child(child_i))), Some((key_j, payload_j))) if child_i.as_tagged().node_is_empty() && key_j.starts_with(key_i) => { + let j_dangling = matches!(payload_j, ValOrChild::Child(child_j) if child_j.as_tagged().node_is_empty()); + key_j.len() > key_i.len() || !j_dangling || i > j + }, + _ => false + }; + if redundant { + items[i] = None; + } + } + let mut new_node = Self::new_in(self.alloc.clone()); + let [item0, item1] = items; + match (item0, item1) { + (Some((key0, payload0)), Some((key1, payload1))) => { + unsafe{ new_node.set_payload_owned::<0>(key0, payload0); } + unsafe{ new_node.set_payload_owned::<1>(key1, payload1); } + }, + (Some((key, payload)), None) | (None, Some((key, payload))) => { + unsafe{ new_node.set_payload_owned::<0>(key, payload); } + }, + (None, None) => unreachable!() + } + debug_assert!(validate_node(&new_node)); + AlgebraicResult::Element(TrieNodeODRc::new_in(new_node, self.alloc.clone())) + } + + /// See [node_drop_dangling] + pub(crate) fn drop_dangling(&self, src: Option>) -> DropDangling { + let (key0, key1) = self.get_both_keys(); + let slot_result = |key: &[u8], is_child: bool, child: fn(&Self) -> &TrieNodeODRc| { + if is_child { + node_drop_dangling(child(self), meet_src_child(src, key)) + } else { + DropDangling::Unchanged + } + }; + let result0 = if self.is_used::<0>() { + slot_result(key0, self.is_child_ptr::<0>(), |node| unsafe{ node.child_in_slot::<0>() }) + } else { + DropDangling::Empty + }; + let result1 = if self.is_used::<1>() { + slot_result(key1, self.is_child_ptr::<1>(), |node| unsafe{ node.child_in_slot::<1>() }) + } else { + DropDangling::Empty + }; + let is_unchanged = |result: &DropDangling, used: bool| matches!(result, DropDangling::Unchanged) || (!used && matches!(result, DropDangling::Empty)); + if is_unchanged(&result0, self.is_used::<0>()) && is_unchanged(&result1, self.is_used::<1>()) { + return DropDangling::Unchanged + } + let payload = |result: DropDangling, slot: usize| match result { + DropDangling::Unchanged => if slot == 0 { self.clone_payload::<0>() } else { self.clone_payload::<1>() }, + DropDangling::Empty => None, + DropDangling::New(node) => Some(ValOrChild::Child(node)), + }; + let mut new_node = Self::new_in(self.alloc.clone()); + match (payload(result0, 0), payload(result1, 1)) { + (Some(payload0), Some(payload1)) => { + unsafe{ new_node.set_payload_owned::<0>(key0, payload0); } + unsafe{ new_node.set_payload_owned::<1>(key1, payload1); } + }, + (Some(payload), None) => unsafe{ new_node.set_payload_owned::<0>(key0, payload); }, + (None, Some(payload)) => unsafe{ new_node.set_payload_owned::<0>(key1, payload); }, + (None, None) => return DropDangling::Empty, + } + debug_assert!(validate_node(&new_node)); + DropDangling::New(TrieNodeODRc::new_in(new_node, self.alloc.clone())) + } + /// Part of the implementation of methods the remove subtries from a node fn remove_subtries(&mut self, remove_0: bool, remove_1: bool, key0_starts_with: bool, prune: bool, key_len: usize) { //NOTE: the order here is important because removing slot_0 first might shift the @@ -3932,4 +4103,46 @@ mod tests { m.write_zipper().restrict(&o.read_zipper()); assert_eq!(keys(&m), ["bx"]); } + + /// Joining a list node with a `CellByteNode` (left behind by a `ZipperHead`) + #[test] + fn list_node_join_with_cell_node() { + use crate::PathMap; + use crate::zipper::*; + let mut cell = PathMap::::new(); + { + let zh = cell.zipper_head(); + for b in [1u8, 2] { + let mut wz = zh.write_zipper_at_exclusive_path(&[b]).unwrap(); + wz.set_val(b as u64); + } + } + assert!(cell.root().unwrap().as_tagged().tag() == CELL_BYTE_NODE_TAG, "the layout this test needs"); + let mut list = PathMap::::new(); + list.set_val_at(&[0u8, 5], 7); + assert!(list.root().unwrap().as_tagged().tag() == LINE_LIST_NODE_TAG, "the layout this test needs"); + + let joined = list.join(&cell); + let vals: Vec<(Vec, u64)> = joined.iter().map(|(k, v)| (k, *v)).collect(); + assert_eq!(vals, vec![(vec![0, 5], 7), (vec![1], 1), (vec![2], 2)]); + let mut into = list.clone(); + into.write_zipper().join_map_into(cell.clone()); + assert_eq!(into.iter().map(|(k, v)| (k, *v)).collect::>(), vals); + } + + /// Joining two nodes whose children under the same key are both empty + #[test] + fn merge_empty_children_under_same_key() { + let node = || { + let mut n = LineListNode::::new_in(global_alloc()); + let child = LineListNode::::new_in(global_alloc()); + unsafe { n.set_child_0(&[0, 3], TrieNodeODRc::new_in(child, global_alloc())); } + n + }; + let (a, b) = (node(), node()); + match merge_list_nodes(&a, &b) { + Ok(AlgebraicResult::Identity(mask)) => assert_eq!(mask, SELF_IDENT | COUNTER_IDENT), + _ => panic!("expected an identity"), + } + } } diff --git a/src/merkleization.rs b/src/merkleization.rs index 6840abb3..99382831 100644 --- a/src/merkleization.rs +++ b/src/merkleization.rs @@ -64,7 +64,8 @@ pub(crate) fn merkleize_impl( it = next; path.hash(&mut hasher); let (child_hash, replace); - if let Some(child) = child { + //An empty child (a dangling path) has nothing to share, and can't be replaced in its parent + if let Some(child) = child.filter(|child| !child.is_empty()) { (child_hash, replace) = merkleize_impl(counters, memo, child, val); if let Some(replace) = replace { let node = replacement.get_or_insert_with(|| { @@ -136,4 +137,18 @@ mod tests { eprintln!("```mermaid\n{}```", std::str::from_utf8(&after).unwrap()); } } + + /// Several dangling paths, which are links to the empty node + #[test] + fn merkleize_with_dangling_paths() { + use crate::zipper::*; + let mut map = crate::PathMap::::new(); + for p in [&[1u8, 1, 5][..], &[2, 2, 5], &[3, 3, 5], &[4]] { map.set_val_at(p, 1); } + for p in [&[1u8, 1, 7][..], &[2, 2, 7], &[3, 3, 7], &[1, 9], &[2, 9]] { map.create_path(p); } + let before: Vec> = { let mut z = map.read_zipper(); let mut v = vec![]; while z.to_next_step() { v.push(z.path().to_vec()) } v }; + let _ = map.merkleize(); + let after: Vec> = { let mut z = map.read_zipper(); let mut v = vec![]; while z.to_next_step() { v.push(z.path().to_vec()) } v }; + assert_eq!(before, after); + assert_eq!(map.val_count(), 4); + } } diff --git a/src/overlay_zipper.rs b/src/overlay_zipper.rs index 211c8aa3..d9f1ae94 100644 --- a/src/overlay_zipper.rs +++ b/src/overlay_zipper.rs @@ -159,6 +159,10 @@ impl ZipperMoving #[inline] fn focus_byte(&self) -> Option { + //The sources may be rooted at different paths, so at the root their bytes differ + if self.depth() == 0 { + return None; + } let byte = self.a.focus_byte(); debug_assert_eq!(byte, self.b.focus_byte()); byte @@ -211,7 +215,7 @@ impl ZipperMoving self.a.ascend(depth_a - depth_o); depth_o } else { - self.a.descend_to(&path[depth_o..depth_a]); + self.b.descend_to(&path[depth_o..depth_a]); depth_a } } else { @@ -577,4 +581,38 @@ mod tests { assert_eq!(moved, true); assert_eq!(observed, oz.path(), "observer must match the resulting path"); } + + /// Sources rooted at different paths: no focus byte, and no sibling step, at the root + #[test] + fn overlay_sources_at_different_roots() { + use crate::zipper::ZipperIteration; + let mut a = PathMap::::new(); + for p in [&[1u8, 5][..], &[1, 6], &[2, 5, 1]] { a.set_val_at(p, 1); } + let mut z = OverlayZipper::new(a.read_zipper_at_path(&[1u8]), a.read_zipper_at_path(&[2u8])); + assert_eq!(z.focus_byte(), None); + assert_eq!(z.to_next_sibling_byte(), None); + assert_eq!(z.to_prev_sibling_byte(), None); + let mut steps = vec![]; + while z.to_next_step() { steps.push(z.path().to_vec()); assert!(steps.len() < 16); } + assert_eq!(steps, vec![vec![5], vec![5, 1], vec![6]]); + let mut o = Vec::new(); + while z.to_next_val_observed(&mut o) { assert_eq!(&o[..], z.path()); } + } + + /// `descend_to_val` keeps both sources at the same place when the second one stops first + #[test] + fn overlay_descend_to_val_second_stops_first() { + let mut a = PathMap::::new(); + a.set_val_at(&[1u8, 2, 3], 1); + let mut b = PathMap::::new(); + b.set_val_at(&[1u8, 7], 2); + for (x, y) in [(&a, &b), (&b, &a)] { + let mut z = OverlayZipper::new(x.read_zipper(), y.read_zipper()); + assert_eq!(z.descend_to_val(&[1u8, 2, 3, 4, 5]), 3); + assert_eq!(z.depth(), 3); + assert_eq!(z.path(), &[1u8, 2, 3]); + assert_eq!(z.ascend(3), 3); + assert_eq!(z.depth(), 0); + } + } } diff --git a/src/prefix_zipper.rs b/src/prefix_zipper.rs index 14823612..25c4244d 100644 --- a/src/prefix_zipper.rs +++ b/src/prefix_zipper.rs @@ -59,6 +59,8 @@ pub struct PrefixZipper<'prefix, Z> { prefix: Cow<'prefix, [u8]>, origin_depth: usize, position: PrefixPos, + /// The zipper's own root is off the trie, as for a fork taken past a diverged prefix + off_root: bool, } impl<'prefix, Z> PrefixZipper<'prefix, Z> @@ -83,6 +85,7 @@ impl<'prefix, Z> PrefixZipper<'prefix, Z> prefix, origin_depth: 0, position, + off_root: false, } } @@ -110,7 +113,9 @@ impl<'prefix, Z> PrefixZipper<'prefix, Z> fn set_valid(&mut self, valid: usize) { debug_assert!(valid <= self.prefix.len(), "valid prefix can't be outside prefix"); - self.position = if valid == self.prefix.len() - self.origin_depth { + self.position = if self.off_root { + PrefixPos::PrefixOff { valid: 0, invalid: 0 } + } else if valid == self.prefix.len() - self.origin_depth { PrefixPos::Source } else { PrefixPos::Prefix { valid } @@ -122,6 +127,9 @@ impl<'prefix, Z> PrefixZipper<'prefix, Z> /// Returns `true` if the focus moved. The descended bytes are appended to this zipper's path /// buffer and reported to `obs`. Does nothing if the focus is already within the source. fn consume_prefix(&mut self, obs: &mut Obs) -> bool { + if !self.source.path_exists() { + return false; + } match self.position.prefixed_depth() { Some(prefixed_depth) => { let prefix_rest = &self.prefix[self.origin_depth + prefixed_depth..]; @@ -331,7 +339,9 @@ impl<'prefix, Z> Zipper for PrefixZipper<'prefix, Z> { fn path_exists(&self) -> bool { match self.position { - PrefixPos::Prefix {..} => true, + //The source stays at its root while the focus is in the prefix, and the prefix only + // exists if the source's root does + PrefixPos::Prefix {..} => self.source.path_exists(), PrefixPos::PrefixOff {..} => false, PrefixPos::Source => self.source.path_exists(), } @@ -344,17 +354,18 @@ impl<'prefix, Z> Zipper for PrefixZipper<'prefix, Z> } fn child_count(&self) -> usize { match self.position { - PrefixPos::Prefix {..} => 1, + PrefixPos::Prefix {..} => self.source.path_exists() as usize, PrefixPos::PrefixOff {..} => 0, PrefixPos::Source => self.source.child_count(), } } fn child_mask(&self) -> ByteMask { match self.position { - PrefixPos::Prefix { valid } => { + PrefixPos::Prefix { valid } if self.source.path_exists() => { let byte = self.prefix[self.origin_depth + valid]; ByteMask::from(byte) }, + PrefixPos::Prefix {..} => ByteMask::EMPTY, PrefixPos::PrefixOff {..} => ByteMask::EMPTY, PrefixPos::Source => self.source.child_mask(), } @@ -373,7 +384,7 @@ impl<'prefix, Z> ZipperMoving for PrefixZipper<'prefix, Z> fn at_root(&self) -> bool { match self.position { PrefixPos::Prefix { valid } => valid == 0, - PrefixPos::PrefixOff {..} => false, + PrefixPos::PrefixOff { valid, invalid } => self.off_root && valid == 0 && invalid == 0, PrefixPos::Source => self.prefix.len() <= self.origin_depth && self.source.at_root(), } } @@ -404,7 +415,7 @@ impl<'prefix, Z> ZipperMoving for PrefixZipper<'prefix, Z> if let PrefixPos::Prefix { valid } = &self.position { let valid = *valid; let rest_prefix = &self.prefix[self.origin_depth + valid..]; - let overlap = find_prefix_overlap(rest_prefix, path); + let overlap = if self.source.path_exists() { find_prefix_overlap(rest_prefix, path) } else { 0 }; path = &path[overlap..]; self.set_valid(valid + overlap); descended += overlap; @@ -558,7 +569,7 @@ impl<'prefix, Z> ZipperIteration for PrefixZipper<'prefix, Z> } fn descend_first_k_path_observed(&mut self, k: usize, obs: &mut Obs) -> bool { - if self.position.is_invalid() { + if k == 0 || self.position.is_invalid() || !self.source.path_exists() { return false; } //The prefix is a single forced path, so the bytes it contributes always exist and never @@ -623,12 +634,19 @@ impl<'prefix, Z, V> ZipperForking for PrefixZipper<'prefix, Z> { type ReadZipperT<'a> = PrefixZipper<'prefix, Z::ReadZipperT<'a>> where Self: 'a; fn fork_read_zipper<'a>(&'a self) -> >::ReadZipperT<'a> { + //The fork is rooted at the focus: in the source, partway along the prefix, or off the trie + let (prefix, position, off_root) = match self.position { + PrefixPos::Source => (Cow::Borrowed(&[][..]), PrefixPos::Source, false), + PrefixPos::Prefix { valid } => (Cow::Owned(self.prefix[self.origin_depth + valid..].to_vec()), PrefixPos::Prefix { valid: 0 }, false), + PrefixPos::PrefixOff {..} => (Cow::Borrowed(&[][..]), PrefixPos::PrefixOff { valid: 0, invalid: 0 }, true), + }; PrefixZipper { path: Vec::new(), - position: PrefixPos::Prefix { valid: 0 }, + position, source: self.source.fork_read_zipper(), - prefix: self.prefix.clone(), + prefix, origin_depth: 0, + off_root, } } } @@ -954,4 +972,91 @@ mod tests { //...so `descend_until` must report that it moved assert_eq!(moved, true); } + + /// `descend_first_k_path(0)` returns `false` without moving, wherever the focus is + #[test] + fn prefix_zipper_descend_first_k_path_zero() { + let mut map = PathMap::::new(); + map.set_val_at(&[0x22u8], 1); + map.set_val_at(&[0x22u8, 1], 2); + let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); + for setup in 0..4 { + z.reset(); + match setup { + 0 => {}, + 1 => { z.descend_to_byte(2); }, + 2 => { z.descend_to_byte(2); z.descend_last_path(); z.descend_last_path(); }, + _ => { z.descend_to(&[2u8, 3, 0x22]); }, + } + let path = z.path().to_vec(); + assert!(!z.descend_first_k_path(0), "setup {setup}"); + assert_eq!(z.path(), &path[..], "setup {setup}"); + } + } + + /// A prefix in front of a source rooted at a missing path doesn't exist either + #[test] + fn prefix_zipper_over_missing_source() { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8], 1); + let mut z = PrefixZipper::new(&[0u8, 7][..], map.read_zipper_at_path(&[5u8])); + assert!(!z.path_exists()); + assert_eq!(z.child_count(), 0); + assert_eq!(z.descend_first_byte(), None); + assert!(!z.to_next_step()); + assert!(!z.to_next_val()); + assert!(!z.descend_until()); + assert!(!z.descend_first_k_path(1)); + assert_eq!(z.path(), &[] as &[u8]); + assert_eq!(z.descend_to_existing(&[0u8, 7]), 0); + z.descend_to(&[0u8]); + assert!(!z.path_exists()); + assert_eq!(z.descend_first_byte(), None); + + //With the source present the prefix exists as before + let mut z = PrefixZipper::new(&[0u8, 7][..], map.read_zipper()); + assert!(z.path_exists()); + assert_eq!(z.descend_first_byte(), Some(0)); + assert!(z.path_exists()); + let mut steps = vec![]; + z.reset(); + while z.to_next_step() { steps.push(z.path().to_vec()); } + assert_eq!(steps, vec![vec![0], vec![0, 7], vec![0, 7, 0]]); + } + + /// A fork is rooted at the focus, wherever the focus is + #[test] + fn prefix_zipper_fork_at_focus() { + use crate::zipper::ZipperForking; + let mut map = PathMap::::new(); + map.set_val_at(&[5u8], 1); + map.set_val_at(&[5u8, 6], 2); + fn steps(z: &mut Z) -> Vec> { + let mut v = vec![]; + while z.to_next_step() { v.push(z.path().to_vec()); assert!(v.len() < 16); } + v + } + let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); + for (at, want) in [ + (&[][..], vec![vec![2], vec![2, 3], vec![2, 3, 5], vec![2, 3, 5, 6]]), + (&[2u8][..], vec![vec![3], vec![3, 5], vec![3, 5, 6]]), + (&[2u8, 3, 5][..], vec![vec![6]]), + (&[9u8][..], vec![]), + ] { + z.reset(); + z.descend_to(at); + let mut f = z.fork_read_zipper(); + assert!(f.at_root(), "{at:?}"); + assert_eq!(f.path_exists(), z.path_exists(), "{at:?}"); + assert_eq!(steps(&mut f), want, "{at:?}"); + f.descend_to(&[1u8, 1]); + assert_eq!(f.ascend(5), 2, "{at:?}"); + assert!(f.at_root(), "{at:?}"); + } + //An empty prefix + let mut z = PrefixZipper::new(&[][..], map.read_zipper()); + z.set_root_prefix_path(&[]).unwrap(); + let mut f = z.fork_read_zipper(); + assert_eq!(steps(&mut f), vec![vec![5], vec![5, 6]]); + } } diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 9f2be3e9..75a6952f 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -136,7 +136,7 @@ impl<'factor_z, 'trie, V: Clone + Send + Sync + Unpin, A: Allocator> ProductZipp /// `product_zipper_test4` for more discussion. #[inline] fn ensure_descend_next_factor(&mut self) { - if self.factor_paths.len() < self.secondaries.len() && self.z.child_count() == 0 { + if self.factor_paths.len() < self.secondaries.len() && self.z.child_count() == 0 && self.z.path_exists() { //We don't want to push the same factor on the stack twice if let Some(factor_path_len) = self.factor_paths.last() { @@ -148,6 +148,14 @@ impl<'factor_z, 'trie, V: Clone + Send + Sync + Unpin, A: Allocator> ProductZipp self.enroll_next_factor(); } } + /// The secondary factor whose root node is the focus, if any. Its node is not a child of the + /// node above it, so the core zipper can't look it up. + fn factor_root(&self) -> Option<&TrieRef<'trie, V, A>> { + match self.factor_paths.last() { + Some(&start) if start == self.depth() => self.secondaries.get(self.factor_paths.len() - 1), + _ => None + } + } /// Internal method to make sure `self.factor_paths` is correct after an ascend method #[inline] fn fix_after_ascend(&mut self) { @@ -175,8 +183,22 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper self.z.reset() } fn val_count(&self) -> usize { - debug_assert!(self.focus_factor() == self.factor_count() - 1); - self.z.val_count() + //Values below the focus can be spread over later factors, so walk a copy of the zipper + let mut walker = ProductZipper { + z: self.z.clone(), + secondaries: self.secondaries.clone(), + factor_paths: self.factor_paths.clone(), + source_zippers: Vec::new(), + }; + let focus = self.path(); + let mut count = self.is_val() as usize; + while walker.to_next_val() { + if walker.path().len() <= focus.len() || !walker.path().starts_with(focus) { + break + } + count += 1; + } + count } fn descend_to_existing>(&mut self, k: K) -> usize { let k = k.as_ref(); @@ -266,16 +288,24 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper moved } fn to_next_sibling_byte(&mut self) -> Option { - if self.factor_paths.last().cloned().unwrap_or(0) == self.depth() { + //Stepping sideways leaves a factor entered at this depth, but at the root there's no sideways + if self.depth() > 0 && self.factor_paths.last().cloned() == Some(self.depth()) { + //Take the factor's root off the core zipper too; a failed step enters it again below self.factor_paths.pop(); + self.z.deregularize(); + self.z.regularize(); } let moved = self.z.to_next_sibling_byte(); self.ensure_descend_next_factor(); moved } fn to_prev_sibling_byte(&mut self) -> Option { - if self.factor_paths.last().cloned().unwrap_or(0) == self.depth() { + //Stepping sideways leaves a factor entered at this depth, but at the root there's no sideways + if self.depth() > 0 && self.factor_paths.last().cloned() == Some(self.depth()) { + //Take the factor's root off the core zipper too; a failed step enters it again below self.factor_paths.pop(); + self.z.deregularize(); + self.z.regularize(); } let moved = self.z.to_prev_sibling_byte(); self.ensure_descend_next_factor(); @@ -362,8 +392,19 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper } impl ZipperConcrete for ProductZipper<'_, '_, V, A> { - fn shared_node_id(&self) -> Option { self.z.shared_node_id() } - fn is_shared(&self) -> bool { self.z.is_shared() } + fn shared_node_id(&self) -> Option { + match self.factor_root() { + Some(_) if self.z.is_val() => None, + Some(factor) => factor.shared_node_id(), + None => self.z.shared_node_id(), + } + } + fn is_shared(&self) -> bool { + match self.factor_root() { + Some(factor) => factor.is_shared(), + None => self.z.is_shared(), + } + } } impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> ZipperPathBuffer for ProductZipper<'_, 'trie, V, A> { @@ -497,6 +538,10 @@ impl<'trie, PrimaryZ, SecondaryZ, V> ProductZipperG<'trie, PrimaryZ, SecondaryZ, /// a combination between `to_next_sibling` and `to_prev_sibling` fn to_sibling_byte(&mut self, next: bool) -> Option { + //`focus_byte` is unspecified at the root, and the root has no siblings + if self.depth() == 0 { + return None; + } let byte = self.focus_byte()?; let ascended = self.ascend(1); debug_assert_eq!(ascended, 1, "must ascend"); @@ -1975,6 +2020,133 @@ mod tests { |btm: &mut PathMap<()>, path: &[u8]| -> _ { ProductZipperG::new::<[ReadZipperUntracked<()>; 0]>(btm.read_zipper_at_path(path), []) }); + + /// `val_count` counts the values below the focus in every later factor + #[test] + fn product_zipper_val_count_in_every_factor() { + let mut a = PathMap::::new(); + for p in [&[1u8, 2][..], &[1u8], &[3u8]] { a.set_val_at(p, 1); } + let mut b = PathMap::::new(); + for p in [&[4u8][..], &[4u8, 5], &[6u8]] { b.set_val_at(p, 2); } + let mut c = PathMap::::new(); + for p in [&[7u8][..], &[8u8, 9]] { c.set_val_at(p, 3); } + + //Every value path in the product, from a full walk + let mut all = vec![]; + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper(), c.read_zipper()]); + while z.to_next_val() { all.push(z.path().to_vec()); } + assert!(!all.is_empty()); + + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper(), c.read_zipper()]); + loop { + let focus = z.path().to_vec(); + let expected = all.iter().filter(|p| p.starts_with(&focus)).count(); + assert_eq!(z.val_count(), expected, "{focus:?}"); + if !z.to_next_step() { break } + } + } + + /// k-path walks and sibling steps keep factor bookkeeping in step with the core zipper, including + /// with a primary rooted at a missing path or at a leaf + #[test] + fn product_zipper_k_path_walk_from_the_root() { + let mut a = PathMap::<()>::new(); + for p in [&[2u8, 2, 0, 0][..], &[0xe7]] { a.set_val_at(p, ()); } + let b = a.clone(); + for root in [&[0xaau8, 0x77][..], &[0xe7u8][..], &[][..]] { + for k in 1..4 { + let mut z = ProductZipper::new(a.read_zipper_at_path(root), [b.read_zipper()]); + let mut paths = vec![]; + if z.descend_first_k_path(k) { + paths.push(z.path().to_vec()); + while paths.len() < 64 && z.to_next_k_path(k) { paths.push(z.path().to_vec()); } + } + assert_eq!(z.path(), &[] as &[u8], "{root:?} k={k}"); + let _ = (z.is_shared(), z.shared_node_id(), z.child_mask()); + assert!(paths.iter().all(|p| p.len() == k), "{root:?} k={k}: {paths:?}"); + if root == &[0xaau8, 0x77][..] { + assert!(paths.is_empty(), "a missing primary has no paths: {paths:?}"); + } + let mut z = ProductZipper::new(a.read_zipper_at_path(root), [b.read_zipper()]); + let _ = (z.to_next_sibling_byte(), z.to_prev_sibling_byte()); + let _ = (z.is_shared(), z.child_mask(), z.descend_first_byte(), z.is_shared()); + } + } + } + + /// Sibling steps and value walks at the root of a `ProductZipperG` whose primary is rooted below the map root + #[test] + fn product_zipper_g_sibling_step_at_root() { + let mut a = PathMap::::new(); + for p in [&[2u8, 3, 2, 1, 0, 7][..], &[2, 3, 2, 1, 0, 8, 1], &[9]] { a.set_val_at(p, 1); } + let b = a.clone(); + let mut z = ProductZipperG::new(a.read_zipper_at_path(&[2u8, 3, 2, 1, 0]), [b.read_zipper()]); + assert_eq!(z.to_next_sibling_byte(), None); + assert_eq!(z.to_prev_sibling_byte(), None); + assert_eq!(z.path(), &[] as &[u8]); + let mut n = 0; + while z.to_next_val() { n += 1; assert!(n < 100); } + assert!(n > 0); + assert_eq!(z.path(), &[] as &[u8]); + } + + /// A failed sibling step at the root of a factor stays in that factor + #[test] + fn product_zipper_no_sibling_at_factor_root() { + let mut a = PathMap::::new(); + a.set_val_at(&[0u8, 0], 1); + let mut b = PathMap::::new(); + b.set_val_at(&[5u8], 2); + for prev in [false, true] { + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); + assert!(z.to_next_val()); + assert_eq!(z.path(), &[0, 0]); + let moved = if prev { z.to_prev_sibling_byte() } else { z.to_next_sibling_byte() }; + assert_eq!(moved, None); + assert_eq!(z.child_count(), 1); + let _ = (z.is_shared(), z.shared_node_id()); + assert!(z.to_next_val()); + assert_eq!((z.path(), z.val()), (&[0u8, 0, 5][..], Some(&2))); + } + } + + /// Sibling steps out of a factor entered below an empty node, as a dropped head writer leaves + #[test] + fn product_zipper_k_path_past_empty_node() { + let mut a = PathMap::::new(); + for p in [&[0u8][..], &[0, 0, 0], &[1]] { a.set_val_at(p, 0); } + { + let zh = a.zipper_head(); + zh.write_zipper_at_exclusive_path(&[1u8, 0]).unwrap().set_val(0); + let _w = zh.write_zipper_at_exclusive_path(&[0u8, 0, 0, 0]).unwrap(); + } + let b = a.clone(); + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); + let mut paths = vec![]; + if z.descend_first_k_path(3) { + paths.push(z.path().to_vec()); + while paths.len() < 64 && z.to_next_k_path(3) { paths.push(z.path().to_vec()); } + } + assert!(paths.iter().all(|p| p.len() == 3), "{paths:?}"); + } + + /// `is_shared` and `shared_node_id` across factor boundaries + #[test] + fn product_zipper_is_shared_across_factors() { + let mut a = PathMap::::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 2, 1, 3, 3], &[0], &[2, 2]] { a.set_val_at(p, 7); } + let b = a.clone(); + let mut z = ProductZipper::new(a.read_zipper_at_path(&[1u8, 2, 1]), [b.read_zipper()]); + let mut factor_roots = 0; + while z.to_next_step() { + let _ = (z.is_shared(), z.shared_node_id()); + if z.factor_root().is_some() { + factor_roots += 1; + assert!(z.is_shared(), "{:?}", z.path()); + } + } + assert!(factor_roots > 0); + } } //POSSIBLE FUTURE DIRECTION: diff --git a/src/ring.rs b/src/ring.rs index 4b9b1d45..0452fea1 100644 --- a/src/ring.rs +++ b/src/ring.rs @@ -425,36 +425,6 @@ impl FatAlgebraicResult { pub(crate) const fn new(identity_mask: u64, element: Option) -> Self { Self {identity_mask, element} } - /// Converts an [AlgebraicResult] into a `FatAlgebraicResult`, assuming the source `result` was the - /// output of a binary operation (two arguments). - #[inline] - pub(crate) fn from_binary_op_result(result: AlgebraicResult, a: &V, b: &V) -> Self - where V: Clone - { - match result { - AlgebraicResult::None => FatAlgebraicResult::none(), - AlgebraicResult::Element(v) => FatAlgebraicResult::element(v), - AlgebraicResult::Identity(mask) => { - debug_assert!(mask <= (SELF_IDENT | COUNTER_IDENT)); - if mask & SELF_IDENT > 0 { - FatAlgebraicResult::new(mask, Some(a.clone())) - } else { - debug_assert_eq!(mask, COUNTER_IDENT); - FatAlgebraicResult::new(mask, Some(b.clone())) - } - } - } - } - /// Maps a `FatAlgebraicResult` to `FatAlgebraicResult` by applying a function to a contained value - #[inline] - pub fn map(self, f: F) -> FatAlgebraicResult - where F: FnOnce(V) -> U, - { - FatAlgebraicResult:: { - identity_mask: self.identity_mask, - element: self.element.map(f) - } - } /// The result of an operation between non-none arguments that results in None #[inline(always)] pub(crate) const fn none() -> Self { @@ -754,6 +724,22 @@ fn option_subtract_test() { assert_eq!(Some(Some(Some(()))).psubtract(&Some(Some(Some(())))), AlgebraicResult::None); } +/// Subtracting a value that isn't there leaves the destination alone, and the integer placeholders +/// have to say so with `Identity(SELF_IDENT)`. Returning `Element(*self)` is the same value, but +/// the node algebra propagates identity *masks*, not values, so an `Element` anywhere below a node +/// forces the whole node -- and with it `subtract_into` -- to report `Element` for a trie that did +/// not change. +#[test] +fn integer_subtract_is_self_identity() { + assert_eq!(3u64.psubtract(&5), AlgebraicResult::Identity(SELF_IDENT)); + assert_eq!(3u64.psubtract(&3), AlgebraicResult::None); + assert_eq!(3u16.psubtract(&5), AlgebraicResult::Identity(SELF_IDENT)); + assert_eq!(3u16.psubtract(&3), AlgebraicResult::None); + //The same, seen through `Option`, which is what the co-free node payloads use + assert_eq!(Some(3u64).psubtract(&Some(5)), AlgebraicResult::Identity(SELF_IDENT)); + assert_eq!(Some(3u64).psubtract(&Some(3)), AlgebraicResult::None); +} + // =-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-= // =-* `Option<&V>` *-= @@ -851,15 +837,24 @@ impl Lattice for () { fn pmeet(&self, _other: &Self) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) } } +/// Left-biased join for the plain integer placeholders: the result is always `self`, but when +/// the two are equal it is also `other`, and the node algebra needs to know that to report an +/// unchanged join as `Identity` (a join evaluated with swapped operands, e.g. against a +/// `TinyRefNode`, otherwise never sees a self-identity) +#[inline] +fn left_biased_pjoin(a: &T, b: &T) -> AlgebraicResult { + if a == b { AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) } else { AlgebraicResult::Identity(SELF_IDENT) } +} + //GOAT trash impl Lattice for usize { - fn pjoin(&self, _other: &usize) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &usize) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &usize) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } //GOAT trash impl Lattice for u64 { - fn pjoin(&self, _other: &u64) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u64) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u64) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } @@ -867,19 +862,19 @@ impl Lattice for u64 { impl DistributiveLattice for u64 { fn psubtract(&self, other: &Self) -> AlgebraicResult where Self: Sized { if self == other { AlgebraicResult::None } - else { AlgebraicResult::Element(*self) } + else { AlgebraicResult::Identity(SELF_IDENT) } } } //GOAT trash impl Lattice for u32 { - fn pjoin(&self, _other: &u32) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u32) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u32) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } //GOAT trash impl Lattice for u16 { - fn pjoin(&self, _other: &u16) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u16) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u16) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } @@ -887,13 +882,13 @@ impl Lattice for u16 { impl DistributiveLattice for u16 { fn psubtract(&self, other: &Self) -> AlgebraicResult { if self == other { AlgebraicResult::None } - else { AlgebraicResult::Element(*self) } + else { AlgebraicResult::Identity(SELF_IDENT) } } } //GOAT trash impl Lattice for u8 { - fn pjoin(&self, _other: &u8) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u8) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u8) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } diff --git a/src/tiny_node.rs b/src/tiny_node.rs index 6ece3c72..c76ad615 100644 --- a/src/tiny_node.rs +++ b/src/tiny_node.rs @@ -149,33 +149,6 @@ impl<'a, V: Clone + Send + Sync, A: Allocator> TrieNode for TinyRefNode<'a } fn node_get_child_mut(&mut self, _key: &[u8]) -> Option<(usize, &mut TrieNodeODRc)> { unreachable!() } fn node_replace_child(&mut self, _key: &[u8], _new_node: TrieNodeODRc) { unreachable!() } - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - if self.node_is_empty() { - return true - } - let mut requested_contained_item = false; // This node type only has one item - let self_key = self.key(); - debug_assert!(results.len() >= keys.len()); - for ((key, expect_val), (result_key_len, payload_ref)) in keys.into_iter().zip(results.into_iter()) { - if starts_with(key, self_key) { - let self_key_len = self_key.len(); - if self.is_child_ptr() { - if !*expect_val || self_key_len < key.len() { - requested_contained_item = true; - *result_key_len = self_key_len; - *payload_ref = PayloadRef::Child(unsafe{ &*self.payload.child }); - } - } else { - if *expect_val && self_key_len == key.len() { - requested_contained_item = true; - *result_key_len = self_key_len; - *payload_ref = PayloadRef::Val(unsafe{ &**self.payload.val }); - } - } - } - } - requested_contained_item - } fn node_contains_val(&self, key: &[u8]) -> bool { if self.is_used_val() { let node_key = self.key(); diff --git a/src/trie_map.rs b/src/trie_map.rs index efe75ed6..b5e0b611 100644 --- a/src/trie_map.rs +++ b/src/trie_map.rs @@ -876,6 +876,37 @@ mod tests { // c-> List("aaa") } + /// `subtract` must keep a value when the other map only has a *longer* path through its location. + /// Needs a byte-node root on the left (a value-and-child key plus a second branch) and a line-node + /// root on the right; every smaller shape already worked. + #[test] + fn map_subtract_keeps_value_under_other_path() { + let mut a: PathMap = PathMap::new(); + a.insert([0], 0); + a.insert([0, 0], 0); + a.insert([0, 0, 0], 0); + a.insert([1, 0, 0], 0); + let mut b: PathMap = PathMap::new(); + b.insert([0, 0], 0); + b.insert([1, 0, 0], 0); + + let diff = a.subtract(&b); + let paths: Vec<(Vec, u64)> = diff.iter().map(|(k, v)| (k.to_vec(), *v)).collect(); + assert_eq!(paths, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); + + // and with distinct values, so the surviving value is visibly the left one + let mut a: PathMap = PathMap::new(); + a.insert([0], 5); + a.insert([0, 0], 6); + a.insert([0, 0, 0], 7); + a.insert([1, 0, 0], 8); + let diff = a.subtract(&b); + assert_eq!(diff.val_at([0]), Some(&5)); + assert_eq!(diff.val_at([0, 0]), Some(&6)); // 6 - 0 is Element(6) for u64 + assert_eq!(diff.val_at([0, 0, 0]), Some(&7)); + assert_eq!(diff.val_at([1, 0, 0]), Some(&8)); + } + #[test] fn map_insert_test() { let keys = [ diff --git a/src/trie_node.rs b/src/trie_node.rs index 27366659..2ef8c8c3 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -5,7 +5,6 @@ use core::ptr::NonNull; use std::collections::HashMap; use dyn_clone::*; use local_or_heap::LocalOrHeap; -use arrayvec::ArrayVec; use crate::utils::ByteMask; use crate::alloc::Allocator; @@ -72,35 +71,6 @@ pub(crate) trait TrieNode: TrieNodeDowncas /// cheaper, but it is adequate for the places that call it fn node_replace_child(&mut self, key: &[u8], new_node: TrieNodeODRc); - /// Retrieves multiple values or child links from the node, associated with elements from `keys`, - /// and places them into the respective element in `results` - /// - /// The `bool` in `keys` indicates whether a value is expected at the requested key. `true` will be - /// passed to indicate a **value**. (WARNING: This is different from the convention in some node types) - /// - /// If a node contains both an onward link and a value at the same key, the `bool` specifies which to - /// return; however, a node may be returned for a requested value, if the path to the node is a prefix - /// to the path to the requested value. This is because the value may live within a child node. On - /// the other hand, a value will only be returned if it is an exact match with the key provided. - /// - /// The `usize` in `results` functions the same way as the returned `usize` in [TrieNode::node_get_child], - /// to indicate the number of key bytes matched by the key contained within the node. - /// - /// The implementation may assume `keys` will be in sorted order, and `false` sorts before `true` if - /// both a value and a node at the same key are requested. - /// - /// Returns `true` if the requested `keys` completely enumerate the set of elements contained within - /// the node, or `false` if the node contains additional elements that were not requested - /// - /// If a result is not found for a given key, the implementation does not guarantee the corresponding - /// element in `results` will be set to [`PayloadRef::None`], therefore the caller should init `results` - /// to default values. - /// - /// Panics if `keys.len() > results.len()` - /// - /// NOTE: It perfectly fine for multiple keys to share a prefix, and sometimes that means multiple - /// results will be identical if the node represents only the prefix portion of the key. - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool; /// Returns `true` if the node contains a value at the specified key, otherwise returns `false` /// @@ -526,32 +496,6 @@ impl Default for PayloadRef<'_, V, A> { } } -impl<'a, V: Clone + Send + Sync, A: Allocator> PayloadRef<'a, V, A> { - pub fn is_none(&self) -> bool { - match self { - Self::None => true, - _ => false - } - } - pub fn is_val(&self) -> bool { - match self { - Self::Val(_) => true, - _ => false - } - } - pub fn child(&self) -> &'a TrieNodeODRc { - match self { - Self::Child(child) => child, - _ => panic!() - } - } - pub fn val(&self) -> &'a V { - match self { - Self::Val(val) => val, - _ => panic!() - } - } -} #[derive(Clone)] pub(crate) enum ValOrChild { @@ -630,53 +574,236 @@ impl ValOrChildUnion { } } -/// An implementation of pmeet_dyn that should be correct for any two node types, Although it -/// certainly won't be optimally efficient. -/// -/// WARNING: just like [TrieNode::node_get_payloads], the keys in `self_payloads` must be in -/// sorted order. -// -//NOTE: I have confirmed that this function behaves no more conservatively than the function it replaced. -// In other words, I have confirmed that, in tests where the old function was behaving correctly, this -// function returns *identical* results. Furthermore those same tests are the ones where the 20% slowdown -// was observed. Therefore the the ~20% slowdown is simply the higher overheads of this generic function. -// -//The next port of call for optimization is probably to remove the recursion -pub(crate) fn pmeet_generic(self_payloads: &[(&[u8], PayloadRef)], other: TaggedNodeRef, merge_f: MergeF) -> AlgebraicResult> - where - MergeF: FnOnce(&mut [Option>]) -> TrieNodeODRc, - V: Clone + Send + Sync + Lattice -{ - let mut request_keys = ArrayVec::<(&[u8], bool), MAX_PAYLOAD_CNT>::new(); - let mut element_results = ArrayVec::>, MAX_PAYLOAD_CNT>::new(); - let mut request_results = ArrayVec::<(usize, PayloadRef), MAX_PAYLOAD_CNT>::new(); - for (self_key, self_payload) in self_payloads.iter() { - debug_assert!(!self_payload.is_none()); - request_keys.push((self_key, self_payload.is_val())); - element_results.push(FatAlgebraicResult::none()); - request_results.push((0, PayloadRef::default())); +/// Where `key` lands in `node`: follows the onward links that cover a strict prefix of `key`, and +/// returns the node that holds the rest of the key, together with that rest (never empty) +#[inline] +pub(crate) fn meet_locate_key<'n, 'k, V: Clone + Send + Sync, A: Allocator>(mut node: TaggedNodeRef<'n, V, A>, mut key: &'k [u8]) -> (TaggedNodeRef<'n, V, A>, &'k [u8]) { + debug_assert!(key.len() > 0); + while let Some((consumed, child)) = node.node_get_child(key) { + if consumed >= key.len() { + break + } + node = child.as_tagged(); + key = &key[consumed..]; } + (node, key) +} - let is_exhaustive = pmeet_generic_internal::(self_payloads, &mut request_keys[..], &mut request_results[..], &mut element_results[..], other); - let mut is_none = true; - let mut combined_mask = SELF_IDENT | COUNTER_IDENT; - let mut result_payloads = ArrayVec::>, MAX_PAYLOAD_CNT>::new(); - for result in element_results { - combined_mask &= result.identity_mask; - is_none = is_none && result.element.is_none(); - result_payloads.push(result.element); +/// The onward node exactly at `key` in `src`, if there is one +#[inline] +pub(crate) fn meet_src_child<'a, V: Clone + Send + Sync, A: Allocator>(src: Option>, key: &[u8]) -> Option> { + let (node, rest) = meet_locate_key(src?, key); + match node.node_get_child(rest) { + Some((consumed, child)) if consumed == rest.len() => Some(child.as_tagged()), + _ => None } +} + +/// The outcome of [node_drop_dangling] +pub(crate) enum DropDangling { + /// The node is kept as it is + Unchanged, + /// No value is left below the node's root + Empty, + /// The node with its dangling paths dropped + New(TrieNodeODRc), +} - if is_none { - return AlgebraicResult::None +/// Drops the dangling paths below the root of `node`, keeping only the locations on the way to a +/// value. `src` is the node standing at the same location in the source of a pruned meet: a node +/// shared with it is skipped rather than walked, so a dangling path inside a shared node survives. +pub(crate) fn node_drop_dangling(node: &TrieNodeODRc, src: Option>) -> DropDangling { + let tagged = node.as_tagged(); + if tagged.node_is_empty() { + return DropDangling::Empty } - if !is_exhaustive { - combined_mask &= !COUNTER_IDENT; + if let Some(src) = src { + if tagged.shared_node_id() == src.shared_node_id() { + return DropDangling::Unchanged + } + } + match tagged.tag() { + DENSE_BYTE_NODE_TAG => unsafe{ tagged.as_dense_unchecked() }.drop_dangling(src), + LINE_LIST_NODE_TAG => unsafe{ tagged.as_list_unchecked() }.drop_dangling(src), + CELL_BYTE_NODE_TAG => unsafe{ tagged.as_cell_unchecked() }.drop_dangling(src), + _ => unreachable!() + } +} + +/// The number of branches below `key` in the trie rooted at `node` +fn meet_count_branches_at(node: TaggedNodeRef, key: &[u8]) -> usize { + if key.is_empty() { + return node.count_branches(&[]) + } + let (node, rest) = meet_locate_key(node, key); + match node.node_get_child(rest) { + Some((consumed, child)) if consumed == rest.len() => child.as_tagged().count_branches(&[]), + _ => node.count_branches(rest) + } +} + +/// What one slot of a list node contributes to a meet; see [meet_list_slot] +pub(crate) enum SlotMeet { + /// The slot was left out of the meet + Skipped, + /// Not even the first byte of the slot's key exists in the other operand + Nothing, + /// The slot's own payload, unchanged + Keep, + /// A value at the slot's key + Val(V), + /// An onward node at the slot's key + Child(TrieNodeODRc), + /// A dangling path along the first `n` bytes of the slot's key + Dangling(usize), +} + +impl SlotMeet { + #[inline] + pub(crate) fn is_nothing(&self) -> bool { + matches!(self, Self::Nothing | Self::Skipped) + } + /// How many bytes of the slot's key exist in the result + #[inline] + pub(crate) fn reach(&self, key_len: usize) -> usize { + match self { + Self::Skipped | Self::Nothing => 0, + Self::Dangling(n) => *n, + _ => key_len + } + } + /// The key and payload of the contribution; `keep` supplies the slot's own payload + #[inline] + pub(crate) fn into_item<'k, F: FnOnce() -> ValOrChild>(self, key: &'k [u8], keep: F) -> Option<(&'k [u8], ValOrChild)> { + match self { + Self::Skipped | Self::Nothing => None, + Self::Keep => Some((key, keep())), + Self::Val(val) => Some((key, ValOrChild::Val(val))), + Self::Child(node) => Some((key, ValOrChild::Child(node))), + Self::Dangling(n) => Some((&key[..n], ValOrChild::Child(TrieNodeODRc::new_empty()))), + } + } +} + +/// Meets one slot of a list node, `payload` at `key`, against `other`, following the rule that a +/// location survives exactly when both operands have it, and a value exactly when both hold one. +/// +/// Returns `(self_ident, counter_ok, contribution)`. `self_ident` is true exactly when the +/// contribution equals the slot. `counter_ok` is true when, along this slot's key and below it, +/// the contribution holds everything `other` holds there; [meet_other_within_slots] checks the rest. +/// `swapped` orients the meet as in [crate::line_list_node::LineListNode::pmeet_dyn_oriented]. +pub(crate) fn meet_list_slot(key: &[u8], payload: PayloadRef, other: TaggedNodeRef, swapped: bool) -> (bool, bool, SlotMeet) { + let (node, rest) = meet_locate_key(other, key); + let exact_child = match node.node_get_child(rest) { + Some((consumed, child)) if consumed == rest.len() => Some(child), + _ => None + }; + let other_val = node.node_get_val(rest); + if exact_child.is_none() && other_val.is_none() && !node.node_contains_partial_key(rest) { + //`other` stops partway along the key: the part both have is a dangling path + let reach = key.len() - rest.len() + node.node_key_overlap(rest); + return (false, true, if reach > 0 { SlotMeet::Dangling(reach) } else { SlotMeet::Nothing }) + } + + match payload { + PayloadRef::Val(self_val) => match other_val { + Some(other_val) => { + let result = if swapped { other_val.pmeet(self_val).invert_identity() } else { self_val.pmeet(other_val) }; + match result { + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { + (true, mask & COUNTER_IDENT > 0, SlotMeet::Keep) + } else { + (false, true, SlotMeet::Val(other_val.clone())) + }, + AlgebraicResult::Element(val) => (false, false, SlotMeet::Val(val)), + AlgebraicResult::None => (false, false, SlotMeet::Dangling(key.len())), + } + }, + None => (false, true, SlotMeet::Dangling(key.len())), + }, + PayloadRef::Child(self_child) => { + let onward = match exact_child { + Some(child) => AbstractNodeRef::BorrowedRc(child), + None => node.get_node_at_key(rest), + }; + let self_empty = self_child.as_tagged().node_is_empty(); + let other_empty = match onward.try_as_tagged() { + Some(below) => below.node_is_empty(), + None => true + }; + if self_empty || other_empty { + //Nothing below the key survives, which leaves the key itself + let contribution = if self_empty { SlotMeet::Keep } else { SlotMeet::Dangling(key.len()) }; + return (self_empty, other_empty, contribution) + } + let result = { + let other_below = onward.as_tagged(); + if swapped { + other_below.pmeet_dyn(self_child.as_tagged()).invert_identity() + } else { + self_child.as_tagged().pmeet_dyn(other_below) + } + }; + match result { + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { + (true, mask & COUNTER_IDENT > 0, SlotMeet::Keep) + } else { + (false, true, SlotMeet::Child(onward.into_option().unwrap())) + }, + AlgebraicResult::Element(node) => (false, false, SlotMeet::Child(node)), + AlgebraicResult::None => (false, false, SlotMeet::Dangling(key.len())), + } + }, + PayloadRef::None => unreachable!() } - if combined_mask > 0 { - return AlgebraicResult::Identity(combined_mask) +} + +/// Given the slots `(key, is_child, reach)` of a list node that have each been met against `other` +/// with `counter_ok` (see [meet_list_slot]), is all of `other` inside the result? That holds when +/// `other` branches nowhere off the paths the slots reach, and holds no value on them except where a +/// value slot stands. Below a slot's onward node, the node meet has already answered. +pub(crate) fn meet_other_within_slots(other: TaggedNodeRef, slots: &[Option<(&[u8], bool, usize)>; 2]) -> bool { + //The distinct bytes the slots continue with after `prefix` + let branches_expected = |prefix: &[u8]| -> usize { + let depth = prefix.len(); + let mut first = None; + let mut count = 0; + for (key, _, reach) in slots.iter().flatten() { + if *reach > depth && &key[..depth] == prefix { + let byte = key[depth]; + if first != Some(byte) { + count += 1; + first = Some(byte); + } + } + } + count + }; + if meet_count_branches_at(other, &[]) != branches_expected(&[]) { + return false + } + for (key, _, reach) in slots.iter().flatten() { + for depth in 1..=*reach { + let prefix = &key[..depth]; + //A value slot can share its key with an onward-node slot, which then answers for below + let below_is_met = slots.iter().flatten().any(|(slot_key, slot_is_child, slot_reach)| { + *slot_is_child && *slot_reach == depth && *slot_key == prefix + }); + if !below_is_met && meet_count_branches_at(other, prefix) != branches_expected(prefix) { + return false + } + let (node, rest) = meet_locate_key(other, prefix); + if node.node_get_val(rest).is_some() { + let covered = slots.iter().flatten().any(|(slot_key, slot_is_child, slot_reach)| { + !*slot_is_child && *slot_reach == depth && *slot_key == prefix + }); + if !covered { + return false + } + } + } } - AlgebraicResult::Element(merge_f(&mut result_payloads[..])) + true } pub(crate) fn node_count_branches_recursive(node: TaggedNodeRef, key: &[u8]) -> usize { @@ -696,124 +823,7 @@ pub(crate) fn node_count_branches_recursive(self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>], other_node: TaggedNodeRef<'trie, V, A>) -> bool - where V: Clone + Send + Sync + Lattice -{ - //If is_exhaustive gets set to `false`, then the pmeet method cannot return a `COUNTER_IDENTITY` result - let mut is_exhaustive = true; - - //Get the payload results from the node - if !other_node.node_get_payloads(&keys[..], request_results) { - is_exhaustive = false; - } - - //Divide the results into groups based on the returned node. Because keys must be - // in sorted order, we can assume that query results returning the same node will - // be contiguous. - //NOTE: It's theoretically possible (although pretty unlikely) that a node will - // have multiple discontinuous internal paths leading to the same child node, however - // the TrieNodeODRc pointers will be different in that case, so this logic is still - // correct. - let mut cur_group: Option<(usize, &TrieNodeODRc)> = None; - for idx in 0..keys.len() { - let (consumed_bytes, payload) = core::mem::take(request_results.get_mut(idx).unwrap()); - if !payload.is_none() { - let is_val = keys[idx].1; - if consumed_bytes < keys[idx].0.len() { - keys[idx].0 = &keys[idx].0[consumed_bytes..]; - debug_assert!(!payload.is_val()); - let child = payload.child(); - - //Continue to grow range, or do the recursive call, depending on whether - // we have the same node as the previous time through the loop - if cur_group.is_some() { - if (cur_group.as_ref().unwrap().1 as *const TrieNodeODRc) != (child as *const TrieNodeODRc) { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); - cur_group = Some((idx, child)); - } - } else { - cur_group = Some((idx, child)); - } - } else { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); - - //We've arrived at a contained value or onward link that has a correspondence - // to one of the values or links in `self` - debug_assert_eq!(consumed_bytes, keys[idx].0.len()); - debug_assert_eq!(is_val, payload.is_val()); - let result = match &self_payloads[idx].1 { - PayloadRef::Child(self_link) => { - let other_link = payload.child(); - let result = self_link.pmeet(other_link); - FatAlgebraicResult::from_binary_op_result(result, self_link, other_link) - .map(|child| ValOrChild::Child(child)) - }, - PayloadRef::Val(self_val) => { - let other_val = payload.val(); - let result = (*self_val).pmeet(other_val); - FatAlgebraicResult::from_binary_op_result(result, *self_val, other_val) - .map(|val| ValOrChild::Val(val)) - }, - _ => unreachable!() - }; - debug_assert!(results[idx].element.is_none()); - debug_assert_eq!(results[idx].identity_mask, 0); - results[idx] = result; - } - } else { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); - - let result = match &self_payloads[idx].1 { - PayloadRef::Child(self_link) => { - match other_node.get_node_at_key(keys[idx].0).into_option() { - Some(other_onward_node) => { - let result = self_link.as_tagged().pmeet_dyn(other_onward_node.as_tagged()); - FatAlgebraicResult::from_binary_op_result(result, self_link, &other_onward_node) - .map(|child| ValOrChild::Child(child)) - }, - None => { - //Check to see if we have a dangling path, because a dangling path meet with a value should result in a path, but no value - if self_link.is_empty() && other_node.node_get_val(keys[idx].0).is_some() { - FatAlgebraicResult::new(SELF_IDENT, Some(ValOrChild::Child(TrieNodeODRc::new_empty()))) - } else { - FatAlgebraicResult::new(COUNTER_IDENT, None) - } - } - } - }, - PayloadRef::Val(_self_val) => { - //If self_payload is a val and we didn't get a corresponding val, then this result is None - FatAlgebraicResult::new(COUNTER_IDENT, None) - }, - _ => unreachable!() - }; - results[idx] = result; - } - } - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, keys.len(), self_payloads, keys, request_results, results); - - is_exhaustive -} -/// Effectively part of `pmeet_generic_internal`, but factored out separately because it's called in -/// several different places. Resets the `cur_group` state and does a recursive call of `pmeet_generic_internal` -#[inline] -fn pmeet_generic_recursive_reset<'trie, const MAX_PAYLOAD_CNT: usize, V, A: Allocator>(cur_group: &mut Option<(usize, &'trie TrieNodeODRc)>, is_exhaustive: &mut bool, idx: usize, self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>]) - where V: Clone + Send + Sync + Lattice -{ - match core::mem::take(cur_group) { - Some((group_start, next_node)) => { - let group_keys = &mut keys[group_start..idx]; - let group_results = &mut results[group_start..idx]; - let group_self_payloads = &self_payloads[group_start..idx]; - if !pmeet_generic_internal::(group_self_payloads, group_keys, request_results, group_results, next_node.as_tagged()) { - *is_exhaustive = false; - } - }, - None => {} - } -} /// An abstracted reference to the node at the zipper's focus, returned by [`crate::zipper::ZipperInfallibleSubtries::get_focus`] /// @@ -1265,15 +1275,6 @@ mod tagged_node_ref { } } - pub(crate) fn node_get_payloads<'res>(&self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'a, V, A>)]) -> bool { - match self { - Self::DenseByteNode(node) => node.node_get_payloads(keys, results), - Self::LineListNode(node) => node.node_get_payloads(keys, results), - Self::CellByteNode(node) => node.node_get_payloads(keys, results), - Self::TinyRefNode(node) => node.node_get_payloads(keys, results), - Self::EmptyNode => true, - } - } pub fn node_contains_val(&self, key: &[u8]) -> bool { match self { @@ -1912,17 +1913,6 @@ mod tagged_node_ref { _ => unsafe{ unreachable_unchecked() } } } - pub fn node_get_payloads<'res>(&self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'a, V, A>)]) -> bool { - let (ptr, tag) = self.ptr.get_raw_parts(); - match tag { - EMPTY_NODE_TAG => true, - DENSE_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - LINE_LIST_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - CELL_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - TINY_REF_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - _ => unsafe{ unreachable_unchecked() } - } - } pub fn node_contains_val(&self, key: &[u8]) -> bool { let (ptr, tag) = self.ptr.get_raw_parts(); match tag { @@ -2192,7 +2182,7 @@ mod tagged_node_ref { } let (ptr, tag) = self.ptr.get_raw_parts(); match tag { - EMPTY_NODE_TAG => AlgebraicResult::None, + EMPTY_NODE_TAG => crate::empty_node::EmptyNode.pmeet_dyn(other), DENSE_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.pmeet_dyn(other), LINE_LIST_NODE_TAG => unsafe{ &*ptr.cast::>() }.pmeet_dyn(other), CELL_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.pmeet_dyn(other), @@ -2662,8 +2652,12 @@ pub(crate) fn node_along_path_mut<'a, 'k, V: Clone + Send + Sync, A: Allocator>( /// Ensures the node is a CellByteNode /// /// Returns `true` if the node was upgraded and `false` if it already was a CellByteNode -pub(crate) fn make_cell_node(node: &mut TrieNodeODRc) -> bool { - if !node.as_tagged().is_cell_node() { +pub(crate) fn make_cell_node(node: &mut TrieNodeODRc, alloc: A) -> bool { + if node.is_empty() { + //The empty sentinel can't be made mutable; there is nothing in it to keep + *node = TrieNodeODRc::new_in(crate::dense_byte_node::CellByteNode::new_in(alloc.clone()), alloc); + true + } else if !node.as_tagged().is_cell_node() { let replacement = node.make_mut().convert_to_cell_node(); *node = replacement; true @@ -3446,6 +3440,106 @@ mod tests { use crate::PathMap; use crate::zipper::*; + fn mk(ps: &[(&[u8], u64)]) -> PathMap { + let mut m = PathMap::::new(); + for (p, v) in ps { m.set_val_at(p, *v); } + m + } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { + m.iter().map(|(p, v)| (p.to_vec(), *v)).collect() + } + + /// `Lattice for u64` keeps `self` on a collision, so a meet carries the *left* operand's value. + /// That must not depend on which node type each operand happens to be stored in: a byte node + /// meeting a list node used to run the meet with the operands swapped and returned the list + /// node's values whichever side it was on. Found by lean/differential.py. + #[test] + fn meet_value_bias_is_left_regardless_of_node_layout() { + let two_payload_list = mk(&[(&[0], 0), (&[0, 0], 0), (&[3, 0], 1)]); + let single_line = mk(&[(&[0], 1)]); + let dense = mk(&[(&[0], 0), (&[1], 0), (&[2], 0), (&[3], 0), (&[4], 0)]); + for (a, b, expect) in [ + (&two_payload_list, &single_line, 0u64), + (&single_line, &two_payload_list, 1), + (&dense, &single_line, 0), + (&single_line, &dense, 1), + (&dense, &two_payload_list, 0), + (&two_payload_list, &dense, 0), + ] { + let mut out = PathMap::::new(); + { let mut wz = out.write_zipper(); wz.meet_2(&a.read_zipper(), &b.read_zipper()); } + assert_eq!(out.get_val_at(&[0]), Some(&expect), "meet_2 of {:?} and {:?}", vals(a), vals(b)); + + let mut into = a.clone(); + { let mut wz = into.write_zipper(); wz.meet_into(&b.read_zipper(), false); } + assert_eq!(into.get_val_at(&[0]), Some(&expect), "meet_into of {:?} and {:?}", vals(a), vals(b)); + } + } + + /// The same for joins: `PathMap::join` and `join_into` keep the left operand's value, whether + /// the left operand is a list node joining into a byte node or the other way round. + #[test] + fn join_value_bias_is_left_regardless_of_node_layout() { + let line = mk(&[(&[1], 0)]); + let dense = mk(&[(&[0], 0), (&[1], 1), (&[2], 0)]); + assert_eq!(line.join(&dense).get_val_at(&[1]), Some(&0)); + assert_eq!(dense.join(&line).get_val_at(&[1]), Some(&1)); + + let mut into = line.clone(); + { let mut wz = into.write_zipper(); wz.join_into(&dense.read_zipper()); } + assert_eq!(vals(&into), vec![(vec![0], 0), (vec![1], 0), (vec![2], 0)]); + let mut into = dense.clone(); + { let mut wz = into.write_zipper(); wz.join_into(&line.read_zipper()); } + assert_eq!(vals(&into), vec![(vec![0], 0), (vec![1], 1), (vec![2], 0)]); + + //A deeper collision, so the child-node join is exercised as well as the value join + let line = mk(&[(&[1, 5], 0), (&[1, 6], 0)]); + let dense = mk(&[(&[0], 0), (&[1, 5], 1), (&[2], 0)]); + assert_eq!(line.join(&dense).get_val_at(&[1, 5]), Some(&0)); + assert_eq!(dense.join(&line).get_val_at(&[1, 5]), Some(&1)); + } + + /// `join_k_path_into` joins the surviving subtries in path order (`PathMap.dropHead` in the + /// Lean model folds over the k-paths in sorted order), so on a collision the value from the + /// lexicographically first k-path survives. The byte node used to fold from the highest byte + /// down, and the list node's two-slot merge used to join with the second slot on the left. + #[test] + fn join_k_path_into_keeps_lexicographically_first_value() { + let mut m = mk(&[(&[0, 0, 0, 2], 0), (&[0, 1, 0, 2], 1), (&[0, 1, 0, 2, 0], 0)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(3, false); } + assert_eq!(vals(&m), vec![(vec![2], 0), (vec![2, 0], 0)]); + + let mut m = mk(&[(&[1, 0, 3], 0), (&[0], 0), (&[0, 0, 0], 0), (&[0, 0, 3], 1)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(2, false); } + assert_eq!(vals(&m), vec![(vec![0], 0), (vec![3], 1)]); + + let mut m = mk(&[(&[0, 0, 0], 0), (&[0], 0), (&[1, 0, 0], 1)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(2, false); } + assert_eq!(vals(&m), vec![(vec![0], 0)]); + + //Three-plus branches at the root make it a byte node + let mut m = mk(&[(&[0, 0, 7], 0), (&[1, 0, 7], 1), (&[2, 0, 7], 2), (&[3, 0, 7], 3)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(2, false); } + assert_eq!(vals(&m), vec![(vec![7], 0)]); + } + + /// `LineListNode::drop_head_dyn` with both keys longer than `byte_cnt`: the shortened keys come + /// out in the opposite order ([0,0] from slot 0 and [0] from slot 1), so the slots are swapped + /// before `factor_prefix` merges them. The merge joined the swapped slot 0 (the original + /// slot 1, the later k-path) on the left, so its value won the collision at [0,0]. + #[test] + fn join_k_path_into_keeps_first_value_when_shortened_keys_reorder() { + let mut m = PathMap::::new(); + m.set_val_at(&[3u8, 0, 0, 0], 173); + m.set_val_at(&[0u8], 0); + m.set_val_at(&[3u8, 1, 0, 0], 0); + m.set_val_at(&[3u8, 1, 0, 1, 2], 82); + m.set_val_at(&[0u8, 0, 2], 196); + m.set_val_at(&[0u8, 0, 3, 1], 38); + m.write_zipper().join_k_path_into(2, false); + assert_eq!(vals(&m), vec![(vec![0, 0], 173), (vec![0, 1, 2], 82), (vec![2], 196), (vec![3, 1], 38)]); + } + #[test] fn slim_ptrs_test1() { let map = PathMap::<()>::new(); diff --git a/src/trie_ref.rs b/src/trie_ref.rs index 67b59d70..3e781e8c 100644 --- a/src/trie_ref.rs +++ b/src/trie_ref.rs @@ -101,7 +101,7 @@ fn trie_ref_from_key_and_path_in<'a, 'paths, V, A, R, RootValF, BuildF, InvalidF mut node: &'a TrieNodeODRc, root_val_f: RootValF, node_key: &'paths [u8], - mut path: &'paths [u8], + path: &'paths [u8], alloc: A, build: BuildF, invalid: InvalidF, @@ -115,31 +115,28 @@ where { // A temporary buffer on the stack, if we need to assemble a combined key from both the `node_key` and `path`. let mut temp_key_buf: [MaybeUninit; MAX_NODE_KEY_BYTES] = [MaybeUninit::uninit(); MAX_NODE_KEY_BYTES]; + let heap_key: Vec; + let mut path: &[u8] = path; let node_key_len = node_key.len(); let path_len = path.len(); // Copy the existing node key and the first chunk of the path into the temporary buffer, then try to descend one step. if node_key_len > 0 && path_len > 0 { - let next_node_path = unsafe { - // SAFETY: `temp_key_buf` has capacity for `MAX_NODE_KEY_BYTES` bytes. We copy exactly - // `node_key_len` bytes from `node_key`, which is a valid slice, then append at most the - // remaining buffer capacity from the valid slice `path`. Both destination ranges are - // within the stack buffer and do not overlap the sources. - let src_ptr = node_key.as_ptr(); - let dst_ptr = temp_key_buf.as_mut_ptr().cast::(); - core::ptr::copy_nonoverlapping(src_ptr, dst_ptr, node_key_len); - - let remaining_len = (MAX_NODE_KEY_BYTES - node_key_len).min(path_len); - let src_ptr = path.as_ptr(); - let dst_ptr = temp_key_buf.as_mut_ptr().cast::().add(node_key_len); - core::ptr::copy_nonoverlapping(src_ptr, dst_ptr, remaining_len); - - let total_buf_len = node_key_len + remaining_len; - // SAFETY: The first `total_buf_len` bytes of `temp_key_buf` were initialized by the - // copies above, and `total_buf_len <= MAX_NODE_KEY_BYTES`, so this slice is valid for - // reads for the duration of this function. - core::slice::from_raw_parts(temp_key_buf.as_mut_ptr().cast::(), total_buf_len) + let next_node_path: &[u8] = if node_key_len + path_len <= MAX_NODE_KEY_BYTES { + unsafe { + // SAFETY: `temp_key_buf` holds `MAX_NODE_KEY_BYTES` bytes and we copy + // `node_key_len + path_len <= MAX_NODE_KEY_BYTES` bytes from two valid slices into it, + // so the resulting slice is initialized and in bounds. + let dst_ptr = temp_key_buf.as_mut_ptr().cast::(); + core::ptr::copy_nonoverlapping(node_key.as_ptr(), dst_ptr, node_key_len); + core::ptr::copy_nonoverlapping(path.as_ptr(), dst_ptr.add(node_key_len), path_len); + core::slice::from_raw_parts(dst_ptr, node_key_len + path_len) + } + } else { + //Too long for the stack buffer + heap_key = [node_key, path].concat(); + &heap_key }; match node.as_tagged().node_get_child(next_node_path) { @@ -455,7 +452,7 @@ impl ZipperConcrete for TrieRefBor } fn is_shared(&self) -> bool { match self.focus_node { - Some(node) => self.node_key().is_empty() && node.refcount() > 1, + Some(node) => self.node_key().is_empty() && !node.is_empty() && node.refcount() > 1, None => false, } } @@ -782,7 +779,7 @@ impl ZipperConcrete for TrieRefOwn } fn is_shared(&self) -> bool { match &self.focus_node { - Some(node) => self.node_key().is_empty() && node.refcount() > 1, + Some(node) => self.node_key().is_empty() && !node.is_empty() && node.refcount() > 1, None => false } } @@ -1448,4 +1445,47 @@ mod tests { assert!(!trie_ref.is_shared()); assert_eq!(trie_ref.shared_node_id(), None); } + + /// `val_at` and `trie_ref_at_path` where the focus key plus the path are longer than a node key + #[test] + fn trie_ref_long_node_key_and_path() { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8; 70], 5); + map.set_val_at(&[1u8], 6); + for (focus, rest) in [(10usize, 60usize), (30, 40), (47, 23), (69, 1)] { + let mut rz = map.read_zipper(); + rz.descend_to(&vec![0u8; focus]); + assert_eq!(rz.val_at(&vec![0u8; rest]), Some(&5), "{focus}+{rest}"); + assert_eq!(rz.trie_ref_at_path(&vec![0u8; rest]).val(), Some(&5), "{focus}+{rest}"); + assert_eq!(rz.val_at(&vec![0u8; rest + 1]), None, "{focus}+{rest}"); + } + //A focus far below anything in the trie + let mut rz = map.read_zipper(); + rz.descend_to(&[7u8; 60]); + assert_eq!(rz.val_at(&[1u8]), None); + assert_eq!(rz.val_at(&[7u8; 60]), None); + } + + /// `is_shared` where the focus is the empty sentinel node + #[test] + fn trie_ref_is_shared_on_empty_node() { + let mut map = PathMap::::new(); + map.set_val_at(&[1u8], 1); + map.remove_val_at(&[1u8], false); + let empty = PathMap::::new(); + for m in [&map, &empty] { + for path in [&[][..], &[1u8][..]] { + let t = m.trie_ref_at_path(path); + let _ = (t.is_shared(), t.shared_node_id()); + } + } + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 3], 1); + { + let mut wz = src.write_zipper_at_path(&[2u8]); + wz.remove_branches(false); + } + let t = src.trie_ref_at_path(&[2u8]); + let _ = (t.is_shared(), t.shared_node_id()); + } } diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 0892e475..2b24156a 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -1343,6 +1343,10 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC /// Internal method to re-borrow a WriteZipperCore without the `'path` lifetime fn as_static_path_zipper(&mut self) -> &mut WriteZipperCore<'a, 'static, V, A> { self.prepare_buffers(); + //The path is in `prefix_buf` now, so drop the borrowed copy + if self.key.origin_path.len() > 0 { + self.key.origin_path = SliceOrLen::new_owned(self.key.origin_path.len()); + } debug_assert!(!self.key.origin_path.is_slice() || self.key.origin_path.len() == 0); unsafe{ &mut *(self as *mut WriteZipperCore).cast() } } @@ -1552,6 +1556,26 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC ; *focus_node = replacement; } + /// One masked branch of [Self::graft_masked_branches]: `self`'s branch at `byte` becomes the + /// source's, value included. + /// + /// When the source has nothing at `byte` -- no value and no node with contents, a dangling + /// source branch included -- grafting nothing removes the branch's contents and value, but + /// the location is neither created nor destroyed: an existing branch survives as a dangling + /// path (as after `graft` of an empty source, or the model's `graftBelow` + `removeVal`), + /// and an absent one stays absent. + fn graft_masked_branch>(&mut self, src: &Z, byte: u8) { + let src_node = src.get_focus_at([byte]); + let src_has_node = !src_node.is_none() && !src_node.as_tagged().node_is_empty(); + self.descend_to_byte(byte); + if src_has_node || src.val_at([byte]).is_some() { + self.graft_src_at(src, [byte]); + } else if self.path_exists() { + self.remove_branches(false); + self.remove_val(false); + } + self.ascend_byte(); + } /// See [ZipperWriting::graft_masked_branches] pub fn graft_masked_branches>(&mut self, src: &Z, child_mask: ByteMask, remove_unset: bool) { // The dense-node merge handles both pieces of the contract directly: it removes @@ -1570,9 +1594,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } // SAFETY: this arm is selected only when `child_mask` has one bit. let byte = unsafe { child_mask.indexed_bit::(0).unwrap_unchecked() }; - self.descend_to_byte(byte); - self.graft_src_at(src, &[byte]); - self.ascend_byte(); + self.graft_masked_branch(src, byte); } 2 => { if remove_unset { @@ -1580,15 +1602,11 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } // SAFETY: this arm is selected only when `child_mask` has two bits. let first_byte = unsafe { child_mask.indexed_bit::(0).unwrap_unchecked() }; - self.descend_to_byte(first_byte); - self.graft_src_at(src, &[first_byte]); - self.ascend_byte(); + self.graft_masked_branch(src, first_byte); // SAFETY: `first_byte` is one of the two set bits, so it has a successor. let second_byte = unsafe { child_mask.next_bit(first_byte).unwrap_unchecked() }; - self.descend_to_byte(second_byte); - self.graft_src_at(src, &[second_byte]); - self.ascend_byte(); + self.graft_masked_branch(src, second_byte); } _ => { let src_focus = src.get_focus(); @@ -1599,11 +1617,18 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC // to hand out) has no node to merge into even after the split, so the // branches are merged into a fresh node that is grafted in afterwards. // This used to unwrap the missing node. + // + // A focus that does not exist at all is not split either: the split + // would create the path, and grafting nothing must not create a + // location. Its branches also go into a fresh node, which is grafted + // (creating the path) only if the merge produced something. let mut fresh_node: Option> = None; let self_focus_node = match self.try_borrow_focus_mut() { Some(node) => node, None => { - self.split_at_focus(); + if self.path_exists() { + self.split_at_focus(); + } match self.try_borrow_focus_mut() { Some(node) => node, None => { @@ -1653,7 +1678,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC if remove_unset { self.remove_branches(false); } else { - self.remove_unmasked_branches(child_mask.not(), false); + self.empty_masked_branches(child_mask); } }, } @@ -1667,7 +1692,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC if remove_unset { self.remove_branches(false); } else { - self.remove_unmasked_branches(child_mask.not(), false); + self.empty_masked_branches(child_mask); } } } @@ -1675,6 +1700,21 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } } + /// [Self::graft_masked_branches] when the source has no node at all to take branches from: + /// every existing branch of `self` named by `mask` is emptied -- contents and value -- but + /// survives as a dangling path, and no branch is created. This is what grafting nothing + /// onto each branch amounts to; removing the branches outright would prune locations that + /// `graft` of an empty source would have left standing. + fn empty_masked_branches(&mut self, mask: ByteMask) { + let existing = mask & self.child_mask(); + for byte in existing.iter() { + self.descend_to_byte(byte); + self.remove_branches(false); + self.remove_val(false); + self.ascend_byte(); + } + } + /// Optimized implementation of [ZipperWriting::graft_child_maps] for WriteZipperCore /// /// This implementation constructs a new node with the appropriate children directly, @@ -1693,7 +1733,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC for child_byte in child_mask.iter() { let map = maps_iter.next().expect("maps iterator returned fewer items than the number of set bits in child_mask"); let (src_root_node, src_root_val) = map.into_root(); - if let Some(node) = src_root_node { + if let Some(node) = src_root_node.filter(|n| !n.as_tagged().node_is_empty()) { new_node.set_child(child_byte, node); } if let Some(val) = src_root_val { @@ -1701,7 +1741,14 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } } let new_node_odrc = TrieNodeODRc::new_in(new_node, self.alloc.clone()); - self.graft_internal(Some(new_node_odrc)); + if new_node_odrc.as_tagged().node_is_empty() { + // Every map was empty: all that is left of the contract is "remove the unset + // branches", and that must not create the focus if it does not exist. Grafting + // an empty node would leave a dangling path made out of nothing. + self.remove_branches(false); + } else { + self.graft_internal(Some(new_node_odrc)); + } } else { // If we don't have enough children to justify forcing a new ByteNode, just set the nodes if remove_unset { @@ -1712,7 +1759,21 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC let map = maps_iter.next().expect("maps iterator returned fewer items than the number of set bits in child_mask"); let (src_root_node, src_root_val) = map.into_root(); - if let Some(node) = src_root_node { + // Each branch is *replaced* by its map, the way `graft` replaces a subtrie: + // whatever `self` had at `child_byte` -- children and value -- goes first, so an + // empty map empties the branch and a map without a root value removes the value. + // The location itself is neither created nor destroyed (an existing branch that + // receives nothing survives as a dangling path, as after `graft` of an empty + // source). With `remove_unset` everything below the focus is already gone. + if !remove_unset { + self.descend_to_byte(child_byte); + if self.path_exists() { + self.remove_branches(false); + self.remove_val(false); + } + self.ascend_byte(); + } + if let Some(node) = src_root_node.filter(|n| !n.as_tagged().node_is_empty()) { self.set_node_at_child_path(&[child_byte], node) } if let Some(val) = src_root_val { @@ -1916,8 +1977,21 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } else { PathMap::new_in(self.alloc.clone()) }; + //`prune` drops the dangling paths from the meet; the focus itself is never removed + let temp_map = if prune { + let alloc = temp_map.alloc.clone(); + let (root, root_val) = temp_map.into_root(); + let root = root.and_then(|root| match node_drop_dangling(&root, None) { + DropDangling::Unchanged => Some(root), + DropDangling::Empty => None, + DropDangling::New(new_root) => Some(new_root), + }); + PathMap::new_with_root_in(root, root_val, alloc) + } else { + temp_map + }; if temp_map.is_empty() { - self.remove_branches(prune); + self.remove_branches(false); false } else { self.graft_map(temp_map); @@ -1989,6 +2063,8 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } /// See [ZipperWriting::meet_into] pub fn meet_into>(&mut self, read_zipper: &Z, prune: bool) -> AlgebraicStatus where V: Lattice { + //The focus is never removed, with or without `prune`: only what is below it can change, + // along with the focus value let src_root_val = read_zipper.val(); #[cfg(not(feature = "graft_root_vals"))] let _ = src_root_val; @@ -1997,60 +2073,77 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC (Some(self_val), Some(src_val)) => { let new_status = match self_val.pmeet(src_val) { AlgebraicResult::Element(new_val) => {self.set_val(new_val); AlgebraicStatus::Element }, - AlgebraicResult::None => {self.remove_val(prune); AlgebraicStatus::None }, + AlgebraicResult::None => {self.remove_val(false); AlgebraicStatus::None }, AlgebraicResult::Identity(_) => { AlgebraicStatus::Identity } }; (new_status, false) }, (None, Some(_)) => { (AlgebraicStatus::None, true) }, - (Some(_), None) => { self.remove_val(prune); (AlgebraicStatus::None, false) }, + (Some(_), None) => { self.remove_val(false); (AlgebraicStatus::None, false) }, (None, None) => { (AlgebraicStatus::None, true) }, }; - let node_was_none; - let node_status = match self.get_focus().try_as_tagged() { - Some(self_node) => { - if !self_node.node_is_empty() { - node_was_none = false; - let src = read_zipper.get_focus(); - if src.is_none() { - self.graft_internal(None); + let self_focus = self.get_focus(); + let node_was_none = match self_focus.try_as_tagged() { + Some(self_node) => self_node.node_is_empty(), + None => true + }; + let node_status = if node_was_none { + AlgebraicStatus::None + } else { + let src = read_zipper.get_focus(); + let result = match src.try_as_tagged() { + Some(src_node) => self_focus.as_tagged().pmeet_dyn(src_node), + None => AlgebraicResult::None, + }; + //With `prune`, only the locations on the way to a value are kept. A node shared with the + // source may be left as it is. + let drop_dangling = |node: TrieNodeODRc, src: Option>| -> Option> { + match node_drop_dangling(&node, src) { + DropDangling::Unchanged => Some(node), + DropDangling::Empty => None, + DropDangling::New(new_node) => Some(new_node), + } + }; + let (unchanged, new_node) = match result { + AlgebraicResult::Element(intersection) => { + (false, if prune { drop_dangling(intersection, src.try_as_tagged()) } else { Some(intersection) }) + }, + AlgebraicResult::None => (false, None), + AlgebraicResult::Identity(mask) => { + if mask & SELF_IDENT > 0 { if prune { - self.prune_path(); + let self_rc = self_focus.into_option().unwrap(); + match node_drop_dangling(&self_rc, src.try_as_tagged()) { + DropDangling::Unchanged => (true, None), + DropDangling::Empty => (false, None), + DropDangling::New(new_node) => (false, Some(new_node)), + } + } else { + (true, None) } - AlgebraicStatus::None } else { - match self_node.pmeet_dyn(src.as_tagged()) { - AlgebraicResult::Element(intersection) => { - self.graft_internal(Some(intersection)); - AlgebraicStatus::Element - }, - AlgebraicResult::None => { - self.graft_internal(None); - if prune { - self.prune_path(); - } - AlgebraicStatus::None - }, - AlgebraicResult::Identity(mask) => { - if mask & SELF_IDENT > 0 { - AlgebraicStatus::Identity - } else { - debug_assert_eq!(mask, COUNTER_IDENT); //It's gotta be self or other - self.graft_internal(Some(src.into_option().unwrap())); - AlgebraicStatus::Element - } - }, - } + debug_assert_eq!(mask, COUNTER_IDENT); //It's gotta be self or other + //The source's own node is shared with the source, so pruning may skip it + let src_is_shared = matches!(src.0, AbstractNodeRef::BorrowedRc(_)); + let src_rc = src.into_option().unwrap(); + (false, if prune && !src_is_shared { drop_dangling(src_rc, None) } else { Some(src_rc) }) + } + }, + }; + if unchanged { + AlgebraicStatus::Identity + } else { + match new_node { + Some(new_node) => { + self.graft_internal(Some(new_node)); + AlgebraicStatus::Element + }, + None => { + self.graft_internal(None); + AlgebraicStatus::None } - } else { - node_was_none = true; - AlgebraicStatus::None } - }, - None => { - node_was_none = true; - AlgebraicStatus::None } }; @@ -2059,6 +2152,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC #[cfg(feature = "graft_root_vals")] return node_status.merge(val_status, node_was_none, val_was_none) } + /// See [WriteZipper::meet_2] pub fn meet_2, ZB: ZipperInfallibleSubtries>(&mut self, rz_a: &ZA, rz_b: &ZB) -> AlgebraicStatus where V: Lattice { let a_focus = rz_a.get_focus(); @@ -2269,9 +2363,26 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } /// See [WriteZipper::remove_unmasked_branches] pub fn remove_unmasked_branches(&mut self, mask: ByteMask, prune: bool) { - let mut focus_node = self.focus_stack.top_mut().unwrap(); let node_key = self.key.node_key(); - if node_key.len() > 0 { + + //`get_child_mut` declines to hand out an empty child node, so a focus that sits at or + // below such a dangling stub never descends and lands in the `None` arm below, where the + // node would mistake the stub's own key for one of its branches. (`LineListNode` asserts + // on exactly that.) A stub has nothing below it, so there are no branches to filter, and + // the dangling path itself must survive. + let below_dangling_stub = node_key.len() > 0 + && match self.focus_stack.top() { + Some(focus_node) => match focus_node.node_get_child(node_key) { + Some((_consumed_bytes, child_node)) => child_node.is_empty(), + None => false + }, + None => false + }; + + let mut focus_node = self.focus_stack.top_mut().unwrap(); + if below_dangling_stub { + //Nothing to do + } else if node_key.len() > 0 { match focus_node.node_get_child_mut(node_key) { Some((consumed_bytes, child_node)) => { if node_key.len() >= consumed_bytes { @@ -2439,28 +2550,37 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC RetryF: FnOnce(&mut TaggedNodeRefMut<'_, V, A>, &[u8]) -> R, { let key = self.key.node_key(); - let mut focus_node = self.focus_stack.top_mut().unwrap(); - if let Some((key_bytes, child_node)) = focus_node.node_get_child_mut(key) { - debug_assert_eq!(key_bytes, key.len()); - let (key, node) = node_along_path_mut(child_node, path, true); - let mut node_ref = node.make_mut(); - match node_f(&mut node_ref, key) { - Ok(result) => result, - Err(replacement_node) => { - *node = replacement_node; - retry_f(&mut node.make_mut(), key) - }, + // At the root the focus node *is* the top of the stack and there is no key to look up; + // `node_get_child_mut` must never see an empty key (a `DenseByteNode` indexes `key[0]`). + if key.len() > 0 { + let mut focus_node = self.focus_stack.top_mut().unwrap(); + if let Some((key_bytes, child_node)) = focus_node.node_get_child_mut(key) { + debug_assert_eq!(key_bytes, key.len()); + let (key, node) = node_along_path_mut(child_node, path, true); + let mut node_ref = node.make_mut(); + return match node_f(&mut node_ref, key) { + Ok(result) => result, + Err(replacement_node) => { + *node = replacement_node; + retry_f(&mut node.make_mut(), key) + }, + } } - } else { + } + { self.in_zipper_mut_static_result( |focus_node, partial_key| { - let mut key_buf = [0u8; MAX_NODE_KEY_BYTES]; - key_buf[0..partial_key.len()].copy_from_slice(partial_key); - //GOAT, currently this will panic if the path is too long to fit in the buffer, which means this internal API - // isn't suitable for general-purpose path-based ops yet, but we're using it to deal with single-byte ops - key_buf[partial_key.len()..partial_key.len()+path.len()].copy_from_slice(path); - let full_key = &key_buf[0..partial_key.len()+path.len()]; - node_f(focus_node, full_key) + let full_len = partial_key.len() + path.len(); + if full_len <= MAX_NODE_KEY_BYTES { + let mut key_buf = [0u8; MAX_NODE_KEY_BYTES]; + key_buf[0..partial_key.len()].copy_from_slice(partial_key); + key_buf[partial_key.len()..full_len].copy_from_slice(path); + node_f(focus_node, &key_buf[0..full_len]) + } else { + //Too long for the stack buffer + let full_key = [partial_key, path].concat(); + node_f(focus_node, &full_key) + } }, retry_f ) @@ -2572,6 +2692,12 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC if should_ascend { self.key.prefix_buf.truncate(temp_path.len()); + } else if ascended { + //The loop above walked the node stack up to where pruning stopped, but the zipper has not + // moved. Walk it back down towards the focus, as far as nodes still exist, or the stack + // and `prefix_idx` describe an ancestor while the path still names the focus, and the next + // write through the focus lands in the wrong node + self.descend_to_internal(); } pruned_bytes @@ -3687,7 +3813,9 @@ mod tests { assert_eq!(btm.path_exists_at(&[1, 255, 0]), true); assert_eq!(btm.path_exists_at(&[0, 255, 0]), true); - // Test 3: meet from a higher level with all dangling paths and prune=true + // Test 3: meet from a higher level with all dangling paths and prune=true. With prune, only + // the locations on the way to a value survive, so the dangling path both sides have goes + // too, leaving nothing below the focus; the focus itself stays. let mut btm2: PathMap<()> = PathMap::new(); btm2.create_path(&[0, 255, 0]); btm2.create_path(&[0, 255, 1]); @@ -3698,16 +3826,456 @@ mod tests { let mut wz = zh2.write_zipper_at_exclusive_path(&[0]).unwrap(); let rz = zh2.read_zipper_at_path(&[1]).unwrap(); let alg_result = wz.meet_into(&rz, true); - assert_eq!(alg_result, AlgebraicStatus::Element); + assert_eq!(alg_result, AlgebraicStatus::None); drop(wz); drop(rz); drop(zh2); // Verify the meet operation did what it should have assert_eq!(btm2.path_exists_at(&[1, 255, 0]), true); - assert_eq!(btm2.path_exists_at(&[0, 255, 0]), true); + assert_eq!(btm2.path_exists_at(&[0]), true); + assert_eq!(btm2.path_exists_at(&[0, 255]), false); assert_eq!(btm2.path_exists_at(&[0, 200, 5]), false); assert_eq!(btm2.path_exists_at(&[0, 255, 1]), false); + + // Test 4: the same without prune keeps the path both sides have, and only that + let mut btm3: PathMap<()> = PathMap::new(); + btm3.create_path(&[0, 255, 0]); + btm3.create_path(&[0, 255, 1]); + btm3.create_path(&[0, 200, 5]); + btm3.create_path(&[1, 255, 0]); + let zh3 = btm3.zipper_head(); + + let mut wz = zh3.write_zipper_at_exclusive_path(&[0]).unwrap(); + let rz = zh3.read_zipper_at_path(&[1]).unwrap(); + assert_eq!(wz.meet_into(&rz, false), AlgebraicStatus::Element); + drop(wz); + drop(rz); + drop(zh3); + assert_eq!(btm3.path_exists_at(&[0, 255, 0]), true); + assert_eq!(btm3.path_exists_at(&[0, 200]), false); + assert_eq!(btm3.path_exists_at(&[0, 255, 1]), false); + } + + /// Every existing location in `map` -- dangling paths included -- with its value, in + /// depth-first order + fn all_locations(map: &PathMap) -> Vec<(Vec, Option)> { + let mut rz = map.read_zipper(); + let mut locations = vec![]; + loop { + locations.push((rz.path().to_vec(), rz.val().cloned())); + if !rz.to_next_step() { break } + } + locations + } + + /// A DenseByteNode subtracted by a node of another type (list or tiny) must drop a dangling + /// path the source reaches, and must not report `Identity` for having done so. Covers both + /// shapes a dangling slot takes in a dense node: an empty onward link, and a CoFree holding + /// neither a link nor a value. + #[test] + fn write_zipper_subtract_into_dense_drops_reached_dangling_path() { + // Empty onward link at [2]; the source (a LineListNode) reaches [2] + let mut dst = PathMap::::new(); + for b in [1u8, 3, 4] { dst.set_val_at(&[b], 1); } + dst.create_path(&[2u8]); + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 0, 1], 246); + let mut wz = dst.write_zipper(); + assert_eq!(wz.subtract_into(&src.read_zipper(), false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![(vec![], None), (vec![1], Some(1)), (vec![3], Some(1)), (vec![4], Some(1))]); + + // A dangling [2] left by a meet that came to nothing, next to a value-and-link at [0]; the + // source is a TinyRefNode reaching [2] + let mut dst = PathMap::::new(); + dst.set_val_at(&[2u8, 0, 0, 0, 0], 0); + dst.set_val_at(&[0u8, 0, 0, 0, 0], 0); + dst.set_val_at(&[0u8], 0); + let mut srcs = PathMap::::new(); + srcs.set_val_at(&[1u8, 2, 0], 0); + let mut wz = dst.write_zipper(); + wz.descend_to_byte(2); + let ra = srcs.read_zipper_at_path(&[1u8]); + let rb = srcs.read_zipper_at_path(&[1u8, 0]); + assert_eq!(wz.meet_2(&ra, &rb), AlgebraicStatus::None); + wz.reset(); + assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[2u8]); probe.path_exists() }, true, "the meet should leave [2] dangling"); + assert_eq!(wz.subtract_into(&ra, false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![ + (vec![], None), (vec![0], Some(0)), (vec![0, 0], None), (vec![0, 0, 0], None), + (vec![0, 0, 0, 0], None), (vec![0, 0, 0, 0, 0], Some(0)), + ]); + + // `graft_child_maps` of an empty map leaves a dangling [0] beside [1, 0] + let mut dst = PathMap::::new(); + dst.set_val_at(&[], 0); + dst.set_val_at(&[0u8], 0); + dst.set_val_at(&[0u8, 0], 0); + dst.set_val_at(&[1u8, 0], 0); + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 3, 0, 0], 0); + let mut wz = dst.write_zipper(); + wz.graft_child_maps(ByteMask::from_iter([0u8]), vec![PathMap::new()], false); + assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[0u8]); probe.path_exists() }, true, "the graft should leave [0] dangling"); + let rz = src.read_zipper_at_path(&[2u8, 3]); + assert_eq!(wz.subtract_into(&rz, false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![(vec![], Some(0)), (vec![1], None), (vec![1, 0], Some(0))]); + } + + /// `prune_path` does not move the zipper, but it used to leave the node stack where its upward + /// walk stopped. When the focus sat in a child node -- as it does after a `graft` at the zipper's + /// root -- the stack then described an ancestor while the path still named the focus, and the next + /// write through the focus went to the wrong node: `get_val_or_set_mut` panicked on the value it + /// had just set. `meet_into(.., true)` reached this through its own `prune_path`. + #[test] + fn write_zipper_write_after_prune_path_below_a_graft() { + let build = || { + let mut m0 = PathMap::::new(); + let mut m1 = PathMap::::new(); + m1.set_val_at(&[1u8, 0, 0, 0, 0], 7); + m0.create_path(&[0u8, 0]); + m1.create_path(&[1u8]); + (m0, m1) + }; + + // prune_path directly + let (mut m0, m1) = build(); + { + let mut wz = m0.write_zipper_at_path(&[0u8, 0]); + let rz = m1.read_zipper_at_path(&[1u8]); + wz.graft(&rz); + wz.descend_last_byte(); + wz.remove_branches(false); + wz.prune_path(); + assert_eq!(wz.path(), &[0u8]); + assert_eq!(*wz.get_val_or_set_mut_with(|| 3), 3); + assert_eq!(wz.val(), Some(&3)); + } + assert_eq!(all_locations(&m0), vec![(vec![], None), (vec![0], None), (vec![0, 0], None), (vec![0, 0, 0], Some(3))]); + + // through meet_into with prune + let (mut m0, m1) = build(); + { + let mut wz = m0.write_zipper_at_path(&[0u8, 0]); + let rz = m1.read_zipper_at_path(&[1u8]); + wz.graft(&rz); + wz.descend_last_byte(); + wz.meet_into(&rz, true); + assert_eq!(wz.path(), &[0u8]); + assert_eq!(*wz.get_val_or_set_mut_with(|| 3), 3); + assert_eq!(wz.val(), Some(&3)); + } + assert_eq!(m0.get_val_at(&[0u8, 0, 0]), Some(&3)); + } + + /// The meet rule, dangling paths included. + /// + /// With `prune = false` a path exists in the result exactly when it exists in both operands, and + /// a value exists exactly where both hold one, as the meet of the two. So a meet with an equal + /// trie -- a clone that shares its nodes, or an independent copy -- changes nothing and reports + /// `Identity`, whatever node types the two sides use. + /// + /// With `prune = true` the values are the same, but only the locations on the way to a value + /// survive: every dangling path is dropped, including one both operands had -- except that a node + /// shared with the source may be skipped rather than walked, so a dangling path inside it may + /// survive. The focus itself is never removed. + #[test] + fn write_zipper_meet_into_dangling_paths() { + type Build = fn() -> PathMap; + type Locations = Vec<(Vec, Option)>; + + // meet({[0,0]:-}, {[0,1]:-}) -> {[0]:-}, and with prune nothing but the root + let a = || { let mut m = PathMap::::new(); m.create_path(&[0u8, 0]); m }; + let b = || { let mut m = PathMap::::new(); m.create_path(&[0u8, 1]); m }; + assert_eq!(all_locations(&a().meet(&b())), vec![(vec![], None), (vec![0], None)]); + let mut d = a(); + assert_eq!(d.write_zipper().meet_into(&b().read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![0], None)]); + let mut d = a(); + assert_eq!(d.write_zipper().meet_into(&b().read_zipper(), true), AlgebraicStatus::None); + assert_eq!(all_locations(&d), vec![(vec![], None)]); + + // A dangling path met against a value keeps the path without the value, in both orders + let left = || { let mut m = PathMap::::new(); m.create_path([7u8, 1, 0]); m }; + let right = || { let mut m = PathMap::::new(); m.set_val_at([7u8, 1, 0], 10); m.set_val_at([7u8, 2, 0], 20); m.create_path([7u8, 3]); m }; + let expected: Locations = vec![(vec![], None), (vec![7], None), (vec![7, 1], None), (vec![7, 1, 0], None)]; + assert_eq!(all_locations(&left().meet(&right())), expected); + assert_eq!(all_locations(&right().meet(&left())), expected); + + // Meeting an equal trie: unchanged without prune, dangling paths dropped with it + let equal_cases: [(&str, Build, Locations); 3] = [ + ("create_path", || { let mut m = PathMap::new(); m.create_path(&[1u8]); m }, vec![(vec![], None)]), + ("value removed", || { let mut m = PathMap::new(); m.set_val_at(&[1u8], 5); m.remove_val_at(&[1u8], false); m }, vec![(vec![], None)]), + ("dangling beside values", || { + let mut m = PathMap::new(); + m.set_val_at(&[1u8, 2], 5); + m.set_val_at(&[1u8, 4, 4, 4], 5); + m.set_val_at(&[9u8], 5); + m.write_zipper_at_path(&[1u8, 4]).remove_branches(false); + m + }, vec![(vec![], None), (vec![1], None), (vec![1, 2], Some(5)), (vec![9], Some(5))]), + ]; + for (name, build, pruned) in equal_cases { + let expected = all_locations(&build()); + let mut shared = build(); + let clone = shared.clone(); + assert_eq!(shared.write_zipper().meet_into(&clone.read_zipper(), false), AlgebraicStatus::Identity, "{name}: meet with a clone"); + assert_eq!(all_locations(&shared), expected, "{name}: meet with a clone"); + let mut unshared = build(); + assert_eq!(unshared.write_zipper().meet_into(&build().read_zipper(), false), AlgebraicStatus::Identity, "{name}: meet with a copy"); + assert_eq!(all_locations(&unshared), expected, "{name}: meet with a copy"); + assert_eq!(all_locations(&build().meet(&build())), expected, "{name}: PathMap::meet"); + + // With a clone every node is shared, so pruning may skip all of it: the result lies + // between fully pruned and not pruned at all, with the values unchanged + let mut shared = build(); + let clone = shared.clone(); + shared.write_zipper().meet_into(&clone.read_zipper(), true); + let got = all_locations(&shared); + assert!(pruned.iter().all(|l| got.contains(l)) && got.iter().all(|l| expected.contains(l)), + "{name}: pruned meet with a clone: {got:?} is not between {pruned:?} and {expected:?}"); + let mut unshared = build(); + unshared.write_zipper().meet_into(&build().read_zipper(), true); + assert_eq!(all_locations(&unshared), pruned, "{name}: pruned meet with a copy"); + } + + // Pruning never removes the focus, below the root or at a zipper's root + let dst = || { let mut m = PathMap::::new(); m.set_val_at(&[9u8], 9); m.create_path(&[5u8, 0]); m }; + let src = || { let mut m = PathMap::::new(); m.create_path(&[5u8, 1]); m }; + let mut d = dst(); + let s = src(); + { let mut wz = d.write_zipper(); wz.descend_to(&[5u8]); assert_eq!(wz.meet_into(&s.read_zipper_at_path(&[5u8]), true), AlgebraicStatus::None); } + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![5], None), (vec![9], Some(9))]); + let mut d = dst(); + { let mut wz = d.write_zipper_at_path(&[5u8]); assert_eq!(wz.meet_into(&s.read_zipper_at_path(&[5u8]), true), AlgebraicStatus::None); } + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![5], None), (vec![9], Some(9))]); + + // A dangling [2] in the destination, against sources of different node types + let dense_dst: Build = || { let mut d = PathMap::new(); for b in [1u8, 3, 4] { d.set_val_at(&[b], 1); } d.create_path(&[2u8]); d }; + let list_dst: Build = || { let mut d = PathMap::new(); d.set_val_at(&[1u8], 1); d.create_path(&[2u8]); d }; + let list_src_below_2: Build = || { let mut s = PathMap::new(); s.set_val_at(&[1u8], 1); s.set_val_at(&[2u8, 0, 1], 246); s }; + let dense_src_below_2: Build = || { let mut s = PathMap::new(); for b in [1u8, 3, 4] { s.set_val_at(&[b], 1); } s.set_val_at(&[2u8, 0, 1], 246); s }; + let dense_src_without_2: Build = || { let mut s = PathMap::new(); for b in [1u8, 3, 4] { s.set_val_at(&[b], 1); } s }; + let dense_all: Locations = vec![(vec![], None), (vec![1], Some(1)), (vec![2], None), (vec![3], Some(1)), (vec![4], Some(1))]; + let cases: [(&str, Build, Build, AlgebraicStatus, Locations); 6] = [ + ("list dst, list src with [2]", list_dst, list_src_below_2, AlgebraicStatus::Identity, + vec![(vec![], None), (vec![1], Some(1)), (vec![2], None)]), + ("list dst, list src dangling at [2]", list_dst, list_dst, AlgebraicStatus::Identity, + vec![(vec![], None), (vec![1], Some(1)), (vec![2], None)]), + ("dense dst, dense src with [2]", dense_dst, dense_src_below_2, AlgebraicStatus::Identity, dense_all.clone()), + ("dense dst, dense src dangling at [2]", dense_dst, dense_dst, AlgebraicStatus::Identity, dense_all), + ("dense dst, list src with [2]", dense_dst, list_src_below_2, AlgebraicStatus::Element, + vec![(vec![], None), (vec![1], Some(1)), (vec![2], None)]), + ("dense dst, src without [2]", dense_dst, dense_src_without_2, AlgebraicStatus::Element, + vec![(vec![], None), (vec![1], Some(1)), (vec![3], Some(1)), (vec![4], Some(1))]), + ]; + let mut failures = vec![]; + for (name, dst, src, status, expected) in cases { + let mut d = dst(); + let s = src(); + let st = d.write_zipper().meet_into(&s.read_zipper(), false); + let got = all_locations(&d); + let whole = all_locations(&dst().meet(&src())); + if st != status || got != expected || whole != expected { + failures.push(format!("{name}: meet_into {st:?} {got:?}, PathMap::meet {whole:?}; expected {status:?} {expected:?}")); + } + } + assert!(failures.is_empty(), "\n{}", failures.join("\n")); + } + + /// The tag of the root node of `map` + fn root_tag(map: &PathMap) -> usize { + map.root().unwrap().as_tagged().tag() + } + + /// Two dense nodes whose co-frees at a byte meet to nothing still share the byte, so it survives + /// the meet as a dangling path. A dangling byte -- a co-free with neither a value nor an onward + /// node, or one whose onward node is empty -- is what such a meet leaves, so it meets anything at + /// that byte as an identity of the dangling side. + #[test] + fn write_zipper_meet_into_dense_keeps_bytes_that_meet_to_nothing() { + type Build = fn() -> PathMap; + type Locations = Vec<(Vec, Option)>; + const BYTES: [u8; 4] = [1, 2, 3, 4]; + let values_at_0: Build = || { let mut m = PathMap::::new(); for b in BYTES { m.set_val_at(&[b, 0], 1); } m }; + let values_at_1: Build = || { let mut m = PathMap::::new(); for b in BYTES { m.set_val_at(&[b, 1], 2); } m }; + let dangling: Build = || { let mut m = PathMap::::new(); for b in BYTES { m.create_path(&[b]); } m }; + let emptied: Build = || { + let mut m = PathMap::::new(); + for b in BYTES { m.set_val_at(&[b, 0], 1); } + for b in BYTES { m.write_zipper_at_path(&[b]).remove_branches(false); } + m + }; + for build in [values_at_0, values_at_1, dangling, emptied] { + assert_eq!(root_tag(&build()), DENSE_BYTE_NODE_TAG); + } + let bare: Locations = core::iter::once((vec![], None)).chain(BYTES.iter().map(|b| (vec![*b], None))).collect(); + assert_eq!(all_locations(&dangling()), bare); + assert_eq!(all_locations(&emptied()), bare); + + // The values below each byte meet to nothing; the bytes stay + assert_eq!(all_locations(&values_at_0().meet(&values_at_1())), bare); + let mut dst = values_at_0(); + assert_eq!(dst.write_zipper().meet_into(&values_at_1().read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&dst), bare); + + // A dangling side is an identity for the meet, whichever form the dangling byte takes + for dangling_side in [dangling, emptied] { + for other in [values_at_0, values_at_1, dangling, emptied] { + let mut dst = dangling_side(); + assert_eq!(dst.write_zipper().meet_into(&other().read_zipper(), false), AlgebraicStatus::Identity); + assert_eq!(all_locations(&dst), bare); + let mut dst = other(); + let expected = if all_locations(&dst) == bare { AlgebraicStatus::Identity } else { AlgebraicStatus::Element }; + assert_eq!(dst.write_zipper().meet_into(&dangling_side().read_zipper(), false), expected); + assert_eq!(all_locations(&dst), bare); + } + } + } + + /// A list node meets another node slot by slot, and each slot keeps the deepest prefix of its + /// key that the other side also has -- through onward links of any node type, and whether the + /// slot holds a value or an onward node. The result must not depend on which side is the list + /// node, so every pair is checked in both orders. + #[test] + fn write_zipper_meet_into_list_keeps_shared_key_prefix() { + type Build = fn() -> PathMap; + type Locations = Vec<(Vec, Option)>; + let list_val: Build = || { let mut m = PathMap::new(); m.set_val_at(&[5u8, 6, 7], 1); m }; + let list_child: Build = || { let mut m = PathMap::new(); m.set_val_at(&[5u8, 6, 7, 8, 9], 1); m }; + let list_dangling: Build = || { let mut m = PathMap::new(); m.create_path(&[5u8, 6]); m }; + let dense_dangling: Build = || { let mut m = PathMap::new(); for b in [1u8, 2, 3] { m.set_val_at(&[b], 3); } m.create_path(&[5u8, 6]); m }; + let dense_branch: Build = || { let mut m = PathMap::new(); for b in [1u8, 2, 3] { m.set_val_at(&[b], 3); } m.set_val_at(&[5u8, 6, 0], 4); m }; + let dense_value: Build = || { let mut m = PathMap::new(); for b in [1u8, 2, 3] { m.set_val_at(&[b], 3); } m.set_val_at(&[5u8, 6], 4); m }; + assert_eq!(root_tag(&list_val()), LINE_LIST_NODE_TAG); + assert_eq!(root_tag(&dense_dangling()), DENSE_BYTE_NODE_TAG); + + let upto_6: Locations = vec![(vec![], None), (vec![5], None), (vec![5, 6], None)]; + let cases: [(&str, Build, Build, Locations); 7] = [ + ("list value, dense dangling", list_val, dense_dangling, upto_6.clone()), + ("list value, dense branch", list_val, dense_branch, upto_6.clone()), + ("list value, dense value", list_val, dense_value, upto_6.clone()), + ("list child, dense branch", list_child, dense_branch, upto_6.clone()), + ("list child, list dangling", list_child, list_dangling, upto_6.clone()), + ("list value, list dangling", list_val, list_dangling, upto_6.clone()), + ("list dangling, dense value", list_dangling, dense_value, upto_6.clone()), + ]; + let mut failures = vec![]; + for (name, a, b, expected) in cases { + for (order, dst, src) in [("a,b", a, b), ("b,a", b, a)] { + let got_map = all_locations(&dst().meet(&src())); + let mut d = dst(); + let status = d.write_zipper().meet_into(&src().read_zipper(), false); + let got = all_locations(&d); + let expected_status = if all_locations(&dst()) == expected { AlgebraicStatus::Identity } else { AlgebraicStatus::Element }; + if got_map != expected || got != expected || status != expected_status { + failures.push(format!("{name} ({order}): PathMap::meet {got_map:?}, meet_into {status:?} {got:?}; expected {expected_status:?} {expected:?}")); + } + } + } + assert!(failures.is_empty(), "\n{}", failures.join("\n")); + } + + /// A meet with an equal trie reports `Identity`, also when the destination is a dense node and + /// the source a list node, so that the list node does the walking and must say exactly when the + /// result is all of the dense node. Here one byte holds both a value and an onward node, which + /// the list node keeps in two slots under the same key. + #[test] + fn write_zipper_meet_into_dense_with_equal_list_is_identity() { + let build_list = || { + let mut m = PathMap::::new(); + m.set_val_at(&[2u8], 9); + m.set_val_at(&[2u8, 1, 0], 7); + m.set_val_at(&[2u8, 2, 0], 8); + m + }; + let build_dense = || { + let mut m = build_list(); + for b in [1u8, 3] { m.set_val_at(&[b], 1); } + for b in [1u8, 3] { m.remove_val_at(&[b], true); } + m + }; + assert_eq!(root_tag(&build_list()), LINE_LIST_NODE_TAG); + assert_eq!(root_tag(&build_dense()), DENSE_BYTE_NODE_TAG); + assert_eq!(all_locations(&build_dense()), all_locations(&build_list())); + + let mut dst = build_dense(); + assert_eq!(dst.write_zipper().meet_into(&build_list().read_zipper(), false), AlgebraicStatus::Identity); + assert_eq!(all_locations(&dst), all_locations(&build_list())); + let mut dst = build_list(); + assert_eq!(dst.write_zipper().meet_into(&build_dense().read_zipper(), false), AlgebraicStatus::Identity); + + // Anything more in the dense node is not in the result + let mut dst = build_dense(); + dst.create_path(&[2u8, 3]); + assert_eq!(dst.write_zipper().meet_into(&build_list().read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&dst), all_locations(&build_list())); + } + + /// `meet_into` never removes its focus: not when the focus value goes because the source has + /// none, and not when nothing is left below it, with or without `prune`. `prune` drops only the + /// dangling paths below the focus. + #[test] + fn write_zipper_meet_into_keeps_focus() { + type Locations = Vec<(Vec, Option)>; + let dst = || { let mut m = PathMap::::new(); m.set_val_at(&[5u8], 1); m.set_val_at(&[5u8, 0], 2); m.set_val_at(&[9u8], 9); m }; + let src = || { let mut m = PathMap::::new(); m.set_val_at(&[5u8, 1], 3); m }; + let focus_left: Locations = vec![(vec![], None), (vec![5], None), (vec![9], Some(9))]; + for prune in [false, true] { + let mut d = dst(); + let s = src(); + assert_eq!(d.write_zipper_at_path(&[5u8]).meet_into(&s.read_zipper_at_path(&[5u8]), prune), AlgebraicStatus::None, "prune = {prune}"); + assert_eq!(all_locations(&d), focus_left, "prune = {prune}"); + } + + // The source's focus value is gone, but a dangling path both sides have survives unless pruned + let dst = || { let mut m = PathMap::::new(); m.set_val_at(&[5u8], 1); m.create_path(&[5u8, 0, 0]); m.set_val_at(&[9u8], 9); m }; + let src = || { let mut m = PathMap::::new(); m.create_path(&[5u8, 0, 0]); m }; + let mut d = dst(); + assert_eq!(d.write_zipper_at_path(&[5u8]).meet_into(&src().read_zipper_at_path(&[5u8]), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![5], None), (vec![5, 0], None), (vec![5, 0, 0], None), (vec![9], Some(9))]); + let mut d = dst(); + assert_eq!(d.write_zipper_at_path(&[5u8]).meet_into(&src().read_zipper_at_path(&[5u8]), true), AlgebraicStatus::None); + assert_eq!(all_locations(&d), focus_left); + } + + /// `meet_k_path_into` meets the subtries `k` bytes below the focus, dangling paths included, and + /// with `prune` drops the dangling paths from the result. The focus stays either way. + #[test] + fn write_zipper_meet_k_path_into_dangling_paths() { + let build = || { + let mut m = PathMap::::new(); + m.set_val_at(&[8u8], 8); + m.set_val_at(&[9u8, 1, 2], 5); + m.create_path(&[9u8, 1, 3]); + m.set_val_at(&[9u8, 2, 2], 6); + m.create_path(&[9u8, 2, 3]); + m + }; + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, false), true); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None), (vec![9, 2], Some(5)), (vec![9, 3], None)]); + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, true), true); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None), (vec![9, 2], Some(5))]); + + // Only a dangling path in common: kept without prune, and with prune nothing is left below + let build = || { + let mut m = PathMap::::new(); + m.set_val_at(&[8u8], 8); + m.create_path(&[9u8, 1, 3]); + m.create_path(&[9u8, 2, 3]); + m + }; + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, false), true); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None), (vec![9, 3], None)]); + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, true), false); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None)]); } /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value @@ -3754,6 +4322,113 @@ mod tests { assert_eq!(map.iter().count(), 1); } + /// A value must survive `subtract_into` when the source only passes *through* its location on the + /// way to a deeper value. The destination root here is a byte node (three branches worth of + /// payloads) and the source a line node, so the subtraction goes through `psubtract_abstract`, + /// which used to drop the value at `[0]` because the source has no value at that exact byte. + #[test] + fn write_zipper_subtract_into_value_under_source_path() { + let mut map: PathMap = PathMap::new(); + map.insert([0], 0); + map.insert([0, 0], 0); + map.insert([0, 0, 0], 0); + map.insert([1, 0, 0], 0); + let mut src: PathMap = PathMap::new(); + src.insert([0, 0], 0); + src.insert([1, 0, 0], 0); + + assert_eq!(map.write_zipper().subtract_into(&src.read_zipper(), false), AlgebraicStatus::Element); + let remaining: Vec<(Vec, u64)> = map.iter().map(|(k, v)| (k.to_vec(), *v)).collect(); + assert_eq!(remaining, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); + + // The same shape reached the way the fuzzer found it: join the source in first, then take it out again + let mut map: PathMap = PathMap::new(); + map.insert([], 0); + map.insert([0], 0); + map.insert([0, 0, 0], 0); + let mut src: PathMap = PathMap::new(); + src.insert([], 0); + src.insert([0, 0], 0); + src.insert([1, 0, 0], 0); + let mut wz = map.write_zipper(); + wz.join_into(&src.read_zipper()); + wz.subtract_into(&src.read_zipper(), false); + drop(wz); + let remaining: Vec<(Vec, u64)> = map.iter().map(|(k, v)| (k.to_vec(), *v)).collect(); + assert_eq!(remaining, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); + } + + /// `subtract_into` where every value of the source collides with a *different* value in the + /// destination. Nothing annihilates, so the destination comes back untouched and the status + /// has to be `Identity`. The integer `psubtract` used to answer `Element(*self)`, and the node + /// algebra -- which propagates identity masks, not values -- turned that into `Element` for the + /// whole trie. + #[test] + fn write_zipper_subtract_into_unequal_values_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + let mut dst = mk(&[(&[0], 1), (&[0, 0], 2), (&[1], 3), (&[2], 4)]); + let before = vals(&dst); + let src = mk(&[(&[0], 9), (&[0, 0], 9), (&[1], 9), (&[2], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //An equal value still annihilates, and that is an `Element`, not an identity + let mut dst = mk(&[(&[0], 1), (&[0, 0], 2), (&[1], 3), (&[2], 4)]); + let src = mk(&[(&[0], 9), (&[0, 0], 2), (&[1], 9), (&[2], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 1), (vec![1], 3), (vec![2], 4)]); + } + + /// `subtract_into` into a byte node whose location holds a value *and* an empty onward link. + /// The link carries nothing, since the value already holds the location, so dropping it while + /// the value survives leaves the trie as it was and the status has to be `Identity`. It used to + /// be `Element`. + #[test] + fn write_zipper_subtract_into_value_beside_an_empty_link_is_identity() { + let mut src = PathMap::::new(); + for (p, v) in [(&[0u8, 0][..], 0), (&[0, 1], 0), (&[0, 3], 0), (&[1], 1)] { src.set_val_at(p, v); } + + //These steps leave `dst` with the content of `src`, and an empty link beside the value at [1] + let mk_dst = || { + let mut dst = PathMap::::new(); + let rz = src.read_zipper(); + let mut wz = dst.write_zipper(); + wz.join_into(&rz); + wz.insert_prefix(&[1u8]); + wz.meet_into(&rz, false); + wz.join_into(&rz); + drop(wz); + let link = dst.root().unwrap().as_tagged().node_get_child(&[1]).map(|(_, child)| child.as_tagged().node_is_empty()); + assert_eq!(link, Some(true), "the layout this test needs"); + dst + }; + let locations = |m: &PathMap| { + let mut z = m.read_zipper(); + let mut locs = vec![(z.path().to_vec(), z.val().copied())]; + while z.to_next_step() { locs.push((z.path().to_vec(), z.val().copied())); } + locs + }; + + //Source {[0]:0, [1]:0, [3]:0}: the value at [1] differs, so nothing annihilates + let mut dst = mk_dst(); + let before = locations(&dst); + let st = dst.write_zipper().subtract_into(&src.read_zipper_at_path(&[0u8]), false); + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(locations(&dst), before); + + //An equal value annihilates, and the location goes with it + let mut dst = mk_dst(); + let mut sub = PathMap::::new(); + sub.set_val_at(&[1u8], 1); + let st = dst.write_zipper().subtract_into(&sub.read_zipper(), false); + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(locations(&dst), vec![(vec![], None), (vec![0], None), (vec![0, 0], Some(0)), (vec![0, 1], Some(0)), (vec![0, 3], Some(0))]); + } + /// Tests how `subtract_into` handles dangling paths, including situations with extraneous empty nodes hanging around #[test] fn write_zipper_subtract_into_test2() { @@ -4775,6 +5450,45 @@ mod tests { wz.remove_unmasked_branches(ByteMask::EMPTY, false); } + /// `remove_unmasked_branches` with the focus on a dangling path. A dangling path has no + /// branches below it, so the call must do nothing at all: the dangling path itself survives + /// (`prune` is `false`) and the rest of the trie is untouched. + /// + /// A dangling path is stored as an onward link to the empty-node sentinel, which + /// `get_child_mut` declines to hand out, so the zipper used to fail to descend through it and + /// hand the stub's own key to the focus node as if it were one of the node's branches. In a + /// `LineListNode` that tripped a `debug_assert!(!self.is_child_ptr::())`. + #[test] + fn write_zipper_test_remove_unmasked_branches_dangling_focus() { + //A LineListNode root: slot 0 is an onward link at [0], slot 1 the dangling stub at [1, 0] + let mut map = PathMap::::new(); + map.set_val_at([0u8], 0); + map.set_val_at([0u8, 0], 1); + assert!(map.create_path([1u8, 0])); + + //Focus exactly on the dangling path + let mut wz = map.write_zipper_at_path(&[1u8, 0]); + assert!(wz.path_exists()); + wz.remove_unmasked_branches(ByteMask::EMPTY, false); + assert!(wz.path_exists()); + drop(wz); + + //Focus below the dangling path + let mut wz = map.write_zipper_at_path(&[1u8, 0, 7]); + wz.remove_unmasked_branches(ByteMask::EMPTY, false); + drop(wz); + + //Nothing may have changed + assert_eq!(map.val_at([0u8]), Some(&0)); + assert_eq!(map.val_at([0u8, 0]), Some(&1)); + assert_eq!(map.val_at([1u8, 0]), None); + let mut rz = map.read_zipper(); + rz.descend_to([1u8, 0]); + assert!(rz.path_exists()); + drop(rz); + assert_eq!(map.val_count(), 2); + } + #[test] fn write_zipper_test_zipper_conversion() { let keys = [ @@ -6052,6 +6766,167 @@ mod tests { assert_eq!(keys(&m), ["cax", "cbx", "cdx", "d"]); } + /// lean/FINDINGS.md #15, `graft_child_maps_dense`: grafting nothing must neither create nor + /// destroy a location, and `graft_child_maps` must not abort on a dense destination. + /// + /// The contract for one masked branch is that of `graft` at that branch: the source's + /// contents and value replace whatever was there. A source with nothing at the branch + /// (absent, or a dangling path) therefore empties an existing branch -- which then survives + /// as a dangling path, exactly as after `graft` of an empty source -- and leaves an absent + /// branch absent. `remove_unset` removes the unset branches outright first. + /// Joining child slots whose onward nodes are empty, as left by dropped head writers + #[test] + fn join_k_path_into_empty_child_nodes() { + let mut map = PathMap::::new(); + map.set_val_at(&[0, 0], 1); + { + let zh = map.zipper_head(); + let _w0 = zh.write_zipper_at_exclusive_path(&[0, 2]).unwrap(); + let _w1 = zh.write_zipper_at_exclusive_path(&[1, 2]).unwrap(); + } + map.write_zipper().join_k_path_into(1, false); + assert_eq!(map.get_val_at(&[0]), Some(&1)); + assert_eq!(map.val_count(), 1); + } + + /// `graft_child_maps` and `graft_masked_branches` below a root path too long for one node key + #[test] + fn graft_child_maps_long_root() { + for root_len in [47usize, 48, 60, 200] { + let root = vec![0u8; root_len]; + let mut map = PathMap::::new(); + { + let mut wz = map.write_zipper_at_path(&root); + wz.graft_child_maps(ByteMask::from_iter([1u8, 3]), [PathMap::single([2u8], 5), PathMap::single([], 6)], false); + } + let mut want = root.clone(); + want.extend([1u8, 2]); + assert_eq!(map.get_val_at(&want), Some(&5), "root {root_len}"); + want.truncate(root_len); + want.push(3); + assert_eq!(map.get_val_at(&want).is_some(), cfg!(feature = "graft_root_vals"), "root {root_len}"); + + let mut src = PathMap::::new(); + src.set_val_at([4u8, 4], 9); + let mut map = PathMap::::new(); + { + let mut wz = map.write_zipper_at_path(&root); + wz.graft_masked_branches(&src.read_zipper(), ByteMask::from_iter([4u8]), false); + } + let mut want = root.clone(); + want.extend([4u8, 4]); + assert_eq!(map.get_val_at(&want), Some(&9), "root {root_len}"); + } + } + + #[test] + fn graft_child_maps_dense() { + use crate::utils::BitMask; + fn mask(bytes: &[u8]) -> ByteMask { + let mut m = ByteMask::EMPTY; + for b in bytes { m.set_bit(*b); } + m + } + fn dump(m: &PathMap) -> Vec<(Vec, Option)> { + // Every location, dangling ones included, with its value. + let mut z = m.read_zipper(); + let mut out = Vec::new(); + while z.to_next_step() { out.push((z.path().to_vec(), z.val().copied())); } + out + } + fn mk(kvs: &[(&[u8], u64)]) -> PathMap { + let mut m = PathMap::new(); + for (k, v) in kvs { m.set_val_at(k, *v); } + m + } + + // (a) A dense destination (>= 3 branches) used to reach `node_get_child_mut` with an + // empty key and abort. + let mut dst = mk(&[(&[0, 0], 1), (&[1, 0], 1), (&[2, 0], 1), (&[3, 0], 1)]); + let child = mk(&[(&[], 7), (&[3], 8)]); + dst.write_zipper().graft_child_maps(mask(&[0]), vec![child], false); + assert_eq!(dump(&dst), vec![ + (vec![0], Some(7)), (vec![0, 3], Some(8)), + (vec![1], None), (vec![1, 0], Some(1)), + (vec![2], None), (vec![2, 0], Some(1)), + (vec![3], None), (vec![3, 0], Some(1)), + ]); + + // (b) Empty maps at a non-existent focus: nothing is created, both with and without + // `remove_unset`, and the same for `graft_masked_branches` from an empty source. + for remove_unset in [true, false] { + let mut m = mk(&[(&[1], 1)]); + { + let mut wz = m.write_zipper(); + wz.descend_to(&[0u8, 0]); + wz.graft_child_maps(mask(&[0, 1, 2]), vec![PathMap::new(); 3], remove_unset); + assert!(!wz.path_exists()); + } + assert_eq!(dump(&m), vec![(vec![1], Some(1))]); + } + for bits in [&[0u8][..], &[0, 2], &[0, 2, 3]] { + let mut m = mk(&[(&[1], 1)]); + let src = mk(&[(&[1, 0], 5)]); + { + let mut wz = m.write_zipper(); + wz.descend_to(&[2u8, 2]); + wz.graft_masked_branches(&src.read_zipper(), mask(bits), false); + assert!(!wz.path_exists(), "{bits:?}"); + } + assert_eq!(dump(&m), vec![(vec![1], Some(1))], "{bits:?}"); + } + + // (c) A masked branch the source lacks is emptied but its location survives; a branch + // the source has is replaced, value included (and a missing source value removes the + // old one). One-, two- and three-bit masks take different code paths. + let src = mk(&[(&[1, 0], 5)]); + for bits in [&[0u8][..], &[0, 1], &[0, 1, 2]] { + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[1, 1], 1), (&[3], 3)]); + m.write_zipper().graft_masked_branches(&src.read_zipper(), mask(bits), false); + let mut expected = vec![(vec![0], None)]; + if bits.contains(&1) { + expected.extend([(vec![1], None), (vec![1, 0], Some(5))]); + } else { + expected.extend([(vec![1], Some(1)), (vec![1, 1], Some(1))]); + } + expected.push((vec![3], Some(3))); + assert_eq!(dump(&m), expected, "{bits:?}"); + } + // A source whose focus is a bare value (no node) or a dangling path takes yet another + // path, and must behave as an empty source. + for src in [mk(&[(&[], 4)]), { let mut s = PathMap::::new(); s.write_zipper().create_path(); s }] { + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[3], 3)]); + m.write_zipper().graft_masked_branches(&src.read_zipper(), mask(&[0, 1, 2]), false); + assert_eq!(dump(&m), vec![(vec![0], None), (vec![1], None), (vec![3], Some(3))]); + } + // With `remove_unset` the branches go outright. + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[3], 3)]); + m.write_zipper().graft_masked_branches(&src.read_zipper(), mask(&[0, 1, 2]), true); + assert_eq!(dump(&m), vec![(vec![1], None), (vec![1, 0], Some(5))]); + + // (d) `graft_child_maps` fed the source's own child subtries agrees with (c). + for bits in [&[0u8][..], &[0, 1], &[0, 1, 2]] { + let maps: Vec> = bits.iter().map(|b| { + let mut z = src.read_zipper(); + z.descend_to_byte(*b); + z.make_map() + }).collect(); + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[1, 1], 1), (&[3], 3)]); + let mut reference = m.clone(); + m.write_zipper().graft_child_maps(mask(bits), maps, false); + reference.write_zipper().graft_masked_branches(&src.read_zipper(), mask(bits), false); + assert_eq!(dump(&m), dump(&reference), "{bits:?}"); + } + // A map with contents but no root value removes the old value; an empty map empties + // the branch and keeps the location. + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1)]); + m.write_zipper().graft_child_maps(mask(&[0]), vec![mk(&[(&[3], 3)])], false); + assert_eq!(dump(&m), vec![(vec![0], None), (vec![0, 3], Some(3)), (vec![1], Some(1))]); + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1)]); + m.write_zipper().graft_child_maps(mask(&[0]), vec![PathMap::new()], false); + assert_eq!(dump(&m), vec![(vec![0], None), (vec![1], Some(1))]); + } + #[test] fn write_zipper_graft_masked_branches_test4() { // Upper bound 0: remove_unset=true with an empty mask. @@ -6672,4 +7547,253 @@ mod tests { } assert_eq!(keys(&m), ["cx", "cy", "d"]); } + + /// `join_into` with the source focus partway into a line node. The focus node is then a + /// `TinyRefNode`, and the join used to be evaluated with the operands swapped and the + /// identity mask *not* swapped back. A destination that already held everything the + /// source has reported `COUNTER_IDENT`, which `join_into` takes as "the result is the + /// source" -- so a dense destination was overwritten by the source (the data loss of + /// FINDINGS.md #1, in the form that survived the empty-destination fix), and a list + /// destination reported `Element` for a join that changed nothing. + #[test] + fn write_zipper_join_into_mid_key_source_keeps_destination() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + let src = mk(&[(&[0, 0, 0], 7)]); + + //Dense destination: was replaced by `[0]=7` + let mut dst = mk(&[(&[0], 7), (&[1], 1), (&[2], 2), (&[3], 3)]); + let before = vals(&dst); + let st = { let mut wz = dst.write_zipper(); let mut rz = src.read_zipper(); rz.descend_to(&[0, 0]); wz.join_into(&rz) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //List destination: was `Element` for an unchanged trie + let mut dst = mk(&[(&[0], 7), (&[0, 0], 0)]); + let before = vals(&dst); + let st = { let mut wz = dst.write_zipper(); let mut rz = src.read_zipper(); rz.descend_to(&[0, 0]); wz.join_into(&rz) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //And a join that does add something still says so, with the destination intact + let mut dst = mk(&[(&[1], 1), (&[2], 2), (&[3], 3)]); + let st = { let mut wz = dst.write_zipper(); let mut rz = src.read_zipper(); rz.descend_to(&[0, 0]); wz.join_into(&rz) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 7), (vec![1], 1), (vec![2], 2), (vec![3], 3)]); + } + + /// `join_into` where a destination slot holds an onward child under a key that is a + /// prefix of the source's longer key. `merge_guts` joined the child with the source's + /// remainder but always reported the pair as `Element`, so a source already contained in + /// that child made the whole join report `Element` although nothing changed. + #[test] + fn write_zipper_join_into_contained_under_child_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + let mut dst = mk(&[(&[0, 0], 0), (&[0, 1], 0)]); + let before = vals(&dst); + let src = mk(&[(&[0, 0], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.join_into(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //The mirror image: the source holds the child, the destination the longer key + let mut dst = mk(&[(&[0, 0], 0)]); + let src = mk(&[(&[0, 0], 0), (&[0, 1], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.join_into(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vals(&src)); + } + + /// `restrict` between two dense nodes, where the source has branches the destination lacks. + /// `restrict` is non-commutative: only the destination's branches can be dropped, so branches + /// that exist only in the source say nothing about whether the destination changed. The dense + /// restrict nevertheless required the two masks to be equal before it would report `Identity`. + #[test] + fn write_zipper_restrict_wider_source_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //Both roots are dense. Every path in `dst` is prefixed by a path to a value in `src`, so + // the restriction keeps all of `dst`; `src`'s extra branches are irrelevant. + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3)]); + let before = vals(&dst); + let src = mk(&[(&[0], 0), (&[1], 0), (&[2], 0), (&[3], 0), (&[4], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //A restriction that really does drop a branch still reports it + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3)]); + let src = mk(&[(&[0], 0), (&[1], 0), (&[3], 0), (&[4], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 1), (vec![1], 2)]); + } + + /// `restrict` of a dense node against a node that can't be iterated as a dense one (a list or + /// a tiny node) walks the destination's entries. An entry holding *both* a value and an + /// onward link, where the source has no value at that byte, loses its value -- the empty path + /// never validates -- but the identity flag was only cleared by the onward link's own result, + /// so an onward link that restricted to an identity made the whole node report `Identity` and + /// the dropped value stayed in the map. + #[test] + fn write_zipper_restrict_drops_value_beside_kept_child() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //Build a dense root, then meet it down to the single byte-0 branch, which keeps the dense + // node type. That branch holds a value (at `[0]`) beside an onward link (to `[0, 0]`). + let mut dst = mk(&[(&[0], 7), (&[0, 0], 1), (&[1], 2), (&[2], 3)]); + let filter = mk(&[(&[0], 0), (&[0, 0], 0), (&[5], 0), (&[6], 0)]); + { let mut wz = dst.write_zipper(); wz.meet_into(&filter.read_zipper(), false); } + assert_eq!(vals(&dst), vec![(vec![0], 7), (vec![0, 0], 1)]); + + //`src` has no value at `[0]`, so `[0]` is not kept, while `[0, 0]` is + let src = mk(&[(&[0, 0], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(vals(&dst), vec![(vec![0, 0], 1)]); + assert_eq!(st, AlgebraicStatus::Element); + } + + /// A dangling path -- one that exists but leads to no value -- is kept by `restrict` only when + /// it is validated, i.e. when the source carries a value at some non-empty prefix of it. In a + /// dense node a dangling path is a co-free with neither a value nor an onward link; the + /// destination-walking `prestrict_abstract` dropped such a co-free from the result but never + /// cleared its identity flag, so the node reported `Identity` and the caller kept `self` -- + /// dangling path included. That is a wrong answer, not just an imprecise status. + #[test] + fn write_zipper_restrict_drops_dangling_branch() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //Build a dense root and meet it down to bytes 0 and 1, which keeps the dense node type, + // then strip the value at [0] without pruning so that [0] is left dangling. + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3), (&[3], 4)]); + let filter = mk(&[(&[0], 0), (&[1], 0)]); + { let mut wz = dst.write_zipper(); wz.meet_into(&filter.read_zipper(), false); } + dst.remove_val_at(&[0u8], false); + assert_eq!(dst.path_exists_at(&[0u8]), true, "[0] should be left dangling"); + assert_eq!(vals(&dst), vec![(vec![1], 2)]); + + //`src` is a list node: it has a path through byte 0 but no value at [0], so the dangling + // [0] is not validated and must go, while [1] carries a value and is kept. Every byte of + // `dst` is present in `src`, so nothing else can clear the identity flag. + let src = mk(&[(&[0, 9], 0), (&[1], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(dst.path_exists_at(&[0u8]), false); + assert_eq!(vals(&dst), vec![(vec![1], 2)]); + + //A dangling path that *is* validated stays: `src` has a value at [0]. + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3), (&[3], 4)]); + { let mut wz = dst.write_zipper(); wz.meet_into(&filter.read_zipper(), false); } + dst.remove_val_at(&[0u8], false); + let src = mk(&[(&[0], 0), (&[1], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(dst.path_exists_at(&[0u8]), true); + assert_eq!(vals(&dst), vec![(vec![1], 2)]); + + //The shrunk differential reproducer, spelled out: `meet_into` against a source whose focus + // is a leaf leaves an empty child node at [0], and the following `restrict` kept it. + let mut map0 = PathMap::::new(); + map0.set_val_at(&[0u8], 0); + let mut map1 = PathMap::::new(); + map1.set_val_at(&[0u8, 0, 0, 0], 0); + map1.set_val_at(&[1u8], 0); + { + let mut wz = map0.write_zipper_at_path(&[]); + let mut rz = map1.read_zipper_at_path(&[]); + wz.join_into(&rz); + wz.descend_first_byte(); + rz.to_next_val(); + wz.subtract_into(&rz, false); + rz.to_next_val(); + wz.meet_into(&rz, false); + } + assert_eq!(map0.path_exists_at(&[0u8]), true, "meet_into leaves [0] dangling"); + let st = { let mut wz = map0.write_zipper(); wz.restrict(&map1.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(map0.path_exists_at(&[0u8]), false); + assert_eq!(vals(&map0), vec![(vec![1], 0)]); + } + + /// A list node may hold a value and an onward child under the *same* key -- that is how a path + /// that both ends and continues is stored. Emptying the child without pruning leaves the link + /// in place, and the slot is then carrying nothing at all: there is nothing below an empty + /// link, and the path it stands at is already there because of the value beside it. + /// + /// `subtract_into` and `meet_into` both drop such a slot, correctly -- a dangling path the + /// source reaches survives neither operation. But dropping something that was carrying + /// nothing is not a change, and both used to rebuild the node around the surviving slot and + /// report `Element` for a destination holding exactly what it held before. + /// + /// The shadowing matters: an empty link under a key of its *own* really does take a path away + /// with it, and that is an `Element`. Only a link sharing its key with the slot beside it is + /// invisible. + #[test] + fn write_zipper_shadowed_dangling_slot_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //`insert_prefix` puts a real onward node under `[0]`, a value is set beside it, and + // `remove_branches` then empties the node but leaves the link. Both slots stand at `[0]`. + fn dst_with_shadowed_dangling() -> PathMap { + let seed = mk(&[(&[0, 0], 0)]); + let mut dst = PathMap::::new(); + { + let mut wz = dst.write_zipper(); + wz.graft(&seed.read_zipper()); + wz.descend_to_byte(0); + wz.insert_prefix(&[0]); + wz.get_val_or_set_mut(1); + wz.remove_branches(false); + } + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + dst + } + + //Nothing of `src` collides with the value at `[0]`, so the subtraction takes nothing away + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[0, 0], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + + //The meet keeps the value at `[0]` and drops the dangling link, which changes nothing + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[0], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.meet_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + + //A subtraction that really does annihilate the value beside the dangling link still says so + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[0], 1)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::None); + assert_eq!(vals(&dst), vec![]); + + //...and so does a meet that drops it + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[1], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.meet_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::None); + assert_eq!(vals(&dst), vec![]); + + //An empty link that is *not* shadowed stands at a path of its own, `[0, 0]` here, and + // dropping it takes that path away -- a change, and still reported as one + let mut dst = mk(&[(&[0], 1), (&[0, 0], 2)]); + { + let empty = PathMap::::new(); + let mut wz = dst.write_zipper_at_path(&[0, 0]); + wz.graft(&empty.read_zipper()); + } + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + let src = mk(&[(&[0, 0], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + } } diff --git a/src/zipper.rs b/src/zipper.rs index 57a70141..59e67e17 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -416,18 +416,29 @@ pub trait ZipperMoving: Zipper { /// where the index passed is 1 more than the index of the current focus position. fn to_next_sibling_byte(&mut self) -> Option { let cur_byte = self.focus_byte()?; + //The focus is allowed to sit on a path that isn't in the trie, and such a focus still has + // well-defined siblings because the siblings come from the parent's `child_mask`. Remember + // whether the focus exists, so the no-sibling arm can check it restores exactly what it found. + #[cfg(debug_assertions)] + let focus_existed = self.path_exists(); if !self.ascend_byte() { return None } let mask = self.child_mask(); match mask.next_bit(cur_byte) { Some(byte) => { + //`byte` came out of the parent's `child_mask`, so the sibling we land on exists self.descend_to_byte(byte); debug_assert!(self.path_exists()); Some(byte) }, None => { + //There is no next sibling, so the focus goes back exactly where it started. That + // location need not be an existing path, so all we can check is that putting it + // back didn't change whether it exists. self.descend_to_byte(cur_byte); + #[cfg(debug_assertions)] + debug_assert_eq!(self.path_exists(), focus_existed); None } } @@ -442,19 +453,27 @@ pub trait ZipperMoving: Zipper { /// where the index passed is 1 less than the index of the current focus position. fn to_prev_sibling_byte(&mut self) -> Option { let cur_byte = self.focus_byte()?; + //See the note in `to_next_sibling_byte`; the focus may legitimately be off the trie + #[cfg(debug_assertions)] + let focus_existed = self.path_exists(); if !self.ascend_byte() { return None } let mask = self.child_mask(); match mask.prev_bit(cur_byte) { Some(byte) => { + //`byte` came out of the parent's `child_mask`, so the sibling we land on exists self.descend_to_byte(byte); debug_assert!(self.path_exists()); Some(byte) }, None => { + //There is no previous sibling, so the focus goes back exactly where it started. + // That location need not be an existing path, so all we can check is that putting + // it back didn't change whether it exists. self.descend_to_byte(cur_byte); - debug_assert!(self.path_exists()); + #[cfg(debug_assertions)] + debug_assert_eq!(self.path_exists(), focus_existed); None } } @@ -1117,6 +1136,8 @@ fn k_path_default_internal(z: &mut if z.depth() == base_idx + k { return true } } } + //Back at the base: nothing (more) below it, and its own siblings are out of bounds + if z.depth() == base_idx { return false } //A sibling step replaces the last byte rather than adding one, so the observer sees the old //byte retracted before the new one arrives if let Some(byte) = z.to_next_sibling_byte() { @@ -1490,10 +1511,13 @@ impl<'a, V: Clone + Send + Sync + Unpin + 'a, A: Allocator + 'a> ZipperReadOnlyP } impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ReadZipperTracked<'a, 'path, V, A> { - /// See [ReadZipperCore::new_with_node_and_path] - pub(crate) fn new_with_node_and_path_in(root_node: &'a TrieNodeODRc, owned_root: bool, path: &'path [u8], root_prefix_len: usize, root_key_start: usize, root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { - let core = ReadZipperCore::new_with_node_and_path_in(root_node, owned_root, path, root_prefix_len, root_key_start, root_val, alloc); - Self { z: core, tracker } + /// See [ReadZipperCore::new_isolated_in] + pub(crate) fn new_isolated_in(root_node: &'a TrieNodeODRc, path: &'path [u8], root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { + Self { z: ReadZipperCore::new_isolated_in(root_node, path, root_val, alloc), tracker } + } + /// See [ReadZipperCore::new_isolated_cloned_path_in] + pub(crate) fn new_isolated_cloned_path_in(root_node: &'a TrieNodeODRc, path: &[u8], root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { + Self { z: ReadZipperCore::new_isolated_cloned_path_in(root_node, path, root_val, alloc), tracker } } /// See [ReadZipperCore::new_with_node_and_cloned_path] pub(crate) fn new_with_node_and_cloned_path_in(root_node: &'a TrieNodeODRc, owned_root: bool, path: &[u8], root_prefix_len: usize, root_key_start: usize, root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { @@ -1997,7 +2021,7 @@ pub(crate) mod read_zipper_core { PathMap::new_with_root_in(root_node, root_val, self.alloc.clone()) } fn get_trie_ref(&self) -> TrieRef<'_, V, A> { - TrieRefBorrowed::new_with_key_and_path_in(self.focus_parent_borrowed(), || self.val(), self.node_key(), b"", self.alloc.clone()).into() + TrieRefBorrowed::new_with_key_and_path_in(self.focus_parent(), || self.val(), self.node_key(), b"", self.alloc.clone()).into() } fn get_focus(&self) -> OpaqueAbstractNodeRef<'_, V, A> { self.get_focus_at([]) @@ -2584,7 +2608,8 @@ pub(crate) mod read_zipper_core { if let Some((parent, _iter_tok, _prefix_offset)) = self.ancestors.last() { parent.node_get_val(self.parent_key()) } else { - if self.root_val.is_some() { + if self.root_val.is_some() || self.root_parent_key_start == usize::MAX { + //No parent key: the zipper root is the root node itself, and its value is `root_val` self.root_val } else { //We know the node in the witness and the node in self.root_node are the same, @@ -2625,11 +2650,11 @@ pub(crate) mod read_zipper_core { let (_key_len, focus_node) = parent.node_get_child(self.parent_key()).unwrap(); !focus_node.is_empty() && focus_node.refcount() > 1 } else { - match &self.root_node { - OwnedOrBorrowed::Owned(root) => !root.is_empty() && root.refcount() > 1, - OwnedOrBorrowed::Borrowed(root) => !root.is_empty() && root.refcount() > 1, - OwnedOrBorrowed::None => false, - } + let focus = match &self.root_node { + OwnedOrBorrowed::None => return false, + _ => self.focus_parent(), + }; + !focus.is_empty() && focus.refcount() > 1 } } } @@ -2798,6 +2823,42 @@ pub(crate) mod read_zipper_core { new_zipper.make_static_path() } + /// Like [Self::new_with_node_and_path_in] with an owned root, but the root is a private node holding + /// only the entry at `path`. A ZipperHead reader must not share a node that live writers point into + pub(crate) fn new_isolated_in(root_node: &'a TrieNodeODRc, path: &'path [u8], root_val: Option<&'a V>, alloc: A) -> Self { + let Some((&last, parent_path)) = path.split_last() else { + return Self::new_with_node_and_path_in(root_node, true, path, 0, 0, root_val, alloc) + }; + let (parent, key, _) = node_along_path(root_node, parent_path, None, false); + let mut entry_key = key.to_vec(); + entry_key.push(last); + let parent = parent.as_tagged(); + let mut root = TrieNodeODRc::new_in(crate::dense_byte_node::DenseByteNode::new_in(alloc.clone()), alloc.clone()); + let val = parent.node_get_val(&entry_key).cloned(); + let child = parent.get_node_at_key(&entry_key).into_option(); + let dangling = val.is_none() && child.is_none() && parent.node_contains_partial_key(&entry_key); + if let Some(val) = val { + if let Err(n) = root.make_mut().node_set_val(&[last], val) { root = n } + } + if let Some(child) = child { + if let Err(n) = root.make_mut().node_set_branch(&[last], child) { root = n } + } + if dangling { + if let Err(n) = root.make_mut().node_create_dangling(&[last]) { root = n } + } + //The root value is read from `root`, via `root_parent_key_start` + Self::new_with_node_and_path_internal_in(OwnedOrBorrowed::Owned(root), path, path.len() - 1, None, alloc) + } + /// Same as [Self::new_isolated_in], but with a `'static` path + pub(crate) fn new_isolated_cloned_path_in(root_node: &'a TrieNodeODRc, path: &[u8], root_val: Option<&'a V>, alloc: A) -> ReadZipperCore<'a, 'static, V, A> { + let mut new_zipper = ReadZipperCore::<'a, '_, V, A>::new_isolated_in(root_node, path, root_val, alloc); + new_zipper.prefix_buf = Vec::with_capacity(EXPECTED_PATH_LEN); + new_zipper.prefix_buf.extend(path); + new_zipper.origin_path = SliceOrLen::new_owned(path.len()); + new_zipper.ancestors = Vec::with_capacity(EXPECTED_DEPTH); + new_zipper.make_static_path() + } + /// Makes a version of `self` that has an allocated path buffer and a `'static`` path lifetime #[inline] pub(crate) fn make_static_path(mut self) -> ReadZipperCore<'a, 'static, V, A> { @@ -2880,7 +2941,12 @@ pub(crate) mod read_zipper_core { // we currently share the same implementation between `val()` and `get_val()` because the only difference is the return // lifetime, and the current ZipperHead implementation is actually ok with referencing the value in the root of the ZipperHead. // debug_assert!(self.root_node.is_borrowed()); - self.root_val + if self.root_val.is_some() || self.root_parent_key_start == usize::MAX || !self.root_node.is_owned() { + self.root_val + } else { + //SAFETY: see the note on this method + self.root_node.as_ref().as_tagged().node_get_val(self.root_node_key()).map(|v| unsafe{ &*(v as *const V) }) + } } } } @@ -3009,6 +3075,10 @@ pub(crate) mod read_zipper_core { if parent_key.len() == 0 { return self.root_node.as_ref() } + if self.ancestors.is_empty() { + //At the root, with the focus on a child of the root node + return self.root_node.as_ref().as_tagged().node_get_child(parent_key).unwrap().1 + } self.focus_parent_borrowed() } @@ -3231,8 +3301,10 @@ pub(crate) mod read_zipper_core { } else { if let Some((parent, _iter_tok, _prefix_offset)) = self.ancestors.last() { parent.node_contains_val(self.parent_key()) - } else { + } else if self.root_val.is_some() || self.root_parent_key_start == usize::MAX || !self.root_node.is_owned() { self.root_val.is_some() + } else { + self.root_node.as_ref().as_tagged().node_contains_val(self.root_node_key()) } } } @@ -3319,6 +3391,8 @@ pub(crate) mod read_zipper_core { if self.prefix_buf.len() > 0 { let key_start = if self.ancestors.len() > 1 { unsafe{ self.ancestors.get_unchecked(self.ancestors.len()-2) }.2 + } else if self.ancestors.is_empty() && self.root_parent_key_start != usize::MAX { + self.root_parent_key_start } else { self.root_key_start }; @@ -3391,6 +3465,41 @@ pub(crate) mod read_zipper_core { } } + /// The default k-path walk ends when there is nothing below its base, and `k = 0` returns `false` + #[test] + fn default_k_path_walk_at_a_leaf() { + use crate::zipper::ProductZipperG; + let mut leaf = PathMap::::new(); + leaf.set_val_at(&[1u8], 1); + let empty = PathMap::::new(); + for (map, path) in [(&empty, &[][..]), (&leaf, &[1u8][..]), (&leaf, &[][..])] { + for k in 0..3 { + let mut z = ProductZipperG::new(map.read_zipper(), [empty.read_zipper()]); + z.descend_to(path); + let found = z.descend_first_k_path(k); + assert_eq!(found, map.val_count() > 0 && path.is_empty() && k == 1, "{path:?} k={k}"); + if !found { + assert_eq!(z.path(), path, "{path:?} k={k}"); + } + } + } + } + + /// `get_val_with_witness` agrees with `val` on owned read zippers, including at a root without a value + #[test] + fn read_zipper_owned_get_val_with_witness() { + let mut map = PathMap::::new(); + for p in [&[1u8][..], &[1, 2], &[3, 4, 5]] { map.set_val_at(p, p.len() as u64); } + for root in [&[][..], &[1u8], &[3u8], &[3u8, 4], &[9u8]] { + let mut z = map.clone().into_read_zipper(root); + loop { + let w = z.witness(); + assert_eq!(z.get_val_with_witness(&w), z.val(), "{root:?} {:?}", z.path()); + if !z.to_next_step() { break } + } + } + } + /// Validate we don't accidentially reallocate the path buffer when we don't need to #[test] fn read_zipper_reserve_buffer_test() { @@ -6401,6 +6510,76 @@ mod tests { assert_eq!(z.to_prev_sibling_byte(), None); } + /// A sibling step from a focus that is not in the trie. `descend_to` an absent path is + /// legal and leaves `path_exists()` false, and the siblings of that focus are still well + /// defined because they come from the parent's `child_mask`. The `ZipperMoving` default + /// `to_prev_sibling_byte` asserted `path_exists()` on the arm that finds no previous + /// sibling and puts the focus back where it started, so every write zipper (which takes the + /// default impl) panicked in a debug build instead of returning `None`. `to_next_sibling_byte` + /// had no assert on its matching arm and so answered correctly; both are checked here. + #[test] + fn sibling_step_from_a_focus_that_does_not_exist() { + //The shrunk fuzz case: an empty map, so the root's child mask is empty + let mut empty = PathMap::::new(); + let mut wz = empty.write_zipper(); + wz.descend_to(&[0u8]); + assert!(!wz.path_exists()); + assert_eq!(wz.to_prev_sibling_byte(), None); + assert_eq!(wz.path(), &[0u8]); + assert!(!wz.path_exists()); + assert_eq!(wz.to_next_sibling_byte(), None); + assert_eq!(wz.path(), &[0u8]); + drop(wz); + + let mut map = PathMap::::new(); + map.insert(&[1u8, 3], 13); + map.insert(&[1u8, 5], 15); + map.insert(&[7u8], 7); + + //A focus that is missing from an existing parent answers from the parent's children: + // `None` when nothing lies to that side, and the real neighbour when something does + for (byte, prev, next) in [(2u8, None, Some(3u8)), (4, Some(3), Some(5)), (6, Some(5), None)] { + let mut wz = map.write_zipper_at_path(&[1u8]); + wz.descend_to(&[byte]); + assert!(!wz.path_exists(), "byte {byte}"); + assert_eq!(wz.to_prev_sibling_byte(), prev, "byte {byte}"); + //A step that moved landed on a real sibling; one that didn't left the focus off-trie + assert_eq!(wz.path_exists(), prev.is_some(), "byte {byte}"); + assert_eq!(wz.path(), &[prev.unwrap_or(byte)], "byte {byte}"); + drop(wz); + + let mut wz = map.write_zipper_at_path(&[1u8]); + wz.descend_to(&[byte]); + assert_eq!(wz.to_next_sibling_byte(), next, "byte {byte}"); + drop(wz); + + //The read zipper's native impls must agree with the write zipper's default impls + let mut rz = map.read_zipper_at_path(&[1u8]); + rz.descend_to(&[byte]); + assert!(!rz.path_exists(), "byte {byte}"); + assert_eq!(rz.to_prev_sibling_byte(), prev, "byte {byte}"); + let mut rz = map.read_zipper_at_path(&[1u8]); + rz.descend_to(&[byte]); + assert_eq!(rz.to_next_sibling_byte(), next, "byte {byte}"); + } + + //A focus whose parent is itself missing has no siblings at all, and the failed step + // must leave the focus exactly where it was + let mut wz = map.write_zipper(); + wz.descend_to(&[9u8, 9]); + assert!(!wz.path_exists()); + assert_eq!(wz.to_prev_sibling_byte(), None); + assert_eq!(wz.path(), &[9u8, 9]); + assert_eq!(wz.to_next_sibling_byte(), None); + assert_eq!(wz.path(), &[9u8, 9]); + assert!(!wz.path_exists()); + //...and the zipper is still usable afterwards + wz.ascend(2); + assert_eq!(wz.to_next_sibling_byte(), None); + wz.descend_to(&[7u8]); + assert_eq!(wz.val(), Some(&7)); + } + /// Tests iteration behavior of to_next_val implementations, comparing the default impl /// against the native imple, and a third run that interleaves calls to each #[test] @@ -6594,6 +6773,30 @@ mod tests { assert_eq!(z.path(), &[3]); } + /// A location holding both a value and a child is stored in a list node as two slots with the same + /// key. Stepping to it as the *previous* sibling looked for the onward child only in the slot + /// whose byte it found first, which can be the value slot, so the zipper landed on the location + /// without its node: `child_count` said 0 and nothing below could be reached. + #[test] + fn read_zipper_prev_sibling_onto_a_value_and_child_location() { + let mut m = PathMap::::new(); + m.set_val_at(&[2u8, 0], 0); + m.set_val_at(&[2u8, 1], 0); + m.set_val_at(&[2u8], 0); + for start in [&[3u8][..], &[9u8]] { + let mut z = m.read_zipper(); + z.descend_to(start); + assert!(!z.path_exists()); + assert_eq!(z.to_prev_sibling_byte(), Some(2), "from {start:?}"); + assert_eq!(z.path(), &[2u8]); + assert_eq!(z.val(), Some(&0)); + assert_eq!(z.child_count(), 2, "from {start:?}"); + assert_eq!(z.child_mask().iter().collect::>(), vec![0u8, 1]); + assert_eq!(z.descend_first_byte(), Some(0), "from {start:?}"); + assert_eq!(z.path(), &[2u8, 0]); + } + } + #[test] fn read_zipper_prev_sibling_cases() { let m: PathMap<()> = [&[10u8][..], &[20], &[70]].into_iter().collect(); // dense node diff --git a/src/zipper_head.rs b/src/zipper_head.rs index 1ab41673..6ef68575 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -209,7 +209,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip // logic makes sure conflicting paths aren't permitted, so we should not get aliased &mut borrows let root_node: &'trie TrieNodeODRc = unsafe{ core::mem::transmute(root_node) }; let root_val: Option<&'trie V> = root_val.map(|v| unsafe{ &*v.as_ptr() } ); - let new_zipper = ReadZipperTracked::new_with_node_and_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), zipper_tracker); + let new_zipper = ReadZipperTracked::new_isolated_in(root_node, path, root_val, z.alloc.clone(), zipper_tracker); Ok(new_zipper) }) } @@ -229,7 +229,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip #[cfg(not(debug_assertions))] let zipper_tracker = None; - ReadZipperTracked::new_with_node_and_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), zipper_tracker) + ReadZipperTracked::new_isolated_in(root_node, path, root_val, z.alloc.clone(), zipper_tracker) }) } fn read_zipper_at_path<'a, K: AsRef<[u8]>>(&'a self, path: K) -> Result, Conflict> where 'trie: 'a { @@ -242,7 +242,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip let root_node: &'trie TrieNodeODRc = unsafe{ core::mem::transmute(root_node) }; let root_val: Option<&'trie V> = root_val.map(|v| unsafe{ &*v.as_ptr() } ); - let new_zipper = ReadZipperTracked::new_with_node_and_cloned_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), Some(zipper_tracker)); + let new_zipper = ReadZipperTracked::new_isolated_cloned_path_in(root_node, path, root_val, z.alloc.clone(), Some(zipper_tracker)); Ok(new_zipper) }) } @@ -263,7 +263,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip #[cfg(not(debug_assertions))] let zipper_tracker = None; - ReadZipperTracked::new_with_node_and_cloned_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), zipper_tracker) + ReadZipperTracked::new_isolated_cloned_path_in(root_node, path, root_val, z.alloc.clone(), zipper_tracker) }) } fn write_zipper_at_exclusive_path<'a, K: AsRef<[u8]>>(&'a self, path: K) -> Result, Conflict> where 'trie: 'a { @@ -343,7 +343,7 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + debug_assert_eq!(z.focus_stack.depth(), 1); z.focus_stack.to_root(); let stack_root = z.focus_stack.root_mut().unwrap(); - make_cell_node(stack_root); + make_cell_node(stack_root, z.alloc.clone()); let root_val = z.root_val.as_mut().unwrap(); return (stack_root, unsafe{ &mut **root_val }) } @@ -368,7 +368,11 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + let cell_node = end_node.make_mut().into_cell_node().unwrap(); let (exclusive_node, val) = cell_node.prepare_cf(last_path_byte); + //With an empty `path`, the popped byte was the zipper's own; put it back z.key.prefix_buf.truncate(original_path_len); + if z.key.prefix_buf.len() < original_path_len { + z.key.prefix_buf.push(last_path_byte); + } return (exclusive_node, val) }, @@ -382,7 +386,7 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + |node, key| { let new_node = if key.len() > 0 { if let Some(mut remaining) = node.take_node_at_key(key, false) { - make_cell_node(&mut remaining); + make_cell_node(&mut remaining, alloc.clone()); remaining } else { TrieNodeODRc::new_in(CellByteNode::new_in(alloc.clone()), alloc) @@ -409,11 +413,15 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + } else { //CASE 4 z.key.prefix_buf.truncate(original_path_len); + if z.key.prefix_buf.len() < original_path_len { + z.key.prefix_buf.push(last_path_byte); + } //If the node on top of the stack is not a cell node, we need to upgrade it if !z.focus_stack.top().unwrap().is_cell_node() { + let alloc = z.alloc.clone(); swap_top_node(&mut z.focus_stack, &z.key, |mut existing_node| { - make_cell_node(&mut existing_node); + make_cell_node(&mut existing_node, alloc); existing_node }); } @@ -432,9 +440,9 @@ fn prepare_node_at_path_end<'a, V: Clone + Send + Sync, A: Allocator>(start_node let mut node_ref = node.make_mut(); let mut new_parent = match node_ref.take_node_at_key(remaining_key, false) { Some(downward_node) => downward_node, - None => TrieNodeODRc::new_in(CellByteNode::new_in(alloc.clone()), alloc) + None => TrieNodeODRc::new_in(CellByteNode::new_in(alloc.clone()), alloc.clone()) }; - make_cell_node(&mut new_parent); + make_cell_node(&mut new_parent, alloc.clone()); let result = node_ref.node_set_branch(remaining_key, new_parent); match result { Ok(_) => { }, @@ -445,7 +453,7 @@ fn prepare_node_at_path_end<'a, V: Clone + Send + Sync, A: Allocator>(start_node node = child_node; } else { //Otherwise just upgrade node - make_cell_node(node); + make_cell_node(node, alloc); } node } @@ -1494,4 +1502,170 @@ mod tests { assert_eq!(rz.is_val(), false); drop(rz); } + + /// An exclusive zipper at the head's own root, requested more than once, from a head whose + /// root sits partway into a node + #[test] + fn exclusive_path_at_head_root_twice() { + let sample = || { + let mut m = PathMap::::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 2, 1, 3, 3], &[0], &[2, 2]] { m.set_val_at(p, 7); } + m + }; + + let zh = sample().into_zipper_head(&[1u8]); + for (path, v) in [(&[][..], 1), (&[9u8][..], 2), (&[][..], 3)] { + let mut wz = zh.write_zipper_at_exclusive_path(path).unwrap(); + wz.descend_to(&[5u8]); + wz.set_val(v); + } + let map = zh.into_map(); + assert_eq!(map.get_val_at(&[1u8, 5]), Some(&3)); + assert_eq!(map.get_val_at(&[1u8, 9, 5]), Some(&2)); + assert_eq!(map.get_val_at(&[1u8, 2, 1, 0]), Some(&7)); + assert_eq!(map.val_count(), 7); + + let mut map = sample(); + { + let mut wz = map.write_zipper(); + wz.descend_to(&[1u8]); + let zh = wz.zipper_head(); + for (path, v) in [(&[][..], 1), (&[][..], 2), (&[9u8][..], 3)] { + let mut child = zh.write_zipper_at_exclusive_path(path).unwrap(); + child.descend_to(&[5u8]); + child.set_val(v); + } + } + assert_eq!(map.get_val_at(&[1u8, 5]), Some(&2)); + assert_eq!(map.get_val_at(&[1u8, 9, 5]), Some(&3)); + assert_eq!(map.val_count(), 7); + } + + /// A `ZipperHead` from a write zipper made with a borrowed path, and the zipper used afterwards + #[test] + fn zipper_head_from_write_zipper_at_borrowed_path() { + let mut map = PathMap::::new(); + map.set_val_at(&[1u8, 2, 3], 7); + { + let path = [1u8, 2]; + let mut wz = map.write_zipper_at_path(&path); + { + let zh = wz.zipper_head(); + let mut child = zh.write_zipper_at_exclusive_path(&[4u8]).unwrap(); + child.set_val(1); + } + assert_eq!(wz.origin_path(), &[1u8, 2]); + wz.descend_to(&[5u8]); + wz.set_val(2); + assert_eq!(wz.origin_path(), &[1u8, 2, 5]); + } + assert_eq!(map.get_val_at(&[1u8, 2, 4]), Some(&1)); + assert_eq!(map.get_val_at(&[1u8, 2, 5]), Some(&2)); + assert_eq!(map.get_val_at(&[1u8, 2, 3]), Some(&7)); + } + + /// A reader must not hold a node that live writers point into + #[test] + fn head_reader_beside_live_writers() { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8, 0], 1); + let zh = map.into_zipper_head(&[]); + { + let mut w1 = zh.write_zipper_at_exclusive_path(&[0x11u8]).unwrap(); + //Not in the trie, so it used to hold the root node, which the next writer then copied + let r0 = zh.read_zipper_at_path(&[0x22u8, 0, 0]).unwrap(); + let w0 = zh.write_zipper_at_exclusive_path(&[0u8]).unwrap(); + assert!(!r0.path_exists()); + drop(r0); + w1.set_val(5); + drop(w0); + drop(w1); + } + let map = zh.into_map(); + assert_eq!(map.get_val_at(&[0x11u8]), Some(&5)); + assert_eq!(map.get_val_at(&[0u8, 0]), Some(&1)); + } + + /// `get_trie_ref`, `get_focus` and forks from a head's read zipper, which owns its root node + #[test] + fn head_read_zipper_trie_refs() { + let mut map = PathMap::::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 3], &[0]] { map.set_val_at(p, 7); } + let zh = map.zipper_head(); + for path in [&[][..], &[1u8], &[1u8, 2], &[9u8]] { + let mut rz = zh.read_zipper_at_path(path).unwrap(); + for step in [&[][..], &[2u8], &[2u8, 1]] { + rz.reset(); + rz.descend_to(step); + let tr = rz.get_trie_ref(); + assert_eq!(tr.val(), rz.val(), "{path:?} {step:?}"); + assert_eq!(tr.child_mask(), rz.child_mask(), "{path:?} {step:?}"); + let _ = rz.get_focus(); + let fork = rz.fork_read_zipper(); + assert_eq!(fork.get_trie_ref().val(), rz.val(), "{path:?} {step:?}"); + assert_eq!(fork.trie_ref_at_path(&[0u8]).val(), rz.val_at(&[0u8]), "{path:?} {step:?}"); + } + } + } + + /// Exclusive paths from a head whose focus node is the empty sentinel + #[test] + fn exclusive_path_over_empty_node() { + let setups: [fn(&mut PathMap); 3] = [ + |m| { m.write_zipper_at_path(&[0u8, 0]).remove_branches(false); }, + |m| { let e = PathMap::::new(); m.write_zipper_at_path(&[0u8, 0]).graft(&e.read_zipper()); }, + |m| { m.write_zipper_at_path(&[0u8, 0]).take_map(false); }, + ]; + for (i, setup) in setups.iter().enumerate() { + for paths in [[&[][..], &[5u8][..]], [&[5u8, 6][..], &[][..]], [&[0u8, 0][..], &[1u8][..]]] { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8, 0, 1, 2], 9); + map.set_val_at(&[7u8], 9); + setup(&mut map); + { + let mut wz = map.write_zipper_at_path(&[0u8, 0]); + let zh = wz.zipper_head(); + for (n, p) in paths.iter().enumerate() { + let mut w = zh.write_zipper_at_exclusive_path(p).unwrap(); + w.set_val(n as u64); + } + } + assert_eq!(map.get(&[7u8]), Some(&9), "setup {i} {paths:?}"); + for (n, p) in paths.iter().enumerate() { + let full: Vec = [&[0u8, 0][..], p].concat(); + assert_eq!(map.get(&full), Some(&(n as u64)), "setup {i} {paths:?}"); + } + } + } + } + + /// An exclusive zipper rooted at a link emptied by `remove_branches`, `take_map` or a graft of nothing + #[test] + fn exclusive_zipper_at_emptied_link() { + let preps: [fn(&mut WriteZipperUntracked); 4] = [ + |z| { z.remove_branches(false); }, + |z| { z.take_map(false); }, + |z| { let e = PathMap::::new(); z.graft(&e.read_zipper()); }, + |z| { let e = PathMap::::new(); z.restrict(&e.read_zipper()); }, + ]; + for (i, prep) in preps.iter().enumerate() { + let mut map = PathMap::::new(); + for p in [&[0u8, 0, 1][..], &[0, 0, 1, 2], &[0]] { map.set_val_at(p, 1); } + { + let mut wz = map.write_zipper_at_path(&[0u8, 0]); + prep(&mut wz); + let zh = wz.zipper_head(); + let mut w = zh.write_zipper_at_exclusive_path(&[]).unwrap(); + assert!(!w.descend_to_existing_byte(1), "prep {i}"); + w.descend_to(&[7u8, 7]); + assert!(!w.path_exists(), "prep {i}"); + assert_eq!(w.ascend_until(), 2, "prep {i}"); + w.descend_to(&[7u8, 7]); + w.set_val(5); + } + assert_eq!(map.get(&[0u8, 0, 7, 7]), Some(&5), "prep {i}"); + assert_eq!(map.get(&[0u8]), Some(&1), "prep {i}"); + assert_eq!(map.val_count(), 2, "prep {i}"); + } + } }