diff --git a/CHANGELOG.md b/CHANGELOG.md index 273c00e9..e47139e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ScanIterator`'s zone-map pruning check now decodes the column's compact zone-map table (once per scan, cached) instead of fetching each chunk's own full data segment just to read its embedded stats — over HTTP, checking whether a chunk could be pruned used to cost exactly as much as fetching it outright, so pruning saved nothing for the checked column itself (only for other, unfiltered columns in the same row window). ([#380](https://github.com/dfa1/vortex-java/issues/380)) - `MaskedEncodingEncoder` (nullable columns) now computes `MIN`/`MAX` from only the row-valid elements instead of trusting whichever inner encoder ran over the dense, placeholder-filled values array — a `long[]`/`int[]`/... has no way to represent "no value", so invalid slots carry a placeholder (commonly `0`) that was silently folding into the reported stats; a column with any nulls could report a wrong non-null `MIN`/`MAX` (e.g. `0`) instead of excluding those rows, corrupting both pruning and any consumer trusting the zone-map's actual values. Found while verifying the #378 fix against the real Rust reader on the same data. ([#381](https://github.com/dfa1/vortex-java/pull/381)) - `AlpRdEncodingEncoder` (`vortex.alprd`) now computes `MIN`/`MAX` from the input `double[]`/`float[]` instead of hardcoding `null` on every path — unlike its sibling `AlpEncodingEncoder` (fixed under #379's pattern), ALP-RD never kept a running min/max alongside its dictionary training and exception encoding, so any column the cascade routed through ALP-RD lost zone-map pruning entirely regardless of how narrow or disjoint a range filter was from the data's actual domain. ([#382](https://github.com/dfa1/vortex-java/issues/382)) +- `ConstantEncodingEncoder`, `RunEndEncodingEncoder`, and `ZigZagEncodingEncoder` now report `MIN`/`MAX` zone-map stats instead of hardcoding `null` — same bug as #382, three more encoders. ([#384](https://github.com/dfa1/vortex-java/issues/384), [#385](https://github.com/dfa1/vortex-java/issues/385), [#386](https://github.com/dfa1/vortex-java/issues/386)) +- `SequenceEncodingEncoder`, `PcoEncodingEncoder`, `RleEncodingEncoder`, `SparseEncodingEncoder`, `PatchedEncodingEncoder`, `ZstdEncodingEncoder`, `FsstEncodingEncoder`, `VarBinViewEncodingEncoder`, `ExtEncodingEncoder` (cascade path), and `DateTimePartsEncodingEncoder` — same bug again, found by auditing every remaining encoder for the pattern rather than waiting for another report. A new `ZoneMapStatsCoverageTest` fitness function now asserts every stats-eligible encoder in the default registry actually reports stats, so a future encoder missing this can't land silently. ([#387](https://github.com/dfa1/vortex-java/pull/387)) ## [0.14.1] — 2026-09-06 diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java index fea4a1d5..1e10f0f5 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadeStep.java @@ -46,6 +46,21 @@ public static CascadeStep notApplicable() { return new CascadeStep(null, List.of(), List.of(), null, null, false); } + /// Convenience: applicable step with open children, taking a `{min, max}`-or-`null` stats pair + /// (as returned by e.g. [PrimitiveEncodingEncoder#minMaxStats] / [VarBinEncodingEncoder#minMaxStats]) + /// instead of two independently-nullable arguments. + /// + /// @param partialRoot partially-assembled root encode node + /// @param ownedBuffers buffers owned directly by the root node + /// @param openChildren child slots to be filled recursively by the cascading compressor + /// @param stats a `{min, max}` pair, or `null` when neither stat is available + /// @return an applicable [CascadeStep] with `stats` unpacked into `statsMin`/`statsMax` + public static CascadeStep open(EncodeNode partialRoot, List ownedBuffers, + List openChildren, byte[][] stats) { + return new CascadeStep(partialRoot, ownedBuffers, openChildren, + stats != null ? stats[0] : null, stats != null ? stats[1] : null, true); + } + /// Returns `true` if this step has no open child slots. /// /// @return `true` if the step is terminal (no open children) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 1391b4e1..0f2cf563 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -54,7 +54,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } long firstRaw = readFirstRaw(data, ptype); ProtoScalarValue scalar = buildScalar(ptype, firstRaw); - return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, MemorySegment.ofArray(scalar.encode())); + byte[] scalarBytes = scalar.encode(); + // A constant array's min and max are both the one repeated value, by construction -- no + // scan needed. Empty arrays report no stats, matching every other encoder's convention. + byte[] stats = arrayLength(data, ptype) > 0 ? scalarBytes : null; + return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, MemorySegment.ofArray(scalarBytes), stats, stats); } @Override @@ -105,9 +109,8 @@ private static long readFirstRaw(Object data, PType ptype) { }; } - private static boolean isConstant(Object data, PType ptype) { - long firstRaw = readFirstRaw(data, ptype); - int len = switch (ptype) { + private static int arrayLength(Object data, PType ptype) { + return switch (ptype) { case I8, U8 -> ((byte[]) data).length; case I16, U16 -> ((short[]) data).length; case I32, U32 -> ((int[]) data).length; @@ -116,6 +119,11 @@ private static boolean isConstant(Object data, PType ptype) { case F64 -> ((double[]) data).length; default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); }; + } + + private static boolean isConstant(Object data, PType ptype) { + long firstRaw = readFirstRaw(data, ptype); + int len = arrayLength(data, ptype); for (int i = 1; i < len; i++) { long raw = switch (ptype) { case I8, U8 -> ((byte[]) data)[i]; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java index cf0c506d..fc18572d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoder.java @@ -88,7 +88,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment.ofArray(metaBytes), new EncodeNode[]{daysNode, secondsNode, subsecondsNode}, new int[]{}); - return new EncodeResult(root, List.copyOf(allBuffers), null, null); + // The extension's zone-map min/max is its storage primitive's, unwrapped -- here the raw + // i64 timestamp before it's split into days/seconds/subseconds, not any of the three parts + // individually (signed i64 order matches chronological order regardless of the split). + return EncodeResult.of(root, List.copyOf(allBuffers), PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps())); } @Override @@ -136,6 +139,10 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC new ChildSlot(DType.I64, seconds, 1), new ChildSlot(DType.I64, subseconds, 2)); - return new CascadeStep(partialRoot, List.of(), children, null, null, true); + // See #encode -- same open-children stats requirement as ExtEncodingEncoder#encodeCascade: + // CascadingCompressor#spliceResult takes the step's own stats verbatim, never deriving them + // from a resolved child, so the raw pre-split timestamp's stats must be computed here. + return CascadeStep.open(partialRoot, List.of(), children, + PrimitiveEncodingEncoder.minMaxStats(PType.I64, d.timestamps())); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java index e0c8ab62..e5684585 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java @@ -44,4 +44,16 @@ public static EncodeResult simple(EncodingId encodingId, MemorySegment data) { public boolean hasStats() { return statsMin != null && statsMax != null; } + + /// Convenience factory taking a `{min, max}`-or-`null` stats pair (as returned by e.g. + /// [PrimitiveEncodingEncoder#minMaxStats] / [VarBinEncodingEncoder#minMaxStats]) instead of two + /// independently-nullable arguments. + /// + /// @param rootNode the root encode node describing the encoding tree structure + /// @param buffers flat list of data buffers in the order referenced by `rootNode` + /// @param stats a `{min, max}` pair, or `null` when neither stat is available + /// @return an [EncodeResult] with `stats` unpacked into `statsMin`/`statsMax` + public static EncodeResult of(EncodeNode rootNode, List buffers, byte[][] stats) { + return new EncodeResult(rootNode, buffers, stats != null ? stats[0] : null, stats != null ? stats[1] : null); + } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java index 9b6179b2..666d0508 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoder.java @@ -59,6 +59,13 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { } EncodeNode partialRoot = new EncodeNode(EncodingId.VORTEX_EXT, null, new EncodeNode[1], new int[0]); ChildSlot slot = new ChildSlot(ext.storageDType(), data, 0); - return new CascadeStep(partialRoot, List.of(), List.of(slot), null, null, true); + // CascadingCompressor#spliceResult takes a step's stats verbatim -- it never derives them + // from a resolved open child -- so an open storage slot needs its stats computed here, + // independently of whatever encoding the cascade eventually picks for it (matching + // ZoneMapStatCodec#zoneMinMaxDtype: an Extension's zone-map min/max is its storage + // primitive's, unwrapped). + byte[][] stats = ext.storageDType() instanceof DType.Primitive p + ? PrimitiveEncodingEncoder.minMaxStats(p.ptype(), data) : null; + return CascadeStep.open(partialRoot, List.of(), List.of(slot), stats); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java index a5e3f90b..5fbaf396 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java @@ -79,9 +79,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { new EncodeNode[]{uncompLensNode, codesOffNode}, new int[]{0, 1, 2}); - return new EncodeResult(root, - List.of(c.symBuf(), c.symLenBuf(), c.compBuf(), uncompLenBuf, codesOffBuf), - null, null); + // Zone-map min/max is lexicographic-string-only, matching VarBinEncodingEncoder: a + // Binary blob isn't usefully zone-mapped. + byte[][] stats = data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; + return EncodeResult.of(root, + List.of(c.symBuf(), c.symLenBuf(), c.compBuf(), uncompLenBuf, codesOffBuf), stats); } /// Cascading FSST: expose the per-row uncompressed-length and code-offset children as open @@ -109,11 +111,12 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { MemorySegment.ofArray(c.metaBytes()), new EncodeNode[]{null, null}, new int[]{0, 1, 2}); - return new CascadeStep(partialRoot, + byte[][] stats = data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; + return CascadeStep.open(partialRoot, List.of(c.symBuf(), c.symLenBuf(), c.compBuf()), List.of(new ChildSlot(new DType.Primitive(c.uncompLenPType(), false), uncompLens, 0), new ChildSlot(new DType.Primitive(c.codesOffPType(), false), codesOffsets, 1)), - null, null, true); + stats); } /// The FSST-specific product of compression: the symbol-table buffers, the wire code stream, the diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java index 72927873..058173f6 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java @@ -78,14 +78,14 @@ static CascadeStep encodeCascade(DType dtype, Object data) { DType u32Dtype = DType.U32; DType u16Dtype = DType.U16; - return new CascadeStep(partialRoot, List.of(), + return CascadeStep.open(partialRoot, List.of(), List.of( new ChildSlot(dtype, fromLongs(pd.inner, ptype), 0), new ChildSlot(u32Dtype, pd.laneOffsets, 1), new ChildSlot(u16Dtype, pd.patchIndices, 2), new ChildSlot(dtype, fromLongs(pd.patchValues, ptype), 3) ), - null, null, true); + PrimitiveEncodingEncoder.minMaxStats(ptype, data)); } static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { @@ -133,7 +133,8 @@ static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode root = new EncodeNode(EncodingId.VORTEX_PATCHED, MemorySegment.ofArray(metaBytes), new EncodeNode[]{innerNode, laneNode, idxNode, valNode}, new int[]{}); - return new EncodeResult(root, List.of(innerBuf, laneOffsBuf, patchIdxBuf, patchValBuf), null, null); + return EncodeResult.of(root, List.of(innerBuf, laneOffsBuf, patchIdxBuf, patchValBuf), + PrimitiveEncodingEncoder.minMaxStats(ptype, data)); } private static PatchedData computePatchedData(long[] longs, PType ptype, int n) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java index 703934ea..83460550 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoder.java @@ -102,7 +102,7 @@ static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { int[] allBufIdxs = IntStream.range(0, buffers.size()).toArray(); MemorySegment metaBuf = buildMetadata(chunks); EncodeNode node = new EncodeNode(EncodingId.VORTEX_PCO, metaBuf, new EncodeNode[0], allBufIdxs); - return new EncodeResult(node, buffers, null, null); + return EncodeResult.of(node, buffers, PrimitiveEncodingEncoder.minMaxStats(ptype, data)); } private static ChunkResult encodeChunk(long[] latents, PType ptype, int dtypeSize, Arena arena) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java index 911381c5..95e535e4 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java @@ -183,7 +183,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment.ofArray(metaBytes), new EncodeNode[]{valuesNode, indicesNode, offsetsNode}, new int[0]); - return new EncodeResult(root, List.of(valuesSeg, indicesSeg, offsetsSeg), null, null); + return EncodeResult.of(root, List.of(valuesSeg, indicesSeg, offsetsSeg), + PrimitiveEncodingEncoder.minMaxStats(ptype, data)); } private static int rleEncode(long[] input, long[] chunkValues, short[] chunkIndices) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java index f9ccbc2d..5dea435d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoRunEndMetadata; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; import java.util.ArrayList; @@ -109,13 +110,24 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } PType ptype = p.ptype(); int n = arrayLength(data, ptype); + boolean unsign = ptype.isUnsigned(); List ends = new ArrayList<>(); List values = new ArrayList<>(); + long minVal = 0L; + long maxVal = 0L; if (n > 0) { long runVal = readLong(data, ptype, 0); + minVal = runVal; + maxVal = runVal; for (int i = 1; i < n; i++) { long cur = readLong(data, ptype, i); + if (unsign ? Long.compareUnsigned(cur, minVal) < 0 : cur < minVal) { + minVal = cur; + } + if (unsign ? Long.compareUnsigned(cur, maxVal) > 0 : cur > maxVal) { + maxVal = cur; + } if (cur != runVal) { ends.add(i); values.add(runVal); @@ -149,7 +161,16 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode valuesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1); EncodeNode root = new EncodeNode(EncodingId.VORTEX_RUNEND, MemorySegment.ofArray(metaBytes), new EncodeNode[]{endsNode, valuesNode}, new int[0]); - return new EncodeResult(root, List.of(endsBuf, valuesBuf), null, null); + byte[] statsMin = n > 0 ? statsBytes(ptype, minVal) : null; + byte[] statsMax = n > 0 ? statsBytes(ptype, maxVal) : null; + return new EncodeResult(root, List.of(endsBuf, valuesBuf), statsMin, statsMax); + } + + private static byte[] statsBytes(PType ptype, long value) { + if (ptype.isUnsigned()) { + return ProtoScalarValue.ofUint64Value(value).encode(); + } + return ProtoScalarValue.ofInt64Value(value).encode(); } private static int arrayLength(Object data, PType ptype) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index b018a0f0..8c314b1a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -66,7 +66,18 @@ private static EncodeResult encodeInteger(PType pt, Object data) { } ProtoScalarValue baseScalar = buildIntScalar(pt, base); ProtoScalarValue mulScalar = buildIntScalar(pt, multiplier); - return buildResult(baseScalar, mulScalar); + // A perfect arithmetic sequence is monotonic (or constant) end to end, so its extremes are + // always the first and last element -- no separate scan needed. + boolean unsign = pt.isUnsigned(); + byte[] statsMin = null; + byte[] statsMax = null; + if (n > 0) { + long last = base + (long) (n - 1) * multiplier; + boolean baseIsMin = unsign ? Long.compareUnsigned(base, last) <= 0 : base <= last; + statsMin = buildIntScalar(pt, baseIsMin ? base : last).encode(); + statsMax = buildIntScalar(pt, baseIsMin ? last : base).encode(); + } + return buildResult(baseScalar, mulScalar, statsMin, statsMax); } private static EncodeResult encodeF32(float[] data) { @@ -77,7 +88,14 @@ private static EncodeResult encodeF32(float[] data) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } } - return buildResult(ProtoScalarValue.ofF32Value(base), ProtoScalarValue.ofF32Value(mul)); + byte[] statsMin = null; + byte[] statsMax = null; + if (data.length > 0) { + float last = base + (data.length - 1) * mul; + statsMin = ProtoScalarValue.ofF32Value(Math.min(base, last)).encode(); + statsMax = ProtoScalarValue.ofF32Value(Math.max(base, last)).encode(); + } + return buildResult(ProtoScalarValue.ofF32Value(base), ProtoScalarValue.ofF32Value(mul), statsMin, statsMax); } private static EncodeResult encodeF64(double[] data) { @@ -88,7 +106,14 @@ private static EncodeResult encodeF64(double[] data) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } } - return buildResult(ProtoScalarValue.ofF64Value(base), ProtoScalarValue.ofF64Value(mul)); + byte[] statsMin = null; + byte[] statsMax = null; + if (data.length > 0) { + double last = base + (data.length - 1) * mul; + statsMin = ProtoScalarValue.ofF64Value(Math.min(base, last)).encode(); + statsMax = ProtoScalarValue.ofF64Value(Math.max(base, last)).encode(); + } + return buildResult(ProtoScalarValue.ofF64Value(base), ProtoScalarValue.ofF64Value(mul), statsMin, statsMax); } private static EncodeResult encodeF16(short[] data) { @@ -102,16 +127,26 @@ private static EncodeResult encodeF16(short[] data) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } } + byte[] statsMin = null; + byte[] statsMax = null; + if (data.length > 0) { + float lastF = baseF + (data.length - 1) * mulF; + short minShort = Float.floatToFloat16(Math.min(baseF, lastF)); + short maxShort = Float.floatToFloat16(Math.max(baseF, lastF)); + statsMin = ProtoScalarValue.ofF16Value(Short.toUnsignedLong(minShort)).encode(); + statsMax = ProtoScalarValue.ofF16Value(Short.toUnsignedLong(maxShort)).encode(); + } return buildResult( ProtoScalarValue.ofF16Value(Short.toUnsignedLong(baseShort)), - ProtoScalarValue.ofF16Value(Short.toUnsignedLong(mulShort))); + ProtoScalarValue.ofF16Value(Short.toUnsignedLong(mulShort)), + statsMin, statsMax); } - private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul) { + private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, byte[] statsMin, byte[] statsMax) { ProtoSequenceMetadata meta = new ProtoSequenceMetadata(base, mul); MemorySegment metaBuf = MemorySegment.ofArray(meta.encode()); EncodeNode node = new EncodeNode(EncodingId.VORTEX_SEQUENCE, metaBuf, new EncodeNode[0], new int[]{}); - return new EncodeResult(node, List.of(), null, null); + return new EncodeResult(node, List.of(), statsMin, statsMax); } private static ProtoScalarValue buildIntScalar(PType pt, long value) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java index 7cf3d14b..a0000865 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java @@ -117,7 +117,8 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { DType idxDtype = new DType.Primitive(idxPtype, false); ChildSlot idxSlot = new ChildSlot(idxDtype, idxArr, 0); ChildSlot valSlot = new ChildSlot(dtype, valArr, 1); - return new CascadeStep(partialRoot, List.of(fillBuf), List.of(idxSlot, valSlot), null, null, true); + return CascadeStep.open(partialRoot, List.of(fillBuf), List.of(idxSlot, valSlot), + PrimitiveEncodingEncoder.minMaxStats(ptype, data)); } private static Object idxArr(List patchIdx, PType idxPtype) { @@ -323,7 +324,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2); EncodeNode root = new EncodeNode(EncodingId.VORTEX_SPARSE, MemorySegment.ofArray(metaBytes), new EncodeNode[]{idxNode, valNode}, new int[]{0}); - return new EncodeResult(root, List.of(fillBuf, idxBuf, valBuf), null, null); + return EncodeResult.of(root, List.of(fillBuf, idxBuf, valBuf), + PrimitiveEncodingEncoder.minMaxStats(ptype, data)); } private static int arrayLength(Object data, PType ptype) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java index 0982ffa3..2408c9ae 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java @@ -68,6 +68,9 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } EncodeNode root = new EncodeNode(EncodingId.VORTEX_VARBINVIEW, null, new EncodeNode[0], bufIndices); - return new EncodeResult(root, buffers, null, null); + // Zone-map min/max is lexicographic-string-only, matching VarBinEncodingEncoder: a + // Binary blob isn't usefully zone-mapped. + byte[][] stats = data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; + return EncodeResult.of(root, buffers, stats); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java index 294d82f0..5fa4b43e 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -30,12 +31,27 @@ public boolean accepts(DType dtype) { @Override public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { PType signed = ((DType.Primitive) dtype).ptype(); + // Zigzag's bit-interleaving is not order-preserving (e.g. -1 maps to 1, 1 maps to 2), so + // stats must be tracked over the original signed values in the same pass, not read back + // from the transformed unsigned output afterward. + long[] minMax = new long[2]; + int n = arrayLength(data, signed); MemorySegment seg = switch (signed) { case I8 -> { byte[] arr = (byte[]) data; MemorySegment s = ctx.arena().allocate(arr.length); + if (n > 0) { + minMax[0] = arr[0]; + minMax[1] = arr[0]; + } for (int i = 0; i < arr.length; i++) { byte v = arr[i]; + if (v < minMax[0]) { + minMax[0] = v; + } + if (v > minMax[1]) { + minMax[1] = v; + } s.set(ValueLayout.JAVA_BYTE, i, (byte) ((v << 1) ^ (v >> 7))); } yield s; @@ -43,8 +59,18 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { case I16 -> { short[] arr = (short[]) data; MemorySegment s = ctx.arena().allocate((long) arr.length * 2, 2); + if (n > 0) { + minMax[0] = arr[0]; + minMax[1] = arr[0]; + } for (int i = 0; i < arr.length; i++) { short v = arr[i]; + if (v < minMax[0]) { + minMax[0] = v; + } + if (v > minMax[1]) { + minMax[1] = v; + } s.setAtIndex(VortexFormat.LE_SHORT, i, (short) ((v << 1) ^ (v >> 15))); } yield s; @@ -52,8 +78,18 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { case I32 -> { int[] arr = (int[]) data; MemorySegment s = ctx.arena().allocate((long) arr.length * 4, 4); + if (n > 0) { + minMax[0] = arr[0]; + minMax[1] = arr[0]; + } for (int i = 0; i < arr.length; i++) { int v = arr[i]; + if (v < minMax[0]) { + minMax[0] = v; + } + if (v > minMax[1]) { + minMax[1] = v; + } s.setAtIndex(VortexFormat.LE_INT, i, (v << 1) ^ (v >> 31)); } yield s; @@ -61,8 +97,18 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { case I64 -> { long[] arr = (long[]) data; MemorySegment s = ctx.arena().allocate((long) arr.length * 8, 8); + if (n > 0) { + minMax[0] = arr[0]; + minMax[1] = arr[0]; + } for (int i = 0; i < arr.length; i++) { long v = arr[i]; + if (v < minMax[0]) { + minMax[0] = v; + } + if (v > minMax[1]) { + minMax[1] = v; + } s.setAtIndex(VortexFormat.LE_LONG, i, (v << 1) ^ (v >> 63)); } yield s; @@ -71,6 +117,18 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { }; EncodeNode child = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ZIGZAG, null, new EncodeNode[]{child}, new int[0]); - return new EncodeResult(root, List.of(seg), null, null); + byte[] statsMin = n > 0 ? ProtoScalarValue.ofInt64Value(minMax[0]).encode() : null; + byte[] statsMax = n > 0 ? ProtoScalarValue.ofInt64Value(minMax[1]).encode() : null; + return new EncodeResult(root, List.of(seg), statsMin, statsMax); + } + + private static int arrayLength(Object data, PType ptype) { + return switch (ptype) { + case I8 -> ((byte[]) data).length; + case I16 -> ((short[]) data).length; + case I32 -> ((int[]) data).length; + case I64 -> ((long[]) data).length; + default -> throw new VortexException(EncodingId.VORTEX_ZIGZAG, "unsupported ptype: " + ptype); + }; } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java index c7351a0e..0d9fd92d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoZstdFrameMetadata; import io.github.dfa1.vortex.core.proto.ProtoZstdMetadata; @@ -115,7 +116,10 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { throw new VortexException(EncodingId.VORTEX_ZSTD, "non-nullable " + dtype + " contains null"); } - return encodeVarBin(encoded, ctx.arena()); + // Zone-map min/max is lexicographic-string-only, matching VarBinEncodingEncoder: a + // Binary blob (e.g. audio bytes) isn't usefully zone-mapped. + byte[][] stats = data instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; + return encodeVarBin(encoded, ctx.arena(), stats); } throw new VortexException(EncodingId.VORTEX_ZSTD, "unsupported dtype: " + dtype); } @@ -124,21 +128,22 @@ private EncodeResult encodePrimitive(DType.Primitive dt, Object data, Arena aren int byteWidth = dt.ptype().byteSize(); MemorySegment raw = primitiveToLeBytes(dt.ptype(), data, arena); long n = primitiveLength(dt.ptype(), data); - return buildResult(raw, uniformLayout(n, byteWidth), arena); + byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(dt.ptype(), data); + return buildResult(raw, uniformLayout(n, byteWidth), arena, stats); } - private EncodeResult encodeVarBin(byte[][] encoded, Arena arena) { + private EncodeResult encodeVarBin(byte[][] encoded, Arena arena, byte[][] stats) { MemorySegment raw = buildLengthPrefixed(encoded, arena); - return buildResult(raw, varBinLayout(raw, encoded.length), arena); + return buildResult(raw, varBinLayout(raw, encoded.length), arena, stats); } - private EncodeResult buildResult(MemorySegment raw, FrameLayout layout, Arena arena) { + private EncodeResult buildResult(MemorySegment raw, FrameLayout layout, Arena arena, byte[][] stats) { // Zero-copy: each frame is an arena-native slice of raw, compressed straight into another // arena segment. A single-value-per-array config yields one frame (the prior behavior). Frames frames = compressFrames(raw, layout, arena); EncodeNode root = new EncodeNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(frames.metadata()), new EncodeNode[0], frameBufferIndices(frames.compressed().size(), 0)); - return new EncodeResult(root, List.copyOf(frames.compressed()), null, null); + return EncodeResult.of(root, List.copyOf(frames.compressed()), stats); } private EncodeResult encodeNullablePrimitive(DType.Primitive dt, NullableData nd, EncodeContext ctx) { @@ -149,7 +154,11 @@ private EncodeResult encodeNullablePrimitive(DType.Primitive dt, NullableData nd // reference). The decoder scatters them back over the validity mask carried by child[0]. MemorySegment full = primitiveToLeBytes(dt.ptype(), nd.values(), arena); MemorySegment packed = packValidBytes(full, validity, byteWidth, arena); - return buildNullableResult(packed, uniformLayout(countValid(validity), byteWidth), validity, ctx); + // Stats must come from only the valid elements -- the dense values array carries + // placeholder garbage (commonly 0) at invalid positions, same as MaskedEncodingEncoder. + Object compacted = PrimitiveArrays.compact(dt.ptype(), nd.values(), validity); + byte[][] stats = PrimitiveEncodingEncoder.minMaxStats(dt.ptype(), compacted); + return buildNullableResult(packed, uniformLayout(countValid(validity), byteWidth), validity, ctx, stats); } private EncodeResult encodeNullableVarBin(NullableData nd, EncodeContext ctx) { @@ -157,11 +166,13 @@ private EncodeResult encodeNullableVarBin(NullableData nd, EncodeContext ctx) { // reference). The decoder scatters them back over the validity mask carried by child[0]. byte[][] valid = stripNulls(VarBinBytes.toRawByteArrays(nd.values())); MemorySegment packed = buildLengthPrefixed(valid, ctx.arena()); - return buildNullableResult(packed, varBinLayout(packed, valid.length), nd.validity(), ctx); + // minMaxStats already skips null entries itself, so the un-stripped values array is fine. + byte[][] stats = nd.values() instanceof String[] strings ? VarBinEncodingEncoder.minMaxStats(strings) : null; + return buildNullableResult(packed, varBinLayout(packed, valid.length), nd.validity(), ctx, stats); } private EncodeResult buildNullableResult( - MemorySegment raw, FrameLayout layout, boolean[] validity, EncodeContext ctx) { + MemorySegment raw, FrameLayout layout, boolean[] validity, EncodeContext ctx, byte[][] stats) { Frames frames = compressFrames(raw, layout, ctx.arena()); int frameCount = frames.compressed().size(); @@ -176,7 +187,7 @@ private EncodeResult buildNullableResult( EncodeNode root = new EncodeNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(frames.metadata()), new EncodeNode[]{validityNode}, frameBufferIndices(frameCount, 0)); - return new EncodeResult(root, buffers, null, null); + return EncodeResult.of(root, buffers, stats); } /// Byte spans and value counts of each frame; spans sum to the payload size. diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java index 4e57cf61..96353a16 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java @@ -293,6 +293,56 @@ void encodeCascade_mixedValues_notApplicable() { } } + /// #384: encode() built its EncodeResult via EncodeResult.simple(...), which defaults stats to + /// null regardless of the constant value — even though min == max == that value by construction. + @Nested + class Stats { + + @Test + void encode_i64_reportsValueAsMinAndMax() throws java.io.IOException { + // Given + long constant = -12345L; + + // When + EncodeResult result = ENCODER.encode(DTypes.I64, new long[]{constant, constant}, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(constant); + assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(constant); + } + + @Test + void encode_f64_reportsValueAsMinAndMax() throws java.io.IOException { + // Given + double constant = 3.5; + + // When + EncodeResult result = ENCODER.encode(DTypes.F64, new double[]{constant, constant, constant}, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).f64_value()).isEqualTo(constant); + assertThat(scalar(result.statsMax()).f64_value()).isEqualTo(constant); + } + + @Test + void encode_empty_statsAreNull() { + // Given + long[] data = {}; + + // When + EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(result.statsMin()).isNull(); + assertThat(result.statsMax()).isNull(); + } + + private static ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException { + MemorySegment seg = MemorySegment.ofArray(bytes); + return ProtoScalarValue.decode(seg, 0, seg.byteSize()); + } + } + /// Rust can write a constant array whose scalar is null (proto null_value tag). /// The decoder must return a [NullArray] — not 0 / false (#246). @Nested diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java index 3541a836..f1f881ab 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java @@ -176,4 +176,54 @@ void encode_i64_metadata_numRuns_andEndsPtype() throws Exception { assertThat(meta.ends_ptype().value()).isEqualTo(2); } } + + /// #385: encode() hardcoded (null, null) regardless of the run values actually encoded. + @Nested + class Stats { + + @Test + void encode_i64_reportsMinMaxAcrossRuns() throws java.io.IOException { + // Given — min/max must come from every run's value, not just the first/last run + long[] data = {10L, 10L, -5L, -5L, 3L, 3L, 3L}; + + // When + EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(-5L); + assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(10L); + } + + @Test + void encode_u32_usesUnsignedComparison() throws java.io.IOException { + // Given — a raw-bit value that would look negative under signed comparison but is a + // huge positive magnitude as U32; a signed `<`/`>` comparison would pick the wrong min. + int[] data = {1, 1, -1, -1}; // -1 raw bits == 4294967295 unsigned + + // When + EncodeResult result = ENCODER.encode(DTypes.U32, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).uint64_value()).isEqualTo(1L); + assertThat(scalar(result.statsMax()).uint64_value()).isEqualTo(4294967295L); + } + + @Test + void encode_empty_statsAreNull() { + // Given + long[] data = {}; + + // When + EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(result.statsMin()).isNull(); + assertThat(result.statsMax()).isNull(); + } + + private static io.github.dfa1.vortex.core.proto.ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException { + MemorySegment seg = MemorySegment.ofArray(bytes); + return io.github.dfa1.vortex.core.proto.ProtoScalarValue.decode(seg, 0, seg.byteSize()); + } + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java index 9b2fb2e7..1f1904e2 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java @@ -228,4 +228,55 @@ void encodeDecode_i64_isLossless(long[] data) { } } } + + /// #386: encode() hardcoded (null, null) regardless of the signed input. Stats must come from + /// the original values, not the zigzag-transformed (not order-preserving) output. + @Nested + class Stats { + + @Test + void encode_i32_reportsMinMaxOfOriginalValues() throws java.io.IOException { + // Given — the most negative value zigzag-maps to the largest unsigned code (-1 -> 1, + // 5 -> 10); a bug reading stats off the transformed output would report min=0, max=10 + int[] data = {5, -1, 0, 3}; + + // When + EncodeResult result = ENCODER.encode(DTypes.I32, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(-1L); + assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(5L); + } + + @Test + void encode_i64_reportsMinMaxOfOriginalValues() throws java.io.IOException { + // Given + long[] data = {Long.MIN_VALUE, Long.MAX_VALUE, 0L}; + + // When + EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).int64_value()).isEqualTo(Long.MIN_VALUE); + assertThat(scalar(result.statsMax()).int64_value()).isEqualTo(Long.MAX_VALUE); + } + + @Test + void encode_empty_statsAreNull() { + // Given + int[] data = {}; + + // When + EncodeResult result = ENCODER.encode(DTypes.I32, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(result.statsMin()).isNull(); + assertThat(result.statsMax()).isNull(); + } + + private static io.github.dfa1.vortex.core.proto.ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException { + MemorySegment seg = MemorySegment.ofArray(bytes); + return io.github.dfa1.vortex.core.proto.ProtoScalarValue.decode(seg, 0, seg.byteSize()); + } + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsCoverageTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsCoverageTest.java new file mode 100644 index 00000000..22b832fd --- /dev/null +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZoneMapStatsCoverageTest.java @@ -0,0 +1,116 @@ +package io.github.dfa1.vortex.writer.encode; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.TimeUnit; +import io.github.dfa1.vortex.core.testing.DTypes; +import io.github.dfa1.vortex.writer.WriteRegistry; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.lang.foreign.MemorySegment; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Fitness function for #382/#384/#385/#386 and the sibling bugs found in the same audit +/// (`Pco`/`Rle`/`Sparse`/`Patched`/`Zstd`/`Fsst`/`VarBinView`/`Ext`/`DateTimeParts`/`Sequence`): +/// every registered [EncodingEncoder] that can encode a `Primitive`/`Extension`/`Utf8` value must +/// report zone-map `MIN`/`MAX` stats for representative non-empty input, or `RowFilter` pruning +/// silently no-ops for any column that encoder wins. +/// +/// Two checks: +/// - [#everyStatsEligibleDefaultEncoder_hasACoverageCase] — registry-driven: every default encoder +/// whose [EncodingEncoder#accepts] matches one of a few representative dtypes must appear in +/// [#coverageCases]. Add an encoder capable of `Primitive`/`Extension`/`Utf8` without adding a +/// case here and this fails — the whole point, so the next such bug can't land silently. +/// - [#encode_reportsMinMaxStats] — the actual per-encoder assertion, run over every case. +/// +/// Deliberately excluded (no zone-map `MIN`/`MAX` concept at all, per +/// `ZoneMapStatCodec#zoneMinMaxDtype`): `Decimal`/`DecimalByteParts` (dtype `Decimal`, not in the +/// codec's supported set), `Bool`/`ByteBool` (dtype `Bool`), every structural/collection encoder +/// (`Chunked`, `FixedSizeList`, `List`, `ListView`, `Map`, `Null`, `Struct`, `Variant`) — none +/// accept a `Primitive`/`Extension`/`Utf8` dtype, so they never surface via the registry probe +/// below. `MaskedEncodingEncoder` is excluded too: its `accepts()` is unconditionally `false` (it +/// is special-dispatched for nullable columns, never registry-selected), so it cannot appear via +/// this probe either — its own stats correctness (#381) is covered by its dedicated test class. +class ZoneMapStatsCoverageTest { + + private static final DType TIMESTAMP_MS = new DType.Extension( + "vortex.timestamp", DType.I64, MemorySegment.ofArray(new byte[]{(byte) TimeUnit.Milliseconds.ordinal(), 0, 0}), false); + + /// Representative dtypes to probe every default-registered encoder's [EncodingEncoder#accepts] + /// with. Deliberately not exhaustive over every `PType` -- just enough to surface every + /// stats-eligible encoder class at least once (an integer, a float, a string, an extension). + private static final DType[] PROBE_DTYPES = {DTypes.I64, DTypes.F64, DTypes.UTF8, TIMESTAMP_MS}; + + @Test + void everyStatsEligibleDefaultEncoder_hasACoverageCase() { + // Given -- every encoder the default registry would actually select + WriteRegistry registry = WriteRegistry.builder().registerDefaults().build(); + + // When -- narrowed to those that can encode at least one representative comparable dtype + Set> statsEligible = new HashSet<>(); + for (EncodingEncoder encoder : registry.encoderMap().values()) { + for (DType probe : PROBE_DTYPES) { + if (encoder.accepts(probe)) { + statsEligible.add(encoder.getClass()); + break; + } + } + } + Set> covered = coverageCases().map(a -> a.get()[1].getClass()).collect(java.util.stream.Collectors.toSet()); + + // Then -- every stats-eligible encoder has a coverage case (new encoder + no case here = failure) + assertThat(covered).containsAll(statsEligible); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("coverageCases") + void encode_reportsMinMaxStats(String label, EncodingEncoder encoder, DType dtype, Object data) { + // Given / When + EncodeResult result = encoder.encode(dtype, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(result.hasStats()).as(label).isTrue(); + } + + static Stream coverageCases() { + return Stream.of( + Arguments.of("Alp/f64", new AlpEncodingEncoder(), DTypes.F64, new double[]{1.1, 2.2, 3.3, 4.4}), + Arguments.of("AlpRd/f64", new AlpRdEncodingEncoder(), DTypes.F64, new double[]{0.5, -3.25, 10.0, 2.0}), + Arguments.of("Bitpacked/i64", new BitpackedEncodingEncoder(), DTypes.I64, new long[]{1L, 2L, 3L, 4L, 5L}), + Arguments.of("Constant/i64", new ConstantEncodingEncoder(), DTypes.I64, new long[]{7L, 7L, 7L}), + Arguments.of("DateTimeParts/timestamp", new DateTimePartsEncodingEncoder(), TIMESTAMP_MS, + new DateTimePartsData(new long[]{1_700_000_000_000L, 1_700_000_100_000L, 1_699_999_900_000L}, false)), + Arguments.of("Delta/i64", new DeltaEncodingEncoder(), DTypes.I64, new long[]{10L, 20L, 15L, 30L}), + Arguments.of("Dict/i32", new DictEncodingEncoder(), DTypes.I32, new int[]{1, 1, 2, 2, 3}), + Arguments.of("Ext/timestamp", new ExtEncodingEncoder(), TIMESTAMP_MS, new long[]{100L, 200L, 300L}), + Arguments.of("FrameOfReference/i64", new FrameOfReferenceEncodingEncoder(), DTypes.I64, new long[]{1000L, 1001L, 1002L, 1003L}), + Arguments.of("Fsst/utf8", new FsstEncodingEncoder(), DTypes.UTF8, new String[]{"hello", "world", "hello"}), + Arguments.of("Patched/i32", new PatchedEncodingEncoder(), DTypes.I32, new int[]{1, 2, 3, 4, 1_000_000}), + Arguments.of("Pco/i64", new PcoEncodingEncoder(), DTypes.I64, longRange(0, 4096)), + Arguments.of("Primitive/i32", new PrimitiveEncodingEncoder(), DTypes.I32, new int[]{1, 2, 3}), + Arguments.of("Rle/i32", new RleEncodingEncoder(), DTypes.I32, new int[]{1, 1, 2, 2, 3, 3}), + Arguments.of("RunEnd/i64", new RunEndEncodingEncoder(), DTypes.I64, new long[]{1L, 1L, 2L, 2L, 3L}), + Arguments.of("Sequence/i64", new SequenceEncodingEncoder(), DTypes.I64, new long[]{10L, 20L, 30L, 40L}), + Arguments.of("Sparse/i32", new SparseEncodingEncoder(), DTypes.I32, new int[]{0, 0, 5, 0, 0}), + Arguments.of("VarBin/utf8", new VarBinEncodingEncoder(), DTypes.UTF8, new String[]{"apple", "banana"}), + Arguments.of("VarBinView/utf8", new VarBinViewEncodingEncoder(), DTypes.UTF8, new String[]{"apple", "banana"}), + Arguments.of("ZigZag/i32", new ZigZagEncodingEncoder(), DTypes.I32, new int[]{-1, 1, -2, 2}), + Arguments.of("Zstd/i64", new ZstdEncodingEncoder(), DTypes.I64, new long[]{1L, 2L, 3L, 4L, 5L}) + ); + } + + private static long[] longRange(long start, int n) { + long[] a = new long[n]; + for (int i = 0; i < n; i++) { + a[i] = start + i; + } + return a; + } +}