From cd7d2641935a8178d4591ff81ed49584e4e3c3b9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 9 Sep 2026 20:16:37 +0200 Subject: [PATCH 1/4] fix(chunk-grids): enforce one zero-length-axis invariant across model, clamps and metadata Invariant: a chunk edge length is always >= 1; a dimension's extent may be 0, in which case the dimension has zero chunks (ceildiv(0, size) == 0). Zero-length-axis bugs have recurred since 2017 (#150, #241, #303, #972, #1977, #2434, #3711, #4305, #4307, #4328) because the layers disagreed on this invariant and every span-derived chunk spelling clamped on its own: - The metadata layer (common.py, metadata/v3.py) required chunk edges >= 1, but the in-memory FixedDimension allowed size == 0 with four special-case branches left over from #2434, so normalization could build a grid the metadata constructor then rejected. FixedDimension now rejects size < 1 and the four `if self.size == 0` branches are gone. VaryingDimension already required edges > 0 and is unchanged. - `chunks=-1`, `chunks=False`, `chunks="auto"` (_guess_regular_chunks, both the typesize == 0 early return and the np.maximum line) and `shards="auto"` each derived "one chunk covering the axis" independently. They now all go through one helper, `_full_span_chunk_size(span) = max(span, 1)`, which is the single definition of that phrase for a possibly zero-length axis. - Zarr format 2 metadata had no chunk >= 1 check, so a legacy `chunks: [0]` document opened fine and read uninitialised memory after a resize. It now raises a clear ValueError at parse time, matching the format 3 grid. - Rectilinear grids had no creation-time spelling for a zero-length axis: normalize_chunks_1d required sum(edges) == span, which no list of positive edges can satisfy for span 0, even though the same state is reachable via resize((0,)) and round-trips through reopen. For span == 0 any non-empty list of positive edges is now accepted verbatim, producing the same VaryingDimension(edges, extent=0) that resize produces; the strict sum check is kept for span > 0. Tests: the per-spelling regression test from #4328 is replaced by one matrix over {-1, False, "auto", 1, (1,...), [[2, 2]]} x {(0,), (0, 4), (4, 0), (0, 0), ()} x {v2, v3} x {no shards, shards="auto" with and without a byte budget, explicit shards}, with separate small tests for each error case. Tests that constructed FixedDimension(size=0) now assert it raises, and a zero-extent test covers the behaviour the old special cases were guarding. Assisted-by: ClaudeCode:claude-fable-5-1 --- changes/0000.bugfix.md | 1 + docs/user-guide/arrays.md | 6 + src/zarr/core/chunk_grids.py | 75 ++++++---- src/zarr/core/metadata/v2.py | 7 + tests/test_chunk_grids.py | 229 ++++++++++++++++++++++++------- tests/test_metadata/test_v2.py | 12 ++ tests/test_unified_chunk_grid.py | 77 +++++------ 7 files changed, 284 insertions(+), 123 deletions(-) create mode 100644 changes/0000.bugfix.md diff --git a/changes/0000.bugfix.md b/changes/0000.bugfix.md new file mode 100644 index 0000000000..8b224c3f76 --- /dev/null +++ b/changes/0000.bugfix.md @@ -0,0 +1 @@ +Zero-length dimensions are now handled by a single rule instead of a per-spelling patch: a chunk edge length is always at least 1, while a dimension's extent may be 0 (such a dimension simply has zero chunks). Every way of asking for "one chunk covering the axis" — `chunks=-1`, `chunks=False`, `chunks="auto"`, and `shards="auto"` — now derives the chunk size from the same helper, so they agree on chunk size 1 for a zero-length axis in both Zarr formats. Zarr format 2 metadata now rejects a chunk edge length of 0 with a clear error, matching Zarr format 3, instead of reading uninitialised data after a resize. Rectilinear chunk grids (`chunks=[[...], ...]`) can now be created on a zero-length dimension: since no list of positive edge lengths can sum to 0, the given edge lengths are stored as-is and describe the chunks the dimension will grow into on `append` or `resize`, exactly the state a rectilinear dimension is in after being resized down to 0. The private `FixedDimension(size=0, ...)` model, which previously carried its own zero-size special cases, now raises `ValueError`. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 707fc1a1a5..ebc4b9aee1 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -708,6 +708,12 @@ z.append(np.arange(10, dtype='float64')) print(f"After append: shape={z.shape}, chunk_sizes={z.write_chunk_sizes}") ``` +A rectilinear array can also be created with a zero-length dimension: because no +list of positive chunk sizes can sum to 0, the chunk sizes given for such a +dimension are stored as-is and describe the chunks the dimension will grow into +on `append` or `resize` — the same state as resizing an existing rectilinear +dimension down to 0. + ### Compressors and filters Rectilinear arrays work with all codecs — compressors, filters, and checksums. diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index 6242994fdf..f759e914b6 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -45,22 +45,24 @@ @dataclass(frozen=True) class FixedDimension: """Uniform chunk size. Boundary chunks contain less data but are - encoded at full size by the codec pipeline.""" + encoded at full size by the codec pipeline. - size: int # chunk edge length (>= 0) - extent: int # array dimension length + The chunk edge length is always at least 1, matching the invariant the + metadata layer enforces for every stored chunk grid. The extent may be 0: + a zero-length axis simply has zero chunks (``ceildiv(0, size) == 0``). + """ + + size: int # chunk edge length (>= 1) + extent: int # array dimension length (>= 0) nchunks: int = field(init=False, repr=False) ngridcells: int = field(init=False, repr=False) def __post_init__(self) -> None: - if self.size < 0: - raise ValueError(f"FixedDimension size must be >= 0, got {self.size}") + if self.size < 1: + raise ValueError(f"FixedDimension size must be >= 1, got {self.size}") if self.extent < 0: raise ValueError(f"FixedDimension extent must be >= 0, got {self.extent}") - if self.size == 0: - n = 0 - else: - n = ceildiv(self.extent, self.size) + n = ceildiv(self.extent, self.size) object.__setattr__(self, "nchunks", n) object.__setattr__(self, "ngridcells", n) @@ -69,8 +71,6 @@ def index_to_chunk(self, idx: int) -> int: raise IndexError(f"Negative index {idx} is not allowed") if idx >= self.extent: raise IndexError(f"Index {idx} is out of bounds for extent {self.extent}") - if self.size == 0: - return 0 return idx // self.size def chunk_offset(self, chunk_ix: int) -> int: @@ -95,8 +95,6 @@ def data_size(self, chunk_ix: int) -> int: Does not validate *chunk_ix* — callers must ensure it is in ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access. """ - if self.size == 0: - return 0 return max(0, min(self.size, self.extent - chunk_ix * self.size)) @property @@ -110,8 +108,6 @@ def _unique_edge_lengths(self) -> Iterable[int]: return (self.size,) def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: - if self.size == 0: - return np.zeros_like(indices) return indices // self.size def with_extent(self, new_extent: int) -> FixedDimension: @@ -640,6 +636,20 @@ class ChunkLayout(NamedTuple): inner: ChunkLayout | None = None +def _full_span_chunk_size(span: int) -> int: + """The edge length of one chunk covering an entire axis of length *span*. + + This is *the* definition of "one chunk spans the axis" for a possibly + zero-length axis. Chunk edge lengths must be at least 1 (the invariant + shared by `FixedDimension`, `VaryingDimension` and the stored chunk grid + metadata), so a zero-length axis gets chunk size 1 and zero chunks. Every + spelling that derives a chunk size from a span — ``chunks=-1``, + ``chunks=False``, ``chunks="auto"``, ``shards="auto"`` — must route + through this helper rather than clamping on its own. + """ + return max(span, 1) + + def _guess_regular_chunks( shape: tuple[int, ...] | int, typesize: int, @@ -677,11 +687,10 @@ def _guess_regular_chunks( shape = (shape,) if typesize == 0: - return shape + return tuple(_full_span_chunk_size(s) for s in shape) ndims = len(shape) - # require chunks to have non-zero length for all dimensions - chunks = np.maximum(np.array(shape, dtype="=f8"), 1) + chunks = np.array([_full_span_chunk_size(s) for s in shape], dtype="=f8") # Determine the optimal chunk size in bytes using a PyTables expression. # This is kept as a float. @@ -724,13 +733,21 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG the span, and the uniform form is O(1) in the number of chunks — a dimension with `2**62` chunks must not materialize one entry per chunk. - `-1` means "one chunk covering the entire span." + `-1` means "one chunk covering the entire span" (see `_full_span_chunk_size` + for what that means on a zero-length span). Explicit chunk size lists must sum to the span exactly and always produce `VaryingDimension`, even when the sizes happen to be uniform: the input syntax declares the grid kind, so a per-chunk list is preserved as a rectilinear dimension rather than silently collapsed to a regular one, which would change how the dimension grows on resize. For scalar sizes the last chunk may overhang the span. + + The one exception to the sum rule is a zero-length span: no list of + positive edges can sum to 0, so any non-empty list is accepted verbatim + and the edges describe the chunks the axis will grow into on `append` / + `resize`. This is the same state a rectilinear axis reaches when it is + resized down to 0 — `VaryingDimension` allows trailing edges beyond the + extent — so creating at length 0 and shrinking to 0 are indistinguishable. """ # `numbers.Integral` rather than `int` so that numpy integer scalars (which are not # `int` subclasses) take the uniform-chunk path instead of being treated as a sequence. @@ -741,9 +758,7 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG if chunk_size < -1 or chunk_size == 0: raise ValueError(f"Chunk size must be positive or -1, got {chunk_size}") if chunk_size == -1: - # A zero-length span still gets chunk size 1 (chunk sizes must be positive), - # matching the auto-chunking clamp in _guess_regular_chunks. - return FixedDimension(size=max(span, 1), extent=span) + return FixedDimension(size=_full_span_chunk_size(span), extent=span) return FixedDimension(size=chunk_size, extent=span) else: try: @@ -768,7 +783,9 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG ints: list[int] = [int(c) for c in chunk_list] # type: ignore[call-overload] if any(c <= 0 for c in ints): raise ValueError(f"All chunk sizes must be positive, got {ints}") - if sum(ints) != span: + # A zero-length span cannot be covered by positive edges; the edges are the + # chunks the axis will grow into, exactly as after ``resize(0)``. + if span > 0 and sum(ints) != span: raise ValueError(f"Chunk sizes {ints} do not sum to span {span}") return VaryingDimension(ints, extent=span) @@ -809,7 +826,7 @@ def normalize_chunks_nd( ) # handle no chunking: one chunk covering every axis. Routed through the -1 sentinel so - # the zero-length-axis clamp lives in one place (normalize_chunks_1d). + # the zero-length-axis rule lives in one place (_full_span_chunk_size). if chunks is False: chunks = -1 @@ -864,8 +881,10 @@ def _guess_num_chunks_per_axis_shard( In other words the shard would be a (2,2,2) grid of (2,2,2) chunks i.e., prod(chunk_shape) * (returned_val ** len(chunk_shape)) * item_size = 256 bytes. - Degenerate chunk shapes — a 0-dimensional shape, or one containing a zero-length - axis — return 1, as the search loop's stopping conditions can never be met. + Degenerate inputs — a 0-dimensional chunk shape, or a zero-byte chunk (``item_size`` + of 0; chunk edge lengths themselves are always at least 1) — return 1, as the + search loop's stopping conditions can never be met. A zero-length *array* axis + needs no special case: the array-bound check fails immediately for it. Parameters ---------- @@ -886,8 +905,8 @@ def _guess_num_chunks_per_axis_shard( if max_bytes < bytes_per_chunk: return 1 num_axes = len(chunk_shape) - # For a 0-dimensional chunk shape or one with a zero-length axis, both loop - # conditions below are constant, so the loop would never terminate. + # For a 0-dimensional chunk shape or a zero-byte chunk, both loop conditions + # below are constant, so the loop would never terminate. if num_axes == 0 or bytes_per_chunk == 0: return 1 chunks_per_shard = 1 diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 5822a228b9..b1fa2d866d 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -89,6 +89,13 @@ def __init__( """ shape_parsed = parse_shapelike(shape) chunks_parsed = parse_shapelike(chunks) + # Same invariant as the Zarr format 3 chunk grid metadata: every chunk edge + # length is at least 1, even on a zero-length axis. + for dim_idx, chunk in enumerate(chunks_parsed): + if chunk < 1: + raise ValueError( + f"Dimension {dim_idx}: chunk edge length must be >= 1, got {chunk}" + ) compressor_parsed = parse_compressor(compressor) order_parsed = parse_indexing_order(order) dimension_separator_parsed = parse_separator(dimension_separator) diff --git a/tests/test_chunk_grids.py b/tests/test_chunk_grids.py index 40133700a8..86383d31ed 100644 --- a/tests/test_chunk_grids.py +++ b/tests/test_chunk_grids.py @@ -367,66 +367,191 @@ def test_create_0d_array_auto_shards_with_target_shard_size() -> None: assert arr.shards == () -@pytest.mark.parametrize("chunks", [-1, False], ids=["minus-one", "false"]) -@pytest.mark.parametrize("shape", [(0,), (0, 4), (4, 0)], ids=["1d", "2d-lead", "2d-trail"]) +# -- Zero-length dimensions -- +# +# One invariant: a chunk edge length is always >= 1, an extent may be 0. Every spelling +# that derives a chunk size from a span (-1, False, "auto", shards="auto") must agree on +# chunk size 1 for a zero-length axis, in both Zarr formats, with or without sharding. +# Historically each spelling clamped (or failed to clamp) on its own; see #4304, #4305, +# #4307, #4328 and, further back, #150, #241, #303, #972, #1977, #2434, #3711. + +ZeroLengthChunkSpelling = Literal["minus-one", "false", "auto", "one", "ones", "rectilinear"] +ZeroLengthShards = Literal["auto", "auto-budget", "explicit"] | None + +# Spellings whose chunk size is derived from the axis span rather than given explicitly. +_SPAN_DERIVED_SPELLINGS: frozenset[ZeroLengthChunkSpelling] = frozenset( + {"minus-one", "false", "auto"} +) + + +def _zero_length_chunks_arg(spelling: ZeroLengthChunkSpelling, shape: tuple[int, ...]) -> Any: + """Translate a chunk-spelling id into the `chunks=` argument for `shape`.""" + match spelling: + case "minus-one": + return -1 + case "false": + return False + case "auto": + return "auto" + case "one": + return 1 + case "ones": + return (1,) * len(shape) + case "rectilinear": + return [[2, 2]] * len(shape) + + +@pytest.mark.parametrize("spelling", ["minus-one", "false", "auto", "one", "ones", "rectilinear"]) @pytest.mark.parametrize( - ("zarr_format", "shards", "target_shard_size_bytes"), - [ - (2, None, None), - (3, None, None), - (3, "auto", None), - (3, "auto", 128 * 1024 * 1024), - ], - ids=["v2", "v3", "v3-auto-shards", "v3-auto-shards-budget"], + "shape", + [(0,), (0, 4), (4, 0), (0, 0), ()], + ids=["1d", "2d-lead", "2d-trail", "2d-both", "0d"], ) -def test_create_zero_length_array_full_span_chunks( - chunks: int | bool, +@pytest.mark.parametrize( + ("zarr_format", "shards"), + [(2, None), (3, None), (3, "auto"), (3, "auto-budget"), (3, "explicit")], + ids=["v2", "v3", "v3-auto-shards", "v3-auto-shards-budget", "v3-explicit-shards"], +) +def test_create_zero_length_array( + spelling: ZeroLengthChunkSpelling, shape: tuple[int, ...], zarr_format: Literal[2, 3], - shards: Literal["auto"] | None, - target_shard_size_bytes: int | None, + shards: ZeroLengthShards, ) -> None: - """`chunks=-1` and `chunks=False` on a zero-length axis must resolve to chunk size 1. - - Both spellings mean "one chunk covering the whole axis". They used to resolve to chunk - size 0 on zero-length axes, which broke every downstream path differently: a ValueError - from the Zarr format 3 chunk grid metadata, a ZeroDivisionError with shards="auto", an - infinite loop with a shard size budget (https://github.com/zarr-developers/zarr-python/issues/4304), - and invalid `chunks: [0]` metadata for Zarr format 2 that silently corrupted reads after - a resize. + """Every chunk spelling produces a valid, usable grid on a zero-length axis. + + Span-derived spellings resolve to chunk size 1 on zero-length axes (and the full span + elsewhere, for these small shapes); explicit spellings are stored verbatim. In every case + the stored metadata matches `arr.chunks` / `arr.shards`, the array can grow along the + empty axis, round-trip data, and shrink back to empty. """ - expected_chunks = tuple(max(s, 1) for s in shape) + ndim = len(shape) + if spelling == "rectilinear": + if zarr_format == 2: + pytest.skip("Zarr format 2 does not support rectilinear chunk grids") + if shards is not None: + pytest.skip("rectilinear chunks with sharding is not supported") + if ndim == 0: + pytest.skip("a 0-d array has no dimension to chunk rectilinearly") + if shards == "explicit" and ndim == 0: + pytest.skip("a 0-d array has no axis to shard explicitly") + + chunks = _zero_length_chunks_arg(spelling, shape) + expected_chunks: tuple[int, ...] | None + if spelling in _SPAN_DERIVED_SPELLINGS: + expected_chunks = tuple(max(s, 1) for s in shape) + elif spelling == "rectilinear": + expected_chunks = None + else: + expected_chunks = (1,) * ndim + + shards_arg: Any + expected_shards: tuple[int, ...] | None + match shards: + case None: + shards_arg, expected_shards = None, None + case "auto" | "auto-budget": + # Axes this short never split, so the guessed shard equals the chunk. + shards_arg, expected_shards = "auto", expected_chunks + case "explicit": + # A shard larger than the (zero) extent is fine: the axis has zero shards. + shards_arg = tuple(2 if s == 0 else s for s in shape) + expected_shards = shards_arg + warns = ( pytest.warns(ZarrUserWarning, match="Automatic shard shape inference is experimental") - if shards == "auto" + if shards_arg == "auto" else contextlib.nullcontext() ) - with zarr.config.set({"array.target_shard_size_bytes": target_shard_size_bytes}), warns: - arr = zarr.create_array( - store={}, - shape=shape, - dtype="int64", - chunks=chunks, - shards=shards, - zarr_format=zarr_format, - ) - assert arr.chunks == expected_chunks - assert arr.shards == (expected_chunks if shards == "auto" else None) + budget = 128 * 1024 * 1024 if shards == "auto-budget" else None + # The rectilinear flag must stay set for the array's whole life, not just creation. + with zarr.config.set( + {"array.rectilinear_chunks": True, "array.target_shard_size_bytes": budget} + ): + with warns: + arr = zarr.create_array( + store={}, + shape=shape, + dtype="int64", + chunks=chunks, + shards=shards_arg, + zarr_format=zarr_format, + ) - # The stored chunk grid must be the clamped shape, whichever format wrote it. - meta = cast(dict[str, Any], arr.metadata.to_dict()) - if zarr_format == 2: - assert meta["chunks"] == expected_chunks - else: - assert meta["chunk_grid"]["configuration"]["chunk_shape"] == expected_chunks - - # The array must remain usable: grow the empty axis and round-trip data through it. - axis = shape.index(0) - grown = tuple(2 if s == 0 else s for s in shape) - arr.append(np.full(grown, 7, dtype="int64"), axis=axis) - assert arr.shape == grown - np.testing.assert_array_equal(arr[...], np.full(grown, 7, dtype="int64")) - resized = tuple(3 if s == 0 else s for s in shape) - arr.resize(resized) - assert arr.shape == resized - assert int(np.asarray(arr[...]).sum()) == 7 * np.prod(grown) + # In-memory view and stored metadata agree with the invariant. + assert arr.shards == expected_shards + meta = cast(dict[str, Any], arr.metadata.to_dict()) + if spelling == "rectilinear": + grid = meta["chunk_grid"] + assert grid["name"] == "rectilinear" + # Stored verbatim on zero-length axes too, run-length encoded as [size, count]. + assert list(grid["configuration"]["chunk_shapes"]) == [[[2, 2]]] * ndim + assert arr.write_chunk_sizes == tuple(() if s == 0 else (2, 2) for s in shape) + else: + assert arr.chunks == expected_chunks + if zarr_format == 2: + assert meta["chunks"] == expected_chunks + else: + stored = meta["chunk_grid"]["configuration"]["chunk_shape"] + assert stored == (expected_chunks if expected_shards is None else expected_shards) + assert all(c >= 1 for c in arr.chunks) + + # The array must remain usable. + if ndim == 0: + arr[...] = 7 + assert arr[...] == 7 + return + axis = shape.index(0) + grown = tuple(2 if i == axis else s for i, s in enumerate(shape)) + data = np.full(grown, 7, dtype="int64") + arr.append(data, axis=axis) + assert arr.shape == grown + np.testing.assert_array_equal(arr[...], data) + arr.resize(shape) + assert arr.shape == shape + assert np.asarray(arr[...]).shape == shape + + +@pytest.mark.parametrize("zarr_format", [2, 3]) +def test_create_zero_chunk_rejected(zarr_format: Literal[2, 3]) -> None: + """An explicit chunk size of 0 is rejected up front, even for a zero-length axis.""" + with pytest.raises(ValueError, match="Chunk size must be positive or -1, got 0"): + zarr.create_array(store={}, shape=(0,), chunks=(0,), dtype="int64", zarr_format=zarr_format) + + +def test_rectilinear_zero_extent_matches_resize() -> None: + """Creating a rectilinear axis at length 0 equals resizing one down to 0. + + Both leave a `VaryingDimension` whose edges lie entirely beyond the extent, so the + stored grids are identical and both grow into the same chunks on append. + """ + with zarr.config.set({"array.rectilinear_chunks": True}): + created = zarr.create_array(store={}, shape=(0,), chunks=[[2, 2]], dtype="int64") + resized = zarr.create_array(store={}, shape=(4,), chunks=[[2, 2]], dtype="int64") + resized.resize((0,)) + created_meta = cast(dict[str, Any], created.metadata.to_dict()) + resized_meta = cast(dict[str, Any], resized.metadata.to_dict()) + assert created_meta["chunk_grid"] == resized_meta["chunk_grid"] + assert created_meta["shape"] == resized_meta["shape"] == (0,) + + created.append(np.arange(3, dtype="int64")) + resized.append(np.arange(3, dtype="int64")) + np.testing.assert_array_equal(created[...], np.arange(3)) + np.testing.assert_array_equal(resized[...], np.arange(3)) + assert created.write_chunk_sizes == resized.write_chunk_sizes == ((2, 1),) + + +def test_normalize_chunks_1d_zero_span_accepts_any_edges() -> None: + """On a zero-length span the explicit edge list is stored verbatim.""" + dim = normalize_chunks_1d([3, 5], span=0) + assert isinstance(dim, VaryingDimension) + assert dim.edges == (3, 5) + assert dim.extent == 0 + assert dim.nchunks == 0 + assert dim.resize(4) == VaryingDimension([3, 5], extent=4) + + +def test_normalize_chunks_1d_nonzero_span_still_requires_exact_sum() -> None: + """Relaxing the sum rule for span 0 must not leak into positive spans.""" + with pytest.raises(ValueError, match="do not sum to span 1"): + normalize_chunks_1d([3, 5], span=1) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 1358f458d6..ac3ee4c029 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -309,6 +309,18 @@ def test_from_dict_extra_fields() -> None: assert result == expected +@pytest.mark.parametrize(("shape", "chunks"), [((0,), (0,)), ((4, 0), (4, 0)), ((5,), (0,))]) +def test_zero_chunk_edge_rejected(shape: tuple[int, ...], chunks: tuple[int, ...]) -> None: + """A chunk edge length of 0 is invalid metadata, whatever the array shape. + + Older releases could write `chunks: [0]` for a zero-length axis; such documents read + uninitialised memory once resized. The v2 layer now enforces the same `>= 1` rule as + the Zarr format 3 chunk grid. + """ + with pytest.raises(ValueError, match="chunk edge length must be >= 1, got 0"): + ArrayV2Metadata(shape=shape, dtype=Float64(), chunks=chunks, fill_value=0.0, order="C") + + def test_eq_nan_fill_value() -> None: """Two metadata objects with an identical NaN fill_value compare equal. diff --git a/tests/test_unified_chunk_grid.py b/tests/test_unified_chunk_grid.py index b8289d2135..2cf3f7cc10 100644 --- a/tests/test_unified_chunk_grid.py +++ b/tests/test_unified_chunk_grid.py @@ -149,9 +149,9 @@ def test_rectilinear_feature_flag_enabled() -> None: (10, 100, 1, 10, 10, 10, 10), (10, 100, 9, 10, 10, 10, 90), (10, 95, 9, 10, 10, 5, 90), # boundary chunk - (0, 0, None, 0, None, None, None), # zero-size + (10, 0, None, 0, None, None, None), # zero-extent: no chunks, size still >= 1 ], - ids=["start", "middle", "end", "boundary", "zero-size"], + ids=["start", "middle", "end", "boundary", "zero-extent"], ) def test_fixed_dimension( size: int, @@ -190,11 +190,21 @@ def test_fixed_dimension_indices_to_chunks() -> None: @pytest.mark.parametrize( ("size", "extent", "match"), - [(-1, 100, "must be >= 0"), (10, -1, "must be >= 0")], - ids=["negative-size", "negative-extent"], + [ + (-1, 100, "size must be >= 1"), + (0, 100, "size must be >= 1"), + (0, 0, "size must be >= 1"), + (10, -1, "extent must be >= 0"), + ], + ids=["negative-size", "zero-size", "zero-size-zero-extent", "negative-extent"], ) -def test_fixed_dimension_rejects_negative(size: int, extent: int, match: str) -> None: - """FixedDimension raises ValueError for negative size or extent""" +def test_fixed_dimension_rejects_invalid(size: int, extent: int, match: str) -> None: + """FixedDimension raises ValueError for a size below 1 or a negative extent. + + A chunk edge length of 0 is never valid, whatever the extent: the metadata layer + requires every chunk edge length to be >= 1, and the in-memory model enforces the + same invariant so the two can never disagree. + """ with pytest.raises(ValueError, match=match): FixedDimension(size=size, extent=extent) @@ -1421,46 +1431,27 @@ def test_edge_case_chunk_grid_boundary_shape() -> None: # -- Zero-size and zero-extent -- -@pytest.mark.parametrize( - ("size", "extent"), - [(0, 0), (0, 5), (10, 0)], - ids=["zero-size-zero-extent", "zero-size-nonzero-extent", "zero-extent-nonzero-size"], -) -def test_edge_case_zero_size_or_extent(size: int, extent: int) -> None: - """FixedDimension with zero size or extent has zero chunks and getitem returns None""" - d = FixedDimension(size=size, extent=extent) - assert d.nchunks == 0 - g = ChunkGrid(dimensions=(d,)) - assert g[0] is None - - -def test_edge_case_zero_size_data_and_indices() -> None: - """FixedDimension(size=0) handles data_size, index_to_chunk, and indices_to_chunks safely.""" - d = FixedDimension(size=0, extent=0) - # Zero-sized chunks have zero data - assert d.data_size(0) == 0 - # Vectorized lookup maps every index to chunk 0 (avoids division by zero) - indices = np.array([0, 0, 0], dtype=np.intp) - np.testing.assert_array_equal(d.indices_to_chunks(indices), np.zeros(3, dtype=np.intp)) +@pytest.mark.parametrize("size", [1, 10], ids=["size-1", "size-10"]) +def test_fixed_dimension_zero_extent(size: int) -> None: + """A zero-length axis has zero chunks and behaves like an empty grid. - -def test_edge_case_zero_size_nonzero_extent_index() -> None: - """FixedDimension(size=0, extent>0) maps valid indices to chunk 0 without dividing by zero.""" - d = FixedDimension(size=0, extent=5) + The extent may be 0 even though the chunk size may not: `ceildiv(0, size)` is 0, + so there is nothing to look up, and the vectorized index mapping of an empty index + array is empty. + """ + d = FixedDimension(size=size, extent=0) assert d.nchunks == 0 - # index_to_chunk avoids division by zero and returns 0 - assert d.index_to_chunk(0) == 0 - assert d.index_to_chunk(4) == 0 - - -def test_edge_case_zero_size_data_and_index() -> None: - """FixedDimension(size=0) returns zero for data_size and maps indices to chunk 0.""" - d = FixedDimension(size=0, extent=0) - # data_size returns 0 for a zero-sized chunk + assert d.ngridcells == 0 assert d.data_size(0) == 0 - # vectorized indices_to_chunks returns zeros - indices = np.array([0, 0, 0], dtype=np.intp) - np.testing.assert_array_equal(d.indices_to_chunks(indices), np.zeros(3, dtype=np.intp)) + assert d.with_extent(0) == d + assert d.with_extent(3) == FixedDimension(size=size, extent=3) + empty = np.array([], dtype=np.intp) + np.testing.assert_array_equal(d.indices_to_chunks(empty), empty) + with pytest.raises(IndexError): + d.index_to_chunk(0) + g = ChunkGrid(dimensions=(d,)) + assert g[0] is None + assert list(g) == [] # -- 0-d grid -- From a171507980ed67b19cbeaa9e33ed20604b4ba944 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 9 Sep 2026 20:20:39 +0200 Subject: [PATCH 2/4] fix(metadata): read a legacy v2 zero chunk edge on an empty axis as 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zarr-python 2.18.7 writes `chunks: [0]` for `zarr.zeros((0,), chunks=False)` and for `chunks=(0,)`, so stores with that document exist. Rejecting them at open would turn a previously-readable array into an error; leaving the 0 in place read uninitialised memory after a resize. Normalize the edge to 1 with a ZarrUserWarning instead — the same grid every other "one chunk spans the axis" spelling produces — and keep rejecting a zero edge on an axis that has data. Assisted-by: ClaudeCode:claude-fable-5-1 --- changes/0000.bugfix.md | 2 +- src/zarr/core/metadata/v2.py | 22 ++++++++++++++++++---- tests/test_metadata/test_v2.py | 32 +++++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/changes/0000.bugfix.md b/changes/0000.bugfix.md index 8b224c3f76..dc0cd52e28 100644 --- a/changes/0000.bugfix.md +++ b/changes/0000.bugfix.md @@ -1 +1 @@ -Zero-length dimensions are now handled by a single rule instead of a per-spelling patch: a chunk edge length is always at least 1, while a dimension's extent may be 0 (such a dimension simply has zero chunks). Every way of asking for "one chunk covering the axis" — `chunks=-1`, `chunks=False`, `chunks="auto"`, and `shards="auto"` — now derives the chunk size from the same helper, so they agree on chunk size 1 for a zero-length axis in both Zarr formats. Zarr format 2 metadata now rejects a chunk edge length of 0 with a clear error, matching Zarr format 3, instead of reading uninitialised data after a resize. Rectilinear chunk grids (`chunks=[[...], ...]`) can now be created on a zero-length dimension: since no list of positive edge lengths can sum to 0, the given edge lengths are stored as-is and describe the chunks the dimension will grow into on `append` or `resize`, exactly the state a rectilinear dimension is in after being resized down to 0. The private `FixedDimension(size=0, ...)` model, which previously carried its own zero-size special cases, now raises `ValueError`. +Zero-length dimensions are now handled by a single rule instead of a per-spelling patch: a chunk edge length is always at least 1, while a dimension's extent may be 0 (such a dimension simply has zero chunks). Every way of asking for "one chunk covering the axis" — `chunks=-1`, `chunks=False`, `chunks="auto"`, and `shards="auto"` — now derives the chunk size from the same helper, so they agree on chunk size 1 for a zero-length axis in both Zarr formats. Zarr format 2 metadata now applies the same rule: a stored chunk edge length of 0 on a zero-length axis — which zarr-python 2.x wrote for `chunks=False` and `chunks=(0,)` — is read as 1 (with a `ZarrUserWarning`) so those stores stay readable and no longer read uninitialised data after a resize, while a chunk edge of 0 on an axis that has data is rejected with a clear error. Rectilinear chunk grids (`chunks=[[...], ...]`) can now be created on a zero-length dimension: since no list of positive edge lengths can sum to 0, the given edge lengths are stored as-is and describe the chunks the dimension will grow into on `append` or `resize`, exactly the state a rectilinear dimension is in after being resized down to 0. The private `FixedDimension(size=0, ...)` model, which previously carried its own zero-size special cases, now raises `ValueError`. diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index b1fa2d866d..7d6e43fc61 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -90,12 +90,26 @@ def __init__( shape_parsed = parse_shapelike(shape) chunks_parsed = parse_shapelike(chunks) # Same invariant as the Zarr format 3 chunk grid metadata: every chunk edge - # length is at least 1, even on a zero-length axis. - for dim_idx, chunk in enumerate(chunks_parsed): + # length is at least 1. zarr-python 2.x wrote `chunks: [0]` for a zero-length + # axis created with `chunks=False` or `chunks=(0,)`. Such an axis holds no + # chunks, so those documents are readable; the edge is normalized to 1 so a + # later resize does not divide by zero. On an axis that has data, 0 is invalid. + normalized_chunks: list[int] = [] + for dim_idx, (extent, chunk) in enumerate(zip(shape_parsed, chunks_parsed, strict=False)): if chunk < 1: - raise ValueError( - f"Dimension {dim_idx}: chunk edge length must be >= 1, got {chunk}" + if chunk < 0 or extent != 0: + raise ValueError( + f"Dimension {dim_idx}: chunk edge length must be >= 1, got {chunk}" + ) + warnings.warn( + f"Dimension {dim_idx}: chunk edge length 0 on a zero-length axis " + "(as written by zarr-python 2.x) is treated as 1.", + ZarrUserWarning, + stacklevel=2, ) + chunk = 1 + normalized_chunks.append(chunk) + chunks_parsed = tuple(normalized_chunks) + chunks_parsed[len(shape_parsed) :] compressor_parsed = parse_compressor(compressor) order_parsed = parse_indexing_order(order) dimension_separator_parsed = parse_separator(dimension_separator) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index ac3ee4c029..39f5961259 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -309,15 +309,33 @@ def test_from_dict_extra_fields() -> None: assert result == expected -@pytest.mark.parametrize(("shape", "chunks"), [((0,), (0,)), ((4, 0), (4, 0)), ((5,), (0,))]) -def test_zero_chunk_edge_rejected(shape: tuple[int, ...], chunks: tuple[int, ...]) -> None: - """A chunk edge length of 0 is invalid metadata, whatever the array shape. +@pytest.mark.parametrize( + ("shape", "chunks", "expected"), + [((0,), (0,), (1,)), ((4, 0), (4, 0), (4, 1)), ((0, 0), (0, 0), (1, 1))], +) +def test_zero_chunk_edge_on_empty_axis_normalized( + shape: tuple[int, ...], chunks: tuple[int, ...], expected: tuple[int, ...] +) -> None: + """A stored chunk edge of 0 on a zero-length axis is read as 1, with a warning. - Older releases could write `chunks: [0]` for a zero-length axis; such documents read - uninitialised memory once resized. The v2 layer now enforces the same `>= 1` rule as - the Zarr format 3 chunk grid. + zarr-python 2.x wrote `chunks: [0]` for such an axis (`chunks=False` or + `chunks=(0,)`), and those documents must stay readable. Left at 0, a later resize + read uninitialised memory; normalizing to 1 gives the axis the same grid every other + "one chunk spans the axis" spelling produces. """ - with pytest.raises(ValueError, match="chunk edge length must be >= 1, got 0"): + with pytest.warns(ZarrUserWarning, match="chunk edge length 0 on a zero-length axis"): + meta = ArrayV2Metadata( + shape=shape, dtype=Float64(), chunks=chunks, fill_value=0.0, order="C" + ) + assert meta.chunks == expected + + +@pytest.mark.parametrize(("shape", "chunks"), [((5,), (0,)), ((4, 3), (4, 0))]) +def test_zero_chunk_edge_with_data_rejected( + shape: tuple[int, ...], chunks: tuple[int, ...] +) -> None: + """A chunk edge of 0 on an axis that has data is invalid metadata.""" + with pytest.raises(ValueError, match="chunk edge length must be >= 1"): ArrayV2Metadata(shape=shape, dtype=Float64(), chunks=chunks, fill_value=0.0, order="C") From 921be0d73916abefd50aa1843d4fa35d841ca17c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 9 Sep 2026 20:20:47 +0200 Subject: [PATCH 3/4] chore: rename changelog fragment to the PR number Assisted-by: ClaudeCode:claude-fable-5-1 --- changes/{0000.bugfix.md => 4334.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{0000.bugfix.md => 4334.bugfix.md} (100%) diff --git a/changes/0000.bugfix.md b/changes/4334.bugfix.md similarity index 100% rename from changes/0000.bugfix.md rename to changes/4334.bugfix.md From 047a92e3246e9ead0329be04afd5f22e12ba04bf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 9 Sep 2026 20:24:28 +0200 Subject: [PATCH 4/4] docs: state what 2.x actually did with a zero chunk edge Measured against zarr 2.18.7: `zeros((0,), chunks=False)`, `chunks=-1` and `chunks=(0,)` all write `chunks: [0]`, after which nchunks, read, write, append, resize and reopen-then-read every raise ZeroDivisionError. There was never a working behaviour to preserve; normalizing the edge to 1 makes such arrays usable for the first time. Say so in the comment and fragment instead of claiming the stores were previously readable. Assisted-by: ClaudeCode:claude-fable-5-1 --- changes/4334.bugfix.md | 2 +- src/zarr/core/metadata/v2.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/changes/4334.bugfix.md b/changes/4334.bugfix.md index dc0cd52e28..8a04bec7e1 100644 --- a/changes/4334.bugfix.md +++ b/changes/4334.bugfix.md @@ -1 +1 @@ -Zero-length dimensions are now handled by a single rule instead of a per-spelling patch: a chunk edge length is always at least 1, while a dimension's extent may be 0 (such a dimension simply has zero chunks). Every way of asking for "one chunk covering the axis" — `chunks=-1`, `chunks=False`, `chunks="auto"`, and `shards="auto"` — now derives the chunk size from the same helper, so they agree on chunk size 1 for a zero-length axis in both Zarr formats. Zarr format 2 metadata now applies the same rule: a stored chunk edge length of 0 on a zero-length axis — which zarr-python 2.x wrote for `chunks=False` and `chunks=(0,)` — is read as 1 (with a `ZarrUserWarning`) so those stores stay readable and no longer read uninitialised data after a resize, while a chunk edge of 0 on an axis that has data is rejected with a clear error. Rectilinear chunk grids (`chunks=[[...], ...]`) can now be created on a zero-length dimension: since no list of positive edge lengths can sum to 0, the given edge lengths are stored as-is and describe the chunks the dimension will grow into on `append` or `resize`, exactly the state a rectilinear dimension is in after being resized down to 0. The private `FixedDimension(size=0, ...)` model, which previously carried its own zero-size special cases, now raises `ValueError`. +Zero-length dimensions are now handled by a single rule instead of a per-spelling patch: a chunk edge length is always at least 1, while a dimension's extent may be 0 (such a dimension simply has zero chunks). Every way of asking for "one chunk covering the axis" — `chunks=-1`, `chunks=False`, `chunks="auto"`, and `shards="auto"` — now derives the chunk size from the same helper, so they agree on chunk size 1 for a zero-length axis in both Zarr formats. Zarr format 2 metadata now applies the same rule: a stored chunk edge length of 0 on a zero-length axis — which zarr-python 2.x wrote for `chunks=False`, `chunks=-1` and `chunks=(0,)` and then could never read, write, append to or resize (every operation raised `ZeroDivisionError`), and which 3.0–3.3 opened but lost data on append — is read as 1 with a `ZarrUserWarning`, making such arrays usable, while a chunk edge of 0 on an axis that has data is rejected with a clear error. Rectilinear chunk grids (`chunks=[[...], ...]`) can now be created on a zero-length dimension: since no list of positive edge lengths can sum to 0, the given edge lengths are stored as-is and describe the chunks the dimension will grow into on `append` or `resize`, exactly the state a rectilinear dimension is in after being resized down to 0. The private `FixedDimension(size=0, ...)` model, which previously carried its own zero-size special cases, now raises `ValueError`. diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 7d6e43fc61..a098132ab9 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -91,9 +91,12 @@ def __init__( chunks_parsed = parse_shapelike(chunks) # Same invariant as the Zarr format 3 chunk grid metadata: every chunk edge # length is at least 1. zarr-python 2.x wrote `chunks: [0]` for a zero-length - # axis created with `chunks=False` or `chunks=(0,)`. Such an axis holds no - # chunks, so those documents are readable; the edge is normalized to 1 so a - # later resize does not divide by zero. On an axis that has data, 0 is invalid. + # axis created with `chunks=False`, `-1` or `(0,)`, and then could not read, + # write, append to or resize the array (every operation divided by zero); + # 3.0-3.3 opened such documents but lost data on append. The axis holds no + # chunks, so the edge is normalized to 1 — the grid every other "one chunk + # spans the axis" spelling produces — which makes the array usable at last. + # On an axis that has data, 0 is invalid and any data was never stored. normalized_chunks: list[int] = [] for dim_idx, (extent, chunk) in enumerate(zip(shape_parsed, chunks_parsed, strict=False)): if chunk < 1: