From 54ffce850567a508778d32aef8490f4f3bdef21b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 12 Jun 2026 14:20:30 +0200 Subject: [PATCH 1/9] Fix structured dtype v2 round-trip and silent loss of field offsets/padding Bug 1: Nested structured dtypes failed the Zarr V2 JSON round-trip. The inner type guard check_structured_dtype_v2_inner recursed into itself for a nested field's last element, but that element is a list of [name, dtype] field pairs, not a single pair. It now validates that element with check_structured_dtype_name_v2, so metadata written by to_json(zarr_format=2) can be read back. Bug 2: Structured dtypes with non-default (aligned / padded) field layouts were silently re-packed contiguously on round-trip, changing field offsets and itemsize and corrupting stored bytes. from_native_dtype now detects non-packed layouts (recursively) and raises a clear ValueError instead. Reading existing packed data is unaffected. Co-Authored-By: Claude Fable 5 --- src/zarr/core/dtype/common.py | 5 +- src/zarr/core/dtype/npy/structured.py | 45 +++++++++- tests/test_dtype/test_npy/test_structured.py | 89 ++++++++++++++++++++ 3 files changed, 137 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/dtype/common.py b/src/zarr/core/dtype/common.py index 61cbfe0360..7a507892b8 100644 --- a/src/zarr/core/dtype/common.py +++ b/src/zarr/core/dtype/common.py @@ -80,7 +80,10 @@ def check_structured_dtype_v2_inner(data: object) -> TypeGuard[StructuredName_V2 if isinstance(data[-1], str): return True elif isinstance(data[-1], Sequence): - return check_structured_dtype_v2_inner(data[-1]) + # A nested structured dtype's field has the form [name, [[sub_name, sub_dtype], ...]], + # i.e. the last element is itself a sequence of field pairs rather than a single + # [name, dtype] pair, so it must be validated as a list of fields, not a single field. + return check_structured_dtype_name_v2(data[-1]) return False diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index dcc523d1d2..68b8e54525 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -2,7 +2,7 @@ from collections.abc import Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, cast, overload +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, TypeGuard, cast, overload import numpy as np @@ -30,6 +30,34 @@ StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int +def _is_default_packed(dtype: np.dtype[np.void]) -> bool: + """ + Check whether a structured numpy dtype uses the default contiguous (packed) field layout. + + The Zarr structured/struct metadata only records ``(name, dtype)`` pairs and reconstructs the + native dtype by packing the fields contiguously (see ``Structured.to_native_dtype``). A dtype + created with ``align=True``, or with explicit field offsets / extra padding, therefore cannot be + represented faithfully: its field offsets and itemsize would silently change on round-trip, + corrupting stored bytes. This function returns ``False`` for any such dtype. + + The check is recursive so that padding within a nested field dtype is detected even when the + outer dtype is itself packed. + """ + names = dtype.names + if names is None: # pragma: no cover - only called on structured dtypes + return True + repacked_fields: list[tuple[str, np.dtype[Any]]] = [] + for name in names: + field_dtype = dtype.fields[name][0] # type: ignore[index] + if field_dtype.names is not None and not _is_default_packed(field_dtype): + return False + repacked_fields.append((name, field_dtype)) + repacked = np.dtype(repacked_fields) + if repacked.itemsize != dtype.itemsize: + return False + return all(dtype.fields[n][1] == repacked.fields[n][1] for n in names) # type: ignore[index] + + class StructuredJSON_V2(DTypeConfig_V2[StructuredName_V2, None]): """ A wrapper around the JSON representation of the ``Structured`` data type in Zarr V2. @@ -185,6 +213,21 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] if cls._check_native_dtype(dtype): + if not _is_default_packed(dtype): + # NOTE: this is a ValueError rather than a DataTypeValidationError on purpose. + # The data type registry suppresses DataTypeValidationError (treating it as + # "this dtype does not match"), but a non-packed layout *does* match this dtype + # class -- it simply cannot be represented faithfully -- so we must raise an + # error the registry propagates to the caller. + raise ValueError( + f"Cannot serialize the structured data type {dtype}. It uses a non-default " + "field layout (e.g. it was created with align=True, or has explicit field " + "offsets or padding), which the Zarr structured data type metadata cannot " + "represent: only the field names and dtypes are stored, and the fields are " + "always packed contiguously on read. Serializing this dtype would silently " + "change its field offsets and itemsize, corrupting stored data. Use a packed " + "structured dtype (without align=True or explicit offsets) instead." + ) # fields of a structured numpy dtype are either 2-tuples or 3-tuples. we only # care about the first element in either case. for key, (dtype_instance, *_) in dtype.fields.items(): # type: ignore[union-attr] diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index 554c3b4e41..105d11c5da 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -14,6 +14,7 @@ Struct, Structured, UInt8, + get_data_type_from_json, ) @@ -260,3 +261,91 @@ def test_struct_from_native_dtype() -> None: struct = Struct.from_native_dtype(dtype) assert struct.fields[0][0] == "field1" assert struct.fields[1][0] == "field2" + + +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +@pytest.mark.parametrize( + "dtype", + [ + # one level of nesting + np.dtype([("a", " None: + """ + Regression test for nested structured dtypes failing the Zarr V2 JSON round-trip. + + ``Struct.to_json(zarr_format=2)`` emits a nested field as ``[name, [[sub, dt], ...]]``, and + ``get_data_type_from_json`` must be able to read that form back. Previously the inner type + guard recursed as a single ``[name, dtype]`` pair and rejected the list-of-fields form, + so Zarr wrote V2 metadata it could not read back. + """ + zdtype = Struct.from_native_dtype(dtype) + json_v2 = zdtype.to_json(zarr_format=2) + recovered = get_data_type_from_json(json_v2, zarr_format=2) + assert recovered == zdtype + assert recovered.to_native_dtype() == dtype + + +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +def test_nested_structured_v2_array_round_trip() -> None: + """End-to-end test: write and read a Zarr V2 array with a nested structured dtype.""" + import zarr + + dtype = np.dtype([("a", " None: + """ + Regression test: structured dtypes with non-default (padded / aligned) field layouts must + fail loudly rather than silently dropping the padding. + + The Zarr structured metadata records only ``(name, dtype)`` pairs and re-packs fields + contiguously on read, so an aligned dtype would round-trip to a smaller itemsize, silently + corrupting stored chunk bytes. ``from_native_dtype`` detects this and raises instead. + """ + with pytest.raises(ValueError, match="non-default field layout"): + Struct.from_native_dtype(dtype) + + +def test_packed_structured_dtype_round_trips() -> None: + """ + A packed (default-layout) structured dtype, including nested ones, must continue to round-trip + unchanged. This guards the loud-failure path for aligned dtypes against false positives, and + ensures existing data written with packed layouts keeps working. + """ + for dtype in ( + np.dtype([("a", "i1"), ("b", "i8")]), + np.dtype([("a", " Date: Sat, 12 Sep 2026 17:43:11 +0200 Subject: [PATCH 2/9] fix(dtype): reject unrepresentable structured dtypes and fix nested v2 round trip Nested structured dtypes written with Zarr format 2 could not be read back, because the inner data type name check recursed as if the nested field were a single [name, dtype] pair instead of a list of fields. Structured dtypes that NumPy allows but the Zarr struct metadata cannot record were accepted and rebuilt as something else: aligned or explicitly offset layouts came back packed with a different itemsize, a titled field came back as two fields (the fields mapping lists titles as extra keys), and a subarray field came back as raw bytes. Struct.from_native_dtype now raises ValueError naming the offending field for all three, and iterates over names rather than the fields mapping. Add zdtypes() and structured_dtypes() hypothesis strategies to zarr.testing.strategies, and three property tests: every registered data type round-trips through JSON and through NumPy, and structured dtype resolution either raises or returns the input dtype exactly. The last property is the one every bug above violated. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/188.bugfix.md | 1 + src/zarr/core/dtype/npy/structured.py | 97 ++++++++------ src/zarr/testing/strategies.py | 125 ++++++++++++++++++- tests/test_dtype/test_npy/test_structured.py | 66 ++++++---- tests/test_properties.py | 75 +++++++++++ 5 files changed, 300 insertions(+), 64 deletions(-) create mode 100644 changes/188.bugfix.md diff --git a/changes/188.bugfix.md b/changes/188.bugfix.md new file mode 100644 index 0000000000..8658aa887b --- /dev/null +++ b/changes/188.bugfix.md @@ -0,0 +1 @@ +Structured (``struct``) data types no longer silently corrupt data on the way in or out. Nested structured dtypes written with Zarr format 2 could not be read back, because the inner data type name check rejected the nested list-of-fields form that ``Struct.to_json`` itself produces. Structured dtypes that NumPy allows but the Zarr struct metadata cannot record used to be accepted and rebuilt as something else: aligned or explicitly offset layouts came back packed with a different itemsize, a titled field came back as two fields, and a subarray field came back as raw bytes. ``Struct.from_native_dtype`` now raises a ``ValueError`` naming the offending field for all three. The ``zarr.testing.strategies`` module gains ``zdtypes`` and ``structured_dtypes`` strategies, and the property tests now check that every registered data type round-trips through JSON and NumPy, and that structured dtype resolution either raises or returns the input dtype exactly. diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index 68b8e54525..e3082f4f72 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -2,7 +2,7 @@ from collections.abc import Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self, TypeGuard, cast, overload +from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, cast, overload import numpy as np @@ -30,32 +30,49 @@ StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int -def _is_default_packed(dtype: np.dtype[np.void]) -> bool: +def _check_representable(dtype: np.dtype[np.void]) -> str | None: """ - Check whether a structured numpy dtype uses the default contiguous (packed) field layout. - - The Zarr structured/struct metadata only records ``(name, dtype)`` pairs and reconstructs the - native dtype by packing the fields contiguously (see ``Structured.to_native_dtype``). A dtype - created with ``align=True``, or with explicit field offsets / extra padding, therefore cannot be - represented faithfully: its field offsets and itemsize would silently change on round-trip, - corrupting stored bytes. This function returns ``False`` for any such dtype. - - The check is recursive so that padding within a nested field dtype is detected even when the - outer dtype is itself packed. + Check whether a structured NumPy dtype can be represented by the Zarr struct data type. + + The Zarr struct metadata records only ``(name, dtype)`` pairs and reconstructs the native + dtype by packing those fields contiguously (see ``Structured.to_native_dtype``). Anything + NumPy allows beyond that is lost on the round trip and would silently change how stored + bytes are interpreted. This function rejects three such features, recursing into nested + fields so that a problem inside a nested field dtype is caught even when the outer dtype + is fine: + + - field titles, e.g. ``np.dtype([(("title", "name"), "i4")])`` + - subarray fields, e.g. ``np.dtype([("name", "i4", (2,))])`` + - non-default field layouts, e.g. ``np.dtype(..., align=True)`` or explicit offsets + + Returns + ------- + str | None + ``None`` if the dtype is representable, otherwise a short description of the problem. """ names = dtype.names - if names is None: # pragma: no cover - only called on structured dtypes - return True - repacked_fields: list[tuple[str, np.dtype[Any]]] = [] + fields = dtype.fields + if names is None or fields is None: # pragma: no cover - only called on structured dtypes + return None for name in names: - field_dtype = dtype.fields[name][0] # type: ignore[index] - if field_dtype.names is not None and not _is_default_packed(field_dtype): - return False - repacked_fields.append((name, field_dtype)) - repacked = np.dtype(repacked_fields) - if repacked.itemsize != dtype.itemsize: - return False - return all(dtype.fields[n][1] == repacked.fields[n][1] for n in names) # type: ignore[index] + field_dtype, _offset, *title = fields[name] + if title: + return f"field {name!r} has a title ({title[0]!r})" + if field_dtype.subdtype is not None: + return f"field {name!r} is a subarray with shape {field_dtype.shape}" + if field_dtype.names is not None: + reason = _check_representable(field_dtype) + if reason is not None: + return f"within field {name!r}: {reason}" + # NumPy dtype equality compares field offsets and itemsize, so rebuilding the dtype from + # its (name, dtype) pairs and comparing detects any padding, alignment or explicit offsets. + repacked = np.dtype([(name, fields[name][0]) for name in names]) + if repacked != dtype: + return ( + "it uses a non-default field layout (e.g. it was created with align=True, or has " + "explicit field offsets or padding)" + ) + return None class StructuredJSON_V2(DTypeConfig_V2[StructuredName_V2, None]): @@ -203,6 +220,10 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: DataTypeValidationError If the input data type is not an instance of np.dtypes.VoidDType with a non-null ``fields`` attribute. + ValueError + If the input is a structured dtype that this data type cannot represent faithfully: + one with field titles, subarray fields, or a non-default (aligned, padded or + explicitly offset) field layout. Notes ----- @@ -213,25 +234,25 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] if cls._check_native_dtype(dtype): - if not _is_default_packed(dtype): + reason = _check_representable(dtype) + if reason is not None: # NOTE: this is a ValueError rather than a DataTypeValidationError on purpose. # The data type registry suppresses DataTypeValidationError (treating it as - # "this dtype does not match"), but a non-packed layout *does* match this dtype - # class -- it simply cannot be represented faithfully -- so we must raise an - # error the registry propagates to the caller. + # "this dtype does not match"), but a dtype with an unrepresentable feature + # *does* match this dtype class -- it simply cannot be represented faithfully -- + # so we must raise an error the registry propagates to the caller. raise ValueError( - f"Cannot serialize the structured data type {dtype}. It uses a non-default " - "field layout (e.g. it was created with align=True, or has explicit field " - "offsets or padding), which the Zarr structured data type metadata cannot " - "represent: only the field names and dtypes are stored, and the fields are " - "always packed contiguously on read. Serializing this dtype would silently " - "change its field offsets and itemsize, corrupting stored data. Use a packed " - "structured dtype (without align=True or explicit offsets) instead." + f"Cannot serialize the structured data type {dtype}: {reason}. The Zarr " + "struct data type records only field names and field data types, and " + "fields are always packed contiguously on read, so serializing this dtype " + "would silently change how the stored bytes are interpreted. Use a packed " + "structured dtype without titles, subarray fields, align=True or explicit " + "offsets instead." ) - # fields of a structured numpy dtype are either 2-tuples or 3-tuples. we only - # care about the first element in either case. - for key, (dtype_instance, *_) in dtype.fields.items(): # type: ignore[union-attr] - dtype_wrapped = get_data_type_from_native_dtype(dtype_instance) + # Iterate over ``names`` rather than ``fields``: the ``fields`` mapping also + # contains an entry for every field title, which would duplicate titled fields. + for key in dtype.names: # type: ignore[union-attr] + dtype_wrapped = get_data_type_from_native_dtype(dtype.fields[key][0]) # type: ignore[index] fields.append((key, dtype_wrapped)) return cls(fields=tuple(fields)) diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index db01697f1e..2a17b01b4c 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,3 +1,4 @@ +import dataclasses import itertools import math import sys @@ -27,7 +28,11 @@ from zarr.core.array import Array, CompressorsLike, SerializerLike from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding from zarr.core.common import JSON, AccessModeLiteral, ZarrFormat -from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype import data_type_registry, get_data_type_from_native_dtype +from zarr.core.dtype.common import HasItemSize +from zarr.core.dtype.npy.common import DATETIME_UNIT +from zarr.core.dtype.npy.structured import Struct +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata from zarr.core.sync import sync @@ -70,6 +75,124 @@ def dtypes() -> st.SearchStrategy[np.dtype[Any]]: ) +_field_names = st.text( + alphabet=st.characters(min_codepoint=97, max_codepoint=122), min_size=1, max_size=4 +) +_field_titles = st.text( + alphabet=st.characters(min_codepoint=65, max_codepoint=90), min_size=1, max_size=4 +) + + +def _leaf_zdtypes(cls: type[ZDType[TBaseDType, TBaseScalar]]) -> SearchStrategy[ZDType[Any, Any]]: + """ + A strategy for instances of a single non-struct ``ZDType`` class, drawing each constructor + parameter the class declares from its valid range. + """ + params = {f.name for f in dataclasses.fields(cls)} + kwargs: dict[str, SearchStrategy[Any]] = {} + if "endianness" in params: + kwargs["endianness"] = st.sampled_from(["little", "big"]) + if "length" in params: + kwargs["length"] = st.integers(min_value=1, max_value=16) + if "unit" in params: + # NumPy spells the microsecond unit "us", so the "μs" alias never round-trips as-is. + kwargs["unit"] = st.sampled_from([u for u in DATETIME_UNIT if u != "μs"]) + kwargs["scale_factor"] = st.integers(min_value=1, max_value=2**31 - 1) + return st.builds(cls, **kwargs).map(_normalize_generic_scale_factor) + return st.builds(cls, **kwargs) + + +def _normalize_generic_scale_factor(zdtype: Any) -> Any: + """ + NumPy's generic time unit carries no scale factor, so only ``scale_factor=1`` has a native + representation for it. + """ + if zdtype.unit == "generic": + return dataclasses.replace(zdtype, scale_factor=1) + return zdtype + + +def _struct_zdtypes( + children: SearchStrategy[ZDType[Any, Any]], +) -> SearchStrategy[ZDType[Any, Any]]: + """A strategy for ``Struct`` instances whose field data types are drawn from ``children``.""" + + @st.composite + def _draw(draw: st.DrawFn) -> ZDType[Any, Any]: + num_fields = draw(st.integers(min_value=1, max_value=4)) + # suffix with the index so that names are unique without filtering + names = [f"{draw(_field_names)}{i}" for i in range(num_fields)] + return Struct(fields=tuple((name, draw(children)) for name in names)) + + return _draw() + + +def zdtypes(*, max_leaves: int = 6) -> SearchStrategy[ZDType[Any, Any]]: + """ + A strategy for instances of every registered ``ZDType`` class, including ``Struct`` with + arbitrarily nested fields. + + Struct fields are restricted to fixed-size data types, since the Zarr struct data type cannot + hold variable-length fields. + """ + leaf_classes = [cls for cls in data_type_registry.contents.values() if cls is not Struct] + leaves = st.one_of([_leaf_zdtypes(cls) for cls in leaf_classes]) + fixed_size_leaves = st.one_of( + [_leaf_zdtypes(cls) for cls in leaf_classes if issubclass(cls, HasItemSize)] + ) + structs = st.recursive(fixed_size_leaves, _struct_zdtypes, max_leaves=max_leaves).filter( + lambda dt: isinstance(dt, Struct) + ) + return leaves | structs + + +@st.composite +def structured_dtypes( + draw: st.DrawFn, *, allow_unrepresentable: bool = False, max_depth: int = 3 +) -> np.dtype[np.void]: + """ + A strategy for native NumPy structured dtypes, flat or nested. + + With ``allow_unrepresentable=False`` (the default) every dtype is packed, has plain field names + and scalar fields, so it can be represented by the Zarr struct data type. With + ``allow_unrepresentable=True`` the strategy also injects the NumPy features that the Zarr + struct data type cannot record: field titles, subarray fields and ``align=True`` layouts. + Each is injected independently at random, so most draws carry at least one and some carry + none. + """ + fixed_size_leaves = st.one_of( + [ + _leaf_zdtypes(cls) + for cls in data_type_registry.contents.values() + if cls is not Struct and issubclass(cls, HasItemSize) + ] + ) + + def build(depth: int) -> np.dtype[np.void]: + num_fields = draw(st.integers(min_value=1, max_value=4)) + # suffix with the index so that names and titles are unique without filtering; titles + # draw from a different alphabet so they never collide with names either + names = [f"{draw(_field_names)}{i}" for i in range(num_fields)] + titles = [f"{draw(_field_titles)}{i}" for i in range(num_fields)] + specs: list[tuple[Any, Any]] = [] + for name, title in zip(names, titles, strict=True): + field_dtype: Any + if depth < max_depth and draw(st.booleans()): + field_dtype = build(depth + 1) + else: + field_dtype = draw(fixed_size_leaves).to_native_dtype() + key: Any = name + if allow_unrepresentable and draw(st.booleans()): + key = (title, name) + if allow_unrepresentable and draw(st.booleans()): + field_dtype = (field_dtype, draw(npst.array_shapes(max_dims=2, max_side=3))) + specs.append((key, field_dtype)) + align = allow_unrepresentable and draw(st.booleans()) + return np.dtype(specs, align=align) + + return build(0) + + def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return dtypes() diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index 105d11c5da..bdb49e0a78 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -1,10 +1,11 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np import pytest +import zarr from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype import ( Float16, @@ -17,6 +18,9 @@ get_data_type_from_json, ) +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + class TestStruct(BaseTestZDType): """Test the canonical 'struct' dtype format.""" @@ -264,9 +268,12 @@ def test_struct_from_native_dtype() -> None: @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +@pytest.mark.parametrize("zarr_format", [2, 3]) @pytest.mark.parametrize( "dtype", [ + # flat + np.dtype([("a", "i1"), ("b", "i8")]), # one level of nesting np.dtype([("a", " None: ), ], ) -def test_nested_structured_v2_round_trip(dtype: np.dtype[np.void]) -> None: +def test_packed_structured_dtype_round_trips( + dtype: np.dtype[np.void], zarr_format: ZarrFormat +) -> None: """ - Regression test for nested structured dtypes failing the Zarr V2 JSON round-trip. + A packed (default-layout) structured dtype, flat or nested, round-trips unchanged through + both the JSON form and the native form. - ``Struct.to_json(zarr_format=2)`` emits a nested field as ``[name, [[sub, dt], ...]]``, and - ``get_data_type_from_json`` must be able to read that form back. Previously the inner type - guard recursed as a single ``[name, dtype]`` pair and rejected the list-of-fields form, + Nested dtypes are the regression case: ``Struct.to_json(zarr_format=2)`` emits a nested + field as ``[name, [[sub, dt], ...]]``, and the inner V2 type guard used to reject that form, so Zarr wrote V2 metadata it could not read back. """ zdtype = Struct.from_native_dtype(dtype) - json_v2 = zdtype.to_json(zarr_format=2) - recovered = get_data_type_from_json(json_v2, zarr_format=2) + recovered = get_data_type_from_json( + zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format + ) assert recovered == zdtype assert recovered.to_native_dtype() == dtype + assert recovered.to_native_dtype().itemsize == dtype.itemsize @pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") def test_nested_structured_v2_array_round_trip() -> None: """End-to-end test: write and read a Zarr V2 array with a nested structured dtype.""" - import zarr - dtype = np.dtype([("a", " None: np.dtype([("a", "i1"), ("b", "i8")], align=True), # outer layout is packed, but a nested field dtype carries padding np.dtype([("a", "i8"), ("nested", np.dtype([("x", "i1"), ("y", "i8")], align=True))]), + # explicit offsets leave a gap without align=True + np.dtype({"names": ["a", "b"], "formats": ["i1", "i1"], "offsets": [0, 4], "itemsize": 8}), ], ) def test_padded_structured_dtype_raises(dtype: np.dtype[np.void]) -> None: """ - Regression test: structured dtypes with non-default (padded / aligned) field layouts must + Structured dtypes with non-default (padded / aligned / explicitly offset) field layouts must fail loudly rather than silently dropping the padding. - The Zarr structured metadata records only ``(name, dtype)`` pairs and re-packs fields - contiguously on read, so an aligned dtype would round-trip to a smaller itemsize, silently - corrupting stored chunk bytes. ``from_native_dtype`` detects this and raises instead. + The Zarr struct metadata records only ``(name, dtype)`` pairs and re-packs fields + contiguously on read, so a padded dtype would round-trip to a different itemsize and + silently misinterpret stored chunk bytes. """ with pytest.raises(ValueError, match="non-default field layout"): Struct.from_native_dtype(dtype) -def test_packed_structured_dtype_round_trips() -> None: +def test_titled_structured_dtype_raises() -> None: """ - A packed (default-layout) structured dtype, including nested ones, must continue to round-trip - unchanged. This guards the loud-failure path for aligned dtypes against false positives, and - ensures existing data written with packed layouts keeps working. + A structured dtype with a field title must be rejected. NumPy's ``fields`` mapping lists the + title as an extra key, so a titled field used to be read back as two separate fields. """ - for dtype in ( - np.dtype([("a", "i1"), ("b", "i8")]), - np.dtype([("a", " None: + """ + A structured dtype with a subarray field must be rejected. The subarray dtype used to be + resolved as raw bytes, silently dropping its shape and element type. + """ + dtype = np.dtype([("f0", "i4", (2,))]) + with pytest.raises(ValueError, match="field 'f0' is a subarray"): + Struct.from_native_dtype(dtype) diff --git a/tests/test_properties.py b/tests/test_properties.py index 2794ad3cb0..831c4abeef 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -19,6 +19,10 @@ from zarr.abc.store import Store from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON +from zarr.core.dtype import get_data_type_from_json, get_data_type_from_native_dtype +from zarr.core.dtype.common import HasItemSize +from zarr.core.dtype.npy.structured import Struct +from zarr.core.dtype.wrapper import ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.testing.strategies import ( @@ -34,7 +38,9 @@ sharded_arrays, simple_arrays, stores, + structured_dtypes, zarr_formats, + zdtypes, ) @@ -361,6 +367,75 @@ def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: in assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" +def _struct_depth(zdtype: ZDType[Any, Any]) -> int: + if not isinstance(zdtype, Struct): + return 0 + return 1 + max(_struct_depth(field_dtype) for _, field_dtype in zdtype.fields) + + +@given(zdtype=zdtypes(), zarr_format=zarr_formats) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_zdtype_json_roundtrip(zdtype: ZDType[Any, Any], zarr_format: int) -> None: + """ + Every registered data type, including arbitrarily nested structs, survives a round trip + through its JSON form for both Zarr formats. + + Zarr format 3 data type names do not carry endianness (the bytes codec does), so for that + format the JSON form is compared instead of the data type instance. + """ + event(f"dtype={type(zdtype).__name__}") + event(f"struct_depth={_struct_depth(zdtype)}") + as_json = zdtype.to_json(zarr_format=zarr_format) # type: ignore[arg-type] + roundtripped = get_data_type_from_json(as_json, zarr_format=zarr_format) + assert roundtripped.to_json(zarr_format=zarr_format) == as_json # type: ignore[arg-type] + if zarr_format == 2: + assert roundtripped == zdtype + + +@given(zdtype=zdtypes()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: + """ + Every registered data type survives a round trip through its native NumPy dtype, and its + reported item size matches the native dtype's itemsize. + + The NumPy object dtype is shared by several Zarr data types, so resolving it is ambiguous by + design and must raise instead. + """ + event(f"dtype={type(zdtype).__name__}") + native = zdtype.to_native_dtype() + if native.kind == "O": + event("native=object") + with pytest.raises(ValueError, match="ambiguous"): + get_data_type_from_native_dtype(native) + return + roundtripped = get_data_type_from_native_dtype(native) + assert roundtripped == zdtype + assert roundtripped.to_native_dtype() == native + if isinstance(zdtype, HasItemSize): + assert zdtype.item_size == native.itemsize + + +@given(dtype=structured_dtypes(allow_unrepresentable=True)) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_structured_dtype_never_silently_changes(dtype: np.dtype[np.void]) -> None: + """ + For any native structured dtype, including ones with field titles, subarray fields or + aligned layouts, resolving a Zarr data type either raises ``ValueError`` or yields a data type + whose native form is exactly the input: same field names, offsets and itemsize. + + This is the property that the silent-corruption bugs in structured dtype handling violated: + a dtype was accepted but came back with different fields, offsets or itemsize. + """ + try: + zdtype = get_data_type_from_native_dtype(dtype) + except ValueError: + event("outcome=rejected") + return + event("outcome=accepted") + assert zdtype.to_native_dtype() == dtype + + # @st.composite # def advanced_indices(draw, *, shape): # basic_idxr = draw( From 0395360aab5524348fcba0e95e1ead42253a5ad7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 12 Sep 2026 17:49:09 +0200 Subject: [PATCH 3/9] chore: renumber changelog fragment to upstream PR 4340 Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/{188.bugfix.md => 4340.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{188.bugfix.md => 4340.bugfix.md} (100%) diff --git a/changes/188.bugfix.md b/changes/4340.bugfix.md similarity index 100% rename from changes/188.bugfix.md rename to changes/4340.bugfix.md From 1c64f197e394030636140a32880ac5d4219a5e3d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 12 Sep 2026 17:57:34 +0200 Subject: [PATCH 4/9] style: use single backticks for code in docstrings Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- src/zarr/core/dtype/npy/structured.py | 14 +++++++------- src/zarr/testing/strategies.py | 14 +++++++------- tests/test_dtype/test_npy/test_structured.py | 8 ++++---- tests/test_properties.py | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index e3082f4f72..19d663f892 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -34,21 +34,21 @@ def _check_representable(dtype: np.dtype[np.void]) -> str | None: """ Check whether a structured NumPy dtype can be represented by the Zarr struct data type. - The Zarr struct metadata records only ``(name, dtype)`` pairs and reconstructs the native - dtype by packing those fields contiguously (see ``Structured.to_native_dtype``). Anything + The Zarr struct metadata records only `(name, dtype)` pairs and reconstructs the native + dtype by packing those fields contiguously (see `Structured.to_native_dtype`). Anything NumPy allows beyond that is lost on the round trip and would silently change how stored bytes are interpreted. This function rejects three such features, recursing into nested fields so that a problem inside a nested field dtype is caught even when the outer dtype is fine: - - field titles, e.g. ``np.dtype([(("title", "name"), "i4")])`` - - subarray fields, e.g. ``np.dtype([("name", "i4", (2,))])`` - - non-default field layouts, e.g. ``np.dtype(..., align=True)`` or explicit offsets + - field titles, e.g. `np.dtype([(("title", "name"), "i4")])` + - subarray fields, e.g. `np.dtype([("name", "i4", (2,))])` + - non-default field layouts, e.g. `np.dtype(..., align=True)` or explicit offsets Returns ------- str | None - ``None`` if the dtype is representable, otherwise a short description of the problem. + `None` if the dtype is representable, otherwise a short description of the problem. """ names = dtype.names fields = dtype.fields @@ -249,7 +249,7 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: "structured dtype without titles, subarray fields, align=True or explicit " "offsets instead." ) - # Iterate over ``names`` rather than ``fields``: the ``fields`` mapping also + # Iterate over `names` rather than `fields`: the `fields` mapping also # contains an entry for every field title, which would duplicate titled fields. for key in dtype.names: # type: ignore[union-attr] dtype_wrapped = get_data_type_from_native_dtype(dtype.fields[key][0]) # type: ignore[index] diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 2a17b01b4c..317a04c69c 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -85,7 +85,7 @@ def dtypes() -> st.SearchStrategy[np.dtype[Any]]: def _leaf_zdtypes(cls: type[ZDType[TBaseDType, TBaseScalar]]) -> SearchStrategy[ZDType[Any, Any]]: """ - A strategy for instances of a single non-struct ``ZDType`` class, drawing each constructor + A strategy for instances of a single non-struct `ZDType` class, drawing each constructor parameter the class declares from its valid range. """ params = {f.name for f in dataclasses.fields(cls)} @@ -104,7 +104,7 @@ def _leaf_zdtypes(cls: type[ZDType[TBaseDType, TBaseScalar]]) -> SearchStrategy[ def _normalize_generic_scale_factor(zdtype: Any) -> Any: """ - NumPy's generic time unit carries no scale factor, so only ``scale_factor=1`` has a native + NumPy's generic time unit carries no scale factor, so only `scale_factor=1` has a native representation for it. """ if zdtype.unit == "generic": @@ -115,7 +115,7 @@ def _normalize_generic_scale_factor(zdtype: Any) -> Any: def _struct_zdtypes( children: SearchStrategy[ZDType[Any, Any]], ) -> SearchStrategy[ZDType[Any, Any]]: - """A strategy for ``Struct`` instances whose field data types are drawn from ``children``.""" + """A strategy for `Struct` instances whose field data types are drawn from `children`.""" @st.composite def _draw(draw: st.DrawFn) -> ZDType[Any, Any]: @@ -129,7 +129,7 @@ def _draw(draw: st.DrawFn) -> ZDType[Any, Any]: def zdtypes(*, max_leaves: int = 6) -> SearchStrategy[ZDType[Any, Any]]: """ - A strategy for instances of every registered ``ZDType`` class, including ``Struct`` with + A strategy for instances of every registered `ZDType` class, including `Struct` with arbitrarily nested fields. Struct fields are restricted to fixed-size data types, since the Zarr struct data type cannot @@ -153,10 +153,10 @@ def structured_dtypes( """ A strategy for native NumPy structured dtypes, flat or nested. - With ``allow_unrepresentable=False`` (the default) every dtype is packed, has plain field names + With `allow_unrepresentable=False` (the default) every dtype is packed, has plain field names and scalar fields, so it can be represented by the Zarr struct data type. With - ``allow_unrepresentable=True`` the strategy also injects the NumPy features that the Zarr - struct data type cannot record: field titles, subarray fields and ``align=True`` layouts. + `allow_unrepresentable=True` the strategy also injects the NumPy features that the Zarr + struct data type cannot record: field titles, subarray fields and `align=True` layouts. Each is injected independently at random, so most draws carry at least one and some carry none. """ diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index bdb49e0a78..5c599cb335 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -292,8 +292,8 @@ def test_packed_structured_dtype_round_trips( A packed (default-layout) structured dtype, flat or nested, round-trips unchanged through both the JSON form and the native form. - Nested dtypes are the regression case: ``Struct.to_json(zarr_format=2)`` emits a nested - field as ``[name, [[sub, dt], ...]]``, and the inner V2 type guard used to reject that form, + Nested dtypes are the regression case: `Struct.to_json(zarr_format=2)` emits a nested + field as `[name, [[sub, dt], ...]]`, and the inner V2 type guard used to reject that form, so Zarr wrote V2 metadata it could not read back. """ zdtype = Struct.from_native_dtype(dtype) @@ -339,7 +339,7 @@ def test_padded_structured_dtype_raises(dtype: np.dtype[np.void]) -> None: Structured dtypes with non-default (padded / aligned / explicitly offset) field layouts must fail loudly rather than silently dropping the padding. - The Zarr struct metadata records only ``(name, dtype)`` pairs and re-packs fields + The Zarr struct metadata records only `(name, dtype)` pairs and re-packs fields contiguously on read, so a padded dtype would round-trip to a different itemsize and silently misinterpret stored chunk bytes. """ @@ -349,7 +349,7 @@ def test_padded_structured_dtype_raises(dtype: np.dtype[np.void]) -> None: def test_titled_structured_dtype_raises() -> None: """ - A structured dtype with a field title must be rejected. NumPy's ``fields`` mapping lists the + A structured dtype with a field title must be rejected. NumPy's `fields` mapping lists the title as an extra key, so a titled field used to be read back as two separate fields. """ dtype = np.dtype([(("title", "f0"), "i4"), ("g", "f8")]) diff --git a/tests/test_properties.py b/tests/test_properties.py index 831c4abeef..77e306000a 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -421,7 +421,7 @@ def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: def test_structured_dtype_never_silently_changes(dtype: np.dtype[np.void]) -> None: """ For any native structured dtype, including ones with field titles, subarray fields or - aligned layouts, resolving a Zarr data type either raises ``ValueError`` or yields a data type + aligned layouts, resolving a Zarr data type either raises `ValueError` or yields a data type whose native form is exactly the input: same field names, offsets and itemsize. This is the property that the silent-corruption bugs in structured dtype handling violated: From f10848e536f22e06b17433e8a851dc1f7ed95f0d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 12 Sep 2026 18:04:47 +0200 Subject: [PATCH 5/9] fix(dtype): preserve padded structured array writes Keep packed conversion for non-default layouts with a user warning, and verify field values survive both Zarr formats. Assisted-by: Codex:GPT-6 --- changes/4340.bugfix.md | 2 +- docs/user-guide/data_types.md | 8 ++++ src/zarr/core/dtype/npy/structured.py | 46 +++++++++++--------- tests/test_dtype/test_npy/test_structured.py | 28 ++++++++---- tests/test_properties.py | 21 ++++++--- 5 files changed, 67 insertions(+), 38 deletions(-) diff --git a/changes/4340.bugfix.md b/changes/4340.bugfix.md index 8658aa887b..a2bf5c7a24 100644 --- a/changes/4340.bugfix.md +++ b/changes/4340.bugfix.md @@ -1 +1 @@ -Structured (``struct``) data types no longer silently corrupt data on the way in or out. Nested structured dtypes written with Zarr format 2 could not be read back, because the inner data type name check rejected the nested list-of-fields form that ``Struct.to_json`` itself produces. Structured dtypes that NumPy allows but the Zarr struct metadata cannot record used to be accepted and rebuilt as something else: aligned or explicitly offset layouts came back packed with a different itemsize, a titled field came back as two fields, and a subarray field came back as raw bytes. ``Struct.from_native_dtype`` now raises a ``ValueError`` naming the offending field for all three. The ``zarr.testing.strategies`` module gains ``zdtypes`` and ``structured_dtypes`` strategies, and the property tests now check that every registered data type round-trips through JSON and NumPy, and that structured dtype resolution either raises or returns the input dtype exactly. +Nested structured dtypes now round-trip through Zarr format 2 metadata. Structured dtypes with field titles or subarray fields now raise `ValueError` instead of losing field information. Aligned, padded, and explicitly offset layouts remain accepted and are converted to packed layouts with a `ZarrUserWarning`: field values are preserved when writing arrays, although offsets and itemsize may change. Use `numpy.lib.recfunctions.repack_fields(data, recurse=True)` to make this conversion explicit. These checks also apply to nested fields. The `zarr.testing.strategies` module gains `zdtypes` and `structured_dtypes` strategies, with property tests for dtype serialization and warned layout normalization. diff --git a/docs/user-guide/data_types.md b/docs/user-guide/data_types.md index 91f828a738..331c354e4b 100644 --- a/docs/user-guide/data_types.md +++ b/docs/user-guide/data_types.md @@ -87,6 +87,14 @@ For these reasons, Zarr V2 uses a special data type encoding for structured data They are stored in JSON as lists of pairs, where the first element is a string, and the second element is a Zarr V2 data type specification. This representation supports recursion. +Zarr stores structured fields in a packed layout. NumPy arrays with alignment, padding, +or explicit field offsets remain accepted, with a `ZarrUserWarning`: writing preserves +field values, but the stored dtype's offsets and itemsize may differ from the input. +To make this conversion explicit, use +`numpy.lib.recfunctions.repack_fields(data, recurse=True)` before creating the array. +Field titles and subarray fields, including those inside nested structures, are rejected +because their field information cannot be represented. These rules apply to both Zarr formats. + For example: ```python exec="true" session="data_types" source="above" result="ansi" diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index 19d663f892..adaa67480f 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from collections.abc import Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, cast, overload @@ -22,7 +23,7 @@ check_json_str, ) from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType -from zarr.errors import DataTypeValidationError +from zarr.errors import DataTypeValidationError, ZarrUserWarning if TYPE_CHECKING: from zarr.core.common import JSON, ZarrFormat @@ -32,18 +33,15 @@ def _check_representable(dtype: np.dtype[np.void]) -> str | None: """ - Check whether a structured NumPy dtype can be represented by the Zarr struct data type. + Check for field features that the Zarr struct data type cannot represent. The Zarr struct metadata records only `(name, dtype)` pairs and reconstructs the native - dtype by packing those fields contiguously (see `Structured.to_native_dtype`). Anything - NumPy allows beyond that is lost on the round trip and would silently change how stored - bytes are interpreted. This function rejects three such features, recursing into nested - fields so that a problem inside a nested field dtype is caught even when the outer dtype - is fine: + dtype by packing those fields contiguously (see `Structured.to_native_dtype`). Padding + can be removed while preserving field values, but titles and subarray fields would lose + field information. This function rejects those features, including in nested fields: - field titles, e.g. `np.dtype([(("title", "name"), "i4")])` - subarray fields, e.g. `np.dtype([("name", "i4", (2,))])` - - non-default field layouts, e.g. `np.dtype(..., align=True)` or explicit offsets Returns ------- @@ -64,14 +62,6 @@ def _check_representable(dtype: np.dtype[np.void]) -> str | None: reason = _check_representable(field_dtype) if reason is not None: return f"within field {name!r}: {reason}" - # NumPy dtype equality compares field offsets and itemsize, so rebuilding the dtype from - # its (name, dtype) pairs and comparing detects any padding, alignment or explicit offsets. - repacked = np.dtype([(name, fields[name][0]) for name in names]) - if repacked != dtype: - return ( - "it uses a non-default field layout (e.g. it was created with align=True, or has " - "explicit field offsets or padding)" - ) return None @@ -222,8 +212,13 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: ``fields`` attribute. ValueError If the input is a structured dtype that this data type cannot represent faithfully: - one with field titles, subarray fields, or a non-default (aligned, padded or - explicitly offset) field layout. + one with field titles or subarray fields. + + Warns + ----- + ZarrUserWarning + If a non-default field layout is converted to a packed layout. Field values are + preserved when writing arrays, but offsets and itemsize may change. Notes ----- @@ -246,8 +241,7 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: "struct data type records only field names and field data types, and " "fields are always packed contiguously on read, so serializing this dtype " "would silently change how the stored bytes are interpreted. Use a packed " - "structured dtype without titles, subarray fields, align=True or explicit " - "offsets instead." + "structured dtype without titles or subarray fields instead." ) # Iterate over `names` rather than `fields`: the `fields` mapping also # contains an entry for every field title, which would duplicate titled fields. @@ -255,7 +249,17 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: dtype_wrapped = get_data_type_from_native_dtype(dtype.fields[key][0]) # type: ignore[index] fields.append((key, dtype_wrapped)) - return cls(fields=tuple(fields)) + result = cls(fields=tuple(fields)) + if result.to_native_dtype() != dtype: + warnings.warn( + "The structured dtype is converted to a packed field layout. " + "Field values are preserved when writing arrays, but field offsets and " + "itemsize may change. To pack explicitly, use " + "numpy.lib.recfunctions.repack_fields(data, recurse=True).", + ZarrUserWarning, + stacklevel=2, + ) + return result raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index 5c599cb335..dbee53b589 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -4,6 +4,7 @@ import numpy as np import pytest +from numpy.lib.recfunctions import repack_fields import zarr from tests.test_dtype.test_wrapper import BaseTestZDType @@ -17,6 +18,7 @@ UInt8, get_data_type_from_json, ) +from zarr.errors import ZarrUserWarning if TYPE_CHECKING: from zarr.core.common import ZarrFormat @@ -334,17 +336,25 @@ def test_nested_structured_v2_array_round_trip() -> None: np.dtype({"names": ["a", "b"], "formats": ["i1", "i1"], "offsets": [0, 4], "itemsize": 8}), ], ) -def test_padded_structured_dtype_raises(dtype: np.dtype[np.void]) -> None: +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +def test_padded_structured_dtype_warns_and_preserves_values( + dtype: np.dtype[np.void], zarr_format: ZarrFormat +) -> None: """ - Structured dtypes with non-default (padded / aligned / explicitly offset) field layouts must - fail loudly rather than silently dropping the padding. - - The Zarr struct metadata records only `(name, dtype)` pairs and re-packs fields - contiguously on read, so a padded dtype would round-trip to a different itemsize and - silently misinterpret stored chunk bytes. + Preserve the existing conversion of padded records to packed records, with a warning. + The layout changes, but field values must survive writing and reopening the array. """ - with pytest.raises(ValueError, match="non-default field layout"): - Struct.from_native_dtype(dtype) + data = np.ones(3, dtype=dtype) + data["a"] = [1, 2, 3] + expected = repack_fields(data, recurse=True) + store = zarr.storage.MemoryStore() + with pytest.warns(ZarrUserWarning, match="packed.*layout"): + zarr.create_array(store, data=data, chunks=(2,), zarr_format=zarr_format) + reopened = zarr.open_array(store) + assert reopened.dtype == expected.dtype + assert reopened.dtype.itemsize == expected.dtype.itemsize + np.testing.assert_array_equal(reopened[:], expected) def test_titled_structured_dtype_raises() -> None: diff --git a/tests/test_properties.py b/tests/test_properties.py index 77e306000a..a2d7e8a9f7 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -1,11 +1,13 @@ import itertools import json import numbers +import warnings from collections.abc import Generator from typing import Any import numpy as np import pytest +from numpy.lib.recfunctions import repack_fields from numpy.testing import assert_array_equal import zarr @@ -25,6 +27,7 @@ from zarr.core.dtype.wrapper import ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync +from zarr.errors import ZarrUserWarning from zarr.testing.strategies import ( array_metadata, arrays, @@ -421,19 +424,23 @@ def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: def test_structured_dtype_never_silently_changes(dtype: np.dtype[np.void]) -> None: """ For any native structured dtype, including ones with field titles, subarray fields or - aligned layouts, resolving a Zarr data type either raises `ValueError` or yields a data type - whose native form is exactly the input: same field names, offsets and itemsize. - - This is the property that the silent-corruption bugs in structured dtype handling violated: - a dtype was accepted but came back with different fields, offsets or itemsize. + aligned layouts, resolution either rejects unsupported field features or preserves the + fields in a packed layout. Any layout change must emit a warning. """ try: - zdtype = get_data_type_from_native_dtype(dtype) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ZarrUserWarning) + zdtype = get_data_type_from_native_dtype(dtype) except ValueError: event("outcome=rejected") return event("outcome=accepted") - assert zdtype.to_native_dtype() == dtype + native = zdtype.to_native_dtype() + assert native == repack_fields(dtype, recurse=True) + layout_warnings = [ + w for w in caught if issubclass(w.category, ZarrUserWarning) and "packed" in str(w.message) + ] + assert bool(layout_warnings) == (native != dtype) # @st.composite From 00cc27148aa27d4d0a18ec26aaa80c966abdd981 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 12 Sep 2026 18:11:37 +0200 Subject: [PATCH 6/9] fix(dtype): warn once for padded structured dtypes and reword the rejection message Repack the dtype once at the top level before resolving fields, so a padded nested field warns once rather than once per level of nesting. Attribute the warning to the first frame outside the zarr package; this reaches the caller for direct uses of the dtype API, while the synchronous array API runs on the event loop thread where no caller frame exists. The title/subarray error message no longer claims that stored bytes would be misinterpreted; the field information simply has no place in the metadata. The changelog now says those dtypes used to fail with an unrelated NumPy error rather than losing information silently. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4340.bugfix.md | 2 +- src/zarr/core/dtype/npy/structured.py | 39 +++++++++++++------- tests/test_dtype/test_npy/test_structured.py | 6 ++- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/changes/4340.bugfix.md b/changes/4340.bugfix.md index a2bf5c7a24..21cc62a70c 100644 --- a/changes/4340.bugfix.md +++ b/changes/4340.bugfix.md @@ -1 +1 @@ -Nested structured dtypes now round-trip through Zarr format 2 metadata. Structured dtypes with field titles or subarray fields now raise `ValueError` instead of losing field information. Aligned, padded, and explicitly offset layouts remain accepted and are converted to packed layouts with a `ZarrUserWarning`: field values are preserved when writing arrays, although offsets and itemsize may change. Use `numpy.lib.recfunctions.repack_fields(data, recurse=True)` to make this conversion explicit. These checks also apply to nested fields. The `zarr.testing.strategies` module gains `zdtypes` and `structured_dtypes` strategies, with property tests for dtype serialization and warned layout normalization. +Nested structured dtypes now round-trip through Zarr format 2 metadata. Structured dtypes with field titles or subarray fields now raise `ValueError` naming the offending field, instead of failing with an unrelated NumPy error. Aligned, padded, and explicitly offset layouts remain accepted and are converted to packed layouts with a `ZarrUserWarning`: field values are preserved when writing arrays, although offsets and itemsize may change. Use `numpy.lib.recfunctions.repack_fields(data, recurse=True)` to make this conversion explicit. These checks also apply to nested fields. The `zarr.testing.strategies` module gains `zdtypes` and `structured_dtypes` strategies, with property tests for dtype serialization and warned layout normalization. diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index adaa67480f..c4f178f485 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -3,9 +3,11 @@ import warnings from collections.abc import Sequence from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Literal, Self, TypeGuard, cast, overload import numpy as np +from numpy.lib.recfunctions import repack_fields from zarr.core.common import NamedConfig from zarr.core.dtype.common import ( @@ -30,6 +32,9 @@ StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int +# The root of the zarr package, used to attribute layout warnings to the caller outside zarr. +_ZARR_PACKAGE_ROOT = str(Path(__file__).parents[3]) + def _check_representable(dtype: np.dtype[np.void]) -> str | None: """ @@ -238,28 +243,34 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: # so we must raise an error the registry propagates to the caller. raise ValueError( f"Cannot serialize the structured data type {dtype}: {reason}. The Zarr " - "struct data type records only field names and field data types, and " - "fields are always packed contiguously on read, so serializing this dtype " - "would silently change how the stored bytes are interpreted. Use a packed " - "structured dtype without titles or subarray fields instead." + "struct data type records only field names and field data types, so a " + "field title or a subarray shape has no representation in the metadata. " + "Use a structured dtype without titles or subarray fields instead." ) - # Iterate over `names` rather than `fields`: the `fields` mapping also - # contains an entry for every field title, which would duplicate titled fields. - for key in dtype.names: # type: ignore[union-attr] - dtype_wrapped = get_data_type_from_native_dtype(dtype.fields[key][0]) # type: ignore[index] - fields.append((key, dtype_wrapped)) - - result = cls(fields=tuple(fields)) - if result.to_native_dtype() != dtype: + # Repack once, at the top level, before resolving the fields. Nested fields then + # reach the registry already packed, so a padded nested field warns exactly once, + # here, rather than once per level of nesting. + packed = cast("np.dtype[np.void]", repack_fields(dtype, recurse=True)) + if packed != dtype: warnings.warn( "The structured dtype is converted to a packed field layout. " "Field values are preserved when writing arrays, but field offsets and " "itemsize may change. To pack explicitly, use " "numpy.lib.recfunctions.repack_fields(data, recurse=True).", ZarrUserWarning, - stacklevel=2, + # Attribute the warning to the first frame outside the zarr package, since + # the depth of the call chain that leads here varies by entry point. This + # reaches the caller for direct uses of the dtype API; the synchronous array + # API runs on the event loop thread, where no caller frame is available. + skip_file_prefixes=(_ZARR_PACKAGE_ROOT,), ) - return result + # Iterate over `names` rather than `fields`: the `fields` mapping also + # contains an entry for every field title, which would duplicate titled fields. + for key in packed.names: # type: ignore[union-attr] + dtype_wrapped = get_data_type_from_native_dtype(packed.fields[key][0]) # type: ignore[index] + fields.append((key, dtype_wrapped)) + + return cls(fields=tuple(fields)) raise DataTypeValidationError( f"Invalid data type: {dtype}. Expected an instance of {cls.dtype_cls}" ) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index dbee53b589..cc115b523e 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -344,13 +344,17 @@ def test_padded_structured_dtype_warns_and_preserves_values( """ Preserve the existing conversion of padded records to packed records, with a warning. The layout changes, but field values must survive writing and reopening the array. + + The warning is emitted exactly once, even for padding inside a nested field. """ data = np.ones(3, dtype=dtype) data["a"] = [1, 2, 3] expected = repack_fields(data, recurse=True) store = zarr.storage.MemoryStore() - with pytest.warns(ZarrUserWarning, match="packed.*layout"): + with pytest.warns(ZarrUserWarning, match="packed.*layout") as record: zarr.create_array(store, data=data, chunks=(2,), zarr_format=zarr_format) + layout_warnings = [w for w in record if issubclass(w.category, ZarrUserWarning)] + assert len(layout_warnings) == 1 reopened = zarr.open_array(store) assert reopened.dtype == expected.dtype assert reopened.dtype.itemsize == expected.dtype.itemsize From 4d6ad7db2f660da698cdca82e68c561885a33d98 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 13 Sep 2026 13:52:20 +0200 Subject: [PATCH 7/9] docs: remove redundant structured dtype packing advice Assisted-by: Codex:GPT-6 --- docs/user-guide/data_types.md | 2 -- src/zarr/core/dtype/npy/structured.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/user-guide/data_types.md b/docs/user-guide/data_types.md index 331c354e4b..f1f73306e0 100644 --- a/docs/user-guide/data_types.md +++ b/docs/user-guide/data_types.md @@ -90,8 +90,6 @@ element is a Zarr V2 data type specification. This representation supports recur Zarr stores structured fields in a packed layout. NumPy arrays with alignment, padding, or explicit field offsets remain accepted, with a `ZarrUserWarning`: writing preserves field values, but the stored dtype's offsets and itemsize may differ from the input. -To make this conversion explicit, use -`numpy.lib.recfunctions.repack_fields(data, recurse=True)` before creating the array. Field titles and subarray fields, including those inside nested structures, are rejected because their field information cannot be represented. These rules apply to both Zarr formats. diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index c4f178f485..77e747c7a2 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -255,8 +255,7 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: warnings.warn( "The structured dtype is converted to a packed field layout. " "Field values are preserved when writing arrays, but field offsets and " - "itemsize may change. To pack explicitly, use " - "numpy.lib.recfunctions.repack_fields(data, recurse=True).", + "itemsize may change.", ZarrUserWarning, # Attribute the warning to the first frame outside the zarr package, since # the depth of the call chain that leads here varies by entry point. This From 1e281483e93617640d494184bc34da6304f22223 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 13 Sep 2026 14:22:56 +0200 Subject: [PATCH 8/9] docs: distinguish structured dtype limitations from format rules Correct unsupported format claims using V2 history and runtime probes. Tighten the generated dtype property so unexpected rejection fails. Assisted-by: Codex:GPT-6 --- changes/4340.bugfix.md | 2 +- docs/user-guide/data_types.md | 14 ++++--- src/zarr/core/dtype/npy/structured.py | 35 +++++++++-------- src/zarr/testing/strategies.py | 27 +++++++------ tests/test_dtype/test_npy/test_structured.py | 8 ++-- tests/test_properties.py | 40 +++++++++++++------- 6 files changed, 73 insertions(+), 53 deletions(-) diff --git a/changes/4340.bugfix.md b/changes/4340.bugfix.md index 21cc62a70c..992cf67ff9 100644 --- a/changes/4340.bugfix.md +++ b/changes/4340.bugfix.md @@ -1 +1 @@ -Nested structured dtypes now round-trip through Zarr format 2 metadata. Structured dtypes with field titles or subarray fields now raise `ValueError` naming the offending field, instead of failing with an unrelated NumPy error. Aligned, padded, and explicitly offset layouts remain accepted and are converted to packed layouts with a `ZarrUserWarning`: field values are preserved when writing arrays, although offsets and itemsize may change. Use `numpy.lib.recfunctions.repack_fields(data, recurse=True)` to make this conversion explicit. These checks also apply to nested fields. The `zarr.testing.strategies` module gains `zdtypes` and `structured_dtypes` strategies, with property tests for dtype serialization and warned layout normalization. +Fix nested structured dtype round-tripping through Zarr format 2 metadata. Native NumPy dtypes with field titles or subarray fields now raise `ValueError` identifying the unsupported field during conversion, rather than being converted to duplicate fields or raw-byte fields. This reflects a limitation of the current Zarr-Python dtype implementation; the V2 format supports subarray fields, as did Zarr-Python 2.x. Padded layouts continue to be converted to packed layouts, now with a `ZarrUserWarning` explaining that field values are preserved when writing arrays but offsets and itemsize may change. These checks also apply to nested fields. Add dtype serialization and layout-conversion property tests using new `zdtypes` and `structured_dtypes` strategies. diff --git a/docs/user-guide/data_types.md b/docs/user-guide/data_types.md index f1f73306e0..4f44fc9576 100644 --- a/docs/user-guide/data_types.md +++ b/docs/user-guide/data_types.md @@ -84,14 +84,18 @@ arbitrary fixed-size byte strings. The `str` attribute of a regular NumPy void data type is the same as the `str` of a NumPy structured data type. This means that the `str` attribute does not convey information about the fields contained in a structured data type. For these reasons, Zarr V2 uses a special data type encoding for structured data types. -They are stored in JSON as lists of pairs, where the first element is a string, and the second -element is a Zarr V2 data type specification. This representation supports recursion. +The [V2 specification](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding) +represents fields as `[fieldname, datatype]` or `[fieldname, datatype, shape]`, where the +optional shape describes a subarray field. Field data types can themselves be structured. +For example, `[["position", " str | None: +def _unsupported_field_feature(dtype: np.dtype[np.void]) -> str | None: """ - Check for field features that the Zarr struct data type cannot represent. + Check for field features unsupported by this implementation's native dtype conversion. - The Zarr struct metadata records only `(name, dtype)` pairs and reconstructs the native - dtype by packing those fields contiguously (see `Structured.to_native_dtype`). Padding - can be removed while preserving field values, but titles and subarray fields would lose - field information. This function rejects those features, including in nested fields: + `Structured.fields` stores `(name, ZDType)` pairs and `to_native_dtype` packs them + contiguously. This conversion does not preserve NumPy field titles or subarray shapes. + Reject those features, including in nested fields, rather than silently losing them: - field titles, e.g. `np.dtype([(("title", "name"), "i4")])` - subarray fields, e.g. `np.dtype([("name", "i4", (2,))])` @@ -51,7 +50,12 @@ def _check_representable(dtype: np.dtype[np.void]) -> str | None: Returns ------- str | None - `None` if the dtype is representable, otherwise a short description of the problem. + `None` if these field features are supported, otherwise a description of the problem. + + Notes + ----- + This is an implementation limitation, not a statement about the V2 format, which has + an encoding for subarray fields. """ names = dtype.names fields = dtype.fields @@ -64,7 +68,7 @@ def _check_representable(dtype: np.dtype[np.void]) -> str | None: if field_dtype.subdtype is not None: return f"field {name!r} is a subarray with shape {field_dtype.shape}" if field_dtype.names is not None: - reason = _check_representable(field_dtype) + reason = _unsupported_field_feature(field_dtype) if reason is not None: return f"within field {name!r}: {reason}" return None @@ -216,8 +220,8 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: If the input data type is not an instance of np.dtypes.VoidDType with a non-null ``fields`` attribute. ValueError - If the input is a structured dtype that this data type cannot represent faithfully: - one with field titles or subarray fields. + If the input has field titles or subarray fields, which this implementation's + native dtype conversion does not support. Warns ----- @@ -234,18 +238,17 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] if cls._check_native_dtype(dtype): - reason = _check_representable(dtype) + reason = _unsupported_field_feature(dtype) if reason is not None: # NOTE: this is a ValueError rather than a DataTypeValidationError on purpose. # The data type registry suppresses DataTypeValidationError (treating it as - # "this dtype does not match"), but a dtype with an unrepresentable feature + # "this dtype does not match"), but a dtype with an unsupported field feature # *does* match this dtype class -- it simply cannot be represented faithfully -- # so we must raise an error the registry propagates to the caller. raise ValueError( - f"Cannot serialize the structured data type {dtype}: {reason}. The Zarr " - "struct data type records only field names and field data types, so a " - "field title or a subarray shape has no representation in the metadata. " - "Use a structured dtype without titles or subarray fields instead." + f"Cannot convert the structured data type {dtype}: {reason}. " + "Zarr-Python's current structured dtype conversion does not support " + "field titles or subarray fields." ) # Repack once, at the top level, before resolving the fields. Nested fields then # reach the registry already packed, so a padded nested field warns exactly once, diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 317a04c69c..cd974367c7 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -129,11 +129,11 @@ def _draw(draw: st.DrawFn) -> ZDType[Any, Any]: def zdtypes(*, max_leaves: int = 6) -> SearchStrategy[ZDType[Any, Any]]: """ - A strategy for instances of every registered `ZDType` class, including `Struct` with - arbitrarily nested fields. + Generate instances of the built-in registered `ZDType` classes, including nested `Struct`. - Struct fields are restricted to fixed-size data types, since the Zarr struct data type cannot - hold variable-length fields. + Struct fields are restricted to fixed-size data types, as required by the V3 `struct` + extension. This strategy samples bounded lengths and normalized datetime units/scales; + it does not cover every valid instance or arbitrary third-party dtype constructors. """ leaf_classes = [cls for cls in data_type_registry.contents.values() if cls is not Struct] leaves = st.one_of([_leaf_zdtypes(cls) for cls in leaf_classes]) @@ -148,17 +148,16 @@ def zdtypes(*, max_leaves: int = 6) -> SearchStrategy[ZDType[Any, Any]]: @st.composite def structured_dtypes( - draw: st.DrawFn, *, allow_unrepresentable: bool = False, max_depth: int = 3 + draw: st.DrawFn, *, allow_extended: bool = False, max_depth: int = 3 ) -> np.dtype[np.void]: """ A strategy for native NumPy structured dtypes, flat or nested. - With `allow_unrepresentable=False` (the default) every dtype is packed, has plain field names - and scalar fields, so it can be represented by the Zarr struct data type. With - `allow_unrepresentable=True` the strategy also injects the NumPy features that the Zarr - struct data type cannot record: field titles, subarray fields and `align=True` layouts. - Each is injected independently at random, so most draws carry at least one and some carry - none. + With `allow_extended=False` (the default), generate packed fields without titles or + subarray shapes. With `allow_extended=True`, also generate field titles, subarray fields, + and `align=True` layouts, independently. The current native dtype conversion rejects + titles and subarray fields and accepts padding with a warning. These are implementation + behaviors, not restrictions imposed by the V2 format. """ fixed_size_leaves = st.one_of( [ @@ -182,12 +181,12 @@ def build(depth: int) -> np.dtype[np.void]: else: field_dtype = draw(fixed_size_leaves).to_native_dtype() key: Any = name - if allow_unrepresentable and draw(st.booleans()): + if allow_extended and draw(st.booleans()): key = (title, name) - if allow_unrepresentable and draw(st.booleans()): + if allow_extended and draw(st.booleans()): field_dtype = (field_dtype, draw(npst.array_shapes(max_dims=2, max_side=3))) specs.append((key, field_dtype)) - align = allow_unrepresentable and draw(st.booleans()) + align = allow_extended and draw(st.booleans()) return np.dtype(specs, align=align) return build(0) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index cc115b523e..7c21528099 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -363,8 +363,8 @@ def test_padded_structured_dtype_warns_and_preserves_values( def test_titled_structured_dtype_raises() -> None: """ - A structured dtype with a field title must be rejected. NumPy's `fields` mapping lists the - title as an extra key, so a titled field used to be read back as two separate fields. + The current conversion rejects titles rather than treating their aliases as extra fields. + NumPy's `fields` mapping lists a string title as an extra key. """ dtype = np.dtype([(("title", "f0"), "i4"), ("g", "f8")]) with pytest.raises(ValueError, match="field 'f0' has a title"): @@ -373,8 +373,8 @@ def test_titled_structured_dtype_raises() -> None: def test_subarray_structured_dtype_raises() -> None: """ - A structured dtype with a subarray field must be rejected. The subarray dtype used to be - resolved as raw bytes, silently dropping its shape and element type. + The current conversion rejects subarrays rather than resolving them as raw bytes and + dropping their shape and element type. V2's support for subarray metadata is separate. """ dtype = np.dtype([("f0", "i4", (2,))]) with pytest.raises(ValueError, match="field 'f0' is a subarray"): diff --git a/tests/test_properties.py b/tests/test_properties.py index a2d7e8a9f7..fcb1076441 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -380,8 +380,8 @@ def _struct_depth(zdtype: ZDType[Any, Any]) -> int: @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zdtype_json_roundtrip(zdtype: ZDType[Any, Any], zarr_format: int) -> None: """ - Every registered data type, including arbitrarily nested structs, survives a round trip - through its JSON form for both Zarr formats. + Generated built-in data types, including nested structs, round-trip through their JSON + forms for both Zarr formats within the strategy's documented parameter ranges. Zarr format 3 data type names do not carry endianness (the bytes codec does), so for that format the JSON form is compared instead of the data type instance. @@ -399,8 +399,8 @@ def test_zdtype_json_roundtrip(zdtype: ZDType[Any, Any], zarr_format: int) -> No @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: """ - Every registered data type survives a round trip through its native NumPy dtype, and its - reported item size matches the native dtype's itemsize. + Generated built-in data types round-trip through their native NumPy dtypes within the + strategy's parameter ranges, and reported item sizes match native dtype itemsizes. The NumPy object dtype is shared by several Zarr data types, so resolving it is ambiguous by design and must raise instead. @@ -419,21 +419,35 @@ def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: assert zdtype.item_size == native.itemsize -@given(dtype=structured_dtypes(allow_unrepresentable=True)) +def _extended_descr_features(descr: list[Any]) -> set[str]: + """Classify NumPy's serialized field records independently of Zarr's dtype conversion.""" + features = set() + for field in descr: + if isinstance(field[0], tuple): + features.add("title") + if len(field) == 3: + features.add("subarray") + if isinstance(field[1], list): + features.update(_extended_descr_features(field[1])) + return features + + +@given(dtype=structured_dtypes(allow_extended=True)) @pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") def test_structured_dtype_never_silently_changes(dtype: np.dtype[np.void]) -> None: """ - For any native structured dtype, including ones with field titles, subarray fields or - aligned layouts, resolution either rejects unsupported field features or preserves the - fields in a packed layout. Any layout change must emit a warning. + Generated structured dtypes with titles or subarrays are rejected by the current + conversion. Other generated dtypes preserve their fields, warning on layout changes. """ - try: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always", ZarrUserWarning) - zdtype = get_data_type_from_native_dtype(dtype) - except ValueError: + unsupported_features = _extended_descr_features(dtype.descr) + if unsupported_features: + with pytest.raises(ValueError, match="|".join(sorted(unsupported_features))): + get_data_type_from_native_dtype(dtype) event("outcome=rejected") return + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", ZarrUserWarning) + zdtype = get_data_type_from_native_dtype(dtype) event("outcome=accepted") native = zdtype.to_native_dtype() assert native == repack_fields(dtype, recurse=True) From 209a235e8ed7771fb4ba63f4bb442e902eaec737 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 13 Sep 2026 17:34:45 +0200 Subject: [PATCH 9/9] docs(dtype): explain generic scale string serialization loss Assisted-by: Codex:GPT-6 --- src/zarr/testing/strategies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index cd974367c7..cd1cfba20a 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -104,8 +104,8 @@ def _leaf_zdtypes(cls: type[ZDType[TBaseDType, TBaseScalar]]) -> SearchStrategy[ def _normalize_generic_scale_factor(zdtype: Any) -> Any: """ - NumPy's generic time unit carries no scale factor, so only `scale_factor=1` has a native - representation for it. + NumPy retains generic scale factors internally, but its dtype string omits them. + Use `scale_factor=1` so the generated dtype survives Zarr V2 string serialization. """ if zdtype.unit == "generic": return dataclasses.replace(zdtype, scale_factor=1)