diff --git a/changes/4340.bugfix.md b/changes/4340.bugfix.md new file mode 100644 index 0000000000..9a23b5abf6 --- /dev/null +++ b/changes/4340.bugfix.md @@ -0,0 +1 @@ +Structured (``struct``) data types no longer silently corrupt data on the way in or out. Nested structured dtypes written with Zarr format 2 could not be read back, because the inner data type name check rejected the nested list-of-fields form that ``Struct.to_json`` itself produces. Structured dtypes that NumPy allows but this implementation does not preserve used to be accepted and rebuilt as something else: aligned or explicitly offset layouts came back packed with a different itemsize, a titled field came back as two fields, and a subarray field came back as raw bytes. ``Struct.from_native_dtype`` now raises a ``ValueError`` naming the offending field for all three. The ``zarr.testing.strategies`` module gains ``zdtypes`` and ``structured_dtypes`` strategies, and the property tests now check that every registered data type round-trips through JSON and NumPy, and that structured dtype resolution either raises or returns the input dtype exactly. diff --git a/changes/4342.bugfix.md b/changes/4342.bugfix.md new file mode 100644 index 0000000000..5b8f4bb342 --- /dev/null +++ b/changes/4342.bugfix.md @@ -0,0 +1 @@ +`DateTime64` and `TimeDelta64` normalize `μs` to `us` and preserve generic-unit scale factors when converting native NumPy dtypes and serializing V2/V3 metadata. V2 writes an explicit suffix such as `[2generic]` to avoid the scale loss in NumPy's `dtype.str` and `dtype.name`. Generic temporal arrays also preserve dtype parameters during CPU buffer allocation and correctly convert byte order; generic datetime fill values can be read back, including in structured fields. diff --git a/packages/zarr-metadata/changes/4342.bugfix.md b/packages/zarr-metadata/changes/4342.bugfix.md new file mode 100644 index 0000000000..240c7c8632 --- /dev/null +++ b/packages/zarr-metadata/changes/4342.bugfix.md @@ -0,0 +1 @@ +Added `numpy_datetime64_configuration`, `numpy_timedelta64_configuration`, and `numpy_time_unit` leaf validators. They validate the documented unit vocabulary and integer scale range `[1, 2**31 - 1]`, including scaled generic units, and normalize `μs` to `us`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py new file mode 100644 index 0000000000..6d93cf3942 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py @@ -0,0 +1,93 @@ +""" +Vocabulary and validation shared by the `numpy.datetime64` and `numpy.timedelta64` data types. + +This module is private (underscore-prefixed); the public names are re-exported by +`zarr_metadata.v3.data_type.numpy_datetime64` and +`zarr_metadata.v3.data_type.numpy_timedelta64`. +""" + +from collections.abc import Mapping +from typing import Final, Literal, cast + +NumpyTimeUnit = Literal[ + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" +] +"""Time unit codes used by numpy.datetime64 and numpy.timedelta64.""" + +NUMPY_TIME_UNIT: Final = ( + "Y", + "M", + "W", + "D", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + "generic", +) +"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" + +MAX_NUMPY_TIME_SCALE_FACTOR: Final = 2**31 - 1 +"""The largest `scale_factor` NumPy accepts for a datetime64 or timedelta64 dtype.""" + +_CONFIGURATION_KEYS: Final = frozenset({"unit", "scale_factor"}) + + +def numpy_time_unit(value: str) -> NumpyTimeUnit: + """Validate `value` as a NumPy time unit and return its canonical spelling. + + The spec lists `"us"` and `"μs"` as equivalent spellings of the microsecond + unit; NumPy itself only ever reports `"us"`, so `"μs"` is returned as `"us"`. + + Raises ValueError if `value` is not one of `NUMPY_TIME_UNIT`. + """ + if value not in NUMPY_TIME_UNIT: + raise ValueError(f"Expected one of {NUMPY_TIME_UNIT}, got {value!r}") + if value == "μs": + return "us" + return cast("NumpyTimeUnit", value) + + +def numpy_time_configuration(value: Mapping[str, object]) -> tuple[NumpyTimeUnit, int]: + """Validate a `numpy.datetime64` / `numpy.timedelta64` configuration object. + + Returns the `(unit, scale_factor)` pair with the unit in its canonical spelling + (see `numpy_time_unit`). + + Raises TypeError if `unit` is not a string or `scale_factor` is not an + integer. Raises ValueError if the object has keys other than exactly `unit` + and `scale_factor`, if `unit` is not a `NumpyTimeUnit`, if `scale_factor` is + outside `[1, MAX_NUMPY_TIME_SCALE_FACTOR]`. + """ + keys = frozenset(value) + if keys != _CONFIGURATION_KEYS: + raise ValueError( + f"Expected exactly the keys {sorted(_CONFIGURATION_KEYS)}, got {sorted(keys)}" + ) + raw_unit = value["unit"] + if not isinstance(raw_unit, str): + raise TypeError(f"Expected 'unit' to be a string, got {raw_unit!r}") + unit = numpy_time_unit(raw_unit) + scale_factor = value["scale_factor"] + if isinstance(scale_factor, bool) or not isinstance(scale_factor, int): + raise TypeError(f"Expected 'scale_factor' to be an integer, got {scale_factor!r}") + if not 1 <= scale_factor <= MAX_NUMPY_TIME_SCALE_FACTOR: + raise ValueError( + f"Expected 'scale_factor' in [1, {MAX_NUMPY_TIME_SCALE_FACTOR}], got {scale_factor}" + ) + return unit, scale_factor + + +__all__ = [ + "MAX_NUMPY_TIME_SCALE_FACTOR", + "NUMPY_TIME_UNIT", + "NumpyTimeUnit", + "numpy_time_configuration", + "numpy_time_unit", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8784160f71..8e6fb83bfb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -4,21 +4,23 @@ See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64 """ +from collections.abc import Mapping from typing import Final, Literal from typing_extensions import ReadOnly, TypedDict +from zarr_metadata.v3.data_type._numpy_time import ( + NumpyTimeUnit, + numpy_time_configuration, + numpy_time_unit, +) + NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" """The `name` field value of the `numpy.datetime64` data type.""" NumpyDatetime64DataTypeName = Literal["numpy.datetime64"] """Literal type of the `name` field of the `numpy.datetime64` data type.""" -NumpyTimeUnit = Literal[ - "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" -] -"""Time unit codes used by numpy.datetime64.""" - class NumpyDatetime64Configuration(TypedDict): """ @@ -36,6 +38,21 @@ class NumpyDatetime64Configuration(TypedDict): scale_factor: ReadOnly[int] +def numpy_datetime64_configuration(value: Mapping[str, object]) -> NumpyDatetime64Configuration: + """Validate `value` as a `numpy.datetime64` configuration and normalize it. + + The returned configuration spells the microsecond unit `"us"` even when the + input used the equivalent `"μs"`. + + Raises TypeError if `unit` is not a string or `scale_factor` is not an + integer. Raises ValueError if `value` does not have exactly the keys `unit` + and `scale_factor`, if `unit` is not a `NumpyTimeUnit`, if `scale_factor` is + outside `[1, 2**31 - 1]`. + """ + unit, scale_factor = numpy_time_configuration(value) + return {"unit": unit, "scale_factor": scale_factor} + + class NumpyDatetime64(TypedDict): """`numpy.datetime64` data type metadata.""" @@ -57,4 +74,6 @@ class NumpyDatetime64(TypedDict): "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", + "numpy_datetime64_configuration", + "numpy_time_unit", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index f5c8c77bf8..c0bcf9c06d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -4,40 +4,24 @@ See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64 """ +from collections.abc import Mapping from typing import Final, Literal from typing_extensions import ReadOnly, TypedDict +from zarr_metadata.v3.data_type._numpy_time import ( + NUMPY_TIME_UNIT, + NumpyTimeUnit, + numpy_time_configuration, + numpy_time_unit, +) + NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" """The `name` field value of the `numpy.timedelta64` data type.""" NumpyTimedelta64DataTypeName = Literal["numpy.timedelta64"] """Literal type of the `name` field of the `numpy.timedelta64` data type.""" -NumpyTimeUnit = Literal[ - "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" -] -"""Time unit codes used by numpy.timedelta64.""" - -NUMPY_TIME_UNIT: Final = ( - "Y", - "M", - "W", - "D", - "h", - "m", - "s", - "ms", - "us", - "μs", - "ns", - "ps", - "fs", - "as", - "generic", -) -"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" - class NumpyTimedelta64Configuration(TypedDict): """ @@ -55,6 +39,23 @@ class NumpyTimedelta64Configuration(TypedDict): scale_factor: ReadOnly[int] +def numpy_timedelta64_configuration( + value: Mapping[str, object], +) -> NumpyTimedelta64Configuration: + """Validate `value` as a `numpy.timedelta64` configuration and normalize it. + + The returned configuration spells the microsecond unit `"us"` even when the + input used the equivalent `"μs"`. + + Raises TypeError if `unit` is not a string or `scale_factor` is not an + integer. Raises ValueError if `value` does not have exactly the keys `unit` + and `scale_factor`, if `unit` is not a `NumpyTimeUnit`, if `scale_factor` is + outside `[1, 2**31 - 1]`. + """ + unit, scale_factor = numpy_time_configuration(value) + return {"unit": unit, "scale_factor": scale_factor} + + class NumpyTimedelta64(TypedDict): """`numpy.timedelta64` data type metadata.""" @@ -77,4 +78,6 @@ class NumpyTimedelta64(TypedDict): "NumpyTimedelta64Configuration", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", + "numpy_time_unit", + "numpy_timedelta64_configuration", ] diff --git a/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64/test_validators.py b/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64/test_validators.py new file mode 100644 index 0000000000..e8a70ef047 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/data_type/numpy_datetime64/test_validators.py @@ -0,0 +1,66 @@ +"""Cover the `numpy_datetime64_configuration` validator. + +The pydantic-driven fixture tests only check the structural shape of a +configuration; the constraints that tie `unit` and `scale_factor` together +live in the validator function and are covered directly here. +""" + +from __future__ import annotations + +import pytest + +from zarr_metadata.v3.data_type.numpy_datetime64 import numpy_datetime64_configuration + +# (input, expected normalized output) +VALID = [ + ({"unit": "ns", "scale_factor": 1}, {"unit": "ns", "scale_factor": 1}), + ({"unit": "s", "scale_factor": 10}, {"unit": "s", "scale_factor": 10}), + ({"unit": "generic", "scale_factor": 1}, {"unit": "generic", "scale_factor": 1}), + ({"unit": "generic", "scale_factor": 2}, {"unit": "generic", "scale_factor": 2}), + ( + {"unit": "generic", "scale_factor": 2**31 - 1}, + {"unit": "generic", "scale_factor": 2**31 - 1}, + ), + ({"unit": "us", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}), + ({"unit": "μs", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}), + ({"unit": "Y", "scale_factor": 2**31 - 1}, {"unit": "Y", "scale_factor": 2**31 - 1}), +] + + +@pytest.mark.parametrize(("value", "expected"), VALID, ids=lambda x: str(x)) +def test_valid(value: dict[str, object], expected: dict[str, object]) -> None: + assert numpy_datetime64_configuration(value) == expected + + +@pytest.mark.parametrize( + "value", + [{}, {"unit": "s"}, {"scale_factor": 1}, {"unit": "s", "scale_factor": 1, "extra": 0}], + ids=lambda x: str(x), +) +def test_wrong_keys(value: dict[str, object]) -> None: + with pytest.raises(ValueError, match="Expected exactly the keys"): + numpy_datetime64_configuration(value) + + +@pytest.mark.parametrize("unit", [1, None], ids=str) +def test_unit_not_a_string(unit: object) -> None: + with pytest.raises(TypeError, match="Expected 'unit' to be a string"): + numpy_datetime64_configuration({"unit": unit, "scale_factor": 1}) + + +@pytest.mark.parametrize("unit", ["", "invalid", "US", "seconds"], ids=str) +def test_unknown_unit(unit: str) -> None: + with pytest.raises(ValueError, match="Expected one of"): + numpy_datetime64_configuration({"unit": unit, "scale_factor": 1}) + + +@pytest.mark.parametrize("scale_factor", [1.0, "1", True, None], ids=str) +def test_scale_factor_not_an_integer(scale_factor: object) -> None: + with pytest.raises(TypeError, match="Expected 'scale_factor' to be an integer"): + numpy_datetime64_configuration({"unit": "s", "scale_factor": scale_factor}) + + +@pytest.mark.parametrize("scale_factor", [-1, 0, 2**31], ids=str) +def test_scale_factor_out_of_range(scale_factor: int) -> None: + with pytest.raises(ValueError, match=r"Expected 'scale_factor' in \[1, 2147483647\]"): + numpy_datetime64_configuration({"unit": "s", "scale_factor": scale_factor}) diff --git a/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64/test_validators.py b/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64/test_validators.py new file mode 100644 index 0000000000..67316176fb --- /dev/null +++ b/packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64/test_validators.py @@ -0,0 +1,66 @@ +"""Cover the `numpy_timedelta64_configuration` validator. + +The pydantic-driven fixture tests only check the structural shape of a +configuration; the constraints that tie `unit` and `scale_factor` together +live in the validator function and are covered directly here. +""" + +from __future__ import annotations + +import pytest + +from zarr_metadata.v3.data_type.numpy_timedelta64 import numpy_timedelta64_configuration + +# (input, expected normalized output) +VALID = [ + ({"unit": "ns", "scale_factor": 1}, {"unit": "ns", "scale_factor": 1}), + ({"unit": "s", "scale_factor": 10}, {"unit": "s", "scale_factor": 10}), + ({"unit": "generic", "scale_factor": 1}, {"unit": "generic", "scale_factor": 1}), + ({"unit": "generic", "scale_factor": 2}, {"unit": "generic", "scale_factor": 2}), + ( + {"unit": "generic", "scale_factor": 2**31 - 1}, + {"unit": "generic", "scale_factor": 2**31 - 1}, + ), + ({"unit": "us", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}), + ({"unit": "μs", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}), + ({"unit": "Y", "scale_factor": 2**31 - 1}, {"unit": "Y", "scale_factor": 2**31 - 1}), +] + + +@pytest.mark.parametrize(("value", "expected"), VALID, ids=lambda x: str(x)) +def test_valid(value: dict[str, object], expected: dict[str, object]) -> None: + assert numpy_timedelta64_configuration(value) == expected + + +@pytest.mark.parametrize( + "value", + [{}, {"unit": "s"}, {"scale_factor": 1}, {"unit": "s", "scale_factor": 1, "extra": 0}], + ids=lambda x: str(x), +) +def test_wrong_keys(value: dict[str, object]) -> None: + with pytest.raises(ValueError, match="Expected exactly the keys"): + numpy_timedelta64_configuration(value) + + +@pytest.mark.parametrize("unit", [1, None], ids=str) +def test_unit_not_a_string(unit: object) -> None: + with pytest.raises(TypeError, match="Expected 'unit' to be a string"): + numpy_timedelta64_configuration({"unit": unit, "scale_factor": 1}) + + +@pytest.mark.parametrize("unit", ["", "invalid", "US", "seconds"], ids=str) +def test_unknown_unit(unit: str) -> None: + with pytest.raises(ValueError, match="Expected one of"): + numpy_timedelta64_configuration({"unit": unit, "scale_factor": 1}) + + +@pytest.mark.parametrize("scale_factor", [1.0, "1", True, None], ids=str) +def test_scale_factor_not_an_integer(scale_factor: object) -> None: + with pytest.raises(TypeError, match="Expected 'scale_factor' to be an integer"): + numpy_timedelta64_configuration({"unit": "s", "scale_factor": scale_factor}) + + +@pytest.mark.parametrize("scale_factor", [-1, 0, 2**31], ids=str) +def test_scale_factor_out_of_range(scale_factor: int) -> None: + with pytest.raises(ValueError, match=r"Expected 'scale_factor' in \[1, 2147483647\]"): + numpy_timedelta64_configuration({"unit": "s", "scale_factor": scale_factor}) diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 8994281b58..859e04396d 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -4,6 +4,7 @@ TYPE_CHECKING, Any, Literal, + cast, ) import numpy as np @@ -155,20 +156,61 @@ def create( ) -> Self: # np.zeros is much faster than np.full, and therefore using it when possible is better. if fill_value is None or (isinstance(fill_value, int) and fill_value == 0): - return cls(np.zeros(shape=tuple(shape), dtype=dtype, order=order)) + data = np.zeros(shape=tuple(shape), dtype=dtype, order=order) else: - return cls(np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order)) + parsed_dtype = np.dtype(dtype) + if ( + isinstance(parsed_dtype, np.dtypes.DateTime64DType | np.dtypes.TimeDelta64DType) + and np.datetime_data( + cast("np.dtypes.DateTime64DType | np.dtypes.TimeDelta64DType", parsed_dtype) + )[0] + == "generic" + ): + # np.full can also drop the requested byte order. Restore the dtype + # before assignment so values are encoded in the requested byte order. + data = np.empty(shape=tuple(shape), dtype=parsed_dtype, order=order).view( + parsed_dtype + ) + data[...] = fill_value + else: + data = np.full(shape=tuple(shape), fill_value=fill_value, dtype=dtype, order=order) + if data.dtype.kind in "mM": + # NumPy allocation can discard generic temporal scales. A view retains them. + data = data.view(dtype=dtype) + return cls(data) @classmethod def empty( cls, shape: tuple[int, ...], dtype: npt.DTypeLike, order: Literal["C", "F"] = "C" ) -> Self: - return cls(np.empty(shape=shape, dtype=dtype, order=order)) + data = np.empty(shape=shape, dtype=dtype, order=order) + if data.dtype.kind in "mM": + data = data.view(dtype=dtype) + return cls(data) @classmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: return cls.from_ndarray_like(np.asanyarray(array_like)) + def astype(self, dtype: npt.DTypeLike, order: Literal["K", "A", "C", "F"] = "K") -> Self: + target = np.dtype(dtype) + if ( + self.dtype.kind in "mM" + and isinstance(target, np.dtypes.DateTime64DType | np.dtypes.TimeDelta64DType) + and target.kind == self.dtype.kind + and np.datetime_data(self.dtype)[0] == "generic" + and np.datetime_data(self.dtype) + == np.datetime_data( + cast("np.dtypes.DateTime64DType | np.dtypes.TimeDelta64DType", target) + ) + ): + # NumPy's generic-time astype can change the byte-order marker without + # swapping the data. Convert the underlying counts for an endian-only cast. + counts = self.as_numpy_array().view(self.dtype.byteorder + "i8") + converted = counts.astype(target.byteorder + "i8", order=order) + return self.__class__(converted.view(target)) + return super().astype(dtype, order=order) + def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). diff --git a/src/zarr/core/dtype/common.py b/src/zarr/core/dtype/common.py index 61cbfe0360..7a507892b8 100644 --- a/src/zarr/core/dtype/common.py +++ b/src/zarr/core/dtype/common.py @@ -80,7 +80,10 @@ def check_structured_dtype_v2_inner(data: object) -> TypeGuard[StructuredName_V2 if isinstance(data[-1], str): return True elif isinstance(data[-1], Sequence): - return check_structured_dtype_v2_inner(data[-1]) + # A nested structured dtype's field has the form [name, [[sub_name, sub_dtype], ...]], + # i.e. the last element is itself a sequence of field pairs rather than a single + # [name, dtype] pair, so it must be validated as a list of fields, not a single field. + return check_structured_dtype_name_v2(data[-1]) return False diff --git a/src/zarr/core/dtype/npy/structured.py b/src/zarr/core/dtype/npy/structured.py index dcc523d1d2..518becc4a7 100644 --- a/src/zarr/core/dtype/npy/structured.py +++ b/src/zarr/core/dtype/npy/structured.py @@ -30,6 +30,54 @@ StructuredScalarLike = list[object] | tuple[object, ...] | bytes | int +def _check_representable(dtype: np.dtype[np.void]) -> str | None: + """ + Check whether this implementation preserves a structured NumPy dtype. + + This implementation records only ``(name, dtype)`` pairs and reconstructs the native + dtype by packing those fields contiguously (see ``Structured.to_native_dtype``). Anything + NumPy allows beyond that is lost on the round trip and would silently change how stored + bytes are interpreted. This function rejects three such features, recursing into nested + fields so that a problem inside a nested field dtype is caught even when the outer dtype + is fine: + + - field titles, e.g. ``np.dtype([(("title", "name"), "i4")])`` + - subarray fields, e.g. ``np.dtype([("name", "i4", (2,))])`` + - non-default field layouts, e.g. ``np.dtype(..., align=True)`` or explicit offsets + + These are implementation restrictions. In particular, Zarr V2 supports subarray + field descriptors; this implementation does not preserve them. + + Returns + ------- + str | None + ``None`` if the dtype is representable, otherwise a short description of the problem. + """ + names = dtype.names + fields = dtype.fields + if names is None or fields is None: # pragma: no cover - only called on structured dtypes + return None + for name in names: + field_dtype, _offset, *title = fields[name] + if title: + return f"field {name!r} has a title ({title[0]!r})" + if field_dtype.subdtype is not None: + return f"field {name!r} is a subarray with shape {field_dtype.shape}" + if field_dtype.names is not None: + reason = _check_representable(field_dtype) + if reason is not None: + return f"within field {name!r}: {reason}" + # NumPy dtype equality compares field offsets and itemsize, so rebuilding the dtype from + # its (name, dtype) pairs and comparing detects any padding, alignment or explicit offsets. + repacked = np.dtype([(name, fields[name][0]) for name in names]) + if repacked != dtype: + return ( + "it uses a non-default field layout (e.g. it was created with align=True, or has " + "explicit field offsets or padding)" + ) + return None + + class StructuredJSON_V2(DTypeConfig_V2[StructuredName_V2, None]): """ A wrapper around the JSON representation of the ``Structured`` data type in Zarr V2. @@ -175,6 +223,10 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: DataTypeValidationError If the input data type is not an instance of np.dtypes.VoidDType with a non-null ``fields`` attribute. + ValueError + If the input is a structured dtype that this data type cannot represent faithfully: + one with field titles, subarray fields, or a non-default (aligned, padded or + explicitly offset) field layout. Notes ----- @@ -185,10 +237,25 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: fields: list[tuple[str, ZDType[TBaseDType, TBaseScalar]]] = [] if cls._check_native_dtype(dtype): - # fields of a structured numpy dtype are either 2-tuples or 3-tuples. we only - # care about the first element in either case. - for key, (dtype_instance, *_) in dtype.fields.items(): # type: ignore[union-attr] - dtype_wrapped = get_data_type_from_native_dtype(dtype_instance) + reason = _check_representable(dtype) + if reason is not None: + # NOTE: this is a ValueError rather than a DataTypeValidationError on purpose. + # The data type registry suppresses DataTypeValidationError (treating it as + # "this dtype does not match"), but a dtype with an unrepresentable feature + # *does* match this dtype class -- it simply cannot be represented faithfully -- + # so we must raise an error the registry propagates to the caller. + raise ValueError( + f"Cannot serialize the structured data type {dtype}: {reason}. The Zarr " + "struct data type records only field names and field data types, and " + "fields are always packed contiguously on read, so serializing this dtype " + "would silently change how the stored bytes are interpreted. Use a packed " + "structured dtype without titles, subarray fields, align=True or explicit " + "offsets instead." + ) + # Iterate over ``names`` rather than ``fields``: the ``fields`` mapping also + # contains an entry for every field title, which would duplicate titled fields. + for key in dtype.names: # type: ignore[union-attr] + dtype_wrapped = get_data_type_from_native_dtype(dtype.fields[key][0]) # type: ignore[index] fields.append((key, dtype_wrapped)) return cls(fields=tuple(fields)) @@ -442,7 +509,22 @@ def default_scalar(self) -> np.void: cast to this structured data type. """ - return self._cast_scalar_unchecked(0) + values: list[object] = [] + for _, field in self.fields: + dtype = field.to_native_dtype() + if isinstance(field, Structured): + value = field.default_scalar() + elif ( + isinstance(dtype, np.dtypes.DateTime64DType) + and np.datetime_data(cast("np.dtypes.DateTime64DType", dtype))[0] == "generic" + ): + # NumPy rejects casting integer zero to generic datetime, but a + # zero count is representable by viewing the integer storage. + value = np.zeros(1, dtype=dtype.byteorder + "i8").view(dtype)[0] + else: + value = np.array([0], dtype=dtype)[0] + values.append(value) + return self._cast_scalar_unchecked(tuple(values)) def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: """ diff --git a/src/zarr/core/dtype/npy/time.py b/src/zarr/core/dtype/npy/time.py index 4efa0be7bb..dddf65fd72 100644 --- a/src/zarr/core/dtype/npy/time.py +++ b/src/zarr/core/dtype/npy/time.py @@ -233,6 +233,11 @@ def __post_init__(self) -> None: raise ValueError(f"scale_factor must be < 2147483648, got {self.scale_factor}.") if self.unit not in get_args(DateTimeUnit): raise ValueError(f"unit must be one of {get_args(DateTimeUnit)}, got {self.unit!r}.") + if self.unit == "μs": + # NumPy spells the microsecond unit "us" and resolves "μs" to it, so an + # instance built with "μs" would not round-trip through to_native_dtype(). + # Store the NumPy spelling; "μs" stays accepted as input and in stored metadata. + object.__setattr__(self, "unit", "us") @classmethod def from_native_dtype(cls, dtype: TBaseDType) -> Self: @@ -256,7 +261,7 @@ def from_native_dtype(cls, dtype: TBaseDType) -> Self: """ if cls._check_native_dtype(dtype): - unit, scale_factor = np.datetime_data(dtype.name) + unit, scale_factor = np.datetime_data(dtype) unit = cast("DateTimeUnit", unit) return cls( unit=unit, @@ -492,7 +497,10 @@ def to_json(self, zarr_format: ZarrFormat) -> TimeDelta64JSON_V2 | TimeDelta64JS If the zarr_format is not 2 or 3. """ if zarr_format == 2: - name = self.to_native_dtype().str + name: str = self.to_native_dtype().str + if self.unit == "generic" and self.scale_factor != 1: + # NumPy omits generic scale from dtype.str; preserve it explicitly. + name += f"[{self.scale_factor}generic]" return {"name": name, "object_codec_id": None} elif zarr_format == 3: return { @@ -777,7 +785,10 @@ def to_json(self, zarr_format: ZarrFormat) -> DateTime64JSON_V2 | DateTime64JSON If the zarr_format is not 2 or 3. """ if zarr_format == 2: - name = self.to_native_dtype().str + name: str = self.to_native_dtype().str + if self.unit == "generic" and self.scale_factor != 1: + # NumPy omits generic scale from dtype.str; preserve it explicitly. + name += f"[{self.scale_factor}generic]" return {"name": name, "object_codec_id": None} elif zarr_format == 3: return { @@ -818,6 +829,9 @@ def _cast_scalar_unchecked(self, data: DateTimeLike) -> np.datetime64: numpy.datetime64 The input cast to a NumPy datetime scalar. """ + if isinstance(data, int): + # The scalar constructor rejects integer counts with a generic unit. + return datetime_from_int(data, unit=self.unit, scale_factor=self.scale_factor) # numpy 2.x stub: datetime64(scalar, formatted_unit_str) is runtime-valid # but no overload matches the dynamic f-string unit argument. return self.to_native_dtype().type(data, f"{self.scale_factor}{self.unit}") # type: ignore[call-overload, no-any-return] diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index db01697f1e..62ff07276e 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,3 +1,4 @@ +import dataclasses import itertools import math import sys @@ -27,7 +28,11 @@ from zarr.core.array import Array, CompressorsLike, SerializerLike from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding from zarr.core.common import JSON, AccessModeLiteral, ZarrFormat -from zarr.core.dtype import get_data_type_from_native_dtype +from zarr.core.dtype import data_type_registry, get_data_type_from_native_dtype +from zarr.core.dtype.common import HasItemSize +from zarr.core.dtype.npy.common import DATETIME_UNIT +from zarr.core.dtype.npy.structured import Struct +from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar, ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata from zarr.core.sync import sync @@ -70,6 +75,112 @@ def dtypes() -> st.SearchStrategy[np.dtype[Any]]: ) +_field_names = st.text( + alphabet=st.characters(min_codepoint=97, max_codepoint=122), min_size=1, max_size=4 +) +_field_titles = st.text( + alphabet=st.characters(min_codepoint=65, max_codepoint=90), min_size=1, max_size=4 +) + + +def _leaf_zdtypes(cls: type[ZDType[TBaseDType, TBaseScalar]]) -> SearchStrategy[ZDType[Any, Any]]: + """ + A strategy for instances of a single non-struct ``ZDType`` class, drawing each constructor + parameter the class declares from its valid range. + """ + params = {f.name for f in dataclasses.fields(cls)} + kwargs: dict[str, SearchStrategy[Any]] = {} + if "endianness" in params: + kwargs["endianness"] = st.sampled_from(["little", "big"]) + if "length" in params: + kwargs["length"] = st.integers(min_value=1, max_value=16) + if "unit" in params: + # The constructor normalizes the microsecond alias; all units accept a scale. + kwargs["unit"] = st.sampled_from(DATETIME_UNIT) + kwargs["scale_factor"] = st.integers(min_value=1, max_value=2**31 - 1) + return st.builds(cls, **kwargs) + + +def _struct_zdtypes( + children: SearchStrategy[ZDType[Any, Any]], +) -> SearchStrategy[ZDType[Any, Any]]: + """A strategy for ``Struct`` instances whose field data types are drawn from ``children``.""" + + @st.composite + def _draw(draw: st.DrawFn) -> ZDType[Any, Any]: + num_fields = draw(st.integers(min_value=1, max_value=4)) + # suffix with the index so that names are unique without filtering + names = [f"{draw(_field_names)}{i}" for i in range(num_fields)] + return Struct(fields=tuple((name, draw(children)) for name in names)) + + return _draw() + + +def zdtypes(*, max_leaves: int = 6) -> SearchStrategy[ZDType[Any, Any]]: + """ + A strategy for instances of every registered ``ZDType`` class, including ``Struct`` with + arbitrarily nested fields. + + Struct fields are restricted to fixed-size data types, since the Zarr struct data type cannot + hold variable-length fields. + """ + leaf_classes = [cls for cls in data_type_registry.contents.values() if cls is not Struct] + leaves = st.one_of([_leaf_zdtypes(cls) for cls in leaf_classes]) + fixed_size_leaves = st.one_of( + [_leaf_zdtypes(cls) for cls in leaf_classes if issubclass(cls, HasItemSize)] + ) + structs = st.recursive(fixed_size_leaves, _struct_zdtypes, max_leaves=max_leaves).filter( + lambda dt: isinstance(dt, Struct) + ) + return leaves | structs + + +@st.composite +def structured_dtypes( + draw: st.DrawFn, *, allow_unrepresentable: bool = False, max_depth: int = 3 +) -> np.dtype[np.void]: + """ + A strategy for native NumPy structured dtypes, flat or nested. + + With ``allow_unrepresentable=False`` (the default) every dtype is packed, has plain field names + and scalar fields, so it can be represented by the Zarr struct data type. With + ``allow_unrepresentable=True`` the strategy also injects NumPy features rejected by this implementation: field titles, subarray fields and ``align=True`` layouts. + Each is injected independently at random, so most draws carry at least one and some carry + none. + """ + fixed_size_leaves = st.one_of( + [ + _leaf_zdtypes(cls) + for cls in data_type_registry.contents.values() + if cls is not Struct and issubclass(cls, HasItemSize) + ] + ) + + def build(depth: int) -> np.dtype[np.void]: + num_fields = draw(st.integers(min_value=1, max_value=4)) + # suffix with the index so that names and titles are unique without filtering; titles + # draw from a different alphabet so they never collide with names either + names = [f"{draw(_field_names)}{i}" for i in range(num_fields)] + titles = [f"{draw(_field_titles)}{i}" for i in range(num_fields)] + specs: list[tuple[Any, Any]] = [] + for name, title in zip(names, titles, strict=True): + field_dtype: Any + if depth < max_depth and draw(st.booleans()): + field_dtype = build(depth + 1) + else: + field_dtype = draw(fixed_size_leaves).to_native_dtype() + key: Any = name + if allow_unrepresentable and draw(st.booleans()): + key = (title, name) + if allow_unrepresentable and draw(st.booleans()): + field_dtype = (field_dtype, draw(npst.array_shapes(max_dims=2, max_side=3))) + specs.append((key, field_dtype)) + align = allow_unrepresentable and draw(st.booleans()) + return np.dtype(specs, align=align) + + return build(0) + + def v3_dtypes() -> st.SearchStrategy[np.dtype[Any]]: return dtypes() diff --git a/tests/test_buffer.py b/tests/test_buffer.py index b4a16ed1de..2f1cdb58f0 100644 --- a/tests/test_buffer.py +++ b/tests/test_buffer.py @@ -238,3 +238,32 @@ def test_empty( assert result.flags.c_contiguous # type: ignore[attr-defined] else: assert result.flags.f_contiguous # type: ignore[attr-defined] + + +@pytest.mark.parametrize("kind", ["M8", "m8"]) +@pytest.mark.parametrize("byteorder", ["<", ">"]) +@pytest.mark.parametrize("scale_factor", [2, 2**31 - 1]) +@pytest.mark.parametrize("fill_value", [None, 0, "NaT"]) +@pytest.mark.parametrize("order", ["C", "F"]) +@pytest.mark.filterwarnings( + "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning" +) +def test_cpu_generic_time_allocation( + kind: str, + byteorder: str, + scale_factor: int, + fill_value: int | str | None, + order: Literal["C", "F"], +) -> None: + """Both allocation paths retain generic scale even when NumPy drops it.""" + dtype = np.dtype(f"{byteorder}{kind}[{scale_factor}generic]") + empty = cpu.NDBuffer.empty((2, 3), dtype=dtype, order=order) + filled = cpu.NDBuffer.create(shape=(2, 3), dtype=dtype, order=order, fill_value=fill_value) + for buffer in (empty, filled): + assert buffer.dtype == dtype + array = buffer.as_numpy_array() + assert array.flags.c_contiguous if order == "C" else array.flags.f_contiguous + expected = -(2**63) if fill_value == "NaT" else 0 + np.testing.assert_array_equal( + filled.as_numpy_array().view(dtype.byteorder + "i8"), np.full((2, 3), expected) + ) diff --git a/tests/test_dtype/test_npy/test_structured.py b/tests/test_dtype/test_npy/test_structured.py index 554c3b4e41..bdb49e0a78 100644 --- a/tests/test_dtype/test_npy/test_structured.py +++ b/tests/test_dtype/test_npy/test_structured.py @@ -1,10 +1,11 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np import pytest +import zarr from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype import ( Float16, @@ -14,8 +15,12 @@ Struct, Structured, UInt8, + get_data_type_from_json, ) +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + class TestStruct(BaseTestZDType): """Test the canonical 'struct' dtype format.""" @@ -260,3 +265,103 @@ def test_struct_from_native_dtype() -> None: struct = Struct.from_native_dtype(dtype) assert struct.fields[0][0] == "field1" assert struct.fields[1][0] == "field2" + + +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize( + "dtype", + [ + # flat + np.dtype([("a", "i1"), ("b", "i8")]), + # one level of nesting + np.dtype([("a", " None: + """ + A packed (default-layout) structured dtype, flat or nested, round-trips unchanged through + both the JSON form and the native form. + + Nested dtypes are the regression case: ``Struct.to_json(zarr_format=2)`` emits a nested + field as ``[name, [[sub, dt], ...]]``, and the inner V2 type guard used to reject that form, + so Zarr wrote V2 metadata it could not read back. + """ + zdtype = Struct.from_native_dtype(dtype) + recovered = get_data_type_from_json( + zdtype.to_json(zarr_format=zarr_format), zarr_format=zarr_format + ) + assert recovered == zdtype + assert recovered.to_native_dtype() == dtype + assert recovered.to_native_dtype().itemsize == dtype.itemsize + + +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +def test_nested_structured_v2_array_round_trip() -> None: + """End-to-end test: write and read a Zarr V2 array with a nested structured dtype.""" + dtype = np.dtype([("a", " None: + """ + Structured dtypes with non-default (padded / aligned / explicitly offset) field layouts must + fail loudly rather than silently dropping the padding. + + The Zarr struct metadata records only ``(name, dtype)`` pairs and re-packs fields + contiguously on read, so a padded dtype would round-trip to a different itemsize and + silently misinterpret stored chunk bytes. + """ + with pytest.raises(ValueError, match="non-default field layout"): + Struct.from_native_dtype(dtype) + + +def test_titled_structured_dtype_raises() -> None: + """ + A structured dtype with a field title must be rejected. NumPy's ``fields`` mapping lists the + title as an extra key, so a titled field used to be read back as two separate fields. + """ + dtype = np.dtype([(("title", "f0"), "i4"), ("g", "f8")]) + with pytest.raises(ValueError, match="field 'f0' has a title"): + Struct.from_native_dtype(dtype) + + +def test_subarray_structured_dtype_raises() -> None: + """ + A structured dtype with a subarray field must be rejected. The subarray dtype used to be + resolved as raw bytes, silently dropping its shape and element type. + """ + dtype = np.dtype([("f0", "i4", (2,))]) + with pytest.raises(ValueError, match="field 'f0' is a subarray"): + Struct.from_native_dtype(dtype) diff --git a/tests/test_dtype/test_npy/test_time.py b/tests/test_dtype/test_npy/test_time.py index 67ba3bd130..900ea9b853 100644 --- a/tests/test_dtype/test_npy/test_time.py +++ b/tests/test_dtype/test_npy/test_time.py @@ -1,15 +1,19 @@ from __future__ import annotations import re -from typing import get_args +from typing import TYPE_CHECKING, get_args import numpy as np import pytest +import zarr from tests.test_dtype.test_wrapper import BaseTestZDType from zarr.core.dtype.npy.common import DateTimeUnit from zarr.core.dtype.npy.time import DateTime64, TimeDelta64, datetime_from_int +if TYPE_CHECKING: + from zarr.core.common import ZarrFormat + class _TestTimeBase(BaseTestZDType): def json_scalar_equals(self, scalar1: object, scalar2: object) -> bool: @@ -43,6 +47,7 @@ class TestDateTime64(_TestTimeBase): valid_json_v3 = ( {"name": "numpy.datetime64", "configuration": {"unit": "ns", "scale_factor": 10}}, {"name": "numpy.datetime64", "configuration": {"unit": "us", "scale_factor": 1}}, + {"name": "numpy.datetime64", "configuration": {"unit": "generic", "scale_factor": 1}}, ) invalid_json_v2 = ( "datetime64", @@ -93,6 +98,7 @@ class TestTimeDelta64(_TestTimeBase): valid_json_v3 = ( {"name": "numpy.timedelta64", "configuration": {"unit": "ns", "scale_factor": 10}}, {"name": "numpy.timedelta64", "configuration": {"unit": "us", "scale_factor": 1}}, + {"name": "numpy.timedelta64", "configuration": {"unit": "generic", "scale_factor": 1}}, ) invalid_json_v2 = ( "timedelta64", @@ -166,6 +172,53 @@ def test_time_scale_factor_too_high() -> None: TimeDelta64(scale_factor=scale_factor) +@pytest.mark.parametrize("cls", [DateTime64, TimeDelta64]) +@pytest.mark.parametrize("unit", get_args(DateTimeUnit)) +@pytest.mark.parametrize("scale_factor", [1, 2, 2**31 - 1]) +@pytest.mark.parametrize("byteorder", ["<", ">"]) +def test_time_dtype_roundtrip( + cls: type[DateTime64 | TimeDelta64], + unit: DateTimeUnit, + scale_factor: int, + byteorder: str, +) -> None: + """Native and JSON conversions must preserve temporal parameters, including generic scale.""" + kind = "M8" if cls is DateTime64 else "m8" + native = np.dtype(f"{byteorder}{kind}[{scale_factor}{unit}]") + expected_unit = "us" if unit == "μs" else unit + dtype = cls.from_native_dtype(native) + assert (dtype.unit, dtype.scale_factor) == (expected_unit, scale_factor) + restored_native = dtype.to_native_dtype() + assert np.datetime_data(restored_native) == (expected_unit, scale_factor) + assert restored_native == native + json_v2 = dtype.to_json(zarr_format=2) + assert np.datetime_data(np.dtype(json_v2["name"])) == (expected_unit, scale_factor) + assert cls.from_json(json_v2, zarr_format=2) == dtype + json_v3 = dtype.to_json(zarr_format=3) + assert json_v3["configuration"]["unit"] == expected_unit + assert json_v3["configuration"]["scale_factor"] == scale_factor + restored_v3 = cls.from_json(json_v3, zarr_format=3) + assert np.datetime_data(restored_v3.to_native_dtype()) == (expected_unit, scale_factor) + + +@pytest.mark.parametrize("cls", [DateTime64, TimeDelta64]) +def test_time_microsecond_alias_normalized(cls: type[DateTime64 | TimeDelta64]) -> None: + """ + Test that the 'μs' unit is stored as NumPy's 'us' spelling. + + The two spellings are equivalent, but NumPy only ever reports 'us', so an instance + that kept 'μs' would compare unequal to itself after a trip through NumPy. Stored + metadata may still spell the unit 'μs' and reads back as the normalized instance. + """ + zdtype = cls(unit="μs", scale_factor=3) + assert zdtype.unit == "us" + assert zdtype == cls(unit="us", scale_factor=3) + assert cls.from_native_dtype(zdtype.to_native_dtype()) == zdtype + json_v3 = {"name": cls._zarr_v3_name, "configuration": {"unit": "μs", "scale_factor": 3}} + assert cls.from_json(json_v3, zarr_format=3) == zdtype + assert zdtype.to_json(zarr_format=3)["configuration"]["unit"] == "us" + + @pytest.mark.parametrize("unit", get_args(DateTimeUnit)) @pytest.mark.parametrize("scale_factor", [1, 10]) @pytest.mark.parametrize("value", [0, 1, 10]) @@ -175,3 +228,41 @@ def test_datetime_from_int(unit: DateTimeUnit, scale_factor: int, value: int) -> """ expected = np.int64(value).view(f"datetime64[{scale_factor}{unit}]") assert datetime_from_int(value, unit=unit, scale_factor=scale_factor) == expected + + +@pytest.mark.parametrize("unit", ["generic", "us"]) +@pytest.mark.parametrize("kind", ["M8", "m8"]) +@pytest.mark.parametrize("byteorder", ["<", ">"]) +@pytest.mark.parametrize("scale_factor", [1, 2, 2**31 - 1]) +@pytest.mark.parametrize("zarr_format", [2, 3]) +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.filterwarnings("ignore::zarr.errors.UnstableSpecificationWarning") +@pytest.mark.filterwarnings( + "ignore:The 'generic' unit for NumPy timedelta is deprecated:DeprecationWarning" +) +def test_generic_time_array_roundtrip( + unit: str, + kind: str, + byteorder: str, + scale_factor: int, + zarr_format: ZarrFormat, + structured: bool, +) -> None: + """Persist counts and generic scale through metadata, chunk IO, and output allocation.""" + leaf = np.dtype(f"{byteorder}{kind}[{scale_factor}{unit}]") + dtype = np.dtype([("time", leaf)]) if structured else leaf + counts = np.array([0, 1, -2, 100], dtype=f"{byteorder}i8") + data = counts.view(dtype) + array = zarr.create_array( + store={}, data=data, chunks=2, zarr_format=zarr_format, compressors=None + ) + array.resize((6,)) + reopened = zarr.open_array(array.store, mode="r") + result = np.asarray(reopened[:]) + values = result["time"] if structured else result + assert np.datetime_data(values.dtype) == (unit, scale_factor) + np.testing.assert_array_equal(values[:4].view(values.dtype.byteorder + "i8"), counts) + expected_fill = 0 if structured else -(2**63) + np.testing.assert_array_equal( + values[4:].view(values.dtype.byteorder + "i8"), [expected_fill, expected_fill] + ) diff --git a/tests/test_properties.py b/tests/test_properties.py index 2794ad3cb0..831c4abeef 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -19,6 +19,10 @@ from zarr.abc.store import Store from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZATTRS_JSON +from zarr.core.dtype import get_data_type_from_json, get_data_type_from_native_dtype +from zarr.core.dtype.common import HasItemSize +from zarr.core.dtype.npy.structured import Struct +from zarr.core.dtype.wrapper import ZDType from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.core.sync import sync from zarr.testing.strategies import ( @@ -34,7 +38,9 @@ sharded_arrays, simple_arrays, stores, + structured_dtypes, zarr_formats, + zdtypes, ) @@ -361,6 +367,75 @@ def test_roundtrip_array_metadata_from_json(data: st.DataObject, zarr_format: in assert deep_equal(orig, rt), f"Roundtrip mismatch:\nOriginal: {orig}\nRoundtripped: {rt}" +def _struct_depth(zdtype: ZDType[Any, Any]) -> int: + if not isinstance(zdtype, Struct): + return 0 + return 1 + max(_struct_depth(field_dtype) for _, field_dtype in zdtype.fields) + + +@given(zdtype=zdtypes(), zarr_format=zarr_formats) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_zdtype_json_roundtrip(zdtype: ZDType[Any, Any], zarr_format: int) -> None: + """ + Every registered data type, including arbitrarily nested structs, survives a round trip + through its JSON form for both Zarr formats. + + Zarr format 3 data type names do not carry endianness (the bytes codec does), so for that + format the JSON form is compared instead of the data type instance. + """ + event(f"dtype={type(zdtype).__name__}") + event(f"struct_depth={_struct_depth(zdtype)}") + as_json = zdtype.to_json(zarr_format=zarr_format) # type: ignore[arg-type] + roundtripped = get_data_type_from_json(as_json, zarr_format=zarr_format) + assert roundtripped.to_json(zarr_format=zarr_format) == as_json # type: ignore[arg-type] + if zarr_format == 2: + assert roundtripped == zdtype + + +@given(zdtype=zdtypes()) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_zdtype_native_roundtrip(zdtype: ZDType[Any, Any]) -> None: + """ + Every registered data type survives a round trip through its native NumPy dtype, and its + reported item size matches the native dtype's itemsize. + + The NumPy object dtype is shared by several Zarr data types, so resolving it is ambiguous by + design and must raise instead. + """ + event(f"dtype={type(zdtype).__name__}") + native = zdtype.to_native_dtype() + if native.kind == "O": + event("native=object") + with pytest.raises(ValueError, match="ambiguous"): + get_data_type_from_native_dtype(native) + return + roundtripped = get_data_type_from_native_dtype(native) + assert roundtripped == zdtype + assert roundtripped.to_native_dtype() == native + if isinstance(zdtype, HasItemSize): + assert zdtype.item_size == native.itemsize + + +@given(dtype=structured_dtypes(allow_unrepresentable=True)) +@pytest.mark.filterwarnings("ignore::zarr.core.dtype.common.UnstableSpecificationWarning") +def test_structured_dtype_never_silently_changes(dtype: np.dtype[np.void]) -> None: + """ + For any native structured dtype, including ones with field titles, subarray fields or + aligned layouts, resolving a Zarr data type either raises ``ValueError`` or yields a data type + whose native form is exactly the input: same field names, offsets and itemsize. + + This is the property that the silent-corruption bugs in structured dtype handling violated: + a dtype was accepted but came back with different fields, offsets or itemsize. + """ + try: + zdtype = get_data_type_from_native_dtype(dtype) + except ValueError: + event("outcome=rejected") + return + event("outcome=accepted") + assert zdtype.to_native_dtype() == dtype + + # @st.composite # def advanced_indices(draw, *, shape): # basic_idxr = draw(