fix(dtype): preserve scaled generic time units and normalize the μs alias

changes/4340.bugfix.md

@@ -0,0 +1 @@
+Structured (``struct``) data types no longer silently corrupt data on the way in or out. Nested structured dtypes written with Zarr format 2 could not be read back, because the inner data type name check rejected the nested list-of-fields form that ``Struct.to_json`` itself produces. Structured dtypes that NumPy allows but this implementation does not preserve used to be accepted and rebuilt as something else: aligned or explicitly offset layouts came back packed with a different itemsize, a titled field came back as two fields, and a subarray field came back as raw bytes. ``Struct.from_native_dtype`` now raises a ``ValueError`` naming the offending field for all three. The ``zarr.testing.strategies`` module gains ``zdtypes`` and ``structured_dtypes`` strategies, and the property tests now check that every registered data type round-trips through JSON and NumPy, and that structured dtype resolution either raises or returns the input dtype exactly.

changes/4342.bugfix.md

@@ -0,0 +1 @@
+`DateTime64` and `TimeDelta64` normalize `μs` to `us` and preserve generic-unit scale factors when converting native NumPy dtypes and serializing V2/V3 metadata. V2 writes an explicit suffix such as `[2generic]` to avoid the scale loss in NumPy's `dtype.str` and `dtype.name`. Generic temporal arrays also preserve dtype parameters during CPU buffer allocation and correctly convert byte order; generic datetime fill values can be read back, including in structured fields.

packages/zarr-metadata/changes/4342.bugfix.md

@@ -0,0 +1 @@
+Added `numpy_datetime64_configuration`, `numpy_timedelta64_configuration`, and `numpy_time_unit` leaf validators. They validate the documented unit vocabulary and integer scale range `[1, 2**31 - 1]`, including scaled generic units, and normalize `μs` to `us`.

packages/zarr-metadata/src/zarr_metadata/v3/data_type/_numpy_time.py

@@ -0,0 +1,93 @@
+"""
+Vocabulary and validation shared by the `numpy.datetime64` and `numpy.timedelta64` data types.
+
+This module is private (underscore-prefixed); the public names are re-exported by
+`zarr_metadata.v3.data_type.numpy_datetime64` and
+`zarr_metadata.v3.data_type.numpy_timedelta64`.
+"""
+
+from collections.abc import Mapping
+from typing import Final, Literal, cast
+
+NumpyTimeUnit = Literal[
+    "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic"
+]
+"""Time unit codes used by numpy.datetime64 and numpy.timedelta64."""
+
+NUMPY_TIME_UNIT: Final = (
+    "Y",
+    "M",
+    "W",
+    "D",
+    "h",
+    "m",
+    "s",
+    "ms",
+    "us",
+    "μs",
+    "ns",
+    "ps",
+    "fs",
+    "as",
+    "generic",
+)
+"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings."""
+
+MAX_NUMPY_TIME_SCALE_FACTOR: Final = 2**31 - 1
+"""The largest `scale_factor` NumPy accepts for a datetime64 or timedelta64 dtype."""
+
+_CONFIGURATION_KEYS: Final = frozenset({"unit", "scale_factor"})
+
+
+def numpy_time_unit(value: str) -> NumpyTimeUnit:
+    """Validate `value` as a NumPy time unit and return its canonical spelling.
+
+    The spec lists `"us"` and `"μs"` as equivalent spellings of the microsecond
+    unit; NumPy itself only ever reports `"us"`, so `"μs"` is returned as `"us"`.
+
+    Raises ValueError if `value` is not one of `NUMPY_TIME_UNIT`.
+    """
+    if value not in NUMPY_TIME_UNIT:
+        raise ValueError(f"Expected one of {NUMPY_TIME_UNIT}, got {value!r}")
+    if value == "μs":
+        return "us"
+    return cast("NumpyTimeUnit", value)
+
+
+def numpy_time_configuration(value: Mapping[str, object]) -> tuple[NumpyTimeUnit, int]:
+    """Validate a `numpy.datetime64` / `numpy.timedelta64` configuration object.
+
+    Returns the `(unit, scale_factor)` pair with the unit in its canonical spelling
+    (see `numpy_time_unit`).
+
+    Raises TypeError if `unit` is not a string or `scale_factor` is not an
+    integer. Raises ValueError if the object has keys other than exactly `unit`
+    and `scale_factor`, if `unit` is not a `NumpyTimeUnit`, if `scale_factor` is
+    outside `[1, MAX_NUMPY_TIME_SCALE_FACTOR]`.
+    """
+    keys = frozenset(value)
+    if keys != _CONFIGURATION_KEYS:
+        raise ValueError(
+            f"Expected exactly the keys {sorted(_CONFIGURATION_KEYS)}, got {sorted(keys)}"
+        )
+    raw_unit = value["unit"]
+    if not isinstance(raw_unit, str):
+        raise TypeError(f"Expected 'unit' to be a string, got {raw_unit!r}")
+    unit = numpy_time_unit(raw_unit)
+    scale_factor = value["scale_factor"]
+    if isinstance(scale_factor, bool) or not isinstance(scale_factor, int):
+        raise TypeError(f"Expected 'scale_factor' to be an integer, got {scale_factor!r}")
+    if not 1 <= scale_factor <= MAX_NUMPY_TIME_SCALE_FACTOR:
+        raise ValueError(
+            f"Expected 'scale_factor' in [1, {MAX_NUMPY_TIME_SCALE_FACTOR}], got {scale_factor}"
+        )
+    return unit, scale_factor
+
+
+__all__ = [
+    "MAX_NUMPY_TIME_SCALE_FACTOR",
+    "NUMPY_TIME_UNIT",
+    "NumpyTimeUnit",
+    "numpy_time_configuration",
+    "numpy_time_unit",
+]

packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py

@@ -4,21 +4,23 @@
 See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64
 """
 
+from collections.abc import Mapping
 from typing import Final, Literal
 
 from typing_extensions import ReadOnly, TypedDict
 
+from zarr_metadata.v3.data_type._numpy_time import (
+    NumpyTimeUnit,
+    numpy_time_configuration,
+    numpy_time_unit,
+)
+
 NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64"
 """The `name` field value of the `numpy.datetime64` data type."""
 
 NumpyDatetime64DataTypeName = Literal["numpy.datetime64"]
 """Literal type of the `name` field of the `numpy.datetime64` data type."""
 
-NumpyTimeUnit = Literal[
-    "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic"
-]
-"""Time unit codes used by numpy.datetime64."""
-
 
 class NumpyDatetime64Configuration(TypedDict):
     """
@@ -36,6 +38,21 @@ class NumpyDatetime64Configuration(TypedDict):
     scale_factor: ReadOnly[int]
 
 
+def numpy_datetime64_configuration(value: Mapping[str, object]) -> NumpyDatetime64Configuration:
+    """Validate `value` as a `numpy.datetime64` configuration and normalize it.
+
+    The returned configuration spells the microsecond unit `"us"` even when the
+    input used the equivalent `"μs"`.
+
+    Raises TypeError if `unit` is not a string or `scale_factor` is not an
+    integer. Raises ValueError if `value` does not have exactly the keys `unit`
+    and `scale_factor`, if `unit` is not a `NumpyTimeUnit`, if `scale_factor` is
+    outside `[1, 2**31 - 1]`.
+    """
+    unit, scale_factor = numpy_time_configuration(value)
+    return {"unit": unit, "scale_factor": scale_factor}
+
+
 class NumpyDatetime64(TypedDict):
     """`numpy.datetime64` data type metadata."""
 
@@ -57,4 +74,6 @@ class NumpyDatetime64(TypedDict):
     "NumpyDatetime64DataTypeName",
     "NumpyDatetime64FillValue",
     "NumpyTimeUnit",
+    "numpy_datetime64_configuration",
+    "numpy_time_unit",
 ]

packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py

@@ -4,40 +4,24 @@
 See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64
 """
 
+from collections.abc import Mapping
 from typing import Final, Literal
 
 from typing_extensions import ReadOnly, TypedDict
 
+from zarr_metadata.v3.data_type._numpy_time import (
+    NUMPY_TIME_UNIT,
+    NumpyTimeUnit,
+    numpy_time_configuration,
+    numpy_time_unit,
+)
+
 NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64"
 """The `name` field value of the `numpy.timedelta64` data type."""
 
 NumpyTimedelta64DataTypeName = Literal["numpy.timedelta64"]
 """Literal type of the `name` field of the `numpy.timedelta64` data type."""
 
-NumpyTimeUnit = Literal[
-    "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic"
-]
-"""Time unit codes used by numpy.timedelta64."""
-
-NUMPY_TIME_UNIT: Final = (
-    "Y",
-    "M",
-    "W",
-    "D",
-    "h",
-    "m",
-    "s",
-    "ms",
-    "us",
-    "μs",
-    "ns",
-    "ps",
-    "fs",
-    "as",
-    "generic",
-)
-"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings."""
-
 
 class NumpyTimedelta64Configuration(TypedDict):
     """
@@ -55,6 +39,23 @@ class NumpyTimedelta64Configuration(TypedDict):
     scale_factor: ReadOnly[int]
 
 
+def numpy_timedelta64_configuration(
+    value: Mapping[str, object],
+) -> NumpyTimedelta64Configuration:
+    """Validate `value` as a `numpy.timedelta64` configuration and normalize it.
+
+    The returned configuration spells the microsecond unit `"us"` even when the
+    input used the equivalent `"μs"`.
+
+    Raises TypeError if `unit` is not a string or `scale_factor` is not an
+    integer. Raises ValueError if `value` does not have exactly the keys `unit`
+    and `scale_factor`, if `unit` is not a `NumpyTimeUnit`, if `scale_factor` is
+    outside `[1, 2**31 - 1]`.
+    """
+    unit, scale_factor = numpy_time_configuration(value)
+    return {"unit": unit, "scale_factor": scale_factor}
+
+
 class NumpyTimedelta64(TypedDict):
     """`numpy.timedelta64` data type metadata."""
 
@@ -77,4 +78,6 @@ class NumpyTimedelta64(TypedDict):
     "NumpyTimedelta64Configuration",
     "NumpyTimedelta64DataTypeName",
     "NumpyTimedelta64FillValue",
+    "numpy_time_unit",
+    "numpy_timedelta64_configuration",
 ]

packages/zarr-metadata/tests/v3/data_type/numpy_datetime64/test_validators.py

@@ -0,0 +1,66 @@
+"""Cover the `numpy_datetime64_configuration` validator.
+
+The pydantic-driven fixture tests only check the structural shape of a
+configuration; the constraints that tie `unit` and `scale_factor` together
+live in the validator function and are covered directly here.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from zarr_metadata.v3.data_type.numpy_datetime64 import numpy_datetime64_configuration
+
+# (input, expected normalized output)
+VALID = [
+    ({"unit": "ns", "scale_factor": 1}, {"unit": "ns", "scale_factor": 1}),
+    ({"unit": "s", "scale_factor": 10}, {"unit": "s", "scale_factor": 10}),
+    ({"unit": "generic", "scale_factor": 1}, {"unit": "generic", "scale_factor": 1}),
+    ({"unit": "generic", "scale_factor": 2}, {"unit": "generic", "scale_factor": 2}),
+    (
+        {"unit": "generic", "scale_factor": 2**31 - 1},
+        {"unit": "generic", "scale_factor": 2**31 - 1},
+    ),
+    ({"unit": "us", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}),
+    ({"unit": "μs", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}),
+    ({"unit": "Y", "scale_factor": 2**31 - 1}, {"unit": "Y", "scale_factor": 2**31 - 1}),
+]
+
+
+@pytest.mark.parametrize(("value", "expected"), VALID, ids=lambda x: str(x))
+def test_valid(value: dict[str, object], expected: dict[str, object]) -> None:
+    assert numpy_datetime64_configuration(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value",
+    [{}, {"unit": "s"}, {"scale_factor": 1}, {"unit": "s", "scale_factor": 1, "extra": 0}],
+    ids=lambda x: str(x),
+)
+def test_wrong_keys(value: dict[str, object]) -> None:
+    with pytest.raises(ValueError, match="Expected exactly the keys"):
+        numpy_datetime64_configuration(value)
+
+
+@pytest.mark.parametrize("unit", [1, None], ids=str)
+def test_unit_not_a_string(unit: object) -> None:
+    with pytest.raises(TypeError, match="Expected 'unit' to be a string"):
+        numpy_datetime64_configuration({"unit": unit, "scale_factor": 1})
+
+
+@pytest.mark.parametrize("unit", ["", "invalid", "US", "seconds"], ids=str)
+def test_unknown_unit(unit: str) -> None:
+    with pytest.raises(ValueError, match="Expected one of"):
+        numpy_datetime64_configuration({"unit": unit, "scale_factor": 1})
+
+
+@pytest.mark.parametrize("scale_factor", [1.0, "1", True, None], ids=str)
+def test_scale_factor_not_an_integer(scale_factor: object) -> None:
+    with pytest.raises(TypeError, match="Expected 'scale_factor' to be an integer"):
+        numpy_datetime64_configuration({"unit": "s", "scale_factor": scale_factor})
+
+
+@pytest.mark.parametrize("scale_factor", [-1, 0, 2**31], ids=str)
+def test_scale_factor_out_of_range(scale_factor: int) -> None:
+    with pytest.raises(ValueError, match=r"Expected 'scale_factor' in \[1, 2147483647\]"):
+        numpy_datetime64_configuration({"unit": "s", "scale_factor": scale_factor})

packages/zarr-metadata/tests/v3/data_type/numpy_timedelta64/test_validators.py

@@ -0,0 +1,66 @@
+"""Cover the `numpy_timedelta64_configuration` validator.
+
+The pydantic-driven fixture tests only check the structural shape of a
+configuration; the constraints that tie `unit` and `scale_factor` together
+live in the validator function and are covered directly here.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from zarr_metadata.v3.data_type.numpy_timedelta64 import numpy_timedelta64_configuration
+
+# (input, expected normalized output)
+VALID = [
+    ({"unit": "ns", "scale_factor": 1}, {"unit": "ns", "scale_factor": 1}),
+    ({"unit": "s", "scale_factor": 10}, {"unit": "s", "scale_factor": 10}),
+    ({"unit": "generic", "scale_factor": 1}, {"unit": "generic", "scale_factor": 1}),
+    ({"unit": "generic", "scale_factor": 2}, {"unit": "generic", "scale_factor": 2}),
+    (
+        {"unit": "generic", "scale_factor": 2**31 - 1},
+        {"unit": "generic", "scale_factor": 2**31 - 1},
+    ),
+    ({"unit": "us", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}),
+    ({"unit": "μs", "scale_factor": 2}, {"unit": "us", "scale_factor": 2}),
+    ({"unit": "Y", "scale_factor": 2**31 - 1}, {"unit": "Y", "scale_factor": 2**31 - 1}),
+]
+
+
+@pytest.mark.parametrize(("value", "expected"), VALID, ids=lambda x: str(x))
+def test_valid(value: dict[str, object], expected: dict[str, object]) -> None:
+    assert numpy_timedelta64_configuration(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value",
+    [{}, {"unit": "s"}, {"scale_factor": 1}, {"unit": "s", "scale_factor": 1, "extra": 0}],
+    ids=lambda x: str(x),
+)
+def test_wrong_keys(value: dict[str, object]) -> None:
+    with pytest.raises(ValueError, match="Expected exactly the keys"):
+        numpy_timedelta64_configuration(value)
+
+
+@pytest.mark.parametrize("unit", [1, None], ids=str)
+def test_unit_not_a_string(unit: object) -> None:
+    with pytest.raises(TypeError, match="Expected 'unit' to be a string"):
+        numpy_timedelta64_configuration({"unit": unit, "scale_factor": 1})
+
+
+@pytest.mark.parametrize("unit", ["", "invalid", "US", "seconds"], ids=str)
+def test_unknown_unit(unit: str) -> None:
+    with pytest.raises(ValueError, match="Expected one of"):
+        numpy_timedelta64_configuration({"unit": unit, "scale_factor": 1})
+
+
+@pytest.mark.parametrize("scale_factor", [1.0, "1", True, None], ids=str)
+def test_scale_factor_not_an_integer(scale_factor: object) -> None:
+    with pytest.raises(TypeError, match="Expected 'scale_factor' to be an integer"):
+        numpy_timedelta64_configuration({"unit": "s", "scale_factor": scale_factor})
+
+
+@pytest.mark.parametrize("scale_factor", [-1, 0, 2**31], ids=str)
+def test_scale_factor_out_of_range(scale_factor: int) -> None:
+    with pytest.raises(ValueError, match=r"Expected 'scale_factor' in \[1, 2147483647\]"):
+        numpy_timedelta64_configuration({"unit": "s", "scale_factor": scale_factor})