refactor v3 data types

src/zarr/api/asynchronous.py

@@ -9,7 +9,13 @@
 import numpy.typing as npt
 from typing_extensions import deprecated
 
-from zarr.core.array import Array, AsyncArray, create_array, get_array_metadata
+from zarr.core.array import (
+    Array,
+    AsyncArray,
+    _get_default_chunk_encoding_v2,
+    create_array,
+    get_array_metadata,
+)
 from zarr.core.array_spec import ArrayConfig, ArrayConfigLike, ArrayConfigParams
 from zarr.core.buffer import NDArrayLike
 from zarr.core.common import (
@@ -21,7 +27,6 @@
     _default_zarr_format,
     _warn_order_kwarg,
     _warn_write_empty_chunks_kwarg,
-    parse_dtype,
 )
 from zarr.core.group import (
     AsyncGroup,
@@ -30,7 +35,7 @@
     create_hierarchy,
 )
 from zarr.core.metadata import ArrayMetadataDict, ArrayV2Metadata, ArrayV3Metadata
-from zarr.core.metadata.v2 import _default_compressor, _default_filters
+from zarr.core.metadata.dtype import get_data_type_from_numpy
 from zarr.errors import NodeTypeValidationError
 from zarr.storage._common import make_store_path
 
@@ -428,11 +433,12 @@ async def save_array(
     shape = arr.shape
     chunks = getattr(arr, "chunks", None)  # for array-likes with chunks attribute
     overwrite = kwargs.pop("overwrite", None) or _infer_overwrite(mode)
+    zarr_dtype = get_data_type_from_numpy(arr.dtype)
     new = await AsyncArray._create(
         store_path,
         zarr_format=zarr_format,
         shape=shape,
-        dtype=arr.dtype,
+        dtype=zarr_dtype,
         chunks=chunks,
         overwrite=overwrite,
         **kwargs,
@@ -978,15 +984,15 @@ async def create(
         _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format)
         or _default_zarr_format()
     )
-
+    dtype_wrapped = get_data_type_from_numpy(dtype)
     if zarr_format == 2:
         if chunks is None:
             chunks = shape
-        dtype = parse_dtype(dtype, zarr_format)
-        if not filters:
-            filters = _default_filters(dtype)
-        if not compressor:
-            compressor = _default_compressor(dtype)
+        default_filters, default_compressor = _get_default_chunk_encoding_v2(dtype_wrapped)
+        if filters is None:
+            filters = default_filters
+        if compressor is None:
+            compressor = default_compressor
     elif zarr_format == 3 and chunk_shape is None:  # type: ignore[redundant-expr]
         if chunks is not None:
             chunk_shape = chunks
@@ -1051,7 +1057,7 @@ async def create(
         store_path,
         shape=shape,
         chunks=chunks,
-        dtype=dtype,
+        dtype=dtype_wrapped,
         compressor=compressor,
         fill_value=fill_value,
         overwrite=overwrite,

src/zarr/codecs/_v2.py

@@ -48,15 +48,15 @@ async def _decode_single(
         # segfaults and other bad things happening
         if chunk_spec.dtype != object:
             try:
-                chunk = chunk.view(chunk_spec.dtype)
+                chunk = chunk.view(chunk_spec.dtype.unwrap())
             except TypeError:
                 # this will happen if the dtype of the chunk
                 # does not match the dtype of the array spec i.g. if
                 # the dtype of the chunk_spec is a string dtype, but the chunk
                 # is an object array. In this case, we need to convert the object
                 # array to the correct dtype.
 
-                chunk = np.array(chunk).astype(chunk_spec.dtype)
+                chunk = np.array(chunk).astype(chunk_spec.dtype.unwrap())
 
         elif chunk.dtype != object:
             # If we end up here, someone must have hacked around with the filters.
@@ -80,7 +80,7 @@ async def _encode_single(
         chunk = chunk_array.as_ndarray_like()
 
         # ensure contiguous and correct order
-        chunk = chunk.astype(chunk_spec.dtype, order=chunk_spec.order, copy=False)
+        chunk = chunk.astype(chunk_spec.dtype.unwrap(), order=chunk_spec.order, copy=False)
 
         # apply filters
         if self.filters:

src/zarr/codecs/blosc.py

@@ -139,11 +139,15 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
         dtype = array_spec.dtype
         new_codec = self
         if new_codec.typesize is None:
-            new_codec = replace(new_codec, typesize=dtype.itemsize)
+            new_codec = replace(new_codec, typesize=dtype.unwrap().itemsize)
         if new_codec.shuffle is None:
             new_codec = replace(
                 new_codec,
-                shuffle=(BloscShuffle.bitshuffle if dtype.itemsize == 1 else BloscShuffle.shuffle),
+                shuffle=(
+                    BloscShuffle.bitshuffle
+                    if dtype.unwrap().itemsize == 1
+                    else BloscShuffle.shuffle
+                ),
             )
 
         return new_codec

src/zarr/codecs/bytes.py

@@ -56,7 +56,7 @@ def to_dict(self) -> dict[str, JSON]:
             return {"name": "bytes", "configuration": {"endian": self.endian.value}}
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        if array_spec.dtype.itemsize == 0:
+        if array_spec.dtype.unwrap().itemsize == 1:
             if self.endian is not None:
                 return replace(self, endian=None)
         elif self.endian is None:
@@ -71,14 +71,8 @@ async def _decode_single(
         chunk_spec: ArraySpec,
     ) -> NDBuffer:
         assert isinstance(chunk_bytes, Buffer)
-        if chunk_spec.dtype.itemsize > 0:
-            if self.endian == Endian.little:
-                prefix = "<"
-            else:
-                prefix = ">"
-            dtype = np.dtype(f"{prefix}{chunk_spec.dtype.str[1:]}")
-        else:
-            dtype = np.dtype(f"|{chunk_spec.dtype.str[1:]}")
+
+        dtype = chunk_spec.dtype.with_endianness(self.endian).unwrap()
 
         as_array_like = chunk_bytes.as_array_like()
         if isinstance(as_array_like, NDArrayLike):

src/zarr/codecs/sharding.py

@@ -50,6 +50,7 @@
     get_indexer,
     morton_order_iter,
 )
+from zarr.core.metadata.dtype import DTypeWrapper
 from zarr.core.metadata.v3 import parse_codecs
 from zarr.registry import get_ndbuffer_class, get_pipeline_class, register_codec
 
@@ -355,9 +356,10 @@ def __init__(
         object.__setattr__(self, "index_location", index_location_parsed)
 
         # Use instance-local lru_cache to avoid memory leaks
-        object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
-        object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
-        object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
+        # TODO: fix these when we don't get hashability errors for certain numpy dtypes
+        # object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
+        # object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
+        # object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
 
     # todo: typedict return type
     def __getstate__(self) -> dict[str, Any]:
@@ -402,7 +404,9 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
             return replace(self, codecs=evolved_codecs)
         return self
 
-    def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None:
+    def validate(
+        self, *, shape: ChunkCoords, dtype: DTypeWrapper[Any, Any], chunk_grid: ChunkGrid
+    ) -> None:
         if len(self.chunk_shape) != len(shape):
             raise ValueError(
                 "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions."
@@ -483,7 +487,10 @@ async def _decode_partial_single(
 
         # setup output array
         out = shard_spec.prototype.nd_buffer.create(
-            shape=indexer.shape, dtype=shard_spec.dtype, order=shard_spec.order, fill_value=0
+            shape=indexer.shape,
+            dtype=shard_spec.dtype.unwrap(),
+            order=shard_spec.order,
+            fill_value=0,
         )
 
         indexed_chunks = list(indexer)

src/zarr/core/_info.py

@@ -7,7 +7,9 @@
 
 from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec
 from zarr.core.common import ZarrFormat
-from zarr.core.metadata.v3 import DataType
+from zarr.core.metadata.dtype import DTypeWrapper
+
+# from zarr.core.metadata.v3 import DataType
 
 
 @dataclasses.dataclass(kw_only=True)
@@ -78,7 +80,7 @@ class ArrayInfo:
 
     _type: Literal["Array"] = "Array"
     _zarr_format: ZarrFormat
-    _data_type: np.dtype[Any] | DataType
+    _data_type: np.dtype[Any] | DTypeWrapper
     _shape: tuple[int, ...]
     _shard_shape: tuple[int, ...] | None = None
     _chunk_shape: tuple[int, ...] | None = None