Allow reading utf-8 encoded json files (#1312)

* read utf-8 in json

* update release

* Update zarr/util.py

Co-authored-by: jakirkham <[email protected]>

* allow str

---------

Co-authored-by: jakirkham <[email protected]>

Author: nhz2

docs/release.rst

@@ -31,6 +31,9 @@ Major changes
 Bug fixes
 ~~~~~~~~~
 
+* Allow reading utf-8 encoded json files
+  By :user:`Nathan Zimmerberg <nhz2>` :issue:`1308`.
+
 * Ensure contiguous data is give to ``FSStore``. Only copying if needed.
   By :user:`Mads R. B. Kristensen <madsbk>` :issue:`1285`.
 * NestedDirectoryStore.listdir now returns chunk keys with the correct '/' dimension_separator.

fixture/utf8attrs/.zattrs

@@ -0,0 +1 @@
+{"foo": "た"}

fixture/utf8attrs/.zgroup

@@ -0,0 +1,3 @@
+{
+    "zarr_format": 2
+}

zarr/meta.py

@@ -92,7 +92,7 @@ class Metadata2:
     ZARR_FORMAT = ZARR_FORMAT
 
     @classmethod
-    def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
+    def parse_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]:
 
         # Here we allow that a store may return an already-parsed metadata object,
         # or a string of JSON that we will parse here. We allow for an already-parsed
@@ -110,7 +110,7 @@ def parse_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
         return meta
 
     @classmethod
-    def decode_array_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
+    def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]:
         meta = cls.parse_metadata(s)
 
         # check metadata format
@@ -198,7 +198,7 @@ def decode_dtype(cls, d) -> np.dtype:
         return np.dtype(d)
 
     @classmethod
-    def decode_group_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
+    def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]:
         meta = cls.parse_metadata(s)
 
         # check metadata format version
@@ -351,7 +351,7 @@ def encode_dtype(cls, d):
             return get_extended_dtype_info(np.dtype(d))
 
     @classmethod
-    def decode_group_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
+    def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]:
         meta = cls.parse_metadata(s)
         # 1 / 0
         # # check metadata format version
@@ -390,7 +390,7 @@ def encode_hierarchy_metadata(cls, meta=None) -> bytes:
 
     @classmethod
     def decode_hierarchy_metadata(
-        cls, s: Union[MappingType, str]
+        cls, s: Union[MappingType, bytes, str]
     ) -> MappingType[str, Any]:
         meta = cls.parse_metadata(s)
         # check metadata format
@@ -495,7 +495,7 @@ def _decode_storage_transformer_metadata(cls, meta: Mapping) -> "StorageTransfor
         return StorageTransformerCls.from_config(transformer_type, conf)
 
     @classmethod
-    def decode_array_metadata(cls, s: Union[MappingType, str]) -> MappingType[str, Any]:
+    def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]:
         meta = cls.parse_metadata(s)
 
         # extract array metadata fields

zarr/tests/test_attrs.py

@@ -4,9 +4,10 @@
 
 from zarr._storage.store import meta_root
 from zarr.attrs import Attributes
-from zarr.storage import KVStore
+from zarr.storage import KVStore, DirectoryStore
 from zarr._storage.v3 import KVStoreV3
 from zarr.tests.util import CountingDict, CountingDictV3
+from zarr.hierarchy import group
 
 
 @pytest.fixture(params=[2, 3])
@@ -42,11 +43,17 @@ def test_storage(self, zarr_version):
         a['baz'] = 42
         assert attrs_key in store
         assert isinstance(store[attrs_key], bytes)
-        d = json.loads(str(store[attrs_key], 'ascii'))
+        d = json.loads(str(store[attrs_key], 'utf-8'))
         if zarr_version == 3:
             d = d['attributes']
         assert dict(foo='bar', baz=42) == d
 
+    def test_utf8_encoding(self, zarr_version):
+
+        # fixture data
+        fixture = group(store=DirectoryStore('fixture'))
+        assert fixture['utf8attrs'].attrs.asdict() == dict(foo='た')
+
     def test_get_set_del_contains(self, zarr_version):
 
         store = _init_store(zarr_version)

zarr/util.py

@@ -56,9 +56,9 @@ def json_dumps(o: Any) -> bytes:
                       separators=(',', ': '), cls=NumberEncoder).encode('ascii')
 
 
-def json_loads(s: str) -> Dict[str, Any]:
+def json_loads(s: Union[bytes, str]) -> Dict[str, Any]:
     """Read JSON in a consistent way."""
-    return json.loads(ensure_text(s, 'ascii'))
+    return json.loads(ensure_text(s, 'utf-8'))
 
 
 def normalize_shape(shape) -> Tuple[int]: