From 091ea1693de56ccabe706997fceeb861c2fc0053 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 8 May 2025 10:39:35 +0100 Subject: [PATCH] Fix typing errors in test_consolidated --- pyproject.toml | 76 ++++++++---------------- src/zarr/core/group.py | 2 +- tests/test_metadata/test_consolidated.py | 75 +++++++++++++---------- tests/test_metadata/test_v2.py | 7 ++- tests/test_metadata/test_v3.py | 14 ++--- 5 files changed, 84 insertions(+), 90 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9244a9ec0b..522cc33ed6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,20 +3,13 @@ requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" [tool.hatch.build.targets.sdist] -exclude = [ - "/.github", - "/bench", - "/docs", - "/notebooks" -] +exclude = ["/.github", "/bench", "/docs", "/notebooks"] [project] name = "zarr" description = "An implementation of chunked, compressed, N-dimensional arrays for Python" readme = { file = "README.md", content-type = "text/markdown" } -authors = [ - { name = "Alistair Miles", email = "alimanfoo@googlemail.com" }, -] +authors = [{ name = "Alistair Miles", email = "alimanfoo@googlemail.com" }] maintainers = [ { name = "Davis Bennett", email = "davis.v.bennett@gmail.com" }, { name = "jakirkham" }, @@ -28,7 +21,7 @@ maintainers = [ { name = "Ryan Abernathey" }, { name = "David Stansby" }, { name = "Tom Augspurger", email = "tom.w.augspurger@gmail.com" }, - { name = "Deepak Cherian" } + { name = "Deepak Cherian" }, ] requires-python = ">=3.11" # If you add a new dependency here, please also add it to .pre-commit-config.yml @@ -40,9 +33,7 @@ dependencies = [ 'donfig>=0.8', ] -dynamic = [ - "version", -] +dynamic = ["version"] classifiers = [ 'Development Status :: 6 - Mature', 'Intended Audience :: Developers', @@ -57,18 +48,13 @@ classifiers = [ 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', ] -license = {text = "MIT License"} +license = { text = "MIT License" } keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] # User extras -remote = [ - "fsspec>=2023.10.0", - "obstore>=0.5.1", -] -gpu = [ - "cupy-cuda12x", -] +remote = ["fsspec>=2023.10.0", "obstore>=0.5.1"] +gpu = ["cupy-cuda12x"] # Development extras test = [ "coverage", @@ -105,7 +91,7 @@ docs = [ 'numcodecs[msgpack]', 'rich', 's3fs>=2023.10.0', - 'astroid<4' + 'astroid<4', ] @@ -117,23 +103,18 @@ Documentation = "https://zarr.readthedocs.io/" Homepage = "https://github.com/zarr-developers/zarr-python" [dependency-groups] -dev = [ - "ipykernel>=6.29.5", - "pip>=25.0.1", -] +dev = ["ipykernel>=6.29.5", "pip>=25.0.1"] [tool.coverage.report] exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", "pragma: ${PY_MAJOR_VERSION} no cover", - '.*\.\.\.' # Ignore "..." lines + '.*\.\.\.', # Ignore "..." lines ] [tool.coverage.run] -omit = [ - "bench/compress_normal.py", -] +omit = ["bench/compress_normal.py"] [tool.hatch] version.source = "vcs" @@ -142,9 +123,7 @@ version.source = "vcs" hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.test] -dependencies = [ - "numpy~={matrix:numpy}", -] +dependencies = ["numpy~={matrix:numpy}"] features = ["test"] [[tool.hatch.envs.test.matrix]] @@ -154,7 +133,9 @@ deps = ["minimal", "optional"] [tool.hatch.envs.test.overrides] matrix.deps.dependencies = [ - {value = "zarr[remote, remote_tests, test, optional]", if = ["optional"]} + { value = "zarr[remote, remote_tests, test, optional]", if = [ + "optional", + ] }, ] [tool.hatch.envs.test.scripts] @@ -177,10 +158,7 @@ fix = "rm -r data/; pytest docs/user-guide --doctest-glob='*.rst' --accept" list-env = "pip list" [tool.hatch.envs.gputest] -dependencies = [ - "numpy~={matrix:numpy}", - "universal_pathlib", -] +dependencies = ["numpy~={matrix:numpy}", "universal_pathlib"] features = ["test", "gpu"] [[tool.hatch.envs.gputest.matrix]] @@ -207,7 +185,7 @@ serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0" python = "3.13" dependencies = [ 'packaging @ git+https://github.com/pypa/packaging', - 'numpy', # from scientific-python-nightly-wheels + 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', 'fsspec @ git+https://github.com/fsspec/filesystem_spec', 's3fs @ git+https://github.com/fsspec/s3fs', @@ -243,7 +221,7 @@ dependencies = [ 'zarr[remote]', 'packaging==22.*', 'numpy==1.25.*', - 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs + 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs 'fsspec==2023.10.0', 's3fs==2023.10.0', 'universal_pathlib==0.0.22', @@ -280,7 +258,7 @@ extend-exclude = [ "buck-out", "build", "dist", - "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6 + "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6 "venv", "docs", "src/zarr/v2/", @@ -357,7 +335,8 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] module = [ "tests.package_with_entrypoint.*", "tests.test_codecs.test_transpose", - "tests.test_config" + "tests.test_config", + "tests.test_metadata.*", ] strict = false @@ -365,9 +344,8 @@ strict = false # and fix the errors [[tool.mypy.overrides]] module = [ - "zarr.testing.stateful", # lots of hypothesis decorator errors + "zarr.testing.stateful", # lots of hypothesis decorator errors "tests.test_codecs.test_codecs", - "tests.test_metadata.*", "tests.test_store.*", "tests.test_group", "tests.test_indexing", @@ -388,9 +366,7 @@ doctest_optionflags = [ "ELLIPSIS", "IGNORE_EXCEPTION_DETAIL", ] -addopts = [ - "--durations=10", "-ra", "--strict-config", "--strict-markers", -] +addopts = ["--durations=10", "-ra", "--strict-config", "--strict-markers"] filterwarnings = [ "error", # TODO: explicitly filter or catch the warnings below where we expect them to be emitted in the tests @@ -403,7 +379,7 @@ filterwarnings = [ "ignore:Duplicate name.*:UserWarning", "ignore:The `compressor` argument is deprecated. Use `compressors` instead.:UserWarning", "ignore:Numcodecs codecs are not in the Zarr version 3 specification and may not be supported by other zarr implementations.:UserWarning", - "ignore:Unclosed client session dict[str, JSON]: } @classmethod - def from_dict(cls, data: dict[str, JSON]) -> ConsolidatedMetadata: + def from_dict(cls, data: Mapping[str, JSON]) -> ConsolidatedMetadata: data = dict(data) kind = data.get("kind") diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index a179982e94..e408fb4655 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -17,6 +17,7 @@ open, open_consolidated, ) +from zarr.api.synchronous import Group from zarr.core.buffer import cpu, default_buffer_prototype from zarr.core.group import ConsolidatedMetadata, GroupMetadata from zarr.core.metadata import ArrayV3Metadata @@ -25,11 +26,11 @@ if TYPE_CHECKING: from zarr.abc.store import Store - from zarr.core.common import ZarrFormat + from zarr.core.common import JSON, ZarrFormat @pytest.fixture -async def memory_store_with_hierarchy(memory_store: Store) -> None: +async def memory_store_with_hierarchy(memory_store: Store) -> Store: g = await group(store=memory_store, attributes={"foo": "bar"}) dtype = "uint8" await g.create_array(name="air", shape=(1, 2, 3), dtype=dtype) @@ -49,15 +50,15 @@ async def memory_store_with_hierarchy(memory_store: Store) -> None: class TestConsolidated: - async def test_open_consolidated_false_raises(self): + async def test_open_consolidated_false_raises(self) -> None: store = zarr.storage.MemoryStore() with pytest.raises(TypeError, match="use_consolidated"): - await zarr.api.asynchronous.open_consolidated(store, use_consolidated=False) + await zarr.api.asynchronous.open_consolidated(store, use_consolidated=False) # type: ignore[arg-type] - def test_open_consolidated_false_raises_sync(self): + def test_open_consolidated_false_raises_sync(self) -> None: store = zarr.storage.MemoryStore() with pytest.raises(TypeError, match="use_consolidated"): - zarr.open_consolidated(store, use_consolidated=False) + zarr.open_consolidated(store, use_consolidated=False) # type: ignore[arg-type] async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: # TODO: Figure out desired keys in @@ -69,7 +70,7 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: await consolidate_metadata(memory_store_with_hierarchy) group2 = await AsyncGroup.open(memory_store_with_hierarchy) - array_metadata = { + array_metadata: dict[str, JSON] = { "attributes": {}, "chunk_key_encoding": { "configuration": {"separator": "/"}, @@ -186,13 +187,11 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: group4 = await open_consolidated(store=memory_store_with_hierarchy) assert group4.metadata == expected - result_raw = json.loads( - ( - await memory_store_with_hierarchy.get( - "zarr.json", prototype=default_buffer_prototype() - ) - ).to_bytes() - )["consolidated_metadata"] + val = await memory_store_with_hierarchy.get( + "zarr.json", prototype=default_buffer_prototype() + ) + assert val is not None + result_raw = json.loads((val).to_bytes())["consolidated_metadata"] assert result_raw["kind"] == "inline" assert sorted(result_raw["metadata"]) == [ "air", @@ -206,7 +205,7 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: "time", ] - def test_consolidated_sync(self, memory_store): + def test_consolidated_sync(self, memory_store: zarr.storage.MemoryStore) -> None: g = zarr.api.synchronous.group(store=memory_store, attributes={"foo": "bar"}) dtype = "uint8" g.create_array(name="air", shape=(1, 2, 3), dtype=dtype) @@ -215,9 +214,9 @@ def test_consolidated_sync(self, memory_store): g.create_array(name="time", shape=(3,), dtype=dtype) zarr.api.synchronous.consolidate_metadata(memory_store) - group2 = zarr.api.synchronous.Group.open(memory_store) + group2 = Group.open(memory_store) - array_metadata = { + array_metadata: dict[str, JSON] = { "attributes": {}, "chunk_key_encoding": { "configuration": {"separator": "/"}, @@ -306,8 +305,8 @@ async def test_non_root_node(self, memory_store_with_hierarchy: Store) -> None: assert "air" not in child.metadata.consolidated_metadata.metadata assert "grandchild" in child.metadata.consolidated_metadata.metadata - def test_consolidated_metadata_from_dict(self): - data = {"must_understand": False} + def test_consolidated_metadata_from_dict(self) -> None: + data: dict[str, JSON] = {"must_understand": False} # missing kind with pytest.raises(ValueError, match="kind='None'"): @@ -329,8 +328,8 @@ def test_consolidated_metadata_from_dict(self): data["metadata"] = {} ConsolidatedMetadata.from_dict(data) - def test_flatten(self): - array_metadata = { + def test_flatten(self) -> None: + array_metadata: dict[str, JSON] = { "attributes": {}, "chunk_key_encoding": { "configuration": {"separator": "/"}, @@ -338,7 +337,7 @@ def test_flatten(self): }, "codecs": ({"configuration": {"endian": "little"}, "name": "bytes"},), "data_type": "float64", - "fill_value": np.float64(0.0), + "fill_value": 0, "node_type": "array", # "shape": (1, 2, 3), "zarr_format": 3, @@ -407,6 +406,17 @@ def test_flatten(self): }, ) result = metadata.flattened_metadata + assert isinstance(metadata.metadata["child"], GroupMetadata) + assert isinstance(metadata.metadata["child"].consolidated_metadata, ConsolidatedMetadata) + assert isinstance( + metadata.metadata["child"].consolidated_metadata.metadata["grandchild"], GroupMetadata + ) + assert isinstance( + metadata.metadata["child"] + .consolidated_metadata.metadata["grandchild"] + .consolidated_metadata, + ConsolidatedMetadata, + ) expected = { "air": metadata.metadata["air"], "lat": metadata.metadata["lat"], @@ -426,7 +436,7 @@ def test_flatten(self): } assert result == expected - def test_invalid_metadata_raises(self): + def test_invalid_metadata_raises(self) -> None: payload = { "kind": "inline", "must_understand": False, @@ -436,9 +446,9 @@ def test_invalid_metadata_raises(self): } with pytest.raises(TypeError, match="key='foo', type='list'"): - ConsolidatedMetadata.from_dict(payload) + ConsolidatedMetadata.from_dict(payload) # type: ignore[arg-type] - def test_to_dict_empty(self): + def test_to_dict_empty(self) -> None: meta = ConsolidatedMetadata( metadata={ "empty": GroupMetadata( @@ -467,7 +477,7 @@ def test_to_dict_empty(self): assert result == expected @pytest.mark.parametrize("zarr_format", [2, 3]) - async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat): + async def test_open_consolidated_raises_async(self, zarr_format: ZarrFormat) -> None: store = zarr.storage.MemoryStore() await AsyncGroup.from_store(store, zarr_format=zarr_format) with pytest.raises(ValueError): @@ -485,12 +495,15 @@ async def v2_consolidated_metadata_empty_dataset( b'{"metadata":{".zgroup":{"zarr_format":2}},"zarr_consolidated_format":1}' ) return AsyncGroup._from_bytes_v2( - None, zgroup_bytes, zattrs_bytes=None, consolidated_metadata_bytes=zmetadata_bytes + None, # type: ignore[arg-type] + zgroup_bytes, + zattrs_bytes=None, + consolidated_metadata_bytes=zmetadata_bytes, ) async def test_consolidated_metadata_backwards_compatibility( - self, v2_consolidated_metadata_empty_dataset - ): + self, v2_consolidated_metadata_empty_dataset: AsyncGroup + ) -> None: """ Test that consolidated metadata handles a missing .zattrs key. This is necessary for backwards compatibility with zarr-python 2.x. See https://github.com/zarr-developers/zarr-python/issues/2694 """ @@ -500,7 +513,7 @@ async def test_consolidated_metadata_backwards_compatibility( result = await zarr.api.asynchronous.open_consolidated(store, zarr_format=2) assert result.metadata == v2_consolidated_metadata_empty_dataset.metadata - async def test_consolidated_metadata_v2(self): + async def test_consolidated_metadata_v2(self) -> None: store = zarr.storage.MemoryStore() g = await AsyncGroup.from_store(store, attributes={"key": "root"}, zarr_format=2) dtype = "uint8" @@ -578,7 +591,7 @@ async def test_use_consolidated_false( @pytest.mark.parametrize("fill_value", [np.nan, np.inf, -np.inf]) async def test_consolidated_metadata_encodes_special_chars( memory_store: Store, zarr_format: ZarrFormat, fill_value: float -): +) -> None: root = await group(store=memory_store, zarr_format=zarr_format) _child = await root.create_group("child", attributes={"test": fill_value}) _time = await root.create_array("time", shape=(12,), dtype=np.float64, fill_value=fill_value) diff --git a/tests/test_metadata/test_v2.py b/tests/test_metadata/test_v2.py index 08b9cb2507..01d4519087 100644 --- a/tests/test_metadata/test_v2.py +++ b/tests/test_metadata/test_v2.py @@ -8,6 +8,7 @@ import zarr.api.asynchronous import zarr.storage +from zarr.core.array import AsyncArray from zarr.core.buffer import cpu from zarr.core.buffer.core import default_buffer_prototype from zarr.core.group import ConsolidatedMetadata, GroupMetadata @@ -18,6 +19,8 @@ from typing import Any from zarr.abc.codec import Codec + from zarr.core.common import JSON + import numcodecs @@ -104,7 +107,7 @@ class TestConsolidated: async def v2_consolidated_metadata( self, memory_store: zarr.storage.MemoryStore ) -> zarr.storage.MemoryStore: - zmetadata = { + zmetadata: dict[str, JSON] = { "metadata": { ".zattrs": { "Conventions": "COARDS", @@ -274,6 +277,7 @@ async def test_getitem_consolidated(self, v2_consolidated_metadata): store = v2_consolidated_metadata group = await zarr.api.asynchronous.open_consolidated(store=store, zarr_format=2) air = await group.getitem("air") + assert isinstance(air, AsyncArray[ArrayV2Metadata]) assert air.metadata.shape == (730,) @@ -335,6 +339,7 @@ def test_structured_dtype_fill_value_serialization(tmp_path, fill_value): zarr.consolidate_metadata(root_group.store, zarr_format=2) root_group = zarr.open_group(group_path, mode="r") + assert isinstance(root_group.metadata.consolidated_metadata, ConsolidatedMetadata) assert ( root_group.metadata.consolidated_metadata.to_dict()["metadata"]["structured_dtype"][ "fill_value" diff --git a/tests/test_metadata/test_v3.py b/tests/test_metadata/test_v3.py index a47cbf43bb..244a429b63 100644 --- a/tests/test_metadata/test_v3.py +++ b/tests/test_metadata/test_v3.py @@ -140,7 +140,7 @@ def test_default_fill_value(dtype_str: str) -> None: (0j, "complex64"), ], ) -def test_parse_fill_value_valid(fill_value: Any, dtype_str: str) -> None: +def test_parse_fill_value_valid(fill_value: bool | float, dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) casts fill_value to the given dtype. """ @@ -154,18 +154,18 @@ def test_parse_fill_value_valid(fill_value: Any, dtype_str: str) -> None: @pytest.mark.parametrize("fill_value", ["not a valid value"]) @pytest.mark.parametrize("dtype_str", [*int_dtypes, *float_dtypes, *complex_dtypes]) -def test_parse_fill_value_invalid_value(fill_value: Any, dtype_str: str) -> None: +def test_parse_fill_value_invalid_value(fill_value: str, dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) raises ValueError for invalid values. This test excludes bool because the bool constructor takes anything. """ with pytest.raises(ValueError): - parse_fill_value(fill_value, dtype_str) + parse_fill_value(fill_value, dtype_str) # type: ignore[arg-type] @pytest.mark.parametrize("fill_value", [[1.0, 0.0], [0, 1], complex(1, 1), np.complex64(0)]) @pytest.mark.parametrize("dtype_str", [*complex_dtypes]) -def test_parse_fill_value_complex(fill_value: Any, dtype_str: str) -> None: +def test_parse_fill_value_complex(fill_value: list[int] | complex, dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) correctly handles complex values represented as length-2 sequences @@ -191,18 +191,18 @@ def test_parse_fill_value_complex_invalid(fill_value: Any, dtype_str: str) -> No f"length {len(fill_value)}." ) with pytest.raises(ValueError, match=re.escape(match)): - parse_fill_value(fill_value=fill_value, dtype=dtype_str) + parse_fill_value(fill_value=fill_value, dtype=dtype_str) # type; ignore[arg-type] @pytest.mark.parametrize("fill_value", [{"foo": 10}]) @pytest.mark.parametrize("dtype_str", [*int_dtypes, *float_dtypes, *complex_dtypes]) -def test_parse_fill_value_invalid_type(fill_value: Any, dtype_str: str) -> None: +def test_parse_fill_value_invalid_type(fill_value: dict[str, int], dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) raises TypeError for invalid non-sequential types. This test excludes bool because the bool constructor takes anything. """ with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): - parse_fill_value(fill_value, dtype_str) + parse_fill_value(fill_value, dtype_str) # type: ignore[arg-type] @pytest.mark.parametrize(