chore: add test for ZarrParser on bucket without list permissions (#919)

Author: maxrjones

pyproject.toml

@@ -228,8 +228,8 @@ check-docs = { cmd = "mkdocs build --strict" }
 readthedocs = { cmd = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" }
 # Define commands to run within the docs environment
 [tool.pixi.feature.minio.tasks]
-run-tests = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py --run-minio-tests --run-network-tests --verbose" }
-run-tests-xml-cov = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py --run-minio-tests --run-network-tests --verbose --cov-report=xml" }
+run-tests = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py virtualizarr/tests/test_parsers/test_zarr.py --run-minio-tests --run-network-tests --verbose" }
+run-tests-xml-cov = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py virtualizarr/tests/test_parsers/test_zarr.py --run-minio-tests --run-network-tests --verbose --cov-report=xml" }
 
 [tool.setuptools_scm]
 fallback_version = "9999"

virtualizarr/tests/conftest.py

@@ -79,3 +79,46 @@ def minio_bucket(container):
         "file": filename,
         "client": client,
     }
+
+
+@pytest.fixture(scope="session")
+def minio_nolist_bucket(container):
+    """Create a MinIO bucket whose anonymous policy allows Get but NOT List."""
+    from minio import Minio
+
+    bucket = "nolist-bucket"
+    client = Minio(
+        "localhost:9000",
+        access_key=container["username"],
+        secret_key=container["password"],
+        secure=False,
+    )
+    client.make_bucket(bucket)
+    policy = {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Principal": {"AWS": "*"},
+                "Action": ["s3:GetBucketLocation"],
+                "Resource": f"arn:aws:s3:::{bucket}",
+            },
+            {
+                "Effect": "Allow",
+                "Principal": {"AWS": "*"},
+                "Action": [
+                    "s3:GetObject",
+                ],
+                "Resource": f"arn:aws:s3:::{bucket}/*",
+            },
+        ],
+    }
+    client.set_bucket_policy(bucket, json.dumps(policy))
+    yield {
+        "port": container["port"],
+        "endpoint": container["endpoint"],
+        "username": container["username"],
+        "password": container["password"],
+        "bucket": bucket,
+        "client": client,
+    }

virtualizarr/tests/test_parsers/test_zarr.py

@@ -20,7 +20,7 @@
     get_strategy,
     join_url,
 )
-from virtualizarr.tests import requires_pyarrow
+from virtualizarr.tests import requires_minio, requires_pyarrow
 
 pytestmark = requires_pyarrow
 
@@ -545,3 +545,52 @@ def test_sharded_array_raises_error(tmpdir):
         match="Zarr V3 arrays with sharding are not yet supported",
     ):
         parser(url=filepath, registry=registry)
+
+
+@requires_minio
+@pytest.mark.xfail(
+    reason="ZarrParser does not yet support buckets without list permissions"
+)
+def test_zarr_parser_nolist_bucket(minio_nolist_bucket):
+    """Test that ZarrParser works with a bucket that does not allow list operations."""
+    import obstore as obs
+
+    bucket = minio_nolist_bucket["bucket"]
+    endpoint = minio_nolist_bucket["endpoint"]
+    username = minio_nolist_bucket["username"]
+    password = minio_nolist_bucket["password"]
+
+    # Write a Zarr V3 store directly to the bucket using admin credentials
+    admin_store = obs.store.S3Store(
+        bucket,
+        endpoint_url=endpoint,
+        access_key_id=username,
+        secret_access_key=password,
+        virtual_hosted_style_request=False,
+        client_options={"allow_http": True},
+    )
+    zarr_store = zarr.storage.ObjectStore(store=admin_store)
+    ds = xr.Dataset(
+        {"data": (("x", "y"), np.arange(12, dtype="float32").reshape(3, 4))},
+        coords={"x": np.arange(3), "y": np.arange(4)},
+    )
+    ds.to_zarr(zarr_store, consolidated=False, zarr_format=3)
+
+    # Create an anonymous S3 store (subject to bucket policy which denies list)
+    anon_store = obs.store.S3Store(
+        bucket,
+        endpoint_url=endpoint,
+        skip_signature=True,
+        virtual_hosted_style_request=False,
+        client_options={"allow_http": True},
+    )
+
+    url = f"s3://{bucket}"
+    registry = ObjectStoreRegistry({url: anon_store})
+    parser = ZarrParser()
+    manifeststore = parser(url=url, registry=registry)
+
+    with xr.open_dataset(
+        manifeststore, engine="zarr", consolidated=False, zarr_format=3
+    ) as actual:
+        xr.testing.assert_identical(actual, ds)