diff --git a/pyproject.toml b/pyproject.toml index 2c97b844..13ccce86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,8 +225,8 @@ check-docs = { cmd = "mkdocs build --strict" } readthedocs = { cmd = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" } # Define commands to run within the docs environment [tool.pixi.feature.minio.tasks] -run-tests = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py --run-minio-tests --run-network-tests --verbose" } -run-tests-xml-cov = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py --run-minio-tests --run-network-tests --verbose --cov-report=xml" } +run-tests = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py virtualizarr/tests/test_parsers/test_zarr.py --run-minio-tests --run-network-tests --verbose" } +run-tests-xml-cov = { cmd = "pytest virtualizarr/tests/test_manifests/test_store.py virtualizarr/tests/test_parsers/test_hdf/test_hdf_manifest_store.py virtualizarr/tests/test_parsers/test_zarr.py --run-minio-tests --run-network-tests --verbose --cov-report=xml" } [tool.setuptools_scm] fallback_version = "9999" diff --git a/virtualizarr/tests/conftest.py b/virtualizarr/tests/conftest.py index a73de6e3..55b08eee 100644 --- a/virtualizarr/tests/conftest.py +++ b/virtualizarr/tests/conftest.py @@ -79,3 +79,46 @@ def minio_bucket(container): "file": filename, "client": client, } + + +@pytest.fixture(scope="session") +def minio_nolist_bucket(container): + """Create a MinIO bucket whose anonymous policy allows Get but NOT List.""" + from minio import Minio + + bucket = "nolist-bucket" + client = Minio( + "localhost:9000", + access_key=container["username"], + secret_key=container["password"], + secure=False, + ) + client.make_bucket(bucket) + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["s3:GetBucketLocation"], + "Resource": f"arn:aws:s3:::{bucket}", + }, + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": [ + "s3:GetObject", + ], + "Resource": f"arn:aws:s3:::{bucket}/*", + }, + ], + } + client.set_bucket_policy(bucket, json.dumps(policy)) + yield { + "port": container["port"], + "endpoint": container["endpoint"], + "username": container["username"], + "password": container["password"], + "bucket": bucket, + "client": client, + } diff --git a/virtualizarr/tests/test_parsers/test_zarr.py b/virtualizarr/tests/test_parsers/test_zarr.py index fea74853..5d605477 100644 --- a/virtualizarr/tests/test_parsers/test_zarr.py +++ b/virtualizarr/tests/test_parsers/test_zarr.py @@ -20,7 +20,7 @@ get_strategy, join_url, ) -from virtualizarr.tests import requires_pyarrow +from virtualizarr.tests import requires_minio, requires_pyarrow pytestmark = requires_pyarrow @@ -545,3 +545,52 @@ def test_sharded_array_raises_error(tmpdir): match="Zarr V3 arrays with sharding are not yet supported", ): parser(url=filepath, registry=registry) + + +@requires_minio +@pytest.mark.xfail( + reason="ZarrParser does not yet support buckets without list permissions" +) +def test_zarr_parser_nolist_bucket(minio_nolist_bucket): + """Test that ZarrParser works with a bucket that does not allow list operations.""" + import obstore as obs + + bucket = minio_nolist_bucket["bucket"] + endpoint = minio_nolist_bucket["endpoint"] + username = minio_nolist_bucket["username"] + password = minio_nolist_bucket["password"] + + # Write a Zarr V3 store directly to the bucket using admin credentials + admin_store = obs.store.S3Store( + bucket, + endpoint_url=endpoint, + access_key_id=username, + secret_access_key=password, + virtual_hosted_style_request=False, + client_options={"allow_http": True}, + ) + zarr_store = zarr.storage.ObjectStore(store=admin_store) + ds = xr.Dataset( + {"data": (("x", "y"), np.arange(12, dtype="float32").reshape(3, 4))}, + coords={"x": np.arange(3), "y": np.arange(4)}, + ) + ds.to_zarr(zarr_store, consolidated=False, zarr_format=3) + + # Create an anonymous S3 store (subject to bucket policy which denies list) + anon_store = obs.store.S3Store( + bucket, + endpoint_url=endpoint, + skip_signature=True, + virtual_hosted_style_request=False, + client_options={"allow_http": True}, + ) + + url = f"s3://{bucket}" + registry = ObjectStoreRegistry({url: anon_store}) + parser = ZarrParser() + manifeststore = parser(url=url, registry=registry) + + with xr.open_dataset( + manifeststore, engine="zarr", consolidated=False, zarr_format=3 + ) as actual: + xr.testing.assert_identical(actual, ds)