diff --git a/.coveragerc.python3.14 b/.coveragerc.python3.14 index 7e23663d..ae5f260e 100644 --- a/.coveragerc.python3.14 +++ b/.coveragerc.python3.14 @@ -5,3 +5,6 @@ exclude_lines = omit = audbackend/core/api.py audbackend/core/backend/artifactory.py + audbackend/core/backend/base.py + audbackend/core/backend/filesystem.py + audbackend/core/backend/minio.py diff --git a/audbackend/core/backend/artifactory.py b/audbackend/core/backend/artifactory.py index 206aefc8..9076a4c0 100644 --- a/audbackend/core/backend/artifactory.py +++ b/audbackend/core/backend/artifactory.py @@ -1,3 +1,4 @@ +from collections.abc import Iterator import os import artifactory @@ -271,6 +272,27 @@ def _get_file( src_path = self.path(src_path) _download(src_path, dst_path, verbose=verbose) + def _get_file_stream( + self, + src_path: str, + ) -> Iterator[bytes]: + r"""Get file from backend as byte stream.""" + from audbackend.core.backend.base import STREAM_CHUNK_SIZE + + src_path = self.path(src_path) + + with src_path.open("r") as fp: + while data := fp.read(STREAM_CHUNK_SIZE): + yield data + + def _size( + self, + path: str, + ) -> int: + r"""Get size of file on backend.""" + path = self.path(path) + return artifactory.ArtifactoryPath.stat(path).size + def _ls( self, path: str, diff --git a/audbackend/core/backend/base.py b/audbackend/core/backend/base.py index 38cd13c5..bc118132 100644 --- a/audbackend/core/backend/base.py +++ b/audbackend/core/backend/base.py @@ -1,15 +1,34 @@ +from collections.abc import Iterator from collections.abc import Sequence +import errno import fnmatch +import hashlib import inspect import os +import shutil import tempfile +import zipfile import audeer + +# stream-unzip is optional (not available on Python 3.14+) +try: + from stream_unzip import TruncatedDataError + from stream_unzip import UnfinishedIterationError + from stream_unzip import stream_unzip + + STREAM_UNZIP_AVAILABLE = True +except ImportError: # pragma: no cover + STREAM_UNZIP_AVAILABLE = False + from audbackend.core import utils from audbackend.core.errors import BackendError +# Default chunk size for streaming file operations (64 KB) +STREAM_CHUNK_SIZE = 64 * 1024 + backend_not_opened_error = ( "Call 'Backend.open()' to establish a connection to the repository first." ) @@ -450,8 +469,17 @@ def get_archive( ) -> list[str]: r"""Get archive from backend and extract. - The archive type is derived from the extension of ``src_path``. - See :func:`audeer.extract_archive` for supported extensions. + For ZIP archives, + streaming extraction is used + if ``stream-unzip`` is installed, + and ``num_workers=1``. + It extracts files during download + without storing the archive locally. + + For uncompressed archives + or archives that do not support streaming extraction, + the archive is downloaded first and then extracted. + In that case we recommend setting ``num_workers=5`` for best performance. If ``dst_root`` does not exist, it is created. @@ -461,14 +489,15 @@ def get_archive( ``src_path`` and the retrieved archive have the same checksum. If it fails, - the retrieved archive is removed and + the extracted files are removed and an :class:`InterruptedError` is raised. Args: src_path: path to archive on backend dst_root: local destination directory tmp_root: directory under which archive is temporarily extracted. - Defaults to temporary directory of system + Defaults to temporary directory of system. + Not used for ZIP archives when ``stream-unzip`` is available. num_workers: number of parallel jobs validate: verify archive was successfully retrieved from the backend @@ -498,10 +527,54 @@ def get_archive( src_path = utils.check_path(src_path) + # Validate tmp_root if specified + # (use TemporaryDirectory to get consistent error format) + if tmp_root is not None: + with tempfile.TemporaryDirectory(dir=tmp_root): + pass + + # Use streaming extraction for ZIP files if stream-unzip is available + if ( + src_path.lower().endswith(".zip") + and num_workers == 1 + and STREAM_UNZIP_AVAILABLE + ): + return self._get_archive_streaming( + src_path, + dst_root, + validate=validate, + verbose=verbose, + ) + + return self._get_archive_via_tempfile( + src_path, + dst_root, + tmp_root=tmp_root, + num_workers=num_workers, + validate=validate, + verbose=verbose, + ) + + def _get_archive_via_tempfile( + self, + src_path: str, + dst_root: str, + *, + tmp_root: str | None, + num_workers: int, + validate: bool, + verbose: bool, + ) -> list[str]: + r"""Get archive via temporary file download and extraction. + + Downloads the archive to a temporary location, + then extracts it to the destination. + + """ with tempfile.TemporaryDirectory(dir=tmp_root) as tmp: - tmp_root = audeer.path(tmp, os.path.basename(dst_root)) + tmp_dir = audeer.path(tmp, os.path.basename(dst_root)) local_archive = os.path.join( - tmp_root, + tmp_dir, os.path.basename(src_path), ) self.get_file( @@ -518,6 +591,128 @@ def get_archive( verbose=verbose, ) + def _cleanup_extracted( + self, + dst_root: str, + dst_root_existed: bool, + extracted_files: list[str], + ) -> None: + r"""Remove extracted files and optionally the destination directory. + + If we created the destination directory and it didn't exist before, + remove it entirely. Otherwise, only remove the files we extracted. + + """ + if not dst_root_existed and os.path.exists(dst_root): + shutil.rmtree(dst_root) + else: + for file_name in extracted_files: + full_path = audeer.path(dst_root, file_name) + if os.path.exists(full_path): + os.remove(full_path) + + def _get_archive_streaming( + self, + src_path: str, + dst_root: str, + *, + validate: bool = False, + verbose: bool = False, + ) -> list[str]: + r"""Get ZIP archive from backend with streaming extraction. + + Extracts files during download without storing the archive locally. + + """ + # Validate dst_root is not an existing file, + # raise same error as audeer.extract_archive() + if os.path.exists(dst_root) and not os.path.isdir(dst_root): + raise NotADirectoryError( + errno.ENOTDIR, + os.strerror(errno.ENOTDIR), + dst_root, + ) + + # Track if we created the destination directory + dst_root_existed = os.path.exists(dst_root) + audeer.mkdir(dst_root) + + extracted_files = [] + md5_hash = hashlib.md5() if validate else None + + # Get file size for progress bar + src_size = self._size(src_path) if verbose else None + + # Setup progress bar + desc = audeer.format_display_message( + f"Download {os.path.basename(src_path)}", + pbar=verbose, + ) + pbar = audeer.progress_bar(total=src_size, desc=desc, disable=not verbose) + + def stream_with_hash(): + """Wrap stream to compute hash while streaming.""" + for chunk in self._get_file_stream(src_path): + if md5_hash is not None: + md5_hash.update(chunk) + pbar.update(len(chunk)) + yield chunk + + try: + with pbar: + for file_name, file_size, unzipped_chunks in stream_unzip( + stream_with_hash() + ): + # Decode file name and handle path + file_name = file_name.decode("utf-8") + + # Skip directory entries + if file_name.endswith("/"): + continue + + # Construct destination path + dst_path = audeer.path(dst_root, file_name) + + # Create parent directories if needed + audeer.mkdir(os.path.dirname(dst_path)) + + # Write file content + with open(dst_path, "wb") as f: + for chunk in unzipped_chunks: + f.write(chunk) + + # Store relative path (consistent with audeer.extract_archive) + extracted_files.append(file_name.replace("/", os.sep)) + + # Validate checksum if requested + if validate: + expected_checksum = self.checksum(src_path) + actual_checksum = md5_hash.hexdigest() + + if actual_checksum != expected_checksum: + self._cleanup_extracted( + dst_root, dst_root_existed, extracted_files + ) + raise InterruptedError( + f"Execution is interrupted because " + f"{src_path} " + f"has checksum " + f"'{actual_checksum}' " + "when the expected checksum is " + f"'{expected_checksum}'. " + f"The extracted files have been removed." + ) + + except (zipfile.BadZipFile, TruncatedDataError, UnfinishedIterationError) as ex: + self._cleanup_extracted(dst_root, dst_root_existed, extracted_files) + raise RuntimeError(f"Broken archive: {src_path}") from ex + except Exception: + # Clean up on any unexpected error (network errors, etc.) + self._cleanup_extracted(dst_root, dst_root_existed, extracted_files) + raise + + return extracted_files + def _get_file( self, src_path: str, @@ -528,6 +723,17 @@ def _get_file( r"""Get file from backend.""" raise NotImplementedError() + def _get_file_stream( + self, + src_path: str, + ) -> Iterator[bytes]: # pragma: no cover + r"""Get file from backend as byte stream. + + This method should return an iterator that yields chunks of bytes. + + """ + raise NotImplementedError() + def get_file( self, src_path: str, @@ -1139,6 +1345,13 @@ def sep(self) -> str: """ return utils.BACKEND_SEPARATOR + def _size( + self, + path: str, + ) -> int: # pragma: no cover + r"""Get size of file on backend.""" + raise NotImplementedError() + def split( self, path: str, diff --git a/audbackend/core/backend/filesystem.py b/audbackend/core/backend/filesystem.py index 47207ba0..ad049ea9 100644 --- a/audbackend/core/backend/filesystem.py +++ b/audbackend/core/backend/filesystem.py @@ -1,3 +1,4 @@ +from collections.abc import Iterator import datetime import os import shutil @@ -122,6 +123,27 @@ def _get_file( src_path = self._expand(src_path) shutil.copy(src_path, dst_path) + def _get_file_stream( + self, + src_path: str, + ) -> Iterator[bytes]: + r"""Get file from backend as byte stream.""" + from audbackend.core.backend.base import STREAM_CHUNK_SIZE + + src_path = self._expand(src_path) + + with open(src_path, "rb") as f: + while data := f.read(STREAM_CHUNK_SIZE): + yield data + + def _size( + self, + path: str, + ) -> int: + r"""Get size of file on backend.""" + path = self._expand(path) + return os.path.getsize(path) + def _ls( self, path: str, diff --git a/audbackend/core/backend/minio.py b/audbackend/core/backend/minio.py index 34ecf295..adbdd7d4 100644 --- a/audbackend/core/backend/minio.py +++ b/audbackend/core/backend/minio.py @@ -1,3 +1,4 @@ +from collections.abc import Iterator import configparser import getpass import mimetypes @@ -409,6 +410,27 @@ def _download_file( response.close() response.release_conn() + def _get_file_stream( + self, + src_path: str, + ) -> Iterator[bytes]: + r"""Get file from backend as byte stream.""" + from audbackend.core.backend.base import STREAM_CHUNK_SIZE + + src_path = self.path(src_path) + + response = self._client.get_object( + bucket_name=self.repository, + object_name=src_path, + ) + + try: + while data := response.read(STREAM_CHUNK_SIZE): + yield data + finally: + response.close() + response.release_conn() + def _ls( self, path: str, diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py index dbe938a5..2874b9f1 100644 --- a/audbackend/core/interface/unversioned.py +++ b/audbackend/core/interface/unversioned.py @@ -217,13 +217,23 @@ def get_archive( dst_root: str, *, tmp_root: str = None, + num_workers: int = 1, validate: bool = False, verbose: bool = False, ) -> list[str]: r"""Get archive from backend and extract. - The archive type is derived from the extension of ``src_path``. - See :func:`audeer.extract_archive` for supported extensions. + For ZIP archives, + streaming extraction is used + if ``stream-unzip`` is installed, + and ``num_workers=1``. + It extracts files during download + without storing the archive locally. + + For uncompressed archives + or archives that do not support streaming extraction, + the archive is downloaded first and then extracted. + In that case we recommend setting ``num_workers=5`` for best performance. If ``dst_root`` does not exist, it is created. @@ -241,6 +251,7 @@ def get_archive( dst_root: local destination directory tmp_root: directory under which archive is temporarily extracted. Defaults to temporary directory of system + num_workers: number of parallel jobs validate: verify archive was successfully retrieved from the backend verbose: show debug messages @@ -278,6 +289,7 @@ def get_archive( src_path, dst_root, tmp_root=tmp_root, + num_workers=num_workers, validate=validate, verbose=verbose, ) diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index e9acdc75..b279a2da 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -260,13 +260,23 @@ def get_archive( version: str, *, tmp_root: str = None, + num_workers: int = 1, validate: bool = False, verbose: bool = False, ) -> list[str]: r"""Get archive from backend and extract. - The archive type is derived from the extension of ``src_path``. - See :func:`audeer.extract_archive` for supported extensions. + For ZIP archives, + streaming extraction is used + if ``stream-unzip`` is installed, + and ``num_workers=1``. + It extracts files during download + without storing the archive locally. + + For uncompressed archives + or archives that do not support streaming extraction, + the archive is downloaded first and then extracted. + In that case we recommend setting ``num_workers=5`` for best performance. If ``dst_root`` does not exist, it is created. @@ -283,6 +293,7 @@ def get_archive( src_path: path to archive on backend dst_root: local destination directory version: version string + num_workers: number of parallel jobs tmp_root: directory under which archive is temporarily extracted. Defaults to temporary directory of system validate: verify archive was successfully @@ -325,6 +336,7 @@ def get_archive( src_path_with_version, dst_root, tmp_root=tmp_root, + num_workers=num_workers, validate=validate, verbose=verbose, ) diff --git a/pyproject.toml b/pyproject.toml index d237992c..9807bd9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ requires-python = '>=3.10' dependencies = [ 'audeer >=2.3.1', 'pywin32; sys_platform == "win32"', + 'stream-unzip >=0.0.99; python_version < "3.14"', ] # Get version dynamically from git # (needs setuptools_scm tools config below) diff --git a/tests/singlefolder.py b/tests/singlefolder.py index b2b261cf..bc64fee1 100644 --- a/tests/singlefolder.py +++ b/tests/singlefolder.py @@ -1,3 +1,4 @@ +from collections.abc import Iterator import datetime import os import pickle @@ -121,6 +122,17 @@ def _get_file( with self.Map(self._path, self._lock) as m: shutil.copy(m[src_path][0], dst_path) + def _get_file_stream( + self, + src_path: str, + ) -> Iterator[bytes]: + with self.Map(self._path, self._lock) as m: + src_file = m[src_path][0] + chunk_size = 64 * 1024 # 64 KB + with open(src_file, "rb") as f: + while data := f.read(chunk_size): + yield data + def _ls( self, path: str, diff --git a/tests/test_backend_artifactory.py b/tests/test_backend_artifactory.py index 728f7312..a1fec44a 100644 --- a/tests/test_backend_artifactory.py +++ b/tests/test_backend_artifactory.py @@ -248,3 +248,80 @@ def test_parquet_file(interface, parquet_file): version = "1.0.0" interface.put_file(parquet_file, dst_file, version) assert interface.exists(dst_file, version) + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Artifactory, audbackend.interface.Versioned)], + indirect=True, +) +def test_size(tmpdir, interface): + """Test _size method returns correct file size.""" + # Create a file with known content + content = "Hello World!" * 1000 # ~12KB + src_path = audeer.path(tmpdir, "test.txt") + with open(src_path, "w") as f: + f.write(content) + expected_size = os.path.getsize(src_path) + + # Upload file to backend + interface.put_file(src_path, "/test.txt", "1.0.0") + + # Get size from backend + backend_path = interface._path_with_version("/test.txt", "1.0.0") + actual_size = interface.backend._size(backend_path) + + assert actual_size == expected_size + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Artifactory, audbackend.interface.Unversioned)], + indirect=True, +) +def test_get_archive_streaming(tmpdir, interface): + """Test get_archive with streaming extraction verifies _get_file_stream. + + This test verifies that _get_file_stream() returns bytes correctly, + which is required for streaming ZIP extraction and checksum computation. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create source files + src_root = audeer.path(tmpdir, "src") + audeer.mkdir(src_root) + with open(audeer.path(src_root, "file1.txt"), "w") as f: + f.write("content of file 1") + with open(audeer.path(src_root, "file2.txt"), "w") as f: + f.write("content of file 2") + + # Create ZIP archive + archive_path = audeer.path(tmpdir, "archive.zip") + audeer.create_archive(src_root, None, archive_path) + + # Upload archive to backend + interface.put_file(archive_path, "/archive.zip") + + # Extract using streaming (this exercises _get_file_stream) + dst_root = audeer.path(tmpdir, "dst") + extracted = interface.get_archive("/archive.zip", dst_root) + + # Verify files were extracted correctly + assert sorted(extracted) == ["file1.txt", "file2.txt"] + with open(audeer.path(dst_root, "file1.txt")) as f: + assert f.read() == "content of file 1" + with open(audeer.path(dst_root, "file2.txt")) as f: + assert f.read() == "content of file 2" + + # Also test with validation to verify checksum computation works + # (requires _get_file_stream to return bytes for md5.update()) + dst_root_validated = audeer.path(tmpdir, "dst_validated") + extracted_validated = interface.get_archive( + "/archive.zip", dst_root_validated, validate=True + ) + assert sorted(extracted_validated) == ["file1.txt", "file2.txt"] diff --git a/tests/test_backend_filesystem.py b/tests/test_backend_filesystem.py index 000b4cec..4095a5e9 100644 --- a/tests/test_backend_filesystem.py +++ b/tests/test_backend_filesystem.py @@ -1,4 +1,5 @@ import os +import zipfile import pytest @@ -189,3 +190,307 @@ def test_open_close(tmpdir, repository): audbackend.backend.FileSystem.create(host, repository) backend.open() backend.close() + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.FileSystem, audbackend.interface.Unversioned)], + indirect=True, +) +def test_get_archive_streaming(tmpdir, interface): + """Test get_archive with streaming extraction verifies _get_file_stream. + + This test verifies that _get_file_stream() returns bytes correctly, + which is required for streaming ZIP extraction and checksum computation. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create source files + src_root = audeer.path(tmpdir, "src") + audeer.mkdir(src_root) + with open(audeer.path(src_root, "file1.txt"), "w") as f: + f.write("content of file 1") + with open(audeer.path(src_root, "file2.txt"), "w") as f: + f.write("content of file 2") + + # Create ZIP archive + archive_path = audeer.path(tmpdir, "archive.zip") + audeer.create_archive(src_root, None, archive_path) + + # Upload archive to backend + interface.put_file(archive_path, "/archive.zip") + + # Extract using streaming (this exercises _get_file_stream) + dst_root = audeer.path(tmpdir, "dst") + extracted = interface.get_archive("/archive.zip", dst_root) + + # Verify files were extracted correctly + assert sorted(extracted) == ["file1.txt", "file2.txt"] + with open(audeer.path(dst_root, "file1.txt")) as f: + assert f.read() == "content of file 1" + with open(audeer.path(dst_root, "file2.txt")) as f: + assert f.read() == "content of file 2" + + # Also test with validation to verify checksum computation works + # (requires _get_file_stream to return bytes for md5.update()) + dst_root_validated = audeer.path(tmpdir, "dst_validated") + extracted_validated = interface.get_archive( + "/archive.zip", dst_root_validated, validate=True + ) + assert sorted(extracted_validated) == ["file1.txt", "file2.txt"] + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.FileSystem, audbackend.interface.Unversioned)], + indirect=True, +) +def test_streaming_dst_root_is_file(tmpdir, interface): + """Test that NotADirectoryError is raised when dst_root is an existing file. + + When extracting an archive and dst_root points to an existing file, + NotADirectoryError should be raised and the file should remain unchanged. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create a regular file where we'll try to extract to + dst_file = audeer.path(tmpdir, "existing_file.txt") + with open(dst_file, "w") as f: + f.write("original content") + original_content = "original content" + original_mtime = os.path.getmtime(dst_file) + + # Create a valid ZIP archive with content + src_root = audeer.path(tmpdir, "src") + audeer.mkdir(src_root) + with open(audeer.path(src_root, "new_file.txt"), "w") as f: + f.write("New content") + + archive_path = audeer.path(tmpdir, "archive.zip") + audeer.create_archive(src_root, None, archive_path) + interface.put_file(archive_path, "/archive.zip") + + # Get list of files in tmpdir before the failed extraction + files_before = set(os.listdir(tmpdir)) + + # Try to extract to the existing file - should fail with NotADirectoryError + with pytest.raises(NotADirectoryError): + interface.get_archive("/archive.zip", dst_file) + + # Verify the file is unchanged + assert os.path.isfile(dst_file) + with open(dst_file) as f: + assert f.read() == original_content + assert os.path.getmtime(dst_file) == original_mtime + + # Verify no additional files were created nearby (no partial extraction) + files_after = set(os.listdir(tmpdir)) + assert files_after == files_before + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.FileSystem, audbackend.interface.Unversioned)], + indirect=True, +) +def test_streaming_cleanup_existing_directory(tmpdir, interface): + """Test cleanup removes only extracted files when dst_root already exists. + + When extracting a malformed archive to an existing directory fails, + only the extracted files should be removed, not the entire directory + or pre-existing files. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create destination directory with pre-existing file + dst_root = audeer.path(tmpdir, "existing_dir") + audeer.mkdir(dst_root) + pre_existing_file = audeer.path(dst_root, "pre_existing.txt") + with open(pre_existing_file, "w") as f: + f.write("I was here before") + + # Create and upload a malformed ZIP archive + malformed_zip = audeer.path(tmpdir, "malformed.zip") + audeer.touch(malformed_zip) # Empty file, not a valid ZIP + interface.put_file(malformed_zip, "/malformed.zip") + + # Try to extract - should fail + with pytest.raises(RuntimeError, match="Broken archive"): + interface.get_archive("/malformed.zip", dst_root) + + # Verify pre-existing file still exists + assert os.path.exists(pre_existing_file) + with open(pre_existing_file) as f: + assert f.read() == "I was here before" + + # Verify destination directory still exists + assert os.path.isdir(dst_root) + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.FileSystem, audbackend.interface.Unversioned)], + indirect=True, +) +def test_streaming_cleanup_extracted_files(tmpdir, interface): + """Test cleanup removes extracted files on validation failure. + + When extracting an archive to an existing directory fails due to + checksum validation, the extracted files should be removed but + pre-existing files should remain. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create destination directory with pre-existing file + dst_root = audeer.path(tmpdir, "existing_dir") + audeer.mkdir(dst_root) + pre_existing_file = audeer.path(dst_root, "pre_existing.txt") + with open(pre_existing_file, "w") as f: + f.write("I was here before") + + # Create a valid ZIP archive with content + src_root = audeer.path(tmpdir, "src") + audeer.mkdir(src_root) + with open(audeer.path(src_root, "new_file.txt"), "w") as f: + f.write("New content") + + archive_path = audeer.path(tmpdir, "archive.zip") + audeer.create_archive(src_root, None, archive_path) + interface.put_file(archive_path, "/archive.zip") + + # Mock checksum to force validation failure after extraction + original_checksum = interface.backend.checksum + + def bad_checksum(path): + if path.endswith(".zip"): + return "bad_checksum_value" + return original_checksum(path) + + interface.backend.checksum = bad_checksum + + # Try to extract with validation - should fail after extraction + with pytest.raises(InterruptedError, match="checksum"): + interface.get_archive("/archive.zip", dst_root, validate=True) + + # Restore original checksum + interface.backend.checksum = original_checksum + + # Verify pre-existing file still exists + assert os.path.exists(pre_existing_file) + with open(pre_existing_file) as f: + assert f.read() == "I was here before" + + # Verify extracted file was cleaned up + extracted_file = audeer.path(dst_root, "new_file.txt") + assert not os.path.exists(extracted_file) + + # Verify destination directory still exists + assert os.path.isdir(dst_root) + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.FileSystem, audbackend.interface.Unversioned)], + indirect=True, +) +def test_streaming_zip_with_directory_entries(tmpdir, interface): + """Test streaming extraction handles ZIP archives with directory entries. + + Some ZIP tools create archives with explicit directory entries + (entries ending with '/'). These should be skipped during extraction + while still extracting the files within those directories. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create source files with subdirectory + src_root = audeer.path(tmpdir, "src") + audeer.mkdir(src_root) + subdir = audeer.path(src_root, "subdir") + audeer.mkdir(subdir) + with open(audeer.path(src_root, "file1.txt"), "w") as f: + f.write("content1") + with open(audeer.path(subdir, "file2.txt"), "w") as f: + f.write("content2") + + # Create ZIP with explicit directory entries (unlike audeer.create_archive) + archive_path = audeer.path(tmpdir, "archive_with_dirs.zip") + with zipfile.ZipFile(archive_path, "w") as zf: + # Add directory entry explicitly + zf.write(subdir, "subdir/") + # Add files + zf.write(audeer.path(src_root, "file1.txt"), "file1.txt") + zf.write(audeer.path(subdir, "file2.txt"), os.path.join("subdir", "file2.txt")) + + # Verify the ZIP contains a directory entry + with zipfile.ZipFile(archive_path, "r") as zf: + names = zf.namelist() + assert "subdir/" in names # Directory entry exists + + # Upload archive + interface.put_file(archive_path, "/archive_with_dirs.zip") + + # Extract using streaming + dst_root = audeer.path(tmpdir, "dst") + extracted = interface.get_archive("/archive_with_dirs.zip", dst_root) + + # Verify files were extracted (directory entry should be skipped) + assert "file1.txt" in extracted + assert os.path.join("subdir", "file2.txt") in extracted + assert f"subdir{os.sep}" not in extracted # Directory entry should not be in result + + # Verify actual files exist + assert os.path.exists(audeer.path(dst_root, "file1.txt")) + assert os.path.exists(audeer.path(dst_root, "subdir", "file2.txt")) + with open(audeer.path(dst_root, "file1.txt")) as f: + assert f.read() == "content1" + with open(audeer.path(dst_root, "subdir", "file2.txt")) as f: + assert f.read() == "content2" + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.FileSystem, audbackend.interface.Versioned)], + indirect=True, +) +def test_size(tmpdir, interface): + """Test _size method returns correct file size.""" + # Create a file with known content + content = "Hello World!" * 1000 # ~12KB + src_path = audeer.path(tmpdir, "test.txt") + with open(src_path, "w") as f: + f.write(content) + expected_size = os.path.getsize(src_path) + + # Upload file to backend + interface.put_file(src_path, "/test.txt", "1.0.0") + + # Get size from backend + backend_path = interface._path_with_version("/test.txt", "1.0.0") + actual_size = interface.backend._size(backend_path) + + assert actual_size == expected_size diff --git a/tests/test_backend_minio.py b/tests/test_backend_minio.py index 85b9be61..6c134550 100644 --- a/tests/test_backend_minio.py +++ b/tests/test_backend_minio.py @@ -767,3 +767,80 @@ def test_invalid_timeout_warning(tmpdir, hosts, hide_credentials): timeout = http_client.connection_pool_kw.get("timeout") assert timeout.connect_timeout == 10.0 # default assert timeout.read_timeout is None # default + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Minio, audbackend.interface.Versioned)], + indirect=True, +) +def test_size(tmpdir, interface): + """Test _size method returns correct file size.""" + # Create a file with known content + content = "Hello World!" * 1000 # ~12KB + src_path = audeer.path(tmpdir, "test.txt") + with open(src_path, "w") as f: + f.write(content) + expected_size = os.path.getsize(src_path) + + # Upload file to backend + interface.put_file(src_path, "/test.txt", "1.0.0") + + # Get size from backend + backend_path = interface._path_with_version("/test.txt", "1.0.0") + actual_size = interface.backend._size(backend_path) + + assert actual_size == expected_size + + +@pytest.mark.parametrize( + "interface", + [(audbackend.backend.Minio, audbackend.interface.Unversioned)], + indirect=True, +) +def test_get_archive_streaming(tmpdir, interface): + """Test get_archive with streaming extraction verifies _get_file_stream. + + This test verifies that _get_file_stream() returns bytes correctly, + which is required for streaming ZIP extraction and checksum computation. + + """ + # Skip if stream-unzip is not available + try: + from stream_unzip import stream_unzip # noqa: F401 + except ImportError: + pytest.skip("stream-unzip not available") + + # Create source files + src_root = audeer.path(tmpdir, "src") + audeer.mkdir(src_root) + with open(audeer.path(src_root, "file1.txt"), "w") as f: + f.write("content of file 1") + with open(audeer.path(src_root, "file2.txt"), "w") as f: + f.write("content of file 2") + + # Create ZIP archive + archive_path = audeer.path(tmpdir, "archive.zip") + audeer.create_archive(src_root, None, archive_path) + + # Upload archive to backend + interface.put_file(archive_path, "/archive.zip") + + # Extract using streaming (this exercises _get_file_stream) + dst_root = audeer.path(tmpdir, "dst") + extracted = interface.get_archive("/archive.zip", dst_root) + + # Verify files were extracted correctly + assert sorted(extracted) == ["file1.txt", "file2.txt"] + with open(audeer.path(dst_root, "file1.txt")) as f: + assert f.read() == "content of file 1" + with open(audeer.path(dst_root, "file2.txt")) as f: + assert f.read() == "content of file 2" + + # Also test with validation to verify checksum computation works + # (requires _get_file_stream to return bytes for md5.update()) + dst_root_validated = audeer.path(tmpdir, "dst_validated") + extracted_validated = interface.get_archive( + "/archive.zip", dst_root_validated, validate=True + ) + assert sorted(extracted_validated) == ["file1.txt", "file2.txt"]