Use connection pool for faster download

audbackend/core/backend/artifactory.py

@@ -1,9 +1,11 @@
 from collections.abc import Iterator
+import configparser
 import os
 
 import artifactory
 import dohq_artifactory
 import requests
+from requests.adapters import HTTPAdapter
 
 import audeer
 
@@ -70,6 +72,16 @@ def _download(
 class Artifactory(Base):
     r"""Backend for Artifactory.
 
+    Connection pool settings can be configured
+    via the config file (see :meth:`get_config`):
+
+    * ``pool_connections``: number of connection pools to cache (default: 10)
+    * ``pool_maxsize``: max connections per pool (default: 10)
+    * ``max_retries``: max retries per connection (default: 0)
+
+    For bulk operations downloading many files in parallel,
+    increase ``pool_maxsize`` to match the number of workers.
+
     Args:
         host: host address
         repository: repository name
@@ -167,6 +179,62 @@ def get_authentication(cls, host: str) -> tuple[str, str]:
 
         return username, api_key
 
+    @classmethod
+    def get_config(cls, host: str) -> dict:
+        """Configuration of Artifactory server.
+
+        The default path of the config file
+        (:file:`~/.artifactory_python.cfg`)
+        can be overwritten with the environment variable
+        ``ARTIFACTORY_CONFIG_FILE``.
+
+        If no config file can be found,
+        or no entry for the requested host,
+        an empty dictionary is returned.
+
+        The config file
+        expects one section per host,
+        e.g.
+
+        .. code-block:: ini
+
+            [artifactory.example.com/artifactory]
+            pool_connections = 10
+            pool_maxsize = 100
+            max_retries = 3
+
+        Connection pool settings (for bulk operations with many files):
+
+        * ``pool_connections``: number of connection pools to cache (default: 10)
+        * ``pool_maxsize``: maximum number of connections per pool.
+          Increase this for parallel downloads of many files (default: 10)
+        * ``max_retries``: maximum number of retries per connection (default: 0)
+
+        Args:
+            host: hostname
+
+        Returns:
+            config entries as dictionary
+
+        """
+        config_file = os.getenv(
+            "ARTIFACTORY_CONFIG_FILE",
+            artifactory.default_config_path,
+        )
+        config_file = audeer.path(config_file)
+
+        if os.path.exists(config_file):
+            config = configparser.ConfigParser(allow_no_value=True)
+            config.read(config_file)
+            try:
+                config = dict(config.items(host))
+            except configparser.NoSectionError:
+                config = {}
+        else:
+            config = {}
+
+        return config
+
     def _checksum(
         self,
         path: str,
@@ -327,6 +395,37 @@ def _open(
         r"""Open connection to backend."""
         self._session = requests.Session()
         self._session.auth = self.authentication
+
+        # Configure connection pooling for better performance
+        # with parallel downloads of many files.
+        # Settings can be tuned via backend config:
+        #   - "pool_connections": number of pools to cache (default: 10)
+        #   - "pool_maxsize": max connections per pool (default: 10)
+        #   - "max_retries": max retries per connection (default: 0)
+        config = self.get_config(self.host)
+        pool_connections = utils.parse_config_int(
+            config.get("pool_connections", 10),
+            name="pool_connections",
+            default=10,
+        )
+        pool_maxsize = utils.parse_config_int(
+            config.get("pool_maxsize", 10),
+            name="pool_maxsize",
+            default=10,
+        )
+        max_retries = utils.parse_config_int(
+            config.get("max_retries", 0),
+            name="max_retries",
+            default=0,
+        )
+        adapter = HTTPAdapter(
+            pool_connections=pool_connections,
+            pool_maxsize=pool_maxsize,
+            max_retries=max_retries,
+        )
+        self._session.mount("https://", adapter)
+        self._session.mount("http://", adapter)
+
         path = artifactory.ArtifactoryPath(self.host, session=self._session)
         self._repo = path.find_repository(self.repository)
         if self._repo is None:

audbackend/core/backend/minio.py

@@ -18,12 +18,18 @@
 class Minio(Base):
     r"""Backend for MinIO.
 
-    HTTP timeouts can be configured via the config file
-    (see :meth:`get_config`):
+    HTTP timeouts and connection pool settings can be configured
+    via the config file (see :meth:`get_config`):
 
     * ``connect_timeout``: seconds for connection establishment (default: 10.0)
     * ``read_timeout``: seconds for read operations; ``None`` means no timeout
       (default: ``None``)
+    * ``num_pools``: number of connection pools to cache (default: 10)
+    * ``pool_maxsize``: max connections per pool (default: 10)
+    * ``pool_block``: block when pool is full (default: ``false``)
+
+    For bulk operations downloading many files in parallel,
+    increase ``pool_maxsize`` to match the number of workers.
 
     Alternatively,
     provide a custom ``http_client`` object as ``kwargs``
@@ -84,12 +90,16 @@ def __init__(
         if secure is None:
             secure = config.get("secure", True)
 
-        # Configure HTTP client with timeouts to prevent hanging connections.
+        # Configure HTTP client with timeouts and connection pooling.
         # Users can override by passing their own http_client in kwargs.
-        # Timeouts can be tuned via backend config:
+        # Settings can be tuned via backend config:
         #   - "connect_timeout": seconds for connection establishment (default: 10.0)
         #   - "read_timeout": seconds for read operations; None means no timeout
         #     (default: None)
+        #   - "num_pools": number of connection pools to cache (default: 10)
+        #   - "pool_maxsize": max connections per pool (default: 10)
+        #   - "pool_block": block when pool is full instead of creating new
+        #     connection (default: False)
         if "http_client" not in kwargs:
             connect_timeout = _parse_timeout(
                 config.get("connect_timeout", 10.0),
@@ -102,7 +112,27 @@ def __init__(
                 default=None,
             )
             timeout = urllib3.Timeout(connect=connect_timeout, read=read_timeout)
-            kwargs["http_client"] = urllib3.PoolManager(timeout=timeout)
+            num_pools = utils.parse_config_int(
+                config.get("num_pools", 10),
+                name="num_pools",
+                default=10,
+            )
+            pool_maxsize = utils.parse_config_int(
+                config.get("pool_maxsize", 10),
+                name="pool_maxsize",
+                default=10,
+            )
+            pool_block = _parse_bool(
+                config.get("pool_block", False),
+                name="pool_block",
+                default=False,
+            )
+            kwargs["http_client"] = urllib3.PoolManager(
+                timeout=timeout,
+                num_pools=num_pools,
+                maxsize=pool_maxsize,
+                block=pool_block,
+            )
 
         # Open MinIO client
         self._client = minio.Minio(
@@ -170,7 +200,7 @@ def get_config(cls, host: str) -> dict:
             access_key = "Q3AM3UQ867SPQQA43P2F"
             secret_key = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
 
-        Optional timeout settings can also be configured:
+        Optional timeout and connection pool settings can also be configured:
 
         .. code-block:: ini
 
@@ -179,12 +209,25 @@ def get_config(cls, host: str) -> dict:
             secret_key = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
             connect_timeout = 10.0
             read_timeout = 60.0
+            num_pools = 10
+            pool_maxsize = 100
+            pool_block = false
+
+        Timeout settings:
 
         * ``connect_timeout``: seconds for connection establishment
           (default: 10.0)
         * ``read_timeout``: seconds for read operations;
           use ``None`` for no timeout (default: ``None``)
 
+        Connection pool settings (for bulk operations with many files):
+
+        * ``num_pools``: number of connection pools to cache (default: 10)
+        * ``pool_maxsize``: maximum number of connections per pool.
+          Increase this for parallel downloads of many files (default: 10)
+        * ``pool_block``: if ``true``, block and wait when pool is exhausted
+          instead of creating a new connection outside the pool (default: ``false``)
+
         Args:
             host: hostname
 
@@ -568,6 +611,41 @@ def _metadata(checksum: str):
     }
 
 
+def _parse_bool(
+    value: str | bool,
+    *,
+    name: str,
+    default: bool,
+) -> bool:
+    """Parse a boolean value from config.
+
+    Converts string values to bool, handling common boolean strings.
+    If parsing fails, logs a warning and returns the default value.
+
+    Args:
+        value: boolean value (string from config or bool)
+        name: name of the setting (for warning messages)
+        default: default value to use if parsing fails
+
+    Returns:
+        parsed boolean value or default
+
+    """
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        if value.lower() in ("true", "1", "yes", "on"):
+            return True
+        if value.lower() in ("false", "0", "no", "off"):
+            return False
+    warnings.warn(
+        f"Invalid {name} value '{value}' in config, using default: {default}",
+        UserWarning,
+        stacklevel=4,
+    )
+    return default
+
+
 def _parse_timeout(
     value: str | float | None,
     *,

audbackend/core/utils.py

@@ -4,6 +4,7 @@
 import os
 import re
 import time
+import warnings
 
 from audbackend.core.errors import BackendError
 
@@ -148,3 +149,36 @@ def raise_is_a_directory(path: str):
         os.strerror(errno.EISDIR),
         path,
     )
+
+
+def parse_config_int(
+    value: str | int,
+    *,
+    name: str,
+    default: int,
+) -> int:
+    """Parse an integer value from config.
+
+    Converts string values to int.
+    If parsing fails, logs a warning and returns the default value.
+
+    Args:
+        value: integer value (string from config or int)
+        name: name of the setting (for warning messages)
+        default: default value to use if parsing fails
+
+    Returns:
+        parsed integer value or default
+
+    """
+    if isinstance(value, int):
+        return value
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        warnings.warn(
+            f"Invalid {name} value '{value}' in config, using default: {default}",
+            UserWarning,
+            stacklevel=4,
+        )
+        return default