diff --git a/diracx-core/src/diracx/core/config/sources.py b/diracx-core/src/diracx/core/config/sources.py index f16fffa82..fbf3bbed6 100644 --- a/diracx-core/src/diracx/core/config/sources.py +++ b/diracx-core/src/diracx/core/config/sources.py @@ -7,7 +7,6 @@ import asyncio import logging -import os from abc import ABCMeta, abstractmethod from datetime import datetime, timezone from pathlib import Path @@ -162,7 +161,12 @@ def __init_subclass__(cls) -> None: @classmethod def create(cls): - return cls.create_from_url(backend_url=os.environ["DIRACX_CONFIG_BACKEND_URL"]) + # Avoid circular import + from diracx.core.settings import FactorySettings + + return cls.create_from_url( + backend_url=FactorySettings().diracx_config_backend_url + ) @classmethod def create_from_url( diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index ec4942e78..2117da303 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -14,10 +14,12 @@ import contextlib import json +import os from collections.abc import AsyncIterator from pathlib import Path from typing import Annotated, Any, Self, TypeVar, cast +import dotenv from cryptography.fernet import Fernet from joserfc.jwk import KeySet, KeySetSerialization from pydantic import ( @@ -29,14 +31,18 @@ SecretStr, TypeAdapter, UrlConstraints, + field_validator, model_validator, ) from pydantic_settings import BaseSettings, SettingsConfigDict from signurlarity.aio.client import AsyncClient from signurlarity.exceptions import SignurlarityError +from .config.sources import ConfigSourceUrl +from .extensions import DiracEntryPoint, select_from_extension from .properties import SecurityProperty from .s3 import s3_bucket_exists +from .utils import dotenv_files_from_environment T = TypeVar("T") @@ -350,3 +356,102 @@ def s3_client(self) -> AsyncClient: if self._client is None: raise RuntimeError("S3 client accessed before lifetime function") return self._client + + +def _parse_env_bool(value: str) -> bool: + """Parse a boolean environment variable value.""" + return TypeAdapter(bool).validate_python(value) + + +class FactorySettings(ServiceSettingsBase): + """Factory settings. + + Settings which do not fit into dedicated classes, + or are dynamically generated. + """ + + model_config = SettingsConfigDict(use_attribute_docstrings=True) + + diracx_config_backend_url: ConfigSourceUrl | None = None + """The URL of the configuration backend. + """ + + diracx_legacy_exchange_hashed_api_key: str = "" + """The hashed API key for the legacy exchange endpoint. + """ + + diracx_tasks_redis_url: str = "redis://localhost" + """The url for the redis server to manage tasks""" + + enabled_services: dict[str, bool] = Field(default_factory=dict) + """The following environment variables dictates which routers are enabled.""" + + opensearch_dbs: dict[str, str] = Field(default_factory=dict) + """The following environment variables configure the OpenSearch database connections.""" + + sql_dbs: dict[str, str] = Field(default_factory=dict) + """The following environment variables configure the SQL database connections.""" + + @model_validator(mode="before") + @classmethod + def load_dotenv_files(cls, data: Any) -> Any: + """Load dotenv files before reading settings from environment.""" + for env_file in dotenv_files_from_environment("DIRACX_SERVICE_DOTENV"): + if not dotenv.load_dotenv(env_file): + raise NotImplementedError(f"Could not load dotenv file {env_file}") + return data + + @field_validator("enabled_services", mode="before") + @classmethod + def build_enabled_services(cls, value: Any) -> dict[str, bool]: + """Build enabled services from the installed service entry points.""" + enabled_services: dict[str, bool] = { + entry_point.name: True + for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES) + if "well-known" not in entry_point.name + } + + for service_name in enabled_services: + env_name = f"DIRACX_SERVICE_{service_name.upper()}_ENABLED" + if env_value := os.environ.get(env_name): + enabled_services[service_name] = _parse_env_bool(env_value) + + if isinstance(value, dict): + enabled_services.update(value) + return enabled_services + + @field_validator("opensearch_dbs", mode="before") + @classmethod + def build_opensearch_dbs(cls, value: Any) -> dict[str, str]: + """Build OpenSearch database URLs from the installed entry points.""" + opensearch_dbs: dict[str, str] = { + entry_point.name: "" + for entry_point in select_from_extension(group=DiracEntryPoint.OS_DB) + } + + for db_name in opensearch_dbs: + env_name = f"DIRACX_OS_DB_{db_name.upper()}" + if env_value := os.environ.get(env_name): + opensearch_dbs[db_name] = env_value + + if isinstance(value, dict): + opensearch_dbs.update(value) + return opensearch_dbs + + @field_validator("sql_dbs", mode="before") + @classmethod + def build_sql_dbs(cls, value: Any) -> dict[str, str]: + """Build SQL database URLs from the installed entry points.""" + sql_dbs: dict[str, str] = { + entry_point.name: "" + for entry_point in select_from_extension(group=DiracEntryPoint.SQL_DB) + } + + for db_name in sql_dbs: + env_name = f"DIRACX_DB_URL_{db_name.upper()}" + if env_value := os.environ.get(env_name): + sql_dbs[db_name] = env_value + + if isinstance(value, dict): + sql_dbs.update(value) + return sql_dbs diff --git a/diracx-db/src/diracx/db/os/utils.py b/diracx-db/src/diracx/db/os/utils.py index 1d80b9958..80af29326 100644 --- a/diracx-db/src/diracx/db/os/utils.py +++ b/diracx-db/src/diracx/db/os/utils.py @@ -3,7 +3,6 @@ import contextlib import json import logging -import os from abc import ABCMeta, abstractmethod from collections.abc import AsyncIterator from contextvars import ContextVar @@ -14,6 +13,7 @@ from diracx.core.exceptions import InvalidQueryError from diracx.core.extensions import DiracEntryPoint, select_from_extension +from diracx.core.settings import FactorySettings from diracx.db.exceptions import DBUnavailableError logger = logging.getLogger(__name__) @@ -38,7 +38,8 @@ class BaseOSDB(metaclass=ABCMeta): This method returns a dictionary of database names to connection parameters. The available databases are determined by the `diracx.dbs.os` entrypoint in the `pyproject.toml` file and the connection parameters are taken from the - environment variables prefixed with `DIRACX_OS_DB_{DB_NAME}`. + `opensearch_dbs` field in FactorySettings, which reads from environment variables + prefixed with `DIRACX_OS_DB_{DB_NAME}`. If extensions to DiracX are being used, there can be multiple implementations of the same database. To list the available implementations use @@ -104,19 +105,26 @@ def available_implementations(cls, db_name: str) -> list[type[BaseOSDB]]: def available_urls(cls) -> dict[str, dict[str, Any]]: """Return a dict of available OpenSearch database urls. - The list of available URLs is determined by environment variables + The list of available URLs is determined by the opensearch_dbs field + in FactorySettings, which reads from environment variables prefixed with ``DIRACX_OS_DB_{DB_NAME}``. """ + factory_settings = FactorySettings() + opensearch_dbs = factory_settings.opensearch_dbs + conn_kwargs: dict[str, dict[str, Any]] = {} for entry_point in select_from_extension(group=DiracEntryPoint.OS_DB): db_name = entry_point.name - var_name = f"DIRACX_OS_DB_{entry_point.name.upper()}" - if var_name in os.environ: - try: - conn_kwargs[db_name] = json.loads(os.environ[var_name]) - except Exception: - logger.error("Error loading connection parameters for %s", db_name) - raise + # Get the field value from the OpenSearchDBSettings model + if field_value := opensearch_dbs.get(db_name): + if field_value: + try: + conn_kwargs[db_name] = json.loads(field_value) + except Exception: + logger.error( + "Error loading connection parameters for %s", db_name + ) + raise return conn_kwargs @classmethod diff --git a/diracx-db/src/diracx/db/sql/utils/base.py b/diracx-db/src/diracx/db/sql/utils/base.py index 4a8ddfd65..d5a03fbc7 100644 --- a/diracx-db/src/diracx/db/sql/utils/base.py +++ b/diracx-db/src/diracx/db/sql/utils/base.py @@ -2,7 +2,6 @@ import contextlib import logging -import os import re from abc import ABCMeta from collections.abc import AsyncIterator @@ -53,8 +52,9 @@ class BaseSQLDB(metaclass=ABCMeta): The available databases are discovered by calling `BaseSQLDB.available_urls`. This method returns a mapping of database names to connection URLs. The available databases are determined by the `diracx.dbs.sql` entrypoint in the - `pyproject.toml` file and the connection URLs are taken from the environment - variables of the form `DIRACX_DB_URL_`. + `pyproject.toml` file and the connection URLs are taken from the + `sql_dbs` field in FactorySettings, which reads from environment variables + of the form `DIRACX_DB_URL_`. If extensions to DiracX are being used, there can be multiple implementations of the same database. To list the available implementations use @@ -125,16 +125,21 @@ def available_implementations(cls, db_name: str) -> list[type["BaseSQLDB"]]: def available_urls(cls) -> dict[str, str]: """Return a dict of available database urls. - The list of available URLs is determined by environment variables + The list of available URLs is determined by the sql_dbs field + in FactorySettings, which reads from environment variables prefixed with ``DIRACX_DB_URL_{DB_NAME}``. """ + from diracx.core.settings import FactorySettings + + factory_settings = FactorySettings() + sql_dbs = factory_settings.sql_dbs + db_urls: dict[str, str] = {} for entry_point in select_from_extension(group=DiracEntryPoint.SQL_DB): db_name = entry_point.name - var_name = f"DIRACX_DB_URL_{entry_point.name.upper()}" - if var_name in os.environ: + # Get the field value from the SqlDBSettings model + if db_url := sql_dbs.get(db_name): try: - db_url = os.environ[var_name] if db_url == "sqlite+aiosqlite:///:memory:": db_urls[db_name] = db_url # pydantic does not allow for underscore in scheme diff --git a/diracx-logic/src/diracx/logic/__main__.py b/diracx-logic/src/diracx/logic/__main__.py index e54837092..596edb09e 100644 --- a/diracx-logic/src/diracx/logic/__main__.py +++ b/diracx-logic/src/diracx/logic/__main__.py @@ -93,14 +93,15 @@ async def delete_jwk(args): async def cleanup_authdb(args): """Maintain AuthDB partitions and remove expired flows.""" logger.info("Maintaining AuthDB partitions and removing expired flows") - import os - from diracx.core.settings import AuthSettings + from diracx.core.settings import AuthSettings, FactorySettings from diracx.db.sql import AuthDB from diracx.logic.auth.management import cleanup_expired_data settings = AuthSettings() - db_url = os.environ["DIRACX_DB_URL_AUTHDB"] + factory_settings = FactorySettings() + db_url = factory_settings.sql_dbs.AuthDB + db = AuthDB(db_url) async with db.engine_context(): async with db: diff --git a/diracx-routers/src/diracx/routers/auth/token.py b/diracx-routers/src/diracx/routers/auth/token.py index b61222fa0..6affb658e 100644 --- a/diracx-routers/src/diracx/routers/auth/token.py +++ b/diracx-routers/src/diracx/routers/auth/token.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import os from http import HTTPStatus from typing import Annotated, Literal @@ -21,7 +20,7 @@ RefreshTokenPayload, TokenResponse, ) -from diracx.core.settings import AuthSettings +from diracx.core.settings import AuthSettings, FactorySettings from diracx.db.sql import AuthDB from diracx.logic.auth import create_token from diracx.logic.auth import get_oidc_token as get_oidc_token_bl @@ -182,6 +181,7 @@ async def perform_legacy_exchange( auth_db: AuthDB, available_properties: AvailableSecurityProperties, settings: AuthSettings, + factory_settings: FactorySettings, config: Config, all_access_policies: Annotated[ dict[str, BaseAccessPolicy], Depends(BaseAccessPolicy.all_used_access_policies) @@ -193,9 +193,7 @@ async def perform_legacy_exchange( This route is disabled if DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY is not set in the environment. """ - if not ( - expected_api_key := os.environ.get("DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY") - ): + if not (expected_api_key := factory_settings.diracx_legacy_exchange_hashed_api_key): raise HTTPException( status_code=HTTPStatus.SERVICE_UNAVAILABLE, detail="Legacy exchange is not enabled", diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index 4c26f4c58..7886f25b6 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -6,7 +6,6 @@ import inspect import logging -import os from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Sequence from functools import partial from http import HTTPStatus @@ -14,7 +13,6 @@ from logging import Formatter, StreamHandler from typing import Any, TypeVar, cast -import dotenv from cachetools import TTLCache from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request from fastapi.dependencies.models import Dependant @@ -24,15 +22,13 @@ from fastapi.responses import JSONResponse, Response from fastapi.routing import APIRoute from packaging.version import InvalidVersion, parse -from pydantic import TypeAdapter from starlette.middleware.base import BaseHTTPMiddleware from uvicorn.logging import AccessFormatter, DefaultFormatter from diracx.core.config import ConfigSource from diracx.core.exceptions import DiracError, DiracHttpResponseError, NotReadyError from diracx.core.extensions import DiracEntryPoint, select_from_extension -from diracx.core.settings import ServiceSettingsBase -from diracx.core.utils import dotenv_files_from_environment +from diracx.core.settings import FactorySettings, ServiceSettingsBase from diracx.db.exceptions import DBUnavailableError from diracx.db.os.utils import BaseOSDB from diracx.db.sql.utils import BaseSQLDB @@ -143,7 +139,6 @@ def create_app_inner( # Please see ServiceSettingsBase for more details available_settings_classes: set[type[ServiceSettingsBase]] = set() - for service_settings in all_service_settings: cls = type(service_settings) assert cls not in available_settings_classes @@ -346,17 +341,12 @@ def create_app() -> DiracFastAPI: We attempt to load each setting classes to make sure that the settings are correctly defined. """ - for env_file in dotenv_files_from_environment("DIRACX_SERVICE_DOTENV"): - logger.debug("Loading dotenv file: %s", env_file) - if not dotenv.load_dotenv(env_file): - raise NotImplementedError(f"Could not load dotenv file {env_file}") - # Load all available routers enabled_systems = set() settings_classes = set() + factory_settings = FactorySettings() for entry_point in select_from_extension(group=DiracEntryPoint.SERVICES): - env_var = f"DIRACX_SERVICE_{entry_point.name.upper()}_ENABLED" - enabled = TypeAdapter(bool).validate_json(os.environ.get(env_var, "true")) + enabled = factory_settings.enabled_services.get(entry_point.name, True) logger.debug("Found service %r: enabled=%s", entry_point, enabled) if not enabled: continue @@ -443,6 +433,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError def find_dependents( obj: APIRouter | Iterable[Dependant], cls: type[T] ) -> Iterable[type[T]]: + if isinstance(obj, APIRouter): # TODO: Support dependencies of the router itself # yield from find_dependents(obj.dependencies, cls) diff --git a/diracx-routers/tests/auth/test_legacy_exchange.py b/diracx-routers/tests/auth/test_legacy_exchange.py index 6aabc4646..f89a957e7 100644 --- a/diracx-routers/tests/auth/test_legacy_exchange.py +++ b/diracx-routers/tests/auth/test_legacy_exchange.py @@ -17,17 +17,40 @@ "ConfigSource", "BaseAccessPolicy", "DevelopmentSettings", + "FactorySettings", ] ) @pytest.fixture -def legacy_credentials(monkeypatch): +def legacy_credentials(test_client, monkeypatch): + secret = secrets.token_bytes() valid_token = f"diracx:legacy:{base64.urlsafe_b64encode(secret).decode()}" - monkeypatch.setenv( - "DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY", hashlib.sha256(secret).hexdigest() + hashed_key = hashlib.sha256(secret).hexdigest() + + from diracx.core.settings import FactorySettings + + # FactorySettings is a frozen Pydantic model, so assigning to + # test_factory_settings.diracx_legacy_exchange_hashed_api_key with + # monkeypatch.setattr raises a ValidationError. The application reads + # FactorySettings through FastAPI dependency injection (FactorySettings.create + # is used as the dependency key), so the safe way to alter behavior for this + # test is to replace the dependency provider itself. We locate the existing + # FactorySettings dependency in the app overrides and swap it with a callable + # that returns a new FactorySettings instance where legacy exchange is set + + factory_settings_dependency = next( + dep + for dep in test_client.app.dependency_overrides + if getattr(dep, "__self__", None) is FactorySettings + ) + monkeypatch.setitem( + test_client.app.dependency_overrides, + factory_settings_dependency, + lambda: FactorySettings(diracx_legacy_exchange_hashed_api_key=hashed_key), ) + yield {"Authorization": f"Bearer {valid_token}"} @@ -134,6 +157,7 @@ async def test_refresh_token(test_client, legacy_credentials): async def test_disabled(test_client): + r = test_client.get( "/api/auth/legacy-exchange", params={"preferred_username": "chaen", "scope": "vo:lhcb group:lhcb_user"}, diff --git a/diracx-routers/tests/auth/test_standard.py b/diracx-routers/tests/auth/test_standard.py index 42a855aaf..22cd0dbb2 100644 --- a/diracx-routers/tests/auth/test_standard.py +++ b/diracx-routers/tests/auth/test_standard.py @@ -39,7 +39,7 @@ DIRAC_CLIENT_ID = "myDIRACClientID" pytestmark = pytest.mark.enabled_dependencies( - ["AuthDB", "AuthSettings", "ConfigSource", "BaseAccessPolicy"] + ["AuthDB", "AuthSettings", "ConfigSource", "BaseAccessPolicy", "FactorySettings"] ) diff --git a/diracx-routers/tests/test_generic.py b/diracx-routers/tests/test_generic.py index a3447a226..1bb56e804 100644 --- a/diracx-routers/tests/test_generic.py +++ b/diracx-routers/tests/test_generic.py @@ -13,6 +13,7 @@ "AuthSettings", "OpenAccessPolicy", "DevelopmentSettings", + "FactorySettings", ] ) diff --git a/diracx-tasks/src/diracx/tasks/plumbing/factory.py b/diracx-tasks/src/diracx/tasks/plumbing/factory.py index 17187a056..c27982094 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/factory.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/factory.py @@ -3,7 +3,6 @@ import asyncio import inspect import logging -import os from collections.abc import AsyncIterator, Iterable from contextlib import AsyncExitStack, asynccontextmanager from functools import partial @@ -15,6 +14,7 @@ from fastapi.dependencies.utils import get_dependant from diracx.core.extensions import select_from_extension +from diracx.core.settings import FactorySettings from ._redis_types import LockCoordinator from .base_task import BaseTask @@ -313,7 +313,7 @@ async def setup_dependency_overrides( overrides[os_db_class.session] = partial(_db_context, os_db) # --- Config --- - config_url = os.environ.get("DIRACX_CONFIG_BACKEND_URL") + config_url = FactorySettings().diracx_config_backend_url if config_url: from diracx.core.config import ConfigSource diff --git a/diracx-tasks/src/diracx/tasks/task_run.py b/diracx-tasks/src/diracx/tasks/task_run.py index a9971cfe9..211c12428 100644 --- a/diracx-tasks/src/diracx/tasks/task_run.py +++ b/diracx-tasks/src/diracx/tasks/task_run.py @@ -22,11 +22,13 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Iterable +from diracx.core.settings import FactorySettings + if TYPE_CHECKING: from .plumbing._redis_types import LockCoordinator DEFAULT_REDIS_URL = "redis://localhost" -REDIS_URL_ENV_VAR = "DIRACX_TASKS_REDIS_URL" +_factory_settings = FactorySettings() class DebugOptions(StrEnum): @@ -39,7 +41,8 @@ def _get_redis_url(args: argparse.Namespace) -> str: """Resolve Redis URL from CLI arg, env var, or default.""" if hasattr(args, "redis_url") and args.redis_url: return args.redis_url - return os.environ.get(REDIS_URL_ENV_VAR, DEFAULT_REDIS_URL) + + return _factory_settings.diracx_tasks_redis_url def main() -> None: @@ -106,7 +109,7 @@ def main() -> None: "--redis-url", type=str, default=None, - help=f"Redis URL (default: ${REDIS_URL_ENV_VAR} or {DEFAULT_REDIS_URL})", + help=f"Redis URL (default: ${{REDIS_URL_ENV_VAR}} or {DEFAULT_REDIS_URL})", ) submit_parser.set_defaults( func=lambda args: asyncio.run( @@ -131,7 +134,7 @@ def main() -> None: "--redis-url", type=str, default=None, - help=f"Redis URL (default: ${REDIS_URL_ENV_VAR} or {DEFAULT_REDIS_URL})", + help=f"Redis URL (default: ${{REDIS_URL_ENV_VAR}} or {DEFAULT_REDIS_URL})", ) worker_parser.add_argument( "--worker-size", @@ -158,7 +161,7 @@ def main() -> None: "--redis-url", type=str, default=None, - help=f"Redis URL (default: ${REDIS_URL_ENV_VAR} or {DEFAULT_REDIS_URL})", + help=f"Redis URL (default: ${{REDIS_URL_ENV_VAR}} or {DEFAULT_REDIS_URL})", ) scheduler_parser.set_defaults( func=lambda args: asyncio.run(start_scheduler(redis_url=_get_redis_url(args))) @@ -240,7 +243,7 @@ async def start_scheduler(redis_url: str) -> None: task_classes = load_task_registry() config = None - config_url = os.environ.get("DIRACX_CONFIG_BACKEND_URL") + config_url = _factory_settings.diracx_config_backend_url if config_url: from diracx.core.config import ConfigSource @@ -336,7 +339,7 @@ async def call_task( # Try to connect to Redis for lock acquisition redis: LockCoordinator | None = None - redis_url = os.environ.get(REDIS_URL_ENV_VAR) + redis_url = _factory_settings.diracx_tasks_redis_url if redis_url: from redis.asyncio import Redis diff --git a/diracx-testing/src/diracx/testing/__init__.py b/diracx-testing/src/diracx/testing/__init__.py index 295aaa079..cc94ad65c 100644 --- a/diracx-testing/src/diracx/testing/__init__.py +++ b/diracx-testing/src/diracx/testing/__init__.py @@ -15,6 +15,7 @@ "session_client_factory", "test_auth_settings", "test_dev_settings", + "test_factory_settings", "test_login", "test_sandbox_settings", "verify_entry_points", @@ -38,6 +39,7 @@ session_client_factory, test_auth_settings, test_dev_settings, + test_factory_settings, test_login, test_sandbox_settings, with_cli_login, diff --git a/diracx-testing/src/diracx/testing/utils.py b/diracx-testing/src/diracx/testing/utils.py index 4e02e3140..ae65c702a 100644 --- a/diracx-testing/src/diracx/testing/utils.py +++ b/diracx-testing/src/diracx/testing/utils.py @@ -17,6 +17,7 @@ "session_client_factory", "test_auth_settings", "test_dev_settings", + "test_factory_settings", "test_login", "test_sandbox_settings", "with_cli_login", @@ -51,6 +52,7 @@ from diracx.core.settings import ( AuthSettings, DevelopmentSettings, + FactorySettings, SandboxStoreSettings, ) from diracx.routers.utils import AuthorizedUserInfo @@ -116,6 +118,14 @@ def test_auth_settings(private_key, fernet_key) -> Generator[AuthSettings, None, ) +@pytest.fixture(scope="session") +def test_factory_settings() -> Generator[FactorySettings, None, None]: + + from diracx.core.settings import FactorySettings + + yield FactorySettings() + + @pytest.fixture(scope="session") def aio_moto(worker_id): """Start the moto server in a separate thread and return the base URL. @@ -167,6 +177,7 @@ def __init__( test_auth_settings, test_sandbox_settings, test_dev_settings, + test_factory_settings, ): from diracx.core.config import ConfigSource from diracx.core.extensions import select_from_extension @@ -216,6 +227,7 @@ def enrich_tokens( self.test_auth_settings = test_auth_settings self.test_dev_settings = test_dev_settings + self.test_factory_settings = test_factory_settings all_access_policies = { e.name: [AlwaysAllowAccessPolicy] @@ -235,6 +247,7 @@ def enrich_tokens( test_auth_settings, test_sandbox_settings, test_dev_settings, + test_factory_settings, ], database_urls=database_urls, os_database_conn_kwargs=os_database_conn_kwargs, @@ -412,6 +425,7 @@ def session_client_factory( with_config_repo, tmp_path_factory, test_dev_settings, + test_factory_settings, ): """TODO.""" yield ClientFactory( @@ -420,6 +434,7 @@ def session_client_factory( test_auth_settings, test_sandbox_settings, test_dev_settings, + test_factory_settings, ) diff --git a/docs/admin/reference/env-variables.md b/docs/admin/reference/env-variables.md index 387a594d8..339b9e508 100644 --- a/docs/admin/reference/env-variables.md +++ b/docs/admin/reference/env-variables.md @@ -4,18 +4,127 @@ ## Core +### `DIRACX_SERVICE_DOTENV` + +The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes +\_X, where X is a number. The files will be loaded in order. + +## FactorySettings + +Factory settings. + +``` +Settings which do not fit into dedicated classes, +or are dynamically generated. +``` + ### `DIRACX_CONFIG_BACKEND_URL` +*Optional*, default value: `None` The URL of the configuration backend. -### `DIRACX_SERVICE_DOTENV` +### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` -The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes -\_X, where X is a number. The files will be loaded in order. +*Optional*, default value: \`\` +The hashed API key for the legacy exchange endpoint. + +### `DIRACX_TASKS_REDIS_URL` + +*Optional*, default value: `redis://localhost` +The url for the redis server to manage tasks -### `DIRACX_SERVICE_JOBS_ENABLED` +### `ENABLED_SERVICES` + +*Optional* +The following environment variables dictates which routers are enabled. + +#### `DIRACX_SERVICE_AUTH_ENABLED` + +*Optional*, default value: `True` -Determines whether the jobs service is enabled. +Enable the AUTH router + +#### `DIRACX_SERVICE_CONFIG_ENABLED` + +*Optional*, default value: `True` + +Enable the CONFIG router + +#### `DIRACX_SERVICE_HEALTH_ENABLED` + +*Optional*, default value: `True` + +Enable the HEALTH router + +#### `DIRACX_SERVICE_JOBS_ENABLED` + +*Optional*, default value: `True` + +Enable the JOBS router + +### `OPENSEARCH_DBS` + +*Optional* +The following environment variables configure the OpenSearch database connections. + +#### `DIRACX_OS_DB_JOBPARAMETERSDB` + +*Optional*, default value: \`\` + +A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database JobParametersDB. + +### `SQL_DBS` + +*Optional* +The following environment variables configure the SQL database connections. + +#### `DIRACX_DB_URL_AUTHDB` + +*Optional*, default value: \`\` + +The URL for the SQL database AuthDB. + +#### `DIRACX_DB_URL_JOBDB` + +*Optional*, default value: \`\` + +The URL for the SQL database JobDB. + +#### `DIRACX_DB_URL_JOBLOGGINGDB` + +*Optional*, default value: \`\` + +The URL for the SQL database JobLoggingDB. + +#### `DIRACX_DB_URL_PILOTAGENTSDB` + +*Optional*, default value: \`\` + +The URL for the SQL database PilotAgentsDB. + +#### `DIRACX_DB_URL_RESOURCESTATUSDB` + +*Optional*, default value: \`\` + +The URL for the SQL database ResourceStatusDB. + +#### `DIRACX_DB_URL_SANDBOXMETADATADB` + +*Optional*, default value: \`\` + +The URL for the SQL database SandboxMetadataDB. + +#### `DIRACX_DB_URL_TASKDB` + +*Optional*, default value: \`\` + +The URL for the SQL database TaskDB. + +#### `DIRACX_DB_URL_TASKQUEUEDB` + +*Optional*, default value: \`\` + +The URL for the SQL database TaskQueueDB. ## AuthSettings @@ -139,10 +248,6 @@ Set of security properties available in this DIRAC installation. These properties define various authorization capabilities and are used for access control decisions. Defaults to all available security properties. -### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` - -The hashed API key for the legacy exchange endpoint. - ## SandboxStoreSettings Settings for the sandbox store. @@ -219,16 +324,6 @@ Maximum number of concurrent DB delete chunks during cleaning. Controls parallelism of database DELETE operations. -## Databases - -### `DIRACX_DB_URL_` - -The URL for the SQL database ``. - -### `DIRACX_OS_DB_` - -A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database ``. - ## OTELSettings Settings for the Open Telemetry Configuration. diff --git a/docs/admin/reference/env-variables.md.j2 b/docs/admin/reference/env-variables.md.j2 index 29cb4f078..8f17665e1 100644 --- a/docs/admin/reference/env-variables.md.j2 +++ b/docs/admin/reference/env-variables.md.j2 @@ -1,35 +1,20 @@ -{% from '_render_class.jinja' import render_class %} +{% from '_render_class.jinja' import render_class,render_factory_settings %} # List of environment variables *This page is auto-generated from the settings classes in `diracx.core.settings`.* ## Core -### `DIRACX_CONFIG_BACKEND_URL` -The URL of the configuration backend. ### `DIRACX_SERVICE_DOTENV` The variable points to .env files where configuration may be placed. There could be more than one file, with suffixes _X, where X is a number. The files will be loaded in order. -### `DIRACX_SERVICE_JOBS_ENABLED` -Determines whether the jobs service is enabled. +{{ render_factory_settings() }} -{{ render_class('AuthSettings') }} -### `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY` -The hashed API key for the legacy exchange endpoint. +{{ render_class('AuthSettings') }} {{ render_class('SandboxStoreSettings') }} -## Databases - -### `DIRACX_DB_URL_` - -The URL for the SQL database ``. - -### `DIRACX_OS_DB_` - -A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database ``. - {{ render_class('OTELSettings') }} diff --git a/docs/templates/_render_class.jinja b/docs/templates/_render_class.jinja index 242d6fb4b..10d9a2aa6 100644 --- a/docs/templates/_render_class.jinja +++ b/docs/templates/_render_class.jinja @@ -38,3 +38,144 @@ Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(' {% endif %} {% endmacro %} + +{% macro render_factory_settings() %} +{# Find the FactorySettings class #} +{% set cls = namespace(found=none) %} +{% for c, field_list in classes.items() %} + {% if c.__name__ == 'FactorySettings' %} + {% set cls.found = c %} + {% endif %} +{% endfor %} + +{% if cls.found %} +## {{ cls.found.__name__ }} + +{{ cls.found.__doc__ or "*No description available.*" }} + +{# Render all fields manually #} +{% for field_name, field_info in cls.found.model_fields.items() %} + {% set env_prefix = cls.found.model_config.get('env_prefix', '') %} + {% set env_name = (env_prefix ~ field_name).upper() %} + {% if field_info.validation_alias %} + {% set env_name = field_info.validation_alias %} + {% endif %} + + {# Render the field heading #} +{% if not loop.first %} + +{% endif %}### `{{ env_name|upper }}` + +*{% if field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if has_default_value(field_info) %}, default value: `{{ fix_str_enum_value(field_info.default) }}`{% endif %} + +{% if field_info.description %} +{{ field_info.description }} +{% endif %} + + {# Special handling for enabled_services - render nested fields one level deeper #} + {% if field_name == 'enabled_services' and field_info.annotation %} + {% set nested_model = field_info.annotation %} + {% if nested_model.model_fields is defined %} + + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} + {% if nested_field_info.validation_alias %} + {% set nested_env_name = nested_field_info.validation_alias %} + {% else %} + {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} + {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} + {% endif %} + +#### `{{ nested_env_name|upper }}` + +*{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} + +{% if nested_field_info.description %} +{{ nested_field_info.description }} +{% endif %} + {% endfor %} + {% elif factory_enabled_services is defined %} + {% for entry in factory_enabled_services %} + +#### `{{ entry.env_name|upper }}` + +*Optional*, default value: `{{ entry.default }}` + +{{ entry.description }} + {% endfor %} + {% endif %} + {% endif %} + + {# Special handling for opensearch_dbs - render nested fields one level deeper #} + {% if field_name == 'opensearch_dbs' and field_info.annotation %} + {% set nested_model = field_info.annotation %} + {% if nested_model.model_fields is defined %} + + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} + {% if nested_field_info.validation_alias %} + {% set nested_env_name = nested_field_info.validation_alias %} + {% else %} + {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} + {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} + {% endif %} + +#### `{{ nested_env_name|upper }}` + +*{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} + +{% if nested_field_info.description %} +{{ nested_field_info.description }} +{% endif %} + {% endfor %} + {% elif factory_opensearch_dbs is defined %} + {% for entry in factory_opensearch_dbs %} + +#### `{{ entry.env_name|upper }}` + +*Optional*, default value: `{{ entry.default }}` + +{{ entry.description }} + {% endfor %} + {% endif %} + {% endif %} + + {# Special handling for sql_dbs - render nested fields one level deeper #} + {% if field_name == 'sql_dbs' and field_info.annotation %} + {% set nested_model = field_info.annotation %} + {% if nested_model.model_fields is defined %} + + {% for nested_field_name, nested_field_info in nested_model.model_fields.items() %} + {% if nested_field_info.validation_alias %} + {% set nested_env_name = nested_field_info.validation_alias %} + {% else %} + {% set nested_env_prefix = nested_model.model_config.get('env_prefix', '') %} + {% set nested_env_name = (nested_env_prefix ~ nested_field_name).upper() %} + {% endif %} + +#### `{{ nested_env_name|upper }}` + +*{% if nested_field_info.is_required() %}*Required*{% else %}Optional{% endif %}*{% if nested_field_info.default is not none %}, default value: `{{ nested_field_info.default }}`{% endif %} + +{% if nested_field_info.description %} +{{ nested_field_info.description }} +{% endif %} + {% endfor %} + {% elif factory_sql_dbs is defined %} + {% for entry in factory_sql_dbs %} + +#### `{{ entry.env_name|upper }}` + +*Optional*, default value: `{{ entry.default }}` + +{{ entry.description }} + {% endfor %} + {% endif %} + {% endif %} +{% endfor %} +{% else %} +{# Class not found - provide a helpful error #} +**Error: FactorySettings class not found in diracx.core.settings** + +Available classes: {{ classes.keys() | map(attribute='__name__') | list | join(', ') }} +{% endif %} + +{% endmacro %} diff --git a/scripts/generate_settings_docs.py b/scripts/generate_settings_docs.py index 27e34558f..62389de75 100644 --- a/scripts/generate_settings_docs.py +++ b/scripts/generate_settings_docs.py @@ -29,6 +29,7 @@ from settings_doc.main import _model_fields from settings_doc.template_functions import JINJA_ENV_GLOBALS +from diracx.core.extensions import DiracEntryPoint, select_from_extension from diracx.core.settings import ServiceSettingsBase @@ -194,6 +195,7 @@ def validate_documentation( # Exclude ServiceSettingsBase as it's the base class undocumented.discard("ServiceSettingsBase") + undocumented.discard("FactorySettings") if undocumented: all_documented = False @@ -231,6 +233,44 @@ def generate_all_templates( classes = {cls: list(cls.model_fields.values()) for cls in settings} + factory_enabled_services = [ + { + "env_name": f"DIRACX_SERVICE_{entry_point.name.upper()}_ENABLED", + "description": f"Enable the {entry_point.name.upper()} router", + "default": "True", + } + for entry_point in sorted( + select_from_extension(group=DiracEntryPoint.SERVICES), + key=lambda entry: entry.name, + ) + if "well-known" not in entry_point.name + ] + + factory_opensearch_dbs = [ + { + "env_name": f"DIRACX_OS_DB_{entry_point.name.upper()}", + "description": "A JSON-encoded dictionary of connection keyword arguments" + f" for the OpenSearch database {entry_point.name}.", + "default": "", + } + for entry_point in sorted( + select_from_extension(group=DiracEntryPoint.OS_DB), + key=lambda entry: entry.name, + ) + ] + + factory_sql_dbs = [ + { + "env_name": f"DIRACX_DB_URL_{entry_point.name.upper()}", + "description": f"The URL for the SQL database {entry_point.name}.", + "default": "", + } + for entry_point in sorted( + select_from_extension(group=DiracEntryPoint.SQL_DB), + key=lambda entry: entry.name, + ) + ] + # Set up Jinja2 environment env = Environment( loader=FileSystemLoader([str(docs_dir), str(docs_dir / "templates")]), @@ -247,6 +287,9 @@ def generate_all_templates( "fields": fields, "classes": classes, "all_classes": settings, + "factory_enabled_services": factory_enabled_services, + "factory_opensearch_dbs": factory_opensearch_dbs, + "factory_sql_dbs": factory_sql_dbs, } )