From 4c1166405ede4e1aba34d8767f01ec8980c8e0a8 Mon Sep 17 00:00:00 2001 From: ademicho123 Date: Sat, 25 Jul 2026 13:47:13 +0100 Subject: [PATCH 1/3] feat(prometheus): support wildcard metric name in prometheus_metrics_config for default label filtering --- litellm/types/integrations/prometheus.py | 22 ++- .../test_prometheus_wildcard_labels.py | 154 ++++++++++++++++++ 2 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_wildcard_labels.py diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 318ba1f5956..9ac31e8ac4b 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -68,6 +68,7 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]: append(ch) return "".join(parts) +PROMETHEUS_METRICS_WILDCARD = "*" @dataclass class MetricValidationError: @@ -775,7 +776,13 @@ class PrometheusMetricLabels: @staticmethod def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: - default_labels = getattr(PrometheusMetricLabels, label_name) + # Some entries in DEFINED_PROMETHEUS_METRICS (e.g. litellm_in_flight_requests) + # have no matching class attribute here, because that metric is created + # with a fixed, empty label set and never goes through get_labels_for_metric(). + # Default to no labels instead of raising, so anything that legitimately + # iterates over every defined metric name (e.g. a wildcard config group) + # doesn't blow up on a metric like this. + default_labels = getattr(PrometheusMetricLabels, label_name, []) custom_labels = [] # Add custom metadata labels @@ -836,7 +843,6 @@ def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: "api_key_hash": "hashed_api_key", } - @dataclass(frozen=True, init=False) class UserAPIKeyLabelValues: """ @@ -933,7 +939,17 @@ def model_dump(self) -> Dict[str, Any]: @dataclass class PrometheusMetricsConfig: - """Configuration for filtering Prometheus metrics (parsed once from proxy config).""" + """ + Configuration for filtering Prometheus metrics (parsed once from proxy config). + + `metrics` is normally a list of names from DEFINED_PROMETHEUS_METRICS. It + can also be `[PROMETHEUS_METRICS_WILDCARD]` ("*") to apply `include_labels` + as the default label set for every metric, instead of listing all of them. + A wildcard group only sets defaults - it never disables a metric, and a + group naming a specific metric still wins over the wildcard for that one + metric. A group can't mix the wildcard with real metric names; use two + groups instead (see PrometheusLogger._build_label_filters). + """ group: str metrics: List[str] diff --git a/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py new file mode 100644 index 00000000000..9545fd794a3 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py @@ -0,0 +1,154 @@ +import sys +from typing import get_args + +import pytest + +sys.path.insert(0, "../../../..") + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.integrations.prometheus import ( + DEFINED_PROMETHEUS_METRICS, + PROMETHEUS_METRICS_WILDCARD, + PrometheusMetricsConfig, +) + + +def _bare_logger() -> PrometheusLogger: + """A PrometheusLogger instance with __init__ skipped, for testing the + config-parsing helpers in isolation without re-registering ~70 real + metrics against prometheus_client's global registry on every test.""" + return PrometheusLogger.__new__(PrometheusLogger) + + +def test_wildcard_sets_default_labels_for_every_metric(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="defaults", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=["hashed_api_key", "team"], + ) + ] + + label_filters = logger._build_label_filters(configs) + + assert set(label_filters.keys()) == set(get_args(DEFINED_PROMETHEUS_METRICS)) + for labels in label_filters.values(): + assert labels == ["hashed_api_key", "team"] + + +def test_named_group_overrides_wildcard_for_that_metric(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="defaults", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=["hashed_api_key", "team"], + ), + PrometheusMetricsConfig( + group="spend_needs_more", + metrics=["litellm_spend_metric"], + include_labels=["hashed_api_key", "team", "end_user"], + ), + ] + + label_filters = logger._build_label_filters(configs) + + assert label_filters["litellm_spend_metric"] == [ + "hashed_api_key", + "team", + "end_user", + ] + assert label_filters["litellm_total_tokens_metric"] == ["hashed_api_key", "team"] + + +def test_wildcard_group_does_not_disable_other_metrics(): + """A wildcard-only config should leave every metric enabled - it's a + label filter, not an allowlist.""" + logger = _bare_logger() + litellm.prometheus_metrics_config = [ + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + } + ] + + logger._parse_prometheus_config() + + assert logger.enabled_metrics == set() + assert logger._is_metric_enabled("litellm_spend_metric") is True + assert logger._is_metric_enabled("litellm_mcp_tool_calls_total") is True + + +def test_wildcard_combined_with_named_enable_list(): + """Wildcard for labels + a separate group naming which metrics are + enabled - the two concerns are independent.""" + logger = _bare_logger() + litellm.prometheus_metrics_config = [ + { + "group": "enabled", + "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"], + }, + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + }, + ] + + label_filters = logger._parse_prometheus_config() + + assert logger.enabled_metrics == { + "litellm_spend_metric", + "litellm_total_tokens_metric", + } + assert logger._is_metric_enabled("litellm_spend_metric") is True + assert logger._is_metric_enabled("litellm_cache_hits_metric") is False + assert label_filters["litellm_spend_metric"] == ["team"] + + +def test_wildcard_mixed_with_named_metric_in_same_group_raises(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="broken", + metrics=[PROMETHEUS_METRICS_WILDCARD, "litellm_spend_metric"], + include_labels=["team"], + ) + ] + + with pytest.raises(ValueError, match="mixes the wildcard"): + logger._validate_all_configurations(configs) + + +def test_wildcard_with_unknown_label_is_reported(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="typo", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=["taem"], # typo, should be "team" + ) + ] + + results = logger._validate_all_configurations(configs) + + assert results.has_errors + assert any("taem" in err.message for err in results.label_errors) + + +def test_wildcard_with_no_include_labels_is_a_noop(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="pointless", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=None, + ) + ] + + label_filters = logger._build_label_filters(configs) + + assert label_filters == {} \ No newline at end of file From aa44477008f107cf136c149c4b5793a2da57aeac Mon Sep 17 00:00:00 2001 From: ademicho123 Date: Sat, 25 Jul 2026 21:36:38 +0100 Subject: [PATCH 2/3] fix: apply wildcard label-filter logic to PrometheusLogger, fix test config leak --- litellm/integrations/prometheus.py | 72 ++++++++++++++++--- .../test_prometheus_wildcard_labels.py | 49 +++++++------ 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 64d4dd578b2..da2547a7556 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -669,14 +669,21 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]: parsed_config = group_config parsed_configs.append(parsed_config) - self.enabled_metrics.update(parsed_config.metrics) + + # A wildcard group only sets a default label filter - it should + # never narrow down which metrics get created. Only groups that + # name real metrics feed into enabled_metrics, same as before. + if parsed_config.metrics != [PROMETHEUS_METRICS_WILDCARD]: + self.enabled_metrics.update(parsed_config.metrics) # Validate all configurations validation_results = self._validate_all_configurations(parsed_configs) if validation_results.has_errors: self._pretty_print_validation_errors(validation_results) - error_message = "Configuration validation failed:\n" + "\n".join(validation_results.all_error_messages) + error_message = "Configuration validation failed:\n" + "\n".join( + validation_results.all_error_messages + ) raise ValueError(error_message) # Build label filters from valid configurations @@ -688,20 +695,53 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]: def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults: """Validate all metric configurations and return collected errors""" + from typing import get_args + metric_errors = [] label_errors = [] for config in parsed_configs: + if PROMETHEUS_METRICS_WILDCARD in config.metrics: + if len(config.metrics) > 1: + raise ValueError( + f"Prometheus metrics group '{config.group}' mixes the " + f"wildcard '{PROMETHEUS_METRICS_WILDCARD}' with named " + f"metrics ({config.metrics}). Put named-metric overrides " + "in a separate group - the wildcard group only sets the " + "default label filter, it can't also be scoped." + ) + + if config.include_labels: + all_valid_labels = { + label + for metric_name in get_args(DEFINED_PROMETHEUS_METRICS) + for label in PrometheusMetricLabels.get_labels(metric_name) + } + unknown_labels = [ + label + for label in config.include_labels + if label not in all_valid_labels + ] + if unknown_labels: + label_errors.append( + LabelValidationError( + metric_name=PROMETHEUS_METRICS_WILDCARD, + invalid_labels=unknown_labels, + valid_labels=sorted(all_valid_labels), + ) + ) + continue + for metric_name in config.metrics: - # Validate metric name metric_error = self._validate_single_metric_name(metric_name) if metric_error: metric_errors.append(metric_error) - continue # Skip label validation if metric name is invalid + continue - # Validate labels if provided if config.include_labels: - label_error = self._validate_single_metric_labels(metric_name, config.include_labels) + label_error = self._validate_single_metric_labels( + metric_name, config.include_labels + ) if label_error: label_errors.append(label_error) @@ -738,12 +778,26 @@ def _validate_single_metric_labels(self, metric_name: str, labels: List[str]) -> def _build_label_filters(self, parsed_configs: List) -> Dict[str, List[str]]: """Build label filters from validated configurations""" - label_filters = {} + from typing import get_args - for config in parsed_configs: + label_filters: Dict[str, List[str]] = {} + + wildcard_configs = [ + c for c in parsed_configs if c.metrics == [PROMETHEUS_METRICS_WILDCARD] + ] + named_configs = [ + c for c in parsed_configs if c.metrics != [PROMETHEUS_METRICS_WILDCARD] + ] + + for config in wildcard_configs: + if not config.include_labels: + continue + for metric_name in get_args(DEFINED_PROMETHEUS_METRICS): + label_filters[metric_name] = config.include_labels + + for config in named_configs: for metric_name in config.metrics: if config.include_labels: - # Only add if metric name is valid (validation already passed) if self._validate_single_metric_name(metric_name) is None: label_filters[metric_name] = config.include_labels diff --git a/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py index 9545fd794a3..b99ca75b1ac 100644 --- a/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py +++ b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py @@ -63,17 +63,21 @@ def test_named_group_overrides_wildcard_for_that_metric(): assert label_filters["litellm_total_tokens_metric"] == ["hashed_api_key", "team"] -def test_wildcard_group_does_not_disable_other_metrics(): +def test_wildcard_group_does_not_disable_other_metrics(monkeypatch): """A wildcard-only config should leave every metric enabled - it's a label filter, not an allowlist.""" logger = _bare_logger() - litellm.prometheus_metrics_config = [ - { - "group": "defaults", - "metrics": [PROMETHEUS_METRICS_WILDCARD], - "include_labels": ["team"], - } - ] + monkeypatch.setattr( + litellm, + "prometheus_metrics_config", + [ + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + } + ], + ) logger._parse_prometheus_config() @@ -82,21 +86,25 @@ def test_wildcard_group_does_not_disable_other_metrics(): assert logger._is_metric_enabled("litellm_mcp_tool_calls_total") is True -def test_wildcard_combined_with_named_enable_list(): +def test_wildcard_combined_with_named_enable_list(monkeypatch): """Wildcard for labels + a separate group naming which metrics are enabled - the two concerns are independent.""" logger = _bare_logger() - litellm.prometheus_metrics_config = [ - { - "group": "enabled", - "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"], - }, - { - "group": "defaults", - "metrics": [PROMETHEUS_METRICS_WILDCARD], - "include_labels": ["team"], - }, - ] + monkeypatch.setattr( + litellm, + "prometheus_metrics_config", + [ + { + "group": "enabled", + "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"], + }, + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + }, + ], + ) label_filters = logger._parse_prometheus_config() @@ -108,7 +116,6 @@ def test_wildcard_combined_with_named_enable_list(): assert logger._is_metric_enabled("litellm_cache_hits_metric") is False assert label_filters["litellm_spend_metric"] == ["team"] - def test_wildcard_mixed_with_named_metric_in_same_group_raises(): logger = _bare_logger() configs = [ From 550b47233d3e2c8e22a84d73698056a7c603c0c4 Mon Sep 17 00:00:00 2001 From: ademicho123 Date: Sun, 26 Jul 2026 17:55:24 +0100 Subject: [PATCH 3/3] style: ruff format --- litellm/integrations/prometheus.py | 22 +++++----------------- litellm/types/integrations/prometheus.py | 3 +++ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index da2547a7556..8ce2971ea0d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -681,9 +681,7 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]: if validation_results.has_errors: self._pretty_print_validation_errors(validation_results) - error_message = "Configuration validation failed:\n" + "\n".join( - validation_results.all_error_messages - ) + error_message = "Configuration validation failed:\n" + "\n".join(validation_results.all_error_messages) raise ValueError(error_message) # Build label filters from valid configurations @@ -717,11 +715,7 @@ def _validate_all_configurations(self, parsed_configs: List) -> ValidationResult for metric_name in get_args(DEFINED_PROMETHEUS_METRICS) for label in PrometheusMetricLabels.get_labels(metric_name) } - unknown_labels = [ - label - for label in config.include_labels - if label not in all_valid_labels - ] + unknown_labels = [label for label in config.include_labels if label not in all_valid_labels] if unknown_labels: label_errors.append( LabelValidationError( @@ -739,9 +733,7 @@ def _validate_all_configurations(self, parsed_configs: List) -> ValidationResult continue if config.include_labels: - label_error = self._validate_single_metric_labels( - metric_name, config.include_labels - ) + label_error = self._validate_single_metric_labels(metric_name, config.include_labels) if label_error: label_errors.append(label_error) @@ -782,12 +774,8 @@ def _build_label_filters(self, parsed_configs: List) -> Dict[str, List[str]]: label_filters: Dict[str, List[str]] = {} - wildcard_configs = [ - c for c in parsed_configs if c.metrics == [PROMETHEUS_METRICS_WILDCARD] - ] - named_configs = [ - c for c in parsed_configs if c.metrics != [PROMETHEUS_METRICS_WILDCARD] - ] + wildcard_configs = [c for c in parsed_configs if c.metrics == [PROMETHEUS_METRICS_WILDCARD]] + named_configs = [c for c in parsed_configs if c.metrics != [PROMETHEUS_METRICS_WILDCARD]] for config in wildcard_configs: if not config.include_labels: diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 9ac31e8ac4b..6e37d8a0da5 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -68,8 +68,10 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]: append(ch) return "".join(parts) + PROMETHEUS_METRICS_WILDCARD = "*" + @dataclass class MetricValidationError: """Error for invalid metric name""" @@ -843,6 +845,7 @@ def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: "api_key_hash": "hashed_api_key", } + @dataclass(frozen=True, init=False) class UserAPIKeyLabelValues: """