feat(prometheus): support wildcard metric name in prometheus_metrics_…

litellm/integrations/prometheus.py

@@ -669,7 +669,12 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]:
                 parsed_config = group_config
 
             parsed_configs.append(parsed_config)
-            self.enabled_metrics.update(parsed_config.metrics)
+
+            # A wildcard group only sets a default label filter - it should
+            # never narrow down which metrics get created. Only groups that
+            # name real metrics feed into enabled_metrics, same as before.
+            if parsed_config.metrics != [PROMETHEUS_METRICS_WILDCARD]:
+                self.enabled_metrics.update(parsed_config.metrics)
 
         # Validate all configurations
         validation_results = self._validate_all_configurations(parsed_configs)
@@ -688,18 +693,45 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]:
 
     def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults:
         """Validate all metric configurations and return collected errors"""
+        from typing import get_args
+
         metric_errors = []
         label_errors = []
 
         for config in parsed_configs:
+            if PROMETHEUS_METRICS_WILDCARD in config.metrics:
+                if len(config.metrics) > 1:
+                    raise ValueError(
+                        f"Prometheus metrics group '{config.group}' mixes the "
+                        f"wildcard '{PROMETHEUS_METRICS_WILDCARD}' with named "
+                        f"metrics ({config.metrics}). Put named-metric overrides "
+                        "in a separate group - the wildcard group only sets the "
+                        "default label filter, it can't also be scoped."
+                    )
+
+                if config.include_labels:
+                    all_valid_labels = {
+                        label
+                        for metric_name in get_args(DEFINED_PROMETHEUS_METRICS)
+                        for label in PrometheusMetricLabels.get_labels(metric_name)
+                    }
+                    unknown_labels = [label for label in config.include_labels if label not in all_valid_labels]
+                    if unknown_labels:
+                        label_errors.append(
+                            LabelValidationError(
+                                metric_name=PROMETHEUS_METRICS_WILDCARD,
+                                invalid_labels=unknown_labels,
+                                valid_labels=sorted(all_valid_labels),
+                            )
+                        )
+                continue
+
             for metric_name in config.metrics:
-                # Validate metric name
                 metric_error = self._validate_single_metric_name(metric_name)
                 if metric_error:
                     metric_errors.append(metric_error)
-                    continue  # Skip label validation if metric name is invalid
+                    continue
 
-                # Validate labels if provided
                 if config.include_labels:
                     label_error = self._validate_single_metric_labels(metric_name, config.include_labels)
                     if label_error:
@@ -738,12 +770,22 @@ def _validate_single_metric_labels(self, metric_name: str, labels: List[str]) ->
 
     def _build_label_filters(self, parsed_configs: List) -> Dict[str, List[str]]:
         """Build label filters from validated configurations"""
-        label_filters = {}
+        from typing import get_args
 
-        for config in parsed_configs:
+        label_filters: Dict[str, List[str]] = {}
+
+        wildcard_configs = [c for c in parsed_configs if c.metrics == [PROMETHEUS_METRICS_WILDCARD]]
+        named_configs = [c for c in parsed_configs if c.metrics != [PROMETHEUS_METRICS_WILDCARD]]
+
+        for config in wildcard_configs:
+            if not config.include_labels:
+                continue
+            for metric_name in get_args(DEFINED_PROMETHEUS_METRICS):
+                label_filters[metric_name] = config.include_labels
+
+        for config in named_configs:
             for metric_name in config.metrics:
                 if config.include_labels:
-                    # Only add if metric name is valid (validation already passed)
                     if self._validate_single_metric_name(metric_name) is None:
                         label_filters[metric_name] = config.include_labels
 

litellm/types/integrations/prometheus.py

@@ -69,6 +69,9 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]:
     return "".join(parts)
 
 
+PROMETHEUS_METRICS_WILDCARD = "*"
+
+
 @dataclass
 class MetricValidationError:
     """Error for invalid metric name"""
@@ -775,7 +778,13 @@ class PrometheusMetricLabels:
 
     @staticmethod
     def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]:
-        default_labels = getattr(PrometheusMetricLabels, label_name)
+        # Some entries in DEFINED_PROMETHEUS_METRICS (e.g. litellm_in_flight_requests)
+        # have no matching class attribute here, because that metric is created
+        # with a fixed, empty label set and never goes through get_labels_for_metric().
+        # Default to no labels instead of raising, so anything that legitimately
+        # iterates over every defined metric name (e.g. a wildcard config group)
+        # doesn't blow up on a metric like this.
+        default_labels = getattr(PrometheusMetricLabels, label_name, [])
         custom_labels = []
 
         # Add custom metadata labels
@@ -933,7 +942,17 @@ def model_dump(self) -> Dict[str, Any]:
 
 @dataclass
 class PrometheusMetricsConfig:
-    """Configuration for filtering Prometheus metrics (parsed once from proxy config)."""
+    """
+    Configuration for filtering Prometheus metrics (parsed once from proxy config).
+
+    `metrics` is normally a list of names from DEFINED_PROMETHEUS_METRICS. It
+    can also be `[PROMETHEUS_METRICS_WILDCARD]` ("*") to apply `include_labels`
+    as the default label set for every metric, instead of listing all of them.
+    A wildcard group only sets defaults - it never disables a metric, and a
+    group naming a specific metric still wins over the wildcard for that one
+    metric. A group can't mix the wildcard with real metric names; use two
+    groups instead (see PrometheusLogger._build_label_filters).
+    """
 
     group: str
     metrics: List[str]

tests/test_litellm/integrations/test_prometheus_wildcard_labels.py

@@ -0,0 +1,161 @@
+import sys
+from typing import get_args
+
+import pytest
+
+sys.path.insert(0, "../../../..")
+
+import litellm
+from litellm.integrations.prometheus import PrometheusLogger
+from litellm.types.integrations.prometheus import (
+    DEFINED_PROMETHEUS_METRICS,
+    PROMETHEUS_METRICS_WILDCARD,
+    PrometheusMetricsConfig,
+)
+
+
+def _bare_logger() -> PrometheusLogger:
+    """A PrometheusLogger instance with __init__ skipped, for testing the
+    config-parsing helpers in isolation without re-registering ~70 real
+    metrics against prometheus_client's global registry on every test."""
+    return PrometheusLogger.__new__(PrometheusLogger)
+
+
+def test_wildcard_sets_default_labels_for_every_metric():
+    logger = _bare_logger()
+    configs = [
+        PrometheusMetricsConfig(
+            group="defaults",
+            metrics=[PROMETHEUS_METRICS_WILDCARD],
+            include_labels=["hashed_api_key", "team"],
+        )
+    ]
+
+    label_filters = logger._build_label_filters(configs)
+
+    assert set(label_filters.keys()) == set(get_args(DEFINED_PROMETHEUS_METRICS))
+    for labels in label_filters.values():
+        assert labels == ["hashed_api_key", "team"]
+
+
+def test_named_group_overrides_wildcard_for_that_metric():
+    logger = _bare_logger()
+    configs = [
+        PrometheusMetricsConfig(
+            group="defaults",
+            metrics=[PROMETHEUS_METRICS_WILDCARD],
+            include_labels=["hashed_api_key", "team"],
+        ),
+        PrometheusMetricsConfig(
+            group="spend_needs_more",
+            metrics=["litellm_spend_metric"],
+            include_labels=["hashed_api_key", "team", "end_user"],
+        ),
+    ]
+
+    label_filters = logger._build_label_filters(configs)
+
+    assert label_filters["litellm_spend_metric"] == [
+        "hashed_api_key",
+        "team",
+        "end_user",
+    ]
+    assert label_filters["litellm_total_tokens_metric"] == ["hashed_api_key", "team"]
+
+
+def test_wildcard_group_does_not_disable_other_metrics(monkeypatch):
+    """A wildcard-only config should leave every metric enabled - it's a
+    label filter, not an allowlist."""
+    logger = _bare_logger()
+    monkeypatch.setattr(
+        litellm,
+        "prometheus_metrics_config",
+        [
+            {
+                "group": "defaults",
+                "metrics": [PROMETHEUS_METRICS_WILDCARD],
+                "include_labels": ["team"],
+            }
+        ],
+    )
+
+    logger._parse_prometheus_config()
+
+    assert logger.enabled_metrics == set()
+    assert logger._is_metric_enabled("litellm_spend_metric") is True
+    assert logger._is_metric_enabled("litellm_mcp_tool_calls_total") is True
+
+
+def test_wildcard_combined_with_named_enable_list(monkeypatch):
+    """Wildcard for labels + a separate group naming which metrics are
+    enabled - the two concerns are independent."""
+    logger = _bare_logger()
+    monkeypatch.setattr(
+        litellm,
+        "prometheus_metrics_config",
+        [
+            {
+                "group": "enabled",
+                "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"],
+            },
+            {
+                "group": "defaults",
+                "metrics": [PROMETHEUS_METRICS_WILDCARD],
+                "include_labels": ["team"],
+            },
+        ],
+    )
+
+    label_filters = logger._parse_prometheus_config()
+
+    assert logger.enabled_metrics == {
+        "litellm_spend_metric",
+        "litellm_total_tokens_metric",
+    }
+    assert logger._is_metric_enabled("litellm_spend_metric") is True
+    assert logger._is_metric_enabled("litellm_cache_hits_metric") is False
+    assert label_filters["litellm_spend_metric"] == ["team"]
+
+def test_wildcard_mixed_with_named_metric_in_same_group_raises():
+    logger = _bare_logger()
+    configs = [
+        PrometheusMetricsConfig(
+            group="broken",
+            metrics=[PROMETHEUS_METRICS_WILDCARD, "litellm_spend_metric"],
+            include_labels=["team"],
+        )
+    ]
+
+    with pytest.raises(ValueError, match="mixes the wildcard"):
+        logger._validate_all_configurations(configs)
+
+
+def test_wildcard_with_unknown_label_is_reported():
+    logger = _bare_logger()
+    configs = [
+        PrometheusMetricsConfig(
+            group="typo",
+            metrics=[PROMETHEUS_METRICS_WILDCARD],
+            include_labels=["taem"],  # typo, should be "team"
+        )
+    ]
+
+    results = logger._validate_all_configurations(configs)
+
+    assert results.has_errors
+    assert any("taem" in err.message for err in results.label_errors)
+
+
+def test_wildcard_with_no_include_labels_is_a_noop():
+    logger = _bare_logger()
+    configs = [
+        PrometheusMetricsConfig(
+            group="pointless",
+            metrics=[PROMETHEUS_METRICS_WILDCARD],
+            include_labels=None,
+        )
+    ]
+
+    label_filters = logger._build_label_filters(configs)
+
+    assert label_filters == {}