diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 64d4dd578b2..8ce2971ea0d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -669,7 +669,12 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]: parsed_config = group_config parsed_configs.append(parsed_config) - self.enabled_metrics.update(parsed_config.metrics) + + # A wildcard group only sets a default label filter - it should + # never narrow down which metrics get created. Only groups that + # name real metrics feed into enabled_metrics, same as before. + if parsed_config.metrics != [PROMETHEUS_METRICS_WILDCARD]: + self.enabled_metrics.update(parsed_config.metrics) # Validate all configurations validation_results = self._validate_all_configurations(parsed_configs) @@ -688,18 +693,45 @@ def _parse_prometheus_config(self) -> Dict[str, List[str]]: def _validate_all_configurations(self, parsed_configs: List) -> ValidationResults: """Validate all metric configurations and return collected errors""" + from typing import get_args + metric_errors = [] label_errors = [] for config in parsed_configs: + if PROMETHEUS_METRICS_WILDCARD in config.metrics: + if len(config.metrics) > 1: + raise ValueError( + f"Prometheus metrics group '{config.group}' mixes the " + f"wildcard '{PROMETHEUS_METRICS_WILDCARD}' with named " + f"metrics ({config.metrics}). Put named-metric overrides " + "in a separate group - the wildcard group only sets the " + "default label filter, it can't also be scoped." + ) + + if config.include_labels: + all_valid_labels = { + label + for metric_name in get_args(DEFINED_PROMETHEUS_METRICS) + for label in PrometheusMetricLabels.get_labels(metric_name) + } + unknown_labels = [label for label in config.include_labels if label not in all_valid_labels] + if unknown_labels: + label_errors.append( + LabelValidationError( + metric_name=PROMETHEUS_METRICS_WILDCARD, + invalid_labels=unknown_labels, + valid_labels=sorted(all_valid_labels), + ) + ) + continue + for metric_name in config.metrics: - # Validate metric name metric_error = self._validate_single_metric_name(metric_name) if metric_error: metric_errors.append(metric_error) - continue # Skip label validation if metric name is invalid + continue - # Validate labels if provided if config.include_labels: label_error = self._validate_single_metric_labels(metric_name, config.include_labels) if label_error: @@ -738,12 +770,22 @@ def _validate_single_metric_labels(self, metric_name: str, labels: List[str]) -> def _build_label_filters(self, parsed_configs: List) -> Dict[str, List[str]]: """Build label filters from validated configurations""" - label_filters = {} + from typing import get_args - for config in parsed_configs: + label_filters: Dict[str, List[str]] = {} + + wildcard_configs = [c for c in parsed_configs if c.metrics == [PROMETHEUS_METRICS_WILDCARD]] + named_configs = [c for c in parsed_configs if c.metrics != [PROMETHEUS_METRICS_WILDCARD]] + + for config in wildcard_configs: + if not config.include_labels: + continue + for metric_name in get_args(DEFINED_PROMETHEUS_METRICS): + label_filters[metric_name] = config.include_labels + + for config in named_configs: for metric_name in config.metrics: if config.include_labels: - # Only add if metric name is valid (validation already passed) if self._validate_single_metric_name(metric_name) is None: label_filters[metric_name] = config.include_labels diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 318ba1f5956..6e37d8a0da5 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -69,6 +69,9 @@ def _sanitize_prometheus_label_value(value: Optional[Any]) -> Optional[str]: return "".join(parts) +PROMETHEUS_METRICS_WILDCARD = "*" + + @dataclass class MetricValidationError: """Error for invalid metric name""" @@ -775,7 +778,13 @@ class PrometheusMetricLabels: @staticmethod def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]: - default_labels = getattr(PrometheusMetricLabels, label_name) + # Some entries in DEFINED_PROMETHEUS_METRICS (e.g. litellm_in_flight_requests) + # have no matching class attribute here, because that metric is created + # with a fixed, empty label set and never goes through get_labels_for_metric(). + # Default to no labels instead of raising, so anything that legitimately + # iterates over every defined metric name (e.g. a wildcard config group) + # doesn't blow up on a metric like this. + default_labels = getattr(PrometheusMetricLabels, label_name, []) custom_labels = [] # Add custom metadata labels @@ -933,7 +942,17 @@ def model_dump(self) -> Dict[str, Any]: @dataclass class PrometheusMetricsConfig: - """Configuration for filtering Prometheus metrics (parsed once from proxy config).""" + """ + Configuration for filtering Prometheus metrics (parsed once from proxy config). + + `metrics` is normally a list of names from DEFINED_PROMETHEUS_METRICS. It + can also be `[PROMETHEUS_METRICS_WILDCARD]` ("*") to apply `include_labels` + as the default label set for every metric, instead of listing all of them. + A wildcard group only sets defaults - it never disables a metric, and a + group naming a specific metric still wins over the wildcard for that one + metric. A group can't mix the wildcard with real metric names; use two + groups instead (see PrometheusLogger._build_label_filters). + """ group: str metrics: List[str] diff --git a/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py new file mode 100644 index 00000000000..b99ca75b1ac --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_wildcard_labels.py @@ -0,0 +1,161 @@ +import sys +from typing import get_args + +import pytest + +sys.path.insert(0, "../../../..") + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.integrations.prometheus import ( + DEFINED_PROMETHEUS_METRICS, + PROMETHEUS_METRICS_WILDCARD, + PrometheusMetricsConfig, +) + + +def _bare_logger() -> PrometheusLogger: + """A PrometheusLogger instance with __init__ skipped, for testing the + config-parsing helpers in isolation without re-registering ~70 real + metrics against prometheus_client's global registry on every test.""" + return PrometheusLogger.__new__(PrometheusLogger) + + +def test_wildcard_sets_default_labels_for_every_metric(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="defaults", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=["hashed_api_key", "team"], + ) + ] + + label_filters = logger._build_label_filters(configs) + + assert set(label_filters.keys()) == set(get_args(DEFINED_PROMETHEUS_METRICS)) + for labels in label_filters.values(): + assert labels == ["hashed_api_key", "team"] + + +def test_named_group_overrides_wildcard_for_that_metric(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="defaults", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=["hashed_api_key", "team"], + ), + PrometheusMetricsConfig( + group="spend_needs_more", + metrics=["litellm_spend_metric"], + include_labels=["hashed_api_key", "team", "end_user"], + ), + ] + + label_filters = logger._build_label_filters(configs) + + assert label_filters["litellm_spend_metric"] == [ + "hashed_api_key", + "team", + "end_user", + ] + assert label_filters["litellm_total_tokens_metric"] == ["hashed_api_key", "team"] + + +def test_wildcard_group_does_not_disable_other_metrics(monkeypatch): + """A wildcard-only config should leave every metric enabled - it's a + label filter, not an allowlist.""" + logger = _bare_logger() + monkeypatch.setattr( + litellm, + "prometheus_metrics_config", + [ + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + } + ], + ) + + logger._parse_prometheus_config() + + assert logger.enabled_metrics == set() + assert logger._is_metric_enabled("litellm_spend_metric") is True + assert logger._is_metric_enabled("litellm_mcp_tool_calls_total") is True + + +def test_wildcard_combined_with_named_enable_list(monkeypatch): + """Wildcard for labels + a separate group naming which metrics are + enabled - the two concerns are independent.""" + logger = _bare_logger() + monkeypatch.setattr( + litellm, + "prometheus_metrics_config", + [ + { + "group": "enabled", + "metrics": ["litellm_spend_metric", "litellm_total_tokens_metric"], + }, + { + "group": "defaults", + "metrics": [PROMETHEUS_METRICS_WILDCARD], + "include_labels": ["team"], + }, + ], + ) + + label_filters = logger._parse_prometheus_config() + + assert logger.enabled_metrics == { + "litellm_spend_metric", + "litellm_total_tokens_metric", + } + assert logger._is_metric_enabled("litellm_spend_metric") is True + assert logger._is_metric_enabled("litellm_cache_hits_metric") is False + assert label_filters["litellm_spend_metric"] == ["team"] + +def test_wildcard_mixed_with_named_metric_in_same_group_raises(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="broken", + metrics=[PROMETHEUS_METRICS_WILDCARD, "litellm_spend_metric"], + include_labels=["team"], + ) + ] + + with pytest.raises(ValueError, match="mixes the wildcard"): + logger._validate_all_configurations(configs) + + +def test_wildcard_with_unknown_label_is_reported(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="typo", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=["taem"], # typo, should be "team" + ) + ] + + results = logger._validate_all_configurations(configs) + + assert results.has_errors + assert any("taem" in err.message for err in results.label_errors) + + +def test_wildcard_with_no_include_labels_is_a_noop(): + logger = _bare_logger() + configs = [ + PrometheusMetricsConfig( + group="pointless", + metrics=[PROMETHEUS_METRICS_WILDCARD], + include_labels=None, + ) + ] + + label_filters = logger._build_label_filters(configs) + + assert label_filters == {} \ No newline at end of file