diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 338afe04e5e7..023caf06d12b 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -35,6 +35,15 @@ span orphaned into its own trace). The anchor — a contextvar inherited by thos child tasks — gives a stable parent in both cases. DB/service spans keep ambient parenting so an auth DB lookup still nests under `auth`. +The anchor is also what `litellm.request.route` is read from: `request_root_http_route` +returns the server span's own `http.route`, so the LLM call span cannot disagree with +its parent about which endpoint served the request. That means the route template on a +normal route and the literal path on a passthrough prefix, because the passthrough hook +rewrote the attribute; an MCP call anchors the same server span, so it reports the +`/mcp` mount point. Attributes stay readable after a span ends, so the async close +callback reads the same value. Where no server span was anchored at all, the route the +proxy recorded at auth (`metadata.user_api_key_request_route`) is the backstop. + **Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's service-logging layer instruments many internal functions, but only some are traceable units of work: diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index a550dca6cc89..5519896a961e 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -49,6 +49,7 @@ from litellm.integrations.otel.plumbing.context import ( is_recordable_span, mcp_message_transport_span, + request_root_http_route, request_root_span, resolve_mcp_span_context, resolve_parent_context, @@ -541,6 +542,7 @@ def _finish_carrier( payload, capture_content=self.config.capture_span_content, time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, + request_route=request_root_http_route(), ) end_time_ns: Final = to_ns(end_time) if carrier is not None and carrier.span is not None: diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 3ac92b04c27b..33457f5de164 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -89,6 +89,7 @@ class GenAIMapper: f"{LiteLLM.COST_PREFIX}margin_percent": lambda d: d.cost.margin_percent, f"{LiteLLM.COST_PREFIX}margin_total_amount": lambda d: d.cost.margin_total_amount, LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming, + LiteLLM.REQUEST_ROUTE: lambda d: d.request_route, } _TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = { diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 062b2ca20b46..ee116aca46b7 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -64,6 +64,7 @@ class RequestIdentity: # completes (routing has picked a deployment), so it's absent from the # auth-time seed and filled only from the payload. provider_model: str | None = None + request_route: str | None = None metadata: Mapping[str, str] = field(default_factory=dict) @classmethod @@ -87,6 +88,7 @@ def from_payload(cls, payload: StandardLoggingPayload) -> RequestIdentity: key_hash=as_str(raw_meta.get("user_api_key_hash")), end_user=as_str(payload.get("end_user")) or as_str(raw_meta.get("user_api_key_end_user_id")), provider_model=resolve_provider_model(payload), + request_route=as_str(raw_meta.get("user_api_key_request_route")), metadata=metadata, ) diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index e8ed269f6cb4..d0959a6c2e99 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -386,6 +386,7 @@ class LLMCallSpanData: # keeps routes the convention folds into one operation distinguishable. output_type: GenAIOutputType | None = None call_type: str | None = None + request_route: str | None = None @classmethod def from_standard_logging_payload( @@ -393,6 +394,7 @@ def from_standard_logging_payload( payload: StandardLoggingPayload, capture_content: bool = False, time_to_first_chunk_seconds: float | None = None, + request_route: str | None = None, ) -> LLMCallSpanData: params: Final = cast(Mapping[str, object], payload.get("model_parameters") or {}) # The single parse of the request's metadata — the request-vs-provider @@ -433,6 +435,7 @@ def from_standard_logging_payload( time_to_first_chunk_seconds=time_to_first_chunk_seconds, output_type=resolve_output_type(call_type), call_type=call_type or None, + request_route=request_route or context.identity.request_route, ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index f7a6280f95b3..af5327cbd41a 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -295,6 +295,7 @@ class LiteLLM: # ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``. PROVIDER_MODEL: Final = "litellm.provider.model" REQUEST_STREAMING: Final = "litellm.request.streaming" + REQUEST_ROUTE: Final = "litellm.request.route" TOOLS_DECLARED: Final = "litellm.request.tools.declared" GUARDRAIL_NAME: Final = "litellm.guardrail.name" GUARDRAIL_MODE: Final = "litellm.guardrail.mode" diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index 159a84b121f4..aa7cc8e2afd2 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -6,6 +6,7 @@ from opentelemetry import baggage from opentelemetry.context import Context, get_current +from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.trace import ( Link, NonRecordingSpan, @@ -18,6 +19,8 @@ TraceContextTextMapPropagator, ) +from litellm.integrations.otel.model.semconv import HTTP + _PROPAGATOR: Final = TraceContextTextMapPropagator() # The request's root span — the FastAPI-owned SERVER span — captured ONCE when the @@ -55,6 +58,25 @@ def request_root_span() -> "Span | None": return span if is_recordable_span(span) else None +def request_root_http_route() -> str | None: + """``http.route`` exactly as the request's root SERVER span reports it. + + Read off the span rather than re-derived, so the LLM call span cannot disagree + with its own parent about which endpoint served the request: the template the + instrumentation matched, or the literal path where + ``mount._passthrough_span_name_hook`` rewrote it, are already in the attribute. + An MCP call anchors that same server span, so it reports the ``/mcp`` mount + point the instrumentation matched. Attributes stay readable after a span ends, + so this answers just as well from the async logging callback. + + None when no server span is anchored, which is the SDK path and any deployment + where the FastAPI instrumentation did not mount. + """ + span: Final = request_root_span() + route: Final = span.attributes.get(HTTP.ROUTE) if isinstance(span, ReadableSpan) and span.attributes else None + return route if isinstance(route, str) and route else None + + # The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the # MCP client propagated in the current request's ``params._meta``. The MCP gateway # sets it per message so the MCP span can record the client's span as a span diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 4973bda29e09..b735abaf7bf1 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -188,6 +188,62 @@ def test_streaming_span_carries_time_to_first_chunk(): assert span.attributes[GenAI.RESPONSE_TIME_TO_FIRST_CHUNK] == pytest.approx(0.75) +def test_llm_call_span_reports_the_server_spans_route(): + """``litellm.request.route`` is the anchored server span's own ``http.route``, + so an operator can group LLM spans by endpoint without joining to the parent.""" + logger, exporter = _logger() + root = logger.tracer.start_span("POST /engines/{model:path}/chat/completions") + root.set_attribute("http.route", "/engines/{model:path}/chat/completions") + set_request_root_span(root) + + _emit_llm(logger, ambient=root) + root.end() + + llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT) + assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/engines/{model:path}/chat/completions" + + +def test_llm_call_span_omits_the_route_without_a_server_span(): + """An SDK call has no server span, so the key is absent rather than empty.""" + logger, exporter = _logger() + _emit_llm(logger) + (span,) = exporter.get_finished_spans() + assert LiteLLM.REQUEST_ROUTE not in span.attributes + + +def test_failed_llm_call_span_reports_the_server_spans_route(): + """The failure leg builds the same span data, so an errored call is still + attributable to the endpoint it came in on.""" + logger, exporter = _logger() + root = logger.tracer.start_span("POST /v1/responses/{response_id}") + root.set_attribute("http.route", "/v1/responses/{response_id}") + set_request_root_span(root) + + _emit_llm(logger, ambient=root, fail=True) + root.end() + + llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT) + assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/v1/responses/{response_id}" + + +def test_deferred_llm_call_span_reports_the_server_spans_route(): + """``pre_call`` driven from a thread pool sees no recordable parent, so the span + is created in the close callback instead. That branch has to carry the route + too, and it can: the worker context still holds the anchor.""" + logger, exporter = _logger() + root = logger.tracer.start_span("POST /v1/messages") + root.set_attribute("http.route", "/v1/messages") + set_request_root_span(root) + + # no ``ambient``: pre_call runs with no recordable span active, which is what + # defers creation to the close callback + _emit_llm(logger) + root.end() + + llm_span = next(s for s in exporter.get_finished_spans() if s.kind is SpanKind.CLIENT) + assert llm_span.attributes[LiteLLM.REQUEST_ROUTE] == "/v1/messages" + + def test_non_streaming_span_has_no_time_to_first_chunk(): logger, exporter = _logger() kwargs = { diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py index 0cd71db4ae1f..e2007486a403 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py @@ -5,6 +5,8 @@ """ +from datetime import datetime, timezone + import pytest @@ -18,6 +20,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, ) +from opentelemetry import trace # noqa: E402 from opentelemetry.trace import SpanKind # noqa: E402 from litellm.integrations.otel.model.config import ( # noqa: E402 @@ -30,6 +33,23 @@ _passthrough_span_name_hook, instrument_fastapi_app, ) +from litellm.integrations.otel.plumbing.context import ( # noqa: E402 + request_root_http_route, + set_request_root_span, +) + + +@pytest.fixture(autouse=True) +def _reset_request_root_span(): + """Clear the root-span anchor around every test. Production gets a fresh + contextvar copy per request task; the test process shares one context.""" + from litellm.integrations.otel.plumbing import context as _otel_context + + _otel_context._request_root_span.set(None) + _otel_context._mcp_message_transport_span.set(None) + yield + _otel_context._request_root_span.set(None) + _otel_context._mcp_message_transport_span.set(None) @pytest.fixture(autouse=True) @@ -128,6 +148,85 @@ def test_passthrough_hook_ignores_non_recording_span(): assert span.name is None +def test_llm_span_route_is_read_off_the_server_span(monkeypatch): + """``request_root_http_route`` answers with the SERVER span's own ``http.route``. + + Driven through ``instrument_fastapi_app`` and the same + ``create_litellm_proxy_request_started_span`` call the proxy makes per request, + so breaking either the mount or the anchor capture fails this.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "1") + is_otel_v2_enabled.cache_clear() + app = fastapi.FastAPI() + seen = {} + + def _anchor_then_read(key): + logger.create_litellm_proxy_request_started_span(start_time=datetime.now(timezone.utc), headers=None) + seen[key] = request_root_http_route() + + @app.post("/engines/{model:path}/chat/completions") + async def engines(model: str): + _anchor_then_read("templated") + return {} + + @app.post("/openai/{endpoint:path}") + async def openai_passthrough(endpoint: str): + _anchor_then_read("passthrough") + return {} + + logger = OpenTelemetryV2(config=OpenTelemetryV2Config(exporter="in_memory")) + exporter = InMemorySpanExporter() + logger._tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + # instrument_fastapi_app passes no provider, so it binds to the OTel global the + # way the proxy does once proxy_startup_event publishes one. set_tracer_provider + # is a once-per-process door, so place it directly and let monkeypatch undo it. + monkeypatch.setattr(trace, "_TRACER_PROVIDER", logger._tracer_provider) + instrument_fastapi_app(app) + + client = TestClient(app) + client.post("/engines/gpt-4o-mini/chat/completions") + client.post("/openai/v1/responses/resp_abc123") + + routes = { + (s.attributes or {})["http.route"] for s in exporter.get_finished_spans() if s.kind is SpanKind.SERVER + } + # a parameterized route keeps its template; the passthrough hook rewrote the + # catch-all to the literal path, and both spans have to follow their own span + assert routes == {"/engines/{model:path}/chat/completions", "/openai/v1/responses/resp_abc123"} + assert seen["templated"] == "/engines/{model:path}/chat/completions" + assert seen["passthrough"] == "/openai/v1/responses/resp_abc123" + + +def test_server_span_route_survives_the_span_ending(): + """The LLM span closes in an async callback that can run after the server span + has ended, so the attribute has to still be readable then.""" + from opentelemetry.sdk.trace import TracerProvider + + span = TracerProvider().get_tracer("t").start_span("POST /v1/responses/{response_id}") + span.set_attribute("http.route", "/v1/responses/{response_id}") + set_request_root_span(span) + span.end() + + assert request_root_http_route() == "/v1/responses/{response_id}" + + +def test_no_server_span_means_no_route(): + """An SDK call has no anchored server span, so the attribute is omitted rather + than reported as empty.""" + assert request_root_http_route() is None + + +def test_blank_route_on_the_server_span_is_omitted(): + """An excluded or unmatched path leaves the server span without a usable route. + Report nothing rather than a span attribute whose value is the empty string.""" + from opentelemetry.sdk.trace import TracerProvider + + span = TracerProvider().get_tracer("t").start_span("GET") + span.set_attribute("http.route", "") + set_request_root_span(span) + + assert request_root_http_route() is None + + def test_known_passthrough_prefixes_present(): """Guard the prefix set against accidental edits.""" assert {"openai", "anthropic", "vertex_ai", "bedrock"} <= PASSTHROUGH_PREFIXES diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 99d706a9c44d..addadf8e5984 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -722,6 +722,39 @@ def test_request_identity_falls_back_to_legacy_team_keys(): assert ident.team_alias == "legacy" +def test_llm_span_carries_proxy_request_route(): + """The LLM span records the proxy route the request arrived on, so it can be + filtered by endpoint (``/v1/responses`` vs ``/v1/chat/completions``) without + joining back to the root SERVER span's ``http.route``. The value is that + span's ``http.route`` verbatim, so a parameterized route reports the template + the SERVER span reports and not the path the caller happened to send.""" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(metadata={"user_api_key_request_route": "/v1/responses/resp_abc123"}), + request_route="/v1/responses/{response_id}", + ) + attrs: Final = GenAIMapper().map(data) + + assert attrs[LiteLLM.REQUEST_ROUTE] == "/v1/responses/{response_id}" + + +def test_llm_span_falls_back_to_the_logged_route_without_a_server_span(): + """The route the proxy recorded at auth is the backstop for a deployment whose + FastAPI instrumentation never mounted: there is no server span to disagree with + there, and an endpoint name is worth more than an absent attribute.""" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(metadata={"user_api_key_request_route": "/v1/responses"}) + ) + + assert GenAIMapper().map(data)[LiteLLM.REQUEST_ROUTE] == "/v1/responses" + + +def test_llm_span_omits_request_route_off_the_proxy(): + """An SDK call has no inbound route, so the key is absent rather than empty.""" + attrs: Final = GenAIMapper().map(LLMCallSpanData.from_standard_logging_payload(_sample_payload(metadata={}))) + + assert LiteLLM.REQUEST_ROUTE not in attrs + + def test_guardrail_span_data_block_carries_verdict_and_error(): from litellm.integrations.otel.model.payloads import GuardrailSpanData