diff --git a/Directory.Packages.props b/Directory.Packages.props index f37bd594e4c..9c25c8b1d13 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -26,6 +26,7 @@ + diff --git a/cspell.json b/cspell.json index d03eb57d0a5..02982d55c59 100644 --- a/cspell.json +++ b/cspell.json @@ -73,6 +73,7 @@ "ndjson", "netcoreapp", "newtonsoft", + "nosuspend", "Notfication", "NTLM", "nupkg", diff --git a/documentation/configuration/README.md b/documentation/configuration/README.md index 36d6eab747f..77189bc3d8b 100644 --- a/documentation/configuration/README.md +++ b/documentation/configuration/README.md @@ -17,6 +17,7 @@ - **[Metrics Configuration](./metrics-configuration.md)** - Configuration of the `/metrics` endpoint for live metrics collection - **[Egress Configuration](./egress-configuration.md)** - When `dotnet-monitor` is used to produce artifacts such as dumps or traces, an egress provider enables the artifacts to be stored in a manner suitable for the hosting environment rather than streamed back directly.] - **[In-Process Features Configuration](./in-process-features-configuration.md)** - Some features of `dotnet monitor` require loading libraries into target applications that may have performance impact on memory and CPU utilization +- **[OpenTelemetry Configuration](./opentelemetry-configuration.md)** - Configure out-of-process telemetry forwarding of logs, metrics, and traces to any OTLP-compatible backend - **[Garbage Collector Mode](#garbage-collector-mode)** - Configure which GC mode is used by the `dotnet monitor` process. ## Kestrel Configuration diff --git a/documentation/configuration/opentelemetry-configuration.md b/documentation/configuration/opentelemetry-configuration.md new file mode 100644 index 00000000000..b318aefba9a --- /dev/null +++ b/documentation/configuration/opentelemetry-configuration.md @@ -0,0 +1,384 @@ +# OpenTelemetry Configuration + +`dotnet monitor` can collect diagnostics from a target .NET application and forward them as OpenTelemetry signals (logs, metrics, and traces) to any OTLP-compatible backend. This enables out-of-process telemetry collection without requiring any changes to the target application. + +> [!NOTE] +> OpenTelemetry forwarding works in both `Connect` and `Listen` diagnostic port connection modes. See [Diagnostic Port Configuration](./diagnostic-port-configuration.md) for details on each mode. + +## Overview + +When OpenTelemetry is configured, `dotnet monitor` connects to a target process via EventPipe and subscribes to telemetry event streams. It converts the incoming data into OpenTelemetry SDK objects and exports them via OTLP to a configured backend (e.g., Jaeger, Grafana, Datadog, or any OpenTelemetry Collector). + +The three supported signal types are: + +- **Logs** — Forwarded from the target application's `ILogger` output +- **Metrics** — Collected from .NET `Meter` and `EventCounter` instruments +- **Traces** — Collected from `System.Diagnostics.ActivitySource` activity data + +## Prerequisites + +- A [Default Process](./default-process-configuration.md) must be configured so `dotnet monitor` knows which process to collect from. +- An OTLP-compatible backend or collector must be reachable from `dotnet monitor`. + +## Exporter Configuration + +The exporter defines where telemetry data is sent. Currently, only the OpenTelemetry Protocol (OTLP) exporter is supported. + +```json +{ + "OpenTelemetry": { + "Exporter": { + "Type": "otlp", + "Settings": { + "Defaults": { + "Protocol": "HttpProtobuf", + "BaseUrl": "http://localhost:4318" + } + } + } + } +} +``` + +| Property | Type | Description | +|---|---|---| +| `Exporter.Type` | string | The exporter type. Currently only `"otlp"` is supported. | +| `Exporter.Settings.Defaults.Protocol` | string | The OTLP protocol to use (e.g., `"HttpProtobuf"`). | +| `Exporter.Settings.Defaults.BaseUrl` | string | The base URL of the OTLP endpoint. Also accepts `Url` as an alternative key. | +| `Exporter.Settings.Defaults.Headers` | object | Optional dictionary of headers to send with OTLP export requests. | + +Per-signal endpoint overrides can be configured under `Settings.Logs`, `Settings.Metrics`, and `Settings.Traces`: + +```json +{ + "OpenTelemetry": { + "Exporter": { + "Type": "otlp", + "Settings": { + "Defaults": { + "BaseUrl": "http://localhost:4318" + }, + "Logs": { + "Url": "http://logs-collector:4318/v1/logs" + } + } + } + } +} +``` + +## Logs Configuration + +Configure log forwarding using the `Logs` section. Categories are specified as a dictionary where the key is the category prefix and the value is the log level. Use the `Default` key to set the default log level. + +```json +{ + "OpenTelemetry": { + "Logs": { + "Categories": { + "Default": "Information", + "Microsoft": "Warning", + "System.Net.Http": "Debug" + }, + "IncludeScopes": true, + "Batch": { + "ExportIntervalMilliseconds": 5000, + "MaxQueueSize": 2048, + "MaxExportBatchSize": 512 + } + } + } +} +``` + +| Property | Type | Description | +|---|---|---| +| `Categories` | dictionary | A dictionary of category prefixes to log levels. The special key `"Default"` sets the default log level for all categories. Valid log levels: `Trace`, `Debug`, `Information`, `Warning`, `Error`, `Critical`. Defaults to `Warning` if not specified or invalid. | +| `IncludeScopes` | bool | Whether to include logging scopes in exported log records. Default: `false`. | +| `Batch` | object | Optional batch export settings. See [Batch Options](#batch-options). | + +## Metrics Configuration + +Configure which meters and instruments to collect using the `Metrics` section. Meters are specified as a dictionary where the key is the meter name and the value is an array of instrument names (use an empty array to collect all instruments from a meter). + +```json +{ + "OpenTelemetry": { + "Metrics": { + "Meters": { + "System.Runtime": [], + "Microsoft.AspNetCore.Hosting": [ + "http.server.request.duration", + "http.server.active_requests" + ] + }, + "PeriodicExporting": { + "ExportIntervalMilliseconds": 10000 + }, + "MaxHistograms": 2000, + "MaxTimeSeries": 1000 + } + } +} +``` + +| Property | Type | Description | +|---|---|---| +| `Meters` | dictionary | A dictionary where keys are meter names and values are arrays of instrument names to collect. An empty array (`[]`) collects all instruments from that meter. | +| `PeriodicExporting.ExportIntervalMilliseconds` | int | How often (in milliseconds) to export collected metrics. Default: `60000`. | +| `PeriodicExporting.ExportTimeoutMilliseconds` | int | Timeout for each export attempt. Default: `30000`. | +| `PeriodicExporting.AggregationTemporalityPreference` | string | Aggregation temporality: `"Cumulative"` or `"Delta"`. Default: `"Cumulative"`. | +| `MaxHistograms` | int | Maximum number of histogram aggregations to track. Default: `10`. | +| `MaxTimeSeries` | int | Maximum number of time series (unique dimension combinations) to track per metric. Default: `1000`. | + +## Tracing Configuration + +Configure which activity sources to trace and how sampling is applied using the `Traces` section. + +```json +{ + "OpenTelemetry": { + "Traces": { + "Sources": [ + "System.Net.Http", + "Microsoft.AspNetCore" + ], + "Sampler": { + "Type": "ParentBased", + "Settings": { + "RootSampler": { + "Type": "TraceIdRatio", + "Settings": { + "SamplingRatio": 0.5 + } + } + } + }, + "Batch": { + "ExportIntervalMilliseconds": 5000 + } + } + } +} +``` + +The `Sampler` also supports a shorthand format — set it to a double value to use a `ParentBased` sampler with a `TraceIdRatio` root sampler at the specified ratio: + +```json +{ + "OpenTelemetry": { + "Traces": { + "Sources": ["Microsoft.AspNetCore"], + "Sampler": 1.0 + } + } +} +``` + +| Property | Type | Description | +|---|---|---| +| `Sources` | array | List of `ActivitySource` names to collect traces from. | +| `Sampler` | object or double | Sampling configuration. Use a double (e.g., `1.0`) as shorthand for `ParentBased` with `TraceIdRatio`. Or use the object format with `Type` and `Settings` properties. Supported types: `"ParentBased"`, `"TraceIdRatio"`. | +| `Batch` | object | Optional batch export settings. See [Batch Options](#batch-options). | + +## Batch Options + +The `Batch` section is available for Logs and Traces. It controls how telemetry records are batched before export. + +| Property | Type | Default | Description | +|---|---|---|---| +| `MaxQueueSize` | int | `2048` | Maximum number of items in the export queue. | +| `MaxExportBatchSize` | int | `512` | Maximum number of items per export batch. | +| `ExportIntervalMilliseconds` | int | `5000` | How often (in milliseconds) to export a batch. | +| `ExportTimeoutMilliseconds` | int | `30000` | Timeout for each export attempt. | + +## Resource Configuration + +Configure resource attributes that are attached to all exported telemetry. Values can reference environment variables from the target process using `${env:VARIABLE_NAME}` syntax. + +```json +{ + "OpenTelemetry": { + "Resource": { + "service.name": "my-service", + "service.version": "1.0.0", + "deployment.environment": "${env:ASPNETCORE_ENVIRONMENT}" + } + } +} +``` + +> [!NOTE] +> The `service.name` and `service.instance.id` attributes are automatically populated from the target process name and process ID if not explicitly set. + +## Using Connect Mode + +In `Connect` mode (the default), `dotnet monitor` connects to a running process's diagnostic port. No special diagnostic port configuration is needed—just ensure the target process is running and a [Default Process](./default-process-configuration.md) filter is configured. + +```json +{ + "DefaultProcess": { + "Filters": [{ + "ProcessName": "MyApp" + }] + }, + "OpenTelemetry": { + "Exporter": { + "Type": "otlp", + "Settings": { + "Defaults": { + "BaseUrl": "http://localhost:4318" + } + } + }, + "Metrics": { + "Meters": { + "System.Runtime": [] + } + } + } +} +``` + +Start `dotnet monitor`: + +```bash +dotnet monitor collect +``` + +> [!NOTE] +> In `Connect` mode, `dotnet monitor` attaches to the target process after it has started. Telemetry events that occur during process startup will not be captured. + +## Using Listen Mode + +In `Listen` mode, `dotnet monitor` creates a diagnostic port endpoint and waits for target processes to connect. This enables capturing telemetry from the very start of a process, including assembly load events and early traces. + +```json +{ + "DiagnosticPort": { + "ConnectionMode": "Listen", + "EndpointName": "/diag/port.sock" + }, + "DefaultProcess": { + "Filters": [{ + "ProcessName": "MyApp" + }] + }, + "OpenTelemetry": { + "Exporter": { + "Type": "otlp", + "Settings": { + "Defaults": { + "BaseUrl": "http://localhost:4318" + } + } + }, + "Logs": { + "Categories": { + "Default": "Information" + } + }, + "Metrics": { + "Meters": { + "System.Runtime": [] + } + }, + "Traces": { + "Sources": ["System.Net.Http"] + } + } +} +``` + +Start `dotnet monitor`: + +```bash +dotnet monitor collect +``` + +Configure the target .NET application to connect to `dotnet monitor`: + +```bash +export DOTNET_DiagnosticPorts="/diag/port.sock,suspend" +``` + +> [!IMPORTANT] +> In `Listen` mode with `suspend`, the target process will pause at startup until `dotnet monitor` connects and resumes it. Use `nosuspend` instead if the process should start immediately regardless of whether `dotnet monitor` is available: +> ```bash +> export DOTNET_DiagnosticPorts="/diag/port.sock,nosuspend" +> ``` + +See [Diagnostic Port Configuration](./diagnostic-port-configuration.md) for more details on connection modes. + +## Full Example + +The following shows a complete configuration that collects all three signal types and exports them to an OpenTelemetry Collector running locally: + +```json +{ + "DiagnosticPort": { + "ConnectionMode": "Listen", + "EndpointName": "/diag/port.sock" + }, + "DefaultProcess": { + "Filters": [{ + "ProcessName": "MyApp" + }] + }, + "OpenTelemetry": { + "Exporter": { + "Type": "otlp", + "Settings": { + "Defaults": { + "Protocol": "HttpProtobuf", + "BaseUrl": "http://localhost:4318" + } + } + }, + "Logs": { + "Categories": { + "Default": "Warning", + "MyApp": "Information" + }, + "IncludeScopes": true, + "Batch": { + "ExportIntervalMilliseconds": 5000 + } + }, + "Metrics": { + "Meters": { + "System.Runtime": [], + "Microsoft.AspNetCore.Hosting": ["http.server.request.duration"] + }, + "PeriodicExporting": { + "ExportIntervalMilliseconds": 10000 + }, + "MaxHistograms": 2000, + "MaxTimeSeries": 1000 + }, + "Traces": { + "Sources": [ + "System.Net.Http", + "Microsoft.AspNetCore" + ], + "Sampler": 1.0, + "Batch": { + "ExportIntervalMilliseconds": 5000 + } + } + } +} +``` + +## Validation + +At least one signal must be configured for OpenTelemetry forwarding to start: + +- **Logs** — Set a `Default` entry in `Categories` or provide other category entries +- **Metrics** — Provide at least one entry in `Meters` +- **Traces** — Provide at least one entry in `Sources` + +If none of these are configured, `dotnet monitor` will log a warning and wait for a configuration change. + +> [!NOTE] +> OpenTelemetry configuration supports dynamic reloading. Changes to the configuration are applied without restarting `dotnet monitor`. diff --git a/generate-dev-sln.ps1 b/generate-dev-sln.ps1 index fec6c0823db..ed3e529ccf6 100644 --- a/generate-dev-sln.ps1 +++ b/generate-dev-sln.ps1 @@ -10,6 +10,7 @@ $ErrorActionPreference = 'Stop' $resolvedPath = Resolve-Path $DiagRepoRoot $env:DIAGNOSTICS_REPO_ROOT=$resolvedPath +$env:ENABLE_OTEL="true" #Generates a solution that spans both the diagnostics and the dotnet-monitor repo. #This can be used to build both projects in VS. @@ -37,4 +38,4 @@ $slnFile = Get-Content $devSln #dotnet sln uses an older ProjectType Guid $slnFile -replace 'FAE04EC0-301F-11D3-BF4B-00C04F79EFBC', '9A19103F-16F7-4668-BE54-9A1E7A4F7556' | Out-File $devSln -& "$PSScriptRoot\startvs.cmd" $PSScriptRoot\dotnet-monitor.dev.sln \ No newline at end of file +& "$PSScriptRoot\startvs.cmd" $PSScriptRoot\dotnet-monitor.dev.sln diff --git a/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/IMetricsStore.cs b/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/IMetricsStore.cs index 868e270a44e..f1ee1fe4890 100644 --- a/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/IMetricsStore.cs +++ b/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/IMetricsStore.cs @@ -3,6 +3,7 @@ using Microsoft.Diagnostics.Monitoring.EventPipe; using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -18,6 +19,65 @@ internal interface IMetricsStore : IDisposable Task SnapshotMetrics(Stream stream, CancellationToken token); + void SnapshotMetrics(out MetricsSnapshot snapshot, bool deltaAggregation = false); + void Clear(); } + + internal sealed class MetricsSnapshot + { + public DateTime ProcessStartTimeUtc { get; } + + public DateTime LastCollectionStartTimeUtc { get; } + + public DateTime LastCollectionEndTimeUtc { get; } + + public IReadOnlyList Meters { get; } + + public MetricsSnapshot( + DateTime processStartTimeUtc, + DateTime lastCollectionStartTimeUtc, + DateTime lastCollectionEndTimeUtc, + IReadOnlyList meters) + { + ProcessStartTimeUtc = processStartTimeUtc; + LastCollectionStartTimeUtc = lastCollectionStartTimeUtc; + LastCollectionEndTimeUtc = lastCollectionEndTimeUtc; + Meters = meters; + } + } + + internal sealed class MetricsSnapshotMeter + { + public string MeterName { get; } + + public string MeterVersion { get; } + + public IReadOnlyList Instruments { get; } + + public MetricsSnapshotMeter( + string meterName, + string meterVersion, + IReadOnlyList instruments) + { + MeterName = meterName; + MeterVersion = meterVersion; + Instruments = instruments; + } + } + + internal sealed class MetricsSnapshotInstrument + { + public CounterMetadata Metadata { get; } + + public IReadOnlyList MetricPoints { get; } + + public MetricsSnapshotInstrument( + CounterMetadata metadata, + IReadOnlyList metricPoints) + { + Metadata = metadata; + MetricPoints = metricPoints; + } + } } diff --git a/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsStore.cs b/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsStore.cs index 57f37690e4d..39497f7741c 100644 --- a/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsStore.cs +++ b/src/Microsoft.Diagnostics.Monitoring.WebApi/Metrics/MetricsStore.cs @@ -32,6 +32,7 @@ public override int GetHashCode() HashCode code = new HashCode(); code.Add(_metric.CounterMetadata.ProviderName); code.Add(_metric.CounterMetadata.CounterName); + code.Add(_metric.ValueTags); return code.ToHashCode(); } @@ -45,14 +46,21 @@ public override bool Equals(object? obj) } } - private Dictionary> _allMetrics = new Dictionary>(); + private readonly Dictionary> _allMetrics = new Dictionary>(); private readonly int _maxMetricCount; - private ILogger _logger; + private readonly ILogger _logger; + private readonly DateTime _processStartTimeUtc; + private DateTime _lastCollectionStartTimeUtc; private HashSet _observedErrorMessages = new(); private HashSet<(string provider, string counter)> _observedEndedCounters = new(); public MetricsStore(ILogger logger, int maxMetricCount) + : this(logger, maxMetricCount, DateTime.UtcNow) + { + } + + public MetricsStore(ILogger logger, int maxMetricCount, DateTime processStartTimeUtc) { if (maxMetricCount < 1) { @@ -60,6 +68,8 @@ public MetricsStore(ILogger logger, int maxMetricCount) } _maxMetricCount = maxMetricCount; _logger = logger; + _processStartTimeUtc = processStartTimeUtc; + _lastCollectionStartTimeUtc = processStartTimeUtc; } public void AddMetric(ICounterPayload metric) @@ -101,65 +111,390 @@ public void AddMetric(ICounterPayload metric) var metricKey = new MetricKey(metric); if (!_allMetrics.TryGetValue(metricKey, out Queue? metrics)) { + if (_allMetrics.Count > _maxMetricCount) + { + return; + } + metrics = new Queue(); _allMetrics.Add(metricKey, metrics); } metrics.Enqueue(metric); - if (metrics.Count > _maxMetricCount) + } + } + + public void SnapshotMetrics(out MetricsSnapshot snapshot, bool deltaAggregation = false) + { + var meterLookup = new Dictionary< + (string, string), + Dictionary)>>(); + + DateTime lastCollectionStartTimeUtc; + DateTime lastCollectionEndTimeUtc; + + lock (_allMetrics) + { + foreach (var metricGroup in _allMetrics) { - metrics.Dequeue(); + var measurements = metricGroup.Value; + if (measurements.Count <= 0) + { + continue; + } + + var firstMeasurement = measurements.Dequeue(); + + var metadata = firstMeasurement.CounterMetadata; + + var meterKey = (metadata.ProviderName, metadata.ProviderVersion); + if (!meterLookup.TryGetValue(meterKey, out var meter)) + { + meter = new(); + meterLookup[meterKey] = meter; + } + + if (!meter.TryGetValue(metadata.CounterName, out var instrument)) + { + instrument = new(metadata, new()); + meter[metadata.CounterName] = instrument; + } + + if (firstMeasurement is RatePayload ratePayload) + { + if (measurements.Count > 1) + { + var rate = ratePayload.Rate; + + foreach (var measurement in measurements) + { + if (measurement is RatePayload nextRatePayload) + { + rate += nextRatePayload.Rate; + } + } + + var aggregated = new RatePayload( + metadata, + displayName: null, + displayUnits: null, + firstMeasurement.ValueTags, + rate, + firstMeasurement.Interval, + firstMeasurement.Timestamp); + + instrument.Item2.Add(aggregated); + + measurements.Clear(); + + if (!deltaAggregation) + { + measurements.Enqueue(aggregated); + } + } + else + { + instrument.Item2.Add(firstMeasurement); + if (!deltaAggregation) + { + measurements.Enqueue(firstMeasurement); + } + } + } + else if (firstMeasurement is UpDownCounterPayload upDownCounterPayload) + { + if (measurements.Count > 1) + { + var rate = upDownCounterPayload.Rate; + + foreach (var measurement in measurements) + { + if (measurement is UpDownCounterPayload nextUpDownCounterPayload) + { + rate += nextUpDownCounterPayload.Rate; + } + } + + var aggregated = new UpDownCounterPayload( + metadata, + displayName: null, + displayUnits: null, + firstMeasurement.ValueTags, + rate, + firstMeasurement.Value, + firstMeasurement.Timestamp); + + instrument.Item2.Add(aggregated); + + measurements.Clear(); + + if (!deltaAggregation) + { + measurements.Enqueue(aggregated); + } + } + else + { + instrument.Item2.Add(firstMeasurement); + if (!deltaAggregation) + { + measurements.Enqueue(firstMeasurement); + } + } + } + else if (firstMeasurement is AggregatePercentilePayload aggregatePercentilePayload) + { + if (measurements.Count > 1) + { + var count = aggregatePercentilePayload.Count; + var sum = aggregatePercentilePayload.Sum; + + foreach (var measurement in measurements) + { + if (measurement is AggregatePercentilePayload nextAggregatePercentilePayload) + { + count += nextAggregatePercentilePayload.Count; + sum += nextAggregatePercentilePayload.Sum; + } + } + + var aggregated = new AggregatePercentilePayload( + metadata, + displayName: null, + displayUnits: null, + firstMeasurement.ValueTags, + count, + sum, + aggregatePercentilePayload.Quantiles, + firstMeasurement.Timestamp); + + instrument.Item2.Add(aggregated); + + measurements.Clear(); + + if (!deltaAggregation) + { + measurements.Enqueue(aggregated); + } + } + else + { + instrument.Item2.Add(firstMeasurement); + if (!deltaAggregation) + { + measurements.Enqueue(firstMeasurement); + } + } + } + else + { + var lastMeasurement = measurements.Count > 0 + ? measurements.Last() + : firstMeasurement; + + instrument.Item2.Add(lastMeasurement); + + if (measurements.Count > 1) + { + measurements.Clear(); + } + + if (!deltaAggregation) + { + measurements.Enqueue(lastMeasurement); + } + } } - // CONSIDER We only keep 1 histogram representation per snapshot. Is it meaningful for Prometheus to see previous histograms? These are not timestamped. - if ((metrics.Count > 1) && (metric is AggregatePercentilePayload)) + lastCollectionStartTimeUtc = _lastCollectionStartTimeUtc; + lastCollectionEndTimeUtc = _lastCollectionStartTimeUtc = DateTime.UtcNow; + } + + var meters = new List(); + foreach (var meter in meterLookup) + { + var instruments = new List(); + foreach (var instrument in meter.Value) { - metrics.Dequeue(); + instruments.Add( + new(instrument.Value.Item1, instrument.Value.Item2)); } + + meters.Add( + new( + meterName: meter.Key.Item1, + meterVersion: meter.Key.Item2, + instruments)); } + + snapshot = new(_processStartTimeUtc, lastCollectionStartTimeUtc, lastCollectionEndTimeUtc, meters); } public async Task SnapshotMetrics(Stream outputStream, CancellationToken token) { - Dictionary>? copy = null; + Dictionary snapshot = new Dictionary(); lock (_allMetrics) { - copy = new Dictionary>(); foreach (var metricGroup in _allMetrics) { - copy.Add(metricGroup.Key, new Queue(metricGroup.Value)); + var measurements = metricGroup.Value; + + var firstMeasurement = measurements.Dequeue(); + + if (firstMeasurement is RatePayload ratePayload) + { + if (measurements.Count > 1) + { + var rate = ratePayload.Rate; + + foreach (var measurement in measurements) + { + if (measurement is RatePayload nextRatePayload) + { + rate += nextRatePayload.Rate; + } + } + + var aggregated = new RatePayload( + firstMeasurement.CounterMetadata, + displayName: null, + displayUnits: null, + firstMeasurement.ValueTags, + rate, + firstMeasurement.Interval, + firstMeasurement.Timestamp); + + snapshot.Add(metricGroup.Key, aggregated); + + measurements.Clear(); + + measurements.Enqueue(aggregated); + } + else + { + snapshot.Add(metricGroup.Key, firstMeasurement); + measurements.Enqueue(firstMeasurement); + } + } + else if (firstMeasurement is UpDownCounterPayload upDownCounterPayload) + { + if (measurements.Count > 1) + { + var rate = upDownCounterPayload.Rate; + + foreach (var measurement in measurements) + { + if (measurement is UpDownCounterPayload nextUpDownCounterPayload) + { + rate += nextUpDownCounterPayload.Rate; + } + } + + var aggregated = new UpDownCounterPayload( + firstMeasurement.CounterMetadata, + displayName: null, + displayUnits: null, + firstMeasurement.ValueTags, + rate, + firstMeasurement.Value, + firstMeasurement.Timestamp); + + snapshot.Add(metricGroup.Key, aggregated); + + measurements.Clear(); + + measurements.Enqueue(aggregated); + } + else + { + snapshot.Add(metricGroup.Key, firstMeasurement); + measurements.Enqueue(firstMeasurement); + } + } + else if (firstMeasurement is AggregatePercentilePayload aggregatePercentilePayload) + { + if (measurements.Count > 1) + { + var count = aggregatePercentilePayload.Count; + var sum = aggregatePercentilePayload.Sum; + + foreach (var measurement in measurements) + { + if (measurement is AggregatePercentilePayload nextAggregatePercentilePayload) + { + count += nextAggregatePercentilePayload.Count; + sum += nextAggregatePercentilePayload.Sum; + } + } + + var aggregated = new AggregatePercentilePayload( + firstMeasurement.CounterMetadata, + displayName: null, + displayUnits: null, + firstMeasurement.ValueTags, + count, + sum, + aggregatePercentilePayload.Quantiles, + firstMeasurement.Timestamp); + + snapshot.Add(metricGroup.Key, aggregated); + + measurements.Clear(); + + measurements.Enqueue(aggregated); + } + else + { + snapshot.Add(metricGroup.Key, firstMeasurement); + measurements.Enqueue(firstMeasurement); + } + } + else + { + var lastMeasurement = measurements.Count > 0 + ? measurements.Last() + : firstMeasurement; + + snapshot.Add(metricGroup.Key, lastMeasurement); + + if (measurements.Count > 1) + { + measurements.Clear(); + } + + measurements.Enqueue(lastMeasurement); + } } + + _allMetrics.Clear(); } await using var writer = new StreamWriter(outputStream, EncodingCache.UTF8NoBOMNoThrow, bufferSize: 1024, leaveOpen: true); writer.NewLine = "\n"; - foreach (var metricGroup in copy) + foreach (var metricGroup in snapshot) { - ICounterPayload metricInfo = metricGroup.Value.First(); + ICounterPayload metric = metricGroup.Value; - string metricName = PrometheusDataModel.GetPrometheusNormalizedName(metricInfo.CounterMetadata.ProviderName, metricInfo.CounterMetadata.CounterName, metricInfo.Unit); + string metricName = PrometheusDataModel.GetPrometheusNormalizedName(metric.CounterMetadata.ProviderName, metric.CounterMetadata.CounterName, metric.Unit); - await WriteMetricHeader(metricInfo, writer, metricName); + await WriteMetricHeader(metric, writer, metricName); - foreach (var metric in metricGroup.Value) + if (metric is AggregatePercentilePayload aggregatePayload) { - if (metric is AggregatePercentilePayload aggregatePayload) - { - // Summary quantiles must appear from smallest to largest - foreach (Quantile quantile in aggregatePayload.Quantiles.OrderBy(q => q.Percentage)) - { - string metricValue = PrometheusDataModel.GetPrometheusNormalizedValue(metric.Unit, quantile.Value); - string metricLabels = GetMetricLabels(metric, quantile.Percentage); - await WriteMetricDetails(writer, metric, metricName, metricValue, metricLabels); - } - } - else + // Summary quantiles must appear from smallest to largest + foreach (Quantile quantile in aggregatePayload.Quantiles.OrderBy(q => q.Percentage)) { - string metricValue = PrometheusDataModel.GetPrometheusNormalizedValue(metric.Unit, metric.Value); - string metricLabels = GetMetricLabels(metric, quantile: null); + string metricValue = PrometheusDataModel.GetPrometheusNormalizedValue(metric.Unit, quantile.Value); + string metricLabels = GetMetricLabels(metric, quantile.Percentage); await WriteMetricDetails(writer, metric, metricName, metricValue, metricLabels); } } + else + { + string metricValue = PrometheusDataModel.GetPrometheusNormalizedValue(metric.Unit, metric.Value); + string metricLabels = GetMetricLabels(metric, quantile: null); + await WriteMetricDetails(writer, metric, metricName, metricValue, metricLabels); + } } } diff --git a/src/Tools/dotnet-monitor/Commands/CollectCommandHandler.cs b/src/Tools/dotnet-monitor/Commands/CollectCommandHandler.cs index a04bba9c099..6fd2d0b4461 100644 --- a/src/Tools/dotnet-monitor/Commands/CollectCommandHandler.cs +++ b/src/Tools/dotnet-monitor/Commands/CollectCommandHandler.cs @@ -5,8 +5,11 @@ using Microsoft.Diagnostics.Monitoring; using Microsoft.Diagnostics.Monitoring.WebApi; using Microsoft.Diagnostics.Tools.Monitor.Auth; -using Microsoft.Diagnostics.Tools.Monitor.Stacks; using Microsoft.Diagnostics.Tools.Monitor.OpenApi; +#if BUILDING_OTEL +using Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry; +#endif +using Microsoft.Diagnostics.Tools.Monitor.Stacks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -130,7 +133,7 @@ private static IHostBuilder Configure(this IHostBuilder builder, StartupAuthenti services.ConfigureCollectionRules(); services.ConfigureLibrarySharing(); - // + // // The order of the below calls is **important**. // - ConfigureInProcessFeatures needs to be called before ConfigureProfiler and ConfigureStartupHook // because these features will configure themselves depending on environment variables set by InProcessFeaturesEndpointInfoSourceCallbacks. @@ -153,6 +156,9 @@ private static IHostBuilder Configure(this IHostBuilder builder, StartupAuthenti services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); +#if BUILDING_OTEL + services.AddOpenTelemetry(); +#endif services.ConfigureCapabilities(noHttpEgress); diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/Extensions/OpenTelemetryServiceCollectionExtensions.cs b/src/Tools/dotnet-monitor/OpenTelemetry/Extensions/OpenTelemetryServiceCollectionExtensions.cs new file mode 100644 index 00000000000..7d61ae8c7fb --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/Extensions/OpenTelemetryServiceCollectionExtensions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry; + +internal static class OpenTelemetryServiceCollectionExtensions +{ + public static IServiceCollection AddOpenTelemetry( + this IServiceCollection services) + => AddOpenTelemetry(services, configurationSectionName: "OpenTelemetry"); + + public static IServiceCollection AddOpenTelemetry( + this IServiceCollection services, + string configurationSectionName) + { + services.ConfigureOpenTelemetry(configurationSectionName); + + services.AddHostedService(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/Logging/OpenTelemetryLogRecordLogger.cs b/src/Tools/dotnet-monitor/OpenTelemetry/Logging/OpenTelemetryLogRecordLogger.cs new file mode 100644 index 00000000000..83846850f17 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/Logging/OpenTelemetryLogRecordLogger.cs @@ -0,0 +1,208 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Monitoring.EventPipe; +using Microsoft.Extensions.Logging; + +using System.Collections.Concurrent; +using OpenTelemetry; +using OpenTelemetry.Configuration; +using OpenTelemetry.Logging; +using OpenTelemetry.Resources; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Logging; + +internal sealed class OpenTelemetryLogRecordLogger : ILogRecordLogger +{ + [ThreadStatic] + private static List>? s_ThreadAttributeStorage; + + private readonly ConcurrentDictionary _scopeCache = new(); + + private static readonly string[] LogLevels = new string[] + { + nameof(LogLevel.Trace), + nameof(LogLevel.Debug), + nameof(LogLevel.Information), + nameof(LogLevel.Warning), + nameof(LogLevel.Error), + nameof(LogLevel.Critical), + nameof(LogLevel.None), + }; + + private readonly ILoggerFactory _LoggerFactory; + private readonly Resource _Resource; + private readonly OpenTelemetryOptions _Options; + + private ILogRecordProcessor? _LogRecordProcessor; + + public OpenTelemetryLogRecordLogger( + ILoggerFactory loggerFactory, + Resource resource, + OpenTelemetryOptions options) + { + _LoggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + _Resource = resource ?? throw new ArgumentNullException(nameof(resource)); + _Options = options ?? throw new ArgumentNullException(nameof(options)); + + if (_Options.ExporterOptions.ExporterType != "OpenTelemetryProtocol" + || _Options.ExporterOptions.OpenTelemetryProtocolExporterOptions == null) + { + throw new InvalidOperationException("Options were invalid."); + } + } + + public Task PipelineStarted(CancellationToken token) + { + try + { + _LogRecordProcessor ??= OpenTelemetryFactory.CreateLogRecordBatchExportProcessorAsync( + _LoggerFactory, + _Resource, + _Options.ExporterOptions, + _Options.LoggingOptions.BatchOptions); + } + catch (Exception ex) + { + _LoggerFactory.CreateLogger() + .LogError(ex, "OpenTelemetryLogRecordLogger failed to initialize OpenTelemetry SDK"); + } + + return Task.CompletedTask; + } + + public async Task PipelineStopped(CancellationToken token) + { + var logRecordProcessor = _LogRecordProcessor; + if (logRecordProcessor != null) + { + await logRecordProcessor.ShutdownAsync(token); + logRecordProcessor.Dispose(); + _LogRecordProcessor = null; + } + } + + public void Log( + in Monitoring.EventPipe.LogRecord log, + ReadOnlySpan> attributes, + in LogRecordScopeContainer scopes) + { + var logRecordProcessor = _LogRecordProcessor; + if (logRecordProcessor == null) + { + return; + } + + InstrumentationScope scope = _scopeCache.GetOrAdd(log.CategoryName, static name => new InstrumentationScope(name)); + + var attributeStorage = s_ThreadAttributeStorage; + if (attributeStorage == null) + { + attributeStorage = s_ThreadAttributeStorage = new(attributes.Length); + } + else + { + attributeStorage.EnsureCapacity(attributes.Length); + } + + attributeStorage.AddRange(attributes); + + scopes.ForEachScope(ScopeCallback, ref attributeStorage); + + /* Begin: Experimental attributes not part of the OTel semantic conventions */ + if (log.EventId.Id != default) + { + attributeStorage.Add(new("log.record.id", log.EventId.Id)); + } + + if (!string.IsNullOrEmpty(log.EventId.Name)) + { + attributeStorage.Add(new("log.record.name", log.EventId.Name)); + } + + if (log.MessageTemplate != null + && log.FormattedMessage != null) + { + attributeStorage.Add(new("log.record.original", log.FormattedMessage)); + } + /* End: Experimental attributes not part of the OTel semantic conventions */ + + if (log.Exception != default) + { + if (!string.IsNullOrEmpty(log.Exception.ExceptionType)) + { + attributeStorage.Add(new("exception.type", log.Exception.ExceptionType)); + } + + if (!string.IsNullOrEmpty(log.Exception.Message)) + { + attributeStorage.Add(new("exception.message", log.Exception.Message)); + } + + if (!string.IsNullOrEmpty(log.Exception.StackTrace)) + { + attributeStorage.Add(new("exception.stacktrace", log.Exception.StackTrace)); + } + } + + LogRecordSeverity severity; + string? severityText; + uint intLogLevel = (uint)log.LogLevel; + if (intLogLevel < 6) + { + severity = (LogRecordSeverity)(intLogLevel * 4 + 1); + severityText = LogLevels[intLogLevel]; + } + else + { + severity = LogRecordSeverity.Unspecified; + severityText = null; + } + + var spanContext = new ActivityContext(log.TraceId, log.SpanId, log.TraceFlags); + + var logRecordInfo = new LogRecordInfo(scope) + { + TimestampUtc = log.Timestamp, + Body = log.MessageTemplate ?? log.FormattedMessage, + Severity = severity, + SeverityText = severityText, + }; + + var logRecord = new global::OpenTelemetry.Logging.LogRecord( + in spanContext, + in logRecordInfo) + { + Attributes = CollectionsMarshal.AsSpan(attributeStorage) + }; + + logRecordProcessor.ProcessEmittedLogRecord(in logRecord); + + attributeStorage.Clear(); + + static void ScopeCallback( + ReadOnlySpan> attributes, + ref List> state) + { + foreach (var attribute in attributes) + { + if (attribute.Key == "{OriginalFormat}" || string.IsNullOrEmpty(attribute.Key)) + { + continue; + } + + state.Add(attribute); + } + } + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryCountersLogger.cs b/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryCountersLogger.cs new file mode 100644 index 00000000000..6d8915109e7 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryCountersLogger.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Diagnostics.Monitoring.EventPipe; +using Microsoft.Diagnostics.Monitoring.WebApi; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Configuration; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Metrics; + +internal sealed class OpenTelemetryCountersLogger : ICountersLogger +{ + private readonly ILoggerFactory _LoggerFactory; + private readonly Resource _Resource; + private readonly OpenTelemetryOptions _Options; + private readonly MetricsStore _MetricsStore; + + private IMetricReader? _MetricReader; + + public OpenTelemetryCountersLogger( + ILoggerFactory loggerFactory, + Resource resource, + OpenTelemetryOptions options) + { + _LoggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + _Resource = resource ?? throw new ArgumentNullException(nameof(resource)); + _Options = options ?? throw new ArgumentNullException(nameof(options)); + + if (_Options.ExporterOptions.ExporterType != "OpenTelemetryProtocol" + || _Options.ExporterOptions.OpenTelemetryProtocolExporterOptions == null) + { + throw new InvalidOperationException("Options were invalid."); + } + + _MetricsStore = new MetricsStore( + loggerFactory.CreateLogger(), + maxMetricCount: 50_000); + } + + public Task PipelineStarted(CancellationToken token) + { + try + { + _MetricReader ??= OpenTelemetryFactory.CreatePeriodicExportingMetricReaderAsync( + _LoggerFactory, + _Resource, + _Options.ExporterOptions, + _Options.MetricsOptions.PeriodicExportingOptions, + [new OpenTelemetryMetricProducerFactory(_MetricsStore)]); + } + catch (Exception ex) + { + _LoggerFactory.CreateLogger() + .LogError(ex, "OpenTelemetryCountersLogger failed to initialize OpenTelemetry SDK"); + } + + return Task.CompletedTask; + } + + public async Task PipelineStopped(CancellationToken token) + { + var metricReader = _MetricReader; + if (metricReader != null) + { + await metricReader.ShutdownAsync(token); + metricReader.Dispose(); + _MetricReader = null; + _MetricsStore.Clear(); + } + } + + public void Log(ICounterPayload counter) + { + if (counter.IsMeter) + { + _MetricsStore.AddMetric(counter); + } + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryMetricProducer.cs b/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryMetricProducer.cs new file mode 100644 index 00000000000..3376a94dc77 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryMetricProducer.cs @@ -0,0 +1,224 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using Microsoft.Diagnostics.Monitoring.EventPipe; +using Microsoft.Diagnostics.Monitoring.WebApi; + +using OTel = OpenTelemetry.Metrics; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Metrics; + +internal sealed class OpenTelemetryMetricProducer : OTel.MetricProducer +{ + private readonly MetricsStore _MetricsStore; + private readonly OTel.AggregationTemporality _AggregationTemporality; + + public OpenTelemetryMetricProducer( + MetricsStore metricsStore, + OTel.AggregationTemporality aggregationTemporality) + { + Debug.Assert(metricsStore != null); + + _MetricsStore = metricsStore; + _AggregationTemporality = aggregationTemporality; + } + + public override bool WriteTo(OTel.MetricWriter writer) + { + OTel.AggregationTemporality aggregationTemporality = _AggregationTemporality; + + _MetricsStore.SnapshotMetrics( + out var snapshot, + deltaAggregation: aggregationTemporality == OTel.AggregationTemporality.Delta); + + foreach (var meter in snapshot.Meters) + { + writer.BeginInstrumentationScope( + new(meter.MeterName) + { + Version = meter.MeterVersion + }); + + foreach (var instrument in meter.Instruments) + { + OTel.Metric? otelMetric = null; + + foreach (var metricPoint in instrument.MetricPoints) + { + if (otelMetric == null) + { + switch (metricPoint.EventType) + { + case EventType.Rate: + otelMetric = new OTel.Metric( + OTel.MetricType.DoubleSum, + instrument.Metadata.CounterName, + aggregationTemporality) + { + Unit = instrument.Metadata.CounterUnit, + Description = instrument.Metadata.CounterDescription + }; + break; + case EventType.Gauge: + otelMetric = new OTel.Metric( + OTel.MetricType.DoubleGauge, + instrument.Metadata.CounterName, + OTel.AggregationTemporality.Cumulative) + { + Unit = instrument.Metadata.CounterUnit, + Description = instrument.Metadata.CounterDescription + }; + break; + case EventType.UpDownCounter: + otelMetric = new OTel.Metric( + OTel.MetricType.DoubleSumNonMonotonic, + instrument.Metadata.CounterName, + OTel.AggregationTemporality.Cumulative) + { + Unit = instrument.Metadata.CounterUnit, + Description = instrument.Metadata.CounterDescription + }; + break; + case EventType.Histogram: + otelMetric = new OTel.Metric( + OTel.MetricType.Histogram, + instrument.Metadata.CounterName, + aggregationTemporality) + { + Unit = instrument.Metadata.CounterUnit, + Description = instrument.Metadata.CounterDescription + }; + break; + default: + return false; + } + + writer.BeginMetric(otelMetric); + } + + DateTime startTimeUtc = otelMetric.AggregationTemporality == OTel.AggregationTemporality.Cumulative + ? snapshot.ProcessStartTimeUtc + : snapshot.LastCollectionStartTimeUtc; + DateTime endTimeUtc = snapshot.LastCollectionEndTimeUtc; + + switch (otelMetric.MetricType) + { + case OTel.MetricType.DoubleSum: + case OTel.MetricType.DoubleGauge: + case OTel.MetricType.DoubleSumNonMonotonic: + WriteNumberMetricPoint(writer, startTimeUtc, endTimeUtc, metricPoint); + break; + case OTel.MetricType.Histogram: + if (metricPoint is AggregatePercentilePayload aggregatePercentilePayload) + { + WriteHistogramMetricPoint(writer, startTimeUtc, endTimeUtc, aggregatePercentilePayload); + } + break; + } + } + + if (otelMetric != null) + { + writer.EndMetric(); + } + } + + writer.EndInstrumentationScope(); + } + + return true; + } + + private static void WriteNumberMetricPoint( + OTel.MetricWriter writer, + DateTime startTimeUtc, + DateTime endTimeUtc, + ICounterPayload payload) + { + double value = payload is IRatePayload ratePayload + ? ratePayload.Rate + : payload.Value; + + var numberMetricPoint = new OTel.NumberMetricPoint( + startTimeUtc, + endTimeUtc, + value); + + writer.WriteNumberMetricPoint( + in numberMetricPoint, + ParseAttributes(payload), + exemplars: default); + } + + private static void WriteHistogramMetricPoint( + OTel.MetricWriter writer, + DateTime startTimeUtc, + DateTime endTimeUtc, + AggregatePercentilePayload payload) + { + var histogramMetricPoint = new OTel.HistogramMetricPoint( + startTimeUtc, + endTimeUtc, + features: OTel.HistogramMetricPointFeatures.None, + min: default, + max: default, + payload.Sum, + payload.Count); + + writer.WriteHistogramMetricPoint( + in histogramMetricPoint, + buckets: default, + ParseAttributes(payload), + exemplars: default); + } + + [ThreadStatic] + private static List>? s_ThreadAttributeStorage; + + private static ReadOnlySpan> ParseAttributes(ICounterPayload payload) + { + List> attributes = s_ThreadAttributeStorage ??= new(); + attributes.Clear(); + + ReadOnlySpan metadata = payload.ValueTags; + + while (!metadata.IsEmpty) + { + int commaIndex = metadata.IndexOf(','); + + ReadOnlySpan kvPair; + + if (commaIndex < 0) + { + kvPair = metadata; + metadata = default; + } + else + { + kvPair = metadata[..commaIndex]; + metadata = metadata.Slice(commaIndex + 1); + } + + int colonIndex = kvPair.IndexOf('='); + if (colonIndex < 0) + { + attributes.Clear(); + break; + } + + string metadataKey = kvPair[..colonIndex].ToString(); + string metadataValue = kvPair.Slice(colonIndex + 1).ToString(); + attributes.Add(new(metadataKey, metadataValue)); + } + + return CollectionsMarshal.AsSpan(attributes); + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryMetricProducerFactory.cs b/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryMetricProducerFactory.cs new file mode 100644 index 00000000000..02fb9242645 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/Metrics/OpenTelemetryMetricProducerFactory.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System.Diagnostics; +using Microsoft.Diagnostics.Monitoring.WebApi; +using OpenTelemetry.Metrics; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Metrics; + +internal sealed class OpenTelemetryMetricProducerFactory : IMetricProducerFactory +{ + private readonly MetricsStore _MetricsStore; + + public OpenTelemetryMetricProducerFactory( + MetricsStore metricsStore) + { + Debug.Assert(metricsStore != null); + + _MetricsStore = metricsStore; + } + + public MetricProducer Create(MetricProducerOptions options) + => new OpenTelemetryMetricProducer(_MetricsStore, options.AggregationTemporality); +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointInfoSourceCallbacks.cs b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointInfoSourceCallbacks.cs new file mode 100644 index 00000000000..0ef5365a6d5 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointInfoSourceCallbacks.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Monitoring.WebApi; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry; + +internal sealed class OpenTelemetryEndpointInfoSourceCallbacks : IEndpointInfoSourceCallbacks +{ + private readonly OpenTelemetryEndpointManager _openTelemetryManager; + + public OpenTelemetryEndpointInfoSourceCallbacks(OpenTelemetryEndpointManager openTelemetryManager) + { + _openTelemetryManager = openTelemetryManager; + } + + Task IEndpointInfoSourceCallbacks.OnAddedEndpointInfoAsync(IEndpointInfo endpointInfo, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + Task IEndpointInfoSourceCallbacks.OnBeforeResumeAsync(IEndpointInfo endpointInfo, CancellationToken cancellationToken) + { + _openTelemetryManager.StartListeningToEndpoint(endpointInfo); + return Task.CompletedTask; + } + + Task IEndpointInfoSourceCallbacks.OnRemovedEndpointInfoAsync(IEndpointInfo endpointInfo, CancellationToken cancellationToken) + { + return _openTelemetryManager.StopListeningToEndpointAsync(endpointInfo); + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointListener.cs b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointListener.cs new file mode 100644 index 00000000000..58586a080b4 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointListener.cs @@ -0,0 +1,326 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Diagnostics.Monitoring.EventPipe; +using Microsoft.Diagnostics.Monitoring.WebApi; +using Microsoft.Diagnostics.NETCore.Client; +using Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Logging; +using Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Metrics; +using Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Tracing; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OpenTelemetry.Configuration; +using OpenTelemetry.Resources; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry; + +internal sealed class OpenTelemetryEndpointListener +{ + private readonly ILoggerFactory _LoggerFactory; + private readonly ILogger _Logger; + private readonly IOptionsMonitor _OpenTelemetryOptions; + private readonly IEndpointInfo _EndpointInfo; + private readonly DiagnosticsClient _DiagnosticsClient; + private readonly object _lock = new(); + private Task? _ProcessTask; + private CancellationTokenSource? _StoppingToken; + + public OpenTelemetryEndpointListener( + ILoggerFactory loggerFactory, + IOptionsMonitor openTelemetryOptions, + IEndpointInfo endpointInfo) + { + _LoggerFactory = loggerFactory; + _OpenTelemetryOptions = openTelemetryOptions; + _EndpointInfo = endpointInfo; + _DiagnosticsClient = new DiagnosticsClient(endpointInfo.Endpoint); + + _Logger = loggerFactory.CreateLogger(); + } + + public void StartListening() + { + lock (_lock) + { + if (_StoppingToken == null) + { + _StoppingToken = new(); + _ProcessTask = Task.Run(() => Process(_StoppingToken.Token)); + } + } + } + + public async Task StopListeningAsync() + { + Task? processTask; + CancellationTokenSource? stoppingToken; + + lock (_lock) + { + stoppingToken = _StoppingToken; + processTask = _ProcessTask; + _StoppingToken = null; + _ProcessTask = null; + } + + if (stoppingToken != null) + { + stoppingToken.Cancel(); + try + { + await processTask!; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _Logger.LogError(ex, "Unexpected error while stopping OpenTelemetry listener for process."); + } + catch (OperationCanceledException) + { + } + stoppingToken.Dispose(); + } + } + + private async Task Process(CancellationToken stoppingToken) + { + var scopeValue = new KeyValueLogScope(); + scopeValue.AddArtifactEndpointInfo(_EndpointInfo); + + using var scope = _Logger.BeginScope(scopeValue); + + while (!stoppingToken.IsCancellationRequested) + { + var options = _OpenTelemetryOptions.Get(name: null); + + var logsEnabled = options.LoggingOptions.DefaultLogLevel != null + || options.LoggingOptions.CategoryOptions.Any(); + var metricsEnabled = options.MetricsOptions.MeterOptions.Any(); + var tracingEnabled = options.TracingOptions.Sources.Any(); + + if (!logsEnabled && !metricsEnabled && !tracingEnabled) + { + _Logger.LogInformation("No OpenTelemetry configuration found process will not be monitored"); + await OpenTelemetryService.WaitForOptionsReloadOrStop(_OpenTelemetryOptions, stoppingToken); + continue; + } + + if (options.ExporterOptions.ExporterType != "OpenTelemetryProtocol") + { + _Logger.LogInformation("OpenTelemetry configuration ExporterType '{ExporterType}' is not supported", options.ExporterOptions.ExporterType); + await OpenTelemetryService.WaitForOptionsReloadOrStop(_OpenTelemetryOptions, stoppingToken); + continue; + } + + var resource = await BuildResourceForProcess(_EndpointInfo, _DiagnosticsClient, options, stoppingToken); + + _Logger.LogInformation("Resource created for process with {NumberOfKeys} keys.", resource.Attributes.Length); + + using var optionsTokenSource = new CancellationTokenSource(); + + using var _ = _OpenTelemetryOptions.OnChange(o => optionsTokenSource.SafeCancel()); + + using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource( + stoppingToken, + optionsTokenSource.Token); + + var tasks = new List(); + + if (logsEnabled) + { + tasks.Add( + RunPipelineAsync("logs", () => ListenToLogs(options, _DiagnosticsClient, resource, linkedTokenSource.Token))); + } + + if (metricsEnabled) + { + tasks.Add( + RunPipelineAsync("metrics", () => ListenToMetrics(options, _DiagnosticsClient, _EndpointInfo.RuntimeVersion, resource, linkedTokenSource.Token))); + } + + if (tracingEnabled) + { + tasks.Add( + RunPipelineAsync("traces", () => ListenToTraces(options, _DiagnosticsClient, resource, linkedTokenSource.Token))); + } + + await Task.WhenAll(tasks); + } + } + + private async Task RunPipelineAsync(string signalType, Func pipelineFunc) + { + try + { + await pipelineFunc(); + } + catch (OperationCanceledException) + { + } + catch (Exception ex) + { + _Logger.LogError(ex, "OpenTelemetry {SignalType} pipeline failed.", signalType); + } + } + + private async Task BuildResourceForProcess( + IEndpointInfo endpointInfo, + DiagnosticsClient client, + OpenTelemetryOptions options, + CancellationToken stoppingToken) + { + // Note: The full process environment is passed to CreateResource so that ${env:VAR} + // expressions in resource attribute configuration can be resolved. Only attributes explicitly + // configured in ResourceOptions are included in the exported resource; the full environment + // dictionary is not forwarded to the OTLP backend. + var environment = await client.GetProcessEnvironmentAsync(stoppingToken); + + var processInfo = await ProcessInfoImpl.FromEndpointInfoAsync(endpointInfo, stoppingToken); + + var resource = OpenTelemetryFactory.CreateResource( + options.ResourceOptions, + out var unresolvedAttributes, + environment, + processInfo.ProcessName, + endpointInfo.ProcessId.ToString()); + + foreach (var attribute in unresolvedAttributes) + { + _Logger.LogWarning( + "Resource key '{Key}' could not be resolved from value/expression '{ValueOrExpression}' in ProcessId '{ProcessId}'.", + attribute.Key, + attribute.ValueOrExpression, + endpointInfo.ProcessId); + } + + return resource; + } + + private async Task ListenToLogs( + OpenTelemetryOptions options, + DiagnosticsClient client, + Resource resource, + CancellationToken stoppingToken) + { + _Logger.LogInformation("Starting log collection"); + + var filterSpecs = new Dictionary(); + + foreach (var category in options.LoggingOptions.CategoryOptions) + { + if (!Enum.TryParse(category.LogLevel, out LogLevel logLevel)) + { + _Logger.LogWarning("Could not parse LogLevel '{LogLevel}' for category '{CategoryPrefix}', defaulting to Warning.", category.LogLevel, category.CategoryPrefix); + logLevel = LogLevel.Warning; + } + + filterSpecs.Add(category.CategoryPrefix, logLevel); + } + + if (!Enum.TryParse(options.LoggingOptions.DefaultLogLevel, out LogLevel defaultLogLevel)) + { + _Logger.LogWarning("Could not parse default LogLevel '{LogLevel}', defaulting to Warning.", options.LoggingOptions.DefaultLogLevel); + defaultLogLevel = LogLevel.Warning; + } + + var settings = new EventLogsPipelineSettings + { + CollectScopes = options.LoggingOptions.IncludeScopes, + LogLevel = defaultLogLevel, + UseAppFilters = true, + FilterSpecs = filterSpecs, + Duration = Timeout.InfiniteTimeSpan, + }; + + await using var pipeline = new EventLogsPipeline( + client, + settings, + new OpenTelemetryLogRecordLogger(_LoggerFactory, resource, options)); + + await pipeline.RunAsync(stoppingToken); + + _Logger.LogInformation("Stopped log collection"); + } + + private async Task ListenToMetrics( + OpenTelemetryOptions options, + DiagnosticsClient client, + Version? runtimeVersion, + Resource resource, + CancellationToken stoppingToken) + { + _Logger.LogInformation("Starting metrics collection"); + + var counterGroups = new List(); + + + foreach (var meter in options.MetricsOptions.MeterOptions) + { + var counterGroup = new EventPipeCounterGroup() { ProviderName = meter.MeterName, Type = CounterGroupType.Meter }; + + counterGroup.CounterNames = meter.Instruments.ToArray(); + + counterGroups.Add(counterGroup); + } + + var settings = new MetricsPipelineSettings + { + CounterIntervalSeconds = options.MetricsOptions.PeriodicExportingOptions.ExportIntervalMilliseconds / 1000, + CounterGroups = counterGroups.ToArray(), + UseSharedSession = runtimeVersion?.Major >= 8, + MaxHistograms = options.MetricsOptions.MaxHistograms, + MaxTimeSeries = options.MetricsOptions.MaxTimeSeries, + Duration = Timeout.InfiniteTimeSpan, + }; + + await using var pipeline = new MetricsPipeline( + client, + settings, + new ICountersLogger[] { new OpenTelemetryCountersLogger(_LoggerFactory, resource, options) }); + + await pipeline.RunAsync(stoppingToken); + + _Logger.LogInformation("Stopped metrics collection"); + } + + private async Task ListenToTraces( + OpenTelemetryOptions options, + DiagnosticsClient client, + Resource resource, + CancellationToken stoppingToken) + { + _Logger.LogInformation("Starting traces collection"); + + var sources = options.TracingOptions.Sources.ToArray(); + + double samplingRatio = options.TracingOptions.SamplerOptions.SamplerType == "ParentBased" + && options.TracingOptions.SamplerOptions.ParentBasedOptions?.RootSamplerOptions?.SamplerType == "TraceIdRatio" + ? options.TracingOptions.SamplerOptions.ParentBasedOptions.RootSamplerOptions.TraceIdRatioBasedOptions?.SamplingRatio ?? 1.0D + : 1.0D; + + var settings = new DistributedTracesPipelineSettings + { + SamplingRatio = samplingRatio, + Sources = sources, + Duration = Timeout.InfiniteTimeSpan, + }; + + await using var pipeline = new DistributedTracesPipeline( + client, + settings, + new IActivityLogger[] { new OpenTelemetryActivityLogger(_LoggerFactory, resource, options) }); + + await pipeline.RunAsync(stoppingToken); + + _Logger.LogInformation("Stopped traces collection"); + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointManager.cs b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointManager.cs new file mode 100644 index 00000000000..0e933fb8d92 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryEndpointManager.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Monitoring.WebApi; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OpenTelemetry.Configuration; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry; + +internal sealed class OpenTelemetryEndpointManager : IAsyncDisposable +{ + private readonly ILoggerFactory _LoggerFactory; + private readonly ILogger _Logger; + private readonly IOptionsMonitor _OpenTelemetryOptions; + private readonly Dictionary _Endpoints = new(); + + public OpenTelemetryEndpointManager( + ILoggerFactory loggerFactory, + IOptionsMonitor openTelemetryOptions) + { + _LoggerFactory = loggerFactory; + _OpenTelemetryOptions = openTelemetryOptions; + + _Logger = loggerFactory.CreateLogger(); + } + + public void StartListeningToEndpoint(IEndpointInfo endpointInfo) + { + OpenTelemetryEndpointListener endpointListener; + + lock (_Endpoints) + { + if (_Endpoints.ContainsKey(endpointInfo.ProcessId)) + { + _Logger.LogWarning("Process {ProcessId} connected but is already subscribed", endpointInfo.ProcessId); + return; + } + + _Logger.LogInformation("Process {ProcessId} connected", endpointInfo.ProcessId); + + endpointListener = new OpenTelemetryEndpointListener( + _LoggerFactory, + _OpenTelemetryOptions, + endpointInfo); + + _Endpoints[endpointInfo.ProcessId] = endpointListener; + } + + endpointListener.StartListening(); + } + + public async Task StopListeningToEndpointAsync(IEndpointInfo endpointInfo) + { + OpenTelemetryEndpointListener? endpointListener; + + lock (_Endpoints) + { + if (!_Endpoints.TryGetValue(endpointInfo.ProcessId, out endpointListener)) + { + _Logger.LogWarning("Process {ProcessId} disconnected but a subscription could not be found", endpointInfo.ProcessId); + return; + } + + _Logger.LogInformation("Process {ProcessId} disconnected", endpointInfo.ProcessId); + + _Endpoints.Remove(endpointInfo.ProcessId); + } + + await endpointListener.StopListeningAsync(); + } + + public async ValueTask DisposeAsync() + { + List listeners; + + lock (_Endpoints) + { + listeners = new List(_Endpoints.Values); + _Endpoints.Clear(); + } + + foreach (OpenTelemetryEndpointListener listener in listeners) + { + await listener.StopListeningAsync(); + } + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryService.cs b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryService.cs new file mode 100644 index 00000000000..5aa19fe3c66 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/OpenTelemetryService.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Monitoring.WebApi; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry; + +internal sealed class OpenTelemetryService : BackgroundService +{ + private readonly ILogger _Logger; + private readonly IOptionsMonitor _ProcessOptions; + private readonly OpenTelemetryEndpointManager _OpenTelemetryEndpointManager; + private readonly IDiagnosticServices _DiagnosticServices; + + public OpenTelemetryService( + ILogger logger, + IOptionsMonitor processOptions, + OpenTelemetryEndpointManager openTelemetryEndpointManager, + IDiagnosticServices diagnosticServices) + { + _Logger = logger; + _ProcessOptions = processOptions; + _OpenTelemetryEndpointManager = openTelemetryEndpointManager; + _DiagnosticServices = diagnosticServices; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + var processOptions = _ProcessOptions.Get(name: null); + + if (processOptions.Filters.Count != 1) + { + _Logger.LogInformation("DefaultProcess configuration was not found or was invalid OpenTelemetry monitoring will only be enabled in passive mode"); + + await WaitForOptionsReloadOrStop(_ProcessOptions, stoppingToken); + + continue; + } + + IProcessInfo processInfo; + try + { + processInfo = await _DiagnosticServices.GetProcessAsync(processKey: null, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + // Process discovery can take time, especially on machines with many .NET processes. + // Log without the stack trace to avoid alarming users during normal startup. + _Logger.LogInformation("Waiting for target process to be discovered. Retrying in 5 seconds."); + _Logger.LogDebug(e, "Process discovery failed with exception."); + await Task.Delay(5000, stoppingToken); + continue; + } + + stoppingToken.ThrowIfCancellationRequested(); + + _OpenTelemetryEndpointManager.StartListeningToEndpoint(processInfo.EndpointInfo); + + await WaitForOptionsReloadOrStop(_ProcessOptions, stoppingToken); + + await _OpenTelemetryEndpointManager.StopListeningToEndpointAsync(processInfo.EndpointInfo); + } + } + + internal static async Task WaitForOptionsReloadOrStop(IOptionsMonitor options, CancellationToken stoppingToken) + { + var cts = new TaskCompletionSource(); + + using var token = options.OnChange(o => cts.TrySetResult()); + + await cts.WithCancellation(stoppingToken); + } +} +#endif diff --git a/src/Tools/dotnet-monitor/OpenTelemetry/Tracing/OpenTelemetryActivityLogger.cs b/src/Tools/dotnet-monitor/OpenTelemetry/Tracing/OpenTelemetryActivityLogger.cs new file mode 100644 index 00000000000..4b74dee1e05 --- /dev/null +++ b/src/Tools/dotnet-monitor/OpenTelemetry/Tracing/OpenTelemetryActivityLogger.cs @@ -0,0 +1,115 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#if BUILDING_OTEL +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Diagnostics.Monitoring.EventPipe; +using Microsoft.Extensions.Logging; + +using System.Collections.Concurrent; +using OpenTelemetry; +using OpenTelemetry.Configuration; +using OpenTelemetry.Resources; +using OpenTelemetry.Tracing; + +namespace Microsoft.Diagnostics.Tools.Monitor.OpenTelemetry.Tracing; + +internal sealed class OpenTelemetryActivityLogger : IActivityLogger +{ + private readonly ConcurrentDictionary<(string Name, string? Version), InstrumentationScope> _scopeCache = new(); + private readonly ILoggerFactory _LoggerFactory; + private readonly Resource _Resource; + private readonly OpenTelemetryOptions _Options; + + private ISpanProcessor? _SpanProcessor; + + public OpenTelemetryActivityLogger( + ILoggerFactory loggerFactory, + Resource resource, + OpenTelemetryOptions options) + { + _LoggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + _Resource = resource ?? throw new ArgumentNullException(nameof(resource)); + _Options = options ?? throw new ArgumentNullException(nameof(options)); + + if (_Options.ExporterOptions.ExporterType != "OpenTelemetryProtocol" + || _Options.ExporterOptions.OpenTelemetryProtocolExporterOptions == null) + { + throw new InvalidOperationException("Options were invalid."); + } + } + + public Task PipelineStarted(CancellationToken token) + { + try + { + _SpanProcessor ??= OpenTelemetryFactory.CreateSpanBatchExportProcessorAsync( + _LoggerFactory, + _Resource, + _Options.ExporterOptions, + _Options.TracingOptions.BatchOptions); + } + catch (Exception ex) + { + _LoggerFactory.CreateLogger() + .LogError(ex, "OpenTelemetryActivityLogger failed to initialize OpenTelemetry SDK"); + } + + return Task.CompletedTask; + } + + public async Task PipelineStopped(CancellationToken token) + { + var spanProcessor = _SpanProcessor; + if (spanProcessor != null) + { + await spanProcessor.ShutdownAsync(token); + spanProcessor.Dispose(); + _SpanProcessor = null; + } + } + + public void Log( + in ActivityData activity, + ReadOnlySpan> tags) + { + var spanProcessor = _SpanProcessor; + if (spanProcessor == null) + { + return; + } + + InstrumentationScope scope = _scopeCache.GetOrAdd( + (activity.Source.Name, activity.Source.Version), + static key => new InstrumentationScope(key.Name) { Version = key.Version }); + + var spanInfo = new SpanInfo( + scope, + name: activity.DisplayName ?? activity.OperationName) + { + TraceId = activity.TraceId, + SpanId = activity.SpanId, + TraceFlags = activity.TraceFlags, + TraceState = activity.TraceState, + ParentSpanId = activity.ParentSpanId, + Kind = activity.Kind, + StartTimestampUtc = activity.StartTimeUtc, + EndTimestampUtc = activity.EndTimeUtc, + StatusCode = activity.Status, + StatusDescription = activity.StatusDescription + }; + + var span = new Span(in spanInfo) + { + Attributes = tags + }; + + spanProcessor.ProcessEndedSpan(in span); + } +} +#endif diff --git a/src/Tools/dotnet-monitor/dotnet-monitor.csproj b/src/Tools/dotnet-monitor/dotnet-monitor.csproj index 34993f9604f..a38477bb388 100644 --- a/src/Tools/dotnet-monitor/dotnet-monitor.csproj +++ b/src/Tools/dotnet-monitor/dotnet-monitor.csproj @@ -28,6 +28,10 @@ + + + + @@ -36,6 +40,10 @@ + + $(DefineConstants);BUILDING_OTEL + + diff --git a/test-otel-config.json b/test-otel-config.json new file mode 100644 index 00000000000..2476824b0ce --- /dev/null +++ b/test-otel-config.json @@ -0,0 +1,48 @@ +{ + "DefaultProcess": { + "Filters": [{ + "ProcessName": "WebApplication2" + }] + }, + "OpenTelemetry": { + "Exporter": { + "Type": "otlp", + "Settings": { + "Defaults": { + "Protocol": "HttpProtobuf", + "BaseUrl": "http://localhost:4318" + } + } + }, + "Logs": { + "Categories": { + "Default": "Information" + }, + "IncludeScopes": true, + "Batch": { + "ExportIntervalMilliseconds": 5000 + } + }, + "Metrics": { + "Meters": { + "System.Runtime": [], + "Microsoft.AspNetCore.Hosting": [], + "Microsoft.AspNetCore.Server.Kestrel": [] + }, + "PeriodicExporting": { + "ExportIntervalMilliseconds": 10000 + } + }, + "Traces": { + "Sampler": 1.0, + "Sources": [ + "Microsoft.AspNetCore", + "System.Net.Http", + "System.Net.Http.HttpClient" + ], + "Batch": { + "ExportIntervalMilliseconds": 5000 + } + } + } +}