Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions components/egress/docs/opentelemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,18 @@ This page lists the OpenTelemetry metrics currently implemented in egress.
| `egress.system.memory.usage_bytes` | Observable Gauge | `By` | System memory used bytes (Linux: gopsutil; non-Linux build: `0`). |
| `egress.system.cpu.utilization` | Observable Gauge | `1` | CPU busy ratio in `[0,1]` (Linux: gopsutil; non-Linux build: `0`). |

`egress.dns.query.duration` declares its bucket boundaries explicitly:

```
0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10
```

They span a cache hit (sub-millisecond) to the upstream timeout
(`OPENSANDBOX_EGRESS_DNS_UPSTREAM_TIMEOUT`, 5s by default), with 10s as an overflow guard.
Do not drop them: the instrument records **seconds**, while the SDK default boundaries are
the spec's millisecond ladder (`0, 5, 10, … 10000`), so every realistic latency would fall
into the single `le=5` bucket and the quantiles would be meaningless.

## Shared Attributes

All egress metrics may include shared attributes:
Expand Down
2 changes: 1 addition & 1 deletion components/egress/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/stretchr/testify v1.11.1
go.opentelemetry.io/otel v1.43.0
go.opentelemetry.io/otel/metric v1.43.0
go.opentelemetry.io/otel/sdk/metric v1.43.0
go.uber.org/automaxprocs v1.6.0
golang.org/x/sys v0.45.0
k8s.io/apimachinery v0.34.2
Expand All @@ -30,7 +31,6 @@ require (
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
go.opentelemetry.io/otel/trace v1.43.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand Down
8 changes: 8 additions & 0 deletions components/egress/pkg/telemetry/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,14 @@ func registerEgressMetrics() error {
"egress.dns.query.duration",
metric.WithDescription("DNS forward latency"),
metric.WithUnit("s"),
// Explicit boundaries: this instrument records seconds, but the SDK default
// boundaries are the spec's millisecond ladder (0, 5, 10, ... 10000), so every
// realistic DNS latency lands in the same bucket and the quantiles are noise.
// The ladder below spans a cache hit (sub-ms) to the upstream timeout
// (DefaultDNSUpstreamTimeoutSec = 5s), with 10s as the overflow guard.
metric.WithExplicitBucketBoundaries(
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
Comment thread
ferponse marked this conversation as resolved.
),
)
if err != nil {
return err
Expand Down
60 changes: 60 additions & 0 deletions components/egress/pkg/telemetry/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@
package telemetry

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"

"github.com/alibaba/opensandbox/egress/pkg/constants"
inttelemetry "github.com/alibaba/opensandbox/internal/telemetry"
)

Expand All @@ -45,3 +51,57 @@ func TestAppendMetricAttrsFromKeyValuePairs(t *testing.T) {
out = inttelemetry.AppendAttrsFromKeyValuePairs(nil, "novalue=,=bad,nokv")
assert.Len(t, out, 0)
}

// The instrument records seconds, so it needs boundaries on a seconds ladder. With the
// SDK default (the spec's millisecond ladder) every realistic DNS latency collapses into
// one bucket and the quantiles are meaningless.
func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) {
reader := sdkmetric.NewManualReader()
previous := otel.GetMeterProvider()
otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
t.Cleanup(func() { otel.SetMeterProvider(previous) })

require.NoError(t, registerEgressMetrics())

// Cache hit, LAN upstream, slow upstream, and the default upstream timeout.
for _, seconds := range []float64{0.0008, 0.012, 0.4, 5} {
RecordDNSForward(seconds)
}

var rm metricdata.ResourceMetrics
require.NoError(t, reader.Collect(context.Background(), &rm))

dp := dnsDurationDataPoint(t, &rm)
require.NotEmpty(t, dp.Bounds)
assert.Less(t, dp.Bounds[0], 0.01,
"boundaries look like the millisecond default, not a seconds ladder")
assert.GreaterOrEqual(t, dp.Bounds[len(dp.Bounds)-1], float64(constants.DefaultDNSUpstreamTimeoutSec),
"the top boundary should cover an upstream timeout")

populated := 0
for _, count := range dp.BucketCounts {
if count > 0 {
populated++
}
}
assert.Equal(t, 4, populated,
"the four latencies must land in four different buckets, got counts %v for bounds %v",
dp.BucketCounts, dp.Bounds)
}

func dnsDurationDataPoint(t *testing.T, rm *metricdata.ResourceMetrics) metricdata.HistogramDataPoint[float64] {
t.Helper()
for _, sm := range rm.ScopeMetrics {
for _, m := range sm.Metrics {
if m.Name != "egress.dns.query.duration" {
continue
}
hist, ok := m.Data.(metricdata.Histogram[float64])
require.True(t, ok, "unexpected aggregation %T", m.Data)
require.Len(t, hist.DataPoints, 1)
return hist.DataPoints[0]
}
}
t.Fatal("egress.dns.query.duration not collected")
return metricdata.HistogramDataPoint[float64]{}
}
Loading