From 8e909413d6799cdb823b40206ea1051c7e3638b5 Mon Sep 17 00:00:00 2001 From: ferponse <47328511+ferponse@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:03:44 +0200 Subject: [PATCH 1/4] fix(egress): give the DNS latency histogram buckets that match its unit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit egress.dns.query.duration records seconds but declared no boundaries, so it got the SDK default — the spec's millisecond ladder (0, 5, 10, ... 10000). Every realistic DNS latency, from a sub-millisecond cache hit to a multi-second timeout, landed in the single le=5 bucket, which makes any quantile over it an interpolation inside (0, 5] rather than a measurement. Declare a seconds ladder spanning a cache hit to DefaultDNSUpstreamTimeoutSec, with 10s as an overflow guard. Keeping seconds rather than switching to milliseconds preserves the declared unit and the OTel semantic conventions; the other components already record milliseconds, which is why their defaults fit. The test records four latencies an operator would care about and asserts they land in four different buckets, so a future change back to the defaults fails instead of silently producing a flat histogram. Fixes #1404 --- components/egress/docs/opentelemetry.md | 12 ++++ components/egress/go.mod | 2 +- components/egress/pkg/telemetry/metrics.go | 8 +++ .../egress/pkg/telemetry/metrics_test.go | 60 +++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/components/egress/docs/opentelemetry.md b/components/egress/docs/opentelemetry.md index d7779c5c9..ce3920796 100644 --- a/components/egress/docs/opentelemetry.md +++ b/components/egress/docs/opentelemetry.md @@ -17,6 +17,18 @@ This page lists the OpenTelemetry metrics currently implemented in egress. | `egress.system.memory.usage_bytes` | Observable Gauge | `By` | System memory used bytes (Linux: gopsutil; non-Linux build: `0`). | | `egress.system.cpu.utilization` | Observable Gauge | `1` | CPU busy ratio in `[0,1]` (Linux: gopsutil; non-Linux build: `0`). | +`egress.dns.query.duration` declares its bucket boundaries explicitly: + +``` +0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 +``` + +They span a cache hit (sub-millisecond) to the upstream timeout +(`OPENSANDBOX_EGRESS_DNS_UPSTREAM_TIMEOUT`, 5s by default), with 10s as an overflow guard. +Do not drop them: the instrument records **seconds**, while the SDK default boundaries are +the spec's millisecond ladder (`0, 5, 10, … 10000`), so every realistic latency would fall +into the single `le=5` bucket and the quantiles would be meaningless. + ## Shared Attributes All egress metrics may include shared attributes: diff --git a/components/egress/go.mod b/components/egress/go.mod index 5e6842b32..3799b6d44 100644 --- a/components/egress/go.mod +++ b/components/egress/go.mod @@ -9,6 +9,7 @@ require ( github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 + go.opentelemetry.io/otel/sdk/metric v1.43.0 go.uber.org/automaxprocs v1.6.0 golang.org/x/sys v0.45.0 k8s.io/apimachinery v0.34.2 @@ -30,7 +31,6 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/components/egress/pkg/telemetry/metrics.go b/components/egress/pkg/telemetry/metrics.go index aa585b50e..433d3b22b 100644 --- a/components/egress/pkg/telemetry/metrics.go +++ b/components/egress/pkg/telemetry/metrics.go @@ -74,6 +74,14 @@ func registerEgressMetrics() error { "egress.dns.query.duration", metric.WithDescription("DNS forward latency"), metric.WithUnit("s"), + // Explicit boundaries: this instrument records seconds, but the SDK default + // boundaries are the spec's millisecond ladder (0, 5, 10, ... 10000), so every + // realistic DNS latency lands in the same bucket and the quantiles are noise. + // The ladder below spans a cache hit (sub-ms) to the upstream timeout + // (DefaultDNSUpstreamTimeoutSec = 5s), with 10s as the overflow guard. + metric.WithExplicitBucketBoundaries( + 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, + ), ) if err != nil { return err diff --git a/components/egress/pkg/telemetry/metrics_test.go b/components/egress/pkg/telemetry/metrics_test.go index a757c4548..a15ee1c21 100644 --- a/components/egress/pkg/telemetry/metrics_test.go +++ b/components/egress/pkg/telemetry/metrics_test.go @@ -15,11 +15,17 @@ package telemetry import ( + "context" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "github.com/alibaba/opensandbox/egress/pkg/constants" inttelemetry "github.com/alibaba/opensandbox/internal/telemetry" ) @@ -45,3 +51,57 @@ func TestAppendMetricAttrsFromKeyValuePairs(t *testing.T) { out = inttelemetry.AppendAttrsFromKeyValuePairs(nil, "novalue=,=bad,nokv") assert.Len(t, out, 0) } + +// The instrument records seconds, so it needs boundaries on a seconds ladder. With the +// SDK default (the spec's millisecond ladder) every realistic DNS latency collapses into +// one bucket and the quantiles are meaningless. +func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) { + reader := sdkmetric.NewManualReader() + previous := otel.GetMeterProvider() + otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))) + t.Cleanup(func() { otel.SetMeterProvider(previous) }) + + require.NoError(t, registerEgressMetrics()) + + // Cache hit, LAN upstream, slow upstream, and the default upstream timeout. + for _, seconds := range []float64{0.0008, 0.012, 0.4, 5} { + RecordDNSForward(seconds) + } + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + + dp := dnsDurationDataPoint(t, &rm) + require.NotEmpty(t, dp.Bounds) + assert.Less(t, dp.Bounds[0], 0.01, + "boundaries look like the millisecond default, not a seconds ladder") + assert.GreaterOrEqual(t, dp.Bounds[len(dp.Bounds)-1], float64(constants.DefaultDNSUpstreamTimeoutSec), + "the top boundary should cover an upstream timeout") + + populated := 0 + for _, count := range dp.BucketCounts { + if count > 0 { + populated++ + } + } + assert.Equal(t, 4, populated, + "the four latencies must land in four different buckets, got counts %v for bounds %v", + dp.BucketCounts, dp.Bounds) +} + +func dnsDurationDataPoint(t *testing.T, rm *metricdata.ResourceMetrics) metricdata.HistogramDataPoint[float64] { + t.Helper() + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != "egress.dns.query.duration" { + continue + } + hist, ok := m.Data.(metricdata.Histogram[float64]) + require.True(t, ok, "unexpected aggregation %T", m.Data) + require.Len(t, hist.DataPoints, 1) + return hist.DataPoints[0] + } + } + t.Fatal("egress.dns.query.duration not collected") + return metricdata.HistogramDataPoint[float64]{} +} From 4f41ebfb285f07674490059c51fe7ad24c885d9a Mon Sep 17 00:00:00 2001 From: ferponse <47328511+ferponse@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:14:05 +0200 Subject: [PATCH 2/4] fix(egress): extend the DNS histogram tail to cover the resolver retry chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recorded duration wraps the whole forward(), which walks the resolvers serially with the full timeout each, so a query can legitimately take timeout x len(upstreams) — 15s for three resolvers at the default. A ladder topping out at 10s pushed those into +Inf, where Prometheus can only report the highest finite boundary, so the tail the metric exists to show was flat again. Add 15, 30, 60 and 120: 15s is the default three-resolver chain, 120s is the cap a single exchange can be configured to wait. Fixed boundaries rather than boundaries derived from the configured timeout and upstream count: those vary per deployment, and histograms with different le sets cannot be aggregated in one backend. There is no finite worst case to cover anyway, since OPENSANDBOX_EGRESS_DNS_UPSTREAM accepts an unbounded resolver list, so past 120s the lookup has failed and _count is the signal rather than a quantile. The test now records a three-resolver chain and asserts the +Inf bucket stays empty, so this specific regression fails rather than passing quietly. --- components/egress/docs/opentelemetry.md | 17 ++++++++++++++--- components/egress/pkg/telemetry/metrics.go | 11 +++++++++-- components/egress/pkg/telemetry/metrics_test.go | 17 +++++++++++------ 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/components/egress/docs/opentelemetry.md b/components/egress/docs/opentelemetry.md index ce3920796..afe795d4f 100644 --- a/components/egress/docs/opentelemetry.md +++ b/components/egress/docs/opentelemetry.md @@ -20,15 +20,26 @@ This page lists the OpenTelemetry metrics currently implemented in egress. `egress.dns.query.duration` declares its bucket boundaries explicitly: ``` -0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 +0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 15 30 60 120 ``` -They span a cache hit (sub-millisecond) to the upstream timeout -(`OPENSANDBOX_EGRESS_DNS_UPSTREAM_TIMEOUT`, 5s by default), with 10s as an overflow guard. Do not drop them: the instrument records **seconds**, while the SDK default boundaries are the spec's millisecond ladder (`0, 5, 10, … 10000`), so every realistic latency would fall into the single `le=5` bucket and the quantiles would be meaningless. +The head resolves a cache hit (sub-millisecond) up to one upstream timeout +(`OPENSANDBOX_EGRESS_DNS_UPSTREAM_TIMEOUT`, 5s by default). The coarse tail exists because +the recorded duration covers the **whole resolver chain**: forwarding walks the upstreams +serially, each with the full timeout, so a query can legitimately take +`timeout x len(upstreams)` — 15s is three resolvers at the default, and 120s is the cap a +single exchange can be configured to wait. The chain has no finite worst case +(`OPENSANDBOX_EGRESS_DNS_UPSTREAM` accepts an unbounded resolver list), so anything past +120s falls in `+Inf` on purpose: at that point the lookup has failed and `_count` is the +signal, not a quantile. + +Note both successful and failed lookups feed this histogram, so its tail mixes slow +resolutions with exhausted retry chains. + ## Shared Attributes All egress metrics may include shared attributes: diff --git a/components/egress/pkg/telemetry/metrics.go b/components/egress/pkg/telemetry/metrics.go index 433d3b22b..3ce9ced19 100644 --- a/components/egress/pkg/telemetry/metrics.go +++ b/components/egress/pkg/telemetry/metrics.go @@ -77,10 +77,17 @@ func registerEgressMetrics() error { // Explicit boundaries: this instrument records seconds, but the SDK default // boundaries are the spec's millisecond ladder (0, 5, 10, ... 10000), so every // realistic DNS latency lands in the same bucket and the quantiles are noise. - // The ladder below spans a cache hit (sub-ms) to the upstream timeout - // (DefaultDNSUpstreamTimeoutSec = 5s), with 10s as the overflow guard. + // + // The head spans a cache hit (sub-ms) to one upstream timeout + // (DefaultDNSUpstreamTimeoutSec = 5s). The coarse tail covers the retry chain: + // forward() walks the resolvers serially, each with the full timeout, and the + // recorded duration is the whole chain — so a query can legitimately take + // timeout x len(upstreams). 15s is three resolvers at the default; 120s is the + // cap a single exchange can be configured to wait. Past that a lookup has simply + // failed, and _count is the signal, not a quantile. metric.WithExplicitBucketBoundaries( 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, + 15, 30, 60, 120, ), ) if err != nil { diff --git a/components/egress/pkg/telemetry/metrics_test.go b/components/egress/pkg/telemetry/metrics_test.go index a15ee1c21..16ce7dcc1 100644 --- a/components/egress/pkg/telemetry/metrics_test.go +++ b/components/egress/pkg/telemetry/metrics_test.go @@ -63,8 +63,9 @@ func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) { require.NoError(t, registerEgressMetrics()) - // Cache hit, LAN upstream, slow upstream, and the default upstream timeout. - for _, seconds := range []float64{0.0008, 0.012, 0.4, 5} { + // Cache hit, LAN upstream, slow upstream, one upstream timeout, and a serial retry + // through three resolvers at the default timeout. + for _, seconds := range []float64{0.0008, 0.012, 0.4, 5, 15} { RecordDNSForward(seconds) } @@ -75,8 +76,10 @@ func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) { require.NotEmpty(t, dp.Bounds) assert.Less(t, dp.Bounds[0], 0.01, "boundaries look like the millisecond default, not a seconds ladder") - assert.GreaterOrEqual(t, dp.Bounds[len(dp.Bounds)-1], float64(constants.DefaultDNSUpstreamTimeoutSec), - "the top boundary should cover an upstream timeout") + // forward() retries resolvers serially with the full timeout each and records the + // whole chain, so the tail has to reach well past a single timeout. + assert.Greater(t, dp.Bounds[len(dp.Bounds)-1], float64(constants.DefaultDNSUpstreamTimeoutSec), + "the top boundary must leave room for a serial retry chain, not just one timeout") populated := 0 for _, count := range dp.BucketCounts { @@ -84,9 +87,11 @@ func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) { populated++ } } - assert.Equal(t, 4, populated, - "the four latencies must land in four different buckets, got counts %v for bounds %v", + assert.Equal(t, 5, populated, + "the five latencies must land in five different buckets, got counts %v for bounds %v", dp.BucketCounts, dp.Bounds) + assert.Zero(t, dp.BucketCounts[len(dp.BucketCounts)-1], + "a retry-chain latency fell into +Inf, where it cannot be distinguished or interpolated") } func dnsDurationDataPoint(t *testing.T, rm *metricdata.ResourceMetrics) metricdata.HistogramDataPoint[float64] { From 6fe49498c1242ffa554d0892d5f96d383cd9f8a0 Mon Sep 17 00:00:00 2001 From: ferponse <47328511+ferponse@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:37:18 +0200 Subject: [PATCH 3/4] docs(egress): publish the DNS latency buckets in docs/ The boundaries were documented only in components/egress/docs, which nothing in docs/ links to. Per AGENTS.md operations-visible content belongs in docs/, and the Observability section of docs/components/egress.md named no metrics at all. Whoever tunes these needs the reason they exist, so the note leads with the consequence: the instrument records seconds, and the SDK default boundaries are the spec's millisecond ladder, which silently flattens every quantile. --- docs/components/egress.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/components/egress.md b/docs/components/egress.md index eeeb610b5..d059f79c0 100644 --- a/docs/components/egress.md +++ b/docs/components/egress.md @@ -181,6 +181,29 @@ See [Credential Vault](/guides/credential-vault) for full API usage, binding rul Egress can export **OTLP metrics**; application logs use the **native zap** logger (JSON to stdout by default, configurable via `OPENSANDBOX_LOG_OUTPUT` / `OPENSANDBOX_EGRESS_LOG_LEVEL`). OTLP log export is not used. +#### DNS latency buckets + +`egress.dns.query.duration` is recorded in **seconds** and declares its bucket boundaries +explicitly: + +``` +0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 15 30 60 120 +``` + +The head resolves a cache hit up to one upstream timeout +(`OPENSANDBOX_EGRESS_DNS_UPSTREAM_TIMEOUT`, 5s by default). The coarse tail is there because +the recorded duration covers the **whole resolver chain** — forwarding walks the upstreams +serially with the full timeout each, so a query can legitimately take +`timeout x len(upstreams)`. Anything past 120s falls in `+Inf` by design: the lookup has +failed, and `_count` is the signal rather than a quantile. + +If you tune these, keep them on a seconds ladder. The SDK default boundaries are the spec's +millisecond ladder (`0, 5, 10, … 10000`), which would put every realistic DNS latency in the +single `le=5` bucket and make `histogram_quantile()` return an interpolation rather than a +measurement. + +Full metric inventory and attribute semantics: [egress OpenTelemetry reference](https://github.com/opensandbox-group/OpenSandbox/blob/main/components/egress/docs/opentelemetry.md). + ## Build & Run ### Build Docker Image From 035f2d9d309756671134e03d1fa05c34e3d80b01 Mon Sep 17 00:00:00 2001 From: ferponse <47328511+ferponse@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:08:47 +0200 Subject: [PATCH 4/4] fix(egress): extend the DNS histogram tail past the configurable timeout cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My justification for stopping at 120s was wrong. I argued anything beyond it had failed, so a quantile there was meaningless — but forward() continues to the next resolver after a timed-out exchange, so a query can *succeed* late: two resolvers each burning close to the configurable 120s cap put a successful lookup past the last finite boundary, where Prometheus can only report that boundary. The quantile loss this branch exists to fix, recreated for an accepted configuration. Add 300 and 600, covering the per-exchange cap across a handful of resolvers, and say plainly what the tail holds: late successes as well as exhausted chains. The chain still has no finite worst case, since the resolver list is unbounded, so past the last boundary _count is what remains — but that now requires a genuinely pathological configuration rather than a documented one. The test records a 240s late success and keeps asserting +Inf stays empty. --- components/egress/docs/opentelemetry.md | 12 +++++++----- components/egress/pkg/telemetry/metrics.go | 11 +++++++---- components/egress/pkg/telemetry/metrics_test.go | 11 ++++++----- docs/components/egress.md | 8 +++++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/components/egress/docs/opentelemetry.md b/components/egress/docs/opentelemetry.md index afe795d4f..08b852fef 100644 --- a/components/egress/docs/opentelemetry.md +++ b/components/egress/docs/opentelemetry.md @@ -20,7 +20,7 @@ This page lists the OpenTelemetry metrics currently implemented in egress. `egress.dns.query.duration` declares its bucket boundaries explicitly: ``` -0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 15 30 60 120 +0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 15 30 60 120 300 600 ``` Do not drop them: the instrument records **seconds**, while the SDK default boundaries are @@ -32,10 +32,12 @@ The head resolves a cache hit (sub-millisecond) up to one upstream timeout the recorded duration covers the **whole resolver chain**: forwarding walks the upstreams serially, each with the full timeout, so a query can legitimately take `timeout x len(upstreams)` — 15s is three resolvers at the default, and 120s is the cap a -single exchange can be configured to wait. The chain has no finite worst case -(`OPENSANDBOX_EGRESS_DNS_UPSTREAM` accepts an unbounded resolver list), so anything past -120s falls in `+Inf` on purpose: at that point the lookup has failed and `_count` is the -signal, not a quantile. +single exchange can be configured to wait. A late **success** lands in the tail too, not only an exhausted failure: a query can +succeed on the second resolver after the first burned a full timeout. The chain has no finite +worst case either (`OPENSANDBOX_EGRESS_DNS_UPSTREAM` accepts an unbounded resolver list), so +past the last boundary quantile resolution is lost by construction and `_count` is what +remains. A configuration that gets there — several resolvers each waiting close to the 120s +per-exchange cap — has bigger problems than a percentile. Note both successful and failed lookups feed this histogram, so its tail mixes slow resolutions with exhausted retry chains. diff --git a/components/egress/pkg/telemetry/metrics.go b/components/egress/pkg/telemetry/metrics.go index 3ce9ced19..6405f3fb8 100644 --- a/components/egress/pkg/telemetry/metrics.go +++ b/components/egress/pkg/telemetry/metrics.go @@ -82,12 +82,15 @@ func registerEgressMetrics() error { // (DefaultDNSUpstreamTimeoutSec = 5s). The coarse tail covers the retry chain: // forward() walks the resolvers serially, each with the full timeout, and the // recorded duration is the whole chain — so a query can legitimately take - // timeout x len(upstreams). 15s is three resolvers at the default; 120s is the - // cap a single exchange can be configured to wait. Past that a lookup has simply - // failed, and _count is the signal, not a quantile. + // timeout x len(upstreams), and a late *success* lands there too, not just an + // exhausted failure. 15s is three resolvers at the default; 600s covers the 120s + // per-exchange cap across a handful of them. The chain has no finite worst case + // (OPENSANDBOX_EGRESS_DNS_UPSTREAM takes an unbounded resolver list), so past the + // last boundary quantile resolution is lost by construction and _count is what + // remains — a configuration that gets there has bigger problems than a percentile. metric.WithExplicitBucketBoundaries( 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, - 15, 30, 60, 120, + 15, 30, 60, 120, 300, 600, ), ) if err != nil { diff --git a/components/egress/pkg/telemetry/metrics_test.go b/components/egress/pkg/telemetry/metrics_test.go index 16ce7dcc1..e4e33bea7 100644 --- a/components/egress/pkg/telemetry/metrics_test.go +++ b/components/egress/pkg/telemetry/metrics_test.go @@ -63,9 +63,10 @@ func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) { require.NoError(t, registerEgressMetrics()) - // Cache hit, LAN upstream, slow upstream, one upstream timeout, and a serial retry - // through three resolvers at the default timeout. - for _, seconds := range []float64{0.0008, 0.012, 0.4, 5, 15} { + // Cache hit, LAN upstream, slow upstream, one upstream timeout, a serial retry through + // three resolvers at the default timeout, and a late success after two resolvers each + // burning the configurable 120s maximum. + for _, seconds := range []float64{0.0008, 0.012, 0.4, 5, 15, 240} { RecordDNSForward(seconds) } @@ -87,8 +88,8 @@ func TestDNSQueryDurationBucketsSpanRealisticLatencies(t *testing.T) { populated++ } } - assert.Equal(t, 5, populated, - "the five latencies must land in five different buckets, got counts %v for bounds %v", + assert.Equal(t, 6, populated, + "the six latencies must land in six different buckets, got counts %v for bounds %v", dp.BucketCounts, dp.Bounds) assert.Zero(t, dp.BucketCounts[len(dp.BucketCounts)-1], "a retry-chain latency fell into +Inf, where it cannot be distinguished or interpolated") diff --git a/docs/components/egress.md b/docs/components/egress.md index d059f79c0..bac58311d 100644 --- a/docs/components/egress.md +++ b/docs/components/egress.md @@ -187,15 +187,17 @@ Egress can export **OTLP metrics**; application logs use the **native zap** logg explicitly: ``` -0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 15 30 60 120 +0.001 0.0025 0.005 0.01 0.025 0.05 0.1 0.25 0.5 1 2.5 5 10 15 30 60 120 300 600 ``` The head resolves a cache hit up to one upstream timeout (`OPENSANDBOX_EGRESS_DNS_UPSTREAM_TIMEOUT`, 5s by default). The coarse tail is there because the recorded duration covers the **whole resolver chain** — forwarding walks the upstreams serially with the full timeout each, so a query can legitimately take -`timeout x len(upstreams)`. Anything past 120s falls in `+Inf` by design: the lookup has -failed, and `_count` is the signal rather than a quantile. +`timeout x len(upstreams)`. A late **success** lands in the tail too, not only an +exhausted failure: a query can succeed on the second resolver after the first burned a full +timeout. Past the last boundary quantile resolution is lost by construction — the chain has no +finite worst case, since the resolver list is unbounded — and `_count` is what remains. If you tune these, keep them on a seconds ladder. The SDK default boundaries are the spec's millisecond ladder (`0, 5, 10, … 10000`), which would put every realistic DNS latency in the