From e7b3dcd74717ae82df3d3c054520d99a65543719 Mon Sep 17 00:00:00 2001 From: simonhammes Date: Mon, 16 Mar 2026 21:47:53 +0100 Subject: [PATCH 1/3] reverseproxy: Add per-upstream metrics Fixes #4140 --- modules/caddyhttp/reverseproxy/metrics.go | 56 +++++++++++++++++++ .../caddyhttp/reverseproxy/reverseproxy.go | 3 + 2 files changed, 59 insertions(+) diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go index 2488427304e..f20bc86c302 100644 --- a/modules/caddyhttp/reverseproxy/metrics.go +++ b/modules/caddyhttp/reverseproxy/metrics.go @@ -11,11 +11,14 @@ import ( "go.uber.org/zap/zapcore" "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/internal/metrics" ) var reverseProxyMetrics = struct { once sync.Once upstreamsHealthy *prometheus.GaugeVec + upstreamRequests *prometheus.CounterVec + upstreamDuration *prometheus.HistogramVec logger *zap.Logger }{} @@ -23,6 +26,8 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) { const ns, sub = "caddy", "reverse_proxy" upstreamsLabels := []string{"upstream"} + upstreamRequestLabels := []string{"upstream", "code", "method"} + reverseProxyMetrics.once.Do(func() { reverseProxyMetrics.upstreamsHealthy = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: ns, @@ -30,6 +35,21 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) { Name: "upstreams_healthy", Help: "Health status of reverse proxy upstreams.", }, upstreamsLabels) + + reverseProxyMetrics.upstreamRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: sub, + Name: "upstream_requests_total", + Help: "Counter of requests made to upstreams.", + }, upstreamRequestLabels) + + reverseProxyMetrics.upstreamDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: ns, + Subsystem: sub, + Name: "upstream_request_duration_seconds", + Help: "Histogram of request durations to upstreams.", + Buckets: prometheus.DefBuckets, + }, upstreamsLabels) }) // duplicate registration could happen if multiple sites with reverse proxy are configured; so ignore the error because @@ -43,6 +63,22 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) { panic(err) } + if err := registry.Register(reverseProxyMetrics.upstreamRequests); err != nil && + !errors.Is(err, prometheus.AlreadyRegisteredError{ + ExistingCollector: reverseProxyMetrics.upstreamRequests, + NewCollector: reverseProxyMetrics.upstreamRequests, + }) { + panic(err) + } + + if err := registry.Register(reverseProxyMetrics.upstreamDuration); err != nil && + !errors.Is(err, prometheus.AlreadyRegisteredError{ + ExistingCollector: reverseProxyMetrics.upstreamDuration, + NewCollector: reverseProxyMetrics.upstreamDuration, + }) { + panic(err) + } + reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics") } @@ -97,3 +133,23 @@ func (m *metricsUpstreamsHealthyUpdater) update() { reverseProxyMetrics.upstreamsHealthy.With(labels).Set(gaugeValue) } } + +func recordUpstreamMetrics(upstream string, method string, statusCode int, duration time.Duration) { + // Guard for test cases that bypass Provision() + if reverseProxyMetrics.upstreamRequests == nil { + return + } + + code := metrics.SanitizeCode(statusCode) + method = metrics.SanitizeMethod(method) + + reverseProxyMetrics.upstreamRequests.With(prometheus.Labels{ + "upstream": upstream, + "code": code, + "method": method, + }).Inc() + + reverseProxyMetrics.upstreamDuration.With(prometheus.Labels{ + "upstream": upstream, + }).Observe(duration.Seconds()) +} diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index 2169d17173f..9838959cdef 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -1007,6 +1007,7 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe if c := logger.Check(zapcore.DebugLevel, logMessage); c != nil { c.Write(zap.Error(err)) } + recordUpstreamMetrics(di.Upstream.String(), req.Method, http.StatusBadGateway, duration) return err } if c := logger.Check(zapcore.DebugLevel, logMessage); c != nil { @@ -1019,6 +1020,8 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe ) } + recordUpstreamMetrics(di.Upstream.String(), req.Method, res.StatusCode, duration) + // duration until upstream wrote response headers (roundtrip duration) repl.Set("http.reverse_proxy.upstream.latency", duration) repl.Set("http.reverse_proxy.upstream.latency_ms", duration.Seconds()*1e3) // multiply seconds to preserve decimal (see #4666) From fba858e0a9db6c0ec0a9f29ad5bef39424ca1024 Mon Sep 17 00:00:00 2001 From: simonhammes Date: Fri, 20 Mar 2026 19:39:32 +0100 Subject: [PATCH 2/3] Harmonize metric labels --- modules/caddyhttp/reverseproxy/metrics.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go index f20bc86c302..12cddc06e24 100644 --- a/modules/caddyhttp/reverseproxy/metrics.go +++ b/modules/caddyhttp/reverseproxy/metrics.go @@ -49,7 +49,7 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) { Name: "upstream_request_duration_seconds", Help: "Histogram of request durations to upstreams.", Buckets: prometheus.DefBuckets, - }, upstreamsLabels) + }, upstreamRequestLabels) }) // duplicate registration could happen if multiple sites with reverse proxy are configured; so ignore the error because @@ -140,16 +140,12 @@ func recordUpstreamMetrics(upstream string, method string, statusCode int, durat return } - code := metrics.SanitizeCode(statusCode) - method = metrics.SanitizeMethod(method) - - reverseProxyMetrics.upstreamRequests.With(prometheus.Labels{ + labels := prometheus.Labels{ "upstream": upstream, - "code": code, - "method": method, - }).Inc() + "method": metrics.SanitizeMethod(method), + "code": metrics.SanitizeCode(statusCode), + } - reverseProxyMetrics.upstreamDuration.With(prometheus.Labels{ - "upstream": upstream, - }).Observe(duration.Seconds()) + reverseProxyMetrics.upstreamRequests.With(labels).Inc() + reverseProxyMetrics.upstreamDuration.With(labels).Observe(duration.Seconds()) } From d3ac6923d7a26a7c2830be37dc05616e772dd81a Mon Sep 17 00:00:00 2001 From: simonhammes Date: Fri, 20 Mar 2026 19:39:54 +0100 Subject: [PATCH 3/3] Update metric descriptions --- modules/caddyhttp/reverseproxy/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go index 12cddc06e24..ef70a5d9b0a 100644 --- a/modules/caddyhttp/reverseproxy/metrics.go +++ b/modules/caddyhttp/reverseproxy/metrics.go @@ -40,14 +40,14 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) { Namespace: ns, Subsystem: sub, Name: "upstream_requests_total", - Help: "Counter of requests made to upstreams.", + Help: "Counter of requests made to reverse proxy upstreams.", }, upstreamRequestLabels) reverseProxyMetrics.upstreamDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: ns, Subsystem: sub, Name: "upstream_request_duration_seconds", - Help: "Histogram of request durations to upstreams.", + Help: "Histogram of request durations to reverse proxy upstreams.", Buckets: prometheus.DefBuckets, }, upstreamRequestLabels) })