reverseproxy: Add per-upstream metrics

modules/caddyhttp/reverseproxy/metrics.go

@@ -11,25 +11,45 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/internal/metrics"
 )
 
 var reverseProxyMetrics = struct {
 	once             sync.Once
 	upstreamsHealthy *prometheus.GaugeVec
+	upstreamRequests *prometheus.CounterVec
+	upstreamDuration *prometheus.HistogramVec
 	logger           *zap.Logger
 }{}
 
 func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
 	const ns, sub = "caddy", "reverse_proxy"
 
 	upstreamsLabels := []string{"upstream"}
+	upstreamRequestLabels := []string{"upstream", "code", "method"}
+
 	reverseProxyMetrics.once.Do(func() {
 		reverseProxyMetrics.upstreamsHealthy = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: ns,
 			Subsystem: sub,
 			Name:      "upstreams_healthy",
 			Help:      "Health status of reverse proxy upstreams.",
 		}, upstreamsLabels)
+
+		reverseProxyMetrics.upstreamRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: ns,
+			Subsystem: sub,
+			Name:      "upstream_requests_total",
+			Help:      "Counter of requests made to reverse proxy upstreams.",
+		}, upstreamRequestLabels)
+
+		reverseProxyMetrics.upstreamDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: ns,
+			Subsystem: sub,
+			Name:      "upstream_request_duration_seconds",
+			Help:      "Histogram of request durations to reverse proxy upstreams.",
+			Buckets:   prometheus.DefBuckets,
+		}, upstreamRequestLabels)
 	})
 
 	// duplicate registration could happen if multiple sites with reverse proxy are configured; so ignore the error because
@@ -43,6 +63,22 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
 		panic(err)
 	}
 
+	if err := registry.Register(reverseProxyMetrics.upstreamRequests); err != nil &&
+		!errors.Is(err, prometheus.AlreadyRegisteredError{
+			ExistingCollector: reverseProxyMetrics.upstreamRequests,
+			NewCollector:      reverseProxyMetrics.upstreamRequests,
+		}) {
+		panic(err)
+	}
+
+	if err := registry.Register(reverseProxyMetrics.upstreamDuration); err != nil &&
+		!errors.Is(err, prometheus.AlreadyRegisteredError{
+			ExistingCollector: reverseProxyMetrics.upstreamDuration,
+			NewCollector:      reverseProxyMetrics.upstreamDuration,
+		}) {
+		panic(err)
+	}
+
 	reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics")
 }
 
@@ -97,3 +133,19 @@ func (m *metricsUpstreamsHealthyUpdater) update() {
 		reverseProxyMetrics.upstreamsHealthy.With(labels).Set(gaugeValue)
 	}
 }
+
+func recordUpstreamMetrics(upstream string, method string, statusCode int, duration time.Duration) {
+	// Guard for test cases that bypass Provision()
+	if reverseProxyMetrics.upstreamRequests == nil {
+		return
+	}
+
+	labels := prometheus.Labels{
+		"upstream": upstream,
+		"method":   metrics.SanitizeMethod(method),
+		"code":     metrics.SanitizeCode(statusCode),
+	}
+
+	reverseProxyMetrics.upstreamRequests.With(labels).Inc()
+	reverseProxyMetrics.upstreamDuration.With(labels).Observe(duration.Seconds())
+}

modules/caddyhttp/reverseproxy/reverseproxy.go

@@ -1007,6 +1007,7 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
 		if c := logger.Check(zapcore.DebugLevel, logMessage); c != nil {
 			c.Write(zap.Error(err))
 		}
+		recordUpstreamMetrics(di.Upstream.String(), req.Method, http.StatusBadGateway, duration)
 		return err
 	}
 	if c := logger.Check(zapcore.DebugLevel, logMessage); c != nil {
@@ -1019,6 +1020,8 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
 		)
 	}
 
+	recordUpstreamMetrics(di.Upstream.String(), req.Method, res.StatusCode, duration)
+
 	// duration until upstream wrote response headers (roundtrip duration)
 	repl.Set("http.reverse_proxy.upstream.latency", duration)
 	repl.Set("http.reverse_proxy.upstream.latency_ms", duration.Seconds()*1e3) // multiply seconds to preserve decimal (see #4666)