Limit response sizes from query-frontends

CHANGELOG.md

@@ -30,6 +30,7 @@
 * [CHANGE] Query-frontend: Removed support for calculating 'cache-adjusted samples processed' query statistic. The `-query-frontend.cache-samples-processed-stats` CLI flag has been deprecated and will be removed in a future release. Setting it has now no effect. #13582
 * [CHANGE] Querier: Renamed experimental flag `-querier.prefer-availability-zone` to `-querier.prefer-availability-zones` and changed it to accept a comma-separated list of availability zones. All zones in the list are given equal priority when querying ingesters and store-gateways. #13756 #13758
 * [CHANGE] Ingester: Stabilize experimental flag `-ingest-storage.write-logs-fsync-before-kafka-commit-concurrency` to fsync write logs before the offset is committed to Kafka. Remove `-ingest-storage.write-logs-fsync-before-kafka-commit-enabled` since this is always enabled now. #13591
+* [CHANGE] Query-frontend: Enforce a limit on the size of responses returned by query-frontends. Defaults to 128MB and can be configured with `-query-frontend.max-response-size-bytes`. #13829 
 * [FEATURE] Distributor: add `-distributor.otel-label-name-underscore-sanitization` and `-distributor.otel-label-name-preserve-underscores` that control sanitization of underscores during OTLP translation. #13133
 * [FEATURE] Query-frontends: Automatically adjust features used in query plans generated for remote execution based on what the available queriers support. #13017 #13164 #13544
 * [FEATURE] Memberlist: Add experimental support for zone-aware routing in order to reduce memberlist cross-AZ data transfer. #13129 #13651 #13664

cmd/mimir/config-descriptor.json

@@ -8199,6 +8199,17 @@
           "fieldType": "boolean",
           "fieldCategory": "experimental"
         },
+        {
+          "kind": "field",
+          "name": "max_response_size_bytes",
+          "required": false,
+          "desc": "Maximum allowed response size for query responses, in bytes.",
+          "fieldValue": null,
+          "fieldDefaultValue": 134217728,
+          "fieldFlag": "query-frontend.max-response-size-bytes",
+          "fieldType": "int",
+          "fieldCategory": "experimental"
+        },
         {
           "kind": "field",
           "name": "extra_propagated_headers",

cmd/mimir/help-all.txt.tmpl

@@ -2427,6 +2427,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL.
   -query-frontend.max-query-expression-size-bytes int
     	Max size of the raw query, in bytes. This limit is enforced by the query-frontend for instant, range and remote read queries. 0 to not apply a limit to the size of the query.
+  -query-frontend.max-response-size-bytes uint
+    	[experimental] Maximum allowed response size for query responses, in bytes. (default 134217728)
   -query-frontend.max-retries-per-request int
     	Maximum number of retries for a single request; beyond this, the downstream error is returned. (default 5)
   -query-frontend.max-total-query-length duration

docs/sources/mimir/configure/about-versioning.md

@@ -214,6 +214,7 @@ The following features are currently experimental:
   - Rewriting of queries to optimize processing: `-query-frontend.rewrite-histogram-queries` and `-query-frontend.rewrite-propagate-matchers`
   - Enable experimental Prometheus extended range selector modifiers `smoothed` and `anchored` (`-query-frontend.enabled-promql-extended-range-selectors=smoothed,anchored`)
   - Experimental PromQL functions and aggregations, including `mad_over_time`, `ts_of_min_over_time`, `ts_of_max_over_time`, `ts_of_first_over_time`, `ts_of_last_over_time`, `sort_by_label`, `sort_by_label_desc`, `limitk` and `limit_ratio` (`-query-frontend.enabled-promql-experimental-functions=...`)
+  - Limiting response size for queries (`-query-frontend.max-response-size-bytes`)
 
 - Query-scheduler
   - `-query-scheduler.querier-forget-delay`

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -1996,6 +1996,10 @@ results_cache:
 # CLI flag: -query-frontend.use-active-series-decoder
 [use_active_series_decoder: <boolean> | default = false]
 
+# (experimental) Maximum allowed response size for query responses, in bytes.
+# CLI flag: -query-frontend.max-response-size-bytes
+[max_response_size_bytes: <int> | default = 134217728]
+
 # (advanced) Comma-separated list of request header names to allow to pass
 # through to the rest of the query path. This is in addition to a list of
 # required headers that the read path needs.

go.mod

@@ -387,3 +387,5 @@ replace github.com/prometheus/otlptranslator => github.com/grafana/mimir-otlptra
 // Use a fork of client_golang with changes from:
 // - https://github.com/prometheus/client_golang/pull/1925
 replace github.com/prometheus/client_golang => github.com/colega/prometheus-client_golang v1.19.1-0.20251204143415-11cda2079634
+
+replace github.com/json-iterator/go => github.com/charleskorn/json-iterator-go v0.0.0-20251215054129-8df468a86247

go.sum

@@ -213,6 +213,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charleskorn/go-grpc v0.0.0-20231024023642-e9298576254f h1:P1GSPnbxmhUafKGBcaY2qqx34mBdC4GVDm/RN3iKKuE=
 github.com/charleskorn/go-grpc v0.0.0-20231024023642-e9298576254f/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
+github.com/charleskorn/json-iterator-go v0.0.0-20251215054129-8df468a86247 h1:m/mfd1NT13+CjBba/8U0Lm2X8kk/p+1wK/Bv2iPf7yE=
+github.com/charleskorn/json-iterator-go v0.0.0-20251215054129-8df468a86247/go.mod h1:TBzl5BIHNXfS9+C35ZyJaklL7mLDbgUkcgXzSLa8Tk0=
 github.com/chromedp/cdproto v0.0.0-20210526005521-9e51b9051fd0/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
 github.com/chromedp/cdproto v0.0.0-20210706234513-2bc298e8be7f/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
 github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 h1:aPflPkRFkVwbW6dmcVqfgwp1i+UWGFH6VgR1Jim5Ygc=
@@ -723,9 +725,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -833,7 +832,6 @@ github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6U
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=

operations/mimir/mimir-flags-defaults.json

@@ -557,6 +557,7 @@
   "query-frontend.max-retries-per-request": 5,
   "query-frontend.not-running-timeout": 2000000000,
   "query-frontend.query-sharding-target-series-per-shard": 0,
+  "query-frontend.max-response-size-bytes": 134217728,
   "query-frontend.extra-propagated-headers": "",
   "query-frontend.query-result-response-format": "protobuf",
   "query-frontend.client-cluster-validation.label": "",

pkg/frontend/querymiddleware/codec.go

@@ -43,6 +43,7 @@ import (
 	"github.com/grafana/mimir/pkg/streamingpromql/compat"
 	"github.com/grafana/mimir/pkg/util"
 	"github.com/grafana/mimir/pkg/util/chunkinfologger"
+	"github.com/grafana/mimir/pkg/util/globalerror"
 	"github.com/grafana/mimir/pkg/util/propagation"
 	"github.com/grafana/mimir/pkg/util/spanlogger"
 )
@@ -54,6 +55,8 @@ var (
 	errRequestNoQuery = apierror.New(apierror.TypeBadData, "the request has no query")
 	allFormats        = []string{formatJSON, formatProtobuf}
 
+	responseSizeTooLargeErrorFormat = globalerror.MaxResponseSizeBytes.MessageWithPerInstanceLimitConfig("the query response exceeded the maximum allowed size (limit: %d bytes)", maxResponseSizeBytesFlag)
+
 	// List of HTTP headers to propagate when a Prometheus request is encoded into a HTTP request.
 	// Read consistency level and max delay headers are propagated as HTTP header -> Request.Context -> Request.Header, so there's no need to explicitly propagate it here.
 	codecPropagateHeadersMetrics = []string{compat.ForceFallbackHeaderName, chunkinfologger.ChunkInfoLoggingHeader, api.ReadConsistencyOffsetsHeader, querier.FilterQueryablesHeader}
@@ -226,6 +229,7 @@ type Codec struct {
 	lookbackDelta                                   time.Duration
 	preferredQueryResultResponseFormat              string
 	propagateHeadersMetrics, propagateHeadersLabels []string
+	formats                                         []formatter
 	injector                                        propagation.Injector
 }
 
@@ -240,21 +244,21 @@ type formatter interface {
 	ContentType() v1.MIMEType
 }
 
-var jsonFormatterInstance = jsonFormatter{}
-
-var knownFormats = []formatter{
-	jsonFormatterInstance,
-	ProtobufFormatter{},
-}
-
 func NewCodec(
 	registerer prometheus.Registerer,
 	lookbackDelta time.Duration,
 	queryResultResponseFormat string,
 	propagateHeaders []string,
 	injector propagation.Injector,
+	maxResponseSizeBytes uint64,
 ) Codec {
 	return Codec{
+		formats: []formatter{
+			newJSONFormatter(maxResponseSizeBytes),
+			ProtobufFormatter{
+				maxEncodedSize: maxResponseSizeBytes,
+			},
+		},
 		metrics:                            newCodecMetrics(registerer),
 		lookbackDelta:                      lookbackDelta,
 		preferredQueryResultResponseFormat: queryResultResponseFormat,
@@ -929,7 +933,7 @@ func (c Codec) DecodeMetricsQueryResponse(ctx context.Context, r *http.Response,
 		}
 	}
 
-	formatter := findFormatter(contentType)
+	formatter := c.findFormatter(contentType)
 	if formatter == nil {
 		return nil, apierror.Newf(apierror.TypeInternal, "unknown response content type '%v'", contentType)
 	}
@@ -990,7 +994,7 @@ func (c Codec) DecodeLabelsSeriesQueryResponse(ctx context.Context, r *http.Resp
 		}
 	}
 
-	formatter := findFormatter(contentType)
+	formatter := c.findFormatter(contentType)
 	if formatter == nil {
 		return nil, apierror.Newf(apierror.TypeInternal, "unknown response content type '%v'", contentType)
 	}
@@ -1042,8 +1046,8 @@ func (c Codec) DecodeLabelsSeriesQueryResponse(ctx context.Context, r *http.Resp
 	return response, nil
 }
 
-func findFormatter(contentType string) formatter {
-	for _, f := range knownFormats {
+func (c Codec) findFormatter(contentType string) formatter {
+	for _, f := range c.formats {
 		if f.ContentType().String() == contentType {
 			return f
 		}
@@ -1073,7 +1077,8 @@ func (c Codec) EncodeMetricsQueryResponse(ctx context.Context, req *http.Request
 	start := time.Now()
 	b, err := formatter.EncodeQueryResponse(a)
 	if err != nil {
-		return nil, apierror.Newf(apierror.TypeInternal, "error encoding response: %v", err)
+		typ := apierror.TypeForError(err, apierror.TypeInternal)
+		return nil, apierror.Newf(typ, "error encoding response: %v", err)
 	}
 
 	encodeDuration := time.Since(start)
@@ -1138,7 +1143,8 @@ func (c Codec) EncodeLabelsSeriesQueryResponse(ctx context.Context, req *http.Re
 		var err error
 		b, err = formatter.EncodeLabelsResponse(a)
 		if err != nil {
-			return nil, apierror.Newf(apierror.TypeInternal, "error encoding response: %v", err)
+			typ := apierror.TypeForError(err, apierror.TypeInternal)
+			return nil, apierror.Newf(typ, "error encoding response: %v", err)
 		}
 	case true:
 		a, ok := res.(*PrometheusSeriesResponse)
@@ -1153,7 +1159,8 @@ func (c Codec) EncodeLabelsSeriesQueryResponse(ctx context.Context, req *http.Re
 		var err error
 		b, err = formatter.EncodeSeriesResponse(a)
 		if err != nil {
-			return nil, apierror.Newf(apierror.TypeInternal, "error encoding response: %v", err)
+			typ := apierror.TypeForError(err, apierror.TypeInternal)
+			return nil, apierror.Newf(typ, "error encoding response: %v", err)
 		}
 	}
 
@@ -1172,13 +1179,13 @@ func (c Codec) EncodeLabelsSeriesQueryResponse(ctx context.Context, req *http.Re
 	return &resp, nil
 }
 
-func (Codec) negotiateContentType(acceptHeader string) (string, formatter) {
+func (c Codec) negotiateContentType(acceptHeader string) (string, formatter) {
 	if acceptHeader == "" {
-		return jsonMimeType, jsonFormatterInstance
+		return c.formats[0].ContentType().String(), c.formats[0]
 	}
 
 	for _, clause := range goautoneg.ParseAccept(acceptHeader) {
-		for _, formatter := range knownFormats {
+		for _, formatter := range c.formats {
 			if formatter.ContentType().Satisfies(clause) {
 				return formatter.ContentType().String(), formatter
 			}

pkg/frontend/querymiddleware/codec_json.go

@@ -6,15 +6,33 @@
 package querymiddleware
 
 import (
+	"errors"
+
+	jsoniter "github.com/json-iterator/go"
 	v1 "github.com/prometheus/prometheus/web/api/v1"
+
+	apierror "github.com/grafana/mimir/pkg/api/error"
 )
 
 const jsonMimeType = "application/json"
 
-type jsonFormatter struct{}
+type jsonFormatter struct {
+	encoder jsoniter.API
+}
+
+func newJSONFormatter(maxEncodedSize uint64) jsonFormatter {
+	cfg := jsoniter.Config{
+		EscapeHTML:             false, // No HTML in our responses.
+		SortMapKeys:            true,
+		ValidateJsonRawMessage: true,
+		MaxMarshalledBytes:     maxEncodedSize,
+	}
+
+	return jsonFormatter{encoder: cfg.Froze()}
+}
 
 func (j jsonFormatter) EncodeQueryResponse(resp *PrometheusResponse) ([]byte, error) {
-	return json.Marshal(resp)
+	return j.marshal(resp)
 }
 
 func (j jsonFormatter) DecodeQueryResponse(buf []byte) (*PrometheusResponse, error) {
@@ -28,7 +46,7 @@ func (j jsonFormatter) DecodeQueryResponse(buf []byte) (*PrometheusResponse, err
 }
 
 func (j jsonFormatter) EncodeLabelsResponse(resp *PrometheusLabelsResponse) ([]byte, error) {
-	return json.Marshal(resp)
+	return j.marshal(resp)
 }
 
 func (j jsonFormatter) DecodeLabelsResponse(buf []byte) (*PrometheusLabelsResponse, error) {
@@ -42,7 +60,7 @@ func (j jsonFormatter) DecodeLabelsResponse(buf []byte) (*PrometheusLabelsRespon
 }
 
 func (j jsonFormatter) EncodeSeriesResponse(resp *PrometheusSeriesResponse) ([]byte, error) {
-	return json.Marshal(resp)
+	return j.marshal(resp)
 }
 
 func (j jsonFormatter) DecodeSeriesResponse(buf []byte) (*PrometheusSeriesResponse, error) {
@@ -62,3 +80,19 @@ func (j jsonFormatter) Name() string {
 func (j jsonFormatter) ContentType() v1.MIMEType {
 	return v1.MIMEType{Type: "application", SubType: "json"}
 }
+
+func (j jsonFormatter) marshal(v interface{}) ([]byte, error) {
+	b, err := j.encoder.Marshal(v)
+	if err != nil {
+		var limitErr jsoniter.ExceededMaxMarshalledBytesError
+
+		if errors.As(err, &limitErr) {
+			return nil, apierror.Newf(apierror.TypeTooLargeEntry, "JSON response is too large: "+responseSizeTooLargeErrorFormat, limitErr.MaxMarshalledBytes)
+		}
+
+		return nil, err
+	}
+
+	return b, nil
+
+}