From 9d857366e11427a5e2ebf036c05420037c332576 Mon Sep 17 00:00:00 2001 From: Mikael Grankvist Date: Fri, 10 Apr 2026 09:29:50 +0300 Subject: [PATCH 1/4] feat: add documentation for setting thresholds Add documentation for default and customized thresholds in load tests. --- .../flow/testing/load-testing/thresholds.adoc | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 articles/flow/testing/load-testing/thresholds.adoc diff --git a/articles/flow/testing/load-testing/thresholds.adoc b/articles/flow/testing/load-testing/thresholds.adoc new file mode 100644 index 0000000000..3a3df3fac5 --- /dev/null +++ b/articles/flow/testing/load-testing/thresholds.adoc @@ -0,0 +1,275 @@ +--- +title: Load Testing Thresholds +page-title: Setting Thresholds for Load Tests +description: How to set and control failure thresholds for the load tests +meta-description: Learn how to set and control failure thresholds for the load tests +order: 60 +version: since:com.vaadin:vaadin@V25.2 +--- + += [since:com.vaadin:vaadin@V25.2]#Load Testing Thresholds# + +== Overview + +k6 thresholds define pass/fail criteria for your load tests. + +When a threshold is breached, k6 marks the test as failed, and the Maven build fails if `k6.failOnThreshold=true` (the default). + +The TestBench k6 converter plugin generates some default thresholds automatically in every k6 script produced. +You can configure the default thresholds, disable individual ones, or add custom thresholds for any k6 metric. + +== Default Thresholds + +Every generated k6 script includes these thresholds out of the box: + +[cols="2,3,1", options="header"] +|=== +| Metric | Expression | Default + +| `http_req_duration` +| 95th percentile response time +| `p(95)<2000` (2 seconds) + +| `http_req_duration` +| 99th percentile response time +| `p(99)<5000` (5 seconds) + +| `checks` +| All checks must pass +| `rate==1` with `abortOnFail: true` +|=== + +This generates the following block in the k6 script: + +[source,javascript] +---- +export const options = { + thresholds: { + checks: [{ threshold: 'rate==1', abortOnFail: true, delayAbortEval: '5s' }], + http_req_duration: ['p(95)<2000', 'p(99)<5000'], + }, +} +---- + +== Editing Default Thresholds + +You can change the default threshold values with Maven properties or through configuration in the POM. + +=== Via command line + +[source,bash] +---- +# Tighter response time requirements +mvn k6:convert -Dk6.harFile=recording.har \ + -Dk6.threshold.httpReqDurationP95=1000 \ + -Dk6.threshold.httpReqDurationP99=3000 + +# Allow checks to fail without aborting the test +mvn k6:convert -Dk6.harFile=recording.har \ + -Dk6.threshold.checksAbortOnFail=false + +# Disable the p95 threshold entirely (set to 0) +mvn k6:convert -Dk6.harFile=recording.har \ + -Dk6.threshold.httpReqDurationP95=0 +---- + +=== Via POM configuration + +[source,xml] +---- + + com.vaadin + testbench-converter-plugin + + + 1000 + + 3000 + + false + + +---- + +=== Default threshold parameters + +[cols="3,1,4", options="header"] +|=== +| Parameter | Default | Description + +| `k6.threshold.httpReqDurationP95` +| `2000` +| 95th percentile response time threshold in ms. Set to `0` to disable. + +| `k6.threshold.httpReqDurationP99` +| `5000` +| 99th percentile response time threshold in ms. Set to `0` to disable. + +| `k6.threshold.checksAbortOnFail` +| `true` +| When `true`, the test aborts immediately on the first check failure. When `false`, failures are recorded but the test continues. +|=== + +== Adding Custom Thresholds + +You can add pass/fail thresholds for any https://grafana.com/docs/k6/latest/using-k6/metrics/[k6 built-in metric] using the `k6.threshold.custom` property. + +The format is a comma-separated list of `metric:expression` pairs: + +[source] +---- +metric1:expression1,metric2:expression2,... +---- + +=== Via command line + +[source,bash] +---- +# Fail if more than 1% of requests fail +mvn k6:run -Dk6.testFile=test.js \ + -Dk6.threshold.custom="http_req_failed:rate<0.01" + +# Multiple custom thresholds +mvn k6:run -Dk6.testFile=test.js \ + -Dk6.threshold.custom="http_req_failed:rate<0.01,http_req_waiting:p(95)<500,http_reqs:count>100" +---- + +=== Via POM configuration + +[source,xml] +---- + + com.vaadin + testbench-converter-plugin + + + http_req_failed:rate<0.01,http_req_waiting:p(95)<500 + + +---- + +NOTE: In XML, use `<` instead of `<` inside threshold expressions. + +=== Combining defaults and custom thresholds + +Custom thresholds are added alongside the defaults. +If you add a custom `http_req_duration` expression, it is merged with the default p95/p99 thresholds: + +[source,bash] +---- +# Adds a median threshold alongside the existing p95 and p99 +mvn k6:convert -Dk6.harFile=recording.har \ + -Dk6.threshold.custom="http_req_duration:p(50)<1000" +---- + +This produces: + +[source,javascript] +---- +export const options = { + thresholds: { + checks: [{ threshold: 'rate==1', abortOnFail: true, delayAbortEval: '5s' }], + http_req_duration: ['p(95)<2000', 'p(99)<5000', 'p(50)<1000'], + }, +} +---- + +== Common Threshold Examples + +Below are commonly used threshold configurations for Vaadin load tests. + +=== Strict performance requirements + +[source,bash] +---- +mvn k6:run -Dk6.testFile=test.js \ + -Dk6.threshold.httpReqDurationP95=500 \ + -Dk6.threshold.httpReqDurationP99=1000 \ + -Dk6.threshold.custom="http_req_failed:rate<0.001,http_req_waiting:p(95)<300" +---- + +Generated output: +[source,javascript] +---- +thresholds: { + checks: [{ threshold: 'rate==1', abortOnFail: true, delayAbortEval: '5s' }], + http_req_duration: ['p(95)<500', 'p(99)<1000'], + http_req_failed: ['rate<0.001'], + http_req_waiting: ['p(95)<300'], +}, +---- + +=== Relaxed thresholds for stress testing + +[source,bash] +---- +mvn k6:run -Dk6.testFile=test.js \ + -Dk6.threshold.httpReqDurationP95=10000 \ + -Dk6.threshold.httpReqDurationP99=0 \ + -Dk6.threshold.checksAbortOnFail=false \ + -Dk6.threshold.custom="http_req_failed:rate<0.05" +---- + +Generated output: +[source,javascript] +---- +thresholds: { + checks: ['rate==1'], + http_req_duration: ['p(95)<10000'], + http_req_failed: ['rate<0.05'], +}, +---- + +=== Throughput minimum + +[source,bash] +---- +mvn k6:run -Dk6.testFile=test.js \ + -Dk6.threshold.custom="http_reqs:count>1000" +---- + +== k6 Metric Reference + +The following k6 built-in metrics can be used with custom thresholds. +For the full reference, see the https://grafana.com/docs/k6/latest/using-k6/metrics/reference/[k6 metrics documentation]. + +[cols="2,2,3", options="header"] +|=== +| Metric | Type | Description + +| `http_req_duration` +| Trend +| Total time for the request (send + wait + receive) + +| `http_req_waiting` +| Trend +| Time spent waiting for response (TTFB) + +| `http_req_connecting` +| Trend +| Time spent establishing TCP connection + +| `http_req_sending` +| Trend +| Time spent sending data + +| `http_req_receiving` +| Trend +| Time spent receiving data + +| `http_req_failed` +| Rate +| Rate of failed requests + +| `http_reqs` +| Counter +| Total number of requests + +| `checks` +| Rate +| Rate of successful checks +|=== + +*Trend* metrics support: `p(N)value` + +*Counter* metrics support: `countvalue` \ No newline at end of file From 693c04161536466ea74f98e7791a14614823f48d Mon Sep 17 00:00:00 2001 From: Mikael Grankvist Date: Fri, 10 Apr 2026 14:04:25 +0300 Subject: [PATCH 2/4] should be in title case --- .../flow/testing/load-testing/thresholds.adoc | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/articles/flow/testing/load-testing/thresholds.adoc b/articles/flow/testing/load-testing/thresholds.adoc index 3a3df3fac5..2f8743bf3b 100644 --- a/articles/flow/testing/load-testing/thresholds.adoc +++ b/articles/flow/testing/load-testing/thresholds.adoc @@ -55,7 +55,7 @@ export const options = { You can change the default threshold values with Maven properties or through configuration in the POM. -=== Via command line +=== Via Command Line [source,bash] ---- @@ -73,7 +73,7 @@ mvn k6:convert -Dk6.harFile=recording.har \ -Dk6.threshold.httpReqDurationP95=0 ---- -=== Via POM configuration +=== Via POM Configuration [source,xml] ---- @@ -91,7 +91,7 @@ mvn k6:convert -Dk6.harFile=recording.har \ ---- -=== Default threshold parameters +=== Default Threshold Parameters [cols="3,1,4", options="header"] |=== @@ -121,7 +121,7 @@ The format is a comma-separated list of `metric:expression` pairs: metric1:expression1,metric2:expression2,... ---- -=== Via command line +=== Via Command Line [source,bash] ---- @@ -134,7 +134,7 @@ mvn k6:run -Dk6.testFile=test.js \ -Dk6.threshold.custom="http_req_failed:rate<0.01,http_req_waiting:p(95)<500,http_reqs:count>100" ---- -=== Via POM configuration +=== Via POM Configuration [source,xml] ---- @@ -150,7 +150,7 @@ mvn k6:run -Dk6.testFile=test.js \ NOTE: In XML, use `<` instead of `<` inside threshold expressions. -=== Combining defaults and custom thresholds +=== Combining Defaults and Custom Thresholds Custom thresholds are added alongside the defaults. If you add a custom `http_req_duration` expression, it is merged with the default p95/p99 thresholds: @@ -178,7 +178,7 @@ export const options = { Below are commonly used threshold configurations for Vaadin load tests. -=== Strict performance requirements +=== Strict Performance Requirements [source,bash] ---- @@ -199,7 +199,7 @@ thresholds: { }, ---- -=== Relaxed thresholds for stress testing +=== Relaxed Thresholds for Stress Testing [source,bash] ---- @@ -220,7 +220,7 @@ thresholds: { }, ---- -=== Throughput minimum +=== Throughput Minimum [source,bash] ---- @@ -243,7 +243,7 @@ For the full reference, see the https://grafana.com/docs/k6/latest/using-k6/metr | `http_req_waiting` | Trend -| Time spent waiting for response (TTFB) +| Time spent waiting for response (Time to First Byte) | `http_req_connecting` | Trend From 134cace5a8d443633b99a0cc31f7f91c150aa1ca Mon Sep 17 00:00:00 2001 From: Mikael Grankvist Date: Tue, 14 Apr 2026 09:32:24 +0300 Subject: [PATCH 3/4] minor fix for tables --- articles/flow/testing/load-testing/thresholds.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/articles/flow/testing/load-testing/thresholds.adoc b/articles/flow/testing/load-testing/thresholds.adoc index 2f8743bf3b..f977e99b3f 100644 --- a/articles/flow/testing/load-testing/thresholds.adoc +++ b/articles/flow/testing/load-testing/thresholds.adoc @@ -22,7 +22,7 @@ You can configure the default thresholds, disable individual ones, or add custom Every generated k6 script includes these thresholds out of the box: -[cols="2,3,1", options="header"] +[cols="2,3,2", options="header"] |=== | Metric | Expression | Default @@ -93,7 +93,7 @@ mvn k6:convert -Dk6.harFile=recording.har \ === Default Threshold Parameters -[cols="3,1,4", options="header"] +[cols="3,1,3", options="header"] |=== | Parameter | Default | Description From 3558f5b4fd6a72997c482b56941b809f1cfc7f0f Mon Sep 17 00:00:00 2001 From: Mikael Grankvist Date: Wed, 15 Apr 2026 07:13:08 +0300 Subject: [PATCH 4/4] fix < showing as < Note that custom overrides default thresholds. --- .../flow/testing/load-testing/thresholds.adoc | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/articles/flow/testing/load-testing/thresholds.adoc b/articles/flow/testing/load-testing/thresholds.adoc index f977e99b3f..a0a69c6bb0 100644 --- a/articles/flow/testing/load-testing/thresholds.adoc +++ b/articles/flow/testing/load-testing/thresholds.adoc @@ -148,11 +148,12 @@ mvn k6:run -Dk6.testFile=test.js \ ---- -NOTE: In XML, use `<` instead of `<` inside threshold expressions. +NOTE: In XML, use `\<` instead of `<` inside threshold expressions. === Combining Defaults and Custom Thresholds Custom thresholds are added alongside the defaults. + If you add a custom `http_req_duration` expression, it is merged with the default p95/p99 thresholds: [source,bash] @@ -174,6 +175,27 @@ export const options = { } ---- +If a custom threshold contains same target as the default, the default gets overridden by the custom one and not duplicated. + +[source,bash] +---- +# Overrides existing default p95 median threshold +mvn k6:convert -Dk6.harFile=recording.har \ + -Dk6.threshold.custom="http_req_duration:p(95)<500" +---- + +This produces: + +[source,javascript] +---- +export const options = { + thresholds: { + checks: [{ threshold: 'rate==1', abortOnFail: true, delayAbortEval: '5s' }], + http_req_duration: ['p(95)<500', 'p(99)<5000'], + }, +} +---- + == Common Threshold Examples Below are commonly used threshold configurations for Vaadin load tests.