From 264dbf5e6e03c971961ce92ad710557fb0cc9634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 11:58:12 +0800 Subject: [PATCH 01/16] feat(benchmark): add pool benchmark with standalone mock server and per-API QPS tracking Standalone Go mock server (lifecycle + execd API per specs/) with configurable per-route response time (uniform/fixed/lognormal), fault injection (create failure, execd failure, poisoned sandboxes), server-side TTL expiry, and exact per-second per-API QPS tracking with full time series. Kotlin/JVM driver runs 7 reproducible scenarios against the mock (cold-start, warm-latency, steady-state, replenish-lag, failure-injection, stale-idle, idle-expiry) and writes JSON + Markdown reports including per-scenario server-side QPS. run.sh orchestrates SDK publish, mock start, and driver run; the mock is reusable from any SDK via ConnectionConfig. --- tests/benchmark/.gitignore | 2 + tests/benchmark/README.md | 238 ++++++++ tests/benchmark/configs/default.json | 43 ++ tests/benchmark/configs/fast.json | 11 + tests/benchmark/configs/slow.json | 13 + tests/benchmark/kotlin/build.gradle.kts | 74 +++ tests/benchmark/kotlin/gradle.properties | 4 + .../kotlin/gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 45633 bytes .../gradle/wrapper/gradle-wrapper.properties | 7 + tests/benchmark/kotlin/gradlew | 248 +++++++++ tests/benchmark/kotlin/settings.gradle.kts | 17 + .../com/alibaba/opensandbox/benchmark/Cli.kt | 141 +++++ .../com/alibaba/opensandbox/benchmark/Main.kt | 190 +++++++ .../alibaba/opensandbox/benchmark/Metrics.kt | 92 +++ .../opensandbox/benchmark/MockControl.kt | 96 ++++ .../opensandbox/benchmark/PoolRunner.kt | 92 +++ .../opensandbox/benchmark/Scenarios.kt | 395 +++++++++++++ .../main/resources/simplelogger.properties | 4 + tests/benchmark/mockserver/config.go | 188 +++++++ tests/benchmark/mockserver/go.mod | 17 + tests/benchmark/mockserver/main.go | 82 +++ tests/benchmark/mockserver/server.go | 522 ++++++++++++++++++ tests/benchmark/mockserver/stats.go | 210 +++++++ tests/benchmark/run.sh | 129 +++++ 24 files changed, 2815 insertions(+) create mode 100644 tests/benchmark/.gitignore create mode 100644 tests/benchmark/README.md create mode 100644 tests/benchmark/configs/default.json create mode 100644 tests/benchmark/configs/fast.json create mode 100644 tests/benchmark/configs/slow.json create mode 100644 tests/benchmark/kotlin/build.gradle.kts create mode 100644 tests/benchmark/kotlin/gradle.properties create mode 100644 tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.jar create mode 100644 tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.properties create mode 100755 tests/benchmark/kotlin/gradlew create mode 100644 tests/benchmark/kotlin/settings.gradle.kts create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt create mode 100644 tests/benchmark/kotlin/src/main/resources/simplelogger.properties create mode 100644 tests/benchmark/mockserver/config.go create mode 100644 tests/benchmark/mockserver/go.mod create mode 100644 tests/benchmark/mockserver/main.go create mode 100644 tests/benchmark/mockserver/server.go create mode 100644 tests/benchmark/mockserver/stats.go create mode 100755 tests/benchmark/run.sh diff --git a/tests/benchmark/.gitignore b/tests/benchmark/.gitignore new file mode 100644 index 000000000..129417619 --- /dev/null +++ b/tests/benchmark/.gitignore @@ -0,0 +1,2 @@ +bin/ +results/ diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md new file mode 100644 index 000000000..7d38efefb --- /dev/null +++ b/tests/benchmark/README.md @@ -0,0 +1,238 @@ +# OpenSandbox Pool Benchmark + +Reproducible, cross-SDK benchmark harness for the sandbox SDK **client pool** +(`SandboxPool` in the Kotlin/JVM SDK; Go/Python/JS pools can reuse the same +mock server). It runs a standalone mock of the lifecycle + execd API with +configurable provisioning latency and fault injection, drives the pool through +scenario workloads, and writes a JSON + Markdown report. + +``` +tests/benchmark/ +├── mockserver/ # standalone Go mock server (lifecycle + execd + control endpoints) +├── kotlin/ # benchmark driver (JVM, uses the published Kotlin SDK) +├── configs/ # mock server scenario configs (latency / fault profiles) +├── run.sh # one-shot orchestration: publish SDK -> start mock -> run driver +└── results/ # reports and mock logs (gitignored) +``` + +## Prerequisites + +- Go (any recent version; mock server uses stdlib only) +- JDK 17+ (driver + Kotlin SDK) +- A Gradle wrapper is included under `kotlin/`. + +## Quick start + +```bash +# default config: create/delete 300-800ms, execd ping 1-5s, others 50-100ms +./run.sh + +# smoke run with fast provisioning +./run.sh --mock-config configs/fast.json -- --scenarios cold-start,warm-latency + +# full control over the driver +./run.sh -- --max-idle 50 --warmup-concurrency 10 --steady-duration-s 120 +``` + +`run.sh` performs three steps: + +1. Publishes the Kotlin SDK to `mavenLocal` (`sdks/sandbox/kotlin/publishToMavenLocal`); + the driver then resolves that exact version (`project.version` from + `gradle.properties`). Pass `--skip-sdk-publish` to reuse the last published + snapshot. +2. Builds and starts the mock server (`go build` + exec). +3. Runs the driver: `./gradlew run` with forwarded `--key value` args. + +Reports land in `results/run-/report.{json,md}`; the mock log is at +`results/mockserver.log`. Exit code is non-zero when a scenario fails. + +## Mock server + +The mock implements the API surface the SDK pool actually drives, per +`specs/sandbox-lifecycle.yml` and `specs/execd-api.yaml`: + +| Endpoint | Behavior | +|---|---| +| `POST /v1/sandboxes` | Simulated provisioning: sleeps `createLatencyMs`, returns `Pending`, flips to `Running` after `bootDelayMs` | +| `GET /v1/sandboxes/{id}` | Sandbox info; 404 after kill/expiry | +| `POST /v1/sandboxes/{id}/renew-expiration` | Applies server-side TTL | +| `DELETE /v1/sandboxes/{id}` | Marks terminated; execd stops responding | +| `GET /v1/sandboxes/{id}/endpoints/{port}` | Returns the execd URL + a per-sandbox access token | +| execd (`/ping`, anything) | 200 when the sandbox is booted and healthy; **404** while `Pending`/expired/poisoned | + +**Why execd answers 404 while a sandbox is not ready**: the SDK's readiness +check polls execd `/ping` (`Sandbox.checkReady`), and its retry interceptor +retries 5xx and transport errors with backoff. A 404 is non-retryable, so the +client polls at its configured `healthCheckPollingInterval` instead of paying +policy backoff — keeping the benchmark's latency numbers clean. The +`execdFailureRate` fault injects 500s when you *want* the retry policy +engaged. + +**Why boot state lives in execd**: readiness is decided by execd `/ping`, not +the lifecycle GET, so the mock attributes each execd request to a sandbox via +the endpoint token and only answers once that sandbox is `Running`. + +Server-side state and counters are exposed for the driver to validate pool +behavior (no over-creation, stale cleanup, hit ratio): + +- `GET /__stats` — counters + per-route QPS/latency + live config +- `POST /__config` — runtime fault injection: `createFailureRate`, + `execdFailureRate`, `bootDelayMs`, `poisonExisting` (flips all alive + sandboxes to a failing state, simulating stale idles); latency knobs are + runtime-mutable too (`createLatencyMs`, `latencyOverrides` — the latter + replaces the whole per-route map) +- `POST /__reset` — zero counters and QPS history + +### Per-API QPS tracking + +The mock records an exact per-second count for every request on each route +(`lifecycle.create|get|delete|renew|endpoint`, `execd.ping|other`) in a ring +buffer (`-stats-window-sec`, default 1800s). `/__stats` returns per route: + +```json +"lifecycle.create": { + "total": 244, "qps1s": 2.0, "qps5s": 3.2, "qps60s": 1.8, + "avgMs": 100.2, "maxMs": 101, + "seriesStartUnixSec": 1786678587, "series": [2, 4, 2, ...] +} +``` + +- `total` counts since the last `__reset`; `series` is the per-second request + count covering `seriesStartUnixSec .. now` (older than the window is + dropped; the driver resets before each scenario so each section is + self-contained). +- The driver attaches a QPS snapshot to **every scenario section** + (`results..mockQps` in `report.json`), and the end-of-run totals + live under `results.mockServerStats`. This is the server-side ground truth + for offline analysis: warmup bursts, reconcile-tick load, replenish spikes, + and per-API request mixes (e.g. how many `renew-expiration` calls each + acquire generates). + +For runs longer than the window, poll `GET /__stats` from your analysis tool +and accumulate the series yourself. + +### Mock config (JSON) + +```json +{ + "createLatencyMs": { "distribution": "lognormal", "meanMs": 800, "stddevMs": 400, "minMs": 50 }, + "createFailureRate": 0.0, + "bootDelayMs": 300, + "execdFailureRate": 0.0, + "defaultTtlSeconds": 3600, + "latencyOverrides": { + "lifecycle.renew": { "distribution": "fixed", "meanMs": 20 }, + "lifecycle.endpoint": { "distribution": "lognormal", "meanMs": 100, "stddevMs": 30, "minMs": 10 }, + "execd.ping": { "distribution": "fixed", "meanMs": 40 } + } +} +``` + +**Response time model** — three independent knobs: + +| Knob | What it controls | +|---|---| +| `createLatencyMs` | `POST /v1/sandboxes` response time (the server sleeps this long) | +| `bootDelayMs` | How long a created sandbox stays `Pending`; during this window execd pings fail **immediately** with 404 (no latency is paid) and the SDK readiness poll keeps retrying at its polling interval | +| `latencyOverrides` | Response time per route: `lifecycle.get`, `lifecycle.delete`, `lifecycle.renew`, `lifecycle.endpoint`, `execd.ping`, `execd.other`. Routes without an override respond immediately; an override for `lifecycle.create` replaces `createLatencyMs`. Execd route latency only applies once the sandbox is booted — not-ready probes fail fast | + +Default profile (no `-config`): create/delete uniform **300-800ms**, execd +`/ping` uniform **1-5s** (readiness probes are slow), all other APIs uniform +**50-100ms**. + +The readiness sequence a client observes is therefore: create latency, then a +few fast `404` polls while the sandbox boots, then a slow successful ping — +typically one ping for the default profile (min 1s ping vs. max 300ms boot +window). + +So the full create-to-ready time a client observes is +`createLatencyMs + bootDelayMs` plus one successful ping (once booted, the +ready execd pays its route latency). All latency knobs take a `LatencySpec`: +`distribution` is `uniform` (random between `minMs` and `maxMs`), `fixed` +(always `meanMs`), or `lognormal` (`meanMs`/`stddevMs`, floored at `minMs`). +They are also runtime-mutable via `POST /__config`; sending `latencyOverrides` +replaces the whole per-route map, and the resulting response times are visible +in `/__stats` per-route `avgMs`/`maxMs` and the QPS series. + +Presets: `default.json`, `fast.json` (smoke tests), `slow.json`. + +## Driver + +Run the driver standalone (mock already up): + +```bash +cd kotlin +./gradlew --console=plain run --args="--mock-base-url http://127.0.0.1:18080 --scenarios all" +``` + +### Scenarios + +| Scenario | What it measures | +|---|---| +| `cold-start` | Time from `pool.start()` until idle buffer is full; over-creation check (server `created` vs `maxIdle`) | +| `warm-latency` | acquire p50/p90/p95/p99/p999 + hit ratio from a warm pool (`N` workers × `M` rounds) | +| `steady-state` | Sustained acquires/sec under concurrent loaders with hold time; idle trajectory (min/mean/empty ratio) | +| `replenish-lag` | Time for a released idle slot to be refilled (completion-driven reconcile) | +| `failure-injection` | Pool behavior at `createFailureRate` 60%: success rate, backoff, DEGRADED transition, recovery after fault removal | +| `stale-idle` | Poisoned idle candidates: retry cost, stale cleanup, refill with fresh sandboxes | +| `idle-expiry` | Self-healing under short server-side TTL: reap + recreate keeps the buffer near `maxIdle` | + +### Driver options + +| Option | Default | Meaning | +|---|---|---| +| `--scenarios` | `all` | Comma-separated list: `cold-start`, `warm-latency`, `steady-state`, `replenish-lag`, `failure-injection`, `stale-idle`, `idle-expiry` | +| `--mock-base-url` | `http://127.0.0.1:18080` | Mock lifecycle base URL | +| `--report-dir` | `results/run-` | Report output directory (`run.sh` passes an absolute path) | +| `--max-idle` | `20` | Pool idle-buffer target | +| `--warmup-concurrency` | `4` | Concurrent warmup creation workers | +| `--reconcile-interval-ms` | `1000` | Pool reconcile tick interval | +| `--idle-timeout-s` | `1800` | Server-side TTL applied to pool-created sandboxes | +| `--acquire-ready-timeout-ms` | `15000` | `checkReady` timeout when acquiring (idle connect + direct create) | +| `--warmup-ready-timeout-ms` | `15000` | `checkReady` timeout for warmup creations | +| `--health-check-polling-interval-ms` | `200` | `checkReady` probe interval (execd ping cadence) | +| `--cold-start-timeout-ms` | `120000` | Max time to wait for the pool to fill | +| `--warm-workers` | `16` | Loader threads in `warm-latency` | +| `--warm-rounds-per-worker` | `150` | Acquire rounds per `warm-latency` worker | +| `--steady-workers` | `16` | Loader threads in `steady-state` | +| `--steady-duration-s` | `60` | `steady-state` run duration | +| `--hold-min-ms` | `1000` | Lower bound of the random hold time per acquired sandbox in `steady-state` | +| `--hold-max-ms` | `5000` | Upper bound of the random hold time per acquired sandbox in `steady-state` | +| `--replenish-rounds` | `20` | Kill-and-wait repetitions in `replenish-lag` | +| `--replenish-wait-timeout-ms` | `15000` | Max time to wait for one replenished slot | +| `--failure-create-rate` | `0.6` | Create failure rate injected in `failure-injection` | +| `--failure-acquires` | `60` | Acquire attempts in `failure-injection` | +| `--stale-acquires` | `100` | Acquire attempts in `stale-idle` | +| `--stale-retries` | `3` | Pool `maxAcquireRetries` in `stale-idle` (idle candidates tried per acquire) | +| `--stale-acquire-ready-timeout-ms` | `3000` | `acquireReadyTimeout` in `stale-idle`; short because the SDK polls a failing execd for the full timeout before discarding a candidate | +| `--idle-expiry-idle-timeout-s` | `20` | `idleTimeout` in `idle-expiry` (short TTL so server-side expiry is exercised) | +| `--idle-expiry-duration-s` | `40` | `idle-expiry` run duration | + +Each scenario resets the mock's counters and QPS history first, so +`report.json`'s per-scenario QPS sections cover exactly that scenario. + +For `warm-latency`, keep `maxIdle` comfortably above `--warm-workers` if you +want to measure pure idle-hit latency: when workers outnumber the idle buffer, +acquires drain it and fall through to direct create (which the `hitRatio` +column will show). + +## Reusing the mock from other SDKs + +Point any SDK's `ConnectionConfig` at the mock: + +```kotlin +ConnectionConfig.builder() + .domain("127.0.0.1:18080") // lifecycle API (no scheme; driver adds /v1) + .protocol("http") + .build() +``` + +The Go SDK's `pool_test.go` shows the same wiring for Go. Health checks and +endpoint lookups behave like a real server, so the mock doubles as a +deterministic test fixture for SDK e2e-style tests. + +## Adding a scenario + +Implement it in `kotlin/.../benchmark/Scenarios.kt`, returning a +`Map` (nested maps render as sections in Markdown), register it in +`ALL_SCENARIOS`, and add a `--` default in `Cli.kt` when it needs knobs. diff --git a/tests/benchmark/configs/default.json b/tests/benchmark/configs/default.json new file mode 100644 index 000000000..21436346c --- /dev/null +++ b/tests/benchmark/configs/default.json @@ -0,0 +1,43 @@ +{ + "createLatencyMs": { + "distribution": "uniform", + "minMs": 300, + "maxMs": 800 + }, + "createFailureRate": 0.0, + "bootDelayMs": 300, + "execdFailureRate": 0.0, + "defaultTtlSeconds": 3600, + "latencyOverrides": { + "lifecycle.delete": { + "distribution": "uniform", + "minMs": 300, + "maxMs": 800 + }, + "lifecycle.get": { + "distribution": "uniform", + "minMs": 50, + "maxMs": 100 + }, + "lifecycle.renew": { + "distribution": "uniform", + "minMs": 50, + "maxMs": 100 + }, + "lifecycle.endpoint": { + "distribution": "uniform", + "minMs": 50, + "maxMs": 100 + }, + "execd.ping": { + "distribution": "uniform", + "minMs": 1000, + "maxMs": 5000 + }, + "execd.other": { + "distribution": "uniform", + "minMs": 50, + "maxMs": 100 + } + } +} diff --git a/tests/benchmark/configs/fast.json b/tests/benchmark/configs/fast.json new file mode 100644 index 000000000..2aa4e2328 --- /dev/null +++ b/tests/benchmark/configs/fast.json @@ -0,0 +1,11 @@ +{ + "createLatencyMs": { + "distribution": "fixed", + "meanMs": 100 + }, + "createFailureRate": 0.0, + "bootDelayMs": 50, + "execdFailureRate": 0.0, + "defaultTtlSeconds": 3600, + "latencyOverrides": {} +} diff --git a/tests/benchmark/configs/slow.json b/tests/benchmark/configs/slow.json new file mode 100644 index 000000000..64af9e101 --- /dev/null +++ b/tests/benchmark/configs/slow.json @@ -0,0 +1,13 @@ +{ + "createLatencyMs": { + "distribution": "lognormal", + "meanMs": 2000, + "stddevMs": 1000, + "minMs": 100 + }, + "createFailureRate": 0.0, + "bootDelayMs": 1000, + "execdFailureRate": 0.0, + "defaultTtlSeconds": 3600, + "latencyOverrides": {} +} diff --git a/tests/benchmark/kotlin/build.gradle.kts b/tests/benchmark/kotlin/build.gradle.kts new file mode 100644 index 000000000..64db29fa9 --- /dev/null +++ b/tests/benchmark/kotlin/build.gradle.kts @@ -0,0 +1,74 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + kotlin("jvm") version "2.2.21" + application +} + +group = "com.alibaba.opensandbox" +version = "1.0.0" + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +repositories { + mavenLocal() + exclusiveContent { + forRepository { + mavenLocal() + } + filter { + includeGroup("com.alibaba.opensandbox") + } + } + mavenCentral() +} + +configurations.configureEach { + resolutionStrategy.cacheDynamicVersionsFor(0, "seconds") + resolutionStrategy.cacheChangingModulesFor(0, "seconds") +} + +dependencies { + // OpenSandbox Kotlin SDK (published to mavenLocal; see tests/benchmark/README.md). + // The version is taken from the SDK's own gradle.properties so run.sh and + // this module cannot drift; pass -PsandboxVersion=... to override. + val sandboxVersion = (project.findProperty("sandboxVersion") as String?) ?: "1.0.18" + implementation("com.alibaba.opensandbox:sandbox:$sandboxVersion") + + implementation("com.squareup.okhttp3:okhttp:4.12.0") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + implementation("org.slf4j:slf4j-simple:2.0.9") +} + +application { + mainClass.set("com.alibaba.opensandbox.benchmark.MainKt") +} + +tasks.withType { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_11) + } +} + +tasks.withType { + sourceCompatibility = "11" + targetCompatibility = "11" +} + diff --git a/tests/benchmark/kotlin/gradle.properties b/tests/benchmark/kotlin/gradle.properties new file mode 100644 index 000000000..e50be0b79 --- /dev/null +++ b/tests/benchmark/kotlin/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m +org.gradle.parallel=true +org.gradle.caching=true +org.gradle.configuration-cache=true diff --git a/tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.jar b/tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..f8e1ee3125fe0768e9a76ee977ac089eb657005e GIT binary patch literal 45633 zcma&NV|1n6wyqu9PQ|uu+csuwn-$x(T~Woh?Nr6KUD3(A)@l1Yd+oj6Z_U=8`RAE` z#vE6_`?!1WLs1443=Ieh3JM4ai0JG2|2{}S&_HrxszP*9^5P7#QX*pVDq?D?;6T8C z{bWO1$9at%!*8ax*TT&F99vwf1Ls+3lklsb|bC`H`~Q z_w}*E9P=Wq;PYlGYhZ^lt#N97bt5aZ#mQcOr~h^B;R>f-b0gf{y(;VA{noAt`RZzU z7vQWD{%|q!urW2j0Z&%ChtL(^9m` zgaU%|B;V#N_?%iPvu0PVkX=1m9=*SEGt-Lp#&Jh%rz6EJXlV^O5B5YfM5j{PCeElx z8sipzw8d=wVhFK+@mgrWyA)Sv3BJq=+q+cL@=wuH$2;LjY z^{&+X4*HFA0{QvlM_V4PTQjIdd;d|2YuN;s|bi!@<)r-G%TuOCHz$O(_-K z)5in&6uNN<0UfwY=K>d;cL{{WK2FR|NihJMN0Q4X+(1lE)$kY?T$7UWleIU`i zQG#X-&&m-8x^(;n@o}$@vPMYRoq~|FqC~CU3MnoiifD{(CwAGd%X#kFHq#4~%_a!{ zeX{XXDT#(DvX7NtAs7S}2ZuiZ>gtd;tCR7E)3{J^`~#Vd**9qz%~JRFAiZf{zt|Dr zvQw!)n7fNUn_gH`o9?8W8t_%x6~=y*`r46bjj(t{YU*qfqd}J}*mkgUfsXTI>Uxl6 z)Fj>#RMy{`wINIR;{_-!xGLgVaTfNJ2-)%YUfO&X5z&3^E#4?k-_|Yv$`fpgYkvnA%E{CiV zP|-zAf8+1@R`sT{rSE#)-nuU7Pwr-z>0_+CLQT|3vc-R22ExKT4ym@Gj77j$aTVns zp4Kri#Ml?t7*n(;>nkxKdhOU9Qbwz%*#i9_%K<`m4T{3aPbQ?J(Mo`6E5cDdbAk%X z+4bN%E#a(&ZXe{G#V!2Nt+^L$msKVHP z|APpBhq7knz(O2yY)$$VyI_Xg4UIC*$!i7qQG~KEZnO@Q1i89@4ZKW*3^Wh?o?zSkfPxdhnTxlO!3tAqe_ zuEqHVcAk3uQIFTpP~C{d$?>7yt3G3Fo>syXTus>o0tJdFpQWC27hDiwC%O09i|xCq z@H6l|+maB;%CYQIChyhu;PVYz9e&5a@EEQs3$DS6dLIS+;N@I0)V}%B`jdYv;JDck zd|xxp(I?aedivE7*19hesoa-@Xm$^EHbbVmh$2^W-&aTejsyc$i+}A#n2W*&0Qt`5 zJS!2A|LVV;L!(*x2N)GjJC;b1RB_f(#D&g_-};a*|BTRvfdIX}Gau<;uCylMNC;UG zzL((>6KQBQ01wr%7u9qI2HLEDY!>XisIKb#6=F?pAz)!_JX}w|>1V>X^QkMdFi@Jr z`1N*V4xUl{qvECHoF?#lXuO#Dg2#gh|AU$Wc=nuIbmVPBEGd(R#&Z`TP9*o%?%#ob zWN%ByU+55yBNfjMjkJnBjT!cVDi}+PR3N&H(f8$d^Pu;A_WV*{)c2Q{IiE7&LPsd4 z!rvkUf{sco_WNSIdW+btM#O+4n`JiceH6%`7pDV zRqJ@lj=Dt(e-Gkz$b!c2>b)H$lf(fuAPdIsLSe(dZ4E~9+Ge!{3j~>nS%r)eQZ;Iq ztWGpp=2Ptc!LK_TQ8cgJXUlU5mRu|7F2{eu*;a>_5S<;bus=t*IXcfzJRPv4xIs;s zt2<&}OM>KxkTxa=dFMfNr42=DL~I}6+_{`HT_YJBiWkpVZND1Diad~Yr*Fuq{zljr z*_+jXk=qVBdwlQkYuIrB4GG*#voba$?h*u0uRNL+87-?AjzG2X_R9mzQ7BJEawutObr|ey~%in>6k%A`K*`pb-|DF5m})!`b=~osoiW2)IFh?_y9y<3Cix_ znvC=bjBX1J820!%%9FaB@v?hAsd05e@w$^ZAvtUp*=Bi+Owkl?rLa6F#yl{s+?563 zmn2 zV95%gySAJ$L!Vvk4kx!n@mo`3Mfi`2lXUkBmd%)u)7C?Pa;oK~zUQ#p0u{a|&0;zNO#9a4`v^3df90X#~l_k$q7n&L5 z?TszF842~g+}tgUP}UG?ObLCE1(Js_$e>XS7m%o7j@@VdxePtg)w{i5an+xK95r?s zDeEhgMO-2$H?@0{p-!4NJ)}zP+3LzZB?FVap)ObHV6wp}Lrxvz$cjBND1T6ln$EfJ zZRPeR2lP}K0p8x`ahxB??Ud;i7$Y5X!5}qBFS+Zp=P^#)08nQi_HuJcN$0=x;2s53 zwoH}He9BlKT4GdWfWt)@o@$4zN$B@5gVIN~aHtwIhh{O$uHiMgYl=&Vd$w#B2 zRv+xK3>4E{!)+LXA2#*K6H~HpovXAQeXV(^Pd%G_>ro0(4_@`{2Ag(+8{9pqJ>Co$ zRRV(oX;nD+Jel_2^BlNO=cQP8q*G#~R3PTERUxvug_C4T3qwb9MQE|^{5(H*nt`fn z^%*p-RwkAhT6(r>E@5w8FaB)Q<{#`H9fTdc6QBuSr9D-x!Tb9f?wI=M{^$cB5@1;0 z+yLHh?3^c-Qte@JI<SW`$bs5Vv9!yWjJD%oY z8Cdc$a(LLy@tB2)+rUCt&0$&+;&?f~W6+3Xk3g zy9L�|d9Zj^A1Dgv5yzCONAB>8LM`TRL&7v_NKg(bEl#y&Z$py}mu<4DrT@8HHjE zqD@4|aM>vt!Yvc2;9Y#V;KJ8M>vPjiS2ycq52qkxInUK*QqA3$&OJ`jZBo zpzw&PT%w0$D94KD%}VN9c)eCueh1^)utGt2OQ+DP(BXszodfc1kFPWl~BQ5Psy*d`UIf zc}zQ8TVw35jdCSc78)MljC-g3$GX2$<0<3MEQXS&i<(ZFClz9WlL}}?%u>S2hhEk_ zyzfm&@Q%YVB-vw3KH|lU#c_)0aeG^;aDG&!bwfOz_9)6gLe;et;h(?*0d-RV0V)1l zzliq#`b9Y*c`0!*6;*mU@&EFSbW>9>L5xUX+unp%@tCW#kLfz)%3vwN{1<-R*g+B_C^W8)>?n%G z<#+`!wU$L&dn)Pz(9DGGI%RlmM2RpeDy9)31OZV$c2T>-Jl&4$6nul&e7){1u-{nP zE$uZs%gyanu+yBcAb+jTYGy(^<;&EzeLeqveN12Lvv)FQFn0o&*qAaH+gLJ)*xT9y z>`Y`W?M#K7%w26w?Oen>j7=R}EbZ;+jcowV&i}P|IfW^C5GJHt5D;Q~)|=gW3iQ;N zQGl4SQFtz=&~BGon6hO@mRnjpmM79ye^LY_L2no{f_M?j80pr`o3BrI7ice#8#Zt4 zO45G97Hpef+AUEU%jN-dLmPYHY(|t#D)9|IeB^i1X|eEq+ymld_Uj$l^zVAPRilx- z^II$sL4G~{^7?sik2BK7;ZV-VIVhrKjUxBIsf^N&K`)5;PjVg-DTm1Xtw4-tGtElU zJgVTCk4^N4#-kPuX=7p~GMf5Jj5A#>)GX)FIcOqY4lf}Vv2gjrOTuFusB@ERW-&fb zTp=E0E?gXkwzn)AMMY*QCftp%MOL-cbsG{02$0~b?-JD{-nwj58 zBHO1YL~yn~RpnZ6*;XA|MSJeBfX-D?afH*E!2uGjT%k!jtx~OG_jJ`Ln}lMQb7W41 zmTIRd%o$pu;%2}}@2J$x%fg{DZEa-Wxdu6mRP~Ea0zD2+g;Dl*to|%sO-5mUrZ`~C zjJ zUe^**YRgBvlxl<(r0LjxjSQKiTx+E<7$@9VO=RYgL9ldTyKzfqR;Y&gu^ub!fVX7u z3H@;8j#tVgga~EMuXv_#Q8<*uK@R{mGzn92eDYkF1sbxh5!P|M-D)T~Ae*SO`@u$Q z7=5s)HM)w~s2j5{I67cqSn6BLLhCMcn0=OTVE?T7bAmY!T+xZ_N3op~wZ3Oxlm6(a5qB({6KghlvBd9HJ#V6YY_zxbj-zI`%FN|C*Q`DiV z#>?Kk7VbuoE*I9tJaa+}=i7tJnMRn`P+(08 za*0VeuAz!eI7giYTsd26P|d^E2p1f#oF*t{#klPhgaShQ1*J7?#CTD@iDRQIV+Z$@ z>qE^3tR3~MVu=%U%*W(1(waaFG_1i5WE}mvAax;iwZKv^g1g}qXY7lAd;!QQa#5e= z1_8KLHje1@?^|6Wb(A{HQ_krJJP1GgE*|?H0Q$5yPBQJlGi;&Lt<3Qc+W4c}Ih~@* zj8lYvme}hwf@Js%Oj=4BxXm15E}7zS0(dW`7X0|$damJ|gJ6~&qKL>gB_eC7%1&Uh zLtOkf7N0b;B`Qj^9)Bfh-( z0or96!;EwEMnxwp!CphwxxJ+DDdP4y3F0i`zZp-sQ5wxGIHIsZCCQz5>QRetx8gq{ zA33BxQ}8Lpe!_o?^u2s3b!a-$DF$OoL=|9aNa7La{$zI#JTu_tYG{m2ly$k?>Yc); zTA9ckzd+ibu>SE6Rc=Yd&?GA9S5oaQgT~ER-|EwANJIAY74|6 z($#j^GP}EJqi%)^jURCj&i;Zl^-M9{=WE69<*p-cmBIz-400wEewWVEd^21}_@A#^ z2DQMldk_N)6bhFZeo8dDTWD@-IVunEY*nYRON_FYII-1Q@@hzzFe(lTvqm}InfjQ2 zN>>_rUG0Lhaz`s;GRPklV?0 z;~t4S8M)ZBW-ED?#UNbCrsWb=??P># zVc}MW_f80ygG_o~SW+Q6oeIUdFqV2Fzys*7+vxr^ZDeXcZZc;{kqK;(kR-DKL zByDdPnUQgnX^>x?1Tz~^wZ%Flu}ma$Xmgtc7pSmBIH%&H*Tnm=L-{GzCv^UBIrTH5 zaoPO|&G@SB{-N8Xq<+RVaM_{lHo@X-q}`zjeayVZ9)5&u*Y>1!$(wh9Qoe>yWbPgw zt#=gnjCaT_+$}w^*=pgiHD8N$hzqEuY5iVL_!Diw#>NP7mEd?1I@Io+?=$?7cU=yK zdDKk_(h_dB9A?NX+&=%k8g+?-f&`vhAR}&#zP+iG%;s}kq1~c{ac1@tfK4jP65Z&O zXj8Ew>l7c|PMp!cT|&;o+(3+)-|SK&0EVU-0-c&guW?6F$S`=hcKi zpx{Z)UJcyihmN;^E?*;fxjE3kLN4|&X?H&$md+Ege&9en#nUe=m>ep3VW#C?0V=aS zLhL6v)|%$G5AO4x?Jxy8e+?*)YR~<|-qrKO7k7`jlxpl6l5H&!C4sePiVjAT#)b#h zEwhfkpFN9eY%EAqg-h&%N>E0#%`InXY?sHyptcct{roG42Mli5l)sWt66D_nG2ed@ z#4>jF?sor7ME^`pDlPyQ(|?KL9Q88;+$C&3h*UV*B+*g$L<{yT9NG>;C^ZmPbVe(a z09K^qVO2agL`Hy{ISUJ{khPKh@5-)UG|S8Sg%xbJMF)wawbgll3bxk#^WRqmdY7qv zr_bqa3{`}CCbREypKd!>oIh^IUj4yl1I55=^}2mZAAW6z}Kpt3_o1b4__sQ;b zv)1=xHO?gE-1FL}Y$0YdD-N!US;VSH>UXnyKoAS??;T%tya@-u zfFo)@YA&Q#Q^?Mtam19`(PS*DL{PHjEZa(~LV7DNt5yoo1(;KT)?C7%^Mg;F!C)q= z6$>`--hQX4r?!aPEXn;L*bykF1r8JVDZ)x4aykACQy(5~POL;InZPU&s5aZm-w1L< z`crCS5=x>k_88n(*?zn=^w*;0+8>ui2i>t*Kr!4?aA1`yj*GXi#>$h8@#P{S)%8+N zCBeL6%!Ob1YJs5+a*yh{vZ8jH>5qpZhz_>(ph}ozKy9d#>gba1x3}`-s_zi+SqIeR z0NCd7B_Z|Fl+(r$W~l@xbeAPl5{uJ{`chq}Q;y8oUN0sUr4g@1XLZQ31z9h(fE_y( z_iQ(KB39LWd;qwPIzkvNNkL(P(6{Iu{)!#HvBlsbm`g2qy&cTsOsAbwMYOEw8!+75D!>V{9SZ?IP@pR9sFG{T#R*6ez2&BmP8*m^6+H2_ z>%9pg(+R^)*(S21iHjLmdt$fmq6y!B9L!%+;wL5WHc^MZRNjpL9EqbBMaMns2F(@h zN0BEqZ3EWGLjvY&I!8@-WV-o@>biD;nx;D}8DPapQF5ivpHVim8$G%3JrHtvN~U&) zb1;=o*lGfPq#=9Moe$H_UhQPBjzHuYw;&e!iD^U2veY8)!QX_E(X@3hAlPBIc}HoD z*NH1vvCi5xy@NS41F1Q3=Jkfu&G{Syin^RWwWX|JqUIX_`}l;_UIsj&(AFQ)ST*5$ z{G&KmdZcO;jGIoI^+9dsg{#=v5eRuPO41<*Ym!>=zHAXH#=LdeROU-nzj_@T4xr4M zJI+d{Pp_{r=IPWj&?%wfdyo`DG1~|=ef?>=DR@|vTuc)w{LHqNKVz9`Dc{iCOH;@H5T{ zc<$O&s%k_AhP^gCUT=uzrzlEHI3q`Z3em0*qOrPHpfl1v=8Xkp{!f9d2p!4 zL40+eJB4@5IT=JTTawIA=Z%3AFvv=l1A~JX>r6YUMV7GGLTSaIn-PUw| z;9L`a<)`D@Qs(@P(TlafW&-87mcZuwFxo~bpa01_M9;$>;4QYkMQlFPgmWv!eU8Ut zrV2<(`u-@1BTMc$oA*fX;OvklC1T$vQlZWS@&Wl}d!72MiXjOXxmiL8oq;sP{)oBe zS#i5knjf`OfBl}6l;BSHeY31w8c~8G>$sJ9?^^!)Z*Z*Xg zbTbkcbBpgFui(*n32hX~sC7gz{L?nlnOjJBd@ zUC4gd`o&YB4}!T9JGTe9tqo0M!JnEw4KH7WbrmTRsw^Nf z^>RxG?2A33VG3>E?iN|`G6jgr`wCzKo(#+zlOIzp-^E0W0%^a>zO)&f(Gc93WgnJ2p-%H-xhe{MqmO z8Iacz=Qvx$ML>Lhz$O;3wB(UI{yTk1LJHf+KDL2JPQ6#m%^bo>+kTj4-zQ~*YhcqS z2mOX!N!Q$d+KA^P0`EEA^%>c12X(QI-Z}-;2Rr-0CdCUOZ=7QqaxjZPvR%{pzd21HtcUSU>u1nw?)ZCy+ zAaYQGz59lqhNXR4GYONpUwBU+V&<{z+xA}`Q$fajmR86j$@`MeH}@zz*ZFeBV9Ot< ze8BLzuIIDxM&8=dS!1-hxiAB-x-cVmtpN}JcP^`LE#2r9ti-k8>Jnk{?@Gw>-WhL=v+H!*tv*mcNvtwo)-XpMnV#X>U1F z?HM?tn^zY$6#|(|S~|P!BPp6mur58i)tY=Z-9(pM&QIHq+I5?=itn>u1FkXiehCRC zW_3|MNOU)$-zrjKnU~{^@i9V^OvOJMp@(|iNnQ%|iojG2_Snnt`1Cqx2t)`vW&w2l zwb#`XLNY@FsnC-~O&9|#Lpvw7n!$wL9azSk)$O}?ygN@FEY({2%bTl)@F2wevCv`; zZb{`)uMENiwE|mti*q5U4;4puX{VWFJ#QIaa*%IHKyrU*HtjW_=@!3SlL~pqLRs?L zoqi&}JLsaP)yEH!=_)zmV-^xy!*MCtc{n|d%O zRM>N>eMG*Qi_XAxg@82*#zPe+!!f#;xBxS#6T-$ziegN-`dLm z=tTN|xpfCPng06|X^6_1JgN}dM<_;WsuL9lu#zLVt!0{%%D9*$nT2E>5@F(>Fxi%Y zpLHE%4LZSJ1=_qm0;^Wi%x56}k3h2Atro;!Ey}#g&*BpbNXXS}v>|nn=Mi0O(5?=1V7y1^1Bdt5h3}oL@VsG>NAH z1;5?|Sth=0*>dbXSQ%MQKB?eN$LRu?yBy@qQVaUl*f#p+sLy$Jd>*q;(l>brvNUbIF0OCf zk%Q;Zg!#0w0_#l)!t?3iz~`X8A>Yd3!P&A4Ov6&EdZmOixeTd4J`*Wutura(}4w@KV>i#rf(0PYL&v^89QiXBP6sj=N;q8kVxS}hA! z|3QaiYz!w+xQ%9&Zg${JgQ*Ip_bg2rmmG`JkX^}&5gbZF!Z(gDD1s5{QwarPK(li- zW9y-CiQ`5Ug1ceN1w7lCxl=2}7c*8_XH8W7y0AICn19qZ`w}z0iCJ$tJ}NjzQCH90 zc!UzpKvk%3;`XfFi2;F*q2eMQQ5fzO{!`KU1T^J?Z64|2Z}b1b6h80_H%~J)J)kbM0hsj+FV6%@_~$FjK9OG7lY}YA zRzyYxxy18z<+mCBiX?3Q{h{TrNRkHsyF|eGpLo0fKUQ|19Z0BamMNE9sW z?vq)r`Qge{9wN|ezzW=@ojpVQRwp##Q91F|B5c`a0A{HaIcW>AnqQ*0WT$wj^5sWOC1S;Xw7%)n(=%^in zw#N*+9bpt?0)PY$(vnU9SGSwRS&S!rpd`8xbF<1JmD&6fwyzyUqk){#Q9FxL*Z9%#rF$} zf8SsEkE+i91VY8d>Fap#FBacbS{#V&r0|8bQa;)D($^v2R1GdsQ8YUk(_L2;=DEyN%X*3 z;O@fS(pPLRGatI93mApLsX|H9$VL2)o(?EYqlgZMP{8oDYS8)3G#TWE<(LmZ6X{YA zRdvPLLBTatiUG$g@WK9cZzw%s6TT1Chmw#wQF&&opN6^(D`(5p0~ zNG~fjdyRsZv9Y?UCK(&#Q2XLH5G{{$9Y4vgMDutsefKVVPoS__MiT%qQ#_)3UUe=2fK)*36yXbQUp#E98ah(v`E$c3kAce_8a60#pa7rq6ZRtzSx6=I^-~A|D%>Riv{Y`F9n3CUPL>d`MZdRmBzCum2K%}z@Z(b7#K!-$Hb<+R@Rl9J6<~ z4Wo8!!y~j(!4nYsDtxPIaWKp+I*yY(ib`5Pg356Wa7cmM9sG6alwr7WB4IcAS~H3@ zWmYt|TByC?wY7yODHTyXvay9$7#S?gDlC?aS147Ed7zW!&#q$^E^_1sgB7GKfhhYu zOqe*Rojm~)8(;b!gsRgQZ$vl5mN>^LDgWicjGIcK9x4frI?ZR4Z%l1J=Q$0lSd5a9 z@(o?OxC72<>Gun*Y@Z8sq@od{7GGsf8lnBW^kl6sX|j~UA2$>@^~wtceTt^AtqMIx zO6!N}OC#Bh^qdQV+B=9hrwTj>7HvH1hfOQ{^#nf%e+l)*Kgv$|!kL5od^ka#S)BNT z{F(miX_6#U3+3k;KxPyYXE0*0CfL8;hDj!QHM@)sekF9uyBU$DRZkka4ie^-J2N8w z3PK+HEv7kMnJU1Y+>rheEpHdQ3_aTQkM3`0`tC->mpV=VtvU((Cq$^(S^p=+$P|@} zueLA}Us^NTI83TNI-15}vrC7j6s_S`f6T(BH{6Jj{Lt;`C+)d}vwPGx62x7WXOX19 z2mv1;f^p6cG|M`vfxMhHmZxkkmWHRNyu2PDTEpC(iJhH^af+tl7~h?Y(?qNDa`|Ogv{=+T@7?v344o zvge%8Jw?LRgWr7IFf%{-h>9}xlP}Y#GpP_3XM7FeGT?iN;BN-qzy=B# z=r$79U4rd6o4Zdt=$|I3nYy;WwCb^`%oikowOPGRUJ3IzChrX91DUDng5_KvhiEZwXl^y z+E!`Z6>}ijz5kq$nNM8JA|5gf_(J-);?SAn^N-(q2r6w31sQh6vLYp^ z<>+GyGLUe_6eTzX7soWpw{dDbP-*CsyKVw@I|u`kVX&6_h5m!A5&3#=UbYHYJ5GK& zLcq@0`%1;8KjwLiup&i&u&rmt*LqALkIqxh-)Exk&(V)gh9@Fn+WU=6-UG^X2~*Q-hnQ$;;+<&lRZ>g0I`~yuv!#84 zy>27(l&zrfDI!2PgzQyV*R(YFd`C`YwR_oNY+;|79t{NNMN1@fp?EaNjuM2DKuG%W z5749Br2aU6K|b=g4(IR39R8_!|B`uQ)bun^C9wR4!8isr$;w$VOtYk+1L9#CiJ#F) z)L}>^6>;X~0q&CO>>ZBo0}|Ex9$p*Hor@Ej9&75b&AGqzpGpM^dx}b~E^pPKau2i5 zr#tT^S+01mMm}z480>-WjU#q`6-gw4BJMWmW?+VXBZ#JPzPW5QQm@RM#+zbQMpr>M zX$huprL(A?yhv8Y81K}pTD|Gxs#z=K(Wfh+?#!I$js5u8+}vykZh~NcoLO?ofpg0! zlV4E9BAY_$pN~e-!VETD&@v%7J~_jdtS}<_U<4aRqEBa&LDpc?V;n72lTM?pIVG+> z*5cxz_iD@3vIL5f9HdHov{o()HQ@6<+c}hfC?LkpBEZ4xzMME^~AdB8?2F=#6ff!F740l&v7FN!n_ zoc1%OfX(q}cg4LDk-1%|iZ^=`x5Vs{oJYhXufP;BgVd*&@a04pSek6OS@*UH`*dAp z7wY#70IO^kSqLhoh9!qIj)8t4W6*`Kxy!j%Bi%(HKRtASZ2%vA0#2fZ=fHe0zDg8^ zucp;9(vmuO;Zq9tlNH)GIiPufZlt?}>i|y|haP!l#dn)rvm8raz5L?wKj9wTG znpl>V@};D!M{P!IE>evm)RAn|n=z-3M9m5J+-gkZHZ{L1Syyw|vHpP%hB!tMT+rv8 zIQ=keS*PTV%R7142=?#WHFnEJsTMGeG*h)nCH)GpaTT@|DGBJ6t>3A)XO)=jKPO<# zhkrgZtDV6oMy?rW$|*NdJYo#5?e|Nj>OAvCXHg~!MC4R;Q!W5xcMwX#+vXhI+{ywS zGP-+ZNr-yZmpm-A`e|Li#ehuWB{{ul8gB&6c98(k59I%mMN9MzK}i2s>Ejv_zVmcMsnobQLkp z)jmsJo2dwCR~lcUZs@-?3D6iNa z2k@iM#mvemMo^D1bu5HYpRfz(3k*pW)~jt8UrU&;(FDI5ZLE7&|ApGRFLZa{yynWx zEOzd$N20h|=+;~w$%yg>je{MZ!E4p4x05dc#<3^#{Fa5G4ZQDWh~%MPeu*hO-6}2*)t-`@rBMoz&gn0^@c)N>z|Ikj8|7Uvdf5@ng296rq2LiM#7KrWq{Jc7;oJ@djxbC1s6^OE>R6cuCItGJ? z6AA=5i=$b;RoVo7+GqbqKzFk>QKMOf?`_`!!S!6;PSCI~IkcQ?YGxRh_v86Q%go2) zG=snIC&_n9G^|`+KOc$@QwNE$b7wxBY*;g=K1oJnw8+ZR)ye`1Sn<@P&HZm0wDJV* z=rozX4l;bJROR*PEfHHSmFVY3M#_fw=4b_={0@MP<5k4RCa-ZShp|CIGvW^9$f|BM#Z`=3&=+=p zp%*DC-rEH3N;$A(Z>k_9rDGGj2&WPH|}=Pe3(g}v3=+`$+A=C5PLB3UEGUMk92-erU%0^)5FkU z^Yx#?Gjyt*$W>Os^Fjk-r-eu`{0ZJbhlsOsR;hD=`<~eP6ScQ)%8fEGvJ15u9+M0c|LM4@D(tTx!T(sRv zWg?;1n7&)-y0oXR+eBs9O;54ZKg=9eJ4gryudL84MAMsKwGo$85q6&cz+vi)9Y zvg#u>v&pQQ1NfOhD#L@}NNZe+l_~BQ+(xC1j-+({Cg3_jrZ(YpI{3=0F1GZsf+3&f z#+sRf=v7DVwTcYw;SiNxi5As}hE-Tpt)-2+lBmcAO)8cP55d0MXS*A3yI5A!Hq&IN zzb+)*y8d8WTE~Vm3(pgOzy%VI_e4lBx&hJEVBu!!P|g}j(^!S=rNaJ>H=Ef;;{iS$$0k-N(`n#J_K40VJP^8*3YR2S`* zED;iCzkrz@mP_(>i6ol5pMh!mnhrxM-NYm0gxPF<%(&Az*pqoRTpgaeC!~-qYKZHJ z2!g(qL_+hom-fp$7r=1#mU~Dz?(UFkV|g;&XovHh~^6 z1eq4BcKE%*aMm-a?zrj+p;2t>oJxxMgsmJ^Cm%SwDO?odL%v6fXU869KBEMoC0&x>qebmE%y+W z51;V2xca9B=wtmln74g7LcEgJe1z7o>kwc1W=K1X7WAcW%73eGwExo&{SSTnXR+pA zRL)j$LV7?Djn8{-8CVk94n|P>RAw}F9uvp$bpNz<>Yw3PgWVJo?zFYH9jzq zU|S+$C6I?B?Jm>V{P67c9aRvK283bnM(uikbL=``ew5E)AfV$SR4b8&4mPDkKT&M3 zok(sTB}>Gz%RzD{hz|7(AFjB$@#3&PZFF5_Ay&V3?c&mT8O;9(vSgWdwcy?@L-|`( z@@P4$nXBmVE&Xy(PFGHEl*K;31`*ilik77?w@N11G7IW!eL@1cz~XpM^02Z?CRv1R z5&x6kevgJ5Bh74Q8p(-u#_-3`246@>kY~V4!XlYgz|zMe18m7Vs`0+D!LQwTPzh?a zp?X169uBrRvG3p%4U@q_(*^M`uaNY!T6uoKk@>x(29EcJW_eY@I|Un z*d;^-XTsE{Vjde=Pp3`In(n!ohHxqB%V`0vSVMsYsbjN6}N6NC+Ea`Hhv~yo@ z|Ab%QndSEzidwOqoXCaF-%oZ?SFWn`*`1pjc1OIk2G8qSJ$QdrMzd~dev;uoh z>SneEICV>k}mz6&xMqp=Bs_0AW81D{_hqJXl6ZWPRNm@cC#+pF&w z{{TT0=$yGcqkPQL>NN%!#+tn}4H>ct#L#Jsg_I35#t}p)nNQh>j6(dfd6ng#+}x3^ zEH`G#vyM=;7q#SBQzTc%%Dz~faHJK+H;4xaAXn)7;)d(n*@Bv5cUDNTnM#byv)DTG zaD+~o&c-Z<$c;HIOc!sERIR>*&bsB8V_ldq?_>fT!y4X-UMddUmfumowO!^#*pW$- z_&)moxY0q!ypaJva)>Bc&tDs?D=Rta*Wc^n@uBO%dd+mnsCi0aBZ3W%?tz844FkZD zzhl+RuCVk=9Q#k;8EpXtSmR;sZUa5(o>dt+PBe96@6G}h`2)tAx(WKR4TqXy(YHIT z@feU+no42!!>y5*3Iv$!rn-B_%sKf6f4Y{2UpRgGg*dxU)B@IRQ`b{ncLrg9@Q)n$ zOZ7q3%zL99j1{56$!W(Wu{#m|@(6BBb-*zV23M!PmH7nzOD@~);0aK^iixd%>#BwR zyIlVF*t4-Ww*IPTGko3RuyJ*^bo-h}wJ{YkHa2y3mIK%U%>PFunkx0#EeIm{u93PX z4L24jUh+37=~WR47l=ug2cn_}7CLR(kWaIpH8ojFsD}GN3G}v6fI-IMK2sXnpgS5O zHt<|^d9q}_znrbP0~zxoJ-hh6o81y+N;i@6M8%S@#UT)#aKPYdm-xlbL@v*`|^%VS(M$ zMQqxcVVEKe5s~61T77N=9x7ndQ=dzWp^+#cX}v`1bbnH@&{k?%I%zUPTDB(DCWY6( zR`%eblFFkL&C{Q}T6PTF0@lW0JViFzz4s5Qt?P?wep8G8+z3QFAJ{Q8 z9J41|iAs{Um!2i{R7&sV=ESh*k(9`2MM2U#EXF4!WGl(6lI!mg_V%pRenG>dEhJug z^oLZ?bErlIPc@Jo&#@jy@~D<3Xo%x$)(5Si@~}ORyawQ{z^mzNSa$nwLYTh6E%!w_ zUe?c`JJ&RqFh1h18}LE47$L1AwR#xAny*v9NWjK$&6(=e0)H_v^+ZIJ{iVg^e_K-I z|L;t=x>(vU{1+G+P5=i7QzubN=dWIe(bqeBJ2fX85qrBYh5pj*f05=8WxcP7do(_h zkfEQ1Fhf^}%V~vr>ed9*Z2aL&OaYSRhJQFWHtirwJFFkfJdT$gZo;aq70{}E#rx((U`7NMIb~uf>{Y@Fy@-kmo{)ei*VjvpSH7AU zQG&3Eol$C{Upe`034cH43cD*~Fgt?^0R|)r(uoq3ZjaJqfj@tiI~`dQnxfcQIY8o| zx?Ye>NWZK8L1(kkb1S9^8Z8O_(anGZY+b+@QY;|DoLc>{O|aq(@x2=s^G<9MAhc~H z+C1ib(J*&#`+Lg;GpaQ^sWw~f&#%lNQ~GO}O<5{cJ@iXSW4#};tQz2#pIfu71!rQ( z4kCuX$!&s;)cMU9hv?R)rQE?_vV6Kg?&KyIEObikO?6Nay}u#c#`ywL(|Y-0_4B_| zZFZ?lHfgURDmYjMmoR8@i&Z@2Gxs;4uH)`pIv#lZ&^!198Fa^Jm;?}TWtz8sulPrL zKbu$b{{4m1$lv0`@ZWKA|0h5U!uIwqUkm{p7gFZ|dl@!5af*zlF% zpT-i|4JMt%M|0c1qZ$s8LIRgm6_V5}6l6_$cFS# z83cqh6K^W(X|r?V{bTQp14v|DQg;&;fZMu?5QbEN|DizzdZSB~$ZB%UAww;P??AT_-JFKAde%=4c z*WK^Iy5_Y`*IZ+cF`jvkCv~Urz3`nP{hF!UT7Z&e;MlB~LBDvL^hy{%; z7t5+&Ik;KwQ5H^i!;(ly8mfp@O>kH67-aW0cAAT~U)M1u`B>fG=Q2uC8k}6}DEV=% z<0n@WaN%dDBTe*&LIe^r-!r&t`a?#mEwYQuwZ69QU3&}7##(|SIP*4@y+}%v^Gb3# zrJ~68hi~77ya4=W-%{<(XErMm>&kvG`{7*$QxRf(jrz|KGXJN3Hs*8BfBx&9|5sZ1 zpFJ1(B%-bD42(%cOiT@2teyYoUBS`L%<(g;$b6nECbs|ADH5$LYxj?i3+2^#L@d{%E(US^chG<>aL7o>Fg~ zW@9wW@Mb&X;BoMz+kUPUcrDQOImm;-%|nxkXJ8xRz|MlPz5zcJHP<+yvqjB4hJAPE zRv>l{lLznW~SOGRU~u77UcOZyR#kuJrIH_){hzx!6NMX z>(OKAFh@s2V;jk|$k5-Q_ufVe;(KCrD}*^oBx{IZq^AB|7z*bH+g_-tkT~8S$bzdU zhbMY*g?Qb;-m|0`&Jm}A8SEI0twaTfXhIc=no}$>)n5^cc)v!C^YmpxLt=|kf%!%f zp5L$?mnzMt!o(fg7V`O^BLyjG=rNa}=$hiZzYo~0IVX$bp^H-hQn!;9JiFAF<3~nt zVhpABVoLWDQ}2vEEF3-?zzUA(yoYw&$YeHB#WGCXkK+YrG=+t0N~!OmTN;fK*k>^! zJW_v+4Q4n2GP7vgBmK;xHg^7zFqyTTfq|0+1^H2lXhn6PpG#TB*``?1STTC#wcaj3 zG~Q9!XHZ#1oPZo zB6h(BVIW5K+S@JG_HctDLHWb;wobZ0h(3xr6(uUspOSK0WoSHeF$ZLw@)cpoIP|kL zu`GnW>gD$rMt}J0qa9kJzn0s`@JNy1Crkb&;ve|()+_%!x%us>1_Xz|BS>9oQeD3O zy#CHX#(q^~`=@_p$XV6N&RG*~oEH$z96b8S16(6wqH)$vPs=ia!(xPVX5o&5OIYQ%E(-QAR1}CnLTIy zgu1MCqL{_wE)gkj0BAezF|AzPJs=8}H2bHAT-Q@Vuff?0GL=)t3hn{$Le?|+{-2N~`HWe24?!1a^UpC~3nK$(yZ_Gp(EzP~a{qe>xK@fN zEETlwEV_%9d1aWU0&?U>p3%4%>t5Pa@kMrL4&S@ zmSn!Dllj>DIO{6w+0^gt{RO_4fDC)f+Iq4?_cU@t8(B^je`$)eOOJh1Xs)5%u3hf; zjw$47aUJ9%1n1pGWTuBfjeBumDI)#nkldRmBPRW|;l|oDBL@cq1A~Zq`dXwO)hZkI zZ=P7a{Azp06yl(!tREU`!JsmXRps!?Z~zar>ix0-1C+}&t)%ist94(Ty$M}ZKn1sDaiZpcoW{q&ns8aWPf$bRkbMdSgG+=2BSRQ6GG_f%Lu#_F z&DxHu+nKZ!GuDhb>_o^vZn&^Sl8KWHRDV;z#6r*1Vp@QUndqwscd3kK;>7H!_nvYH zUl|agIWw_LPRj95F=+Ex$J05p??T9_#uqc|q>SXS&=+;eTYdcOOCJDhz7peuvzKoZhTAj&^RulU`#c?SktERgU|C$~O)>Q^$T8ippom{6Ze0_44rQB@UpR~wB? zPsL@8C)uCKxH7xrDor zeNvVfLLATsB!DD{STl{Fn3}6{tRWwG8*@a2OTysNQz2!b6Q2)r*|tZwIovIK9Ik#- z0k=RUmu97T$+6Lz%WQYdmL*MNII&MI^0WWWGKTTi&~H&*Ay7&^6Bpm!0yoVNlSvkB z;!l3U21sJyqc`dt)82)oXA5p>P_irU*EyG72iH%fEpUkm1K$?1^#-^$$Sb=c8_? zOWxxguW7$&-qzSI=Z{}sRGAqzy3J-%QYz2Cffj6SOU|{CshhHx z6?5L$V_QIUbI)HZ9pwP9S15 zXc%$`dxETq+S3_jrfmi$k=)YO5iUeuQ&uX}rCFvz&ubO?u)tv|^-G_`h$pb+8vn@f z7@eQe#Kx|8^37a4d0GulYIUAW|@I5|NIh%=OqHU{(>(UhKvJ}i_X*>!Geb+Rs0MWf66Lf z-cQ(4QOENSbTX$6w_9w4{5eR?14#?)Jqf2UCk5US4bnz8!e>vFduH6(cZZ=5*_!M# zUTZ_b<4v@}dSQOcH@wt-s;3JhkVDct$6k9!ETdi-tplkaxl^qF=p}Q8KMVm+ zeIa2q?RYr}nM0d_W2YWv%JKyCrGSePj8GrRN)<$Nsq8l$X=>`W;?>0eME3|8t&d$~ zH`XG45lBh>-te_f0Mh0??)=Ee0~zESx=sZPv<#!sAVv$0qTn@CmCUNJU<#=`GC)&P z9zuV~9*3_n2*ZQBUh)2xIi;0yo)9XXJxM-VB*6xpyz{Rx2ZCvFnF$2aPcYFG( zyXkO(B30?mt;5GW&{m^w3?!P`#_o;Y%P2z^A`|4%Bt2@3G?C2dcSPNy1#HMXZ>{+L z3BE#xvqR@Ub}uKfzGC=RO|W%dJpUK#m8p&Dk|6Ub8S+dN3qxf9dJ_|WFdM9CSNQv~ zjaFxIX`xx-($#Fq+EI76uB@kK=B4FS0k=9(c8UQnr(nLQxa2qWbuJyD7%`zuqH|eF zNrpM@SIBy@lKb%*$uLeRJQ->ko3yaG~8&}9|f z*KE`oMHQ(HdHlb&)jIzj5~&z8r}w?IM1KSdR=|GFYzDwbn8-uUfu+^h?80e*-9h%Nr;@)Q-TI#dN1V zQPT2;!Wk)DP`kiY<{o7*{on%It(j0&qSv=fNfg3qeNjT@CW{WT<)1Eig!g9lAGx6& zk9_Zrp2I+w_f!LRFsgxKA}gO=xSPSY``kn=c~orU4+0|^K762LWuk_~oK{!-4N8p8 zUDVu0ZhvoD0fN8!3RD~9Bz5GNEn%0~#+E-Js}NTBX;JXE@29MdGln$Aoa3Nzd@%Z= z^zuGY4xk?r(ax7i4RfxA?IPe27s87(e-2Z_KJ(~YI!7bhMQvfN4QX{!68nj@lz^-& z1Zwf=V5ir;j*30AT$nKSfB;K9(inDFwbI^%ohwEDOglz}2l}0!#LsdS3IW43= zBR#E@135bu#VExrtj?)RH^PM(K4B`d=Z6^kix`8$C1&q)w1<&?bAS?70}9fZwZU7R z5RYFo?2Q>e3RW2dl&3E^!&twE<~Lk+apY?#4PM5GWJb2xuWyZs6aAH-9gqg${<1?M zoK&n+$ZyGIi=hakHqRu{^8T4h@$xl?9OM46t;~1_mPs9}jV58E-sp!_CPH4<^A|Q5 zedUHmiyxTc2zgdxU?4PyQ{ON@r+Ucn1kjWSOsh6WzLV~Bv&vWLaj#Xz4VSDs*F#@M>#e^ixNCQ-J|iC=LcB*M4WUb>?v6C z14^8h9Ktd1>XhO$kb-rRL}SFTH)kSu+Dwds$oed7qL)Jbd zhQys4$Uw~yj03)6Kq+K-BsEDftLgjDZk@qLjAyrb5UMeuO^>D43g%0GoKJ~TO0o!D z9E$WfxEDFTT?~sT?|!7aYY*mpt`}i;WTgY|Cb4{Cscrmzb(?UE+nz1wC3#QSjbg>N zleu?7MGaQ&FtejK#?07Uq$vIZX5FqR*a=(zUm`Fq$VUl){GQ{2MA)_j4H$U8FZ`=A z&GU_an)?g%ULunbBq4EUT7uT=vI6~uapKC|H6uz1#Rqt$G(!hE7|c8_#JH%wp9+F? zX`ZigNe9GzC(|Nr8GlmwPre3*Nfu+ zF=SHtv_g@vvoVpev$Jxs|F7CH`X5#HAI=ke(>G6DQQ=h^U8>*J=t5Z3Fi>eH9}1|6 znwv3k>D=kufcp= zAyK#v05qERJxS_ts79QVns}M?sIf(hCO0Q9hKe49a@PzvqzZXTAde6a)iZLw|8V-) ziK`-s)d(oQSejO?eJki$UtP0ped)5T1b)uVFQJq*`7w8liL4TX*#K`hdS!pY9aLD+ zLt=c$c_wt^$Wp~N^!_nT(HiDVibxyq2oM^dw-jC~+3m-#=n!`h^8JYkDTP2fqcVC& zA`VWy*eJC$Eo7qIe@KK;HyTYo0c{Po-_yp=>J(1h#)aH5nV8WGT(oSP)LPgusH%N$?o%U%2I@Ftso10xd z)Tx(jT_vrmTQJDx0QI%9BRI1i!wMNy(LzFXM_wucgJGRBUefc413a9+)}~*UzvNI{KL# z_t4U&srNV|0+ZqwL(<}<%8QtjUD8kSB&p$v^y}vuEC2wyW{aXp2{LTi$EBEHjVnS# z+4=G$GUllsjw&hTbh6z%D2j=cG>gkNVlh|24QUfD*-x9OMzTO93n*pE(U7Vz7BaL% z@(c!GbEjK~fH}sqbB1JNI!~b+AYb5le<-qxDA9&r2o)|epl9@5Ya7}yVkcM)yW6KY7QOX_0-N=)+M!A$NpG? z6BvZ8Tb}Pw(i9f7S00=KbWmNvJGL(-MsAz3@aR~PM$Z>t)%AiCZu?A|?P*~UdhhFT`;Nb)MxIg*0QlkYVX+46( zSd%WoWR@kYToK7)(J=#qUD-ss;4M&27w#03y6$gk6X<-VL8AJM@NFTx#Z!n)F5T357%njjKyjro(yW8ceP{!%;*Y>DN`&_18p(z2Hg$%K zohbgJcp%+ux%q6F?(sc_mYJ<$;DxgkTEi?yjT6Du@+n(KsKtFHcO%7O z=AsfLSTdE2>7a@0^`;)?Fg|s2XOPV&fo<%Q)Izaw4s&RvrX0^+aPNq|yE?oSa7 zsnNs!+vGcTM4yM|$9so*2Nv;ngDD}b0MjH6i4e|l^O`lzCRj)-qa6f%|afJpmf(S1J2k7Nt^!;Q}0 z4ejPF?^M~Sv+@LYn&IFUk2;1h?kb8lfrT`oMm=JBm{fo5N|HY~yQQ`T*e2?!tF%*t zf+ncx15$NdF82GXrpP5rJ7!PVE3>u`ME$9Hw5RlP zUh+s#pg{9kEOsAhvu2pry#@dvbB3Lti+9VkLxPZSl;fNr9}wv1cTahUw_Py7%Xp;C zaz__|kz*ydKiYbsqK{?cXhqR(!1KMoV-+!mz>3S8S`Va4kD#(aKyqecGXB^nF*>mS z1gG>fKZc?R~Tye>%x+43D8=e zf0eKr-)>VEu7^I{%T}BT-WaGXO3+x<2w2jwnXePdc2#BdofU6wbE)ZWHsyj=_NT3o z)kySji#CTEnx8*-n=88Ld+TuNy;x$+vDpZ)=XwCr_Gx-+N=;=LCE7CqKX9 zQ-0{jIr zktqqWCgBa3PYK*qQqd=BO70DfM#|JvuW*0%zmTE{mBI$55J=Y2b2UoZ)Yk z3M%rrX7!nwk#@CXTr5=J__(3cI-8~*MC+>R);Z)0Zkj2kpsifdJeH)2uhA|9^B;S$ z4lT3;_fF@g%#qFotZ#|r-IB*zSo;fokxbsmMrfNfJEU&&TF%|!+YuN=#8jFS4^f*m zazCA-2krJ-;Tkufh!-urx#z*imYo|n6+NDGT#*EH355(vRfrGnr*x z5PWMD7>3IwEh=lO^V>O>iLP~S!GjrvI5lx<7oOg(d;6uEFqo5>IwptBQz;`>zx`n$ zjZQ#Hb)qJdQy#ML&qcfmb$KT+f_1#uYNo7HHDY}7xAw8qbl;9LWO-cndfI=5$%jBw zb}K3U%88Fg^|&0Vc~99bKl|$3JzdawRZ|`7%1S<8B7>9*rWAT0U<@mHDfnL1`~1U| zDw7m@<@}C|zqeHM(OK@di6~sKHiJvk^I0^S<LBe^_xZsUOzVkYSE)Bxn*NekQYbyTn5SRt!n{EseOo-$u)vjM(PV%6cIG3Kv$>dd}HUyXi;_Lv>}OyUj38dPe8+1Pr?{LXnIBCoTnocD60@vhsz+GG5lJB9ncgP8T6@LwuzZ)J zKETBS~AvzGE!{u^+Rd-|Gn!rc@UUnioP0{@_j_>tg8YI#?y zL-H$=&xXkCJ2Qe7&exbI!z`OyPxBp|4_ zZrrc;OAb%T4Ze%7E}FBB`8t$QN0sA3vpwU>?7QAmE%-ethXdCtby$Qm3v$lNxB2a7 ze6F5eEWV`={#W(G)Va}7?$D65WF|f0nmfZT;?=LE6Yz{{W3CV2h^Ma+LXdZ(HMVKZ z!YXJ*34lo!FA>)jSo@*!Hs_)IwmTo6pBr3c^j2u_amZ~g;&Z2jZIw!}v@w8DtZz7|A%rFksD4^HYB!xFAqX;u0HxPeG!3Z(z z4}+^N5-nckKf2YSR5R_}PD+2?Wq#BOiON74#{`u=4f59WKdy_77EYq~_|X6cNtno{ zZ?WLwbV57Z6uI|uY_;vzv~~`eiiOl($Au7C*X<&MY5v0b`KEu-GW}{2UNfmmrP!^Y zAOczy!}TIJsom=}kxH)9W`&Rp&rR6T7y&~5nXbut;wcs@M?aa^9j{ZDtx=1?P8TV{ zee2kKf%CE$mogyKKT=xQQ#)OCl9bjc)}{p2X$}aG`^B0w0yi-rI!d4e-u9uR$kJK3 zhqBG9Wx<-3DFw5olJ6neF@hB;8o(r(GB_;p1i>}cjN`JNEZg-dlxtLL=8~gfLrBy_ z1~bGh{I>_xqh(}?%bCf1U6~K@+N*i}bTi+pUAW)oM0`D*PeJq=S(-|Plxe9OqxBRg zM((r)xkSH@j!8@+=cA4US0fDL&O?W~x=Mlu>7zvHO2sy7D5_7ulP+YMecP~}F0b*K z3oO2j{o&WHd<&UWcyA(&6hvBJv}qUZ!@R<(mwKB^;y3zeE1>LzbDWSkRD1|5MZPx( zxd=&MsQi1eE@@6W+4N`cF?yh!3R5JlAV--&RONWQ#?SbrQ95<@ag>C{jQmGXpQX{) z1dbFg1_`qLxuDZnX#PKfCW*Jl3F&^7@gO&{>Nb8um$VBcF1!AL=N6`A%BFj=`QaPI z+m^`n+{o)KLif;Gt|7aQ(XXRP@x)jJt}s{&S`I3}jPTY>$@W0BD3Oif^ehs~!H7T1FUSWxLS&W;0q6+azjbWn?3!q$ z9qbmdr4H4Y)p^NOACJ^L>u}NS8T0_5hW)G z%Hv}dAqM}d@t;|hf8>+NHHPi*xePsRlqr46njzhiXXZti7i5+GTKcrlxA->OJ9*Pna`02EIA5~(SMV`T@H6F2VtwwP1$tYujbC1^VE$Yd&I`WSwB^1( zT7NP3|85z#R%&wktjwY_i*n_$RRZPM^ota{LPV%*>=>sAv%fn*cnkCIX{^SJRmwZv z!?f@T&D%Lz@*!mNYTGp{J|7)~PR*ib`;l^E)rQw@)Qn0ECnB8W1S_SbLZWdqcmo?V zX5g0_3qhn4TrN27^x#Qdq*4*G1L|)I^b8GuP_8O{p|M`uvZO6McXa>OSQRW|kQTNPZ#Zyj~SZ<`6B)Y+}jxpn+YT>MhZ!Rxyd@rU>N zP>MkDBLX|<)SJaO?Ge=!D>i+Wq&PgneO?ZXUq4IQuTq z+V{ZGkuw77o~o$!b>4ov`6CKJ)$cf=S6%1ZQyYU!kz_qiuNxY2*Bh;K9J6o_YV6xQ znW|>x+#Mymu&wF9P|3wP*(ZjwE+ou|{eFqMv}d_iEyH zQ?NSf3VX+EpbrIKmp|oD-t_rh(D#e)fp)dYbG{=yPj-3-#l+iu7r+~#w|(#wv@G0` z38`Yhf5CznhyDEhD;jzaz7fc8L?(n-m zR#|5hqq#yRoeTm+h^9J42mnB>BY>HSu&&O-Hxo6j!dqck)dGS&odS@Hsk2-*Z~x z0!%{@gT645S5DeF@JZeE$DFl*nJB8Z|JKvs%7d`KjbJ*AsA_=fEZ&V9=*+K{(TF^( ztjjYr(7@fV^tDs9c*#=8)ZRKO17A5Z`8v*)U+?hS>3sEfgh3`#vFO^7n}&&adV?}n zdy&BY1h|I@eBm=l*kqiJn>vNkOH4l$Op5Hw3K_w8lF!6T@-H)S2W|Km#6!-X#NqLJ zsiVDrc%*@I3^Gen$)6O0C_qw;8{aucF;}U^1%YE`?AYTtb`Z$B$vfhcHQF`VCB(Pf z_G#fV*Colv-k!O+=^nDNe(03?m+RTu&28d%>JrrwFNb{ND&?Ad(=DP@voz$usk1|w z&#gTB7F)#*LtY6@pIb(g72*LcnXRlTPQAD?)ZFnB*EsZqxM&Uk_KGXnR{4}K`I6i- zU9}R>tiO0De1Hx=kAy>7O+nKO@kGQEYOai&S9&WTY+flvR?uhI695W-xZnq4aRMh8 zwfp)+KYWVB#r=5AwwlSdM4@x7-R_{2;1iqz2lXL$7iu1>5W*+I)jlkMs>60=LN)Y= zbPw;;%U+%p_&{2Obemh$BLmbpDd31YxJ8#TpH3~3B8QLUMvx1X5Vl48hWSNN*UTlO zQgQyZbmyjGC-s$3tnB z0mfKUu2+_c`ZVvDVwUy#j3W*l^BSXXQ%=r6Z}C73jx8DAk!t7k{dK^udpHIcUejp# zyx}og$Hr+f>9kaZvno*Om`d|VTUce9tHM=R8thoG!a=NT$s;g@n_rAN%cp7nnLuav z6}j56TSSfPL$p#y#!5TVyqa3zTzi7@#IoeR=E6CdS`JrR+@i2DwZ?T*bh+(k5!a)0 zgRdF93z8XJ|5?>hDN!YAW5cK=+BwDLNT_+otd zqC@*{S0hCKZ+TnN*2&qx+WP;ZjHA`yytPcwKl~)uy)sQ}Q*0-&3X|YFYAjmolaciq zxS$r5^fxICetD*Dw78M9leVvhAOZ$=;SP7L!Vs?+0f1h*YCuTXIt03iAf)0=0KEvZ zB69o-zg`0C#hQ>`4`}1g=a~EID(j9HbjJG^tV-zumR-+fahTPveA{%0u2uQwMZ%}5 zwY!|}i0oTd&>^QSRhIKU+cMC#|C3f>|647?v1B(wH)EWb{vuJEJh~!#|J7%=h!x3| zCH6m}wg;>Q&?@5Ct1%n`lj%*>9a52d@wmvE`=aQjtz$sWj3V;fDns5<7d2*``)u1( zh!Ub>!#N0m=Vz1n1=El zwb2IVRw$6NIFRpGyUoM0iqc$IPehcmm7<0s7F*Yv+zq?_%pf*SS~~}s0M`m(rMbx% zi?|Wjr6fJN`_J8&B2$4+V+iO~m>s~Zr2T3Y3HGREFQ%%pEoU0N));AeSVM#gYQ>l} z0`RhgS`R^pJH31YQ~eTeJiI}g$&^|nv{!h?8mJK{{XDt+sG8D`7)$jvM#hjPI(5sS zfFW4s7wao%Lo| z#pJRC?iZOai;57ANs|vm6%}rPlGo}}Aso1t#xJn}%VW@~1WSjh(@JTgM$0x6ZQ)gB zdiox3f>kqGZY}+R<;wlNoWJ8#X-v)1;wRD*ec*wnvsN06Q@cZuD`deT-Bu&G;2fBC z0FE1%pG@{Yo2O87&dE;w???%`9s1gs=3GpM8xx_}=AB$K9y=cD);^iE*p4;T1RU%B zBPr)yqOBX<2}xt%g9qr>;z&|?4vhhw7@$a}Uy2b%_^VdB^VfzrebKUPnq;hliCNU% zVt3R5EHkhN^Pv`REF+npA@#HdCQN9IbQbqSDs^+zt(A6;rLwN+@Em}WrV5vPEo!w^ zSCd3RZ8{7a@d9@|IF&&G%irS7FHle?@49LctrtTt=rP$W)se*#RkFmyf)D1^U6EYI zfh+N?uH?-))O$9zM19VsuGn8?o~5`scXU?!P@_cWP&1U4PQqGus=sQzrX+YvKG%XBL3nt6!&M<#}wqA;Mo(}qrq<1lNkpQD-T#-y>grt|E+JNU) z2j+g+QPcA9VEFc0k;H(hSNOpp$I+!$ z&d&W6kBM9+c{X%vr_X0}tdB5dvEDyk5H2*T(QW8Yz-#tjvF?up=^Kfym``^!&O-X! z@HdfpHn;}_)y$Xjb-5cR$Q#-XdhKpmJG5pl>h*Q2(u*gt_4(>6?kG)%T3*&TT0qI( zL!aR~4HiJiaHlgdNcOQP6xx1f3AWx&8}(NEps|G!cO>J^rE2@&-t#_Jb7GYgnLnML~1ze1D$?~BwbgA^=pr55tC|d7w42vN11_8bS75u z_MRKqE7Xik8fk>6(VE5{qT}6rSzd|o}Zb>*aI*Bwg%ccE$_ytH;g2H z^i3qY!+aE*&s^BMH9TI6GLm&9c`D6)3{-+?2Pon+040Yuv$2(LqV*krKhTg5CHOj* zquacxc1&~=S(O@gR8aI#?R%)meONmw1rub9E2QzeM$pBBm2wbPNR3tab{op53<oFwaUbARdD5jSA_6zmKX7!VicEP1m)rYnk{P- zruRj;4c8S29Rd#Baf|fq_pA^r3K#qRHS;($XNoLI*`puZjM?bA0tH>FDiVc9qR*|3 zGn#nhqxkvqFwRfCB~2yA0pxWapfjCdAem$utuon-`*6}mUP?l%$CE(FjAwL%Oe7GQbu7*+&q>*(cAofJr^gg>xw>hx-SO7Lx2)I} zJ)tV1XKbkE4sS&La#-smSq>S9gBzGLH%v?KVezdGv%Xs}kDJZJi{lDl(FpLZupBta z3iDlkd6LlkRro}+El?GIObw06D%NTXpL{W}Ve*%u#{wTC=+VHS%o`sAez&cYz|Tn` zcK_~pvN%cd^8FlFypCjTjw9@ulLoJ^!QAK*++^wC2~}CFeoY;q6y~r&f^+0>LR6)n z$hSev@GzzGgDc>)#u5_;{T9^5y5I?m=z7=J!eVId8p6R5>NV8)h|bA}#3KUufq4CPGiWYvGj%0=H@Q66);F)#cDMND4 zX|?rg>Bb28q*a!_sgVF(A=OeC&je$C4>$0%yy;Fla-hl(|9Ww4!@Q#E2hpJMMxpQ2L+R;+ZMpS+|j*F`Fh}p)`a_*<`AaeFzNEq^- zlF$7BFKD%p@K+3$Vx%N{QOayKKWU#JOAwXiLO62cA6=|DiDG_Z=ef;f&gQ5-?+Pb+ z)4NsyEZXCdjq5tgDN39V9!6#w25+R1;PD7ss;hFvQn}Hnl3^3h<`ylzJdVEL>|Jj0 zg>=Pscwx&;pWEzMn`ld**$1F-nhqlMuX;G{lWrT<<4$7MZ^*4a2hAMf)3eYiT$lRz&9({j<=%DWIRpgu zoOns@gF}AQ_6Y5RhySg7yMtJcYQap6^hgy{`zX1Zv26q4<)g@t%aIi|-lmcySuRN8*5f*$aEFi8o#kMKRCMnrAY~l`= zez#50^@Qo+6r508>iKfAbbc3JwCnjnmw;~=mlMG`(H8EJz7W6mh@mdinO&)#zHX=| z&|fo@s`;njVkkCMczSnp+TnW8YPU4w2&QmzEh1}orF~KlT=V+`!!rH|PtULCcL!P*m0EaN0Ad2qBw%Gs40jfu=%`N*k@z2-p?&B?Yum-p+h?7(!D^ z&f2Bn_#t!4HM2y^*1GN;U+_x8T$Z2>U9Yx;p_9Qf=ww z2hxO^*{%p9-CwMKz}C4mTi8xvqhivltE|}Kgq5MK@f6tBT&`@RYzsFFi>*eMZ0Z6Y zKBl`GOh!U%C+PXJ|7PF)V*~#8eS80D@v-NL2U&;i62W}k+vJAC+7xF`eq%c0b?{PVTcqiDr%6jLBdkVcTwLJSd313SP)1r=;2`cORbMzrhqZxMWcTWru5-l_H8;f|?{^M%%7>sU zGx2{fX*t;7SewS|NvPR-6F5p(ji7d}CK#%7y}jsPkgj%F5cUbQ?b7uWpYks^|DL*n zau%X$^(%wXMS3c;C4=p*#q>ahmLH5woLsn-YcZP~mH-rGnRyl#KU4MsLu+G3z90+q zM$HCWgZYR`8_I%8)SYuBltP$sN`-6hcjnzhDsVl+Y}yqMN*4MWsJX_6R>Cyw8cHGQ z1>r%vkDxxc#ACA4+-ZO|QBMUz`YHrS{l-*$> zi(n_;4{Gn+d2gn)TA<9) zibWdKJv#s_f5K}vM=d0NaYrd;5A+Fy^=+WgKC`@bS>!P5@K4fzE#VYfMcNdbbvLPY zeR~!f3xU>|pfq-LOsoF=t94x%K!8>#8tR4KQ2G3Yr?Cb98^KL*+G8``rHMpNUN}-T z5HGAkiLh{WR;N$Nk3X_2^3pW=vOFTOb(LS0Wu)0)I{8sZj>}5ZGtD=va-72l&5`L= zhyzBWie2UrC|?(sTcuk$OwvV4oVlxc3ncXPj|cD%%*6(hoKMd5wzPQs^6g)B0xK#d zemOodB7D(!@v!|eYqMfx@M#b+D)PwAuvimOW#13i-xAR5)Ai; zXNX(A@M*y&+TVZI zGHo$F*Ipg~Rnp`KlMNAl2o86}r%Yv9#!O-oo`pe`880;-Y28tR)b4H%nqXXHxN9m0 zI&#!(XhT=T3$WS$)K4#Y=ceN`MsP0v1X{nIoQ14S2^--MnUp21=V3&Uv8|y}^}7Vl zI5tRbOp#?@ay6uncZFE0hg}kt(k%piw^M8;0yynsK_!l~uP??IqzmKJMUqAW^GG{~ z7Fg)Q&zBlp z%Tj8jOUpuR>YHP6zYsX?)aJ`)_pRwu+Tn8I;brOW_`v$u$`$9T)cO*O$j=?mg>dW$ zw=&3=v||fqCr`-$okN*$S9(Nyrs}+Lu#IwDg2xSBz_VfU*?A&26vwv>&>*U_TT7-7 zS~X}fT%9+q(Xvc0qzOG^8gmMcZE9izi5feqvY(aY=%reP+wVZ&cRd`^y6}-gJ&_6n zR%Wdl3vQ4DOt!X9ry7j%=+7pLPdus*@7dZMBo0_WKZPD1(o{=;D> zyc9_WFI3{URv=d6EXcnOG0$(J(R#8Oz$kmuSFQ{-Y20}1027!FkodTU!fouSybwqn zRO-$2BH(w4)$wiPo<1w-4*p=Q0@YKRm^cgiA>~ho)U8^e>SBk*!@xvr0CdvnLHS#CACVuQfgzF>8qV znqf{oO1}RWhiZ3g!Tx9sk!JfLqcP`>Ksx#vZuLg-DC6h4mT!vlU zqw0`0CzZgY!EN0*{sQnDNFn;T<+e_x$zY|n;p0@d^hK*n!S!=#^;P{*D^6~h!T7r6 zoiMxtovMo-dj*{qZPy*c3gaMBEDQDkINU%d8HeBZVlRuzkCId9rx{?L= z-dLlk$w&JX5wn+8`mtqCpKnx+w+$@6DEUI}8P%xN$MEsw%S1-$9PM6r^jP-@?cS<# zhg$wl0X=s3{8EZ2U9(};p{X_b1@jJuGgx`gDK{6MpF|XON_=Rv%-<Ee1cuuy?nl9xVDa~x=+8ppnOQ9 zN$53qi4QQ!co(;f!#YJ8(=Z>_9UF#(QOVjS7T!g2)*Oecrf-R^)tFugBkQsMVNua# zS;1V^#fJS{h+!O+FgS%0=Pd9;lMa0QHn?-n(<0b2$<|@r>fjiyw6u*UoGmU$ayJM@ zfp;c4@{$b*Z_v9?8ZEp{m6Q(mDHW<``n?jg-ZN)Hhvxn*l=O1f*K%{5s77WCt!ugS?*2oG5-Q)JEJd0+W5=doeD$Wh?U$ZRg)K$v8cmQ{hba9jw_mF&X zi-dV?WITgIz!!0uB~jE?(t`&qo{WGyUspX| zc6+F2K4l5$LqxERF#`I&k^^opVIMZjGhsJ^vI0c%kV+|&_k>~}ueTtj;^Dfb@xHs` z)-39elzVA~D~n_aoyBQ1>Qd2!;E!G*pZM&RX`r*y)b`yxvP2;#vM*;CQGPg|gni)} z47`Log3PUyVfdmJ2zvHBhg7T#D-H=myzkeUa$@);WC(yB4k^*$wda3=S-UH5Q1Hx6 zPcGxMP&kXBa+4$s#Sw3-V?mlHj^8&bLpIN~GkYj;!;M!$ZxvtQY4j&Ngz_mxuQRqx zYTbN6epx@-!0jRV5yiSIJ<^mCZ<|;&x2~a)t+(eAVB!1XpCZok*Z2C5P7&>z-Oy?t zf@F(_FLsSrfCus61+Vt~svP%(u<4pzT5{w*0XqfPV%~|=%aq^$=*U+_trGQaoUxbt zBV#Yqx+ULku8yPJs4gGcC?+3iRt_6)Oi0DNLxdb(!n!cup_XUZ3eDe(!DChZ!IG&L?_;T-1GB!R;;Sk;l3Y*JQ!I|l20_f}ZyC;4D7R@6F z>%z~wV;Bj1b(*kp26Ed!Y-OKxNbt3%t))xxOrazWsmwvW;uaSaJ0ou+{01vXvU>_V z6Ha@+;giVaiyg`J8ENQf)Pq>!Nf22>XFHnXTNk84&jp-^YwmlUqnOll8)5mzlO$o! z#fSMwH8Pn+Fy7O5M5#ZGr$cKfaGf8g;XN)<*TrQjMk<}_oRf&b6qZoR38Q{Zxo{V; zby+J_hCZT1>`4~jnQxo|ji%BQ0=BLzC6c!1=B(jS5+fcp%q)JI)=c3{D|=k5;0&c2 zrbRE|qxkNqah2nvextOvjYA{T43n1c6eO7B9DH)tLqB46E7;0xKM=%#wx-*-+*OY{ zQ#7gMStz%I&2&rbo>#T20OD_#g`WYbt9+!MC08%zSMhqMoRk)7VOk%~`sD%(U6zzO zdmSC9@x0GCv2_)umYc5@#%efP0_cu+=f^}k$H9$N_>piA_(5UM_o{++8+Yf8SJ)?C zDd3l=GGm3EEy;&Z6N=+XP@IM0L=uW^ooyYQYyx1vwFR?@U~BAtAqTu%Mi2 zTCQh$K=UZA{P`Cw0I$xAh_f?fq-Goe`7I38{3L8?K3`lRhSAyB)tHT@4c!Y;bJAAS z3u>Q7qx>9SJs4$EB=hxh)u`W5jp?>^g1s_MV7<1zN zXt{FSt?Mt&8aCy67<)b@eg@h0iCW@%+pF-V>p${fyEk6_Gvp|ms{Whi-9eNId?xzZ zm|MI>F;JSuaUnQp#|}k3o&ddCZEeTI608txuU4~7K(wg9 zg%+}(7h2@(%>LI1F*puF(h$ZD`Q+ar!VoVajPY0-XS$>6F_F?sc6Mr7>SL-&{pC;2 zKx@2{@ULz7RCpaKg$iu2rcY+y*~qaPo0}^7T1K$_(NPS<1;V zTj8-xC%WvgDI_YYEG{bySvyO3M>XKY)oXgGG*eB{yDgNQ3s3)A~@n>!O#lNh0! z(-dqW#_z&mMfq#2+u61N`L^({4UoU8wE5`4c}{SGFzKb(BK8hM%cf_zj_HmC48)M& z398ICVJTGzBaz7K{L+Ew=;z^0xA``wbtPs`r+Wrb^_vzzhukq{;A`t&-ktzb zbqy`Z0#D6fdVAiodjF3J+qI*vu#=OCjiL4bIIXEf4?zmN7(H|+<+WfR7@7jrMx7FY z5*0X1enhay-q^M?j}3Pd^|U9(C3#CQU3=hlc~@y9@NQD{UZNfC^5?Cuuuu{ebn_<7 zEzudv*b@QP%)N^5jP;86nQGb<*SOytCM5wmf-=rH#K{Wd$2(X#S$jF}XIxZC1)zir zU2Wq>hIB44nCTqx2x<{_wiVzLSJR}L%P!Y|lFHtA_=bDj=OqvmmSZ}ffuqPge#V-f zZDk|XX0RK}=73LxL`H%OXxK*^I2!fp&kxatErK~&tM3@j1a(Yrq$z)R()i?}p|0^Y zhW&8!IpRA1jJ3e!p66ZY=eBmEA+$A`!%s+{Cz!s$IA`{_Dh0^jt!vn;+Nw}hx019Q z_Wg=#-G-~&@>l=&H~48$L8`LX)!Bcq%(DFa2Loc91u@WcwlHzJwo{cdur>bQ;{fr_ z`rC5QRQ_)`8EadJzz-{K&sUI~>NX>P|c4l)fKS0gkuGe_P ziaQy!%CK(CtAwj-J8&#kyU=G(k%3y`!gS9dU&1xIrGRL|!&aVMEaezUIpopoET~xE zp`%~`LZfn!Lu^+00?>v4UOfM!HeeQoLZP<#o`^9oi69|$0BM?n17R~tGpY)eJiv@$ zTV-~ZZ*}C1J{a}p`>l$Bx8qRBq91;dLdmp84auzmcd|XzJG%I|r z^E-8Tm~jRn_>as(R=@~z3I2E3<=#hXn>A=0`wfOGIxiP)N2%!cG?&^w=E#TR z`lSY@Mm36zu4p3}+S#67MpL$d{gf@dnP%*ZMW=gCXK-%0E(xAC!^+b7hCSMF$m;Rn zCTErbBK#;a)>kHX5}w6PRmnw(!Gy>m_g*2opfklHyx>eb1bu|_lwJdf!ogxhk}X^v zc+^L;F7ta!8+i%6?M}XvQn4b%aOSCpDW+4#JDDG(wvXC*9%9(XBhbv4LX3R5G&(+@ z)nbdivYRQ5pW;9~@YGf{h~Rm(@MfV8Tj&T@EejO6(C#(+z7FVNBR`@j!#wScHM5ki%j+^GykUJ2m zYgpwm;#Q)~LoozUSV($?r3vQ~#ZU_}ggl~J%z*1dYt_^4K6e7o&qs_ORz{km+D+^a zqDdUO)d}|)v9h(Zz3}#DLWyRVCY!=PMCO{=PA)Upb@)1j?c)||l{6&pI=;U#bS#Jk zOOiwVH3FM!SuJDIPnN$|ZKz5fQwHmzn8f^?B+T2ew%~PSE#X_jk`Wu;a{4}9%AHg7 zZm8^bAee$bdpwklIE`$fV15=pI+tgJpll4uQjIM;Q!gvISFc_{@=lUSc-lABE%U?+ zHW$;!NcH1&F;AS~7RH=n<=!NTKnm3t`B@YeL?8d2{WGrmSjG;yBbY*9$N&DT^e?l2 z|1A2482Or7n7KF_TpRn|nmqD}`-=?QJ0z5q$C9Td^sML&aN7OGi+W$uYjDXKJg+0W@S=FoQP2dBI=48|FH>p2mh zFrdu!AwoG$NkvnZp_KT8HEo=RNNJ4IxucGXLr2N*I5Ao>Efb+pNOm9Zw0_7_s|9ac zS6}W##>$W*cBmksip;43p#a4&iTpM)8(gRGekW+AKm5zb)xpUFT>~b+FOH`Zs!$RDgpSCE z>;CL8Uu|EWeR~TvgDX@K=mtReFed;FZ!M2SjzW35i;UqfyemM?rq5yZS#hK5Y~|wt z2#^`Q6$b~uGT_++C3+B~#(oFHdSL&hh`Z8{t5#=ZkoaWVJoLm)3vT_@5HOnZGa;s~ z;4=E`3Eo@=$BxFjS`Iu|8SALB`<#TPTeE%h(dol+#CzJ=Zb&EHpw*=0H*~8x6 z`G`b<@>L2(AS*J!NVp`DN{g!8R#h(~URslf zC8PwGM$5V}+$WcoT*C~*$WmCpS6Gis&sZo|9OfRiwjX$f*&25Gjv6$YPde1smwGw( zb@y=gbl1!8>hm-il3&~zFca0~aJN!?b97+$E>2$Gn$31OR&UnE=Tm= zH44$Dx2HNN1lrCGjfuwo@+(m2j85w-oxre9FopupEV+6HACFyTbt}s-`lCCJ8om5RIE~T#Yg_DWu1u zyAp%jp;3&%D4;CRaR6g=f*ZvPqw2BadP=*ZYy_~CV3@wFx5YA(E8)jfqx z8tjEkMf>msMqi)zaY2fWrMq`lZzZdiMcluc(@(yxK(4hPEFk0~HO3^CUZk3;?Tv3` ze-rjZ8@hBrVPzA$^4hW?<33{d2)h7Jw?$t%V6(C_m+bNhXl9vXCJcBWmMeQoLDm5b zt9|A5pDHY#Y@(rlEo_WzXila!uaZE*WVc`=IM)SSc`#liZ2Wt*~fHgm9uH^ISX2d@)XGZ)_$qnbx6?J<14_=SS(ITs#LPDk03a&%x;bAuGz=P ze^<4p@tD@J|M;88;~IsEOPpB+&3C4!3q;}Kk2tb*WuuE z2u(BE$1(2AwbbBrmU-YLI4>#K((6&QZ~m2Yp;I14x0N8hos}{uoQuMG)Wy?ogaNayqmc&`I=8y6&dPf{Fky#B7 z#F=Xy213s`NFxjKuMqH3+ibWsFRi=QtH*j$9^)Zy8F|^vSmgj~l5<04MiU;BNyAn) zlM+c20Y#%@>WgdY>5kx}H)7*!D~BZJdg8d5iHx|>(jj=!MEmr)-$kH8?A#;DyBone(uz;e^|=9nIwfuWY?yw; zC|H`;8#O$vTPm5AW1Gg-Up&#Ca$<@!JZkAUDbmd*?X}QSA5$(*c+FZ|l+}F%*L1OH z{ck}P=j@=7>6ga#cqzj|ODXHD>ckIBmOd9Fh=~>?C7$uII_3rEX%UKdywsInR~{t- zg|t`~l=L1P_QPkZN53Q>!^A*QDZ zK(f;%VVQo)n1bsy)LWL#?&|wN`hL~Rnxhd3d-bOvlRQAiybH&=i;SlnwP$3P-!%x3^o)t6aoT-zXU}ARq-l^bOW-zg$@b|19Aua zF+k$V!uO;fNwCUEi;6!|5?4_MKtTq}|C`2gXh8EhWP1bTgZ)DqHZ&-x|E2*6Ka!RZ zS5jsHN&IW7%g1yUln@bn$cO!hR2b+`P~1-3dFIx!6EltRa{a z6Z@Y$_ug)~d%u)K$+?LYfc<87}bupdiK(3|m%hiA$Pc>zKNP0hqBj{X*L0rm@j(0s(f>>t{1L0?w#rS+#E)IdBKcF5|Dq-S zZ*-X3x;NeSuOSxS<3Q%uy1zwQ+?Kj&)Ou~-|2+&J{Zi^T=lx9+&+B^K_lQ;hY2H6D zeZ9T!H&;?$+kt+MLCs%i{8QEVi8<(Pft!mFt`}r~k5Y%93jAjQ!fgoD?Zh|Vi~q5A z27G^+_!lc1Zfo3}625-J{(B@p`IW|R4(!c|yX*Pn?*SA0)3iUGUB11uH>ab1{F$$g z|7q4=O#$9cezU54J)`wKI1_%J{14{0Zj0P3wEcKU`%-=?@(1PW+Zs0qGuI`%??IID dD~*3C;60WFKt@K_BOwYX49GZ$DDV2e{|AYb(KrAA literal 0 HcmV?d00001 diff --git a/tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.properties b/tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..4eac4a84c --- /dev/null +++ b/tests/benchmark/kotlin/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-all.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/tests/benchmark/kotlin/gradlew b/tests/benchmark/kotlin/gradlew new file mode 100755 index 000000000..adff685a0 --- /dev/null +++ b/tests/benchmark/kotlin/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/tests/benchmark/kotlin/settings.gradle.kts b/tests/benchmark/kotlin/settings.gradle.kts new file mode 100644 index 000000000..a78f73ed5 --- /dev/null +++ b/tests/benchmark/kotlin/settings.gradle.kts @@ -0,0 +1,17 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +rootProject.name = "opensandbox-pool-benchmark" diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt new file mode 100644 index 000000000..229fb9f76 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -0,0 +1,141 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import java.time.Duration + +/** + * All benchmark knobs, parsed from `--key value` command-line arguments. + */ +data class BenchmarkConfig( + val mockBaseUrl: String, + val reportDir: String, + val scenarios: List, + val maxIdle: Int, + val warmupConcurrency: Int, + val reconcileIntervalMs: Long, + val idleTimeoutS: Long, + val acquireReadyTimeoutMs: Long, + val warmupReadyTimeoutMs: Long, + val healthCheckPollingIntervalMs: Long, + val coldStartTimeoutMs: Long, + val warmWorkers: Int, + val warmRoundsPerWorker: Int, + val steadyWorkers: Int, + val steadyDurationS: Int, + val holdMinMs: Long, + val holdMaxMs: Long, + val replenishRounds: Int, + val replenishWaitTimeoutMs: Long, + val failureCreateRate: Double, + val failureAcquires: Int, + val staleAcquires: Int, + val staleRetries: Int, + val staleAcquireReadyTimeoutMs: Long, + val idleExpiryIdleTimeoutS: Long, + val idleExpiryDurationS: Int, +) { + val mockDomain: String + get() = mockBaseUrl.removePrefix("http://").removePrefix("https://").trimEnd('/') +} + +object Cli { + private val allKeys = + listOf( + "mock-base-url", + "report-dir", + "scenarios", + "max-idle", + "warmup-concurrency", + "reconcile-interval-ms", + "idle-timeout-s", + "acquire-ready-timeout-ms", + "warmup-ready-timeout-ms", + "health-check-polling-interval-ms", + "cold-start-timeout-ms", + "warm-workers", + "warm-rounds-per-worker", + "steady-workers", + "steady-duration-s", + "hold-min-ms", + "hold-max-ms", + "replenish-rounds", + "replenish-wait-timeout-ms", + "failure-create-rate", + "failure-acquires", + "stale-acquires", + "stale-retries", + "stale-acquire-ready-timeout-ms", + "idle-expiry-idle-timeout-s", + "idle-expiry-duration-s", + ) + + fun parse(args: Array): BenchmarkConfig { + val map = mutableMapOf() + var i = 0 + while (i < args.size) { + val key = args[i] + if (!key.startsWith("--")) { + throw IllegalArgumentException("unexpected argument: $key") + } + val name = key.removePrefix("--") + val value = args.getOrNull(i + 1) ?: throw IllegalArgumentException("missing value for $key") + if (name !in allKeys) { + throw IllegalArgumentException("unknown option: $key") + } + map[name] = value + i += 2 + } + return BenchmarkConfig( + mockBaseUrl = map["mock-base-url"] ?: "http://127.0.0.1:18080", + reportDir = map["report-dir"] ?: "results/run-${System.currentTimeMillis()}", + scenarios = + (map["scenarios"] ?: "all").split(",").map { it.trim() }.filter { it.isNotEmpty() }, + maxIdle = (map["max-idle"] ?: "20").toInt(), + warmupConcurrency = (map["warmup-concurrency"] ?: "4").toInt(), + reconcileIntervalMs = (map["reconcile-interval-ms"] ?: "1000").toLong(), + idleTimeoutS = (map["idle-timeout-s"] ?: "1800").toLong(), + acquireReadyTimeoutMs = (map["acquire-ready-timeout-ms"] ?: "15000").toLong(), + warmupReadyTimeoutMs = (map["warmup-ready-timeout-ms"] ?: "15000").toLong(), + healthCheckPollingIntervalMs = (map["health-check-polling-interval-ms"] ?: "200").toLong(), + coldStartTimeoutMs = (map["cold-start-timeout-ms"] ?: "120000").toLong(), + warmWorkers = (map["warm-workers"] ?: "16").toInt(), + warmRoundsPerWorker = (map["warm-rounds-per-worker"] ?: "150").toInt(), + steadyWorkers = (map["steady-workers"] ?: "16").toInt(), + steadyDurationS = (map["steady-duration-s"] ?: "60").toInt(), + holdMinMs = (map["hold-min-ms"] ?: "1000").toLong(), + holdMaxMs = (map["hold-max-ms"] ?: "5000").toLong(), + replenishRounds = (map["replenish-rounds"] ?: "20").toInt(), + replenishWaitTimeoutMs = (map["replenish-wait-timeout-ms"] ?: "15000").toLong(), + failureCreateRate = (map["failure-create-rate"] ?: "0.6").toDouble(), + failureAcquires = (map["failure-acquires"] ?: "60").toInt(), + staleAcquires = (map["stale-acquires"] ?: "100").toInt(), + staleRetries = (map["stale-retries"] ?: "3").toInt(), + staleAcquireReadyTimeoutMs = (map["stale-acquire-ready-timeout-ms"] ?: "3000").toLong(), + idleExpiryIdleTimeoutS = (map["idle-expiry-idle-timeout-s"] ?: "20").toLong(), + idleExpiryDurationS = (map["idle-expiry-duration-s"] ?: "40").toInt(), + ) + } + + fun usage(): String = + buildString { + appendLine("Usage: pool-benchmark [--key value ...]") + allKeys.forEach { appendLine(" --$it ") } + } +} + +val BenchmarkConfig.acquireTimeout: Duration get() = Duration.ofMinutes(10) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt new file mode 100644 index 000000000..62704646d --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -0,0 +1,190 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import java.io.File +import java.time.Duration +import java.time.Instant + +fun main(args: Array) { + val cfg = + try { + Cli.parse(args) + } catch (e: Exception) { + System.err.println("error: ${e.message}") + System.err.println(Cli.usage()) + kotlin.system.exitProcess(2) + } + + val mock = MockControl(cfg.mockBaseUrl, Duration.ofSeconds(10)) + if (!mock.ping()) { + System.err.println( + "error: mock server not reachable at ${cfg.mockBaseUrl} " + + "(start it first, e.g. via tests/benchmark/run.sh)", + ) + kotlin.system.exitProcess(1) + } + + val scenarios = mutableListOf() + for (name in cfg.scenarios) { + if (name == "all") { + scenarios.addAll(Scenarios.ALL_SCENARIOS.keys) + } else { + if (name !in Scenarios.ALL_SCENARIOS) { + System.err.println("error: unknown scenario '$name' (available: ${Scenarios.ALL_SCENARIOS.keys})") + kotlin.system.exitProcess(2) + } + scenarios.add(name) + } + } + + println("== OpenSandbox pool benchmark ==") + println("mock: ${cfg.mockBaseUrl} scenarios: $scenarios") + println( + "config: maxIdle=${cfg.maxIdle} warmupConcurrency=${cfg.warmupConcurrency} " + + "reconcileIntervalMs=${cfg.reconcileIntervalMs} idleTimeoutS=${cfg.idleTimeoutS}", + ) + + val results = LinkedHashMap() + val perScenarioQps = LinkedHashMap() + for (name in scenarios) { + println("\n-- scenario: $name --") + val t0 = System.nanoTime() + val section = + try { + Scenarios.ALL_SCENARIOS.getValue(name)(cfg, mock) + } catch (t: Throwable) { + System.err.println("scenario $name failed: $t") + mapOf("error" to (t.message ?: t.toString())) + } + val elapsedMs = (System.nanoTime() - t0) / 1_000_000 + println(" completed in ${elapsedMs}ms") + section.forEach { (k, v) -> println(" $k=$v") } + results[name] = section + // Precise per-API QPS observed by the mock during this scenario. The + // mock ring keeps per-second counts; the series is included in the JSON + // report for offline analysis. + perScenarioQps[name] = mock.stats()["qps"] + } + results["perScenarioQps"] = perScenarioQps + + results["mockServerStats"] = mock.stats() + + val report = + buildJsonObject { + put("runId", "${Instant.now().toEpochMilli()}") + put("timestamp", Instant.now().toString()) + put("config", toJsonElement(cfgValues(cfg))) + put("results", toJsonElement(results)) + } + + val outDir = File(cfg.reportDir) + outDir.mkdirs() + File(outDir, "report.json").writeText(Json { prettyPrint = true }.encodeToString(JsonObject.serializer(), report)) + File(outDir, "report.md").writeText(renderMarkdown(cfg, results)) + println("\n== done: report written to ${outDir.absolutePath}/report.{json,md} ==") +} + +private fun cfgValues(cfg: BenchmarkConfig): Map = + linkedMapOf( + "mockBaseUrl" to cfg.mockBaseUrl, + "scenarios" to cfg.scenarios, + "maxIdle" to cfg.maxIdle, + "warmupConcurrency" to cfg.warmupConcurrency, + "reconcileIntervalMs" to cfg.reconcileIntervalMs, + "idleTimeoutS" to cfg.idleTimeoutS, + "acquireReadyTimeoutMs" to cfg.acquireReadyTimeoutMs, + "warmupReadyTimeoutMs" to cfg.warmupReadyTimeoutMs, + "healthCheckPollingIntervalMs" to cfg.healthCheckPollingIntervalMs, + "warmWorkers" to cfg.warmWorkers, + "warmRoundsPerWorker" to cfg.warmRoundsPerWorker, + "steadyWorkers" to cfg.steadyWorkers, + "steadyDurationS" to cfg.steadyDurationS, + "holdMinMs" to cfg.holdMinMs, + "holdMaxMs" to cfg.holdMaxMs, + "failureCreateRate" to cfg.failureCreateRate, + "staleRetries" to cfg.staleRetries, + ) + +private fun toJsonElement(value: Any?): JsonElement = + when (value) { + is JsonElement -> value + is Map<*, *> -> JsonObject(value.entries.associate { (k, v) -> k.toString() to toJsonElement(v) }) + is Iterable<*> -> JsonArray(value.map { toJsonElement(it) }) + is Double -> JsonPrimitive(value) + is Float -> JsonPrimitive(value) + is Long -> JsonPrimitive(value) + is Int -> JsonPrimitive(value) + is Boolean -> JsonPrimitive(value) + is String -> JsonPrimitive(value) + is Number -> JsonPrimitive(value.toDouble()) + null -> JsonPrimitive("") + else -> JsonPrimitive(value.toString()) + } + +private fun renderMarkdown(cfg: BenchmarkConfig, results: Map): String { + val sb = StringBuilder() + sb.appendLine("# OpenSandbox Pool Benchmark") + sb.appendLine() + sb.appendLine("- runId: ${Instant.now().toEpochMilli()}") + sb.appendLine("- mock: ${cfg.mockBaseUrl}") + sb.appendLine( + "- maxIdle: ${cfg.maxIdle}, warmupConcurrency: ${cfg.warmupConcurrency}, " + + "reconcileIntervalMs: ${cfg.reconcileIntervalMs}, idleTimeoutS: ${cfg.idleTimeoutS}", + ) + sb.appendLine() + for ((scenario, section) in results) { + if (scenario == "mockServerStats") continue + sb.appendLine("## $scenario") + sb.appendLine() + if (section is Map<*, *>) { + renderMap(sb, section as Map, " ") + } + sb.appendLine() + } + sb.appendLine("## mock server stats (end of run)") + sb.appendLine() + if (results["mockServerStats"] is Map<*, *>) { + renderMap(sb, results["mockServerStats"] as Map, " ") + } + return sb.toString() +} + +private fun renderMap( + sb: StringBuilder, + map: Map<*, *>, + indent: String, +) { + for ((k, v) in map) { + if (k == "series") continue // full per-second series lives in report.json only + when (v) { + is Map<*, *> -> { + sb.appendLine("$indent$k:") + renderMap(sb, v, "$indent ") + } + is List<*> -> sb.appendLine("$indent$k: ${v.joinToString(",")}") + else -> sb.appendLine("$indent$k: $v") + } + } +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt new file mode 100644 index 000000000..9203edbf9 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +/** + * Thread-safe latency collector. Contention is negligible at the benchmark's + * acquire rates, so a lock-protected ArrayList is sufficient. + */ +class LatencyCollector { + private val lock = Any() + private val samples = ArrayList() + private var failures = 0L + + fun record(elapsedMs: Long) { + synchronized(lock) { + samples.add(elapsedMs) + } + } + + fun recordFailure() { + synchronized(lock) { + failures++ + } + } + + fun snapshot(): LatencyStats { + val sorted: LongArray + var failureCount: Long + synchronized(lock) { + sorted = LongArray(samples.size) + for ((i, v) in samples.withIndex()) sorted[i] = v + sorted.sort() + failureCount = failures + } + return LatencyStats( + n = sorted.size.toLong(), + failures = failureCount, + meanMs = if (sorted.isEmpty()) 0.0 else sorted.average(), + p50 = percentile(sorted, 0.50), + p90 = percentile(sorted, 0.90), + p95 = percentile(sorted, 0.95), + p99 = percentile(sorted, 0.99), + p999 = percentile(sorted, 0.999), + maxMs = sorted.lastOrNull() ?: 0L, + ) + } + + private fun percentile(sorted: LongArray, p: Double): Long { + if (sorted.isEmpty()) return 0L + val idx = ((sorted.size - 1) * p).toInt() + return sorted[idx] + } +} + +data class LatencyStats( + val n: Long, + val failures: Long, + val meanMs: Double, + val p50: Long, + val p90: Long, + val p95: Long, + val p99: Long, + val p999: Long, + val maxMs: Long, +) { + fun toMap(): Map = + mapOf( + "count" to n, + "failures" to failures, + "meanMs" to meanMs, + "p50Ms" to p50, + "p90Ms" to p90, + "p95Ms" to p95, + "p99Ms" to p99, + "p999Ms" to p999, + "maxMs" to maxMs, + ) +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt new file mode 100644 index 000000000..41511fc34 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.put +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import java.time.Duration + +/** + * Client for the mock server's control endpoints: + * `GET /__stats`, `POST /__config`, `POST /__reset`. + */ +class MockControl( + baseUrl: String, + requestTimeout: Duration, +) { + private val client = + OkHttpClient.Builder() + .connectTimeout(requestTimeout) + .readTimeout(requestTimeout) + .build() + private val base = baseUrl.trimEnd('/') + + private val json = Json { ignoreUnknownKeys = true } + + fun ping(): Boolean = + try { + client.newCall(Request.Builder().url("$base/__stats").get().build()).execute().use { it.isSuccessful } + } catch (e: Exception) { + false + } + + fun stats(): Map = get("__stats") + + fun reset() { + post("__reset", buildJsonObject {}) + } + + fun setFaults(createFailureRate: Double? = null, execdFailureRate: Double? = null, poisonExisting: Boolean = false) { + val body = + buildJsonObject { + createFailureRate?.let { put("createFailureRate", it) } + execdFailureRate?.let { put("execdFailureRate", it) } + if (poisonExisting) put("poisonExisting", true) + } + post("__config", body) + } + + private fun get(path: String): Map { + val response = client.newCall(Request.Builder().url("$base/$path").get().build()).execute() + response.use { + if (!it.isSuccessful) { + throw IllegalStateException("mock $path failed: HTTP ${it.code}") + } + val body = it.body?.string() ?: "{}" + return json.parseToJsonElement(body).jsonObject + } + } + + private fun post( + path: String, + body: JsonObject, + ) { + val request = + Request.Builder() + .url("$base/$path") + .post(body.toString().toRequestBody("application/json".toMediaType())) + .build() + client.newCall(request).execute().use { + if (!it.isSuccessful) { + throw IllegalStateException("mock $path failed: HTTP ${it.code} ${it.body?.string()}") + } + } + } +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt new file mode 100644 index 000000000..0f6e866d7 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import com.alibaba.opensandbox.sandbox.pool.SandboxPool +import com.alibaba.opensandbox.sandbox.config.ConnectionConfig +import com.alibaba.opensandbox.sandbox.domain.pool.AcquirePolicy +import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec +import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore +import java.time.Duration + +/** + * Builds a [SandboxPool] wired to the mock server. Health checks stay enabled + * so the execd path (connect + readiness ping) is exercised. + */ +object PoolRunner { + fun build( + cfg: BenchmarkConfig, + poolName: String, + maxIdle: Int = cfg.maxIdle, + warmupConcurrency: Int = cfg.warmupConcurrency, + reconcileIntervalMs: Long = cfg.reconcileIntervalMs, + idleTimeoutS: Long = cfg.idleTimeoutS, + maxAcquireRetries: Int = cfg.staleRetries, + acquireReadyTimeoutMs: Long = cfg.acquireReadyTimeoutMs, + ): SandboxPool { + val connectionConfig = + ConnectionConfig.builder() + .domain(cfg.mockDomain) + .protocol("http") + .requestTimeout(Duration.ofSeconds(30)) + .disableMetrics() + .build() + return SandboxPool.builder() + .poolName(poolName) + .ownerId("bench-owner-$poolName") + .maxIdle(maxIdle) + .stateStore(InMemoryPoolStateStore()) + .connectionConfig(connectionConfig) + .creationSpec( + PoolCreationSpec.builder() + .image("benchmark:mock") + .entrypoint("tail", "-f", "/dev/null") + .build(), + ) + .warmupConcurrency(warmupConcurrency) + .reconcileInterval(Duration.ofMillis(reconcileIntervalMs)) + .acquireReadyTimeout(Duration.ofMillis(acquireReadyTimeoutMs)) + .warmupReadyTimeout(Duration.ofMillis(cfg.warmupReadyTimeoutMs)) + .acquireHealthCheckPollingInterval(Duration.ofMillis(cfg.healthCheckPollingIntervalMs)) + .warmupHealthCheckPollingInterval(Duration.ofMillis(cfg.healthCheckPollingIntervalMs)) + .idleTimeout(Duration.ofSeconds(idleTimeoutS)) + .maxAcquireRetries(maxAcquireRetries) + .build() + } + + val DEFAULT_POLICY = AcquirePolicy.DIRECT_CREATE + val RETRY_POLICY = AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE + + /** Polls snapshot until idleCount reaches [target]; returns elapsed ms or -1 on timeout. */ + fun waitForIdle( + pool: SandboxPool, + target: Int, + timeoutMs: Long, + ): Long { + val deadline = System.nanoTime() + timeoutMs * 1_000_000 + while (true) { + val idle = pool.snapshot().idleCount + if (idle >= target) { + return (System.nanoTime() - (deadline - timeoutMs * 1_000_000)) / 1_000_000 + } + if (System.nanoTime() > deadline) { + return -1L + } + Thread.sleep(100) + } + } +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt new file mode 100644 index 000000000..a5cc1cad7 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -0,0 +1,395 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import com.alibaba.opensandbox.sandbox.Sandbox +import com.alibaba.opensandbox.sandbox.pool.SandboxPool +import com.alibaba.opensandbox.sandbox.domain.pool.PoolState +import kotlinx.serialization.json.JsonPrimitive +import java.util.concurrent.CountDownLatch +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import kotlin.random.Random + +/** + * Benchmark scenarios. Every scenario returns a flat-ish map that the report + * writer renders into JSON and Markdown. Each scenario owns a fresh pool and + * shuts it down before returning. + */ +object Scenarios { + + // ---------- cold-start ---------- + + fun coldStart(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "cold-start") + pool.start() + val t0 = System.nanoTime() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val stats = mock.stats() + pool.shutdown(graceful = false) + + val created = num(stats, "stats.created") + return mapOf( + "fillTimeMs" to fillMs, + "timedOut" to (fillMs < 0), + "serverCreated" to created, + "serverAliveAtFill" to num(stats, "alive"), + "overCreationOvershoot" to (created - cfg.maxIdle).coerceAtLeast(0), + ) + } + + // ---------- warm-pool acquire latency ---------- + + fun warmLatency(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "warm-latency") + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val createdBefore = num(mock.stats(), "stats.created") + + val latency = LatencyCollector() + val start = CountDownLatch(1) + val workers = cfg.warmWorkers + val rounds = cfg.warmRoundsPerWorker + val threads = Executors.newFixedThreadPool(workers) + repeat(workers) { + threads.submit { + start.await() + repeat(rounds) { + val t0 = System.nanoTime() + try { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) + latency.record((System.nanoTime() - t0) / 1_000_000) + killAndClose(sb) + } catch (t: Throwable) { + latency.recordFailure() + } + } + } + } + start.countDown() + threads.shutdown() + threads.awaitTermination(10, TimeUnit.MINUTES) + + val createdDelta = num(mock.stats(), "stats.created") - createdBefore + val acquires = rounds * workers.toLong() + pool.shutdown(graceful = false) + + return mapOf( + "fillTimeMs" to fillMs, + "latency" to latency.snapshot().toMap(), + "acquires" to acquires, + "serverCreatedDelta" to createdDelta, + "hitRatio" to (1.0 - createdDelta.toDouble() / acquires).coerceIn(0.0, 1.0), + ) + } + + // ---------- steady-state throughput ---------- + + fun steadyState(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "steady-state") + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val createdBefore = num(mock.stats(), "stats.created") + val killedBefore = num(mock.stats(), "stats.killed") + + val durationMs = cfg.steadyDurationS * 1000L + val deadline = System.nanoTime() + durationMs * 1_000_000 + val running = AtomicBoolean(true) + val latency = LatencyCollector() + val acquires = AtomicLong(0) + + val idleSamples = CopyOnWriteArrayList() + val sampler = Thread { + while (running.get()) { + idleSamples.add(pool.snapshot().idleCount) + Thread.sleep(500) + } + } + sampler.isDaemon = true + sampler.start() + + val rng = Random(System.nanoTime()) + val threads = Executors.newFixedThreadPool(cfg.steadyWorkers) + repeat(cfg.steadyWorkers) { + threads.submit { + while (System.nanoTime() < deadline) { + val t0 = System.nanoTime() + try { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) + latency.record((System.nanoTime() - t0) / 1_000_000) + acquires.incrementAndGet() + Thread.sleep(rng.nextLong(cfg.holdMinMs, cfg.holdMaxMs + 1)) + killAndClose(sb) + } catch (t: Throwable) { + latency.recordFailure() + Thread.sleep(200) + } + } + } + } + threads.shutdown() + threads.awaitTermination(15, TimeUnit.MINUTES) + running.set(false) + + val createdDelta = num(mock.stats(), "stats.created") - createdBefore + val killedDelta = num(mock.stats(), "stats.killed") - killedBefore + pool.shutdown(graceful = false) + + val idleMin = idleSamples.minOrNull() ?: 0 + val idleMean = if (idleSamples.isEmpty()) 0.0 else idleSamples.average() + val idleZeroRatio = + if (idleSamples.isEmpty()) 0.0 + else idleSamples.count { it == 0 }.toDouble() / idleSamples.size + + return mapOf( + "fillTimeMs" to fillMs, + "durationMs" to durationMs, + "workers" to cfg.steadyWorkers, + "throughputAcquiresPerSec" to (acquires.get().toDouble() / cfg.steadyDurationS), + "latency" to latency.snapshot().toMap(), + "serverCreatedDelta" to createdDelta, + "serverKilledDelta" to killedDelta, + "idleSamples" to idleSamples.size, + "idleMin" to idleMin, + "idleMean" to idleMean, + "idleEmptyRatio" to idleZeroRatio, + ) + } + + // ---------- replenish lag ---------- + + fun replenishLag(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "replenish-lag") + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + + val lags = LatencyCollector() + var timedOut = 0L + repeat(cfg.replenishRounds) { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) + killAndClose(sb) + val t0 = System.nanoTime() + val lag = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.replenishWaitTimeoutMs) + if (lag < 0) { + timedOut++ + } else { + lags.record(lag) + } + Thread.sleep(50) + } + pool.shutdown(graceful = false) + + return mapOf( + "fillTimeMs" to fillMs, + "rounds" to cfg.replenishRounds, + "replenishLagMs" to lags.snapshot().toMap(), + "timedOutRounds" to timedOut, + ) + } + + // ---------- creation-failure injection ---------- + + fun failureInjection(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "failure-injection", maxAcquireRetries = 3) + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + pool.releaseAllIdle() + + mock.setFaults(createFailureRate = cfg.failureCreateRate) + Thread.sleep(500) + + val latency = LatencyCollector() + repeat(cfg.failureAcquires) { + val t0 = System.nanoTime() + try { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.RETRY_POLICY) + latency.record((System.nanoTime() - t0) / 1_000_000) + killAndClose(sb) + } catch (t: Throwable) { + latency.recordFailure() + } + } + Thread.sleep(500) + val snap = pool.snapshot() + + mock.setFaults(createFailureRate = 0.0) + val refillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val stats = mock.stats() + pool.shutdown(graceful = false) + + return mapOf( + "fillTimeMs" to fillMs, + "createFailureRate" to cfg.failureCreateRate, + "acquires" to cfg.failureAcquires, + "latency" to latency.snapshot().toMap(), + "poolStateAfterBurst" to snap.state.name, + "backoffActive" to snap.backoffActive, + "failureCount" to snap.failureCount, + "lastError" to (snap.lastError ?: ""), + "serverCreateFailed" to num(stats, "stats.createFailed"), + "refillTimeMsAfterRecovery" to refillMs, + ) + } + + // ---------- stale idle sandboxes ---------- + + fun staleIdle(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = + PoolRunner.build( + cfg, + "stale-idle", + maxAcquireRetries = cfg.staleRetries, + acquireReadyTimeoutMs = cfg.staleAcquireReadyTimeoutMs, + ) + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val createdBefore = num(mock.stats(), "stats.created") + + // Poison every currently-alive sandbox: their execd endpoints start + // failing, so idle candidates cannot be connected. + mock.setFaults(poisonExisting = true) + + val latency = LatencyCollector() + repeat(cfg.staleAcquires) { + val t0 = System.nanoTime() + try { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.RETRY_POLICY) + latency.record((System.nanoTime() - t0) / 1_000_000) + killAndClose(sb) + } catch (t: Throwable) { + latency.recordFailure() + } + } + val stats = mock.stats() + + // Pool must drain the stale idles and refill with fresh sandboxes. + val refillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + pool.shutdown(graceful = false) + + return mapOf( + "fillTimeMs" to fillMs, + "acquires" to cfg.staleAcquires, + "latency" to latency.snapshot().toMap(), + "serverExecdPoisoned" to num(stats, "stats.execdPoisoned"), + "serverCreatedDelta" to (num(stats, "stats.created") - createdBefore), + "serverAliveAfter" to num(stats, "alive"), + "refillTimeMs" to refillMs, + ) + } + + // ---------- server-side TTL self-heal ---------- + + fun idleExpiry(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val idleTimeoutS = cfg.idleExpiryIdleTimeoutS + val pool = + PoolRunner.build( + cfg, + "idle-expiry", + maxIdle = 10, + warmupConcurrency = 2, + reconcileIntervalMs = 500, + idleTimeoutS = idleTimeoutS, + ) + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, 10, cfg.coldStartTimeoutMs) + val createdBefore = num(mock.stats(), "stats.created") + + val durationMs = cfg.idleExpiryDurationS * 1000L + val deadline = System.nanoTime() + durationMs * 1_000_000 + val running = AtomicBoolean(true) + val idleSamples = CopyOnWriteArrayList() + val sampler = Thread { + while (running.get()) { + idleSamples.add(pool.snapshot().idleCount) + Thread.sleep(250) + } + } + sampler.isDaemon = true + sampler.start() + + while (System.nanoTime() < deadline) { + Thread.sleep(100) + } + running.set(false) + + val stats = mock.stats() + val createdDelta = num(stats, "stats.created") - createdBefore + val idleMean = if (idleSamples.isEmpty()) 0.0 else idleSamples.average() + val idleMin = idleSamples.minOrNull() ?: 0 + pool.shutdown(graceful = false) + + return mapOf( + "idleTimeoutS" to idleTimeoutS, + "fillTimeMs" to fillMs, + "durationMs" to durationMs, + "serverCreatedDelta" to createdDelta, + "serverKilled" to num(stats, "stats.killed"), + "idleMean" to idleMean, + "idleMin" to idleMin, + "idleSamples" to idleSamples.size, + ) + } + + // ---------- helpers ---------- + + private fun killAndClose(sandbox: Sandbox) { + try { + sandbox.kill() + } finally { + try { + sandbox.close() + } catch (_: Exception) { + // ignore + } + } + } + + private fun num(stats: Map, dottedKey: String): Long { + var cur: Any? = stats + for (part in dottedKey.split(".")) { + if (cur !is Map<*, *>) return 0L + cur = cur[part] + } + return when (cur) { + is Number -> cur.toLong() + is String -> cur.toLongOrNull() ?: 0L + is JsonPrimitive -> cur.content.toLongOrNull() ?: 0L + else -> 0L + } + } + + val ALL_SCENARIOS = + mapOf( + "cold-start" to ::coldStart, + "warm-latency" to ::warmLatency, + "steady-state" to ::steadyState, + "replenish-lag" to ::replenishLag, + "failure-injection" to ::failureInjection, + "stale-idle" to ::staleIdle, + "idle-expiry" to ::idleExpiry, + ) +} diff --git a/tests/benchmark/kotlin/src/main/resources/simplelogger.properties b/tests/benchmark/kotlin/src/main/resources/simplelogger.properties new file mode 100644 index 000000000..66abf87b5 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/resources/simplelogger.properties @@ -0,0 +1,4 @@ +org.slf4j.simpleLogger.defaultLogLevel=error +org.slf4j.simpleLogger.showThreadName=false +org.slf4j.simpleLogger.showDateTime=false +org.slf4j.simpleLogger.showLogName=false diff --git a/tests/benchmark/mockserver/config.go b/tests/benchmark/mockserver/config.go new file mode 100644 index 000000000..abf97ace9 --- /dev/null +++ b/tests/benchmark/mockserver/config.go @@ -0,0 +1,188 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "encoding/json" + "fmt" + "math" + "math/rand" + "os" + "time" +) + +// Config models one mock-server run. It is loaded from a JSON file at startup +// and can be mutated at runtime through POST /__config (see FaultConfig). +type Config struct { + // CreateLatencyMs controls how long POST /v1/sandboxes takes before + // returning, mimicking real sandbox provisioning time. + CreateLatencyMs LatencySpec `json:"createLatencyMs"` + // CreateFailureRate is the probability [0,1] that POST /v1/sandboxes + // returns HTTP 500 instead of creating a sandbox. + CreateFailureRate float64 `json:"createFailureRate"` + // BootDelayMs is how long a freshly created sandbox stays in Pending + // state. Its execd endpoint (and therefore the SDK readiness probe) + // only succeeds after the boot delay elapses. + BootDelayMs int64 `json:"bootDelayMs"` + // ExecdFailureRate is the probability [0,1] that any execd request + // returns HTTP 500. + ExecdFailureRate float64 `json:"execdFailureRate"` + // DefaultTtl is the server-side lifetime assigned to created sandboxes + // when the request does not carry a timeout. Sandboxes are reaped at + // expiry: lifecycle lookups and execd pings start failing. + DefaultTtl time.Duration `json:"-"` + DefaultTtlSeconds int64 `json:"defaultTtlSeconds"` + // LatencyOverrides controls the response time of every other API route + // (lifecycle.get, lifecycle.delete, lifecycle.renew, lifecycle.endpoint, + // execd.ping, execd.other). Routes without an override respond + // immediately. An entry for lifecycle.create overrides CreateLatencyMs. + LatencyOverrides map[string]LatencySpec `json:"latencyOverrides"` +} + +type LatencySpec struct { + // Distribution: "lognormal" (default), "uniform", or "fixed". + Distribution string `json:"distribution"` + MeanMs float64 `json:"meanMs"` + StddevMs float64 `json:"stddevMs"` + MinMs float64 `json:"minMs"` + // MaxMs is the upper bound for "uniform" sampling. + MaxMs float64 `json:"maxMs"` +} + +// FaultConfig is the mutable runtime subset of Config, updated via +// POST /__config. Fields are optional; absent fields keep current values. +type FaultConfig struct { + CreateFailureRate *float64 `json:"createFailureRate"` + ExecdFailureRate *float64 `json:"execdFailureRate"` + BootDelayMs *int64 `json:"bootDelayMs"` + CreateLatencyMs *LatencySpec `json:"createLatencyMs"` + // LatencyOverrides replaces the whole per-route response-time map when + // present (even when empty). + LatencyOverrides *map[string]LatencySpec `json:"latencyOverrides"` + // PoisonExisting flips every currently-alive sandbox into a poisoned + // state: its execd endpoint starts failing so SDK connects to it break. + // Newly created sandboxes are unaffected. Used to simulate stale idle + // sandboxes (e.g. sandboxes that died server-side). + PoisonExisting bool `json:"poisonExisting"` +} + +func loadConfig(path string) (*Config, error) { + cfg := &Config{ + // Default response-time profile: create and delete take a uniform + // 300-800ms, execd ping a uniform 1-5s (readiness probes are slow), + // every other API a uniform 50-100ms. + CreateLatencyMs: LatencySpec{ + Distribution: "uniform", + MinMs: 300, + MaxMs: 800, + }, + LatencyOverrides: map[string]LatencySpec{ + "lifecycle.delete": {Distribution: "uniform", MinMs: 300, MaxMs: 800}, + "lifecycle.get": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, + "lifecycle.renew": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, + "lifecycle.endpoint": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, + "execd.ping": {Distribution: "uniform", MinMs: 1000, MaxMs: 5000}, + "execd.other": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, + }, + DefaultTtlSeconds: 3600, + } + if path != "" { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read config: %w", err) + } + // The default latencyOverrides map is pre-seeded, and encoding/json + // merges into existing maps instead of replacing them. Detect whether + // the file mentions the key and drop the defaults first so an empty + // map in the file means "no route latency" rather than "keep defaults". + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + if _, ok := keys["latencyOverrides"]; ok { + cfg.LatencyOverrides = nil + } + if err := json.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + } + if cfg.CreateLatencyMs.Distribution == "" { + cfg.CreateLatencyMs.Distribution = "lognormal" + } + cfg.DefaultTtl = time.Duration(cfg.DefaultTtlSeconds) * time.Second + return cfg, nil +} + +func (cfg *Config) applyFault(f FaultConfig) { + if f.CreateFailureRate != nil { + cfg.CreateFailureRate = *f.CreateFailureRate + } + if f.ExecdFailureRate != nil { + cfg.ExecdFailureRate = *f.ExecdFailureRate + } + if f.BootDelayMs != nil { + cfg.BootDelayMs = *f.BootDelayMs + } + if f.CreateLatencyMs != nil { + cfg.CreateLatencyMs = *f.CreateLatencyMs + } + if f.LatencyOverrides != nil { + cfg.LatencyOverrides = *f.LatencyOverrides + } +} + +// latencyFor returns the latency spec for one API route, or nil when the +// route should respond immediately. +func (cfg *Config) latencyFor(route string) *LatencySpec { + if cfg.LatencyOverrides != nil { + if spec, ok := cfg.LatencyOverrides[route]; ok { + return &spec + } + } + return nil +} + +// sample returns a latency duration drawn from the configured distribution. +// Uses math/rand's global functions, which are goroutine-safe. +func (s *LatencySpec) sample() time.Duration { + switch s.Distribution { + case "fixed": + return time.Duration(s.MeanMs) * time.Millisecond + case "uniform": + lo := s.MinMs + hi := s.MaxMs + if hi <= lo { + hi = s.MeanMs + } + if hi < lo { + hi, lo = lo, hi + } + ms := lo + rand.Float64()*(hi-lo) + return time.Duration(ms) * time.Millisecond + default: // lognormal + mean := math.Max(s.MeanMs, 0.001) + stddev := math.Max(s.StddevMs, 0.001) + mu := math.Log(mean * mean / math.Sqrt(mean*mean+stddev*stddev)) + sigma := math.Sqrt(math.Log(1 + stddev*stddev/(mean*mean))) + v := mu + sigma*rand.NormFloat64() + ms := math.Exp(v) + if ms < s.MinMs { + ms = s.MinMs + } + return time.Duration(ms) * time.Millisecond + } +} diff --git a/tests/benchmark/mockserver/go.mod b/tests/benchmark/mockserver/go.mod new file mode 100644 index 000000000..82aa3ebae --- /dev/null +++ b/tests/benchmark/mockserver/go.mod @@ -0,0 +1,17 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module github.com/alibaba/OpenSandbox/tests/benchmark/mockserver + +go 1.20 diff --git a/tests/benchmark/mockserver/main.go b/tests/benchmark/mockserver/main.go new file mode 100644 index 000000000..bf5371cdc --- /dev/null +++ b/tests/benchmark/mockserver/main.go @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/http" + "strconv" + "time" +) + +func main() { + lifecycleAddr := flag.String("lifecycle-addr", "127.0.0.1:18080", "lifecycle API listen address") + execdAddr := flag.String("execd-addr", "127.0.0.1:18081", "execd API listen address") + configPath := flag.String("config", "", "path to a mock server config JSON file") + statsWindowSec := flag.Int("stats-window-sec", DefaultStatsWindowSec, "per-API QPS history window in seconds") + flag.Parse() + + cfg, err := loadConfig(*configPath) + if err != nil { + log.Fatalf("load config: %v", err) + } + + host, port, err := splitHostPort(*execdAddr) + if err != nil { + log.Fatalf("invalid execd addr: %v", err) + } + mock := newMockServer(cfg, host, port, *statsWindowSec) + + lifecycleMux := http.NewServeMux() + lifecycleMux.HandleFunc("/", mock.handleLifecycle) + execdMux := http.NewServeMux() + execdMux.HandleFunc("/", mock.handleExecd) + + lifecycleSrv := &http.Server{ + Addr: *lifecycleAddr, + Handler: lifecycleMux, + ReadHeaderTimeout: 10 * time.Second, + } + execdSrv := &http.Server{ + Addr: *execdAddr, + Handler: execdMux, + ReadHeaderTimeout: 10 * time.Second, + } + + log.Printf("mock lifecycle server listening on http://%s", *lifecycleAddr) + log.Printf("mock execd server listening on http://%s", *execdAddr) + + errCh := make(chan error, 2) + go func() { errCh <- lifecycleSrv.ListenAndServe() }() + go func() { errCh <- execdSrv.ListenAndServe() }() + log.Fatal(<-errCh) +} + +func splitHostPort(addr string) (string, int, error) { + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return "", 0, fmt.Errorf("invalid addr %q: %w", addr, err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid port %q: %w", portStr, err) + } + return host, port, nil +} diff --git a/tests/benchmark/mockserver/server.go b/tests/benchmark/mockserver/server.go new file mode 100644 index 000000000..4c9c5f00c --- /dev/null +++ b/tests/benchmark/mockserver/server.go @@ -0,0 +1,522 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" +) + +// MockServer implements the OpenSandbox lifecycle API surface used by the +// sandbox SDKs plus a per-sandbox execd listener. It is a standalone, +// language-agnostic mock intended for pool benchmarks: it simulates +// provisioning latency, sandbox boot time, server-side TTL expiry, and +// per-endpoint faults. +type MockServer struct { + cfgMu sync.RWMutex + cfg *Config + mu sync.RWMutex + nextSeq uint64 + sandboxes map[string]*Sandbox + + execdHost string + execdPort int + + stats Stats + qps *QpsRegistry +} + +// Sandbox is the mock's view of a sandbox on the lifecycle side. +type Sandbox struct { + ID string + CreatedAt time.Time + ExpiresAt time.Time + State string // "Pending" | "Running" | "Terminated" + Poisoned bool +} + +func (s *Sandbox) alive(now time.Time) bool { + return s.State != "Terminated" && (s.ExpiresAt.IsZero() || s.ExpiresAt.After(now)) +} + +func (s *Sandbox) running(now time.Time) bool { + return s.alive(now) && s.State == "Running" +} + +func newMockServer(cfg *Config, execdHost string, execdPort int, windowSec int) *MockServer { + return &MockServer{ + cfg: cfg, + sandboxes: make(map[string]*Sandbox), + execdHost: execdHost, + execdPort: execdPort, + qps: newQpsRegistry(windowSec), + } +} + +// recordQps attributes one request to a route. Handlers defer this at entry so +// every outcome (including faults) is counted. +func (m *MockServer) recordQps(route string, start time.Time) { + m.qps.record(route, time.Now(), time.Since(start)) +} + +// applyRouteLatency sleeps for the configured response time of [route], +// if any override exists. +func (m *MockServer) applyRouteLatency(route string) { + cfg := m.cfgSnapshot() + if spec := cfg.latencyFor(route); spec != nil { + time.Sleep(spec.sample()) + } +} + +// ---------- lifecycle handlers ---------- + +func (m *MockServer) handleLifecycle(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case path == "/__stats": + m.handleStats(w, r) + case path == "/__config": + m.handleConfig(w, r) + case path == "/__reset": + m.handleReset(w, r) + case r.Method == http.MethodPost && path == "/v1/sandboxes": + m.handleCreate(w, r) + case r.Method == http.MethodDelete && strings.HasPrefix(path, "/v1/sandboxes/"): + m.handleDelete(w, r, strings.TrimPrefix(path, "/v1/sandboxes/")) + case r.Method == http.MethodPost && strings.HasSuffix(path, "/renew-expiration"): + id := strings.TrimSuffix(strings.TrimPrefix(path, "/v1/sandboxes/"), "/renew-expiration") + m.handleRenew(w, r, id) + case r.Method == http.MethodGet && strings.HasPrefix(path, "/v1/sandboxes/") && strings.Contains(path, "/endpoints/"): + parts := strings.Split(strings.TrimPrefix(path, "/v1/sandboxes/"), "/endpoints/") + m.handleEndpoint(w, r, parts[0]) + case r.Method == http.MethodGet && strings.HasPrefix(path, "/v1/sandboxes/"): + m.handleGet(w, r, strings.TrimPrefix(path, "/v1/sandboxes/")) + default: + writeError(w, http.StatusNotFound, "NOT_FOUND", "no such route: "+path) + } +} + +func (m *MockServer) handleCreate(w http.ResponseWriter, r *http.Request) { + start := time.Now() + defer m.recordQps("lifecycle.create", start) + _, _ = io.Copy(io.Discard, r.Body) + + cfg := m.cfgSnapshot() + if cfg.CreateFailureRate > 0 && rand.Float64() < cfg.CreateFailureRate { + m.stats.incCreateFailed() + writeError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "simulated provisioning failure") + return + } + + latency := cfg.CreateLatencyMs.sample() + if override := cfg.latencyFor("lifecycle.create"); override != nil { + latency = override.sample() + } + time.Sleep(latency) + m.stats.recordCreateLatency(latency) + + now := time.Now().UTC() + expiresAt := now.Add(cfg.DefaultTtl) + id := m.newSandboxID() + sandbox := &Sandbox{ + ID: id, + CreatedAt: now, + ExpiresAt: expiresAt, + State: "Pending", + } + m.mu.Lock() + m.sandboxes[id] = sandbox + m.mu.Unlock() + + go m.bootSandbox(id, cfg.BootDelayMs) + + m.stats.incCreated() + writeJSON(w, http.StatusCreated, map[string]any{ + "id": id, + "status": sandboxStatus(sandbox, now), + "createdAt": now.Format(time.RFC3339), + "expiresAt": expiresAt.Format(time.RFC3339), + "entrypoint": []string{"tail", "-f", "/dev/null"}, + }) +} + +func (m *MockServer) bootSandbox(id string, delayMs int64) { + if delayMs <= 0 { + delayMs = 1 + } + time.Sleep(time.Duration(delayMs) * time.Millisecond) + m.mu.Lock() + if sb, ok := m.sandboxes[id]; ok && sb.State == "Pending" { + sb.State = "Running" + } + m.mu.Unlock() +} + +func (m *MockServer) handleGet(w http.ResponseWriter, r *http.Request, id string) { + start := time.Now() + defer m.recordQps("lifecycle.get", start) + m.applyRouteLatency("lifecycle.get") + _ = r.Body.Close() + m.stats.incSandboxGets() + m.mu.RLock() + sb := m.sandboxes[id] + m.mu.RUnlock() + if sb == nil || !sb.alive(time.Now()) { + writeError(w, http.StatusNotFound, "NOT_FOUND", "sandbox not found: "+id) + return + } + writeJSON(w, http.StatusOK, m.sandboxInfo(sb)) +} + +func (m *MockServer) handleDelete(w http.ResponseWriter, r *http.Request, id string) { + start := time.Now() + defer m.recordQps("lifecycle.delete", start) + m.applyRouteLatency("lifecycle.delete") + _ = r.Body.Close() + m.mu.Lock() + sb := m.sandboxes[id] + if sb != nil { + sb.State = "Terminated" + } + m.mu.Unlock() + m.stats.incKilled() + // DELETE of an unknown sandbox still succeeds (best-effort semantics); + // a killed sandbox's execd endpoint stops responding. + w.WriteHeader(http.StatusNoContent) +} + +func (m *MockServer) handleRenew(w http.ResponseWriter, r *http.Request, id string) { + start := time.Now() + defer m.recordQps("lifecycle.renew", start) + m.applyRouteLatency("lifecycle.renew") + defer r.Body.Close() + m.stats.incRenews() + m.mu.RLock() + sb := m.sandboxes[id] + m.mu.RUnlock() + if sb == nil || !sb.alive(time.Now()) { + writeError(w, http.StatusNotFound, "NOT_FOUND", "sandbox not found: "+id) + return + } + var body struct { + ExpiresAt string `json:"expiresAt"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.ExpiresAt == "" { + writeError(w, http.StatusBadRequest, "INVALID_REQUEST", "expiresAt is required") + return + } + expiresAt, err := time.Parse(time.RFC3339, body.ExpiresAt) + if err != nil { + writeError(w, http.StatusBadRequest, "INVALID_REQUEST", "expiresAt must be RFC3339: "+err.Error()) + return + } + m.mu.Lock() + sb.ExpiresAt = expiresAt + m.mu.Unlock() + writeJSON(w, http.StatusOK, map[string]any{"expiresAt": expiresAt.Format(time.RFC3339)}) +} + +func (m *MockServer) handleEndpoint(w http.ResponseWriter, r *http.Request, id string) { + start := time.Now() + defer m.recordQps("lifecycle.endpoint", start) + m.applyRouteLatency("lifecycle.endpoint") + _ = r.Body.Close() + m.stats.incEndpointGets() + m.mu.RLock() + sb := m.sandboxes[id] + m.mu.RUnlock() + if sb == nil || !sb.alive(time.Now()) { + writeError(w, http.StatusNotFound, "NOT_FOUND", "sandbox not found: "+id) + return + } + // The endpoint URL and token let the execd listener attribute requests + // to this sandbox so boot state and poisoning can be enforced. + writeJSON(w, http.StatusOK, map[string]any{ + "endpoint": fmt.Sprintf("%s:%d", m.execdHost, m.execdPort), + "headers": map[string]string{"X-EXECD-ACCESS-TOKEN": execdToken(id)}, + }) +} + +func (m *MockServer) sandboxInfo(sb *Sandbox) map[string]any { + return map[string]any{ + "id": sb.ID, + "status": sandboxStatus(sb, time.Now()), + "createdAt": sb.CreatedAt.Format(time.RFC3339), + "expiresAt": sb.ExpiresAt.Format(time.RFC3339), + "entrypoint": []string{"tail", "-f", "/dev/null"}, + } +} + +// ---------- execd handlers ---------- + +func (m *MockServer) handleExecd(w http.ResponseWriter, r *http.Request) { + start := time.Now() + route := "execd.other" + if r.URL.Path == "/ping" { + route = "execd.ping" + } + defer m.recordQps(route, start) + m.stats.incExecdRequests() + token := r.Header.Get("X-EXECD-ACCESS-TOKEN") + id := strings.TrimPrefix(token, "mock-token-") + if token != "" && token == execdToken(id) { + m.mu.RLock() + sb := m.sandboxes[id] + m.mu.RUnlock() + if sb == nil || !sb.running(time.Now()) { + // Not booted yet, expired, or killed. Fail fast and answer with + // a non-retryable status (404): the SDK's retry interceptor + // retries 5xx and transport errors with backoff, which would + // pollute the readiness-poll timing this mock is meant to + // measure. 404 makes the SDK poll at its configured interval + // until ready. The readiness probe only starts paying the route + // latency once the sandbox is actually up. + writeError(w, http.StatusNotFound, "NOT_READY", "sandbox not ready: "+id) + return + } + if sb.Poisoned { + m.stats.incExecdPoisoned() + writeError(w, http.StatusNotFound, "POISONED", "sandbox endpoint poisoned") + return + } + } + // A ready sandbox's requests pay the configured route latency. + m.applyRouteLatency(route) + cfg := m.cfgSnapshot() + if cfg.ExecdFailureRate > 0 && rand.Float64() < cfg.ExecdFailureRate { + m.stats.incExecdFailures() + writeError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "simulated execd failure") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"status":"ok"}`)) +} + +// ---------- control handlers ---------- + +func (m *MockServer) handleStats(w http.ResponseWriter, r *http.Request) { + _ = r.Body.Close() + stats := m.stats.snapshot() + m.mu.RLock() + alive := 0 + poisoned := 0 + now := time.Now() + for _, sb := range m.sandboxes { + if sb.alive(now) { + alive++ + } + if sb.Poisoned { + poisoned++ + } + } + m.mu.RUnlock() + writeJSON(w, http.StatusOK, map[string]any{ + "stats": stats, + "alive": alive, + "poisoned": poisoned, + "config": m.cfgSnapshot(), + "qps": m.qps.snapshot(time.Now()), + "serverTime": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (m *MockServer) handleConfig(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var f FaultConfig + if err := json.NewDecoder(r.Body).Decode(&f); err != nil { + writeError(w, http.StatusBadRequest, "INVALID_REQUEST", err.Error()) + return + } + m.cfgMu.Lock() + m.cfg.applyFault(f) + m.cfgMu.Unlock() + if f.PoisonExisting { + m.mu.Lock() + for _, sb := range m.sandboxes { + if sb.alive(time.Now()) { + sb.Poisoned = true + } + } + m.mu.Unlock() + } + writeJSON(w, http.StatusOK, map[string]any{"config": m.cfgSnapshot()}) +} + +func (m *MockServer) handleReset(w http.ResponseWriter, r *http.Request) { + _ = r.Body.Close() + m.stats.reset() + m.qps.reset(time.Now()) + writeJSON(w, http.StatusOK, map[string]any{"reset": true}) +} + +// ---------- helpers ---------- + +func (m *MockServer) newSandboxID() string { + m.mu.Lock() + defer m.mu.Unlock() + m.nextSeq++ + return fmt.Sprintf("sbx-mock-%d-%d", time.Now().UnixNano(), m.nextSeq) +} + +func (m *MockServer) cfgSnapshot() Config { + m.cfgMu.RLock() + defer m.cfgMu.RUnlock() + return *m.cfg +} + +func sandboxStatus(sb *Sandbox, now time.Time) map[string]any { + return map[string]any{ + "state": sb.State, + "lastTransitionAt": sb.CreatedAt.Format(time.RFC3339), + } +} + +func execdToken(id string) string { return "mock-token-" + id } + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeError(w http.ResponseWriter, status int, code, message string) { + writeJSON(w, status, map[string]any{"code": code, "message": message}) +} + +// Stats are the server-side counters exposed via /__stats. They let the +// benchmark driver validate pool behavior (no over-creation, stale cleanup, +// hit rate) and cross-check client-observed numbers. +type Stats struct { + created atomic.Int64 + createFailed atomic.Int64 + killed atomic.Int64 + renews atomic.Int64 + sandboxGets atomic.Int64 + endpointGets atomic.Int64 + execdRequests atomic.Int64 + execdFailures atomic.Int64 + execdPoisoned atomic.Int64 + latencyMu sync.Mutex + createLatencyMs []int64 + maxCreateLatency atomic.Int64 +} + +type StatsSnapshot struct { + Created int64 `json:"created"` + CreateFailed int64 `json:"createFailed"` + Killed int64 `json:"killed"` + Renews int64 `json:"renews"` + SandboxGets int64 `json:"sandboxGets"` + EndpointGets int64 `json:"endpointGets"` + ExecdRequests int64 `json:"execdRequests"` + ExecdFailures int64 `json:"execdFailures"` + ExecdPoisoned int64 `json:"execdPoisoned"` + CreateLatencyMsAvg float64 `json:"createLatencyMsAvg"` + CreateLatencyMsP50 int64 `json:"createLatencyMsP50"` + CreateLatencyMsP95 int64 `json:"createLatencyMsP95"` + CreateLatencyMsP99 int64 `json:"createLatencyMsP99"` + MaxCreateLatencyMs int64 `json:"maxCreateLatencyMs"` +} + +func (s *Stats) incCreated() { s.created.Add(1) } +func (s *Stats) incCreateFailed() { s.createFailed.Add(1) } +func (s *Stats) incKilled() { s.killed.Add(1) } +func (s *Stats) incRenews() { s.renews.Add(1) } +func (s *Stats) incSandboxGets() { s.sandboxGets.Add(1) } +func (s *Stats) incEndpointGets() { s.endpointGets.Add(1) } +func (s *Stats) incExecdRequests() { s.execdRequests.Add(1) } +func (s *Stats) incExecdFailures() { s.execdFailures.Add(1) } +func (s *Stats) incExecdPoisoned() { s.execdPoisoned.Add(1) } + +func (s *Stats) recordCreateLatency(d time.Duration) { + ms := d.Milliseconds() + s.latencyMu.Lock() + s.createLatencyMs = append(s.createLatencyMs, ms) + s.latencyMu.Unlock() + if cur := s.maxCreateLatency.Load(); ms > cur { + s.maxCreateLatency.CompareAndSwap(cur, ms) + } +} + +func (s *Stats) snapshot() StatsSnapshot { + s.latencyMu.Lock() + samples := append([]int64(nil), s.createLatencyMs...) + s.latencyMu.Unlock() + sorted := make([]int64, len(samples)) + copy(sorted, samples) + // insertion sort: sample counts stay small for benchmark runs + for i := 1; i < len(sorted); i++ { + for j := i; j > 0 && sorted[j] < sorted[j-1]; j-- { + sorted[j], sorted[j-1] = sorted[j-1], sorted[j] + } + } + percentile := func(p float64) int64 { + if len(sorted) == 0 { + return 0 + } + idx := int(float64(len(sorted)-1) * p) + return sorted[idx] + } + var sum int64 + for _, v := range sorted { + sum += v + } + avg := 0.0 + if len(sorted) > 0 { + avg = float64(sum) / float64(len(sorted)) + } + return StatsSnapshot{ + Created: s.created.Load(), + CreateFailed: s.createFailed.Load(), + Killed: s.killed.Load(), + Renews: s.renews.Load(), + SandboxGets: s.sandboxGets.Load(), + EndpointGets: s.endpointGets.Load(), + ExecdRequests: s.execdRequests.Load(), + ExecdFailures: s.execdFailures.Load(), + ExecdPoisoned: s.execdPoisoned.Load(), + CreateLatencyMsAvg: avg, + CreateLatencyMsP50: percentile(0.50), + CreateLatencyMsP95: percentile(0.95), + CreateLatencyMsP99: percentile(0.99), + MaxCreateLatencyMs: s.maxCreateLatency.Load(), + } +} + +func (s *Stats) reset() { + s.created.Store(0) + s.createFailed.Store(0) + s.killed.Store(0) + s.renews.Store(0) + s.sandboxGets.Store(0) + s.endpointGets.Store(0) + s.execdRequests.Store(0) + s.execdFailures.Store(0) + s.execdPoisoned.Store(0) + s.maxCreateLatency.Store(0) + s.latencyMu.Lock() + s.createLatencyMs = nil + s.latencyMu.Unlock() +} diff --git a/tests/benchmark/mockserver/stats.go b/tests/benchmark/mockserver/stats.go new file mode 100644 index 000000000..d486181ea --- /dev/null +++ b/tests/benchmark/mockserver/stats.go @@ -0,0 +1,210 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "sync" + "time" +) + +// DefaultStatsWindowSec is how many seconds of per-API request history the +// mock keeps for QPS analysis. 30 minutes covers a full default benchmark +// run; raise it with -stats-window-sec for longer soaks. +const DefaultStatsWindowSec = 1800 + +// qpsTracker records exact per-second request counts for one API route in a +// ring buffer. Snapshots expose totals, recent-window rates, and the full +// per-second series so the driver can plot pool load over time. +type qpsTracker struct { + mu sync.Mutex + window int64 + buckets []int64 // ring of per-second counts + secs []int64 // wall-clock second each bucket covers + startSec int64 // first recorded second (0 until first request/reset) + total int64 + latencySumMs int64 + maxMs int64 +} + +func newQpsTracker(windowSec int) *qpsTracker { + return &qpsTracker{ + window: int64(windowSec), + buckets: make([]int64, windowSec), + secs: make([]int64, windowSec), + } +} + +func (t *qpsTracker) record(now time.Time, elapsed time.Duration) { + sec := now.Unix() + idx := sec % t.window + t.mu.Lock() + if t.secs[idx] != sec { + // Lazy bucket reset: first request in this second. + t.secs[idx] = sec + t.buckets[idx] = 0 + } + t.buckets[idx]++ + t.total++ + if t.startSec == 0 { + t.startSec = sec + } + ms := elapsed.Milliseconds() + t.latencySumMs += ms + if ms > t.maxMs { + t.maxMs = ms + } + t.mu.Unlock() +} + +func (t *qpsTracker) reset(now time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + for i := range t.buckets { + t.buckets[i] = 0 + t.secs[i] = 0 + } + t.total = 0 + t.latencySumMs = 0 + t.maxMs = 0 + t.startSec = now.Unix() +} + +type QpsSnapshot struct { + Total int64 `json:"total"` + Qps1s float64 `json:"qps1s"` + Qps5s float64 `json:"qps5s"` + Qps60s float64 `json:"qps60s"` + SeriesStart int64 `json:"seriesStartUnixSec"` + Series []int64 `json:"series"` + // AvgMs is the average handler latency for this route since the last reset. + AvgMs float64 `json:"avgMs"` + MaxMs int64 `json:"maxMs"` +} + +// snapshot returns totals, rates over the trailing 1s/5s/60s windows (the +// current, possibly partial second counts toward qps1s), and the per-second +// series covering the retained window. Requests older than the ring are +// dropped, so for runs longer than the window the driver should poll /__stats +// and accumulate externally. +func (t *qpsTracker) snapshot(now time.Time) QpsSnapshot { + sec := now.Unix() + t.mu.Lock() + defer t.mu.Unlock() + + if t.startSec == 0 { + t.startSec = sec + } + begin := t.startSec + if sec-begin+1 > t.window { + begin = sec - t.window + 1 + } + + avgLatency := 0.0 + if t.total > 0 { + avgLatency = float64(t.latencySumMs) / float64(t.total) + } + + series := make([]int64, 0, sec-begin+1) + for s := begin; s <= sec; s++ { + idx := s % t.window + count := t.buckets[idx] + if t.secs[idx] != s { + count = 0 + } + series = append(series, count) + } + + rate := func(windowSec int64) float64 { + if windowSec <= 0 { + return 0 + } + start := sec - windowSec + 1 + if start < begin { + start = begin + } + if sec < start { + return 0 + } + var sum int64 + for s := start; s <= sec; s++ { + idx := s % t.window + if t.secs[idx] == s { + sum += t.buckets[idx] + } + } + return float64(sum) / float64(sec-start+1) + } + + return QpsSnapshot{ + Total: t.total, + Qps1s: rate(1), + Qps5s: rate(5), + Qps60s: rate(60), + SeriesStart: begin, + Series: series, + AvgMs: avgLatency, + MaxMs: t.maxMs, + } +} + +// QpsRegistry tracks one tracker per API route. +type QpsRegistry struct { + windowSec int + mu sync.RWMutex + trackers map[string]*qpsTracker +} + +func newQpsRegistry(windowSec int) *QpsRegistry { + return &QpsRegistry{ + windowSec: windowSec, + trackers: make(map[string]*qpsTracker), + } +} + +func (r *QpsRegistry) record(route string, now time.Time, elapsed time.Duration) { + r.mu.RLock() + t := r.trackers[route] + r.mu.RUnlock() + if t == nil { + r.mu.Lock() + t = r.trackers[route] + if t == nil { + t = newQpsTracker(r.windowSec) + r.trackers[route] = t + } + r.mu.Unlock() + } + t.record(now, elapsed) +} + +func (r *QpsRegistry) reset(now time.Time) { + r.mu.RLock() + defer r.mu.RUnlock() + for _, t := range r.trackers { + t.reset(now) + } +} + +func (r *QpsRegistry) snapshot(now time.Time) map[string]QpsSnapshot { + r.mu.RLock() + defer r.mu.RUnlock() + out := make(map[string]QpsSnapshot, len(r.trackers)) + for route, t := range r.trackers { + out[route] = t.snapshot(now) + } + return out +} diff --git a/tests/benchmark/run.sh b/tests/benchmark/run.sh new file mode 100755 index 000000000..9f1da69a6 --- /dev/null +++ b/tests/benchmark/run.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +BENCH_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${BENCH_DIR}/../.." && pwd)" + +# Gradle 9 requires JVM 17+. Override JAVA_HOME when it is unset or points at +# an older JDK; otherwise pick the highest installed major >= 17. +NEEDS_OVERRIDE=false +if [ -z "${JAVA_HOME:-}" ]; then + NEEDS_OVERRIDE=true +elif [ -x "${JAVA_HOME}/bin/java" ]; then + MAJOR="$("${JAVA_HOME}/bin/java" -version 2>&1 | sed -nE 's/.*version "([0-9]+).*/\1/p' | head -1)" + case "${MAJOR}" in + 8|9|10|11|12|13|14|15|16) NEEDS_OVERRIDE=true ;; + esac +fi +if [ "${NEEDS_OVERRIDE}" = "true" ] && command -v /usr/libexec/java_home > /dev/null 2>&1; then + # Highest installed major >= 17, e.g. "17.0.7" or "21.0.4". + VERSION="$( + /usr/libexec/java_home -V 2>&1 \ + | sed -nE 's/^[[:space:]]*([0-9]+)\.([0-9]+).*/\1.\2/p' \ + | awk -F. '$1 >= 17' \ + | sort -t. -k1,1n -k2,2n | tail -1 + )" + if [ -n "${VERSION}" ]; then + JH="$(/usr/libexec/java_home -v "${VERSION}" 2>/dev/null || true)" + if [ -n "${JH}" ]; then + export JAVA_HOME="${JH}" + fi + fi +fi + +LIFECYCLE_ADDR="${LIFECYCLE_ADDR:-127.0.0.1:18080}" +EXECD_ADDR="${EXECD_ADDR:-127.0.0.1:18081}" +MOCK_CONFIG="${MOCK_CONFIG:-${BENCH_DIR}/configs/default.json}" +SKIP_SDK_PUBLISH="${SKIP_SDK_PUBLISH:-false}" +MOCK_PID="" + +cleanup() { + if [ -n "${MOCK_PID}" ]; then + kill "${MOCK_PID}" 2>/dev/null || true + wait "${MOCK_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +usage() { + cat <<'EOF' +Usage: run.sh [--skip-sdk-publish] [--mock-config ] [-- ] + +Environment: + LIFECYCLE_ADDR lifecycle mock listen address (default 127.0.0.1:18080) + EXECD_ADDR execd mock listen address (default 127.0.0.1:18081) + MOCK_CONFIG mock server config JSON (default configs/default.json) + +Driver args (after --) are forwarded to the benchmark driver, e.g.: + ./run.sh -- --max-idle 50 --scenarios warm-latency,steady-state +EOF +} + +# parse flags +DRIVER_ARGS=() +while [ $# -gt 0 ]; do + case "$1" in + --skip-sdk-publish) SKIP_SDK_PUBLISH=true ;; + --mock-config) shift; MOCK_CONFIG="$1" ;; + --) shift; DRIVER_ARGS=("$@"); break ;; + -h|--help) usage; exit 0 ;; + *) DRIVER_ARGS=("$@"); break ;; + esac + shift +done + +# 1. publish the Kotlin SDK to mavenLocal (driver resolves com.alibaba.opensandbox:sandbox:latest.integration) +if [ "${SKIP_SDK_PUBLISH}" != "true" ]; then + echo "== publishing Kotlin SDK to mavenLocal ==" + (cd "${REPO_ROOT}/sdks/sandbox/kotlin" && ./gradlew -q publishToMavenLocal --no-build-cache) +else + echo "== skipping SDK publish (--skip-sdk-publish) ==" +fi + +# 2. build the mock server +echo "== building mock server ==" +(cd "${BENCH_DIR}/mockserver" && go build -o "${BENCH_DIR}/bin/mockserver" .) + +# 3. start the mock server +mkdir -p "${BENCH_DIR}/results" +echo "== starting mock server (lifecycle=${LIFECYCLE_ADDR}, execd=${EXECD_ADDR}, config=${MOCK_CONFIG}) ==" +"${BENCH_DIR}/bin/mockserver" \ + -lifecycle-addr "${LIFECYCLE_ADDR}" \ + -execd-addr "${EXECD_ADDR}" \ + -config "${MOCK_CONFIG}" \ + > "${BENCH_DIR}/results/mockserver.log" 2>&1 & +MOCK_PID=$! + +for _ in $(seq 1 50); do + if curl -fsS "http://${LIFECYCLE_ADDR}/__stats" > /dev/null 2>&1; then + break + fi + sleep 0.2 +done +if ! curl -fsS "http://${LIFECYCLE_ADDR}/__stats" > /dev/null 2>&1; then + echo "error: mock server did not come up" >&2 + cat "${BENCH_DIR}/results/mockserver.log" >&2 + exit 1 +fi + +# 4. run the driver +SDK_VERSION="$(grep '^project.version=' "${REPO_ROOT}/sdks/sandbox/kotlin/gradle.properties" | cut -d= -f2 | tr -d '[:space:]')" +echo "== running benchmark driver (sandbox SDK ${SDK_VERSION}) ==" +DRIVER_ARGS+=("--report-dir" "${BENCH_DIR}/results/run-$(date +%Y%m%d-%H%M%S)") +(cd "${BENCH_DIR}/kotlin" && ./gradlew --console=plain run -PsandboxVersion="${SDK_VERSION}" --args="${DRIVER_ARGS[*]}") + +echo "== done ==" From 1e6c2d34f4f5ad9595ce5ede5f614eb984c2398f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 12:01:13 +0800 Subject: [PATCH 02/16] refactor(benchmark): build Kotlin SDK from source via Gradle composite build Replace the mavenLocal-published artifact dependency with includeBuild on sdks/sandbox/kotlin so the benchmark always runs the checked-out SDK code. Drops the publishToMavenLocal step from run.sh and the -PsandboxVersion injection. --- tests/benchmark/README.md | 20 +++++++++++------- tests/benchmark/kotlin/build.gradle.kts | 24 ++++------------------ tests/benchmark/kotlin/settings.gradle.kts | 4 ++++ tests/benchmark/run.sh | 24 +++++++--------------- 4 files changed, 28 insertions(+), 44 deletions(-) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index 7d38efefb..cb2baf9a9 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -18,8 +18,10 @@ tests/benchmark/ ## Prerequisites - Go (any recent version; mock server uses stdlib only) -- JDK 17+ (driver + Kotlin SDK) -- A Gradle wrapper is included under `kotlin/`. +- JDK 17+ (driver + Kotlin SDK; `run.sh` picks a suitable JDK automatically + on macOS) +- A Gradle wrapper is included under `kotlin/`. The SDK sources are referenced + in place (composite build), so no separate install is needed. ## Quick start @@ -36,13 +38,17 @@ tests/benchmark/ `run.sh` performs three steps: -1. Publishes the Kotlin SDK to `mavenLocal` (`sdks/sandbox/kotlin/publishToMavenLocal`); - the driver then resolves that exact version (`project.version` from - `gradle.properties`). Pass `--skip-sdk-publish` to reuse the last published - snapshot. -2. Builds and starts the mock server (`go build` + exec). +1. Builds the mock server (`go build` + exec). +2. Starts the mock server. 3. Runs the driver: `./gradlew run` with forwarded `--key value` args. +The Kotlin SDK is **built from source**: the driver uses a Gradle composite +build (`includeBuild` in `kotlin/settings.gradle.kts`), so the benchmark always +runs the checked-out SDK code and picks up SDK changes without any +publish/install step. `com.alibaba.opensandbox:sandbox:1.0.18` in +`kotlin/build.gradle.kts` is a module coordinate that the composite build +substitutes with the local `:sandbox` project (the version is informational). + Reports land in `results/run-/report.{json,md}`; the mock log is at `results/mockserver.log`. Exit code is non-zero when a scenario fails. diff --git a/tests/benchmark/kotlin/build.gradle.kts b/tests/benchmark/kotlin/build.gradle.kts index 64db29fa9..f2a0f14d5 100644 --- a/tests/benchmark/kotlin/build.gradle.kts +++ b/tests/benchmark/kotlin/build.gradle.kts @@ -28,29 +28,14 @@ java { } repositories { - mavenLocal() - exclusiveContent { - forRepository { - mavenLocal() - } - filter { - includeGroup("com.alibaba.opensandbox") - } - } mavenCentral() } -configurations.configureEach { - resolutionStrategy.cacheDynamicVersionsFor(0, "seconds") - resolutionStrategy.cacheChangingModulesFor(0, "seconds") -} - dependencies { - // OpenSandbox Kotlin SDK (published to mavenLocal; see tests/benchmark/README.md). - // The version is taken from the SDK's own gradle.properties so run.sh and - // this module cannot drift; pass -PsandboxVersion=... to override. - val sandboxVersion = (project.findProperty("sandboxVersion") as String?) ?: "1.0.18" - implementation("com.alibaba.opensandbox:sandbox:$sandboxVersion") + // OpenSandbox Kotlin SDK, built from source via composite build + // (see settings.gradle.kts). The module coordinate is substituted by the + // included build's :sandbox project; the version is informational only. + implementation("com.alibaba.opensandbox:sandbox:1.0.18") implementation("com.squareup.okhttp3:okhttp:4.12.0") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") @@ -71,4 +56,3 @@ tasks.withType { sourceCompatibility = "11" targetCompatibility = "11" } - diff --git a/tests/benchmark/kotlin/settings.gradle.kts b/tests/benchmark/kotlin/settings.gradle.kts index a78f73ed5..137bf298e 100644 --- a/tests/benchmark/kotlin/settings.gradle.kts +++ b/tests/benchmark/kotlin/settings.gradle.kts @@ -15,3 +15,7 @@ */ rootProject.name = "opensandbox-pool-benchmark" + +// Build the Kotlin SDK from source (Gradle composite build) so the benchmark +// always runs the checked-out SDK code; no mavenLocal publish step needed. +includeBuild("../../../sdks/sandbox/kotlin") diff --git a/tests/benchmark/run.sh b/tests/benchmark/run.sh index 9f1da69a6..d479e050d 100755 --- a/tests/benchmark/run.sh +++ b/tests/benchmark/run.sh @@ -48,7 +48,6 @@ fi LIFECYCLE_ADDR="${LIFECYCLE_ADDR:-127.0.0.1:18080}" EXECD_ADDR="${EXECD_ADDR:-127.0.0.1:18081}" MOCK_CONFIG="${MOCK_CONFIG:-${BENCH_DIR}/configs/default.json}" -SKIP_SDK_PUBLISH="${SKIP_SDK_PUBLISH:-false}" MOCK_PID="" cleanup() { @@ -61,7 +60,7 @@ trap cleanup EXIT usage() { cat <<'EOF' -Usage: run.sh [--skip-sdk-publish] [--mock-config ] [-- ] +Usage: run.sh [--mock-config ] [-- ] Environment: LIFECYCLE_ADDR lifecycle mock listen address (default 127.0.0.1:18080) @@ -77,7 +76,6 @@ EOF DRIVER_ARGS=() while [ $# -gt 0 ]; do case "$1" in - --skip-sdk-publish) SKIP_SDK_PUBLISH=true ;; --mock-config) shift; MOCK_CONFIG="$1" ;; --) shift; DRIVER_ARGS=("$@"); break ;; -h|--help) usage; exit 0 ;; @@ -86,19 +84,11 @@ while [ $# -gt 0 ]; do shift done -# 1. publish the Kotlin SDK to mavenLocal (driver resolves com.alibaba.opensandbox:sandbox:latest.integration) -if [ "${SKIP_SDK_PUBLISH}" != "true" ]; then - echo "== publishing Kotlin SDK to mavenLocal ==" - (cd "${REPO_ROOT}/sdks/sandbox/kotlin" && ./gradlew -q publishToMavenLocal --no-build-cache) -else - echo "== skipping SDK publish (--skip-sdk-publish) ==" -fi - -# 2. build the mock server +# 1. build the mock server echo "== building mock server ==" (cd "${BENCH_DIR}/mockserver" && go build -o "${BENCH_DIR}/bin/mockserver" .) -# 3. start the mock server +# 2. start the mock server mkdir -p "${BENCH_DIR}/results" echo "== starting mock server (lifecycle=${LIFECYCLE_ADDR}, execd=${EXECD_ADDR}, config=${MOCK_CONFIG}) ==" "${BENCH_DIR}/bin/mockserver" \ @@ -120,10 +110,10 @@ if ! curl -fsS "http://${LIFECYCLE_ADDR}/__stats" > /dev/null 2>&1; then exit 1 fi -# 4. run the driver -SDK_VERSION="$(grep '^project.version=' "${REPO_ROOT}/sdks/sandbox/kotlin/gradle.properties" | cut -d= -f2 | tr -d '[:space:]')" -echo "== running benchmark driver (sandbox SDK ${SDK_VERSION}) ==" +# 3. run the driver (Kotlin SDK is built from source via composite build, +# see kotlin/settings.gradle.kts) +echo "== running benchmark driver ==" DRIVER_ARGS+=("--report-dir" "${BENCH_DIR}/results/run-$(date +%Y%m%d-%H%M%S)") -(cd "${BENCH_DIR}/kotlin" && ./gradlew --console=plain run -PsandboxVersion="${SDK_VERSION}" --args="${DRIVER_ARGS[*]}") +(cd "${BENCH_DIR}/kotlin" && ./gradlew --console=plain run --args="${DRIVER_ARGS[*]}") echo "== done ==" From af2294661e8defa6754a25e757fdaf4c83ab77fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 12:05:33 +0800 Subject: [PATCH 03/16] feat(benchmark): support production pool profile knobs (acquireMinRemainingTtl, primaryLockTtl, degradedThreshold) Map a production large-pool / high-frequency profile 1:1 onto driver options; document the example profile and scale caveats in README. --- tests/benchmark/README.md | 37 +++++++++++++++++++ .../com/alibaba/opensandbox/benchmark/Cli.kt | 12 ++++++ .../com/alibaba/opensandbox/benchmark/Main.kt | 3 ++ .../opensandbox/benchmark/PoolRunner.kt | 11 ++++++ 4 files changed, 63 insertions(+) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index cb2baf9a9..b76489c8f 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -194,6 +194,9 @@ cd kotlin | `--warmup-concurrency` | `4` | Concurrent warmup creation workers | | `--reconcile-interval-ms` | `1000` | Pool reconcile tick interval | | `--idle-timeout-s` | `1800` | Server-side TTL applied to pool-created sandboxes | +| `--acquire-min-remaining-ttl-s` | `0` | Idle entries with less remaining TTL than this are discarded on acquire; `0` = SDK auto default (`min(60s, idleTimeout/2)`) | +| `--primary-lock-ttl-s` | `0` | Distributed primary-lock TTL; `0` = SDK default (60s). No effect with the in-memory state store (single node always holds the lock) | +| `--degraded-threshold` | `0` | Consecutive create failures before the pool enters DEGRADED; `0` = SDK default (3) | | `--acquire-ready-timeout-ms` | `15000` | `checkReady` timeout when acquiring (idle connect + direct create) | | `--warmup-ready-timeout-ms` | `15000` | `checkReady` timeout for warmup creations | | `--health-check-polling-interval-ms` | `200` | `checkReady` probe interval (execd ping cadence) | @@ -217,6 +220,40 @@ cd kotlin Each scenario resets the mock's counters and QPS history first, so `report.json`'s per-scenario QPS sections cover exactly that scenario. +### Reproducing a production pool profile + +Any `PoolConfig`-level profile maps 1:1 onto driver knobs. Example — a +large-pool / high-frequency-acquire / high-frequency-replenish production +profile (`maxIdle=13815, warmupConcurrency=1000, idleTtl=4h, +acquireMinRemainingTtl=15min, reconcile=30s, acquireReady=60s, +warmupReady=180s, primaryLockTtl=360s, degradedThreshold=5`): + +```bash +./run.sh -- --max-idle 13815 --warmup-concurrency 1000 \ + --reconcile-interval-ms 30000 --idle-timeout-s 14400 \ + --acquire-min-remaining-ttl-s 900 --primary-lock-ttl-s 360 \ + --degraded-threshold 5 --acquire-ready-timeout-ms 60000 \ + --warmup-ready-timeout-ms 180000 --cold-start-timeout-ms 300000 \ + --scenarios cold-start,steady-state --steady-workers 300 \ + --steady-duration-s 120 --hold-min-ms 200 --hold-max-ms 2000 +``` + +Notes for this kind of run: + +- `--cold-start-timeout-ms` must cover a full fill: with + `warmupConcurrency=1000` and the default profile (~2-6s per sandbox) a + 13815-sandbox pool fills in roughly 30-90s. +- At this scale each scenario still starts a fresh pool, but the mock's + sandbox registry accumulates across scenarios (previous pools are not + killed on non-graceful shutdown) — budget mock memory accordingly or run + one scenario per invocation. +- For higher acquire frequency, shrink the `execd.ping` latency via + `latencyOverrides` in the mock config: each acquire pays one readiness ping + (1-5s by default), which caps the sustainable acquire rate. +- `primaryLockTtl` and `drainTimeout` do not change single-node benchmark + behavior (in-memory state store always grants the lock; the driver never + shuts down gracefully); they are reproduced for config fidelity only. + For `warm-latency`, keep `maxIdle` comfortably above `--warm-workers` if you want to measure pure idle-hit latency: when workers outnumber the idle buffer, acquires drain it and fall through to direct create (which the `hitRatio` diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 229fb9f76..1ab7c52d6 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -29,6 +29,9 @@ data class BenchmarkConfig( val warmupConcurrency: Int, val reconcileIntervalMs: Long, val idleTimeoutS: Long, + val acquireMinRemainingTtlS: Long, + val primaryLockTtlS: Long, + val degradedThreshold: Int, val acquireReadyTimeoutMs: Long, val warmupReadyTimeoutMs: Long, val healthCheckPollingIntervalMs: Long, @@ -63,6 +66,9 @@ object Cli { "warmup-concurrency", "reconcile-interval-ms", "idle-timeout-s", + "acquire-min-remaining-ttl-s", + "primary-lock-ttl-s", + "degraded-threshold", "acquire-ready-timeout-ms", "warmup-ready-timeout-ms", "health-check-polling-interval-ms", @@ -109,6 +115,12 @@ object Cli { warmupConcurrency = (map["warmup-concurrency"] ?: "4").toInt(), reconcileIntervalMs = (map["reconcile-interval-ms"] ?: "1000").toLong(), idleTimeoutS = (map["idle-timeout-s"] ?: "1800").toLong(), + // 0 = leave the SDK's auto-derived default (min(60s, idleTimeout/2)) + acquireMinRemainingTtlS = (map["acquire-min-remaining-ttl-s"] ?: "0").toLong(), + // 0 = leave the SDK default (60s) + primaryLockTtlS = (map["primary-lock-ttl-s"] ?: "0").toLong(), + // 0 = leave the SDK default (3) + degradedThreshold = (map["degraded-threshold"] ?: "0").toInt(), acquireReadyTimeoutMs = (map["acquire-ready-timeout-ms"] ?: "15000").toLong(), warmupReadyTimeoutMs = (map["warmup-ready-timeout-ms"] ?: "15000").toLong(), healthCheckPollingIntervalMs = (map["health-check-polling-interval-ms"] ?: "200").toLong(), diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index 62704646d..9c2894090 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -114,6 +114,9 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "warmupConcurrency" to cfg.warmupConcurrency, "reconcileIntervalMs" to cfg.reconcileIntervalMs, "idleTimeoutS" to cfg.idleTimeoutS, + "acquireMinRemainingTtlS" to cfg.acquireMinRemainingTtlS, + "primaryLockTtlS" to cfg.primaryLockTtlS, + "degradedThreshold" to cfg.degradedThreshold, "acquireReadyTimeoutMs" to cfg.acquireReadyTimeoutMs, "warmupReadyTimeoutMs" to cfg.warmupReadyTimeoutMs, "healthCheckPollingIntervalMs" to cfg.healthCheckPollingIntervalMs, diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt index 0f6e866d7..047e27dbb 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -65,6 +65,17 @@ object PoolRunner { .warmupHealthCheckPollingInterval(Duration.ofMillis(cfg.healthCheckPollingIntervalMs)) .idleTimeout(Duration.ofSeconds(idleTimeoutS)) .maxAcquireRetries(maxAcquireRetries) + .also { builder -> + if (cfg.acquireMinRemainingTtlS > 0) { + builder.acquireMinRemainingTtl(Duration.ofSeconds(cfg.acquireMinRemainingTtlS)) + } + if (cfg.primaryLockTtlS > 0) { + builder.primaryLockTtl(Duration.ofSeconds(cfg.primaryLockTtlS)) + } + if (cfg.degradedThreshold > 0) { + builder.degradedThreshold(cfg.degradedThreshold) + } + } .build() } From 51466eccd4b45f4d25f9e3f19c4fb16bdca463fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 12:13:23 +0800 Subject: [PATCH 04/16] chore(benchmark): execd ping latency to fixed 100ms in the default profile Keeps the high-frequency acquire case realistic without the 1-5s readiness probe capping the sustainable acquire rate. --- tests/benchmark/README.md | 11 ++-- tests/benchmark/configs/default.json | 5 +- .../com/alibaba/opensandbox/benchmark/Cli.kt | 5 ++ .../com/alibaba/opensandbox/benchmark/Main.kt | 1 + .../opensandbox/benchmark/RatePacer.kt | 64 +++++++++++++++++++ .../opensandbox/benchmark/Scenarios.kt | 5 ++ tests/benchmark/mockserver/config.go | 6 +- 7 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/RatePacer.kt diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index b76489c8f..a295348e4 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -143,13 +143,12 @@ and accumulate the series yourself. | `latencyOverrides` | Response time per route: `lifecycle.get`, `lifecycle.delete`, `lifecycle.renew`, `lifecycle.endpoint`, `execd.ping`, `execd.other`. Routes without an override respond immediately; an override for `lifecycle.create` replaces `createLatencyMs`. Execd route latency only applies once the sandbox is booted — not-ready probes fail fast | Default profile (no `-config`): create/delete uniform **300-800ms**, execd -`/ping` uniform **1-5s** (readiness probes are slow), all other APIs uniform -**50-100ms**. +`/ping` fixed **100ms**, all other APIs uniform **50-100ms**. The readiness sequence a client observes is therefore: create latency, then a -few fast `404` polls while the sandbox boots, then a slow successful ping — -typically one ping for the default profile (min 1s ping vs. max 300ms boot -window). +few fast `404` polls while the sandbox boots, then one successful ping — +typically one ping for the default profile (fixed 100ms ping vs. max 300ms +boot window). So the full create-to-ready time a client observes is `createLatencyMs + bootDelayMs` plus one successful ping (once booted, the @@ -249,7 +248,7 @@ Notes for this kind of run: one scenario per invocation. - For higher acquire frequency, shrink the `execd.ping` latency via `latencyOverrides` in the mock config: each acquire pays one readiness ping - (1-5s by default), which caps the sustainable acquire rate. + (100ms by default), which caps the sustainable acquire rate. - `primaryLockTtl` and `drainTimeout` do not change single-node benchmark behavior (in-memory state store always grants the lock; the driver never shuts down gracefully); they are reproduced for config fidelity only. diff --git a/tests/benchmark/configs/default.json b/tests/benchmark/configs/default.json index 21436346c..9e5baa450 100644 --- a/tests/benchmark/configs/default.json +++ b/tests/benchmark/configs/default.json @@ -30,9 +30,8 @@ "maxMs": 100 }, "execd.ping": { - "distribution": "uniform", - "minMs": 1000, - "maxMs": 5000 + "distribution": "fixed", + "meanMs": 100 }, "execd.other": { "distribution": "uniform", diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 1ab7c52d6..5653a9bdb 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -40,6 +40,7 @@ data class BenchmarkConfig( val warmRoundsPerWorker: Int, val steadyWorkers: Int, val steadyDurationS: Int, + val acquireRatePerMin: Int, val holdMinMs: Long, val holdMaxMs: Long, val replenishRounds: Int, @@ -77,6 +78,7 @@ object Cli { "warm-rounds-per-worker", "steady-workers", "steady-duration-s", + "acquire-rate-per-min", "hold-min-ms", "hold-max-ms", "replenish-rounds", @@ -129,6 +131,9 @@ object Cli { warmRoundsPerWorker = (map["warm-rounds-per-worker"] ?: "150").toInt(), steadyWorkers = (map["steady-workers"] ?: "16").toInt(), steadyDurationS = (map["steady-duration-s"] ?: "60").toInt(), + // 0 = unlimited (workers run back-to-back); > 0 paces acquires + // evenly across each minute at this many acquires per minute. + acquireRatePerMin = (map["acquire-rate-per-min"] ?: "0").toInt(), holdMinMs = (map["hold-min-ms"] ?: "1000").toLong(), holdMaxMs = (map["hold-max-ms"] ?: "5000").toLong(), replenishRounds = (map["replenish-rounds"] ?: "20").toInt(), diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index 9c2894090..44e4cbfcb 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -124,6 +124,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "warmRoundsPerWorker" to cfg.warmRoundsPerWorker, "steadyWorkers" to cfg.steadyWorkers, "steadyDurationS" to cfg.steadyDurationS, + "acquireRatePerMin" to cfg.acquireRatePerMin, "holdMinMs" to cfg.holdMinMs, "holdMaxMs" to cfg.holdMaxMs, "failureCreateRate" to cfg.failureCreateRate, diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/RatePacer.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/RatePacer.kt new file mode 100644 index 000000000..3e4d2ff75 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/RatePacer.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong + +/** + * Spreads acquires evenly over wall-clock time at a fixed per-minute rate. + * + * Every caller claims the next absolute slot (`start + k * interval`); slots + * are advanced with CAS so concurrent workers never share one. Slots that + * fall in the past are executed immediately, so a slow worker lets the rate + * catch up on the next slots — the long-run average stays at [ratePerMin] + * without drift. No-op when [ratePerMin] is <= 0. + */ +class RatePacer(private val ratePerMin: Int) { + private val intervalNanos = + if (ratePerMin <= 0) 0L else TimeUnit.MINUTES.toNanos(1) / ratePerMin + private val nextSlot = AtomicLong(0) + + /** Blocks until this caller's slot is due. */ + fun waitForSlot() { + if (intervalNanos <= 0) return + var slot = nextSlot.get() + while (true) { + if (slot == 0L) { + if (nextSlot.compareAndSet(0L, System.nanoTime())) { + slot = System.nanoTime() + break + } + slot = nextSlot.get() + continue + } + val next = slot + intervalNanos + if (nextSlot.compareAndSet(slot, next)) { + break + } + slot = nextSlot.get() + } + val waitMs = (slot - System.nanoTime()) / 1_000_000 + if (waitMs > 0) { + try { + Thread.sleep(waitMs) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } + } +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index a5cc1cad7..46c42a632 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -129,10 +129,12 @@ object Scenarios { sampler.start() val rng = Random(System.nanoTime()) + val pacer = RatePacer(cfg.acquireRatePerMin) val threads = Executors.newFixedThreadPool(cfg.steadyWorkers) repeat(cfg.steadyWorkers) { threads.submit { while (System.nanoTime() < deadline) { + pacer.waitForSlot() val t0 = System.nanoTime() try { val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) @@ -165,6 +167,9 @@ object Scenarios { "fillTimeMs" to fillMs, "durationMs" to durationMs, "workers" to cfg.steadyWorkers, + "targetAcquiresPerMin" to cfg.acquireRatePerMin, + "acquiredCount" to acquires.get(), + "achievedAcquiresPerMin" to (acquires.get().toDouble() * 60_000 / durationMs), "throughputAcquiresPerSec" to (acquires.get().toDouble() / cfg.steadyDurationS), "latency" to latency.snapshot().toMap(), "serverCreatedDelta" to createdDelta, diff --git a/tests/benchmark/mockserver/config.go b/tests/benchmark/mockserver/config.go index abf97ace9..e4caff19c 100644 --- a/tests/benchmark/mockserver/config.go +++ b/tests/benchmark/mockserver/config.go @@ -83,8 +83,8 @@ type FaultConfig struct { func loadConfig(path string) (*Config, error) { cfg := &Config{ // Default response-time profile: create and delete take a uniform - // 300-800ms, execd ping a uniform 1-5s (readiness probes are slow), - // every other API a uniform 50-100ms. + // 300-800ms, execd ping a fixed 100ms, every other API a uniform + // 50-100ms. CreateLatencyMs: LatencySpec{ Distribution: "uniform", MinMs: 300, @@ -95,7 +95,7 @@ func loadConfig(path string) (*Config, error) { "lifecycle.get": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, "lifecycle.renew": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, "lifecycle.endpoint": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, - "execd.ping": {Distribution: "uniform", MinMs: 1000, MaxMs: 5000}, + "execd.ping": {Distribution: "fixed", MeanMs: 100}, "execd.other": {Distribution: "uniform", MinMs: 50, MaxMs: 100}, }, DefaultTtlSeconds: 3600, From d5e6bc60980581bc1fa7998ab51b16ab367dd19d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 12:17:32 +0800 Subject: [PATCH 05/16] feat(benchmark): add client-side instrumentation and explicit success/replenish metrics PoolProbe samples JVM threads, heap/GC, and pool health every 500ms during warm-latency and steady-state. Scenarios now report successRate, replenishRatePerSec/killRatePerSec, directCreateRatio, and the client probe block; steady-state idle stats come from the probe. --- tests/benchmark/README.md | 17 +++ .../opensandbox/benchmark/PoolProbe.kt | 136 ++++++++++++++++++ .../opensandbox/benchmark/Scenarios.kt | 50 ++++--- 3 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index a295348e4..37ab9f0fd 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -258,6 +258,23 @@ want to measure pure idle-hit latency: when workers outnumber the idle buffer, acquires drain it and fall through to direct create (which the `hitRatio` column will show). +### What is measured + +| Concern | Where it shows up | +|---|---| +| Acquire latency (p50/90/95/99/999) | `results..latency` | +| Acquire success rate | `successRate` (successful acquires / attempts, failures counted in `latency.failures`) | +| Idle-hit vs direct-create | `hitRatio` (warm-latency), `directCreateRatio` (steady-state) | +| Pool health | `client.poolIdleCount` (min/mean/max samples), `poolIdleZeroRatio`, `poolDegradedSamples`, `poolBackoffSamples`, `poolInFlightMax`; failure scenarios also report `poolStateAfterBurst`/`backoffActive`/`failureCount` | +| Replenish throughput | `replenishRatePerSec` / `killRatePerSec` (server-observed), plus the per-second `lifecycle.create`/`lifecycle.delete` QPS series under `mockQps` | +| Client threads | `client.threads` (min/mean/max sampled every 500ms) + `client.threadPeakSinceProbeStart` | +| Client memory/GC | `client.heapUsedMb` (min/mean/max), `client.gcCollections`, `client.gcTimeMs` | +| Server QPS (all APIs) | `perScenarioQps..` — per-route totals, 1s/5s/60s rates, and the full per-second series | + +The `client` block is produced by a probe thread sampling +`SandboxPool.snapshot()` plus JVM thread/heap/GC beans every 500ms during the +scenario. + ## Reusing the mock from other SDKs Point any SDK's `ConnectionConfig` at the mock: diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt new file mode 100644 index 000000000..f23636647 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import com.alibaba.opensandbox.sandbox.domain.pool.PoolState +import com.alibaba.opensandbox.sandbox.pool.SandboxPool +import java.lang.management.GarbageCollectorMXBean +import java.lang.management.ManagementFactory +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger + +/** + * Continuous client-side instrumentation: JVM threads, heap/GC, and pool + * health (snapshot-based) sampled every [intervalMs] during a scenario. + */ +class PoolProbe( + private val pool: SandboxPool, + private val intervalMs: Long = 500, +) { + private val running = AtomicBoolean(true) + private val threadCounts = ArrayList() + private val heapUsedMb = ArrayList() + private val idleSamples = ArrayList() + private val degradedSamples = AtomicInteger() + private val backoffSamples = AtomicInteger() + private val inFlightMax = AtomicInteger() + private val lock = Any() + private var thread: Thread? = null + + private val threadBean = ManagementFactory.getThreadMXBean() + private val memoryBean = ManagementFactory.getMemoryMXBean() + private val gcBeans: List = ManagementFactory.getGarbageCollectorMXBeans() + private val gcStartCount = gcBeans.sumOf { it.collectionCount } + private val gcStartTimeMs = gcBeans.sumOf { it.collectionTime } + + init { + // Peak thread count is reported relative to this probe's window. + threadBean.resetPeakThreadCount() + } + + fun start() { + val t = + Thread { + while (running.get()) { + sample() + try { + Thread.sleep(intervalMs) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + break + } + } + } + t.isDaemon = true + t.name = "bench-probe" + thread = t + t.start() + } + + fun stop() { + running.set(false) + thread?.join(5000) + } + + private fun sample() { + val heap = memoryBean.heapMemoryUsage + val snap = pool.snapshot() + synchronized(lock) { + threadCounts.add(threadBean.threadCount) + heapUsedMb.add(heap.used / (1024.0 * 1024.0)) + idleSamples.add(snap.idleCount) + } + if (snap.state == PoolState.DEGRADED) degradedSamples.incrementAndGet() + if (snap.backoffActive) backoffSamples.incrementAndGet() + inFlightMax.accumulateAndGet(snap.inFlightOperations) { a, b -> maxOf(a, b) } + } + + fun report(): Map { + val threads: LongArray + val heap: DoubleArray + val idle: IntArray + synchronized(lock) { + threads = LongArray(threadCounts.size) { threadCounts[it].toLong() } + heap = DoubleArray(heapUsedMb.size) { heapUsedMb[it] } + idle = IntArray(idleSamples.size) { idleSamples[it] } + } + val zeroIdleRatio = + if (idle.isEmpty()) 0.0 else idle.count { it == 0 }.toDouble() / idle.size + return mapOf( + "threads" to stat(threads), + "threadPeakSinceProbeStart" to threadBean.peakThreadCount, + "heapUsedMb" to stat(heap), + "gcCollections" to (gcBeans.sumOf { it.collectionCount } - gcStartCount), + "gcTimeMs" to (gcBeans.sumOf { it.collectionTime } - gcStartTimeMs), + "poolIdleCount" to stat(idle.map { it.toLong() }.toLongArray()), + "poolIdleZeroRatio" to zeroIdleRatio, + "poolDegradedSamples" to degradedSamples.get(), + "poolBackoffSamples" to backoffSamples.get(), + "poolInFlightMax" to inFlightMax.get(), + ) + } + + private fun stat(samples: LongArray): Map { + if (samples.isEmpty()) return mapOf("samples" to 0, "min" to 0L, "mean" to 0.0, "max" to 0L) + return mapOf( + "samples" to samples.size, + "min" to samples.min(), + "mean" to samples.average(), + "max" to samples.max(), + ) + } + + private fun stat(samples: DoubleArray): Map { + if (samples.isEmpty()) return mapOf("samples" to 0, "min" to 0.0, "mean" to 0.0, "max" to 0.0) + return mapOf( + "samples" to samples.size, + "min" to samples.min(), + "mean" to samples.average(), + "max" to samples.max(), + ) + } +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index 46c42a632..e7fceb142 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -69,6 +69,8 @@ object Scenarios { val start = CountDownLatch(1) val workers = cfg.warmWorkers val rounds = cfg.warmRoundsPerWorker + val probe = PoolProbe(pool) + probe.start() val threads = Executors.newFixedThreadPool(workers) repeat(workers) { threads.submit { @@ -88,17 +90,21 @@ object Scenarios { start.countDown() threads.shutdown() threads.awaitTermination(10, TimeUnit.MINUTES) + probe.stop() val createdDelta = num(mock.stats(), "stats.created") - createdBefore val acquires = rounds * workers.toLong() + val latencyStats = latency.snapshot() pool.shutdown(graceful = false) return mapOf( "fillTimeMs" to fillMs, - "latency" to latency.snapshot().toMap(), + "latency" to latencyStats.toMap(), "acquires" to acquires, + "successRate" to successRate(latencyStats), "serverCreatedDelta" to createdDelta, "hitRatio" to (1.0 - createdDelta.toDouble() / acquires).coerceIn(0.0, 1.0), + "client" to probe.report(), ) } @@ -118,15 +124,8 @@ object Scenarios { val latency = LatencyCollector() val acquires = AtomicLong(0) - val idleSamples = CopyOnWriteArrayList() - val sampler = Thread { - while (running.get()) { - idleSamples.add(pool.snapshot().idleCount) - Thread.sleep(500) - } - } - sampler.isDaemon = true - sampler.start() + val probe = PoolProbe(pool) + probe.start() val rng = Random(System.nanoTime()) val pacer = RatePacer(cfg.acquireRatePerMin) @@ -152,17 +151,15 @@ object Scenarios { threads.shutdown() threads.awaitTermination(15, TimeUnit.MINUTES) running.set(false) + probe.stop() val createdDelta = num(mock.stats(), "stats.created") - createdBefore val killedDelta = num(mock.stats(), "stats.killed") - killedBefore + val latencyStats = latency.snapshot() + val client = probe.report() pool.shutdown(graceful = false) - val idleMin = idleSamples.minOrNull() ?: 0 - val idleMean = if (idleSamples.isEmpty()) 0.0 else idleSamples.average() - val idleZeroRatio = - if (idleSamples.isEmpty()) 0.0 - else idleSamples.count { it == 0 }.toDouble() / idleSamples.size - + val idleStat = client["poolIdleCount"] as Map return mapOf( "fillTimeMs" to fillMs, "durationMs" to durationMs, @@ -171,13 +168,19 @@ object Scenarios { "acquiredCount" to acquires.get(), "achievedAcquiresPerMin" to (acquires.get().toDouble() * 60_000 / durationMs), "throughputAcquiresPerSec" to (acquires.get().toDouble() / cfg.steadyDurationS), - "latency" to latency.snapshot().toMap(), + "successRate" to successRate(latencyStats), + "latency" to latencyStats.toMap(), "serverCreatedDelta" to createdDelta, "serverKilledDelta" to killedDelta, - "idleSamples" to idleSamples.size, - "idleMin" to idleMin, - "idleMean" to idleMean, - "idleEmptyRatio" to idleZeroRatio, + "replenishRatePerSec" to (createdDelta.toDouble() / cfg.steadyDurationS), + "killRatePerSec" to (killedDelta.toDouble() / cfg.steadyDurationS), + "directCreateRatio" to + ((createdDelta - killedDelta).coerceAtLeast(0).toDouble() / acquires.get().coerceAtLeast(1)), + "idleSamples" to (idleStat["samples"] as Int), + "idleMin" to (idleStat["min"] as Long), + "idleMean" to (idleStat["mean"] as Double), + "idleEmptyRatio" to (client["poolIdleZeroRatio"] as Double), + "client" to client, ) } @@ -361,6 +364,11 @@ object Scenarios { // ---------- helpers ---------- + private fun successRate(stats: LatencyStats): Double { + val total = stats.n + stats.failures + return if (total == 0L) 0.0 else (stats.n.toDouble() / total) + } + private fun killAndClose(sandbox: Sandbox) { try { sandbox.kill() From 3e19bf9ca3a1dd4c1152bc3acc03eac52789d872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 12:28:41 +0800 Subject: [PATCH 06/16] feat(benchmark): consolidate all metrics into one run directory run.sh creates results/run-/ holding the mock log, mock config, driver args, reports, per-scenario server timeseries CSVs (alive + per-API QPS) and client probe CSVs (threads/heap/idle/inFlight), plus a raw end-of-run mock stats snapshot. Adds failure classification (readyTimeout/poolNotRunning/ storeUnavailable/...), resize, shutdown-race, store-outage scenarios, partial poisoning, and mock alive gauge (max + per-second series). --- tests/benchmark/README.md | 31 ++- .../com/alibaba/opensandbox/benchmark/Cli.kt | 4 + .../benchmark/FailingPoolStateStore.kt | 132 +++++++++++ .../com/alibaba/opensandbox/benchmark/Main.kt | 87 ++++++- .../alibaba/opensandbox/benchmark/Metrics.kt | 17 +- .../opensandbox/benchmark/MockControl.kt | 8 +- .../opensandbox/benchmark/PoolProbe.kt | 82 ++++++- .../opensandbox/benchmark/PoolRunner.kt | 29 ++- .../opensandbox/benchmark/Scenarios.kt | 214 +++++++++++++++++- tests/benchmark/mockserver/config.go | 3 + tests/benchmark/mockserver/main.go | 2 + tests/benchmark/mockserver/server.go | 35 +++ tests/benchmark/mockserver/stats.go | 79 +++++++ tests/benchmark/run.sh | 21 +- 14 files changed, 700 insertions(+), 44 deletions(-) create mode 100644 tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/FailingPoolStateStore.kt diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index 37ab9f0fd..c3c5a0d90 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -49,8 +49,25 @@ publish/install step. `com.alibaba.opensandbox:sandbox:1.0.18` in `kotlin/build.gradle.kts` is a module coordinate that the composite build substitutes with the local `:sandbox` project (the version is informational). -Reports land in `results/run-/report.{json,md}`; the mock log is at -`results/mockserver.log`. Exit code is non-zero when a scenario fails. +Reports and every artifact of a run land in one directory, +`results/run-/`: + +``` +results/run-/ +├── report.json # full results: per-scenario metrics + per-scenario QPS + end stats +├── report.md # human-readable view of report.json +├── server-timeseries-.csv # per-second: alive + each API request count +├── client-.csv # per-500ms: threads, heap MB, idle, inFlight, degraded, backoff +├── mock-stats-end.json # raw /__stats snapshot at end of run +├── mock-config.json # mock server config used for this run +├── driver-args.txt # driver CLI arguments +└── mockserver.log # mock server log +``` + +The `server-timeseries-*.csv` files merge the mock's per-second per-API QPS +series with the alive-sandbox gauge into one table +(`second,alive,create,delete,get,renew,endpoint,execd.ping,execd.other`) for +direct plotting; the full per-second series is also embedded in `report.json`. ## Mock server @@ -179,8 +196,11 @@ cd kotlin | `steady-state` | Sustained acquires/sec under concurrent loaders with hold time; idle trajectory (min/mean/empty ratio) | | `replenish-lag` | Time for a released idle slot to be refilled (completion-driven reconcile) | | `failure-injection` | Pool behavior at `createFailureRate` 60%: success rate, backoff, DEGRADED transition, recovery after fault removal | -| `stale-idle` | Poisoned idle candidates: retry cost, stale cleanup, refill with fresh sandboxes | +| `stale-idle` | Poisoned idle candidates (`--stale-poison-rate`, default 1.0 = all): retry cost, stale cleanup, refill with fresh sandboxes | | `idle-expiry` | Self-healing under short server-side TTL: reap + recreate keeps the buffer near `maxIdle` | +| `resize` | Shrink accuracy and speed (excess idles killed by reconcile), regrow speed, server-side alive check | +| `shutdown-race` | Workers hammering acquire while the pool drains: success rate in the running phase vs rejections during DRAINING (`poolNotRunning`), drain duration | +| `store-outage` | State-store outage (OSEP-0005): DIRECT_CREATE must fall through to direct create and stay available; FAIL_FAST must fail closed with `storeUnavailable`; refill after recovery | ### Driver options @@ -212,6 +232,7 @@ cd kotlin | `--failure-acquires` | `60` | Acquire attempts in `failure-injection` | | `--stale-acquires` | `100` | Acquire attempts in `stale-idle` | | `--stale-retries` | `3` | Pool `maxAcquireRetries` in `stale-idle` (idle candidates tried per acquire) | +| `--stale-poison-rate` | `1.0` | Fraction (0..1] of alive sandboxes poisoned in `stale-idle`; `1.0` = all (partial rates simulate real-world partial failure) | | `--stale-acquire-ready-timeout-ms` | `3000` | `acquireReadyTimeout` in `stale-idle`; short because the SDK polls a failing execd for the full timeout before discarding a candidate | | `--idle-expiry-idle-timeout-s` | `20` | `idleTimeout` in `idle-expiry` (short TTL so server-side expiry is exercised) | | `--idle-expiry-duration-s` | `40` | `idle-expiry` run duration | @@ -263,10 +284,12 @@ column will show). | Concern | Where it shows up | |---|---| | Acquire latency (p50/90/95/99/999) | `results..latency` | -| Acquire success rate | `successRate` (successful acquires / attempts, failures counted in `latency.failures`) | +| Acquire success rate | `successRate` (successful acquires / attempts) | +| Failure breakdown | `latency.failuresByType`: `readyTimeout` / `createFailed`-style `other` / `poolNotRunning` / `poolEmpty` / `acquireFailed` / `storeUnavailable` | | Idle-hit vs direct-create | `hitRatio` (warm-latency), `directCreateRatio` (steady-state) | | Pool health | `client.poolIdleCount` (min/mean/max samples), `poolIdleZeroRatio`, `poolDegradedSamples`, `poolBackoffSamples`, `poolInFlightMax`; failure scenarios also report `poolStateAfterBurst`/`backoffActive`/`failureCount` | | Replenish throughput | `replenishRatePerSec` / `killRatePerSec` (server-observed), plus the per-second `lifecycle.create`/`lifecycle.delete` QPS series under `mockQps` | +| Pool-size trajectory / over-creation | mock `aliveStats.max` + per-second `alive` series (server view); pool idle should never exceed `maxIdle` + in-flight warmups | | Client threads | `client.threads` (min/mean/max sampled every 500ms) + `client.threadPeakSinceProbeStart` | | Client memory/GC | `client.heapUsedMb` (min/mean/max), `client.gcCollections`, `client.gcTimeMs` | | Server QPS (all APIs) | `perScenarioQps..` — per-route totals, 1s/5s/60s rates, and the full per-second series | diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 5653a9bdb..fd335989d 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -50,6 +50,7 @@ data class BenchmarkConfig( val staleAcquires: Int, val staleRetries: Int, val staleAcquireReadyTimeoutMs: Long, + val stalePoisonRate: Double, val idleExpiryIdleTimeoutS: Long, val idleExpiryDurationS: Int, ) { @@ -88,6 +89,7 @@ object Cli { "stale-acquires", "stale-retries", "stale-acquire-ready-timeout-ms", + "stale-poison-rate", "idle-expiry-idle-timeout-s", "idle-expiry-duration-s", ) @@ -143,6 +145,8 @@ object Cli { staleAcquires = (map["stale-acquires"] ?: "100").toInt(), staleRetries = (map["stale-retries"] ?: "3").toInt(), staleAcquireReadyTimeoutMs = (map["stale-acquire-ready-timeout-ms"] ?: "3000").toLong(), + // fraction (0..1] of idle sandboxes to poison in stale-idle; 1.0 = poison all + stalePoisonRate = (map["stale-poison-rate"] ?: "1.0").toDouble(), idleExpiryIdleTimeoutS = (map["idle-expiry-idle-timeout-s"] ?: "20").toLong(), idleExpiryDurationS = (map["idle-expiry-duration-s"] ?: "40").toInt(), ) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/FailingPoolStateStore.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/FailingPoolStateStore.kt new file mode 100644 index 000000000..a77969909 --- /dev/null +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/FailingPoolStateStore.kt @@ -0,0 +1,132 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.benchmark + +import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolStateStoreUnavailableException +import com.alibaba.opensandbox.sandbox.domain.pool.IdleEntry +import com.alibaba.opensandbox.sandbox.domain.pool.PoolDestroyState +import com.alibaba.opensandbox.sandbox.domain.pool.PoolStateStore +import com.alibaba.opensandbox.sandbox.domain.pool.StoreCounters +import com.alibaba.opensandbox.sandbox.domain.pool.TakeIdleResult +import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore +import java.time.Duration +import java.time.Instant +import java.util.concurrent.atomic.AtomicLong + +/** + * A [PoolStateStore] wrapper that can be toggled into "outage" mode, where + * every operation throws [PoolStateStoreUnavailableException]. Used to test + * the pool's store-outage behavior (OSEP-0005): DIRECT_CREATE fallthrough + * keeps acquire available; FAIL_FAST fails closed. + */ +class FailingPoolStateStore( + private val delegate: PoolStateStore = InMemoryPoolStateStore(), +) : PoolStateStore { + @Volatile + private var failing = false + private val errorCount = AtomicLong() + + fun setFailing(f: Boolean) { + failing = f + } + + fun errorCount(): Long = errorCount.get() + + private fun gate(block: () -> T): T { + if (failing) { + errorCount.incrementAndGet() + throw PoolStateStoreUnavailableException("simulated store outage") + } + return block() + } + + override fun tryTakeIdle(poolName: String): String? = gate { delegate.tryTakeIdle(poolName) } + + override fun tryTakeIdle( + poolName: String, + minRemainingTtl: Duration, + ): TakeIdleResult = gate { delegate.tryTakeIdle(poolName, minRemainingTtl) } + + override fun putIdle( + poolName: String, + sandboxId: String, + ) = gate { delegate.putIdle(poolName, sandboxId) } + + override fun removeIdle( + poolName: String, + sandboxId: String, + ) = gate { delegate.removeIdle(poolName, sandboxId) } + + override fun tryAcquirePrimaryLock( + poolName: String, + ownerId: String, + ttl: Duration, + ): Boolean = gate { delegate.tryAcquirePrimaryLock(poolName, ownerId, ttl) } + + override fun renewPrimaryLock( + poolName: String, + ownerId: String, + ttl: Duration, + ): Boolean = gate { delegate.renewPrimaryLock(poolName, ownerId, ttl) } + + override fun releasePrimaryLock( + poolName: String, + ownerId: String, + ) = gate { delegate.releasePrimaryLock(poolName, ownerId) } + + override fun reapExpiredIdle( + poolName: String, + now: Instant, + ) = gate { delegate.reapExpiredIdle(poolName, now) } + + override fun reapExpiredIdle( + poolName: String, + now: Instant, + minRemainingTtl: Duration, + ): List = gate { delegate.reapExpiredIdle(poolName, now, minRemainingTtl) } + + override fun snapshotCounters(poolName: String): StoreCounters = gate { delegate.snapshotCounters(poolName) } + + override fun snapshotIdleEntries(poolName: String): List = gate { delegate.snapshotIdleEntries(poolName) } + + override fun getMaxIdle(poolName: String): Int? = gate { delegate.getMaxIdle(poolName) } + + override fun setMaxIdle( + poolName: String, + maxIdle: Int, + ) = gate { delegate.setMaxIdle(poolName, maxIdle) } + + override fun setIdleEntryTtl( + poolName: String, + idleTtl: Duration, + ) = gate { delegate.setIdleEntryTtl(poolName, idleTtl) } + + override fun getDestroyState(poolName: String): PoolDestroyState = gate { delegate.getDestroyState(poolName) } + + override fun beginDestroy( + poolName: String, + ownerId: String, + ) = gate { delegate.beginDestroy(poolName, ownerId) } + + override fun clearPoolState(poolName: String) = gate { delegate.clearPoolState(poolName) } + + override fun markDestroyed( + poolName: String, + ownerId: String, + tombstoneTtl: Duration?, + ) = gate { delegate.markDestroyed(poolName, ownerId, tombstoneTtl) } +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index 44e4cbfcb..98b25fb6b 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -84,12 +84,15 @@ fun main(args: Array) { results[name] = section // Precise per-API QPS observed by the mock during this scenario. The // mock ring keeps per-second counts; the series is included in the JSON - // report for offline analysis. - perScenarioQps[name] = mock.stats()["qps"] + // report and exported as a per-scenario CSV for offline analysis. + val stats = mock.stats() + perScenarioQps[name] = stats["qps"] + writeTimeseriesCsv(File(cfg.reportDir, "server-timeseries-$name.csv"), name, stats) } results["perScenarioQps"] = perScenarioQps - results["mockServerStats"] = mock.stats() + val endStats = mock.stats() + results["mockServerStats"] = endStats val report = buildJsonObject { @@ -101,6 +104,8 @@ fun main(args: Array) { val outDir = File(cfg.reportDir) outDir.mkdirs() + File(outDir, "mock-stats-end.json") + .writeText(Json { prettyPrint = true }.encodeToString(JsonObject.serializer(), endStats as JsonObject)) File(outDir, "report.json").writeText(Json { prettyPrint = true }.encodeToString(JsonObject.serializer(), report)) File(outDir, "report.md").writeText(renderMarkdown(cfg, results)) println("\n== done: report written to ${outDir.absolutePath}/report.{json,md} ==") @@ -129,6 +134,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "holdMaxMs" to cfg.holdMaxMs, "failureCreateRate" to cfg.failureCreateRate, "staleRetries" to cfg.staleRetries, + "stalePoisonRate" to cfg.stalePoisonRate, ) private fun toJsonElement(value: Any?): JsonElement = @@ -192,3 +198,78 @@ private fun renderMap( } } } + +/** + * Merges the mock's per-second series (per-API QPS + alive gauge) for one + * scenario into a single CSV: `second,alive,create,delete,get,renew, + * endpoint,execd.ping,execd.other` (second = offset from the earliest second + * recorded in the scenario window). + */ +private data class Series( + val start: Long, + val values: List, +) + +private fun writeTimeseriesCsv( + file: File, + scenario: String, + stats: Map, +) { + fun extractSeries( + key: String, + value: Any?, + ): Series? { + if (value !is Map<*, *>) return null + val start = (value["seriesStartUnixSec"] as? JsonPrimitive)?.content?.toLongOrNull() ?: return null + val raw = value["series"] as? List<*> ?: return null + val values = raw.mapNotNull { (it as? JsonPrimitive)?.content?.toLongOrNull() } + return Series(start, values) + } + + val routes = + listOf( + "lifecycle.create", + "lifecycle.delete", + "lifecycle.get", + "lifecycle.renew", + "lifecycle.endpoint", + "execd.ping", + "execd.other", + ) + val qps = stats["qps"] as? Map<*, *> ?: return + val aliveStats = stats["aliveStats"] + val seriesByRoute = routes.associateWith { route -> extractSeries(route, qps[route]) } + val alive = extractSeries("alive", aliveStats) + + val allStarts = seriesByRoute.values.mapNotNull { it?.start } + (alive?.start ?: 0L) + if (allStarts.isEmpty()) return + val begin = allStarts.min() + val end = allStarts.maxOf { start -> + val s = seriesByRoute.values.firstOrNull { it?.start == start } ?: alive + start + (s?.values?.size?.toLong() ?: 1L) - 1 + } + + val sb = StringBuilder() + sb.appendLine("second,alive,create,delete,get,renew,endpoint,execd.ping,execd.other") + for (sec in begin..end) { + val offset = sec - begin + sb.append(offset) + sb.append(',').append(valueAt(alive, sec)) + for (route in routes) { + sb.append(',').append(valueAt(seriesByRoute[route], sec)) + } + sb.appendLine() + } + file.parentFile?.mkdirs() + file.writeText(sb.toString()) +} + +private fun valueAt( + series: Series?, + second: Long, +): Long { + if (series == null) return 0L + val idx = (second - series.start).toInt() + if (idx < 0 || idx >= series.values.size) return 0L + return series.values[idx] +} diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt index 9203edbf9..3b63d4ad0 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Metrics.kt @@ -23,7 +23,7 @@ package com.alibaba.opensandbox.benchmark class LatencyCollector { private val lock = Any() private val samples = ArrayList() - private var failures = 0L + private val failureCounts = LinkedHashMap() fun record(elapsedMs: Long) { synchronized(lock) { @@ -31,24 +31,27 @@ class LatencyCollector { } } - fun recordFailure() { + fun recordFailure(reason: String = "other") { synchronized(lock) { - failures++ + failureCounts[reason] = (failureCounts[reason] ?: 0L) + 1 } } fun snapshot(): LatencyStats { val sorted: LongArray - var failureCount: Long + val failures: Long + val failureByType: Map synchronized(lock) { sorted = LongArray(samples.size) for ((i, v) in samples.withIndex()) sorted[i] = v sorted.sort() - failureCount = failures + failures = failureCounts.values.sum() + failureByType = failureCounts.toMap() } return LatencyStats( n = sorted.size.toLong(), - failures = failureCount, + failures = failures, + failuresByType = failureByType, meanMs = if (sorted.isEmpty()) 0.0 else sorted.average(), p50 = percentile(sorted, 0.50), p90 = percentile(sorted, 0.90), @@ -69,6 +72,7 @@ class LatencyCollector { data class LatencyStats( val n: Long, val failures: Long, + val failuresByType: Map = emptyMap(), val meanMs: Double, val p50: Long, val p90: Long, @@ -81,6 +85,7 @@ data class LatencyStats( mapOf( "count" to n, "failures" to failures, + "failuresByType" to failuresByType, "meanMs" to meanMs, "p50Ms" to p50, "p90Ms" to p90, diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt index 41511fc34..1b6da94ec 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/MockControl.kt @@ -57,12 +57,18 @@ class MockControl( post("__reset", buildJsonObject {}) } - fun setFaults(createFailureRate: Double? = null, execdFailureRate: Double? = null, poisonExisting: Boolean = false) { + fun setFaults( + createFailureRate: Double? = null, + execdFailureRate: Double? = null, + poisonExisting: Boolean = false, + poisonRate: Double? = null, + ) { val body = buildJsonObject { createFailureRate?.let { put("createFailureRate", it) } execdFailureRate?.let { put("execdFailureRate", it) } if (poisonExisting) put("poisonExisting", true) + poisonRate?.let { put("poisonRate", it) } } post("__config", body) } diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt index f23636647..922455ec8 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolProbe.kt @@ -18,27 +18,30 @@ package com.alibaba.opensandbox.benchmark import com.alibaba.opensandbox.sandbox.domain.pool.PoolState import com.alibaba.opensandbox.sandbox.pool.SandboxPool +import java.io.File import java.lang.management.GarbageCollectorMXBean import java.lang.management.ManagementFactory import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicInteger /** * Continuous client-side instrumentation: JVM threads, heap/GC, and pool * health (snapshot-based) sampled every [intervalMs] during a scenario. + * Aggregates go into the report; the raw per-sample series are written to a + * CSV in the run directory for offline analysis. */ class PoolProbe( private val pool: SandboxPool, private val intervalMs: Long = 500, ) { private val running = AtomicBoolean(true) + private val lock = Any() + private val sampleTimesMs = ArrayList() private val threadCounts = ArrayList() private val heapUsedMb = ArrayList() private val idleSamples = ArrayList() - private val degradedSamples = AtomicInteger() - private val backoffSamples = AtomicInteger() - private val inFlightMax = AtomicInteger() - private val lock = Any() + private val inFlightSamples = ArrayList() + private val degradedSamples = ArrayList() + private val backoffSamples = ArrayList() private var thread: Thread? = null private val threadBean = ManagementFactory.getThreadMXBean() @@ -78,25 +81,39 @@ class PoolProbe( private fun sample() { val heap = memoryBean.heapMemoryUsage - val snap = pool.snapshot() + val snap = + try { + pool.snapshot() + } catch (_: Exception) { + // Pool snapshot may fail while the state store is faulted. + return + } + val nowMs = System.currentTimeMillis() synchronized(lock) { + sampleTimesMs.add(nowMs) threadCounts.add(threadBean.threadCount) heapUsedMb.add(heap.used / (1024.0 * 1024.0)) idleSamples.add(snap.idleCount) + inFlightSamples.add(snap.inFlightOperations) + degradedSamples.add(snap.state == PoolState.DEGRADED) + backoffSamples.add(snap.backoffActive) } - if (snap.state == PoolState.DEGRADED) degradedSamples.incrementAndGet() - if (snap.backoffActive) backoffSamples.incrementAndGet() - inFlightMax.accumulateAndGet(snap.inFlightOperations) { a, b -> maxOf(a, b) } } fun report(): Map { val threads: LongArray val heap: DoubleArray val idle: IntArray + val inFlight: IntArray + val degradedCount: Int + val backoffCount: Int synchronized(lock) { threads = LongArray(threadCounts.size) { threadCounts[it].toLong() } heap = DoubleArray(heapUsedMb.size) { heapUsedMb[it] } idle = IntArray(idleSamples.size) { idleSamples[it] } + inFlight = IntArray(inFlightSamples.size) { inFlightSamples[it] } + degradedCount = degradedSamples.count { it } + backoffCount = backoffSamples.count { it } } val zeroIdleRatio = if (idle.isEmpty()) 0.0 else idle.count { it == 0 }.toDouble() / idle.size @@ -108,9 +125,50 @@ class PoolProbe( "gcTimeMs" to (gcBeans.sumOf { it.collectionTime } - gcStartTimeMs), "poolIdleCount" to stat(idle.map { it.toLong() }.toLongArray()), "poolIdleZeroRatio" to zeroIdleRatio, - "poolDegradedSamples" to degradedSamples.get(), - "poolBackoffSamples" to backoffSamples.get(), - "poolInFlightMax" to inFlightMax.get(), + "poolInFlight" to stat(inFlight.map { it.toLong() }.toLongArray()), + "poolDegradedSamples" to degradedCount, + "poolBackoffSamples" to backoffCount, + ) + } + + /** + * Writes the raw per-sample series as CSV (time offset ms, threads, + * heap MB, idle, inFlight, degraded, backoff) for offline analysis. + */ + fun writeCsv(file: File) { + val times: LongArray + val threads: IntArray + val heap: DoubleArray + val idle: IntArray + val inFlight: IntArray + val degraded: BooleanArray + val backoff: BooleanArray + synchronized(lock) { + times = sampleTimesMs.toLongArray() + threads = threadCounts.toIntArray() + heap = heapUsedMb.toDoubleArray() + idle = idleSamples.toIntArray() + inFlight = inFlightSamples.toIntArray() + degraded = degradedSamples.toBooleanArray() + backoff = backoffSamples.toBooleanArray() + } + if (times.isEmpty()) return + val start = times[0] + file.parentFile?.mkdirs() + file.writeText( + buildString { + appendLine("tMs,threads,heapUsedMb,idleCount,inFlight,degraded,backoff") + for (i in times.indices) { + append(times[i] - start) + append(',').append(threads[i]) + append(',').append(heap[i]) + append(',').append(idle[i]) + append(',').append(inFlight[i]) + append(',').append(if (degraded[i]) 1 else 0) + append(',').append(if (backoff[i]) 1 else 0) + appendLine() + } + }, ) } diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt index 047e27dbb..16497c8a0 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -20,6 +20,7 @@ import com.alibaba.opensandbox.sandbox.pool.SandboxPool import com.alibaba.opensandbox.sandbox.config.ConnectionConfig import com.alibaba.opensandbox.sandbox.domain.pool.AcquirePolicy import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec +import com.alibaba.opensandbox.sandbox.domain.pool.PoolStateStore import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore import java.time.Duration @@ -37,6 +38,7 @@ object PoolRunner { idleTimeoutS: Long = cfg.idleTimeoutS, maxAcquireRetries: Int = cfg.staleRetries, acquireReadyTimeoutMs: Long = cfg.acquireReadyTimeoutMs, + stateStore: PoolStateStore = InMemoryPoolStateStore(), ): SandboxPool { val connectionConfig = ConnectionConfig.builder() @@ -49,7 +51,7 @@ object PoolRunner { .poolName(poolName) .ownerId("bench-owner-$poolName") .maxIdle(maxIdle) - .stateStore(InMemoryPoolStateStore()) + .stateStore(stateStore) .connectionConfig(connectionConfig) .creationSpec( PoolCreationSpec.builder() @@ -88,11 +90,32 @@ object PoolRunner { target: Int, timeoutMs: Long, ): Long { - val deadline = System.nanoTime() + timeoutMs * 1_000_000 + val start = System.nanoTime() + val deadline = start + timeoutMs * 1_000_000 while (true) { val idle = pool.snapshot().idleCount if (idle >= target) { - return (System.nanoTime() - (deadline - timeoutMs * 1_000_000)) / 1_000_000 + return (System.nanoTime() - start) / 1_000_000 + } + if (System.nanoTime() > deadline) { + return -1L + } + Thread.sleep(100) + } + } + + /** Polls snapshot until idleCount drops to [target] or below (shrink); returns elapsed ms or -1 on timeout. */ + fun waitForIdleBelow( + pool: SandboxPool, + target: Int, + timeoutMs: Long, + ): Long { + val start = System.nanoTime() + val deadline = start + timeoutMs * 1_000_000 + while (true) { + val idle = pool.snapshot().idleCount + if (idle <= target) { + return (System.nanoTime() - start) / 1_000_000 } if (System.nanoTime() > deadline) { return -1L diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index e7fceb142..9cff3ced6 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -17,9 +17,17 @@ package com.alibaba.opensandbox.benchmark import com.alibaba.opensandbox.sandbox.Sandbox -import com.alibaba.opensandbox.sandbox.pool.SandboxPool +import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolAcquireFailedException +import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolDestroyedException +import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolEmptyException +import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolNotRunningException +import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolStateStoreUnavailableException +import com.alibaba.opensandbox.sandbox.domain.exceptions.SandboxReadyTimeoutException +import com.alibaba.opensandbox.sandbox.domain.pool.AcquirePolicy import com.alibaba.opensandbox.sandbox.domain.pool.PoolState +import com.alibaba.opensandbox.sandbox.pool.SandboxPool import kotlinx.serialization.json.JsonPrimitive +import java.io.File import java.util.concurrent.CountDownLatch import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.Executors @@ -82,7 +90,7 @@ object Scenarios { latency.record((System.nanoTime() - t0) / 1_000_000) killAndClose(sb) } catch (t: Throwable) { - latency.recordFailure() + latency.recordFailure(classifyFailure(t)) } } } @@ -91,6 +99,7 @@ object Scenarios { threads.shutdown() threads.awaitTermination(10, TimeUnit.MINUTES) probe.stop() + probe.writeCsv(File(cfg.reportDir, "client-warm-latency.csv")) val createdDelta = num(mock.stats(), "stats.created") - createdBefore val acquires = rounds * workers.toLong() @@ -142,7 +151,7 @@ object Scenarios { Thread.sleep(rng.nextLong(cfg.holdMinMs, cfg.holdMaxMs + 1)) killAndClose(sb) } catch (t: Throwable) { - latency.recordFailure() + latency.recordFailure(classifyFailure(t)) Thread.sleep(200) } } @@ -152,6 +161,7 @@ object Scenarios { threads.awaitTermination(15, TimeUnit.MINUTES) running.set(false) probe.stop() + probe.writeCsv(File(cfg.reportDir, "client-steady-state.csv")) val createdDelta = num(mock.stats(), "stats.created") - createdBefore val killedDelta = num(mock.stats(), "stats.killed") - killedBefore @@ -236,7 +246,7 @@ object Scenarios { latency.record((System.nanoTime() - t0) / 1_000_000) killAndClose(sb) } catch (t: Throwable) { - latency.recordFailure() + latency.recordFailure(classifyFailure(t)) } } Thread.sleep(500) @@ -276,9 +286,11 @@ object Scenarios { val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) val createdBefore = num(mock.stats(), "stats.created") - // Poison every currently-alive sandbox: their execd endpoints start - // failing, so idle candidates cannot be connected. - mock.setFaults(poisonExisting = true) + // Poison a fraction (default 1.0 = all) of the currently-alive + // sandboxes: their execd endpoints start failing, so idle candidates + // cannot be connected. Partial poisoning simulates real-world failure + // where the pool must skip bad candidates and return good ones. + mock.setFaults(poisonRate = cfg.stalePoisonRate) val latency = LatencyCollector() repeat(cfg.staleAcquires) { @@ -288,7 +300,7 @@ object Scenarios { latency.record((System.nanoTime() - t0) / 1_000_000) killAndClose(sb) } catch (t: Throwable) { - latency.recordFailure() + latency.recordFailure(classifyFailure(t)) } } val stats = mock.stats() @@ -300,7 +312,9 @@ object Scenarios { return mapOf( "fillTimeMs" to fillMs, "acquires" to cfg.staleAcquires, + "poisonRate" to cfg.stalePoisonRate, "latency" to latency.snapshot().toMap(), + "successRate" to successRate(latency.snapshot()), "serverExecdPoisoned" to num(stats, "stats.execdPoisoned"), "serverCreatedDelta" to (num(stats, "stats.created") - createdBefore), "serverAliveAfter" to num(stats, "alive"), @@ -362,6 +376,176 @@ object Scenarios { ) } + // ---------- resize (shrink + regrow) ---------- + + fun resize(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "resize") + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val createdBefore = num(mock.stats(), "stats.created") + val killedBefore = num(mock.stats(), "stats.killed") + + // Shrink: excess idles must be drained and killed by reconcile. + val shrinkTarget = maxOf(1, cfg.maxIdle / 2) + pool.resize(shrinkTarget) + val shrinkMs = PoolRunner.waitForIdleBelow(pool, shrinkTarget, cfg.coldStartTimeoutMs) + Thread.sleep(1000) // let server-side kills settle + val killedDuringShrink = num(mock.stats(), "stats.killed") - killedBefore + + // Regrow back to the original target. + pool.resize(cfg.maxIdle) + val regrowMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val stats = mock.stats() + pool.shutdown(graceful = false) + + return mapOf( + "fillTimeMs" to fillMs, + "shrinkTo" to shrinkTarget, + "shrinkTimeMs" to shrinkMs, + "killedDuringShrink" to killedDuringShrink, + "regrowTimeMs" to regrowMs, + "serverCreatedDelta" to (num(stats, "stats.created") - createdBefore), + "serverAliveAtEnd" to num(stats, "alive"), + ) + } + + // ---------- acquire racing graceful shutdown ---------- + + fun shutdownRace(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val pool = PoolRunner.build(cfg, "shutdown-race") + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + + val shutdownStarted = AtomicBoolean(false) + val stop = AtomicBoolean(false) + // Collector 1: attempts while the pool is RUNNING. Collector 2: the + // race window itself (acquires racing DRAINING/STOPPED). + val runningLatency = LatencyCollector() + val raceLatency = LatencyCollector() + val runningAttempts = AtomicLong(0) + val raceAttempts = AtomicLong(0) + val threads = Executors.newFixedThreadPool(cfg.steadyWorkers) + repeat(cfg.steadyWorkers) { + threads.submit { + while (!stop.get()) { + val inRace = shutdownStarted.get() + if (inRace) raceAttempts.incrementAndGet() else runningAttempts.incrementAndGet() + val latency = if (inRace) raceLatency else runningLatency + try { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) + latency.record(0) + killAndClose(sb) + } catch (t: Throwable) { + latency.recordFailure(classifyFailure(t)) + } + } + } + } + Thread.sleep(2000) // let workers hammer the warm pool + shutdownStarted.set(true) + val shutdownT0 = System.nanoTime() + pool.shutdown(graceful = true) + val shutdownMs = (System.nanoTime() - shutdownT0) / 1_000_000 + stop.set(true) + threads.shutdownNow() + threads.awaitTermination(30, TimeUnit.SECONDS) + + val runningStats = runningLatency.snapshot() + val raceStats = raceLatency.snapshot() + return mapOf( + "fillTimeMs" to fillMs, + "runningPhaseAttempts" to runningAttempts.get(), + "runningPhase" to + mapOf( + "successRate" to successRate(runningStats), + "latency" to runningStats.toMap(), + ), + "raceWindowAttempts" to raceAttempts.get(), + "raceWindow" to + mapOf( + "successRate" to successRate(raceStats), + "latency" to raceStats.toMap(), + "rejectedDuringDraining" to (raceStats.failuresByType["poolNotRunning"] ?: 0L), + ), + "shutdownMs" to shutdownMs, + ) + } + + // ---------- state-store outage (OSEP-0005 fallthrough) ---------- + + fun storeOutage(cfg: BenchmarkConfig, mock: MockControl): Map { + mock.reset() + val store = FailingPoolStateStore() + val pool = + PoolRunner.build( + cfg, + "store-outage", + maxAcquireRetries = 1, + stateStore = store, + ) + pool.start() + val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val createdBefore = num(mock.stats(), "stats.created") + + // Drain part of the idle buffer before the outage so recovery has + // real refill work (the pool cannot commit warmups while the store + // is down, and fallthrough acquires never touch the store). + val drainCount = maxOf(1, cfg.maxIdle / 3) + repeat(drainCount) { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) + killAndClose(sb) + } + + // Phase 1: store down, DIRECT_CREATE policy must fall through to + // direct create (OSEP-0005) and keep acquire available. + store.setFailing(true) + Thread.sleep(500) + val phase1 = LatencyCollector() + repeat(cfg.failureAcquires) { + val t0 = System.nanoTime() + try { + val sb = pool.acquire(cfg.acquireTimeout, PoolRunner.DEFAULT_POLICY) + phase1.record((System.nanoTime() - t0) / 1_000_000) + killAndClose(sb) + } catch (t: Throwable) { + phase1.recordFailure(classifyFailure(t)) + } + } + val phase1Stats = phase1.snapshot() + val errorsPhase1 = store.errorCount() + + // Phase 2: store still down, FAIL_FAST must fail closed and surface + // PoolStateStoreUnavailableException. + val phase2 = LatencyCollector() + repeat(10) { + try { + val sb = pool.acquire(cfg.acquireTimeout, AcquirePolicy.FAIL_FAST) + phase2.record(0) + killAndClose(sb) + } catch (t: Throwable) { + phase2.recordFailure(classifyFailure(t)) + } + } + val phase2Stats = phase2.snapshot() + + // Phase 3: store recovers; the pool must refill. + store.setFailing(false) + val refillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + val stats = mock.stats() + pool.shutdown(graceful = false) + + return mapOf( + "fillTimeMs" to fillMs, + "phase1DirectCreateFallthrough" to phase1Stats.toMap(), + "phase1StoreErrorCount" to errorsPhase1, + "phase2FailFastFailClosed" to phase2Stats.toMap(), + "refillTimeMsAfterRecovery" to refillMs, + "serverCreatedDelta" to (num(stats, "stats.created") - createdBefore), + ) + } + // ---------- helpers ---------- private fun successRate(stats: LatencyStats): Double { @@ -369,6 +553,17 @@ object Scenarios { return if (total == 0L) 0.0 else (stats.n.toDouble() / total) } + private fun classifyFailure(t: Throwable): String = + when (t) { + is SandboxReadyTimeoutException -> "readyTimeout" + is PoolNotRunningException -> "poolNotRunning" + is PoolEmptyException -> "poolEmpty" + is PoolAcquireFailedException -> "acquireFailed" + is PoolDestroyedException -> "poolDestroyed" + is PoolStateStoreUnavailableException -> "storeUnavailable" + else -> "other" + } + private fun killAndClose(sandbox: Sandbox) { try { sandbox.kill() @@ -404,5 +599,8 @@ object Scenarios { "failure-injection" to ::failureInjection, "stale-idle" to ::staleIdle, "idle-expiry" to ::idleExpiry, + "resize" to ::resize, + "shutdown-race" to ::shutdownRace, + "store-outage" to ::storeOutage, ) } diff --git a/tests/benchmark/mockserver/config.go b/tests/benchmark/mockserver/config.go index e4caff19c..5b43153c8 100644 --- a/tests/benchmark/mockserver/config.go +++ b/tests/benchmark/mockserver/config.go @@ -78,6 +78,9 @@ type FaultConfig struct { // Newly created sandboxes are unaffected. Used to simulate stale idle // sandboxes (e.g. sandboxes that died server-side). PoisonExisting bool `json:"poisonExisting"` + // PoisonRate flips a random subset (probability [0,1]) of currently-alive + // sandboxes into the poisoned state, simulating partial failure. + PoisonRate *float64 `json:"poisonRate"` } func loadConfig(path string) (*Config, error) { diff --git a/tests/benchmark/mockserver/main.go b/tests/benchmark/mockserver/main.go index bf5371cdc..5d5488da8 100644 --- a/tests/benchmark/mockserver/main.go +++ b/tests/benchmark/mockserver/main.go @@ -63,6 +63,8 @@ func main() { log.Printf("mock lifecycle server listening on http://%s", *lifecycleAddr) log.Printf("mock execd server listening on http://%s", *execdAddr) + go mock.startAliveTicker() + errCh := make(chan error, 2) go func() { errCh <- lifecycleSrv.ListenAndServe() }() go func() { errCh <- execdSrv.ListenAndServe() }() diff --git a/tests/benchmark/mockserver/server.go b/tests/benchmark/mockserver/server.go index 4c9c5f00c..0d9d6eb42 100644 --- a/tests/benchmark/mockserver/server.go +++ b/tests/benchmark/mockserver/server.go @@ -45,6 +45,7 @@ type MockServer struct { stats Stats qps *QpsRegistry + alive *gaugeTracker } // Sandbox is the mock's view of a sandbox on the lifecycle side. @@ -71,6 +72,28 @@ func newMockServer(cfg *Config, execdHost string, execdPort int, windowSec int) execdHost: execdHost, execdPort: execdPort, qps: newQpsRegistry(windowSec), + alive: newGaugeTracker(windowSec), + } +} + +// startAliveTicker records the alive-sandbox count once per second so the +// driver can see the pool-size trajectory (over-creation, shrink, drift). +func (m *MockServer) startAliveTicker() { + record := func(now time.Time) { + m.mu.RLock() + alive := 0 + for _, sb := range m.sandboxes { + if sb.alive(now) { + alive++ + } + } + m.mu.RUnlock() + m.alive.record(now, int64(alive)) + } + record(time.Now()) + ticker := time.NewTicker(time.Second) + for range ticker.C { + record(time.Now()) } } @@ -335,6 +358,7 @@ func (m *MockServer) handleStats(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "stats": stats, "alive": alive, + "aliveStats": m.alive.snapshot(time.Now()), "poisoned": poisoned, "config": m.cfgSnapshot(), "qps": m.qps.snapshot(time.Now()), @@ -361,6 +385,16 @@ func (m *MockServer) handleConfig(w http.ResponseWriter, r *http.Request) { } m.mu.Unlock() } + if f.PoisonRate != nil { + rate := *f.PoisonRate + m.mu.Lock() + for _, sb := range m.sandboxes { + if sb.alive(time.Now()) && rand.Float64() < rate { + sb.Poisoned = true + } + } + m.mu.Unlock() + } writeJSON(w, http.StatusOK, map[string]any{"config": m.cfgSnapshot()}) } @@ -368,6 +402,7 @@ func (m *MockServer) handleReset(w http.ResponseWriter, r *http.Request) { _ = r.Body.Close() m.stats.reset() m.qps.reset(time.Now()) + m.alive.reset(time.Now()) writeJSON(w, http.StatusOK, map[string]any{"reset": true}) } diff --git a/tests/benchmark/mockserver/stats.go b/tests/benchmark/mockserver/stats.go index d486181ea..79af61437 100644 --- a/tests/benchmark/mockserver/stats.go +++ b/tests/benchmark/mockserver/stats.go @@ -208,3 +208,82 @@ func (r *QpsRegistry) snapshot(now time.Time) map[string]QpsSnapshot { } return out } + +// GaugeSnapshot is the per-second view of a gauge (e.g. alive sandboxes). +type GaugeSnapshot struct { + Max int64 `json:"max"` + SeriesStart int64 `json:"seriesStartUnixSec"` + Series []int64 `json:"series"` +} + +// gaugeTracker keeps the per-second peak of a gauge (ring buffer) plus the +// all-time max since the last reset. +type gaugeTracker struct { + mu sync.Mutex + window int64 + buckets []int64 + secs []int64 + startSec int64 + maxAll int64 +} + +func newGaugeTracker(windowSec int) *gaugeTracker { + return &gaugeTracker{ + window: int64(windowSec), + buckets: make([]int64, windowSec), + secs: make([]int64, windowSec), + } +} + +func (g *gaugeTracker) record(now time.Time, value int64) { + sec := now.Unix() + idx := sec % g.window + g.mu.Lock() + if g.secs[idx] != sec { + g.secs[idx] = sec + g.buckets[idx] = value + } else if value > g.buckets[idx] { + g.buckets[idx] = value + } + if value > g.maxAll { + g.maxAll = value + } + if g.startSec == 0 { + g.startSec = sec + } + g.mu.Unlock() +} + +func (g *gaugeTracker) reset(now time.Time) { + g.mu.Lock() + defer g.mu.Unlock() + for i := range g.buckets { + g.buckets[i] = 0 + g.secs[i] = 0 + } + g.maxAll = 0 + g.startSec = now.Unix() +} + +func (g *gaugeTracker) snapshot(now time.Time) GaugeSnapshot { + sec := now.Unix() + g.mu.Lock() + defer g.mu.Unlock() + if g.startSec == 0 { + g.startSec = sec + } + begin := g.startSec + if sec-begin+1 > g.window { + begin = sec - g.window + 1 + } + series := make([]int64, 0, sec-begin+1) + for s := begin; s <= sec; s++ { + idx := s % g.window + value := g.buckets[idx] + if g.secs[idx] != s { + value = 0 + } + series = append(series, value) + } + return GaugeSnapshot{Max: g.maxAll, SeriesStart: begin, Series: series} +} diff --git a/tests/benchmark/run.sh b/tests/benchmark/run.sh index d479e050d..b73fdde5c 100755 --- a/tests/benchmark/run.sh +++ b/tests/benchmark/run.sh @@ -88,14 +88,21 @@ done echo "== building mock server ==" (cd "${BENCH_DIR}/mockserver" && go build -o "${BENCH_DIR}/bin/mockserver" .) -# 2. start the mock server -mkdir -p "${BENCH_DIR}/results" +# 2. one run directory holds every artifact of this run (reports, CSVs, +# mock config, driver args, mock log) for convenient offline analysis. +RUN_DIR="${BENCH_DIR}/results/run-$(date +%Y%m%d-%H%M%S)" +mkdir -p "${RUN_DIR}" +printf '%s\n' "${DRIVER_ARGS[@]}" > "${RUN_DIR}/driver-args.txt" +cp "${MOCK_CONFIG}" "${RUN_DIR}/mock-config.json" + +# 3. start the mock server echo "== starting mock server (lifecycle=${LIFECYCLE_ADDR}, execd=${EXECD_ADDR}, config=${MOCK_CONFIG}) ==" +echo "== run directory: ${RUN_DIR} ==" "${BENCH_DIR}/bin/mockserver" \ -lifecycle-addr "${LIFECYCLE_ADDR}" \ -execd-addr "${EXECD_ADDR}" \ -config "${MOCK_CONFIG}" \ - > "${BENCH_DIR}/results/mockserver.log" 2>&1 & + > "${RUN_DIR}/mockserver.log" 2>&1 & MOCK_PID=$! for _ in $(seq 1 50); do @@ -106,14 +113,14 @@ for _ in $(seq 1 50); do done if ! curl -fsS "http://${LIFECYCLE_ADDR}/__stats" > /dev/null 2>&1; then echo "error: mock server did not come up" >&2 - cat "${BENCH_DIR}/results/mockserver.log" >&2 + cat "${RUN_DIR}/mockserver.log" >&2 exit 1 fi -# 3. run the driver (Kotlin SDK is built from source via composite build, +# 4. run the driver (Kotlin SDK is built from source via composite build, # see kotlin/settings.gradle.kts) echo "== running benchmark driver ==" -DRIVER_ARGS+=("--report-dir" "${BENCH_DIR}/results/run-$(date +%Y%m%d-%H%M%S)") +DRIVER_ARGS+=("--report-dir" "${RUN_DIR}") (cd "${BENCH_DIR}/kotlin" && ./gradlew --console=plain run --args="${DRIVER_ARGS[*]}") -echo "== done ==" +echo "== done: ${RUN_DIR} ==" From 0f107a9d5c9b6fb68a0251ee84263bbd7bf7b02b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 12:30:43 +0800 Subject: [PATCH 07/16] docs(benchmark): fix stale default-profile comment in quick start --- tests/benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index c3c5a0d90..67ca5e4bf 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -26,7 +26,7 @@ tests/benchmark/ ## Quick start ```bash -# default config: create/delete 300-800ms, execd ping 1-5s, others 50-100ms +# default config: create/delete 300-800ms, execd ping 100ms, others 50-100ms ./run.sh # smoke run with fast provisioning From 51419a106a845e2243c80801135eaa102dfb58d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 14:07:37 +0800 Subject: [PATCH 08/16] fix(benchmark): steady-state await must cover the full configured duration awaitTermination(15min) silently truncated 30-min runs to ~15min. Wait for duration+5min and report the actual loader duration so achieved rate is computed against real runtime. --- .../com/alibaba/opensandbox/benchmark/Scenarios.kt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index 9cff3ced6..a851d283f 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -139,6 +139,7 @@ object Scenarios { val rng = Random(System.nanoTime()) val pacer = RatePacer(cfg.acquireRatePerMin) val threads = Executors.newFixedThreadPool(cfg.steadyWorkers) + val loaderStart = System.nanoTime() repeat(cfg.steadyWorkers) { threads.submit { while (System.nanoTime() < deadline) { @@ -158,7 +159,10 @@ object Scenarios { } } threads.shutdown() - threads.awaitTermination(15, TimeUnit.MINUTES) + // The loaders run for the full configured duration; the wait must not + // truncate them (a 15-min cap would silently halve a 30-min run). + threads.awaitTermination(durationMs / 1000 + 300, TimeUnit.SECONDS) + val loaderDurationMs = (System.nanoTime() - loaderStart) / 1_000_000 running.set(false) probe.stop() probe.writeCsv(File(cfg.reportDir, "client-steady-state.csv")) @@ -173,11 +177,13 @@ object Scenarios { return mapOf( "fillTimeMs" to fillMs, "durationMs" to durationMs, + "actualLoaderDurationMs" to loaderDurationMs, "workers" to cfg.steadyWorkers, "targetAcquiresPerMin" to cfg.acquireRatePerMin, "acquiredCount" to acquires.get(), - "achievedAcquiresPerMin" to (acquires.get().toDouble() * 60_000 / durationMs), - "throughputAcquiresPerSec" to (acquires.get().toDouble() / cfg.steadyDurationS), + "achievedAcquiresPerMin" to + (acquires.get().toDouble() * 60_000 / loaderDurationMs.coerceAtLeast(1)), + "throughputAcquiresPerSec" to (acquires.get().toDouble() * 1000 / loaderDurationMs.coerceAtLeast(1)), "successRate" to successRate(latencyStats), "latency" to latencyStats.toMap(), "serverCreatedDelta" to createdDelta, From 6f26b165b4a04d903ffe185a10ec479b6d858544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 14:39:46 +0800 Subject: [PATCH 09/16] feat(pool): add warmup pipeline diagnostics to isolate warmupConcurrency scaling PoolWarmupDiagnostics records warmup queue-wait, createOneSandbox duration, commit duration, reconcile-tick cadence, in-flight warmup peak, and create failure reasons. Cold-start scenario reports them; --shared-connection-pool flag added to test connection-reuse hypotheses. Diagnosis: at wc=1000 the submission chain fully saturates (in-flight peak 1000, queue wait ~20ms); fill degradation is caused by per-sandbox fresh-TCP connection churn producing intermittent Connection reset failures against the mock listener (accept backlog), amplifying attempts via retries. --- .../sandbox/pool/PoolWarmupDiagnostics.kt | 191 ++++++++++++++++++ .../opensandbox/sandbox/pool/SandboxPool.kt | 13 ++ .../com/alibaba/opensandbox/benchmark/Cli.kt | 6 + .../com/alibaba/opensandbox/benchmark/Main.kt | 1 + .../opensandbox/benchmark/PoolRunner.kt | 5 + .../opensandbox/benchmark/Scenarios.kt | 16 ++ 6 files changed, 232 insertions(+) create mode 100644 sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupDiagnostics.kt diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupDiagnostics.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupDiagnostics.kt new file mode 100644 index 000000000..d2330c589 --- /dev/null +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/PoolWarmupDiagnostics.kt @@ -0,0 +1,191 @@ +/* + * Copyright 2026 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.opensandbox.sandbox.pool + +/** + * Diagnostic counters for the warmup pipeline. + * + * Benchmark/diagnostic aid only — not part of the public API contract. + * Overhead is a single `System.nanoTime()` read plus a locked list append per + * warmup event, and it is a no-op when never read. State is global (all + * pools in the JVM share it), so reset before a single-pool experiment and + * snapshot after it settles. + * + * Measured phases of one warmup task: + * - queue wait: submission -> task picked up by a warmup worker + * - create: `createOneSandbox` (create + readiness + renew) + * - commit: lock + store putIdle when the sandbox enters the idle buffer + * Plus the reconcile-tick cadence (submission driver) and the in-flight + * warmup count trajectory. + */ +object PoolWarmupDiagnostics { + data class PhaseStats( + val count: Long, + val meanMs: Double, + val p50Ms: Long, + val p95Ms: Long, + val maxMs: Long, + ) + + data class Snapshot( + val queueWaitMs: PhaseStats, + val createDurationMs: PhaseStats, + val commitDurationMs: PhaseStats, + val tickIntervalMs: PhaseStats, + val tickDurationMs: PhaseStats, + val submitBurst: PhaseStats, + val submitCalls: Long, + val inFlightPeak: Int, + val inFlightMean: Double, + val createFailures: Map, + ) + + private val lock = Any() + private val queueWaitNanos = ArrayList() + private val createNanos = ArrayList() + private val commitNanos = ArrayList() + private val tickIntervalNanos = ArrayList() + private val tickDurationNanos = ArrayList() + private val submitBursts = ArrayList() + private val failureReasons = LinkedHashMap() + private var submitCalls = 0L + private var inFlightSum = 0L + private var inFlightSamples = 0L + private var inFlightPeak = 0 + private var lastTickNanos = 0L + + fun reset() { + synchronized(lock) { + queueWaitNanos.clear() + createNanos.clear() + commitNanos.clear() + tickIntervalNanos.clear() + tickDurationNanos.clear() + submitBursts.clear() + failureReasons.clear() + submitCalls = 0L + inFlightSum = 0L + inFlightSamples = 0L + inFlightPeak = 0 + lastTickNanos = 0L + } + } + + fun recordQueueWait(nanos: Long) = synchronized(lock) { queueWaitNanos.add(nanos) } + + fun recordCreate(nanos: Long) = synchronized(lock) { createNanos.add(nanos) } + + fun recordCommit(nanos: Long) = synchronized(lock) { commitNanos.add(nanos) } + + /** Records the wall-clock spacing between reconcile ticks plus each tick's execution time. */ + fun recordTick(nowNanos: Long, durationNanos: Long) { + synchronized(lock) { + tickDurationNanos.add(durationNanos) + if (lastTickNanos != 0L) { + tickIntervalNanos.add(nowNanos - lastTickNanos) + } + lastTickNanos = nowNanos + } + } + + fun recordSubmitBurst(size: Int) { + synchronized(lock) { + submitCalls++ + submitBursts.add(size) + } + } + + fun recordInFlight(current: Int) { + synchronized(lock) { + if (current > inFlightPeak) inFlightPeak = current + inFlightSum += current + inFlightSamples++ + } + } + + /** Records a failed createOneSandbox attempt: exception class + first words of the message. */ + fun recordCreateFailure(failure: Throwable) { + val message = failure.message?.trim()?.take(80) ?: "" + val key = "${failure.javaClass.simpleName}: $message" + synchronized(lock) { + failureReasons[key] = (failureReasons[key] ?: 0L) + 1 + } + } + + fun snapshot(): Snapshot { + val queueWait: LongArray + val create: LongArray + val commit: LongArray + val tickInterval: LongArray + val tickDuration: LongArray + val bursts: IntArray + val calls: Long + val peak: Int + val meanInFlight: Double + val failures: Map + synchronized(lock) { + queueWait = queueWaitNanos.toLongArray() + create = createNanos.toLongArray() + commit = commitNanos.toLongArray() + tickInterval = tickIntervalNanos.toLongArray() + tickDuration = tickDurationNanos.toLongArray() + bursts = submitBursts.toIntArray() + calls = submitCalls + peak = inFlightPeak + meanInFlight = if (inFlightSamples == 0L) 0.0 else inFlightSum.toDouble() / inFlightSamples + failures = failureReasons.toMap() + } + return Snapshot( + queueWaitMs = phase(queueWait), + createDurationMs = phase(create), + commitDurationMs = phase(commit), + tickIntervalMs = phase(tickInterval), + tickDurationMs = phase(tickDuration), + submitBurst = burstPhase(bursts), + submitCalls = calls, + inFlightPeak = peak, + inFlightMean = meanInFlight, + createFailures = failures, + ) + } + + private fun phase(nanos: LongArray): PhaseStats { + if (nanos.isEmpty()) return PhaseStats(0, 0.0, 0L, 0L, 0L) + val sorted = nanos.copyOf() + sorted.sort() + return PhaseStats( + count = sorted.size.toLong(), + meanMs = sorted.average() / 1_000_000.0, + p50Ms = sorted[(sorted.size - 1) / 2] / 1_000_000, + p95Ms = sorted[((sorted.size - 1) * 95) / 100] / 1_000_000, + maxMs = sorted.last() / 1_000_000, + ) + } + + private fun burstPhase(bursts: IntArray): PhaseStats { + if (bursts.isEmpty()) return PhaseStats(0, 0.0, 0L, 0L, 0L) + val sorted = bursts.copyOf() + sorted.sort() + return PhaseStats( + count = sorted.size.toLong(), + meanMs = sorted.average(), + p50Ms = sorted[(sorted.size - 1) / 2].toLong(), + p95Ms = sorted[((sorted.size - 1) * 95) / 100].toLong(), + maxMs = sorted.last().toLong(), + ) + } +} diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt index dcbd7a38c..beac87498 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/pool/SandboxPool.kt @@ -906,6 +906,7 @@ class SandboxPool internal constructor( if (!isCurrentRun(run) || lifecycleState.get() != LifecycleState.RUNNING) return if (!isPoolNamespaceActive()) return val reconcileConfig = config.withMaxIdle(resolveMaxIdle()) + val tickStart = System.nanoTime() try { run.primaryOwned.set( PoolReconciler.runReconcileTick( @@ -920,6 +921,8 @@ class SandboxPool internal constructor( } catch (e: Exception) { run.primaryOwned.set(false) throw e + } finally { + PoolWarmupDiagnostics.recordTick(System.nanoTime(), System.nanoTime() - tickStart) } } finally { endOperation(run) @@ -999,6 +1002,7 @@ class SandboxPool internal constructor( run: RunContext, count: Int, ) { + PoolWarmupDiagnostics.recordSubmitBurst(count) repeat(count) { if (!isCurrentRun(run) || lifecycleState.get() != LifecycleState.RUNNING || @@ -1024,20 +1028,26 @@ class SandboxPool internal constructor( private inner class TrackedWarmupTask( private val run: RunContext, ) : Runnable { + private val submittedAtNanos = System.nanoTime() private val completed = AtomicBoolean(false) init { run.warmingCount.incrementAndGet() + PoolWarmupDiagnostics.recordInFlight(run.warmingCount.get()) beginOperation(run) } override fun run() { + PoolWarmupDiagnostics.recordQueueWait(System.nanoTime() - submittedAtNanos) + val createStart = System.nanoTime() val outcome = try { WarmupOutcome.Success(createOneSandbox()) } catch (failure: Throwable) { + PoolWarmupDiagnostics.recordCreateFailure(failure) WarmupOutcome.Failure(failure) } + PoolWarmupDiagnostics.recordCreate(System.nanoTime() - createStart) dispatchCompletion(outcome) } @@ -1066,6 +1076,7 @@ class SandboxPool internal constructor( handleWarmupOutcome(run, outcome) } finally { run.warmingCount.decrementAndGet() + PoolWarmupDiagnostics.recordInFlight(run.warmingCount.get()) endOperation(run) // Only successful completions trigger an immediate reconcile. A failed warmup // frees its slot but must not cause an immediate retry: fast-failing creates @@ -1142,6 +1153,7 @@ class SandboxPool internal constructor( ) { var cleanupSource: String? = null run.commitLock.lock() + val commitStart = System.nanoTime() try { val state = lifecycleState.get() if (!isCurrentRun(run) || (state != LifecycleState.RUNNING && state != LifecycleState.DRAINING)) { @@ -1189,6 +1201,7 @@ class SandboxPool internal constructor( } } } finally { + PoolWarmupDiagnostics.recordCommit(System.nanoTime() - commitStart) run.commitLock.unlock() } cleanupSource?.let { source -> diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index fd335989d..9e7efa219 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -51,6 +51,7 @@ data class BenchmarkConfig( val staleRetries: Int, val staleAcquireReadyTimeoutMs: Long, val stalePoisonRate: Double, + val sharedConnectionPool: Boolean, val idleExpiryIdleTimeoutS: Long, val idleExpiryDurationS: Int, ) { @@ -90,6 +91,7 @@ object Cli { "stale-retries", "stale-acquire-ready-timeout-ms", "stale-poison-rate", + "shared-connection-pool", "idle-expiry-idle-timeout-s", "idle-expiry-duration-s", ) @@ -147,6 +149,10 @@ object Cli { staleAcquireReadyTimeoutMs = (map["stale-acquire-ready-timeout-ms"] ?: "3000").toLong(), // fraction (0..1] of idle sandboxes to poison in stale-idle; 1.0 = poison all stalePoisonRate = (map["stale-poison-rate"] ?: "1.0").toDouble(), + // Share one OkHttp connection pool across all sandbox clients + // (diagnostic: distinguishes connection-establishment bottlenecks + // from the SDK warmup submission chain). + sharedConnectionPool = (map["shared-connection-pool"] ?: "false").toBoolean(), idleExpiryIdleTimeoutS = (map["idle-expiry-idle-timeout-s"] ?: "20").toLong(), idleExpiryDurationS = (map["idle-expiry-duration-s"] ?: "40").toInt(), ) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index 98b25fb6b..748f89cc0 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -135,6 +135,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "failureCreateRate" to cfg.failureCreateRate, "staleRetries" to cfg.staleRetries, "stalePoisonRate" to cfg.stalePoisonRate, + "sharedConnectionPool" to cfg.sharedConnectionPool, ) private fun toJsonElement(value: Any?): JsonElement = diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt index 16497c8a0..120025ef7 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -46,6 +46,11 @@ object PoolRunner { .protocol("http") .requestTimeout(Duration.ofSeconds(30)) .disableMetrics() + .also { builder -> + if (cfg.sharedConnectionPool) { + builder.connectionPool(okhttp3.ConnectionPool()) + } + } .build() return SandboxPool.builder() .poolName(poolName) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index a851d283f..01d1c1813 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -25,6 +25,7 @@ import com.alibaba.opensandbox.sandbox.domain.exceptions.PoolStateStoreUnavailab import com.alibaba.opensandbox.sandbox.domain.exceptions.SandboxReadyTimeoutException import com.alibaba.opensandbox.sandbox.domain.pool.AcquirePolicy import com.alibaba.opensandbox.sandbox.domain.pool.PoolState +import com.alibaba.opensandbox.sandbox.pool.PoolWarmupDiagnostics import com.alibaba.opensandbox.sandbox.pool.SandboxPool import kotlinx.serialization.json.JsonPrimitive import java.io.File @@ -47,12 +48,14 @@ object Scenarios { fun coldStart(cfg: BenchmarkConfig, mock: MockControl): Map { mock.reset() + PoolWarmupDiagnostics.reset() val pool = PoolRunner.build(cfg, "cold-start") pool.start() val t0 = System.nanoTime() val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) val stats = mock.stats() pool.shutdown(graceful = false) + val diagnostics = PoolWarmupDiagnostics.snapshot() val created = num(stats, "stats.created") return mapOf( @@ -61,6 +64,19 @@ object Scenarios { "serverCreated" to created, "serverAliveAtFill" to num(stats, "alive"), "overCreationOvershoot" to (created - cfg.maxIdle).coerceAtLeast(0), + "warmupPipeline" to + mapOf( + "queueWaitMs" to diagnostics.queueWaitMs, + "createDurationMs" to diagnostics.createDurationMs, + "commitDurationMs" to diagnostics.commitDurationMs, + "tickIntervalMs" to diagnostics.tickIntervalMs, + "tickDurationMs" to diagnostics.tickDurationMs, + "submitBurst" to diagnostics.submitBurst, + "submitCalls" to diagnostics.submitCalls, + "inFlightPeak" to diagnostics.inFlightPeak, + "inFlightMean" to diagnostics.inFlightMean, + "createFailures" to diagnostics.createFailures, + ), ) } From 5734b0da6f598ca457786d67ed7dc62bdd3bb8d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 14:47:19 +0800 Subject: [PATCH 10/16] feat(benchmark): shared connection pool size sweep (--shared-connection-pool-size) Replace the boolean flag with a configurable OkHttp ConnectionPool idle size. Evidence: at wc=1000, pool size 0/100/200/500 -> fill 19.9s/9.1s/7.1s/4.3s, connection-reset failures 8024/2696/1341/0, attempt amplification 5x/2.3x/ 1.7x/1x. A properly sized shared pool eliminates the TCP churn entirely. --- .../kotlin/com/alibaba/opensandbox/benchmark/Cli.kt | 12 ++++++------ .../kotlin/com/alibaba/opensandbox/benchmark/Main.kt | 2 +- .../com/alibaba/opensandbox/benchmark/PoolRunner.kt | 11 +++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 9e7efa219..65cfdb5d9 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -51,7 +51,7 @@ data class BenchmarkConfig( val staleRetries: Int, val staleAcquireReadyTimeoutMs: Long, val stalePoisonRate: Double, - val sharedConnectionPool: Boolean, + val sharedConnectionPoolSize: Int, val idleExpiryIdleTimeoutS: Long, val idleExpiryDurationS: Int, ) { @@ -91,7 +91,7 @@ object Cli { "stale-retries", "stale-acquire-ready-timeout-ms", "stale-poison-rate", - "shared-connection-pool", + "shared-connection-pool-size", "idle-expiry-idle-timeout-s", "idle-expiry-duration-s", ) @@ -149,10 +149,10 @@ object Cli { staleAcquireReadyTimeoutMs = (map["stale-acquire-ready-timeout-ms"] ?: "3000").toLong(), // fraction (0..1] of idle sandboxes to poison in stale-idle; 1.0 = poison all stalePoisonRate = (map["stale-poison-rate"] ?: "1.0").toDouble(), - // Share one OkHttp connection pool across all sandbox clients - // (diagnostic: distinguishes connection-establishment bottlenecks - // from the SDK warmup submission chain). - sharedConnectionPool = (map["shared-connection-pool"] ?: "false").toBoolean(), + // Inject a shared OkHttp ConnectionPool with this many idle + // connections across all sandbox clients (0 = each sandbox keeps + // its own fresh connections). Diagnostic: tests connection reuse. + sharedConnectionPoolSize = (map["shared-connection-pool-size"] ?: "0").toInt(), idleExpiryIdleTimeoutS = (map["idle-expiry-idle-timeout-s"] ?: "20").toLong(), idleExpiryDurationS = (map["idle-expiry-duration-s"] ?: "40").toInt(), ) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index 748f89cc0..f83474774 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -135,7 +135,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "failureCreateRate" to cfg.failureCreateRate, "staleRetries" to cfg.staleRetries, "stalePoisonRate" to cfg.stalePoisonRate, - "sharedConnectionPool" to cfg.sharedConnectionPool, + "sharedConnectionPoolSize" to cfg.sharedConnectionPoolSize, ) private fun toJsonElement(value: Any?): JsonElement = diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt index 120025ef7..55f04cd5b 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -23,6 +23,7 @@ import com.alibaba.opensandbox.sandbox.domain.pool.PoolCreationSpec import com.alibaba.opensandbox.sandbox.domain.pool.PoolStateStore import com.alibaba.opensandbox.sandbox.infrastructure.pool.InMemoryPoolStateStore import java.time.Duration +import java.util.concurrent.TimeUnit /** * Builds a [SandboxPool] wired to the mock server. Health checks stay enabled @@ -47,8 +48,14 @@ object PoolRunner { .requestTimeout(Duration.ofSeconds(30)) .disableMetrics() .also { builder -> - if (cfg.sharedConnectionPool) { - builder.connectionPool(okhttp3.ConnectionPool()) + if (cfg.sharedConnectionPoolSize > 0) { + builder.connectionPool( + okhttp3.ConnectionPool( + cfg.sharedConnectionPoolSize, + 5, + TimeUnit.MINUTES, + ), + ) } } .build() From e533355c293024b3126f4c84988ea4c9882be143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 15:07:38 +0800 Subject: [PATCH 11/16] fix(benchmark): serialize warmup pipeline PhaseStats as JSON objects, not strings The diagnostics data classes fell into the generic toString() branch, so queueWaitMs/createDurationMs etc. were strings in report.json. Convert them explicitly so the warmupPipeline block is machine-readable. --- .../opensandbox/benchmark/Scenarios.kt | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index 01d1c1813..acaa3648f 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -66,12 +66,12 @@ object Scenarios { "overCreationOvershoot" to (created - cfg.maxIdle).coerceAtLeast(0), "warmupPipeline" to mapOf( - "queueWaitMs" to diagnostics.queueWaitMs, - "createDurationMs" to diagnostics.createDurationMs, - "commitDurationMs" to diagnostics.commitDurationMs, - "tickIntervalMs" to diagnostics.tickIntervalMs, - "tickDurationMs" to diagnostics.tickDurationMs, - "submitBurst" to diagnostics.submitBurst, + "queueWaitMs" to phaseStats(diagnostics.queueWaitMs), + "createDurationMs" to phaseStats(diagnostics.createDurationMs), + "commitDurationMs" to phaseStats(diagnostics.commitDurationMs), + "tickIntervalMs" to phaseStats(diagnostics.tickIntervalMs), + "tickDurationMs" to phaseStats(diagnostics.tickDurationMs), + "submitBurst" to phaseStats(diagnostics.submitBurst), "submitCalls" to diagnostics.submitCalls, "inFlightPeak" to diagnostics.inFlightPeak, "inFlightMean" to diagnostics.inFlightMean, @@ -570,6 +570,15 @@ object Scenarios { // ---------- helpers ---------- + private fun phaseStats(s: PoolWarmupDiagnostics.PhaseStats): Map = + mapOf( + "count" to s.count, + "meanMs" to s.meanMs, + "p50Ms" to s.p50Ms, + "p95Ms" to s.p95Ms, + "maxMs" to s.maxMs, + ) + private fun successRate(stats: LatencyStats): Double { val total = stats.n + stats.failures return if (total == 0L) 0.0 else (stats.n.toDouble() / total) From cc3903d9076121755d6c30a0f8c989cef078d301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 15:28:02 +0800 Subject: [PATCH 12/16] docs(benchmark): document warmup connection-churn problem and shared pool guidance Adds --shared-connection-pool-size to the driver options table and a 'Warmup throughput and connection reuse' section describing the connection-reset/retry-amplification finding at high warmupConcurrency, the pool-size sweep evidence, and configuration guidance for benchmark and production (including the upcoming SDK default from #1517). --- tests/benchmark/README.md | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index 67ca5e4bf..cb288a139 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -211,6 +211,7 @@ cd kotlin | `--report-dir` | `results/run-` | Report output directory (`run.sh` passes an absolute path) | | `--max-idle` | `20` | Pool idle-buffer target | | `--warmup-concurrency` | `4` | Concurrent warmup creation workers | +| `--shared-connection-pool-size` | `0` | Inject one shared OkHttp `ConnectionPool` with this many idle slots across all sandbox clients (`0` = each sandbox uses its own fresh connections). See "Warmup throughput and connection reuse" below | | `--reconcile-interval-ms` | `1000` | Pool reconcile tick interval | | `--idle-timeout-s` | `1800` | Server-side TTL applied to pool-created sandboxes | | `--acquire-min-remaining-ttl-s` | `0` | Idle entries with less remaining TTL than this are discarded on acquire; `0` = SDK auto default (`min(60s, idleTimeout/2)`) | @@ -279,6 +280,53 @@ want to measure pure idle-hit latency: when workers outnumber the idle buffer, acquires drain it and fall through to direct create (which the `hitRatio` column will show). +### Warmup throughput and connection reuse + +**Problem.** The Kotlin SDK creates a fresh OkHttp client (and fresh TCP +connections) for every sandbox — each warmup create opens ~2-4 connections +(create + endpoint lookups + execd ping + renew). At high +`warmupConcurrency` the resulting connection burst can exceed what the server +listener can absorb (e.g. macOS accept backlog 128), producing intermittent +TCP-level `Connection reset`/`Broken pipe` failures. Failed warmups are +retried (backoff-gated), amplifying attempts several-fold and making fills +slower and burstier than at lower concurrency — measured at `wc=1000`: +**~80% of warmup attempts failed, 5x attempt amplification** (see the +`warmupPipeline.createFailures` block in `cold-start` reports). + +**Fix: share one connection pool.** A shared `ConnectionPool` sized to the +warmup concurrency makes warmup creates reuse connections instead of opening +new ones. Measured fill of 2000 idles at `wc=1000`: + +| shared pool size | fill | create failures | attempt amplification | +|---|---|---|---| +| 0 (baseline) | ~20s | ~8000 | 5x | +| 100 | ~9s | ~2700 | 2.3x | +| 200 | ~7s | ~1300 | 1.7x | +| 500 | ~4s | 0 | 1x | + +**Configuration guidance.** + +- In the benchmark, pass `--shared-connection-pool-size ` (rule of thumb: + `max(warmupConcurrency, 200)`, i.e. ~1:1 with the warmup workers). +- In production, the pool already accepts a shared pool through the standard + `ConnectionConfig` (no SDK change needed): + + ```kotlin + ConnectionConfig.builder() + .connectionPool(ConnectionPool(500, 5, TimeUnit.MINUTES)) // ~= warmupConcurrency + .build() + ``` + + The pool uses this config for every sandbox it creates (warmup, direct + create, idle connect); a user-provided pool is never evicted by the SDK. +- `warmupConcurrency` beyond ~200-300 only pays off together with a shared + pool: without reuse the extra threads mostly produce connection-reset + retries. If you cannot share connections, keep `warmupConcurrency` in the + 200-300 range. +- This will become the SDK default behavior (pool-created shared pool sized + by `warmupConcurrency`) once the companion SDK change is merged + (opensandbox-group/OpenSandbox#1517); until then configure it explicitly. + ### What is measured | Concern | Where it shows up | From fae4b742766b2e7a3fa174a1494bd35c0f792163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 15:29:31 +0800 Subject: [PATCH 13/16] docs: document connection reuse at high warmup concurrency in the client pool guide Move the problem description, sweep evidence, and configuration guidance (shared ConnectionPool via ConnectionConfig, sizing rule of thumb, upcoming SDK default) to docs/guides/client-pool.md. The benchmark README keeps only a pointer plus the --shared-connection-pool-size repro flag. --- docs/guides/client-pool.md | 44 +++++++++++++++++++++++++++++++ tests/benchmark/README.md | 53 +++++++------------------------------- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/docs/guides/client-pool.md b/docs/guides/client-pool.md index f9b36a431..0edd1535e 100644 --- a/docs/guides/client-pool.md +++ b/docs/guides/client-pool.md @@ -136,6 +136,50 @@ canonical reference; refer to the per-language builder or constructor for exact old-template sandbox IDs into the shared buffer during a rolling deploy. - `resize(max_idle)` and `release_all_idle()` can be called from any node. +### Connection reuse at high warmup concurrency + +**Problem.** Pool-created sandboxes go through the SDK transport per sandbox, and by +default each sandbox's HTTP client opens its own TCP connections — a warmup create +typically uses 2-4 fresh connections (create + endpoint lookups + readiness probe + +renew). At high `warmup_concurrency` the resulting connection burst can exceed what the +server listener can absorb (for example macOS accept backlog 128), producing intermittent +TCP-level `Connection reset` / `Broken pipe` failures. Failed warmups are retried +(backoff-gated), amplifying attempts several-fold and making fills slower and burstier +than at lower concurrency — measured in the Kotlin SDK benchmark harness at +`warmup_concurrency=1000`: ~80% of warmup attempts failed with 5x attempt amplification. + +**Fix: share one transport connection pool.** Warmup creates then reuse connections +instead of opening new ones. Measured fill of 2000 idles at `warmup_concurrency=1000` +(Kotlin SDK, mock server): + +| shared pool size | fill time | create failures | attempt amplification | +|---|---|---|---| +| 0 (per-sandbox connections) | ~20 s | ~8000 | 5x | +| 100 | ~9 s | ~2700 | 2.3x | +| 200 | ~7 s | ~1300 | 1.7x | +| 500 | ~4 s | 0 | 1x | + +**Kotlin/Java.** Inject a shared `okhttp3.ConnectionPool` through the standard +`ConnectionConfig` — no pool-specific option is needed: + +```kotlin +ConnectionConfig.builder() + .connectionPool(ConnectionPool(500, 5, TimeUnit.MINUTES)) // ~= warmup_concurrency + .build() +``` + +The pool uses this config for every sandbox it creates (warmup, direct create, idle +connect). A user-provided pool is treated as user-managed and is never evicted by the +SDK. A pool-created shared pool sized by `warmup_concurrency` will become the SDK +default once the companion change lands; until then configure it explicitly. + +**Rule of thumb.** `warmup_concurrency` beyond ~200-300 only pays off together with a +shared connection pool — without reuse the extra threads mostly produce +connection-reset retries. If connections cannot be shared, keep +`warmup_concurrency` in the 200-300 range. The same principle applies to any SDK whose +transport opens connections per sandbox; see the language-local transport docs for how +to share a connection pool. + ## Minimal usage ### Python (sync) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index cb288a139..c276a4be2 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -282,50 +282,15 @@ column will show). ### Warmup throughput and connection reuse -**Problem.** The Kotlin SDK creates a fresh OkHttp client (and fresh TCP -connections) for every sandbox — each warmup create opens ~2-4 connections -(create + endpoint lookups + execd ping + renew). At high -`warmupConcurrency` the resulting connection burst can exceed what the server -listener can absorb (e.g. macOS accept backlog 128), producing intermittent -TCP-level `Connection reset`/`Broken pipe` failures. Failed warmups are -retried (backoff-gated), amplifying attempts several-fold and making fills -slower and burstier than at lower concurrency — measured at `wc=1000`: -**~80% of warmup attempts failed, 5x attempt amplification** (see the -`warmupPipeline.createFailures` block in `cold-start` reports). - -**Fix: share one connection pool.** A shared `ConnectionPool` sized to the -warmup concurrency makes warmup creates reuse connections instead of opening -new ones. Measured fill of 2000 idles at `wc=1000`: - -| shared pool size | fill | create failures | attempt amplification | -|---|---|---|---| -| 0 (baseline) | ~20s | ~8000 | 5x | -| 100 | ~9s | ~2700 | 2.3x | -| 200 | ~7s | ~1300 | 1.7x | -| 500 | ~4s | 0 | 1x | - -**Configuration guidance.** - -- In the benchmark, pass `--shared-connection-pool-size ` (rule of thumb: - `max(warmupConcurrency, 200)`, i.e. ~1:1 with the warmup workers). -- In production, the pool already accepts a shared pool through the standard - `ConnectionConfig` (no SDK change needed): - - ```kotlin - ConnectionConfig.builder() - .connectionPool(ConnectionPool(500, 5, TimeUnit.MINUTES)) // ~= warmupConcurrency - .build() - ``` - - The pool uses this config for every sandbox it creates (warmup, direct - create, idle connect); a user-provided pool is never evicted by the SDK. -- `warmupConcurrency` beyond ~200-300 only pays off together with a shared - pool: without reuse the extra threads mostly produce connection-reset - retries. If you cannot share connections, keep `warmupConcurrency` in the - 200-300 range. -- This will become the SDK default behavior (pool-created shared pool sized - by `warmupConcurrency`) once the companion SDK change is merged - (opensandbox-group/OpenSandbox#1517); until then configure it explicitly. +At high `warmupConcurrency`, per-sandbox HTTP clients opening fresh TCP connections +cause intermittent `Connection reset` failures and retry amplification (see +`warmupPipeline.createFailures` in `cold-start` reports). The problem, evidence, and +configuration guidance (including production `ConnectionConfig` setup) are documented +in the [client pool guide](/guides/client-pool#connection-reuse-at-high-warmup-concurrency). + +In the benchmark, reproduce or verify it with `--shared-connection-pool-size ` +(rule of thumb: `max(warmupConcurrency, 200)`); the flag injects one shared OkHttp +`ConnectionPool` with that many idle slots across all sandbox clients. ### What is measured From 8ba6040d4d4e9fa76d2be8f1b97ce27eb564d368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 15:32:25 +0800 Subject: [PATCH 14/16] feat(benchmark): enable the shared connection pool by default (auto-sized) --shared-connection-pool-size now defaults to auto = max(warmupConcurrency, 200) instead of 0, matching the guidance documented in docs/guides/client-pool.md. Explicit 0 disables sharing (reproduces the connection-reset pathology); N sweeps pool sizes. Report shows the resolved mode ('auto'/0/N). --- tests/benchmark/README.md | 9 +++++---- .../kotlin/com/alibaba/opensandbox/benchmark/Cli.kt | 12 +++++++----- .../kotlin/com/alibaba/opensandbox/benchmark/Main.kt | 2 +- .../com/alibaba/opensandbox/benchmark/PoolRunner.kt | 9 +++++++-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index c276a4be2..c3ed3dad1 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -211,7 +211,7 @@ cd kotlin | `--report-dir` | `results/run-` | Report output directory (`run.sh` passes an absolute path) | | `--max-idle` | `20` | Pool idle-buffer target | | `--warmup-concurrency` | `4` | Concurrent warmup creation workers | -| `--shared-connection-pool-size` | `0` | Inject one shared OkHttp `ConnectionPool` with this many idle slots across all sandbox clients (`0` = each sandbox uses its own fresh connections). See "Warmup throughput and connection reuse" below | +| `--shared-connection-pool-size` | `auto` | Inject one shared OkHttp `ConnectionPool` across all sandbox clients. `auto` = `max(warmupConcurrency, 200)` idle slots (recommended; matches the guidance in the [client pool guide](/guides/client-pool#connection-reuse-at-high-warmup-concurrency)); `0` = per-sandbox fresh connections (reproduces the connection-reset pathology); `N` = explicit idle slots | | `--reconcile-interval-ms` | `1000` | Pool reconcile tick interval | | `--idle-timeout-s` | `1800` | Server-side TTL applied to pool-created sandboxes | | `--acquire-min-remaining-ttl-s` | `0` | Idle entries with less remaining TTL than this are discarded on acquire; `0` = SDK auto default (`min(60s, idleTimeout/2)`) | @@ -288,9 +288,10 @@ cause intermittent `Connection reset` failures and retry amplification (see configuration guidance (including production `ConnectionConfig` setup) are documented in the [client pool guide](/guides/client-pool#connection-reuse-at-high-warmup-concurrency). -In the benchmark, reproduce or verify it with `--shared-connection-pool-size ` -(rule of thumb: `max(warmupConcurrency, 200)`); the flag injects one shared OkHttp -`ConnectionPool` with that many idle slots across all sandbox clients. +In the benchmark, the shared pool is **on by default** (auto-sized to +`max(warmupConcurrency, 200)`), so high-concurrency runs already follow the +guidance. Pass `--shared-connection-pool-size 0` to reproduce the +per-sandbox-connection pathology, or an explicit `N` to sweep pool sizes. ### What is measured diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 65cfdb5d9..8429c7ff6 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -51,7 +51,7 @@ data class BenchmarkConfig( val staleRetries: Int, val staleAcquireReadyTimeoutMs: Long, val stalePoisonRate: Double, - val sharedConnectionPoolSize: Int, + val sharedConnectionPoolSize: Int?, val idleExpiryIdleTimeoutS: Long, val idleExpiryDurationS: Int, ) { @@ -149,10 +149,12 @@ object Cli { staleAcquireReadyTimeoutMs = (map["stale-acquire-ready-timeout-ms"] ?: "3000").toLong(), // fraction (0..1] of idle sandboxes to poison in stale-idle; 1.0 = poison all stalePoisonRate = (map["stale-poison-rate"] ?: "1.0").toDouble(), - // Inject a shared OkHttp ConnectionPool with this many idle - // connections across all sandbox clients (0 = each sandbox keeps - // its own fresh connections). Diagnostic: tests connection reuse. - sharedConnectionPoolSize = (map["shared-connection-pool-size"] ?: "0").toInt(), + // Inject a shared OkHttp ConnectionPool across all sandbox + // clients. null = auto-size to max(warmupConcurrency, 200); + // 0 = each sandbox keeps its own fresh connections (reproduces + // the connection-reset pathology at high concurrency); N = that + // many idle slots. + sharedConnectionPoolSize = map["shared-connection-pool-size"]?.toInt(), idleExpiryIdleTimeoutS = (map["idle-expiry-idle-timeout-s"] ?: "20").toLong(), idleExpiryDurationS = (map["idle-expiry-duration-s"] ?: "40").toInt(), ) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index f83474774..cfac9a8b2 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -135,7 +135,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "failureCreateRate" to cfg.failureCreateRate, "staleRetries" to cfg.staleRetries, "stalePoisonRate" to cfg.stalePoisonRate, - "sharedConnectionPoolSize" to cfg.sharedConnectionPoolSize, + "sharedConnectionPoolSize" to (cfg.sharedConnectionPoolSize ?: "auto"), ) private fun toJsonElement(value: Any?): JsonElement = diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt index 55f04cd5b..5a0eda687 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -48,10 +48,15 @@ object PoolRunner { .requestTimeout(Duration.ofSeconds(30)) .disableMetrics() .also { builder -> - if (cfg.sharedConnectionPoolSize > 0) { + // Shared connection pool by default (auto-sized to the + // warmup concurrency) so high-concurrency runs do not hit + // the per-sandbox fresh-connection churn documented in + // docs/guides/client-pool.md. Explicit 0 disables sharing. + val sharedPoolSize = cfg.sharedConnectionPoolSize ?: maxOf(cfg.warmupConcurrency, 200) + if (sharedPoolSize > 0) { builder.connectionPool( okhttp3.ConnectionPool( - cfg.sharedConnectionPoolSize, + sharedPoolSize, 5, TimeUnit.MINUTES, ), From 9c3ecc7d36424c805cde1774862a0f385adfffe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 15:38:37 +0800 Subject: [PATCH 15/16] feat(benchmark): fixed 500-idle-slot shared pool by default Replace the auto-derived pool size with a plain fixed default (500 idle slots, 5min keep-alive) so the benchmark follows the documented guidance without any derivation logic. 0 still disables sharing to reproduce the connection-reset pathology; N sweeps sizes. --- tests/benchmark/README.md | 10 +++++----- .../kotlin/com/alibaba/opensandbox/benchmark/Cli.kt | 11 +++++------ .../kotlin/com/alibaba/opensandbox/benchmark/Main.kt | 2 +- .../com/alibaba/opensandbox/benchmark/PoolRunner.kt | 11 +++++------ 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index c3ed3dad1..2b286aee8 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -211,7 +211,7 @@ cd kotlin | `--report-dir` | `results/run-` | Report output directory (`run.sh` passes an absolute path) | | `--max-idle` | `20` | Pool idle-buffer target | | `--warmup-concurrency` | `4` | Concurrent warmup creation workers | -| `--shared-connection-pool-size` | `auto` | Inject one shared OkHttp `ConnectionPool` across all sandbox clients. `auto` = `max(warmupConcurrency, 200)` idle slots (recommended; matches the guidance in the [client pool guide](/guides/client-pool#connection-reuse-at-high-warmup-concurrency)); `0` = per-sandbox fresh connections (reproduces the connection-reset pathology); `N` = explicit idle slots | +| `--shared-connection-pool-size` | `500` | Inject one shared OkHttp `ConnectionPool` with this many idle slots across all sandbox clients (matches the guidance in the [client pool guide](/guides/client-pool#connection-reuse-at-high-warmup-concurrency)); `0` = per-sandbox fresh connections (reproduces the connection-reset pathology) | | `--reconcile-interval-ms` | `1000` | Pool reconcile tick interval | | `--idle-timeout-s` | `1800` | Server-side TTL applied to pool-created sandboxes | | `--acquire-min-remaining-ttl-s` | `0` | Idle entries with less remaining TTL than this are discarded on acquire; `0` = SDK auto default (`min(60s, idleTimeout/2)`) | @@ -288,10 +288,10 @@ cause intermittent `Connection reset` failures and retry amplification (see configuration guidance (including production `ConnectionConfig` setup) are documented in the [client pool guide](/guides/client-pool#connection-reuse-at-high-warmup-concurrency). -In the benchmark, the shared pool is **on by default** (auto-sized to -`max(warmupConcurrency, 200)`), so high-concurrency runs already follow the -guidance. Pass `--shared-connection-pool-size 0` to reproduce the -per-sandbox-connection pathology, or an explicit `N` to sweep pool sizes. +In the benchmark, the shared pool is **on by default** (fixed 500 idle +slots), so high-concurrency runs already follow the guidance. Pass +`--shared-connection-pool-size 0` to reproduce the per-sandbox-connection +pathology, or an explicit `N` to sweep pool sizes. ### What is measured diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 8429c7ff6..7f3043cd8 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -51,7 +51,7 @@ data class BenchmarkConfig( val staleRetries: Int, val staleAcquireReadyTimeoutMs: Long, val stalePoisonRate: Double, - val sharedConnectionPoolSize: Int?, + val sharedConnectionPoolSize: Int, val idleExpiryIdleTimeoutS: Long, val idleExpiryDurationS: Int, ) { @@ -150,11 +150,10 @@ object Cli { // fraction (0..1] of idle sandboxes to poison in stale-idle; 1.0 = poison all stalePoisonRate = (map["stale-poison-rate"] ?: "1.0").toDouble(), // Inject a shared OkHttp ConnectionPool across all sandbox - // clients. null = auto-size to max(warmupConcurrency, 200); - // 0 = each sandbox keeps its own fresh connections (reproduces - // the connection-reset pathology at high concurrency); N = that - // many idle slots. - sharedConnectionPoolSize = map["shared-connection-pool-size"]?.toInt(), + // clients (fixed default 500 idle slots; 0 = each sandbox keeps + // its own fresh connections, reproducing the connection-reset + // pathology at high concurrency; N = that many idle slots). + sharedConnectionPoolSize = (map["shared-connection-pool-size"] ?: "500").toInt(), idleExpiryIdleTimeoutS = (map["idle-expiry-idle-timeout-s"] ?: "20").toLong(), idleExpiryDurationS = (map["idle-expiry-duration-s"] ?: "40").toInt(), ) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index cfac9a8b2..f83474774 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -135,7 +135,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "failureCreateRate" to cfg.failureCreateRate, "staleRetries" to cfg.staleRetries, "stalePoisonRate" to cfg.stalePoisonRate, - "sharedConnectionPoolSize" to (cfg.sharedConnectionPoolSize ?: "auto"), + "sharedConnectionPoolSize" to cfg.sharedConnectionPoolSize, ) private fun toJsonElement(value: Any?): JsonElement = diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt index 5a0eda687..f14973ed3 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/PoolRunner.kt @@ -48,15 +48,14 @@ object PoolRunner { .requestTimeout(Duration.ofSeconds(30)) .disableMetrics() .also { builder -> - // Shared connection pool by default (auto-sized to the - // warmup concurrency) so high-concurrency runs do not hit - // the per-sandbox fresh-connection churn documented in + // Shared connection pool by default (fixed 500 idle slots) + // so high-concurrency runs do not hit the per-sandbox + // fresh-connection churn documented in // docs/guides/client-pool.md. Explicit 0 disables sharing. - val sharedPoolSize = cfg.sharedConnectionPoolSize ?: maxOf(cfg.warmupConcurrency, 200) - if (sharedPoolSize > 0) { + if (cfg.sharedConnectionPoolSize > 0) { builder.connectionPool( okhttp3.ConnectionPool( - sharedPoolSize, + cfg.sharedConnectionPoolSize, 5, TimeUnit.MINUTES, ), From f367f7b033b584a4a3293c07eca7c5658f16d1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 14 Aug 2026 16:28:53 +0800 Subject: [PATCH 16/16] feat(benchmark): steady-start-immediately flag for startup-under-load scenario Loaders race the pool fill instead of waiting for idle; fixes the CLI parser for valueless boolean flags (previously swallowed the next --key as its value). --- .../com/alibaba/opensandbox/benchmark/Cli.kt | 18 +++++++++++++++--- .../com/alibaba/opensandbox/benchmark/Main.kt | 1 + .../alibaba/opensandbox/benchmark/Scenarios.kt | 10 +++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt index 7f3043cd8..ceb8ffade 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Cli.kt @@ -41,6 +41,7 @@ data class BenchmarkConfig( val steadyWorkers: Int, val steadyDurationS: Int, val acquireRatePerMin: Int, + val steadyStartImmediately: Boolean, val holdMinMs: Long, val holdMaxMs: Long, val replenishRounds: Int, @@ -81,6 +82,7 @@ object Cli { "steady-workers", "steady-duration-s", "acquire-rate-per-min", + "steady-start-immediately", "hold-min-ms", "hold-max-ms", "replenish-rounds", @@ -105,12 +107,18 @@ object Cli { throw IllegalArgumentException("unexpected argument: $key") } val name = key.removePrefix("--") - val value = args.getOrNull(i + 1) ?: throw IllegalArgumentException("missing value for $key") if (name !in allKeys) { throw IllegalArgumentException("unknown option: $key") } - map[name] = value - i += 2 + // Boolean flags may be passed without a value (--flag == --flag true). + val next = args.getOrNull(i + 1) + if (next == null || next.startsWith("--")) { + map[name] = "true" + i += 1 + } else { + map[name] = next + i += 2 + } } return BenchmarkConfig( mockBaseUrl = map["mock-base-url"] ?: "http://127.0.0.1:18080", @@ -138,6 +146,10 @@ object Cli { // 0 = unlimited (workers run back-to-back); > 0 paces acquires // evenly across each minute at this many acquires per minute. acquireRatePerMin = (map["acquire-rate-per-min"] ?: "0").toInt(), + // Start loaders immediately after pool.start() instead of waiting + // for the idle buffer to fill (pool startup races high-frequency + // acquire — extreme cold-start-under-load scenario). + steadyStartImmediately = (map["steady-start-immediately"] ?: "false").toBoolean(), holdMinMs = (map["hold-min-ms"] ?: "1000").toLong(), holdMaxMs = (map["hold-max-ms"] ?: "5000").toLong(), replenishRounds = (map["replenish-rounds"] ?: "20").toInt(), diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt index f83474774..5f0c38f32 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Main.kt @@ -130,6 +130,7 @@ private fun cfgValues(cfg: BenchmarkConfig): Map = "steadyWorkers" to cfg.steadyWorkers, "steadyDurationS" to cfg.steadyDurationS, "acquireRatePerMin" to cfg.acquireRatePerMin, + "steadyStartImmediately" to cfg.steadyStartImmediately, "holdMinMs" to cfg.holdMinMs, "holdMaxMs" to cfg.holdMaxMs, "failureCreateRate" to cfg.failureCreateRate, diff --git a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt index acaa3648f..f0994626c 100644 --- a/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt +++ b/tests/benchmark/kotlin/src/main/kotlin/com/alibaba/opensandbox/benchmark/Scenarios.kt @@ -139,7 +139,14 @@ object Scenarios { mock.reset() val pool = PoolRunner.build(cfg, "steady-state") pool.start() - val fillMs = PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + // Extreme-cold-start mode: loaders race the fill instead of waiting + // for the idle buffer (fillMs = -1 in that case). + val fillMs = + if (cfg.steadyStartImmediately) { + -1L + } else { + PoolRunner.waitForIdle(pool, cfg.maxIdle, cfg.coldStartTimeoutMs) + } val createdBefore = num(mock.stats(), "stats.created") val killedBefore = num(mock.stats(), "stats.killed") @@ -192,6 +199,7 @@ object Scenarios { val idleStat = client["poolIdleCount"] as Map return mapOf( "fillTimeMs" to fillMs, + "startImmediately" to cfg.steadyStartImmediately, "durationMs" to durationMs, "actualLoaderDurationMs" to loaderDurationMs, "workers" to cfg.steadyWorkers,