From 27c838e8696610616a6593926df0835e00bd8428 Mon Sep 17 00:00:00 2001 From: Avi Basnet Date: Fri, 4 Sep 2026 00:29:24 +0000 Subject: [PATCH] [tinker] Survive completion bursts at the socket layer uvicorn.run: backlog=SKYRL_HTTP_CONNECTION_LIMIT (50k; the effective value is capped by net.core.somaxconn, raise it to match) and timeout_keep_alive=75s. With 131072 outstanding samples and 212k-token results each 2048-result completion burst kept the event loop busy ~16s. uvicorn's 5s keep-alive then closed every idle client connection during the burst, every client reconnected at once, and the 2048-entry accept backlog overflowed, so the kernel refused 109k of 131072 requests. Connections now queue in the kernel while the loop is busy and idle ones survive a burst. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Avi Basnet --- skyrl/tinker/api.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/skyrl/tinker/api.py b/skyrl/tinker/api.py index 22c6a5b1a9..a7e4e1f450 100644 --- a/skyrl/tinker/api.py +++ b/skyrl/tinker/api.py @@ -32,6 +32,7 @@ from sqlmodel import SQLModel, func, select from sqlmodel.ext.asyncio.session import AsyncSession +from skyrl.env_vars import SKYRL_HTTP_CONNECTION_LIMIT from skyrl.tinker import types from skyrl.tinker.config import EngineConfig, add_model, config_to_argv from skyrl.tinker.db_models import ( @@ -71,6 +72,12 @@ # How long retrieve_future waits for a result before returning 408 RETRIEVE_FUTURE_TIMEOUT_SECONDS = 300 +# Idle keep-alive for client connections. Under a burst of completions the +# event loop can be busy for many seconds; with uvicorn's 5s default every +# idle SDK connection is closed during such a burst and all clients reconnect +# at once, overflowing the accept backlog. Hold connections across bursts. +HTTP_KEEP_ALIVE_TIMEOUT_SECONDS = 75 + # How often poll_futures looks for newly finished requests. A single query # covers every waiter, so this can stay tight without the load scaling up with # the number of in-flight requests. @@ -1854,4 +1861,13 @@ async def root(): # Store config in app.state so lifespan can access it app.state.engine_config = engine_config - uvicorn.run(app, host=args.host, port=args.port, log_config=get_uvicorn_log_config()) + uvicorn.run( + app, + host=args.host, + port=args.port, + log_config=get_uvicorn_log_config(), + # Pending connections queue in the kernel while the loop is busy instead + # of being refused (effective value is capped by net.core.somaxconn). + backlog=SKYRL_HTTP_CONNECTION_LIMIT, + timeout_keep_alive=HTTP_KEEP_ALIVE_TIMEOUT_SECONDS, + )