diff --git a/pyproject.toml b/pyproject.toml index e1309f3159..47a52ab3d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ tpu = [ tinker = [ "tinker>=0.3.0,<=0.24.1", "fastapi[standard]", + # Decodes vLLM completion bodies into numeric buffers without per-token + # Python objects (skyrl/tinker/extra/completion_decode.py). + "pysimdjson", "sqlmodel", "sqlalchemy[asyncio]", "aiosqlite", diff --git a/skyrl/benchmarks/load_test_tinker_sampling.py b/skyrl/benchmarks/load_test_tinker_sampling.py index a00da01b8a..e4c014a643 100644 --- a/skyrl/benchmarks/load_test_tinker_sampling.py +++ b/skyrl/benchmarks/load_test_tinker_sampling.py @@ -1,6 +1,12 @@ #!/usr/bin/env python3 """Load test: is the Tinker API server the bottleneck at N concurrent sample requests? +Structure follows Chuck Tang's SQLite QueuePool repro +(https://gist.github.com/j316chuck/f44f35572ffb8584519d13b943f99ef8): a barrier +fake vLLM, the real API server, and a Tinker-SDK-shaped client, extended with +orchestration, server profiling/monitoring, payload sizing and a raw client that +can exceed the SDK's in-flight cap. + Exercises the API server layers on the non-colocated SkyRL-Train path (``backend=megatron``, ``trainer.placement.colocate_all=false``): FastAPI + uvicorn, the SQLite-backed session/model tables, the in-memory @@ -75,6 +81,8 @@ # The Tinker SDK gives up on each retrieve_future poll after 45s and re-polls # the same request_id (the server would hold it for up to 300s). SDK_RETRIEVE_POLL_TIMEOUT_SECONDS = 45.0 +# The SDK gives up on a request after this many consecutive connection errors. +SDK_MAX_CONNECTION_ERROR_RETRIES = 16 # --------------------------------------------------------------------------- # @@ -82,13 +90,75 @@ # --------------------------------------------------------------------------- # +class _SharedCounters: + """Request counters shared by the fake router's worker processes.""" + + def __init__(self) -> None: + import multiprocessing + + self._lock = multiprocessing.Lock() + self._values = { + k: multiprocessing.Value("q", 0, lock=False) for k in ("received", "completed", "in_flight", "peak") + } + + def enter(self) -> None: + with self._lock: + self._values["received"].value += 1 + self._values["in_flight"].value += 1 + self._values["peak"].value = max(self._values["peak"].value, self._values["in_flight"].value) + + def exit(self, completed: bool) -> None: + with self._lock: + self._values["in_flight"].value -= 1 + if completed: + self._values["completed"].value += 1 + + def snapshot(self) -> dict[str, int]: + with self._lock: + v = self._values + return { + "received": v["received"].value, + "completed": v["completed"].value, + "in_flight": v["in_flight"].value, + "peak_in_flight": v["peak"].value, + } + + def run_fake_vllm(args: argparse.Namespace) -> None: + """Serve the fake router; with --router-workers > 1, fork workers sharing the port (SO_REUSEPORT).""" + import multiprocessing + + counters = _SharedCounters() + workers = max(1, args.router_workers) + if workers == 1: + _serve_fake_vllm(args, counters, worker_index=0, workers=1) + return + procs = [ + multiprocessing.Process(target=_serve_fake_vllm, args=(args, counters, i, workers), daemon=True) + for i in range(workers) + ] + for proc in procs: + proc.start() + + def terminate_workers(signum, frame): + # SIGTERM from the orchestrator would otherwise leave the workers + # orphaned (daemon cleanup only runs on a normal exit). + for proc in procs: + proc.terminate() + raise SystemExit(0) + + signal.signal(signal.SIGTERM, terminate_workers) + for proc in procs: + proc.join() + + +def _serve_fake_vllm(args: argparse.Namespace, counters: _SharedCounters, worker_index: int, workers: int) -> None: from aiohttp import web - state = {"received": 0, "completed": 0, "in_flight": 0, "peak_in_flight": 0} barrier_requests = args.barrier_requests or args.num_requests release = asyncio.Event() - admit = asyncio.Semaphore(args.max_num_seqs) + # Each worker admits its share of the engine's concurrent sequences. + admit = asyncio.Semaphore(max(1, args.max_num_seqs // workers)) body_cache: dict[tuple[int, int], bytes] = {} def body_for(n: int, max_tokens: int) -> bytes: @@ -107,33 +177,33 @@ async def barrier_timeout_watch() -> None: if not release.is_set(): print( f"[fake-vllm] barrier timeout after {args.barrier_timeout}s with " - f"{state['received']}/{barrier_requests} received -- releasing", + f"{counters.snapshot()['received']}/{barrier_requests} received -- releasing", flush=True, ) release.set() async def completions(request: web.Request) -> web.Response: payload = await request.json() - state["received"] += 1 - state["in_flight"] += 1 - state["peak_in_flight"] = max(state["peak_in_flight"], state["in_flight"]) + counters.enter() + completed = False try: if args.vllm_mode == "barrier": - if state["received"] >= barrier_requests: + # Barrier mode is single-process: the release is local state. + if counters.snapshot()["received"] >= barrier_requests: release.set() await release.wait() else: async with admit: await asyncio.sleep(args.gen_seconds) - state["completed"] += 1 + completed = True return web.Response( body=body_for(payload.get("n", 1), payload["max_tokens"]), content_type="application/json" ) finally: - state["in_flight"] -= 1 + counters.exit(completed) async def stats(_: web.Request) -> web.Response: - return web.json_response(state) + return web.json_response(counters.snapshot()) async def serve() -> None: app = web.Application(client_max_size=64 * 1024 * 1024) @@ -142,11 +212,15 @@ async def serve() -> None: app.router.add_get("/stats", stats) runner = web.AppRunner(app, access_log=None) await runner.setup() - site = web.TCPSite(runner, "127.0.0.1", args.vllm_port, backlog=65535) + site = web.TCPSite(runner, "127.0.0.1", args.vllm_port, backlog=65535, reuse_port=workers > 1) await site.start() if args.vllm_mode == "barrier": asyncio.create_task(barrier_timeout_watch()) - print(f"[fake-vllm] mode={args.vllm_mode} listening on 127.0.0.1:{args.vllm_port}", flush=True) + if worker_index == 0: + print( + f"[fake-vllm] mode={args.vllm_mode} workers={workers} listening on 127.0.0.1:{args.vllm_port}", + flush=True, + ) await asyncio.Event().wait() asyncio.run(serve()) @@ -197,13 +271,24 @@ def record_error(kind: str, detail: str) -> None: # (its sample dispatch semaphore); results are then awaited with one # long-poll each. Mirror that so the submit burst matches production. submit_gate = asyncio.Semaphore(max(1, args.submit_concurrency)) + # The SDK keeps at most sample_max_concurrent_requests samples in flight + # per SamplingClient (submit through result) and queues the rest. + outstanding_gate = asyncio.Semaphore(max(1, args.max_outstanding)) if args.max_outstanding else None async def one(i: int) -> None: + if outstanding_gate is None: + await _one(i) + else: + async with outstanding_gate: + await _one(i) + + async def _one(i: int) -> None: nonlocal retries_408, reconnects, asample_retries, poll_timeouts, completed t0 = time.monotonic() phase = "asample" try: async with submit_gate: + connection_failures = 0 while True: try: async with session.post(f"{base}/asample", json=payload) as resp: @@ -212,17 +297,25 @@ async def one(i: int) -> None: return request_id = (await resp.json())["request_id"] break - except (aiohttp.ServerDisconnectedError, aiohttp.ClientOSError) as e: - # The SDK re-sends an asample whose connection was - # reset (the server may or may not have accepted - # the first copy). Mirror that, once per request. - if asample_retries >= count: - record_error("asample_retry_budget_exhausted", str(e)) + except ( + aiohttp.ServerDisconnectedError, + aiohttp.ClientOSError, + aiohttp.ClientConnectorError, + asyncio.TimeoutError, + ) as e: + # The SDK re-sends an asample whose connection failed + # or timed out (the server may or may not have + # accepted the first copy), up to 16 times with + # exponential backoff. + connection_failures += 1 + if connection_failures > SDK_MAX_CONNECTION_ERROR_RETRIES: + record_error("asample_connection_retries_exhausted", str(e)) return asample_retries += 1 - await asyncio.sleep(0.05) + await asyncio.sleep(min(2 ** (connection_failures - 1), 30)) submit_latency.append(time.monotonic() - t0) phase = "retrieve" + retrieve_failures = 0 while True: if time.monotonic() - t0 > args.request_timeout: record_error("request_timeout", f"no result after {args.request_timeout}s") @@ -244,16 +337,21 @@ async def one(i: int) -> None: # and immediately re-polls the same request_id. poll_timeouts += 1 continue - except (aiohttp.ServerDisconnectedError, aiohttp.ClientOSError) as e: - # The server dropped the connection (e.g. uvicorn's - # keep-alive timer firing during an event-loop stall). - # retrieve_future is idempotent and the Tinker SDK - # retries it, so do the same. + except ( + aiohttp.ServerDisconnectedError, + aiohttp.ClientOSError, + aiohttp.ClientConnectorError, + ) as e: + # The server dropped or refused the connection (e.g. + # uvicorn's keep-alive timer firing during an event-loop + # stall, or a full accept backlog). retrieve_future is + # idempotent; the SDK retries with backoff, so do the same. reconnects += 1 - if reconnects > args.max_reconnects * count: - record_error("retrieve_reconnect_budget_exhausted", str(e)) + retrieve_failures += 1 + if retrieve_failures > SDK_MAX_CONNECTION_ERROR_RETRIES: + record_error("retrieve_connection_retries_exhausted", str(e)) return - await asyncio.sleep(0.05) + await asyncio.sleep(min(2 ** (retrieve_failures - 1), 30)) e2e_latency.append(time.monotonic() - t0) completed += 1 except aiohttp.ClientConnectorError as e: @@ -427,6 +525,8 @@ def run_load(args: argparse.Namespace) -> dict[str, Any]: str(args.request_timeout), "--max-reconnects", str(args.max_reconnects), + "--max-outstanding", + str(args.max_outstanding // workers if args.max_outstanding else 0), "--poll-timeout", str(args.poll_timeout), "--submit-concurrency", @@ -501,14 +601,20 @@ def run_server(args: argparse.Namespace) -> None: "-c", "import time; time.sleep(10**9)", ] + # Older commits lack some of these knobs; pass only the fields this + # EngineConfig knows so the same harness can baseline them. + optional_fields = { + "forwarding_inference_max_connections": args.forwarding_max_connections, + "forwarding_inference_timeout_sec": args.forwarding_timeout, + "external_future_retrieved_ttl_sec": args.retrieved_ttl, + } tinker_api.app.state.engine_config = EngineConfig( base_model=args.base_model, backend="megatron", backend_config=NON_COLOCATED_MEGATRON_BACKEND_CONFIG, database_url=database_url, checkpoints_base=str(workdir / "checkpoints"), - forwarding_inference_max_connections=args.forwarding_max_connections, - forwarding_inference_timeout_sec=args.forwarding_timeout, + **{k: v for k, v in optional_fields.items() if v is not None and k in EngineConfig.model_fields}, ) profile_path = os.environ.get("TINKER_LOADTEST_PROFILE") if profile_path: @@ -524,7 +630,14 @@ def dump_profile(signum, frame): signal.signal(signal.SIGUSR1, dump_profile) profiler.enable() - uvicorn.run(tinker_api.app, host="127.0.0.1", port=args.api_port, log_config=get_uvicorn_log_config()) + uvicorn.run( + tinker_api.app, + host="127.0.0.1", + port=args.api_port, + log_config=get_uvicorn_log_config(), + backlog=getattr(tinker_api, "SKYRL_HTTP_CONNECTION_LIMIT", 2048), + timeout_keep_alive=getattr(tinker_api, "HTTP_KEEP_ALIVE_TIMEOUT_SECONDS", 5), + ) # --------------------------------------------------------------------------- # @@ -635,6 +748,8 @@ def start_api_server(args: argparse.Namespace, workdir: Path, vllm_url: str) -> ] if args.forwarding_max_connections is not None: cmd += ["--forwarding-max-connections", str(args.forwarding_max_connections)] + if args.retrieved_ttl is not None: + cmd += ["--retrieved-ttl", str(args.retrieved_ttl)] print(f"[orchestrator] starting API server: {' '.join(cmd)}") print(f"[orchestrator] server log: {log_path}") proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, start_new_session=True) @@ -696,6 +811,8 @@ def run_all(args: argparse.Namespace) -> dict[str, Any]: str(args.max_num_seqs), "--gen-seconds", str(args.gen_seconds), + "--router-workers", + str(args.router_workers), ] children.append(subprocess.Popen(vllm_cmd)) wait_for_http(f"{vllm_url}/healthz", 30) @@ -794,6 +911,12 @@ def parse_args() -> argparse.Namespace: default=400, help="max asample POSTs in flight across all workers (Tinker SDK default: 400 per client)", ) + p.add_argument( + "--max-outstanding", + type=int, + default=0, + help="cap on samples in flight across all workers, like the SDK's sample_max_concurrent_requests (0 = unlimited)", + ) p.add_argument( "--max-reconnects", type=float, @@ -818,6 +941,12 @@ def parse_args() -> argparse.Namespace: p.add_argument( "--forwarding-timeout", type=float, default=300.0, help="EngineConfig.forwarding_inference_timeout_sec" ) + p.add_argument( + "--retrieved-ttl", + type=float, + default=None, + help="EngineConfig.external_future_retrieved_ttl_sec for the launched server (memory = rate x size x ttl)", + ) p.add_argument("--server-startup-timeout", type=float, default=120.0) p.add_argument("--api-port", type=int, default=DEFAULT_API_PORT) p.add_argument("--vllm-port", type=int, default=DEFAULT_VLLM_PORT) @@ -826,6 +955,12 @@ def parse_args() -> argparse.Namespace: p.add_argument("--barrier-timeout", type=float, default=600.0, help="release the barrier anyway after this long") p.add_argument("--max-num-seqs", type=int, default=1024, help="latency mode: concurrent generations") p.add_argument("--gen-seconds", type=float, default=2.0, help="latency mode: seconds per generation") + p.add_argument( + "--router-workers", + type=int, + default=1, + help="fake router processes sharing the port (latency mode only); raise when large results saturate one process", + ) p.add_argument("--json-out", type=Path, default=None) # internal p.add_argument("--vllm-url", default=f"http://127.0.0.1:{DEFAULT_VLLM_PORT}") diff --git a/skyrl/tinker/api.py b/skyrl/tinker/api.py index 2328a850cc..d3930f0d4c 100644 --- a/skyrl/tinker/api.py +++ b/skyrl/tinker/api.py @@ -32,6 +32,7 @@ from sqlmodel import SQLModel, func, select from sqlmodel.ext.asyncio.session import AsyncSession +from skyrl.env_vars import SKYRL_HTTP_CONNECTION_LIMIT from skyrl.tinker import types from skyrl.tinker.config import EngineConfig, add_model, config_to_argv from skyrl.tinker.db_models import ( @@ -71,6 +72,12 @@ # How long retrieve_future waits for a result before returning 408 RETRIEVE_FUTURE_TIMEOUT_SECONDS = 300 +# Idle keep-alive for client connections. Under a burst of completions the +# event loop can be busy for many seconds; with uvicorn's 5s default every +# idle SDK connection is closed during such a burst and all clients reconnect +# at once, overflowing the accept backlog. Hold connections across bursts. +HTTP_KEEP_ALIVE_TIMEOUT_SECONDS = 75 + # How often poll_futures looks for newly finished requests. A single query # covers every waiter, so this can stay tight without the load scaling up with # the number of in-flight requests. @@ -1872,4 +1879,13 @@ async def root(): # Store config in app.state so lifespan can access it app.state.engine_config = engine_config - uvicorn.run(app, host=args.host, port=args.port, log_config=get_uvicorn_log_config()) + uvicorn.run( + app, + host=args.host, + port=args.port, + log_config=get_uvicorn_log_config(), + # Pending connections queue in the kernel while the loop is busy instead + # of being refused (effective value is capped by net.core.somaxconn). + backlog=SKYRL_HTTP_CONNECTION_LIMIT, + timeout_keep_alive=HTTP_KEEP_ALIVE_TIMEOUT_SECONDS, + ) diff --git a/skyrl/tinker/extra/completion_decode.py b/skyrl/tinker/extra/completion_decode.py new file mode 100644 index 0000000000..12342c6c04 --- /dev/null +++ b/skyrl/tinker/extra/completion_decode.py @@ -0,0 +1,139 @@ +"""Decode vLLM ``/v1/completions`` bodies straight into numpy arrays. + +A long-output result is almost entirely two numeric arrays (token ids and +logprobs). Decoding it the ordinary way materializes one Python object per +element and then converts the lists to numpy, which for a 262k-token result +(2.8MB of JSON) costs ~38ms and dominates the API server's CPU. pysimdjson +exposes homogeneous numeric arrays as raw buffers, so the same body decodes in +~6ms with no per-token Python objects. orjson is the fallback when pysimdjson +is unavailable or an array is not purely numeric (a null logprob, say). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import orjson + +from skyrl.utils.log import logger + +try: + import simdjson +except ImportError: # pragma: no cover - exercised via the forced-fallback tests + simdjson = None + + +@dataclass +class DecodedChoice: + finish_reason: str | None + tokens: np.ndarray # int32 + logprobs: np.ndarray # float32, same length as tokens + # vLLM's raw prompt_logprobs structure, kept only when the request asked for it. + prompt_logprobs: list[Any] | None = None + + +def _floats_from_list(values: list[Any], expected_len: int) -> np.ndarray: + """Float32 array from a decoded list, filling absent values with zeros. + + vLLM occasionally returns None for logprobs under load; zero-fill so RL + advantage computation doesn't see a ragged shape. + """ + if not values: + if expected_len: + logger.warning("No logprobs returned from vLLM — filling with zeros") + return np.zeros(expected_len, dtype=np.float32) + if any(value is None for value in values): + logger.warning("vLLM returned null logprobs — filling those positions with zeros") + values = [0.0 if value is None else value for value in values] + return np.asarray(values, dtype=np.float32) + + +class CompletionDecoder: + """Decodes completion bodies; one instance per forwarding client.""" + + def __init__(self) -> None: + self._parser = simdjson.Parser() if simdjson is not None else None + + @property + def backend(self) -> str: + return "simdjson" if self._parser is not None else "orjson" + + def decode(self, body: bytes, *, want_prompt_logprobs: bool = False) -> list[DecodedChoice]: + """Return one :class:`DecodedChoice` per ``choices`` entry. + + Raises ``ValueError`` for a body that is not JSON. + """ + if self._parser is not None: + return self._decode_simdjson(body, want_prompt_logprobs) + return self._decode_orjson(body, want_prompt_logprobs) + + # -- pysimdjson -------------------------------------------------------- + + def _decode_simdjson(self, body: bytes, want_prompt_logprobs: bool) -> list[DecodedChoice]: + # The parser owns one document at a time, so everything is copied out + # into numpy / Python objects before this method returns. + doc = self._parser.parse(body) + choices = [] + for choice in doc.get("choices") or (): + tokens = _simd_int32(choice.get("token_ids")) + logprobs_obj = choice.get("logprobs") + raw_logprobs = logprobs_obj.get("token_logprobs") if logprobs_obj is not None else None + logprobs = _simd_float32(raw_logprobs, expected_len=len(tokens)) + prompt_logprobs = None + if want_prompt_logprobs: + raw = choice.get("prompt_logprobs") + prompt_logprobs = raw.as_list() if raw is not None else None + choices.append( + DecodedChoice( + finish_reason=choice.get("finish_reason"), + tokens=tokens, + logprobs=logprobs, + prompt_logprobs=prompt_logprobs, + ) + ) + return choices + + # -- orjson fallback --------------------------------------------------- + + @staticmethod + def _decode_orjson(body: bytes, want_prompt_logprobs: bool) -> list[DecodedChoice]: + result = orjson.loads(body) + choices = [] + for choice in result.get("choices") or (): + tokens = np.asarray(choice.get("token_ids") or (), dtype=np.int32) + raw_logprobs = (choice.get("logprobs") or {}).get("token_logprobs") or [] + choices.append( + DecodedChoice( + finish_reason=choice.get("finish_reason"), + tokens=tokens, + logprobs=_floats_from_list(raw_logprobs, expected_len=len(tokens)), + prompt_logprobs=choice.get("prompt_logprobs") if want_prompt_logprobs else None, + ) + ) + return choices + + +def _simd_int32(array) -> np.ndarray: + if array is None: + return np.zeros(0, dtype=np.int32) + try: + return np.frombuffer(array.as_buffer(of_type="i"), dtype=np.int64).astype(np.int32) + except TypeError: + # Not a homogeneous integer array; take the slow path for this one. + return np.asarray(array.as_list(), dtype=np.int32) + + +def _simd_float32(array, *, expected_len: int) -> np.ndarray: + if array is None: + return _floats_from_list([], expected_len) + try: + values = np.frombuffer(array.as_buffer(of_type="d"), dtype=np.float64).astype(np.float32) + except TypeError: + # Nulls or mixed ints/floats: fall back to a list for this array only. + return _floats_from_list(array.as_list(), expected_len) + if len(values) == 0 and expected_len: + # vLLM returned an empty logprob list for a non-empty sequence. + return _floats_from_list([], expected_len) + return values diff --git a/skyrl/tinker/extra/skyrl_train_inference_forwarding.py b/skyrl/tinker/extra/skyrl_train_inference_forwarding.py index 681768a5d5..01dc2a42f4 100644 --- a/skyrl/tinker/extra/skyrl_train_inference_forwarding.py +++ b/skyrl/tinker/extra/skyrl_train_inference_forwarding.py @@ -8,7 +8,6 @@ from datetime import datetime, timezone import aiohttp -import orjson from sqlmodel.ext.asyncio.session import AsyncSession from skyrl.backends.renderer import render_model_input @@ -17,12 +16,15 @@ from skyrl.tinker.config import EngineConfig from skyrl.tinker.db_models import EngineStateDB, FutureDB, RequestStatus from skyrl.tinker.external_future_store import ExternalFutureStore, PreparedResult +from skyrl.tinker.extra.completion_decode import CompletionDecoder from skyrl.tinker.proto_serialization import ( sample_output_json_from_proto, serialize_sample_output, ) from skyrl.utils.log import logger +_ROUTER_CONNECT_TIMEOUT_SECONDS = 60.0 + class TransientInferenceError(RuntimeError): """A 5xx from vllm-router/vLLM: the request was rejected, not executed, so it is safe to retry.""" @@ -47,6 +49,7 @@ def __init__( self._cache_lock = asyncio.Lock() # Created on first use so it binds to the serving event loop. self._session: aiohttp.ClientSession | None = None + self._decoder = CompletionDecoder() def _get_session(self) -> aiohttp.ClientSession: """Return the shared aiohttp session, creating it on first use. @@ -66,12 +69,18 @@ def _get_session(self) -> aiohttp.ClientSession: max_conn = self.engine_config.forwarding_inference_max_connections # keepalive_timeout must stay under the router's idle timeout so a # pooled connection is never reused after the server closed it. - connector = aiohttp.TCPConnector(limit=max_conn or 0, keepalive_timeout=2) + # Happy Eyeballs is off: a burst of connect timeouts cancels its + # sock_connect calls mid-flight, and under uvloop the closed sockets' + # descriptors get reused before the loop forgets them ("File + # descriptor N is used by transport"), failing unrelated forwards. + connector = aiohttp.TCPConnector(limit=max_conn or 0, keepalive_timeout=2, happy_eyeballs_delay=None) self._session = aiohttp.ClientSession( connector=connector, timeout=aiohttp.ClientTimeout( total=None, - sock_connect=10.0, + # A saturated router can take tens of seconds to accept; + # that is queueing, not failure. + sock_connect=_ROUTER_CONNECT_TIMEOUT_SECONDS, sock_read=self.engine_config.forwarding_inference_timeout_sec, ), ) @@ -230,8 +239,10 @@ async def _forward(self, proxy_url: str, sample_req, model_id: str, *, base_mode if response.status >= 400: raise RuntimeError(f"vLLM /v1/completions returned {response.status}: {body.decode(errors='replace')}") try: - result = orjson.loads(body) - except orjson.JSONDecodeError as e: + # Token ids and logprobs land directly in int32/float32 arrays; + # no per-token Python objects are built (see completion_decode). + choices = self._decoder.decode(body, want_prompt_logprobs=want_prompt_logprobs) + except ValueError as e: # vllm-router can return HTML on transient errors even with 2xx status. raise RuntimeError( f"vLLM /v1/completions returned non-JSON ({response.status}, " @@ -243,26 +254,16 @@ async def _forward(self, proxy_url: str, sample_req, model_id: str, *, base_mode if want_prompt_logprobs: # All `n` choices share one prompt, so vLLM repeats the same prompt # logprobs on each choice; read them off the first. - choices = result.get("choices") or [] - raw = choices[0].get("prompt_logprobs") if choices else None + raw = choices[0].prompt_logprobs if choices else None if raw is None: logger.warning("Requested prompt logprobs but vLLM /v1/completions returned none") prompt_logprobs, topk = convert_vllm_prompt_logprobs(prompt_tokens, raw, topk=topk_prompt_logprobs) - sequences = [] - for choice in result.get("choices", []): - tokens = choice.get("token_ids", []) - lp = choice.get("logprobs") or {} - logprobs = lp.get("token_logprobs") or [] - # vLLM occasionally returns None for logprobs under load; zero-fill so - # RL advantage computation doesn't see a ragged shape. - if not logprobs and tokens: - logger.warning("No logprobs returned from vLLM — filling with zeros") - logprobs = [0.0] * len(tokens) - # Tinker's stop_reason is Literal["stop", "length"]; vLLM emits a wider set. - finish_reason = choice.get("finish_reason") - stop_reason = "stop" if finish_reason in ("stop", "stop_token") else "length" - sequences.append((stop_reason, tokens, logprobs)) + # Tinker's stop_reason is Literal["stop", "length"]; vLLM emits a wider set. + sequences = [ + ("stop" if choice.finish_reason in ("stop", "stop_token") else "length", choice.tokens, choice.logprobs) + for choice in choices + ] # Encode straight to the proto wire form the SDK retrieves; no pydantic # model or JSON text is built for the result (see PreparedResult). diff --git a/tests/tinker/test_completion_decode.py b/tests/tinker/test_completion_decode.py new file mode 100644 index 0000000000..ecc734a089 --- /dev/null +++ b/tests/tinker/test_completion_decode.py @@ -0,0 +1,108 @@ +"""vLLM completion bodies decode to the same arrays with pysimdjson and with the orjson fallback.""" + +import numpy as np +import orjson +import pytest + +from skyrl.tinker.extra import completion_decode +from skyrl.tinker.extra.completion_decode import CompletionDecoder + + +def _body(choices: list[dict]) -> bytes: + return orjson.dumps({"id": "cmpl", "object": "text_completion", "choices": choices, "usage": {}}) + + +def _decoders() -> list[CompletionDecoder]: + fast = CompletionDecoder() + slow = CompletionDecoder() + slow._parser = None + return [fast, slow] + + +@pytest.mark.parametrize("decoder", _decoders(), ids=lambda d: d.backend) +def test_decodes_tokens_and_logprobs_to_typed_arrays(decoder): + choices = decoder.decode( + _body( + [ + { + "token_ids": [5, 6, 70000], + "logprobs": {"token_logprobs": [-0.5, -1.25, -3.0]}, + "finish_reason": "stop", + }, + {"token_ids": [1], "logprobs": {"token_logprobs": [0]}, "finish_reason": "length"}, + ] + ) + ) + + assert [c.finish_reason for c in choices] == ["stop", "length"] + assert choices[0].tokens.dtype == np.int32 and choices[0].logprobs.dtype == np.float32 + assert choices[0].tokens.tolist() == [5, 6, 70000] + assert choices[0].logprobs.tolist() == [-0.5, -1.25, -3.0] + # An integer-valued logprob still comes out as float32. + assert choices[1].logprobs.tolist() == [0.0] + assert choices[0].prompt_logprobs is None + + +@pytest.mark.parametrize("decoder", _decoders(), ids=lambda d: d.backend) +def test_missing_or_null_logprobs_are_zero_filled(decoder): + choices = decoder.decode( + _body( + [ + {"token_ids": [1, 2, 3], "logprobs": None, "finish_reason": "length"}, + {"token_ids": [1, 2], "logprobs": {"token_logprobs": []}, "finish_reason": "length"}, + {"token_ids": [1, 2, 3], "logprobs": {"token_logprobs": [None, -0.5, -1.0]}, "finish_reason": "stop"}, + {"token_ids": [], "logprobs": {"token_logprobs": []}, "finish_reason": "stop"}, + ] + ) + ) + + assert choices[0].logprobs.tolist() == [0.0, 0.0, 0.0] + assert choices[1].logprobs.tolist() == [0.0, 0.0] + assert choices[2].logprobs.tolist() == [0.0, -0.5, -1.0] + assert choices[3].tokens.tolist() == [] and choices[3].logprobs.tolist() == [] + assert all(c.logprobs.dtype == np.float32 for c in choices) + + +@pytest.mark.parametrize("decoder", _decoders(), ids=lambda d: d.backend) +def test_prompt_logprobs_kept_only_when_requested(decoder): + raw = [None, {"5": {"logprob": -0.1, "rank": 1, "decoded_token": "a"}}] + body = _body( + [{"token_ids": [1], "logprobs": {"token_logprobs": [-1.0]}, "finish_reason": "stop", "prompt_logprobs": raw}] + ) + + assert decoder.decode(body)[0].prompt_logprobs is None + assert decoder.decode(body, want_prompt_logprobs=True)[0].prompt_logprobs == raw + + +@pytest.mark.parametrize("decoder", _decoders(), ids=lambda d: d.backend) +def test_non_json_body_raises_value_error(decoder): + with pytest.raises(ValueError): + decoder.decode(b"502 Bad Gateway") + + +@pytest.mark.skipif(completion_decode.simdjson is None, reason="pysimdjson not installed") +def test_fast_and_fallback_agree_on_large_arrays(): + rng = np.random.default_rng(0) + tokens = rng.integers(0, 150_000, size=200_000).tolist() + logprobs = (-rng.random(200_000) * 20).tolist() + body = _body([{"token_ids": tokens, "logprobs": {"token_logprobs": logprobs}, "finish_reason": "length"}]) + fast, slow = _decoders() + + a, b = fast.decode(body)[0], slow.decode(body)[0] + + assert np.array_equal(a.tokens, b.tokens) + assert np.array_equal(a.logprobs, b.logprobs) + assert fast.backend == "simdjson" and slow.backend == "orjson" + + +def test_decoder_can_be_reused_across_bodies(): + decoder = CompletionDecoder() + first = decoder.decode( + _body([{"token_ids": [1, 2], "logprobs": {"token_logprobs": [-1.0, -2.0]}, "finish_reason": "stop"}]) + ) + second = decoder.decode( + _body([{"token_ids": [9], "logprobs": {"token_logprobs": [-9.0]}, "finish_reason": "length"}]) + ) + # Arrays from the first body survive the second parse (they were copied out). + assert first[0].tokens.tolist() == [1, 2] and first[0].logprobs.tolist() == [-1.0, -2.0] + assert second[0].tokens.tolist() == [9] diff --git a/tests/tinker/test_inference_forwarding_config.py b/tests/tinker/test_inference_forwarding_config.py index 9b8a8bdcf1..1987fb1deb 100644 --- a/tests/tinker/test_inference_forwarding_config.py +++ b/tests/tinker/test_inference_forwarding_config.py @@ -33,7 +33,7 @@ async def test_forwarding_client_uses_configured_timeout_and_connection_limit() client = SkyRLTrainInferenceForwardingClient(config, db_engine=None) try: session = client._get_session() - assert session.timeout.sock_connect == 10.0 + assert session.timeout.sock_connect == 60.0 assert session.timeout.sock_read == 1800.0 # No overall deadline: a request may wait in the connector queue for # as long as the engine takes to get to it. diff --git a/uv.lock b/uv.lock index 0a7fc776d0..4f3575aac7 100644 --- a/uv.lock +++ b/uv.lock @@ -7499,6 +7499,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/72/a12b6109c3c274a5a680f96f4b6b108dba94958075116bca4d3d2ace2eab/pyqwest-0.8.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bf5caf0e25a26145e08643be9b65831c0a2c9bf6a19d1e6c9b1ad5a2c06cf3e9", size = 4825893, upload-time = "2026-07-31T03:33:40.887Z" }, ] +[[package]] +name = "pysimdjson" +version = "7.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/24/65e3cad88e74ef8ca59fefded953eb78ebface8a3199c3a97fe318a7387b/pysimdjson-7.0.2.tar.gz", hash = "sha256:44cf276e48912a3b9c7ca362c14da8420a7ac15a9f1a16ec95becff86db3904a", size = 1397812, upload-time = "2025-06-28T20:37:24.071Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/53/6c08667ec90830f42b257a460fe04c0316e6aeb6e5567b20813caddcdbae/pysimdjson-7.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9ef56dff19b004dd52bbaf31bd6b26486d20a07de50bf3fd0e2d655cebadc135", size = 1857080, upload-time = "2025-06-28T20:36:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5f/81f0bc351e6970dcd7448580779791c42706614627807b3aec8cb8095be0/pysimdjson-7.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7db0a4abf3740a33204283c15ae1bc4fd2dd17be7c259d10551a8d32f72fab9", size = 1646685, upload-time = "2025-06-28T20:36:24.79Z" }, + { url = "https://files.pythonhosted.org/packages/7a/bd/8249fd295a1113b3a66eccd68752bd52d4f32df4d1a740f9f0d3db91b517/pysimdjson-7.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b751b44323c763ae51303aba5834bd193eea4d121987230a977ccfbe258e479", size = 2798435, upload-time = "2025-06-28T20:36:25.946Z" }, + { url = "https://files.pythonhosted.org/packages/0d/58/504b6bdfd97c26094bcc50fbc283c806d3b36477c077267d76d07ab96caa/pysimdjson-7.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe3712de488044408ff4a8e59c0745ba74f063ad019a3d0e662c9df9bb96e985", size = 2859387, upload-time = "2025-06-28T20:36:27.078Z" }, + { url = "https://files.pythonhosted.org/packages/47/2f/ca46b61203ab06d9bb45d216a5a635ec92250fe531bf990191c07403096d/pysimdjson-7.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0caeb9edaeae4bbbce9fdc0c2e81d303c29628ef637c11b248942c591eb59b24", size = 3265436, upload-time = "2025-06-28T20:36:28.164Z" }, + { url = "https://files.pythonhosted.org/packages/14/ce/cce78f90c9fb51df6fdb71e8262206887a72a3851734b3a07528f9fd1eec/pysimdjson-7.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc0e934a4bb9b1465628eae80d6f386d0cfd5c6b9e8bc822a9326e30c2b7fb66", size = 2547752, upload-time = "2025-06-28T20:36:29.295Z" }, + { url = "https://files.pythonhosted.org/packages/45/b2/c841750e7cc118bcfaf3f47f904eb405f632f2a5fe24d8c35bb5657933a9/pysimdjson-7.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:39c05ca2d26de21373045557fc1f1a84c70cea35e89f4746e537fbe2948f9c38", size = 3688684, upload-time = "2025-06-28T20:36:30.431Z" }, + { url = "https://files.pythonhosted.org/packages/0f/48/aad6bbd435f47385487b7397a7bb645ee53195bf4637f467d616382c1bf8/pysimdjson-7.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98018ad3e96dc9a5ffcce5100bc1cc0ef20185ff1ab097bb21a2dd1090e644e6", size = 3595563, upload-time = "2025-06-28T20:36:31.857Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9d/dcdafeb3ee0c689b4dbef7d859919760fe89357551a8ddbacfb65244a689/pysimdjson-7.0.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3a05fbc43f22b131246c58d25f332e6e7929826bd4ee88fab2ffb5f3a29305bf", size = 3850967, upload-time = "2025-06-28T20:36:32.95Z" }, + { url = "https://files.pythonhosted.org/packages/6f/db/3aee16daf44b31399dc5d3318978782a96138f0ea32f09bedf172a4473f1/pysimdjson-7.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:755774195a3c7714ec88d08da2f03ed9097d72bcc35ae31b4887b524ae37d435", size = 4214235, upload-time = "2025-06-28T20:36:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3a/79876cd35668e2dfd4aa66091ecda130117940fe706d754c39aaa4609b3a/pysimdjson-7.0.2-cp311-cp311-win32.whl", hash = "sha256:1c7f85f5b0280e57de1cbfb624b3b2535cc590d4490a6955ff65e5a358b09285", size = 1529278, upload-time = "2025-06-28T20:36:35.993Z" }, + { url = "https://files.pythonhosted.org/packages/fc/89/bda298ab3b3407f38b70994efc7e4f0938d9ff34e0e5b180f9d5066cccf7/pysimdjson-7.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:d3ff730a48e666a2f663a43663fd71c10ba5d0393cfce500c4f535f09fae39e7", size = 1573100, upload-time = "2025-06-28T20:36:37.079Z" }, + { url = "https://files.pythonhosted.org/packages/61/81/2a7bee8961e9519084ee290bb7135844f1f786ec8a26f62d48e7fd23a08b/pysimdjson-7.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8ea5ffbdfde6a26b05bec12263ffacf8435d2e51c3793b44aa090fb38e709434", size = 1877768, upload-time = "2025-06-28T20:36:38.463Z" }, + { url = "https://files.pythonhosted.org/packages/b3/55/dfa21b647ff1a54e5925664ebfe3f1f800375546f0665347f3041a52bf5a/pysimdjson-7.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4fbe295c84bd9406ac8fc38ab76a6ff1187df11be9348e5937f9dcc42f41c8f8", size = 1656024, upload-time = "2025-06-28T20:36:39.847Z" }, + { url = "https://files.pythonhosted.org/packages/64/bd/06b744b0b33f4932ad4ed51fdb8ec5eeca6f7980ad502839dbfbe5ac60c9/pysimdjson-7.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abbbd51ef301083c9ee885d1ba8d3c2081c462d56c2d0e2f603cc917a44f7ed5", size = 2771741, upload-time = "2025-06-28T20:36:41.249Z" }, + { url = "https://files.pythonhosted.org/packages/90/a4/c13afff7d4cd2fd001508f0d411063a8a9c451d694178b5230d50c8caf98/pysimdjson-7.0.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14ca76010e5d82f4c0de90586a940e57c28beee937b4a53ef239b88ebee7190e", size = 2823997, upload-time = "2025-06-28T20:36:42.704Z" }, + { url = "https://files.pythonhosted.org/packages/58/da/459c89f3dbb8344f6b2a374850d13522cc9a89726faea4319568034f1f1f/pysimdjson-7.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1de838fc7aa473db24ddacc0b285928bd74d5830755f8471b17c34e78e94840", size = 3248858, upload-time = "2025-06-28T20:36:43.969Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/c9274cb68412b2b119a0d72c71d57b01f05397b59afc7cec9ff0b28a88d5/pysimdjson-7.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:061259784a9a4746d40a3a3f20542a19bd0e403e49af4aa3bd9a1626429ce704", size = 2529651, upload-time = "2025-06-28T20:36:45.266Z" }, + { url = "https://files.pythonhosted.org/packages/95/3b/8f3a3866daa6776ea3d3986b0c21cc678bd0bb5872a19a18170fae396e90/pysimdjson-7.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:27c2e4cde872b8d3a05dc855341508d11d056bb3b25eddbc17e533417a848a52", size = 3664874, upload-time = "2025-06-28T20:36:46.541Z" }, + { url = "https://files.pythonhosted.org/packages/1e/21/376e54868918d8b4831fb8653c1976615f99a11d95e0502ecaaa7a306d32/pysimdjson-7.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:41a18886861d47b63ef6231796a30ccc547bf3772a06fa60b681ee8f00a614ce", size = 3579057, upload-time = "2025-06-28T20:36:47.843Z" }, + { url = "https://files.pythonhosted.org/packages/5f/92/29bf4549ec6d692aca1cc11b1ff8a8bf8f742dd09e834f649e2567eb1438/pysimdjson-7.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fdbd392590613ddbc4922ab5374282dddefa94471fc7a97bc2c1df6a450dd671", size = 3818097, upload-time = "2025-06-28T20:36:49.319Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f8/ff0a6e3ee124eef780f164c95ea95ccca1ac04e4cff483e728aa029e7b36/pysimdjson-7.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cb217ddaedd5f28ca7db16e4ea972f02c6db380827ec312c7e6a9371ca5e4d7c", size = 4201879, upload-time = "2025-06-28T20:36:50.801Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b0/7f60a32fef8b97407f07c80d367fb161c9245bd3c1de1597c9f4cb1c6536/pysimdjson-7.0.2-cp312-cp312-win32.whl", hash = "sha256:bf5af81e19b0cef57679523759f9219e2641e5156a4ee5b854e49e3e6b1690ab", size = 1529773, upload-time = "2025-06-28T20:36:51.97Z" }, + { url = "https://files.pythonhosted.org/packages/28/e7/b127c677f6aa8991ba6f9ea99a08aa167ab93a1844f6da35c65fa4b98179/pysimdjson-7.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:782ee03679eaea5b28d9bc9279bc0f0f03d251c17571396f3ed50ba86023d88f", size = 1574523, upload-time = "2025-06-28T20:36:53.103Z" }, + { url = "https://files.pythonhosted.org/packages/65/65/bf171e0dde8a40a56c6fde4e700daa3b172f1781b26478e92c34317f1225/pysimdjson-7.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a721cc23cd6240430b2c862caff79a411abc987290859cd0f9c5a3e29efa1d2c", size = 1877151, upload-time = "2025-06-28T20:36:54.199Z" }, + { url = "https://files.pythonhosted.org/packages/e2/2d/242c1bebadb960b704066288ae28660da3de7fb5d8f52f655e080e7ffbbf/pysimdjson-7.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fdbbf4246cac27dac38043da8f4d82a46d434b5bc3a4e54c0a55de1dd92631ae", size = 1655651, upload-time = "2025-06-28T20:36:55.336Z" }, + { url = "https://files.pythonhosted.org/packages/49/86/3b25e77ae2998342d2bd376eb58baf17b35e6c2fdb9184e8bc8c31ebfafe/pysimdjson-7.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77bbf9afdea8a9aa220cbf29115cc32e81207f9e8e07963ea145ba8d2e8f4053", size = 2771613, upload-time = "2025-06-28T20:36:56.732Z" }, + { url = "https://files.pythonhosted.org/packages/49/d9/3db962802aa5c95a8f89023dcf00eefa30817e9b9862668d5efb91c44d81/pysimdjson-7.0.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43d42ef0660181b67bd833c13bdcbb2743abd40bc348db8f9e788b5d88717459", size = 2819981, upload-time = "2025-06-28T20:36:57.923Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a0/bfbc3c9a1b216cacad74863229c06c576f108e4f67cb6daa3c4d6071a9ff/pysimdjson-7.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13f2820c95d9c74139407921aeec8099e67546ccfcb309561881e877e4a3aa97", size = 3246918, upload-time = "2025-06-28T20:36:59.458Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fc/1d21538d1fd3e4f2f7a96de605fbcdb1f150ff0eb49ac08f005da83e17c7/pysimdjson-7.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f81638ce66a7393ad1b4f5fae6666c417cc01e5ecb81c86ff727349599bbc83f", size = 2524078, upload-time = "2025-06-28T20:37:00.659Z" }, + { url = "https://files.pythonhosted.org/packages/2d/d3/76c05b4d116adcb947955c68700c9e67ee7f748a38d37ba72e5b1109ef1d/pysimdjson-7.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5ffe83c4dbfdabea5f2231cc64ff1a62b7ecd18f64cb04a61439a5c24d08a0cd", size = 3662263, upload-time = "2025-06-28T20:37:01.835Z" }, + { url = "https://files.pythonhosted.org/packages/5f/4c/7f4c326f4022babab518e1295446c58c7f72b7bfb242b47e9fae421c3783/pysimdjson-7.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:08b576531375fa6b9479b43b5358e5e172490bef8969b0f53d6b6be7c5d7b88a", size = 3576295, upload-time = "2025-06-28T20:37:02.989Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/c4df622caf46284dd1a4d6e403dccea2a874623563c63d6e1cec4f54259a/pysimdjson-7.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1b7e26580d0030b6f7bb6fddc12e7756f4ffae3a9e4f7a8c3522d783173ac459", size = 3813976, upload-time = "2025-06-28T20:37:04.186Z" }, + { url = "https://files.pythonhosted.org/packages/75/b9/e21a5d1f4060ffeca6026a94599f6b68bf62221dd02a7af5962c73040edc/pysimdjson-7.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a8fb78454cd2936f8e27e8948b56b6e44a766eaa162fef02a1436c2d4570053", size = 4197725, upload-time = "2025-06-28T20:37:05.591Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ed/7e4511cabdcb2931cce174ce0ecf17cf4de6039b4d908daca4d313875f1e/pysimdjson-7.0.2-cp313-cp313-win32.whl", hash = "sha256:ef56eacf050e194d4058d6ed818dbbe40d9ec5dcb182ba93a451cad2467aad27", size = 1529585, upload-time = "2025-06-28T20:37:07.016Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fa/3642b49521007362c9eb228ed472927e020b84d6413efa8fd69fd9f7c6b9/pysimdjson-7.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:4ae000c2d45a1af0303fe151e5204188fcbb23acc6cbdf04ac1062ab80538a1b", size = 1574251, upload-time = "2025-06-28T20:37:08.327Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -9151,6 +9195,7 @@ tinker = [ { name = "asyncpg" }, { name = "fastapi", extra = ["standard"] }, { name = "psycopg2-binary" }, + { name = "pysimdjson" }, { name = "sqlalchemy", extra = ["asyncio"] }, { name = "sqlmodel" }, { name = "tinker" }, @@ -9232,6 +9277,7 @@ requires-dist = [ { name = "pybase64", marker = "extra == 'skyrl-train'", specifier = ">=1.4.2" }, { name = "pybind11", marker = "extra == 'skyrl-train'" }, { name = "pymdown-extensions", marker = "extra == 'dev'", specifier = ">=10.7" }, + { name = "pysimdjson", marker = "extra == 'tinker'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-asyncio", marker = "extra == 'dev'" }, { name = "pytest-forked", marker = "extra == 'dev'" },