From eb53a1fbd57bf57589309941a6bb0109e7e1aacb Mon Sep 17 00:00:00 2001 From: Pensieve Intern Agent Date: Sun, 14 Jun 2026 03:35:58 +0000 Subject: [PATCH 1/2] Add Nemotron Super MTP vLLM SPEED-bench config Signed-off-by: Pensieve Intern Agent --- .../specdec_bench_mtp_vllm.yaml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tools/launcher/examples/Nemotron-h/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/specdec_bench_mtp_vllm.yaml diff --git a/tools/launcher/examples/Nemotron-h/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/specdec_bench_mtp_vllm.yaml b/tools/launcher/examples/Nemotron-h/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/specdec_bench_mtp_vllm.yaml new file mode 100644 index 00000000000..d3dbaacf1ec --- /dev/null +++ b/tools/launcher/examples/Nemotron-h/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/specdec_bench_mtp_vllm.yaml @@ -0,0 +1,66 @@ +# SPEED-bench MTP speculative-decoding run for NVIDIA-Nemotron-3-Super-120B-A12B-BF16 via vLLM. +# +# Nemotron-3-Super-120B-A12B is a 120B-total-parameter MoE model, so use +# tensor parallel size 4 for BF16 weights on 80 GB GPUs. +# +# Slurm run on cw_dfw: +# uv run slurm.py --yaml modules/Model-Optimizer/tools/launcher/examples/Nemotron-h/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/specdec_bench_mtp_vllm.yaml --yes + +job_name: NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm + +pipeline: + global_vars: + hf_model: /hf-local/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + + # task_0: SPEED qualitative split + task_0: + script: common/specdec_bench/run.sh + args: + - --dataset speed + - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/qualitative + - --engine VLLM + - --speculative_algorithm MTP + - --draft_length 3 + - --tp_size 4 + - --ep_size 1 + - --concurrency 32 + - --output_length 4096 + - --aa_timing + - --show_progress + - --save_dir /scratchspace/{sweep_name_default}/qualitative + environment: + - HF_MODEL_CKPT: <> + - HF_LOCAL: /hf-local + slurm_config: + _factory_: "slurm_factory" + nodes: 1 + ntasks_per_node: 1 + gpus_per_node: 4 + container: vllm/vllm-openai:latest + + # task_1: SPEED throughput_32k split + task_1: + script: common/specdec_bench/run.sh + args: + - --dataset speed + - --dataset_path /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k + - --engine VLLM + - --speculative_algorithm MTP + - --draft_length 3 + - --tp_size 4 + - --ep_size 1 + - --concurrency 8 + - --num_requests 80 + - --output_length 4096 + - --aa_timing + - --show_progress + - --save_dir /scratchspace/{sweep_name_default}/throughput_32k + environment: + - HF_MODEL_CKPT: <> + - HF_LOCAL: /hf-local + slurm_config: + _factory_: "slurm_factory" + nodes: 1 + ntasks_per_node: 1 + gpus_per_node: 4 + container: vllm/vllm-openai:latest From 829df4f93030722db0dafce6bb4d3665c14a53aa Mon Sep 17 00:00:00 2001 From: pensieve-intern Date: Sun, 14 Jun 2026 04:12:14 +0000 Subject: [PATCH 2/2] =?UTF-8?q?[OMNIML-5100]=20cell=5Ft1=5Fd7=20=E2=80=94?= =?UTF-8?q?=20pensieve-intern=20agent=20draft?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PR_URL.txt | 1 + cell_output.log | 268 ++++++++ qualitative_specbench_results.json | 928 ++++++++++++++++++++++++++ throughput_32k_specbench_results.json | 141 ++++ 4 files changed, 1338 insertions(+) create mode 100644 PR_URL.txt create mode 100644 cell_output.log create mode 100644 qualitative_specbench_results.json create mode 100644 throughput_32k_specbench_results.json diff --git a/PR_URL.txt b/PR_URL.txt new file mode 100644 index 00000000000..669407067b4 --- /dev/null +++ b/PR_URL.txt @@ -0,0 +1 @@ +https://github.com/NVIDIA/Model-Optimizer/pull/1715 diff --git a/cell_output.log b/cell_output.log new file mode 100644 index 00000000000..5853dbae002 --- /dev/null +++ b/cell_output.log @@ -0,0 +1,268 @@ +warning: `VIRTUAL_ENV=/tmp/builds/YQxxH4yPp/3/omniml/integration/nmm-sandbox/.venv-intern-agent` does not match the project environment path `.venv` and will be ignored; use `--active` to target the active environment instead +warning: No `requires-python` value found in the workspace. Defaulting to `>=3.12`. +Configuring global options +Dry run for task __main__:cicd +Resolved Arguments +┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Argument Name ┃ Resolved Value ┃ +┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ detach │ True │ +│ hf_local │ None │ +│ identity │ '/.ssh/id_ed25519' │ +│ job_dir │ '/lustre/fsw/portfolios/coreai/users/chenhany/experiment… │ +│ job_name │ 'NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mt… │ +│ pipeline │ SandboxPipeline( │ +│ │ global_vars=GlobalVariables( │ +│ │ hf_model='/hf-local/nvidia/NVIDIA-Nemotron-3-Super-1… │ +│ │ task_0=SandboxTask0( │ +│ │ script='common/specdec_bench/run.sh', │ +│ │ slurm_config=SlurmConfig( │ +│ │ host='cw-dfw-cs-001-login-01.nvidia.com', │ +│ │ account='coreai_dlalgo_modelopt', │ +│ │ partition='batch', │ +│ │ container='vllm/vllm-openai:latest', │ +│ │ modelopt_install_path='/usr/local/lib/python3.12/d… │ +│ │ container_mounts=['/lustre/fsw/portfolios/coreai/p… │ +│ │ '/lustre:/lustre', '/cm:/cm', │ +│ │ '/var/run/munge:/var/run/munge'], │ +│ │ srun_args=['--no-container-mount-home'], │ +│ │ array=None, │ +│ │ nodes=1, │ +│ │ ntasks_per_node=1, │ +│ │ gpus_per_node=4), │ +│ │ args=['--dataset speed', '--dataset_path │ +│ │ /hf-local/nvidia/SPEED-Bench-Internal/qualitative', │ +│ │ '--engine VLLM', '--speculative_algorithm MTP', │ +│ │ '--draft_length 3', '--tp_size 4', '--ep_size 1', │ +│ │ '--concurrency 32', '--output_length 4096', │ +│ │ '--aa_timing', '--show_progress', '--save_dir │ +│ │ /scratchspace/{sweep_name_default}/qualitative', │ +│ │ '--temperature 1', '--max_seq_len 65536', '--save_dir │ +│ │ /scratchspace/NVIDIA-Nemotron-3-Super-120B-A12B-BF16_mtp… │ +│ │ '--draft_length 7'], │ +│ │ environment=[{'HF_MODEL_CKPT': │ +│ │ '<>'}, {'HF_LOCAL': '/hf-local'}]), │ +│ │ task_1=SandboxTask1( │ +│ │ script='common/specdec_bench/run.sh', │ +│ │ slurm_config=SlurmConfig( │ +│ │ host='cw-dfw-cs-001-login-01.nvidia.com', │ +│ │ account='coreai_dlalgo_modelopt', │ +│ │ partition='batch', │ +│ │ container='vllm/vllm-openai:latest', │ +│ │ modelopt_install_path='/usr/local/lib/python3.12/d… │ +│ │ container_mounts=['/lustre/fsw/portfolios/coreai/p… │ +│ │ '/lustre:/lustre', '/cm:/cm', │ +│ │ '/var/run/munge:/var/run/munge'], │ +│ │ srun_args=['--no-container-mount-home'], │ +│ │ array=None, │ +│ │ nodes=1, │ +│ │ ntasks_per_node=1, │ +│ │ gpus_per_node=4), │ +│ │ args=['--dataset speed', '--dataset_path │ +│ │ /hf-local/nvidia/SPEED-Bench-Internal/throughput_32k', │ +│ │ '--engine VLLM', '--speculative_algorithm MTP', │ +│ │ '--draft_length 3', '--tp_size 4', '--ep_size 1', │ +│ │ '--concurrency 8', '--num_requests 80', '--output_length │ +│ │ 4096', '--aa_timing', '--show_progress', '--save_dir │ +│ │ /scratchspace/{sweep_name_default}/throughput_32k', │ +│ │ '--temperature 1', '--max_seq_len 65536', '--save_dir │ +│ │ /scratchspace/NVIDIA-Nemotron-3-Super-120B-A12B-BF16_mtp… │ +│ │ '--num_requests 80', '--draft_length 7'], │ +│ │ environment=[{'HF_MODEL_CKPT': │ +│ │ '<>'}, {'HF_LOCAL': '/hf-local'}])) │ +│ task │ None │ +│ test_level │ 0 │ +│ user │ 'chenhany' │ +└──────────────────┴───────────────────────────────────────────────────────────┘ +Launching cicd... +============================================================ +Version Report +============================================================ + Launcher 5c34b6a (main) + Model-Optimizer eb53a1fb (pensieve-intern/OMNIML-5095/cell-t1-d7) +============================================================ +────────────── Entering Experiment cicd with id: cicd_1781408325 ─────────────── +job NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm task 0 slurm_config: SlurmConfig(host='cw-dfw-cs-001-login-01.nvidia.com', port=22, account='coreai_dlalgo_modelopt', partition='batch', qos=None, container='vllm/vllm-openai:latest', modelopt_install_path='/usr/local/lib/python3.12/dist-packages/modelopt', container_mounts=['/lustre/fsw/portfolios/coreai/projects/coreai_dlalgo_modelopt/hf-local:/hf-local', '/lustre:/lustre', '/cm:/cm', '/var/run/munge:/var/run/munge'], srun_args=['--no-container-mount-home'], array=None, nodes=1, ntasks_per_node=1, gpus_per_node=4, time='04:00:00', local=False, segment=None) +job NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm task 1 slurm_config: SlurmConfig(host='cw-dfw-cs-001-login-01.nvidia.com', port=22, account='coreai_dlalgo_modelopt', partition='batch', qos=None, container='vllm/vllm-openai:latest', modelopt_install_path='/usr/local/lib/python3.12/dist-packages/modelopt', container_mounts=['/lustre/fsw/portfolios/coreai/projects/coreai_dlalgo_modelopt/hf-local:/hf-local', '/lustre:/lustre', '/cm:/cm', '/var/run/munge:/var/run/munge'], srun_args=['--no-container-mount-home'], array=None, nodes=1, ntasks_per_node=1, gpus_per_node=4, time='04:00:00', local=False, segment=None) +find: ‘modules/Megatron-LM/megatron/*’: No such file or directory +find: ‘modules/Megatron-LM/examples/*’: No such file or directory +find: ‘modules/Megatron-LM/*.py’: No such file or directory +find: ‘modules/Model-Optimizer-Internal/**’: No such file or directory +find: ‘modules/Megatron-LM/megatron/*’: No such file or directory +find: ‘modules/Megatron-LM/examples/*’: No such file or directory +find: ‘modules/Megatron-LM/*.py’: No such file or directory +find: ‘modules/Model-Optimizer-Internal/**’: No such file or directory +[03:38:51] Connecting to client.py:257 + chenhany@cw-dfw-cs-001-login-01.nvidia.com +[03:38:51] INFO Connected (version 2.0, client transport.py:1786 + OpenSSH_8.9p1) +[03:38:52] INFO Authentication (publickey) successful! transport.py:1786 + INFO rsyncing rsync.py:37 + /tmp/pensieve-intern-agent-ukdlhxs6/workspace/ex + periments/cicd/cicd_1781408325 to + /lustre/fsw/portfolios/coreai/users/chenhany/exp + eriments/cicd ... +[03:39:17] INFO Successfully ran `rsync -pthrvz --rsh='ssh -i rsync.py:93 + /.ssh/id_ed25519 -p 22 ' + /tmp/pensieve-intern-agent-ukdlhxs6/workspace/ex + periments/cicd/cicd_1781408325 + chenhany@cw-dfw-cs-001-login-01.nvidia.com:/lust + re/fsw/portfolios/coreai/users/chenhany/experime + nts/cicd` +[03:39:17] Launching job experiment.py:800 + NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_benc + h_mtp_vllm_0 for experiment cicd +[03:39:18] INFO Launched app: launcher.py:116 + slurm_tunnel://nemo_run/12788709 +[03:39:18] Launching job experiment.py:800 + NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_benc + h_mtp_vllm_1 for experiment cicd +[SLURM] Job 12788709 is no longer pending. + INFO Launched app: launcher.py:116 + slurm_tunnel://nemo_run/12788710 +────────────────── Detaching from Experiment cicd_1781408325. ────────────────── + Task specific cleanup won't be run. experiment.py:1212 + Ephemeral logs and artifacts may be lost. +[SLURM] Job 12788710 - State: PENDING, Estimated start: N/A, Current time: 2026-06-14 03:39:18 + +Experiment Status for cicd_1781408325 + +Task 0: NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0 +- Status: PENDING +- Executor: SlurmExecutor on chenhany@cw-dfw-cs-001-login-01.nvidia.com +- Job id: 12788709 +- Local Directory: /tmp/pensieve-intern-agent-ukdlhxs6/workspace/experiments/cicd/cicd_1781408325/NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0 +- Remote Directory: /lustre/fsw/portfolios/coreai/users/chenhany/experiments/cicd/cicd_1781408325/NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0 + +Task 1: NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1 +- Status: SUBMITTED +- Executor: SlurmExecutor on chenhany@cw-dfw-cs-001-login-01.nvidia.com +- Job id: 12788710 +- Local Directory: /tmp/pensieve-intern-agent-ukdlhxs6/workspace/experiments/cicd/cicd_1781408325/NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1 +- Remote Directory: /lustre/fsw/portfolios/coreai/users/chenhany/experiments/cicd/cicd_1781408325/NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1 + + +# The experiment was run with the following tasks: ['NVIDIA-Nemotron-3-Super-120 +# You can inspect and reconstruct this experiment at a later point in time using +experiment = run.Experiment.from_id("cicd_1781408325") +experiment.status() # Gets the overall status +experiment.logs("NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0 +experiment.cancel("NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm + + +# You can inspect this experiment at a later point in time using the CLI as well +nemo experiment status cicd_1781408325 +nemo experiment logs cicd_1781408325 0 +nemo experiment cancel cicd_1781408325 0 + +Found 2 experiment(s): cicd_1781406247 cicd_1781408325 + +=== [2026-06-14 03:39:28] Polling iteration 1/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: RUNNING + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: PENDING + + Summary: 2 succeeded, 0 failed, 0 cancelled, 1 running, 1 pending +Waiting 180s before next poll... + +=== [2026-06-14 03:42:33] Polling iteration 2/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: RUNNING + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: PENDING + + Summary: 2 succeeded, 0 failed, 0 cancelled, 1 running, 1 pending +Waiting 180s before next poll... + +=== [2026-06-14 03:45:40] Polling iteration 3/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: RUNNING + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: PENDING + + Summary: 2 succeeded, 0 failed, 0 cancelled, 1 running, 1 pending +Waiting 180s before next poll... + +=== [2026-06-14 03:48:45] Polling iteration 4/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: RUNNING + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: PENDING + + Summary: 2 succeeded, 0 failed, 0 cancelled, 1 running, 1 pending +Waiting 180s before next poll... + +=== [2026-06-14 03:51:50] Polling iteration 5/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: RUNNING + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: PENDING + + Summary: 2 succeeded, 0 failed, 0 cancelled, 1 running, 1 pending +Waiting 180s before next poll... + +=== [2026-06-14 03:54:55] Polling iteration 6/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: RUNNING + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: PENDING + + Summary: 2 succeeded, 0 failed, 0 cancelled, 1 running, 1 pending +Waiting 180s before next poll... + +=== [2026-06-14 03:58:00] Polling iteration 7/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: RUNNING + + Summary: 3 succeeded, 0 failed, 0 cancelled, 1 running, 0 pending +Waiting 180s before next poll... + +=== [2026-06-14 04:01:05] Polling iteration 8/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: RUNNING + + Summary: 3 succeeded, 0 failed, 0 cancelled, 1 running, 0 pending +Waiting 180s before next poll... + +=== [2026-06-14 04:04:10] Polling iteration 9/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: RUNNING + + Summary: 3 succeeded, 0 failed, 0 cancelled, 1 running, 0 pending +Waiting 180s before next poll... + +=== [2026-06-14 04:07:15] Polling iteration 10/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: RUNNING + + Summary: 3 succeeded, 0 failed, 0 cancelled, 1 running, 0 pending +Waiting 180s before next poll... + +=== [2026-06-14 04:10:20] Polling iteration 11/14400 === + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781406247 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_0: SUCCEEDED + cicd_1781408325 / NVIDIA-Nemotron-3-Super-120B-A12B-BF16_specdec_bench_mtp_vllm_1: SUCCEEDED + + Summary: 4 succeeded, 0 failed, 0 cancelled, 0 running, 0 pending + +All experiments complete. + SUCCEEDED: 4 + FAILED: 0 + CANCELLED: 0 + +=== Fetching experiment logs === +Fetching logs: cicd_1781406247 task 0 +Fetching logs: cicd_1781406247 task 1 +Fetching logs: cicd_1781408325 task 0 +Fetching logs: cicd_1781408325 task 1 +=== Done fetching logs === diff --git a/qualitative_specbench_results.json b/qualitative_specbench_results.json new file mode 100644 index 00000000000..0abdf8a9b26 --- /dev/null +++ b/qualitative_specbench_results.json @@ -0,0 +1,928 @@ +{ + "Request_AL": { + "0daf539b787c4dccbb547330a8b4c3d7": 2.6036036036036037, + "135c7fe91faa48fd83ca5eac94c09f00": 3.4746192893401013, + "37f349607a3d46b8bc5aafcefb5fd518": 2.928240740740741, + "1fd6d82bbde647f29b93afd7b4eb14b0": 3.9315068493150687, + "48579d93b98d459fbf473748a607d748": 3.5672514619883042, + "5ace1d46a306452da3c517a7a8bc2093": 3.75, + "3416d0ce7c7a4632943ff1326913437f": 2.7, + "e281ab7c608141a683519a317e998dd2": 3.5116959064327484, + "20de4633029f426fbd0c3c4eb6bd3853": 3.723076923076923, + "988e25df0fba43039adf10baf818f7a9": 3.5903614457831323, + "0fe036855bd4410c99eca693e49248cc": 3.655637254901961, + "91ec653700af4857857a159f8c6da4c7": 3.0328068043742404, + "154d2a4f6556463793035c90e4250561": 3.514705882352941, + "22f2c7dc2f7745e2857160543c2cf401": 3.0344827586206895, + "80156ace23984b74891e02065411ef87": 3.2432432432432434, + "60868b6276204b9ba3e59ad45067b7fd": 3.7353595255745, + "cd4f5c325d534a3cb5e243c016f8f3b9": 3.617940199335548, + "4b328c08db884b959183f68428cc02cc": 3.6278026905829597, + "21a642e08bdc4040803897dae92c3b6e": 3.7378964941569284, + "7635d4e93e514bf6bce51b2416b2c1d9": 3.1834625322997416, + "b9ec7a3ce76342ed80071dac82c09678": 2.9331742243436754, + "7ea2cb7770904ea3b78f43f466748b6c": 2.9446441409058233, + "8e3677237d6f4ba3b24298f75709dd0b": 3.4941634241245136, + "f5bc6e1054a14617875475b0472fa018": 4.296511627906977, + "ecf92ab7acf1492da70e68fcddd9d0b7": 2.9854227405247813, + "7df494fd138345ff93caadc693a0b1b0": 3.015957446808511, + "97f49a2c2ad14e7d8cf106551b593c69": 3.468729851708575, + "22f088e74c284523b965617fa401840a": 2.9329896907216493, + "a6acc9365c464865ad61f02f8b2360aa": 3.506849315068493, + "c848ad5326ce4238bdc30b6ab0343b92": 3.4876543209876543, + "5126ccb16c2146b5876d6a7da63e9fc1": 3.3044077134986227, + "709706096071415b9f3bd1d558693d1f": 3.716666666666667, + "b76c05c683154ddc867e4d85ec7dde3e": 3.612676056338028, + "dc8c14e68d7a4de59c48ad5d412e8bbb": 3.8, + "90f41b6c201149f0846516c2221552a5": 3.5733882030178328, + "182e7a241cd64f309a013f89f2dde604": 3.0169082125603865, + "a58f4fb5e86543e9862b62598fa4913b": 3.4682472480948348, + "de6fabc20e3b4e1db4b039330d57e641": 3.046767537826685, + "76dfa9e332124de5a5403fcf51e61050": 3.352517985611511, + "f2bf4c564d0d42a39241e98a4e6c493f": 3.386058981233244, + "38603327dbd2472a93a652ce7706ef78": 3.5854922279792745, + "08a5bcca5621439cb1b93a663843d75c": 2.675375571521881, + "42c1487f04e648fdaeb04aed702d05ea": 3.757142857142857, + "0e608dd6d8794d0aa40b1a1b2748b699": 3.3427835051546393, + "bba6da07402a44e59a444de6f6b672d4": 3.1384615384615384, + "18c06caa12064698a1f7377225b37aa2": 3.725, + "e967915d991c4cfe96e56cddc8acbc8e": 3.5310344827586206, + "72556488e2be47fe9980caa01e3fef35": 2.99163179916318, + "eaa1ffd6d1b04beb91923a028c426db3": 3.0296442687747036, + "08f4c31730b4427f90e8146fef066c4c": 4.029411764705882, + "2c3dda5236e14337b8a5cca80754f0a2": 3.720720720720721, + "11713dbc6908422a8fb942ec607cb850": 3.4944099378881988, + "bca053d2f77e4f918fe652c195402919": 3.5750636132315523, + "c486d8856c1544af9e673cc9ee53c8b5": 2.7987616099071206, + "fa27803acfbf4e55ae341d7624b515f7": 2.6580142764438675, + "1dad57f7698846fdb28cf2ba7631f181": 2.989333333333333, + "c72329fe893f4158b58d9fb6576c19ef": 2.952443857331572, + "c49bd7aa449844da8c67a1bb6797314e": 3.5843071786310516, + "9df42173e2c54dcdb6e241fcc2e3f186": 3.751336898395722, + "6be05541db3a4c8f8e82169353ed6aa2": 3.3464566929133857, + "94188a5a701c4394aabfb2214a6572b9": 3.040625, + "74f48ed284614abda4e67c724e4d6af7": 3.668730650154799, + "4ca88202f05d4923b0fa2dcb3b6b637a": 3.7324723247232474, + "eaad9dbf75df4d2a90e328b155fcc978": 3.478048780487805, + "4ded8a39d0e64298848d673579638960": 3.0007326007326007, + "126235195ce847b09b1dab1507421e64": 3.259791122715405, + "73c564f6e702456fa85e903de4824d3c": 3.6095238095238096, + "5de62efe068247c4bb29dd5b37b21d80": 3.0182370820668694, + "2afb354c2f2d428e84a21770ef617b4f": 3.5353846153846153, + "8fbbe76d4116423dae03ecd9e61b90d5": 2.5237215033887863, + "d21f707273ae4e16b80365e6c5394df4": 3.134185303514377, + "84924e6ca98b4dfda5c32c335dc24324": 3.143119266055046, + "a1fa94f6735545c4821bec31321ab404": 3.8732876712328768, + "0a7a83477d19422db773738a353b2795": 3.6610169491525424, + "68a3c3535f404350a2ece3e8c62211b5": 3.43801652892562, + "9bb734657ad7475589f27e94941aad68": 3.345, + "278ebab0c4934903a93d6eb1e213f6cc": 3.4375, + "230db901c3d94ac4adfc7716753ea7e5": 3.029209621993127, + "326e163b8bc24f74abf125254d60051b": 3.5072992700729926, + "c3a8af3647d342c9b7bf92ce6c550b26": 3.206896551724138, + "69d831e14137427fa31d6e67761555e7": 2.9586312563840655, + "b8832921930f4ddd835e982a25655aad": 3.1727343144848956, + "90443bcccb5245b99cf1bd881fd07b33": 2.904471544715447, + "8884100957344de0a9304c38f8303f1d": 3.118333333333333, + "622f7fa890e14cc081e63bc90170c4cc": 3.4513618677042803, + "5359bceeb76b4ec89e7f0f5cdd0eef21": 2.897810218978102, + "6023d8b3dd4e431f884661d68a080e0d": 2.452296819787986, + "4ecf0309246b44bfa5cb3f923ada6909": 2.8149122807017544, + "34bfdceb252d4c9d9bf7292918d0b009": 3.4424460431654675, + "dbcd535e14ac434ab14febcb9fd20591": 2.809116809116809, + "3144677d7cfa40bfac5f79706e0d89ad": 2.983108108108108, + "008581155fe44bb3a8c8ee0fc487accc": 2.522448979591837, + "69b200ed82ad4a4a94d79f71a2e55f60": 2.849217638691323, + "e36c236a0f9e4680b0c912df870c1337": 3.1604938271604937, + "c1cadb71190743d6aa36002687b6a652": 2.645, + "7e9b625014474ddb9802f49d49d3c241": 2.9812332439678286, + "af288f0c78e44a44956c83b24e735a1c": 3.5120274914089347, + "9708caa32e7745be8e02d9a60230070a": 2.2874493927125505, + "5491c23305b24c228345a1851f62abe6": 3.191637630662021, + "7f06f76a0d9b44619dc88333ecf84804": 2.8457683741648108, + "c73c9f3c32904bd3a34abc810fc846b3": 2.8341584158415842, + "f1913a826c9a49489a45c31de573bff0": 2.8603351955307263, + "d02988575e684bb49870395994b73cb7": 3.1171993911719937, + "adb913eeb5554f28a3d351ae09dbd5c9": 2.511392405063291, + "603d30fac52c4b26a7ebc97f259492c8": 2.809327846364883, + "2e67f29f99014dac9506e8171e7a10b2": 2.9441707717569785, + "c5a0f378314a48dba65d0fde859acc5b": 2.4547038327526134, + "97ace957a69d47ed9bf9975c6b771c38": 2.875, + "c357f9ba56504b14bff0705fdd6ba6d8": 2.9011627906976742, + "b04362512df1446380a50bb836eec67e": 2.6876640419947506, + "0e3444a598024eb395c0d3ba4cb1f5a2": 2.5860927152317883, + "b92e85ccee204e2f87a7c45f2a423660": 2.604575163398693, + "91ff40656d5b4634af4ed3c20efa3695": 2.98978102189781, + "edd7f64588494f5b80f58f5bbf4c84ef": 2.8443877551020407, + "b11aaab065b74ff4a1b9332531c0e7cf": 2.847560975609756, + "eeea2b1adde64034a2e4fceb32cdb0f3": 2.6885245901639343, + "3400c61bdf344c1ab4c6487808ef0a86": 2.817135549872123, + "aae8375e3db1404d8def0fbbdaef518c": 3.101123595505618, + "a1c61c51df7d49709a2c75dd1c715469": 2.984888888888889, + "04c2d712762d4e62860763ceda33c092": 2.6183783783783783, + "249cebba509e4795a30e63eb1aaa6cf2": 2.8723702664796633, + "62bc739ee6a347929da2589a6c60e72a": 2.8238021638330757, + "c9349198f0ab457abed1d3d3aa385300": 2.804915514592934, + "e4eaa7ecec01471aad13e693b5ca0ff8": 2.568025078369906, + "622debd5a1214c4dbc5de12fcda92ba9": 2.951008645533141, + "a06319eb76da4f31b5031b3a88225dbd": 2.9148351648351647, + "ccbbb8c5a0714f9d895acb62822ef44c": 3.1039260969976907, + "3c945fde276046cc88de06dc392f2db9": 2.8167372881355934, + "1febf8ed22dc4a21b845c53b4c91b91f": 2.493150684931507, + "993d9dea949a4044a10831ab7b87afc1": 2.817073170731707, + "36e4b2b1560d41e6b82783279b7a5b0b": 3.4565400843881857, + "103461985f0d4571b9f9408660bc1c21": 2.7018469656992083, + "8a23d167221b439680731869814bd2b7": 2.682139253279516, + "56293840be0343cea95fbf14b47f9869": 2.5745454545454547, + "2d938d5a4da349809af3e91e44acb026": 3.5008880994671405, + "770f1aa84d6540c8b2a65d068a6604c3": 3.0929519918283965, + "b956e284892c4ec49a7393002cdc28dd": 3.41025641025641, + "607c9958ffe14ae292370d946cc56b41": 2.915302491103203, + "5e5fdfdf5e034dd3a39fde4ef578af1a": 2.615581098339719, + "d6596f0d56084c2eb2ca545f2a0c9bd1": 2.6105799872530273, + "552dc297eb864b1b87ec612a442ca3f2": 2.688014638609332, + "06a2b184a8864774aa03b38edd7a42a1": 3.282051282051282, + "3e919c3be6ec47449da08474872b327d": 2.6906531091122408, + "1662d26367bc402fbfe14001d3d23ea6": 2.8926553672316384, + "53f4b2efa2d6499f9cfeb2cd7e0400c1": 2.9118683901292597, + "b114e73446bc43649ae46cbde3cc4e3d": 2.9962335216572504, + "6f68f767a7974fa7b93e9c4ae02ad205": 3.128680479825518, + "099ee93beb784a0381dfaaf54c0b838f": 2.6568848758465013, + "5478926013a4460e83766c23182b8558": 3.287319422150883, + "afa5bca3af454a7cb09462034e6d282b": 2.4960390006093847, + "ee33b4e809e84757afb898c91e3f2eff": 2.7566137566137567, + "e7eb0a40ddab49408874cba105c05135": 2.9298998569384835, + "757458aa46b247a9846276810a21155c": 2.4215626892792246, + "e9f9d5948f7247f2bb96c7ad27373695": 2.8201284796573876, + "f4ec8f581c594d39b15da7adeaac5f70": 2.7675675675675677, + "33c7e741d6754d85b7999adf26b437f4": 2.981329350261389, + "051ee8b0529e4406962b514a13bb653c": 2.6597402597402597, + "ed2e6229f77b4746932a8c3665ebeb58": 2.861818181818182, + "bc3f16b8c6234d83bd9fdf99316de4e4": 3.125779625779626, + "c9d8860831a445209a5ce990be3811de": 2.6, + "47534ec0377943708c55cb1bfad6c83d": 3.558644656820156, + "f73440d5701f4feeaeffcacf282d5607": 2.541176470588235, + "10983429e00549a3bf63b7ea733195af": 2.7694388100067613, + "90180668a772489fbcb5a2157c181713": 2.6235632183908044, + "10266986b3974f5f87cb1a292bcdba80": 2.8209366391184574, + "cb90814119944cefa1529113e8ab86fc": 4.415384615384616, + "ac08cc1b70ad47c799bbd8270e4d1a0f": 3.217596229379419, + "d46506160ed243ccaba50a3b9ac06612": 3.0646830530401035, + "632b4df3f76d46c0a4516ad87e1fbd9e": 3.9536679536679538, + "c6020b5671944433a17e177ee0a0e523": 3.1020408163265305, + "8c9dc360e57a44ea8fc574105b4d8843": 3.8085106382978724, + "2b4ee1d6c47d4cfbaa746c653f182254": 2.958097395243488, + "af3a1ee372034c739f8e307ecace453c": 3.9460500963391136, + "bf6f877c479e422eba87f0eeb4b856f3": 3.116531165311653, + "2e78f12cf024427398b6a0b89fb85dee": 3.1727343144848956, + "7c0591ad03564396891682194419ad5e": 2.773188896411645, + "52a56dfe319f4f74beb67271a9fc8936": 3.152360515021459, + "41932dee5763472881ab549878aeb51a": 4.019354838709678, + "31de87f7550648da86e5147a1cc65b2c": 3.9121298949379177, + "b6648a442592439da879588e0e43bb57": 3.4080459770114944, + "fa7b62014c7f401ab6f2dbe9e0fc943e": 3.5128644939965694, + "f89689087e784851bf5fc9720d82e689": 3.61624649859944, + "828d65d40f3a43c1b62a3aeb758bc849": 3.551282051282051, + "99f44333802041ada2a36fa30883ccb7": 2.756393001345895, + "0efb96219a4c46d5b42ebbb12fd51ce4": 2.683381088825215, + "623eab8ad0d047fa957ec6307024718c": 3.1362940275650844, + "b9b5588862dc4c9080f77ec7aee6b769": 4.293103448275862, + "1a732d5227b341c59a3728363879b8cb": 2.9257142857142857, + "1332cc8dbe08499b92ff62a6d120ead7": 4.453781512605042, + "7764656f3730496290cc821a6decd249": 3.1171993911719937, + "3f4b10384fa74b1db40f0e338b385e9f": 2.9872727272727273, + "0c221341bf36432884031235ce2639fb": 3.9747899159663866, + "7867496118734f088d6918f87c9fe3e1": 3.4076539101497505, + "c4f8043e377c42bb9b03ebfafdaf20d8": 2.5648090169067, + "36dd55922f7646e29617b5c9e7e8047e": 4.025316455696203, + "6c2b2664fc9e48f981c86d984a4ad67f": 2.7694388100067613, + "749a8cbad2b84d989cbda0e5067b6f77": 3.69009009009009, + "30f201e7329a49e989edea2ed3d740a8": 3.37123745819398, + "e30bd068509844fe8df187a0960a861d": 2.9236259814418273, + "d50a0698769e4c68951f982f35489fed": 3.1302170283806343, + "8bc49b74f8fc40c5ab4ce9b0ca21e4cf": 3.416180150125104, + "1817ff34804f46b29d6fe10a08053f28": 3.514705882352941, + "3efc6992406f4e08bbb4c562dd668a2c": 3.034074074074074, + "4a8cb21927ed423aab156e0bf9db9a38": 3.4463007159904535, + "cee4837deb3b4f95bc4b556a1c289911": 2.743469524447421, + "573e688d1bf84af4b3c7be620c912c39": 3.263917525773196, + "8efa1ea752e74352be0cee85731b6ba7": 4.233333333333333, + "e4fdfe1b81ae41b2b3a8028ccd330710": 3.0130434782608697, + "a263497e25f04d3a8a77ddc58eff255c": 3.88268156424581, + "bd5b0d9763bd449c9be2d7aaebbc8237": 4.082278481012659, + "843497a1304c4a8984afcec085246405": 2.8927738927738926, + "23470a8261194e41a8379e07be2b35a5": 3.894243641231593, + "1f6a9f62e4344ab2bac87d75d6b5f0f4": 3.079699248120301, + "0766855ff16949e29efb17729184984b": 3.0889894419306185, + "56befc28d0634f6c821403fe8535cec2": 4.25, + "ddd46664ca574850a2b369922495df2d": 3.1411042944785277, + "f4387010eadf460da9502171acaf4d85": 2.951008645533141, + "54e9d000b0d04ff6aff461551f3a4fbe": 3.212549019607843, + "c975ce4e34074b0f95d4f5e07db6b72a": 3.34392523364486, + "8b6d50a2fba949f1890557a860ad9972": 2.8209366391184574, + "790d88a17d904190bef98eef9be43006": 2.9767441860465116, + "14afaf182ede4796b2c5ad7df0ad26b4": 4.158333333333333, + "43c22ce01b0f4992a3749257c79bd565": 3.1850699844479005, + "a87d22e12d4c4b6795323ec85c1f6457": 2.525277435265105, + "a766cbba2c64440db2999dc421834861": 2.842470506592644, + "ff44c6c365af46b7905b7321da5997a7": 3.207517619420517, + "126c70bd6d2240958e92dd1c75b95eeb": 2.9552669552669553, + "f1eee8a8cd814a7e9dc0abb8533700c9": 3.148347425057648, + "4b193822abd64ea5922f52dd9862b147": 3.5641025641025643, + "1194419af28e4f30a9c10b8dc8147781": 2.752688172043011, + "ce03d5987b2a48ad8d20178109dd69a0": 3.0889894419306185, + "66dba8079e7649ad942f7d5c54d14aac": 3.6146907216494846, + "3ff8588681a24a38bfb1dbe87f19a5ff": 3.9422521655437923, + "6f0b49dec7d44a09b21255712867b15c": 2.8209366391184574, + "145cfa1348e749c5908f6cd18f586688": 2.8885754583921015, + "c6d8fb41e00c44f383a76e6f2756b85f": 3.6690140845070425, + "1e0e4ffd12524f2f8c5c4886db4c9768": 3.898550724637681, + "3a59ccd8bdff4b1398478222cf0e360b": 3.065868263473054, + "350cf51bbbfc4c01bcde7e9649e37b8a": 3.3327908868999185, + "b09e1c6564c1470f8ef16fc0eb316c23": 2.8563458856345885, + "a3ac2c931db84417a211b7d8756b5ff1": 3.0384615384615383, + "2caa2289a2574756bb51967df812c420": 3.1901840490797544, + "88ef499cf4f744da8f11a5c0bd2d5076": 3.1580570547417115, + "62a4c1a709c04f9da17f16263d3d997f": 3.0397022332506203, + "25e0e585858b42dc95517ee5be249c0f": 3.21505376344086, + "dff99017bb6b46aeae637e6afc655609": 3, + "a2d03fa1de0d493aaf7a5300d20ac74b": 2.8232044198895028, + "80172db2fabd4685ac88ad275dfdcda0": 2.979775280898876, + "ba990ae0a48f491aa735ea56e4b1e8f9": 2.8644578313253013, + "dcd4ecaca5154d5596d9c928b3f6aec2": 2.7719568567026194, + "c7fa9dac8158497aa0063a3e9a83cca9": 3.106145251396648, + "64664fc85aad4f578f1e7d1a59faa434": 2.973293768545994, + "ff1c648fc35d49cdbdaef9c58032e275": 3.2933333333333334, + "e368f3118d984139b3c924b0a6446601": 2.7449664429530203, + "54c09257d56d4602b71daad5ebc754df": 3.1436567164179103, + "f5e611a4ff9644afac933e79c6fe971c": 2.9551724137931035, + "892ca258e1324df1b3e940b463008b45": 2.8156182212581347, + "95eb5cd3b5384c3d81933ccedce643a4": 3.6202531645569622, + "6f3dbc8404d9411ea298dd8fb7c34d5d": 3.120218579234973, + "a90682e761ad488cbade63c0b82d7d6a": 3.079510703363914, + "b0e1cd874a3847ae9aa54262954d62d7": 2.9176788124156543, + "30d519a5268849f49ec5e3e45113b653": 3.510989010989011, + "99ebcf1d7bf2444ba694b35f3b03272d": 2.948936170212766, + "916b818f8f5f40a493db6477b77c3ea6": 2.866359447004608, + "e8feb75b28244ba0ae0511b10fa69c30": 3.311377245508982, + "53dba76f85cb43a685350b0aaea85527": 3.1062271062271063, + "da8ba3f679ef48fb8f47c2aa4a33412f": 2.88126159554731, + "e4b9605669c44c9bad2448ea9d528a63": 3.141025641025641, + "cd8820a1e8424972844569e672ecd7d4": 2.832460732984293, + "f2e7462fc9c24681a859464564708692": 3.326283987915408, + "5a555855c1214f5c9eac0d56568acb52": 3.045, + "d483ad52d357465e8724c2542ab76da4": 2.590250329380764, + "e190dd6bff8d4054a178665fcf7e6f0c": 2.958064516129032, + "b9a97c3055334bbd98287bd7651322d4": 2.5913978494623655, + "c66000d9ed434659959d302c93527ace": 4.289005235602095, + "5c953850b8cc4464bc074a1a4e8e594b": 2.845679012345679, + "7806bad94c8f4fe48d5101000ce8d12f": 2.6830357142857144, + "ab96316b809843ffb3dee5d509499c3f": 3.02710027100271, + "121692a7e45246b189cd46616d399253": 2.9613526570048307, + "a679342425df4cf4ab6929f024d8603e": 3.0254237288135593, + "5486a5ac3f42451d8b471a0147fa2d3b": 2.919811320754717, + "51e8dae347994759bf1053cbd5bc7909": 3.1770186335403725, + "feaece4949104cf1a98636d67d1dd8b5": 3.115702479338843, + "787f7ae3d85d4e6e9f055b8547a608a7": 3.292604501607717, + "90f1c495485642e7ad513deb1988bd9c": 2.9081225033288947, + "efc7e5af1fc04eec8820ff2b47fdfa6a": 3.016867469879518, + "5a3467ed8eaf4df8a640332288dfa924": 2.8181818181818183, + "7c7a32e1b09e4fceaac9e86ddaff9b49": 2.767088607594937, + "b5079f0f7b6f4e1cb836376c2de02245": 2.9093023255813955, + "69ce786604c14ab68fffc1e9ad723508": 2.49636803874092, + "6ecaec61fd41475eb29cff0d6277829c": 3.1943127962085307, + "3ec172d4dcfa4bc7bb39cc03e8f1c90c": 2.8005284015852046, + "bdde54e51f5a46b4abe61a701deac70d": 3.0216346153846154, + "f6a2a8f72990473780eb77903d2f33ac": 2.731736526946108, + "7f463a3e532b49b88b5cffccdc680e05": 2.9204545454545454, + "85e93a7be81e443ca0167b0d32adfa6f": 3.138996138996139, + "851ea8d297ac4a468c6c1e6d239f304f": 3.0691244239631335, + "91d6fee8c0fa40bbb5db5acf07b95fbe": 2.9151624548736463, + "cdffc005539d40de9d901b8140eeff8c": 2.9652777777777777, + "2b9de4e51059411c90b916f7fb551aef": 2.81491344873502, + "6688448dc9dc409095af2ba569e7e3bc": 3.240549828178694, + "1bfd1c5614334177befddace53cd7007": 3.2465277777777777, + "8670c5f4c62e4f278732a56377127090": 3.0293577981651376, + "b2d9ab1ab4d34417a06ab8688e005307": 3.5131761442441056, + "b9ba803e36444d06a1a0fe9c8a3c212e": 2.7807692307692307, + "b3c61ad852ab43ebb0c26975cdfb3fca": 3.022222222222222, + "1428f429a5b241baac49b0bfd128dfa1": 3.1521035598705502, + "6c8d4ad833c94ef6aa1097ee8adfc9d7": 3.933333333333333, + "043446326ad2464ba2e229e5da57f6d0": 3.2677824267782425, + "baeb957a50ef4b3eb36f1e57203e4bcc": 2.8840579710144927, + "adc5ed5fc2854c94b115dcd34d0459a5": 3.1965317919075145, + "a23b9f605f654b48b33bf848a212ba66": 3.0664451827242525, + "cbc6240bc636468a8f1d6f36bff29e7a": 2.8164983164983166, + "d85b0594cf55419c9413e8f01b8d4d53": 2.7950310559006213, + "e5b733600f8f45c7bb586027e65236f8": 2.8226299694189603, + "ed48fd5da7a947b595e732580e2c5650": 3.356589147286822, + "93effafaba524472851b7662632eacac": 3.061990212071778, + "ebfaccf5647e40ae866ec574dbd4d945": 3.0056657223796033, + "782c9b11490e47218dafaedd7a9c7783": 3.005820721769499, + "6b53a61bb07b4413ac36103a043b57b3": 3.0869565217391304, + "87791f973349454392b3f0f17a09b88c": 3.3767123287671232, + "d8f87fd586584e59924edd9426dd73e2": 2.823529411764706, + "16c110a393f5465285afcaf0bba7a133": 3.0576923076923075, + "0929674f400c44459cb259c55bcb959c": 3.2948717948717947, + "15f1eae72fc04bc39a87699c640692ce": 3.5480427046263343, + "d36a9c0d6ca141bca508334ee0fd8a5e": 2.7241379310344827, + "57e2972168f94400927b2d9bacdca910": 3.43859649122807, + "003d477fd51145f8bcd8f3fe35e1a85c": 3.1724137931034484, + "25d368ee28ee4eb8aa7b3879636043f2": 3.8974358974358974, + "72ed8a0c5a224c58914d9c57f3795875": 3.7257731958762887, + "319e493f6fae43ba87392833e6dc9e0e": 3.2771618625277164, + "20c5348bede34727b2713268caf935cf": 3.9054054054054053, + "ab60309723424b9396f7b2a2993af3f7": 3.5077399380804954, + "0c12db672870460582dd7fe4321bcc09": 3.2507163323782233, + "547c6ca5b7d94c7db11797e2818ff03e": 3.143939393939394, + "b99611be247d4cfab96244d42eb5f568": 3.14, + "7f25eba0e7ca435b9b6072ef1945250a": 4.059574468085106, + "abfc39f4c4fb448286ee4da9ab90bd43": 3.055776892430279, + "74444417b37d47efa666334700a1d687": 2.979591836734694, + "85667316b13046769a68805547c3b04a": 4.044334975369458, + "997bc561acff40b486c1ecb8830eb16a": 3.0697674418604652, + "d505608d1fb84e08948c51b06b689226": 3.7766990291262137, + "bf1ff08bcc22452b8dde9636c150e3d3": 3.84, + "c3eeb459eff44169819720520d038586": 3.1125, + "a0c7dfed9d6142bc91b3ce1fef2930ed": 3.1630434782608696, + "3dd2298c9478497db805c74862f9fa9a": 3.604761904761905, + "805e8b2fdb8f4797bcbc797954451577": 3.8, + "6799f966c4e24422b7efdd705a5ab21c": 3.608108108108108, + "5f0a15a8bc854d5fba26751eee4dac03": 2.9683972911963883, + "33c807be61ab48e3865cf60ab4883b88": 3.8520710059171597, + "ddc4b91449a847c2bb855a38dc2a1433": 3.203166226912929, + "9807e96e75b64fa1ab39dd7f5753c126": 4.256880733944954, + "cf75757524b4492083ee5c257f517e7b": 2.9863013698630136, + "8227a9fd61ed49449776e2c8652247de": 3.119949494949495, + "1d3348678c684a69a3b33c7290894cb2": 3.7777777777777777, + "30aee928ad3c4e7d93e420545ec41f9b": 3.498186215235792, + "665a282e8278486ea3c204fd2e12ed1b": 3.3076923076923075, + "9748ad0778ab435187d0049e6f6de1b7": 3.303901437371663, + "a66255bdfc904a1e82a95d29a9db13d1": 3.0396039603960396, + "6455672462d84e8c8287d32679c71a3e": 3.871313672922252, + "4ee80ea583d144cba410406a4b2c9077": 3.128279883381924, + "8547409ccb0742f1bf6678ea09defa96": 2.779816513761468, + "afcac885694e44a88d6b28a5d46999c5": 3.576547231270358, + "dfd1569351a543239a8ee12497863f84": 3.780821917808219, + "85c384e4142e4e54b32fb5ffeb8acf97": 3.1005291005291005, + "74cd8d6deab5419cbda3d5cadba49763": 3.206299212598425, + "517a61ec86ab4c4cb511574e3f1806b5": 3.9469026548672566, + "025ad33d686a4b14ad6021cc041a400c": 3.0576923076923075, + "217c9418ae674ad9bcc4b236fdaf8801": 3.6893203883495147, + "b3ddda1eca8043e0a5a0116eb1377a2a": 3.2401129943502824, + "2c58e4b8296046c38a82d462e39a0b7b": 3.111111111111111, + "01fa8e8862984a30bc598b65bd77ca95": 4.098522167487685, + "af25231644bf42989a21f89e900bc50e": 3.4013377926421406, + "884b8de7bc174800abedd9c87f24038f": 4.165048543689321, + "e7fd1bab38f84bed934068f00755f85e": 3.5260115606936417, + "24984241d4164639a65fd876f07946a0": 4.4004329004329, + "330006df2dab4792b8c48c97b22dbdf9": 3.0419161676646707, + "bf804731358c47e8888dae6144b9311f": 3.2511415525114153, + "ace0ce03c1654f3bb5ddb5fbd8c059df": 3.353896103896104, + "43c54b3d1e324131adb2187da5e6691a": 3.690537084398977, + "8e4d92f78d17492f8f86b2459cfb3c87": 4.555555555555555, + "325eb186c7f242d2a6431faa2b161157": 3.452991452991453, + "8a4f1370bc814d65b7127cd265574067": 3.3550295857988166, + "f369ce3acfc94c80b00343500e364cb0": 3.0806451612903225, + "7ddae1ad89d246d7aeac324973523fef": 3.4285714285714284, + "8b08c14b779e4eefa06849a15cbf6bf4": 2.970833333333333, + "dfcfc5138c654f90b827014cd10fbd49": 4.333333333333333, + "e546b2d9f6724cf78d24a3ca80c8d005": 3.462962962962963, + "2934786458e74085ab0a5cbb3f025d41": 3.3702290076335877, + "25249f1532d54c36b4b52c9c553a8fb4": 4.147679324894515, + "04dbbad1651540119148d95967058c11": 3.2052785923753664, + "0d41a1be06fe428b951b07bd08c0c4c3": 2.859106529209622, + "2fa028fff27842b99328af080320349c": 3.560693641618497, + "a747d4aa3bd44a31be83b31e6f8551fd": 3.0961538461538463, + "7704487c48cc4bcf85c112a3c8fe3a38": 3.07035175879397, + "0d330b147a83455ba51e6433c4af0e00": 3.4814814814814814, + "48877593fb044393946bc2189c257844": 3.7142857142857144, + "5d5828017de64529a00f69210aa2e735": 3.2506963788300838, + "26317214c6e94d1ea28e7d9ac0ce0a2d": 3.562043795620438, + "37665162ec8a443b8656812eeb4685b5": 3.9580152671755724, + "459e9fdf35f34b8c95d5327ca99320af": 3.3685474189675872, + "8319d90fdbd84f42b18c840420381f34": 2.7612203731719616, + "67f401c0308b450eb0ccf9cfaff76bc6": 2.7957714799820064, + "72f6a3f4f0c849e8b0eb9168edd6e18f": 3.5037406483790523, + "b844f24d694e4faf84e78469d6d583f0": 3.7205882352941178, + "738f723bfc404949ad32acc44c11fc7f": 3.4253611556982344, + "2824eb032a7d4ad691cb7b9da21bbe36": 2.922543352601156, + "b07ac369f4b747e0878ca44352151eb6": 3.7264472190692395, + "4219e4f1495f4e869adcfc74d2db71cb": 2.883986928104575, + "73c0104a36e64dc8bbf3cafdd5aaf6f6": 2.955056179775281, + "4835b88294ad437583974305a471f51a": 2.750663129973475, + "197396b0970342efbbb1272f78ad6554": 3.163568773234201, + "dd90a6125df04e6fae99dc08fb853250": 4.186440677966102, + "8389ac2f20c74f62a2704a88fe36e297": 2.9129092032118593, + "f6e36904f2fd420ab76e2d1484425c7c": 4.114754098360656, + "b29dc4bb22284c63a7f14b9675afa3ff": 3.092948717948718, + "d698a38be25a4323951af9d211049a3f": 2.747373564622526, + "6a2aa29548a54fc9979b903d178542ee": 4.066666666666666, + "fbd9b99ceedf4a70b02a3e6ed72b46fb": 2.963308872581721, + "5fe103e8a3aa494a8856e93a59cd1eab": 3.8160377358490565, + "b13cd37711d74b27b08787f1a626a2f8": 2.883779527559055, + "697c90d5dc6b49c8b505bad4ddef4708": 2.934536298783047, + "cbce698bb3a44cd882f7d9bcbe13175d": 2.9408185840707963, + "597756b38ce1445b99a4de130f07a62c": 3.5283307810107196, + "0af14f1626604fb78c17db8e076be523": 3.130841121495327, + "b8948b102f4e441d8162073c8883007b": 3.9504950495049505, + "b13893f6d1dd4565ada93a7f35a00bdb": 2.8680033416875523, + "237d66a760e04c0f9717603002db502c": 3.972972972972973, + "0042b6eb553f41398140f79a252210fb": 2.94375, + "26f6de622e9c44cba7e732f0240065c2": 3.5268456375838926, + "52059470a72041afb700ba289eb89d2e": 2.9710327455919394, + "511fa197e1cc481984a9ae3b6aa5f984": 3.625514403292181, + "3014f9a16b16444ab74a44c640334490": 2.981333333333333, + "ac4685a7186d48f3abfe7441315a1f44": 2.8463611859838274, + "d1d4aa45b6a649fab79f9e36baf720e9": 3.0787878787878786, + "43469ada6ab74746b90f43b4b986cdf1": 2.870833333333333, + "9c5a534971d74e6b8f3e3ed1805316a5": 2.9386056191467222, + "e262112bee894c1398f42eb4f4ad8950": 3.946564885496183, + "6ad38f254dfc46f6927fcd9f2627f72e": 3.9774774774774775, + "9a3ede14175948b282974d2803fff0d2": 2.8996153846153847, + "b888d611264747c1832326d6674f5dea": 3.038327526132404, + "4c9931a13ffa44d38bfc9648b07308f9": 3.404580152671756, + "b02f239a7d984743a8168280aafa329b": 3.7948717948717947, + "7b37f555e73547ce9fcd0f9fa1ace96e": 2.9066666666666667, + "bf1b0b44d4de4a3c8af566ce9a053318": 2.9133574007220218, + "b465c2ad15a74f91bd512fde379d491e": 3.471875, + "71a2947fa75f490aba088eb9f16037c8": 2.902905740609497, + "fb9a529e53bb4c0684f8fa4bf0979bf5": 3.3758064516129034, + "ed0fbb11a00e42b3afa399c9c3a343a4": 3.460674157303371, + "c1c4e3b172c24c79b69a56a7c7fb057c": 3.160377358490566, + "36f3f8eddede4926bde631d0b5739751": 3.476449275362319, + "ccdfebcaaffe4d8e849e1f03d683bab1": 3.1283783783783785, + "c150ae92a5ba4183817c7a78adfdc94d": 2.91967291967292, + "195331b751c443ce834a63743fcff239": 3.313953488372093, + "8a6440a42b874d999caf35903e36d9ca": 3.496031746031746, + "cefd018f39284d0b9e31e2da0867ebf6": 3.7142857142857144, + "35e47de267b840c680bf7de8a7a9a291": 2.5609756097560976, + "150f42512f514ee6b63b2723fc2a69ff": 2.8674698795180724, + "28ef4c206f79443aa5337b33db4080c0": 3.6623529411764704, + "d8399e70c5564e3a87afb7585a4b3396": 2.8554021121039805, + "dc707b3e22d44b9fb91fe4dbaa0579fd": 2.9012104646622414, + "cc1858924edd451cbf924ea047a8992f": 3.0288875230485557, + "378bbbfce1394cc195be68576a11733a": 3.6701662292213473, + "3ecbec93945546e3adc260536ef6602d": 3.9030732860520096, + "7a9f9338da464e73ad62de6f34688ca4": 3.18, + "2edc2280761049d2ae2a705ec48e8547": 2.870577281191806, + "9d3de33149ca44a3a81009faa3060b0c": 3.4049586776859506, + "2170cc31c6bd4a54a9a46569008b1664": 3.24, + "6a7cbf994f134f4fa1be5e11670b9c67": 3.0045662100456623, + "13027a52cab44af49105732cab0fa27b": 2.7716701902748415, + "62c8e7dfdf7b401b9d57ba662b48b27b": 3.489795918367347, + "392d8332895e4d70b3e86341a0aa6569": 3.0529595015576323, + "faa4b8af57494a5686780d41a703cc7b": 3.1511254019292605, + "fd444d3bd311408c9d70ea39adc9eb87": 2.880672268907563, + "f6a2344dc2c44fe7b93be358ee1eafd1": 2.9526462395543174, + "7005890acb72418886a03079a0fdce94": 3.0306513409961684, + "60b60fb5a46c4662b0c9d7551be59904": 3.473170731707317, + "8878064a509c4c3f9f5be20eac79fa73": 2.9705188679245285, + "79db1de2c50b4f218a451fec510916dd": 3.8334485141672427, + "c2a9e662ce8a4680bf14ac5680fa7536": 2.704626334519573, + "7d07ba32e86544fca7130503b92c21b8": 2.699300699300699, + "ce77d02cc5c342cba9c797dabc3fff26": 2.727030625832224, + "4e52bf06e2d84215a31fb7f850c06276": 2.8723702664796633, + "b19c940f19924bd296d8699bc0a379b8": 3.1209540034071552, + "7c2eb79baed64872b99f05b3e30993a9": 2.533333333333333, + "cee8f4c8644446359f5a0353f3247ebf": 2.6072565245066834, + "48199ed80c954896a66c7479109ed173": 3.8460093896713614, + "64550e344f84467cbb7056fc19dba666": 3.151304347826087, + "815b09995b82434e94dc2b69853e8d74": 3.2482157018239493, + "eb50f9f9953944c0adc3c30c8f5f8e50": 3.1485148514851486, + "7179120179824e739237dff693588141": 2.752688172043011, + "6ffa3e1fad3e4d568caf0b018cfcb63a": 2.9745824255628177, + "0f807ffebf7d496fb8aa2169f692db3e": 2.9323671497584543, + "43f7fa142036413ea613472f14a21642": 2.963932107496464, + "79927e59ce2344c1ab0518e15a1af64e": 3.519461077844311, + "467cf40333434004b985cc9c2c939552": 3.023354564755839, + "7cf9562dac1346eba6a0fdeaf0b1929e": 2.8222523744911805, + "fa0340c7981b4b51adb41672ee977929": 2.900647948164147, + "31d2a4e0c65f4f3bb931a6a62fa209ad": 2.9008498583569406, + "e1f686c45f1d4b1ebb6587aade523e2e": 3.27694090382387, + "8912ec8e3f6e474aa529b60b81cf46ad": 3.054436987322893, + "3233c7aab23b4d7c9c0d6bc6ca4606f4": 3.764705882352941, + "0da0e35a8d8541b489f3b03333ec1e41": 2.9340974212034383, + "b39665a5800247bbb2ac1826d50ae0cd": 2.764621284755513, + "f5558beb886b4936aaf3b1b6f4066e18": 2.7221297836938434, + "5fd91d521bc74c36a1ba03e2015544fe": 2.7619689817936615, + "454ceb6d215646e8abec3fcbf80619f5": 2.6232876712328768, + "8c0db71b04304a4797c64a87fae97d22": 3.039568345323741, + "23fdd5d2491b4370a6a87e9fcb64b51f": 3.177657098525989, + "2b1d624a9878476b883e025d59edab28": 3.2952534191472242, + "ae464da777c746c39f80295fb944825d": 2.7125827814569536, + "eb2d9f0e56ca4b12a0fe01cab1203056": 2.727906976744186, + "3861557b5a6b4573afbe3e800e08da51": 3.0498883097542815, + "497159584fba466885c459e6c06c23be": 3.1981382978723403, + "8c022ddcc854470cb87aa5091ae79598": 2.708994708994709, + "9d9d53db038e41cdb22671a1946b6463": 2.717990275526742, + "1e7fbf8c83c444b583995afbc9b90aa1": 3.089974293059126, + "0bce5094ae47482faa036ee2d263933b": 2.7997265892002736, + "b8139de45ea6434ca0841b76c955db31": 2.972413793103448, + "a04a812d59bc44369c2ef5fb5b3ca696": 3.065868263473054, + "8a06d5ad95894b38847470fcda13dfe8": 3.4770797962648556, + "8b68108c5636475a81587c1fb0b37f8b": 3.1850699844479005, + "5a373bcd3496460188d7bd876259acfa": 3.1455026455026456, + "47510092ba9542f2bd2b252e219b937e": 3.477064220183486, + "6ba4093e4f8c4eb5b5651ea86b43de31": 3.1205357142857144, + "fb0ba83291bf4d76ba39c251190b1021": 2.698717948717949, + "61ffc7a36261431d8b782f50c0a65eb8": 2.628839590443686, + "809cd8ce44404b3faa74ed6673bfa253": 3.0363232023721274, + "e200130aadf94053809ac10b76ab45f2": 2.6736292428198434, + "be34ee995564455da0795118f118eedb": 3.0430906389301633, + "60d411949d2f4c9db8ff65d0bcb398e9": 2.8495508982035926, + "6d7f3d40ee584ef5b46a9bf53ce0c560": 2.635772357723577, + "5348f5d392ab43f7be64c34f575f9344": 3.5128644939965694, + "f6e63ee7d1bf49b58f68dd665c03d0c0": 3.4294478527607364, + "dec93811778141e49e34ad34a33821f4": 2.3380281690140845, + "264679df5d0c4ab0ac89f501217e7268": 2.7942857142857145, + "c3e52c99f7754dd78b5d7d03deb8864a": 3.4019933554817277, + "1978a58390d646e18464a11342e69a2c": 3.0163934426229506, + "a5ad1f30a87c4ba49033564b0f37c1aa": 2.880450070323488, + "00842277fae448d495f50d9abd63510e": 3.324675324675325, + "74e076747ca649a9b9f51ee2251fe8ef": 2.8643356643356643, + "675eebfbee6e4073a092a10270e30f05": 3.025487256371814, + "5659e06b30f34f19ad8c178d4c368efb": 2.6139119336311425, + "1059658605c1432e9f0437ec2ec6acb8": 3.1678267594740914, + "b5c283da9ff0473a83d6df72a7db9e6a": 2.81899518238128, + "69358599fc0a428e9aa011e73d3b5539": 2.7675675675675677, + "d1ee6bc6f1384e298f5a42e922bbebbb": 2.886539816772375, + "8624cf28b65a43a1ba252072d92bd831": 2.7426778242677825, + "a2d05f1cb660400ea91653c79b266bbe": 2.6788750817527798, + "1efebd4a5d4f4ddfb115de00f8ba772a": 2.9832483612527314, + "dade029da0b246e5901ebfcc74d2cc9e": 2.615581098339719, + "1e8fc0242a114a879666e59acccfeabd": 2.8654891304347827, + "029701fe32dc47bfb57ada3721595874": 2.7306666666666666, + "f9254bd1ccc74ebf98f857dcd8a39b36": 2.840499306518724, + "a52b365bece549a789bc9d27b9dbece9": 2.826777087646653, + "02832781df114e4f99719032ef0695fd": 2.7585585585585584, + "dde335ee4a7948e2a03dcd46194b7a62": 3.098339719029374, + "96f0831b31e44e989304e54bd2bfb5ac": 2.859611231101512, + "cb4019095afa4be39eab30fec88b936e": 2.9257142857142857, + "3a004ce5332e468a9376404ddee08e39": 2.7370689655172415, + "bec8b7f8659648118561e17008cc59cf": 3.5702005730659025, + "5beba2e064e244a49a78aa83a0ae23e3": 3.5150862068965516, + "5b30259fc26e4f598e76f752d7f21177": 2.467455621301775, + "f4e6eb18883e4760b5b2e34dd9680a4f": 3.4939024390243905, + "4c9161f2a8bb4be7878500a2a9a7fbc5": 3.0318282753515913, + "b935b2ee56fa4e5e907d4a9158bb6eab": 2.5128834355828222, + "a5d115601ddf473f9266410d3afd0d46": 3.4118268215417107, + "7fc942f907e14ab09d08eb99be931a6d": 2.9446441409058233, + "9d2b4887e3864a2ca8fdf49868563705": 2.8306841741534208, + "4a981eb352aa44a4bfa7cca954c9ed72": 3.153284671532847, + "e854958813e24859bde5180f3a66f2fe": 3.0095518001469506, + "a85d4a80816e4416b4a0e7fb0e66b7f5": 2.740518962075848, + "492bd5d1dc5f4d038c4ec2c56ce8fb3b": 2.918116683725691, + "b83b36147a354cf88d782ea28a935590": 3.0430906389301633, + "f5b2dfa399ba47608e53cbfacbdffeca": 3.1223021582733814, + "1d2aaf5541b04cc59a12f23a5443d275": 2.6133190118152525, + "2572ab5ab5e0488cb41e2384c95d504d": 2.842470506592644, + "da81feabec7644328e6620c3a2695d87": 3.1298076923076925, + "4855039f995f4e2a84ecc18d213754f8": 2.5339195979899496, + "2c2852c41ff7405f8b90e88191dc6bb8": 2.3877551020408165, + "3a6a61dbe9f249e285fa630dc08bc950": 3.0099557522123894, + "8906c3002a1f461387c46af17a86c3b9": 3.2379446640316205, + "f9b42254b76a41759ffbffea8d7e3417": 2.874862788144896, + "cadac91230a14105b4679f8e5b7f2031": 2.4519716885743175, + "fd260fd4dbf6467c9c255c73dde29eee": 3.6441281138790034, + "a513a13d4616426a828a798738ef9873": 2.587492103600758, + "a4642be1cc8d457eaddbeb82f151b02f": 2.763157894736842, + "d8a97d79e4df4e209590bbba75796b6e": 2.773188896411645, + "68082716b22c4463bedf99a8b553a6e5": 3.319286871961102, + "35560bbf930a45ff99ad8f44688ab0e8": 2.816576086956522, + "02ca690c74b14ccb82a44471d97e094e": 2.993710691823899, + "5dd205ad19154f179a4cb8e79da8deaf": 2.7312925170068025, + "30cfe8efe0ca429889c21b68ab9b0957": 2.5545741324921134, + "e084de6190b9492095b8bb4c999e0169": 3.266347687400319, + "da934ac93ef9469785b65b08af270d2b": 2.7485907553551296, + "d7d4940031be4ce99d6668d5739b3866": 2.726502311248074, + "6ce4d69d8a8843e99d85c4760afb457c": 3.129353233830846, + "db2655d27ead40c79457c0bb21cf0360": 3.011764705882353, + "25edce98bf6b4d0c9297a4b317213fb4": 2.4816354816354815, + "449673b3764947cd96776f2bca09c7c3": 2.577721837633732, + "54b194233aef449c95c343da3256495c": 2.81864161849711, + "4e8c5aa454df487abb11cbdd55e2d3f3": 2.708994708994709, + "69a7bffab30948888f629fa4c1692985": 3.223938223938224, + "8e98ae76854e427b83f39aa9f68fa517": 2.913565426170468, + "82e07d182d284df885e0b12558d032b6": 3.0098360655737704, + "adc7fcea6f5b473c8f2b29967dd0946d": 3.51587982832618, + "b406d7fe469e4de78154e07627522624": 2.7509293680297398, + "046f05feedfe413cb872cb5c638f5ea7": 2.4869459623557986, + "f3a5910cbdc742e78fea77d2de4e71f7": 3.1048387096774195, + "4450948de1964c38b1afcbaeeba27d0f": 2.5696361355081554, + "c41e6c6c820d4db5b3157296bb872d36": 2.4547413793103448, + "89b03c4663d14915878d3e6b0ddb4798": 2.526835286859963, + "268ada90e4e449d49b9a411868b1bdfc": 3.1195734958111196, + "3c3d6b88e1b54c59b94f211671d785aa": 2.9074074074074074, + "ea9178c2bf4f41e0ae2421308cbc4d11": 2.4641555285540706, + "892eda701b0f4cbb8636548f7d97a50f": 2.951923076923077, + "200467e1ca3b4cf2ae72f0be35533df8": 2.335149863760218, + "02cdc14782c5405da93af16c2156e493": 3.1678267594740914, + "6e817df322634bb8bf9c93a69b584c72": 2.9722943722943724, + "b51d2e57615d49248a1910c4afdc54f6": 2.9789090909090907, + "c0e0c43ce4f8432ab84c2096cc336ecf": 2.953136265320836, + "4b74fb9e1f9049679c91052d3a998fcb": 3.2979066022544283, + "66b5ac7bf5d64e22825335ae0a6d4716": 2.6597402597402597, + "ab4d9010e2cf4596a1b6355832d38c9d": 2.6766091051805336, + "69d64c0b26004e209e614b335debe3e3": 2.473864610111397, + "770d87ecff044cadb4eb18f930e25a77": 2.839243498817967, + "4c50f2e1abac419fbc6b77ad454b5652": 2.4473029045643155, + "f695b5b0659d418a80521611ffb8f977": 2.758522727272727, + "2e3d370fc8894a1daf2b538cb2af059d": 2.61723700887199, + "413474a25ae9489da1fae6782e807245": 3.187548638132296, + "1574e7b555934e60b662de65184269cc": 2.7705146036161334, + "848baa0bc31747c7b0a745e7525527e9": 3.0843373493975905, + "fcb17f13e3cc43fd845e06b1de1c0fea": 3.029585798816568, + "519c7e1102f846a69400e1b6151aa8ec": 2.3665987780040734, + "33ecf63ce2d94e96923b18100259d043": 3.0638297872340425, + "ac32c5404d994060877464b2ceb89e72": 3.3684210526315788, + "564fdbb54fa14913ba5c0a0ad6c7e2da": 2.985726280436608, + "dad1d8e31bc34ae6bf0d1201718dc4c2": 2.3852596314907872, + "46acc39a953d4b7a9007511f88f8269e": 3.1124620060790273, + "74ebef9b145d4a7d83b08b7e21361d95": 2.7452471482889735, + "6b37910e636747d68332efde376063dd": 3.869565217391304, + "e59cdda8b5f64d7f97b2ffa3628d6382": 3.2131147540983607, + "81fc706ff39f40959d9cea6f76c94bbc": 4.037313432835821, + "31b1dcdaa3c148c38ae719c7b8ec0055": 3.1717791411042944, + "1672523d190a47e599498ce10e972d89": 2.942652329749104, + "e03ac2e089434a088c22c9dfdbe8727c": 3.6597938144329896, + "639037da20014b2382a5f4944076b05c": 2.921708185053381, + "955c628c0ecd4d26a3139238930a323e": 2.885797950219619, + "8d4c592428454da88e04965e12f30ac0": 2.3863636363636362, + "fcb508c707594d9c92d6074b1ad65d69": 3.757798165137615, + "1a3d81e14af44c84a1dbfd31a5b7ab11": 3.5907990314769975, + "8f0d01d4d46d41a6ba4f1fc402c9cf96": 3.3991701244813277, + "2ca502cd6c2c40a8ab11e6b876fa2c7a": 3.4272727272727272, + "2f5322fd5c40448c9518cf04fdec0c31": 3.1354166666666665, + "b7695218960549dcab701b01d91f4f7d": 3.1658767772511847, + "20c9c6aea24643078bf18a2b8ddd2dff": 2.6561338289962824, + "0480238e40b944bbb4bc522bbf84dd94": 3.744510978043912, + "5aeda306dc5947e29e69be4cd795cfec": 3.603960396039604, + "9e957177448d4e78a205142f195516a2": 3.6867686768676866, + "5755f800b87943399edac4ed54b68060": 4.435897435897436, + "47e40cc25b36400da67816d28e5179e1": 3.870431893687708, + "9c0a1a37bc884b1e93c79dc145c23d51": 3.480905233380481, + "4b1cb30c6051425d81a78d9ebb27039b": 3.8582995951417005, + "fb6f658f9c01475a90aee3c2f767edaa": 3.4913793103448274, + "458e21cef03b4dd9b1b0c062d9639119": 3.193359375, + "e57f7da1586c4f5990db304a4121b66c": 3.97444089456869, + "3b09d5f4773c4598ac8de11db4ef39d3": 3.6666666666666665, + "61528e7d09564903a7e6358ed3b3f4c6": 3.721830985915493, + "f70af64e1f00450784178d30734ceb8d": 4.32089552238806, + "acc3998331f7450cbeebc19ee06dcfd9": 4, + "c099247c36c74ae7a0a6ffef05d0b112": 3.632183908045977, + "d0521d12ee4f4cd09e34a213427692b7": 2.7868852459016393, + "ecbcfa084ae14b33aade81eca89af6d4": 3.1666666666666665, + "afc177a0949548b884c12290153fea23": 3.6206896551724137, + "f47d4f0823764c40a8109f271acef230": 3.46875, + "1b2b2f912a1c4ee2849845fdbce89c61": 2.8603066439523, + "74e19c9d28224982b75e9c82c56ba9e6": 4.536585365853658, + "87ed53a16dec49fd918382cacd3eef93": 3.5526315789473686, + "5d08adc123c24c0c9c348e7db0a84b90": 3.427480916030534, + "f568ea39deed41239e8aa961122205c5": 3.2054794520547945, + "7f620504bc24438db937b6bcaa91ec6f": 3.3774058577405857, + "c6f62d0db4f0462689f14d17c39e6544": 3.99375, + "6d84570923b74cae9004df6217c508ed": 3.4732142857142856, + "648baf3e0c21439994e6903390a8bc08": 3.214002642007926, + "bb5086d34e8c4b03aef38c340932e573": 2.6464646464646466, + "a5fece73b7d74a55a4b23addb2a07858": 3.213675213675214, + "55389be511694cd6a04570c8fa178c2e": 3.9689922480620154, + "1bfe2f153ca0465d950ca963fae666ee": 3.4654088050314464, + "217189a6bc624591bc41ddc3789d696e": 2.323809523809524, + "0399d9edccc54efe8a183f992d20d619": 3.8557114228456912, + "d0b50d89c64b47c5ae3daf1bbd2cea25": 2.478102189781022, + "9d86ddeaf1a648dd9df4c084612ed56f": 3.536363636363636, + "6ee6d59c971e493f844b2c9f7cf726eb": 3.105421686746988, + "e26ead48bc0243e4863228712d27a169": 2.7619689817936615, + "a90fbb3335cf4925966b80c25edcbe3c": 4.011450381679389, + "9f3e1cac69114092a3ba08611c188bf3": 3.5142857142857142, + "2e74678ef6344de584bf6f2480e75866": 3.312925170068027, + "37507e6abb1f4176a4b8262f81e597a0": 3.386904761904762, + "9b5e916b7e17497f924f2fc806d0e635": 3.0903614457831323, + "8b54e2ebf5ca4d5c988f3a23ecda505d": 2.737037037037037, + "6aebfdd4784f4cce9c493132f0062407": 3.412451361867704, + "a9c2e0682b9d4bf08652bed14edbbc28": 4.148387096774194, + "a619715abcde49f6948c682f1c3e0b0b": 3.6555555555555554, + "d44a6862770a44309ba39dc6eac72634": 3.3389830508474576, + "47aafc3e7566455f9680ae9a34e32c1e": 3.920863309352518, + "54ba95b82ed0460bb4fe57e3b6043fc5": 3.8626609442060085, + "c67c25dcf5154836ac8f1fbad7729d1f": 3.1456953642384105, + "82cfc8b895e741098c724f6118859f3a": 2.978494623655914, + "4d5c5607856340368426ee9c3cde53d6": 3.2378378378378376, + "b5138291518b4dbfa26f094d90938d66": 3.214783821478382, + "876afcc0586941ecb65606e7a46d560d": 3.3706896551724137, + "4bf9866b9cb74ec58152b129037630ce": 3.7186311787072244, + "82c276a9b0574984b99798c32f3f61f6": 2.7018469656992083, + "4cd848f83b2d4e73bebb8f75ad190553": 4.127118644067797, + "5711ea4cd27a4559b22b0141d51bfdf3": 4.146198830409356, + "0f64ec4115c74bbbb8933b7b065e3b70": 3.8378378378378377, + "85edd9347a094fab87a1a16562b924ca": 3.9602649006622515, + "60afcedeb4b44f38a39ff6a09d7bf429": 3.9615384615384617, + "55f8d054a9854ca89bd211698ace5559": 3.3834586466165413, + "62d39190691043659ac41b012f3d719a": 3.517412935323383, + "de60de7fd5864529a973d3a50e967813": 3.6, + "93226c97d9184a57add0b751291c4535": 3.630434782608696, + "e7845ff9638c4879b417e649ee70218b": 2.859848484848485, + "7e4af3153d2742f7bf4f4347ad4a57af": 3.4375, + "58be00e22c14441bb7bfb2eeeb3161f0": 3.7183098591549295, + "9bb92deb3b8442ce940d8d0452f6adf1": 3.8684210526315788, + "cc3c3e9a0bf2450796fff2935247912d": 3.125, + "aefdae675b944b48acc7260eb1a69f7d": 3.6486486486486487, + "3890524c4ab244cba814efcd1b6cd47b": 3.2694300518134716, + "03d7303ac5ac49a28c55d3280488f8bf": 3.117391304347826, + "935edb568cb743b4ac1bc4651e1656af": 2.914285714285714, + "9746a7c9897848d7a81dd61ec55390db": 2.7641509433962264, + "82dad5260e7c45caa42843ffbfc17a8b": 2.859375, + "90caa54db3ae476aa22601501aeab007": 3.128205128205128, + "4d44b0869879494fa31a6cac17de52d1": 3.3333333333333335, + "8e255899360d4369b01f9b3caf95ffce": 2.9722222222222223, + "1fb51d511414412081798fae4676b8ea": 3.0606060606060606, + "dcb2f52bc1c048fcadf3fd82e6c4092d": 3.3164556962025316, + "fa4ac434be6141598ce4063b4d0fe658": 3.6785714285714284, + "b337255aced44d029eac97b2c27816a3": 3.2666666666666666, + "f45a2ab34a6c4b46b26eb3ef8b7eba54": 3.225806451612903, + "91058e55249d496880af35e58c5e57be": 3.2641509433962264, + "6f82af731c0948f1aa0efa2092f326a3": 2.9363636363636365, + "f8344e397ada406dabef6fa0f7476b98": 3.3541666666666665, + "477c0870c6a342dc9eef6a0636a0ac84": 3.652173913043478, + "bf5bcd508ccc4fe89700b4bc69f716f9": 2.6296296296296298, + "7924bc3a20bc4e1189b209e53f584735": 3.4388489208633093, + "8413f72e2daf41808c70c3c7e63fcea1": 3.69811320754717, + "6df52bf9e69b4d1da59cae467861e453": 3.085714285714286, + "fcb05350b2974a9d8885db85bf049fcf": 3.1493506493506493, + "c9b78f878c994ef8a3eaca4cb053ea6e": 3.174193548387097, + "d43699db597f490b958f65715aa98197": 2.9367088607594938, + "743357b2d28d40329d3fa319502ae5f9": 4.073170731707317, + "fc4227f6f54440ddb0e9f475d7490f86": 3.3015873015873014, + "99fc4d8762da442386b5a909bc405ecb": 2.6666666666666665, + "7e6d70a4a43a4387bf4f56e1a8ce36ff": 3.2209302325581395, + "3adad8818521477aa6058495a3ce4e8e": 3.225806451612903, + "86ffebddbfa4407c893b924e46a3bd06": 3.6739130434782608, + "72ff0d207a4e48c2b94e44318e868c9f": 2.864864864864865, + "7e628056aaea4ca2b74e233cbafc36d1": 2.772727272727273, + "8525af1f0d5b42a8aaa9c9028da29a46": 3.0912280701754384, + "fa4b04e071f0469792c6ab2666dc5c2b": 3.3059701492537314, + "3c9c857925d943da91d9d33e3fabcaf0": 3.238095238095238, + "3d2e77d34dcf4cde940a2d7779b29729": 2.9298245614035086, + "d4c971a1d0414186bed24bb726e984a3": 3.409090909090909, + "b4921bee57bc4f01b7ed4f6b25029f13": 3.4096385542168677, + "6f2e092145244cf8b63611da1db994bd": 2.9661016949152543, + "698a068d8ec5447db92cc9358e8172ac": 3.129496402877698, + "dd2ce65ebb36429a802a23c1e15aa5fd": 3.625, + "7b0c6493cdaf4aaa86809f9e43b710a2": 3.073170731707317, + "23802b227ecf4c52b5346e17018fca83": 2.732142857142857, + "0cc2d06cc57441db894ca5ffdd416f7b": 3.6285714285714286, + "4f056606b669457084990a70268ef8fe": 3.6, + "3a94754278be465092eed0113f3a7727": 3.2439024390243905, + "7ea7171837074c67ab2b3b217d365c77": 3.2857142857142856, + "56454061357646d3a576a9c86655e7c8": 3.6923076923076925, + "50dd34bf722b47c3b7e7a92a93ffc375": 3.5161290322580645, + "7f6a174db4aa445895f754f756d4af64": 3.2735849056603774, + "bf9a29c79ae84c58a90e99ce50775319": 3.3636363636363638, + "fce3565d07b84d3fa8d8ad4c43eda835": 3.4, + "2468860c81604925b8f2e1251b2ee316": 3.054263565891473, + "c940fdcb94544f71a46c7c7fa8601135": 2.9926470588235294, + "fd821dd0415f484b98125d723b94480d": 3.6490066225165565, + "f518fcb451954e35a24f6f2233ef8197": 3.1806167400881056, + "af98c0afce0749a49226373efa057137": 3.4942528735632186, + "e9b64887c86844ab84d2cbd65f9bd1ff": 3.140845070422535, + "3f135fc3c6544d9486b2279821dc8fda": 3.4516129032258065, + "f38b6d082bea4952a97ed3de0d83b3ec": 3.569620253164557, + "4ce3a51a65944bd5945604e0c34aebdf": 3.2083333333333335, + "8f430d98b2444fc5af867f9a1a511913": 2.7901234567901234, + "96f6c3df25ea48829a1030508396ddd2": 3.1780821917808217, + "90815691746e4d77a0cc318c5d4d850f": 2.918032786885246, + "440177f1477f4b9bb1404e112d54cc58": 2.8857142857142857, + "d11b0f5e0b4d4c7c81d687149ec67a45": 3.138888888888889, + "127ba38b163d49009fa41677a99dbbea": 3.6129032258064515, + "155ca955f1aa40f6a46720275e25e14e": 3.7837837837837838, + "c391b65995c34356968d491bd19c6378": 3.1184210526315788, + "2b31a18eca914b98b820b086580a2a2a": 4.15625, + "93ce60397ad747799d49d29e89249330": 3.210526315789474, + "1aae714f580c468fbeea50eb90222833": 3.4871794871794872, + "43bec92a2f674c9d930afd3ced9299c5": 2.398184176394293, + "cbd7ad2dc3a64a169db2b27a549fffc8": 3.3965014577259476, + "f5a6890f732141b2b0d4647f83d6531f": 2.63003663003663, + "cb478e50d4094acebf3471672cfb40e4": 2.8569051580698837, + "4178ae6db2204816b7205900fe768a12": 2.7293042153377347, + "75aea11342b146268fc440243acd1632": 2.1323529411764706, + "2121a48e22c44f978e5615763b67c75c": 2.4172003659652335, + "a6c10766a3ce4875bafe7b0a49b4429f": 2.6236933797909407, + "08da613ad78e414193f946836edb45dc": 2.569037656903766, + "e71896c80d764a76a5133d0420151398": 2.7095357590966125, + "22452f091f9c48bcb7b540da85df5171": 2.102272727272727, + "7c1045fde70b4f74a89902a108550555": 3.104424778761062, + "05c22d1a5713466ea6c7bf63515a8690": 2.514705882352941, + "d3a945cbf6e54513a837af6226da5a8d": 2.43343653250774, + "2c50cff8f70e4338bfcdfe71f164afd1": 2.490740740740741, + "02c8965f0cc54314829c25183822b645": 2.4072538860103627, + "e9d8b95453c94017bb491adc695ffc19": 2.3221153846153846, + "5bcf105a3cbe452a84ab997c82e43eda": 2.4065420560747666, + "c940abe418904c2581ed3ff7d87cc4b1": 2.331288343558282, + "7dc2cc90bf3347ddba3486865a6ee68b": 2.198889916743756, + "7c8d62496f654389ba48fb4d399d1701": 2.8048780487804876, + "a67a52e961cc4ffe9f7f6162c5d51bb3": 2.684563758389262, + "ebf5ba12beb84e3c9ec4dde8e81d0b72": 2.320388349514563, + "feb85d4ad3b546dfa65544fde6c1d11b": 2.426605504587156, + "0c6944cd7b734a298df4d65f97c2eb84": 2.611864406779661, + "c6d2c24631564ddf9e8565b5c949e82e": 2.5775193798449614, + "fa16a94c776e47efac17f72f23650b00": 3.0317460317460316, + "46c758fc50e54a04b85b29cdc3cb4fe5": 2.028169014084507, + "40aeea7dd21f4b5ebb8aae45a443e2f5": 3.12142152023692, + "81bab241b55044b99dbc017a7aa7c6d0": 2.4233409610983982, + "dc5a4b2a18ca492da91ad92b5c04274b": 2.1818181818181817, + "7865fde4fcbc4d7fb316849bd00c93f2": 2.909090909090909, + "0ccd4188bff14abd9fe6ce8c4f438129": 2.349330357142857, + "ad5c9d7fa74b48ce90a0e688bfb81408": 2.723004694835681, + "7d9578ff1d22498fa8784fa48a7f29bf": 2.4121621621621623, + "36e8f9d5cd024f4ba05c3d652eac3f2f": 2.3580246913580245, + "64dec9f872be4906b7f15b159ed6a08c": 2.7635135135135136, + "35491440456f4d66b6cb007542b2ef65": 2.4694835680751175, + "1e02f6e3621e452d8f352b62e3460623": 2.375, + "ad9ae2f31a524ad29a465fbbfde0afc4": 2.4132231404958677, + "c8ba010e205445d194a09e452d4229c3": 2.846805736636245, + "7f03694f465042129059f661ad34b2be": 2.7248677248677247, + "0f65d3d8f4e6482e8421584ef2aa88d0": 2.3580786026200875, + "ca8ab5a12ab94645a67d419a92e6281d": 2.2976653696498053, + "f1850e535eba4bbf92fb791780fbf574": 2.838235294117647, + "d4d67c54891641b9ac5163513e857ba0": 2.5242718446601944, + "f38d28752a534e96bba4a5bf62cf78f0": 2.3435114503816794, + "223da671423c456f874f62bccf348f65": 2.524663677130045, + "c3d0934f196846c5a5e838c95dc6a053": 2.3789473684210525, + "5695bc1cdc3e472d9e4e5b13c10ff9e9": 2.4516129032258065, + "d09225d3671a4caf93429aca07fb72a2": 2.4767676767676767, + "8e65f2c939d24ee999a79f4b66aa47f4": 2.3505535055350553, + "09114ee2c7f34625ba0722f0b18d8881": 2.4473684210526314, + "8ed6e12656454fe0bbcdfe7a49484dbc": 2.5525477707006368, + "e16bc96b50a248ae9adf4d2780cd514e": 2.710144927536232, + "e3284b2cabc44e659a9d6f0fbe8e092f": 2.3636363636363638, + "869ad289e7864c32a3a476273c14dbfc": 2.5684713375796178, + "374976f3d8c94c1c8ecb4700af292b08": 2.6991150442477876, + "8ae19017dc994cada379d127a641513e": 2.3549382716049383, + "5384172790464297a713c9ffb4d4ffbe": 2.8728813559322033, + "e5314d6b101f420fa6d3bce92b8051db": 2.4864130434782608, + "b269779872e34105862b11e244991b2a": 2.6429479034307497, + "d4adce94e54b499c82edee3da0dd77b4": 2.152941176470588, + "de121de3b5d444b6939cf7540e6d5529": 2.288557213930348, + "9a50c7c38d2a43bc903b607267df39ba": 2.4702702702702704, + "aaea968b553047d68af8be49e8456c6a": 2.355421686746988, + "fc17b343c86e429caddbc3e323a0a7e9": 2.8203125, + "0d5fd96e88f149a287de3186091eef30": 2.3225806451612905, + "50a8e717c6d248ada3c9597859a523cd": 2.2538552787663106, + "41d928e4236a4f28934cc35698c8531a": 2.3301043219076005, + "2ea3c574049b416d92129fe48273e34c": 2.7115384615384617, + "70a50d2167824f7cbb86fe15889547d6": 2.4312796208530805, + "ad2488477c5d48b7b31214ff086c57c3": 2.1783783783783783, + "8b54894ac0cf4d2490511326eb1812c6": 2.3708133971291865, + "552974440153459a9a56c3b94f6c0dd2": 2.5228426395939088, + "8c7be26abc8f401090f5ab60a0e15eac": 2.2625, + "ee51619e422e4cf1a7a5c5b3e6b9296b": 2.7777777777777777, + "6d567be1958b43ae9a830a13c5a90ccc": 2.282051282051282, + "7f74d1cf113d472f810ba11e3448d446": 2.308139534883721, + "ebf181f8fd9741a8bc89db9ebde8b55d": 2.5833333333333335 + }, + "Category_AL": { + "coding": 3.360338172249745, + "humanities": 2.8719922283192925, + "math": 3.31279868284246, + "qa": 3.0383647306128307, + "rag": 3.450572296983691, + "reasoning": 3.232919666109211, + "stem": 2.9584957956361433, + "writing": 2.8847227886588818, + "multilingual": 3.457944110646776, + "summarization": 3.273105660855136, + "roleplay": 2.5124589161836814 + }, + "Average_AL": 3.123064822645259, + "Acceptance_Length_Histogram": { + "1": 137619, + "2": 127963, + "3": 97043, + "4": 85915, + "5": 52909, + "6": 33427, + "7": 15316, + "8": 10195 + }, + "Conditional_Acceptance_Rate": { + "1": 1.0, + "2": 0.754421498000489, + "3": 0.697320989289634, + "4": 0.6708230864469734, + "5": 0.565563657325472, + "6": 0.5269519969243699, + "7": 0.43284468424446026, + "8": 0.3996315314962173 + }, + "Joint_Acceptance_Rate": { + "1": 1.0, + "2": 0.754421498000489, + "3": 0.5260739453270686, + "4": 0.35290254770364055, + "5": 0.19958885555874778, + "6": 0.10517374600053177, + "7": 0.04552389687840724, + "8": 0.01819278462919375 + } +} \ No newline at end of file diff --git a/throughput_32k_specbench_results.json b/throughput_32k_specbench_results.json new file mode 100644 index 00000000000..8063aa388d8 --- /dev/null +++ b/throughput_32k_specbench_results.json @@ -0,0 +1,141 @@ +{ + "Request_AL": { + "08a2f4220a4b48bca1961b5548097f6a": 2.48576512455516, + "979f73e74a9042588e91771d5b60ebce": 3.0385756676557865, + "d28872d9ffed438dac2be45dc738ee5e": 3.7304189435336976, + "8580e194938c45d681b22f443ec85256": 2.5565714285714285, + "52f0aa9c608449cfb4a858506e115e4d": 2.938307030129125, + "4a45cf5aa5bc4795aff28ebe0e4f5e8e": 3.1727343144848956, + "aa9f8c1febe646608ca6c2c2d6046209": 2.535796766743649, + "f90d7cdd1f87421586728f96309acf74": 3.71350861287398, + "b17311d52b5144e4b730021c08adc0a9": 4.047430830039525, + "ef5f14c9992e41ad8142b07d153c7c0e": 2.595505617977528, + "b98de9e87f8c4e1fbad70db1019b45da": 3.710144927536232, + "1dcd72e8dc1540dab9e195dfc087c572": 4.648604269293925, + "5b7294277cfe48f798537efc0de8e250": 2.637037037037037, + "3da1e340007f472780658f8fd1e03b51": 3.2533756949960284, + "137a50d86b9d4fd5832a1929cb772206": 4.2533748701973, + "2201c2c5a347433dae6ba246a9b974e3": 4.1878787878787875, + "930c185081b94776b8b8f36f27ab0c1a": 3.1195734958111196, + "6ab1f484298b4456be086a681c234969": 4.2845188284518825, + "0d7bf237116941f58c3c0396607f5acf": 2.5547945205479454, + "69370c0153cb4abaaf6939d7af5d04a0": 3.1392557022809124, + "7b977ef6aa504f04ba0dc8f38d7861f3": 3.957487922705314, + "1ab6894e613e4cc79b779e731abad0a7": 2.519756838905775, + "32e3c445fc5d43f3944ce5aeac813e31": 4.29800629590766, + "d35992d94cc5462f99435a88938a8bfc": 3.647373107747106, + "e30cb9c9a1284e39be8ba8c08d092b65": 2.573820395738204, + "25f3ee1e172341838f6fc634f5d6775c": 3.01620029455081, + "e047a8e8d4c444548263cc3c99338e80": 3.7304189435336976, + "88b6c670d9db47b8a31cc87a6f615f40": 2.4710500490677134, + "fde4d82f81b8423f9e8a28f7a33fc9d3": 3.4133333333333336, + "9a2914363e9b4638813d7e5fb019427e": 4.665148063781321, + "91edb47f061d4cd2a442872be490c7e8": 2.5440993788819877, + "428119247bc54583a6404ec89787bf87": 3.8181818181818183, + "b6161cea54ee4d118d1eccc5521f5f32": 4.209660842754368, + "98d19ae2448543689884a2fedf4d20fb": 2.5477839335180055, + "f0190d653c744f199f5a483f8495c54b": 2.9688995215311005, + "7fcd053b935044a0a97f97b269b40477": 3.757798165137615, + "1850bfc4444c4bc38a043ea1bb37352c": 3.054436987322893, + "95592f04cf2c4463bb8c0c9c34dce322": 3.1232876712328768, + "24c04f36b0d5400493d58604f4e99fd6": 4.530973451327434, + "b3389e8d325b4a47827fd67a2eeba6d5": 2.5267489711934155, + "6fa8c0e2e0a648bcb07ebb49bb1d91e1": 4.385438972162741, + "c188f3ff73f84446bb9b37ec6db8af45": 4.0039100684261975, + "a1b183ebbf744dd7a2148ff7ce5e7958": 3.243531202435312, + "8023b28c50ce4ef89fdb6a6ef03ff0e7": 3.934678194044188, + "3015f6a31ffb4bc9a89c7dc6ecd72c49": 3.7890841813135987, + "505e4694caba46fb9bd3a78fb231ac47": 2.5396995708154506, + "a3d83fb38b6c4830813dee7ae44e5a59": 3.091320754716981, + "6f832727a8d84d2cb126013cde2f8a59": 3.8568738229755177, + "607b01bba0fc43e28ba4da428cbbe426": 3.1925175370226033, + "700fc5a30416471da625a9521a0e947f": 3.0285714285714285, + "b6b412e9f8094b63b3243e17e36ad134": 3.37119341563786, + "86b2a2608252410fbe431355c9e63723": 2.5871647509578546, + "79ae838f311748e682ee21acdb9b92cf": 3.220125786163522, + "8f5f05166322459bb73f09ad1c2f3791": 3.7961075069508805, + "7af145416c8543e0b2bb0fc4440dd27c": 2.972423802612482, + "55091e0182ba4292a024dc71a9f414d1": 3.640888888888889, + "18e782c799264c64b614c9628b773876": 4.149949341438703, + "5c7b6d75bbeb440cb74e73ab34f76273": 2.3135026737967914, + "600b5d0489a4414db452f83c380ee216": 4.289005235602095, + "a6a77115c76a4f4da8f1b9bc5f41b465": 3.6934174932371504, + "99fa1eceebd948b9947fcf7bf2963b72": 2.925675675675676, + "a7ebfc337a3d4315a86c9737d1561c72": 3.22271714922049, + "f44dee1855d746669f62b43989fafb71": 3.9536679536679538, + "6d187acb90ba43f9830b09beb596b69b": 3.574171029668412, + "f2dfde375a524e9fa507c7c7328b4cfa": 3.145098039215686, + "c2fcb9e1095f4f3c8cddb44eac0071c9": 3.7890841813135987, + "459c7cd0a6344f399b59aa6a765c6347": 3.6344276841171252, + "e5675bfbe61e406f961e4bdb97512d3d": 3.6706349206349205, + "e212eeba9c734e009e481373e9bb43fe": 4.534782608695652, + "4fda8da873a5475a81e942659c2d0b38": 2.55045871559633, + "3be4fd4eed58433b8ff8088fa9944741": 3.1356673960612693, + "40b2f9919375454d8f000b920a2ad093": 3.162934362934363, + "986f2e94c0a14e33816d8a405b544f7f": 2.4662162162162162, + "579acbff37be4ebd84ac3f9b22a7af79": 2.887037037037037, + "fcf59b01aa2c4626b1a1929eb6fba4ac": 3.324675324675325, + "4ded974a0c244997bca5826784f16498": 2.368832731648616, + "57d9824d693d4a6c80718b93a9b38fe3": 3.5474452554744524, + "c46f8331574f43c284b6d8e5b01b3b4d": 3.7925925925925927, + "fdd51edefcef4b1eb948ddec3f9f51d2": 2.6797791580400276, + "576bc17ff88e43f095fae03db8b5ed6c": 4.129032258064516 + }, + "Category_AL": { + "high_entropy": 2.771831355057127, + "mixed": 3.439937458588111, + "low_entropy": 3.9174698233402876 + }, + "Average_AL": 3.369649667190861, + "Acceptance_Length_Histogram": { + "1": 15084, + "2": 15768, + "3": 12264, + "4": 15766, + "5": 9255, + "6": 6639, + "7": 2530, + "8": 1617, + "9": 36, + "10": 24, + "11": 16, + "12": 12, + "13": 7, + "14": 1, + "15": 1 + }, + "Conditional_Acceptance_Rate": { + "1": 1.0, + "2": 0.8091116173120729, + "3": 0.7533783783783784, + "4": 0.7453911310413552, + "5": 0.5608845811051694, + "6": 0.5404210944483068, + "7": 0.3899660020215014, + "8": 0.4038642789820924, + "9": 0.05659276546091015, + "10": 0.6288659793814433, + "11": 0.6065573770491803, + "12": 0.5675675675675675, + "13": 0.42857142857142855, + "14": 0.2222222222222222, + "15": 0.5 + }, + "Joint_Acceptance_Rate": { + "1": 1.0, + "2": 0.8091116173120729, + "3": 0.6095671981776766, + "4": 0.45436598329536826, + "5": 0.254846874209061, + "6": 0.1377246266767907, + "7": 0.053707922045051884, + "8": 0.021690711212351305, + "9": 0.0012275373323209313, + "10": 0.0007719564667172866, + "11": 0.00046823588964819025, + "12": 0.00026575550493545934, + "13": 0.00011389521640091114, + "14": 2.5310048089091364e-05, + "15": 1.2655024044545682e-05 + } +} \ No newline at end of file