diff --git a/.github/benchmark/models_atomesh.yaml b/.github/benchmark/models_atomesh.yaml index c1be8bc3f0..368c5866a8 100644 --- a/.github/benchmark/models_atomesh.yaml +++ b/.github/benchmark/models_atomesh.yaml @@ -248,7 +248,7 @@ models: isl: [1048576] osl: 1024 server: - common_args: + common_args: &glm52_agentic_args kv_cache_dtype: fp8 block_size: 16 gpu_memory_utilization: 0.9 @@ -294,36 +294,106 @@ models: run_eval: false name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c1 concurrency: [1] - accuracy: - task: swebench_lite - threshold: 0.50 - agent_workers: 32 - agent_step_limit: 150 - case_timeout: 3600 - agent_timeout: 21600 - score_timeout: 7200 - max_workers: 4 - instance_timeout: 900 - eval_concurrency: [1] - run_eval: true - runner: - time_limit: "12:00:00" + server: + common_args: &glm52_agentic_mtp_args + <<: *glm52_agentic_args + extra_args: "--method mtp --num-speculative-tokens 4 --spec-decode-acceptance-rate 0.5825" - <<: *glm52_agentic_tp4_tp4 name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c2 concurrency: [2] + server: + common_args: *glm52_agentic_mtp_args - <<: *glm52_agentic_tp4_tp4 name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c4 concurrency: [4] + server: + common_args: *glm52_agentic_mtp_args - <<: *glm52_agentic_tp4_tp4 name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c8 concurrency: [8] + server: + common_args: *glm52_agentic_mtp_args - - <<: *glm52_agentic_tp4_tp4 - name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c48 + # CPP4 prefill (PP4 x TP1) paired with TP4+DCP4 decode on one node. + - &glm52_agentic_lmcache_cpp4_dcp4 + <<: *glm52_agentic_tp4_tp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48 + topology: 1p1d_cpp4_dcp4 + pd_worker_layout: single_node concurrency: [48] + server: + common_args: + kv_cache_dtype: fp8 + block_size: 16 + max_num_seqs: 512 + enable_prefix_caching: true + decode_max_num_seqs: 512 + # Empty means "do not pass the flag", so the model config's own + # context length applies. The 16384 the model defaults carry would + # truncate this suite's 1M agentic traces. + max_model_len: "" + max_num_batched_tokens: "" + decode_max_num_batched_tokens: "" + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' + gpu_memory_utilization: 0.85 + extra_args: "--level 3 --method mtp --num-speculative-tokens 3 --spec-decode-acceptance-rate 0.6633" + prefill: + workers: 1 + tp: 1 + cudagraph: none + extra_args: "--pipeline-parallel-size 4 --enforce-eager --max-num-batched-tokens 8192" + decode: + workers: 1 + tp: 4 + cudagraph: >- + [1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256] + extra_args: "--decode-context-parallel-size 4 --cudagraph-mode FULL" + env: + common: + ATOM_MLA_PAGE_SIZE: "1" + ATOM_ONLINE_QUANT_STREAMING: "0" + ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB: "2047" + ATOM_USE_TRITON_MLA: "0" + MAX_JOBS: "16" + PYTHONHASHSEED: "0" + TOPK_FORCE_PATH: "one" + prefill: + HIP_VISIBLE_DEVICES: "0,1,2,3" + VLLM_PP_LAYER_PARTITION: "20,20,20,18" + LMCACHE_LOCAL_CPU: "True" + LMCACHE_MAX_LOCAL_CPU_SIZE: "256" + LMCACHE_CHUNK_SIZE: "256" + # NVMe (L3) tier under the CPU hot cache. Both variables are + # required together; LMCACHE_MAX_LOCAL_CPU_SIZE must stay > 0 + # because LocalDiskBackend stages through the host allocator. + # scale_cpu_size_for_pp splits only the CPU budget across PP + # stages, so this size is per rank: 4 x 500GiB on a pp4 prefill. + # pd_server_atom.sh empties the directory around the run. + # The path follows the default atomesh-cicd fleet, where /mnt is + # local and already holds ATOMESH_MODEL_ROOT. The mi350 and crusoe + # fleets keep their models on /data and /shared_nfs instead, so a + # dispatch onto those needs this pointed at their own local disk. + LMCACHE_LOCAL_DISK: "/mnt/lmcache-job" + LMCACHE_MAX_LOCAL_DISK_SIZE: "500" + OFFLOAD_PROFILE: "1" + OFFLOAD_MIN_LOAD_TOKENS: "0" + PREFILL_KV_TRANSFER_CONFIG: >- + {"kv_connector":"multi","connectors":[{"kv_connector":"mooncake","kv_role":"kv_producer","proxy_ip":"${ROLE_IP}","handshake_port":${HANDSHAKE_PORT},"protocol":"rdma"},{"kv_connector":"lmcache_offload","kv_role":"offload"}]} + decode: + HIP_VISIBLE_DEVICES: "4,5,6,7" + DECODE_KV_TRANSFER_CONFIG: >- + {"kv_connector":"mooncake","kv_role":"kv_consumer","proxy_ip":"${ROLE_IP}","handshake_port":${HANDSHAKE_PORT},"protocol":"rdma"} + + - <<: *glm52_agentic_lmcache_cpp4_dcp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c32 + concurrency: [32] + + - <<: *glm52_agentic_lmcache_cpp4_dcp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40 + concurrency: [40] - &glm52_agentic_lmcache_tp4_dpa <<: *glm52_agentic_tp4_tp4 diff --git a/.github/scripts/atomesh/pd_matrix.py b/.github/scripts/atomesh/pd_matrix.py index b80efb095c..e4bfe92610 100644 --- a/.github/scripts/atomesh/pd_matrix.py +++ b/.github/scripts/atomesh/pd_matrix.py @@ -206,7 +206,9 @@ def role_env( model_cfg.get("env", {}).get(role, {}), suite_cfg.get("env", {}).get(role, {}), ) - env = resolve_env_refs_in_value(env, preserve_names={"ROLE_IP"}) + # ROLE_IP and HANDSHAKE_PORT are filled in by pd_server_atom.sh at + # launch time (host IP and 6301 + ATOMESH_SERVICE_PORT_OFFSET). + env = resolve_env_refs_in_value(env, preserve_names={"ROLE_IP", "HANDSHAKE_PORT"}) return {str(key): str(value) for key, value in env.items()} diff --git a/.github/scripts/atomesh/pd_server_atom.sh b/.github/scripts/atomesh/pd_server_atom.sh index f78eca6aac..19304fb2f0 100644 --- a/.github/scripts/atomesh/pd_server_atom.sh +++ b/.github/scripts/atomesh/pd_server_atom.sh @@ -246,10 +246,38 @@ apply_prefixed_env() { while IFS='=' read -r name raw; do [[ "${name}" == "${prefix}"* ]] || continue value="${raw//\$\{ROLE_IP\}/${role_ip}}" + value="${value//\$\{HANDSHAKE_PORT\}/${HANDSHAKE_PORT}}" export "${name#${prefix}}=${value}" done < <(env) } +# Names the last apply_role_env() exported for a role. +ROLE_ENV_NAMES=() + +# Both servers are launched from this one shell, so a variable exported for one +# role stays in the environment the next role inherits. Only names the two roles +# both define get overwritten; a prefill-only name reaches decode unchanged -- +# VLLM_PP_LAYER_PARTITION from a pp4 prefill aborts a pp1 decode's model build +# with "len(partitions)=4 does not match pp_size=1". Drop the previous role's +# names before applying this one's. +apply_role_env() { + local prefix="$1" + local role_ip="$2" + local name + for name in ${ROLE_ENV_NAMES[@]+"${ROLE_ENV_NAMES[@]}"}; do + unset "${name}" + done + ROLE_ENV_NAMES=() + while IFS='=' read -r name _; do + [[ "${name}" == "${prefix}"* ]] || continue + ROLE_ENV_NAMES+=("${name#${prefix}}") + done < <(env) + # A name the common block also sets was just unset with the previous role's, + # so put the common value back before the role overrides it. + apply_prefixed_env "ATOMESH_ENV_" "${role_ip}" + apply_prefixed_env "${prefix}" "${role_ip}" +} + host_ip="$(echo "${IPADDRS}" | tr ',' '\n' | sed -n "$((NODE_RANK + 1))p")" if [[ -z "${host_ip}" ]]; then host_ip="$(hostname -I 2>/dev/null | awk '{print $1}')" @@ -503,12 +531,38 @@ terminate_process_group() { wait "${pid}" 2>/dev/null || true } +# LMCache's NVMe tier lives on a host bind mount, so unlike the container's own +# /tmp it survives `docker run --rm`. Every concurrency runs as its own job, and +# a tier left behind would both serve the previous job's KV and hold its +# LMCACHE_MAX_LOCAL_DISK_SIZE of disk per rank. Start empty, leave nothing. +lmcache_disk_dir="" + +reset_lmcache_disk() { + local dir="${LMCACHE_LOCAL_DISK:-}" + [[ -n "${dir}" && "${dir}" != "/" ]] || return 0 + # Several prefill workers can share this shell, so only the first one empties + # the tier; a later one would delete a running worker's cache underneath it. + [[ "${lmcache_disk_dir}" != "${dir}" ]] || return 0 + lmcache_disk_dir="${dir}" + rm -rf -- "${dir}" + mkdir -p -- "${dir}" + echo "[lmcache] NVMe tier ${dir} reset (${LMCACHE_MAX_LOCAL_DISK_SIZE:-0}GiB per rank)" +} + +purge_lmcache_disk() { + [[ -n "${lmcache_disk_dir}" ]] || return 0 + rm -rf -- "${lmcache_disk_dir}" + echo "[lmcache] NVMe tier ${lmcache_disk_dir} removed" + lmcache_disk_dir="" +} + cleanup_processes() { local rc=$? local pid for pid in "$@"; do terminate_process_group "${pid}" done + purge_lmcache_disk return "${rc}" } @@ -543,7 +597,8 @@ start_prefill() { local handshake_port="${3:-${HANDSHAKE_PORT}}" local dp_master_port="${4:-${PREFILL_DP_MASTER_PORT}}" local dp_base_port="${5:-${PREFILL_DP_BASE_PORT}}" - apply_prefixed_env "ATOMESH_PREFILL_ENV_" "${host_ip}" + apply_role_env "ATOMESH_PREFILL_ENV_" "${host_ip}" + reset_lmcache_disk local -a prefill_cache_env=() build_server_cache_env "prefill" "${server_port}" prefill_cache_env local -a prefill_dp_env=() @@ -580,7 +635,7 @@ start_decode() { local handshake_port="${3:-${HANDSHAKE_PORT}}" local dp_master_port="${4:-${DECODE_DP_MASTER_PORT}}" local dp_base_port="${5:-${DECODE_DP_BASE_PORT}}" - apply_prefixed_env "ATOMESH_DECODE_ENV_" "${host_ip}" + apply_role_env "ATOMESH_DECODE_ENV_" "${host_ip}" local max_conc max_conc="$(echo "${BENCH_MAX_CONCURRENCY}" | tr 'x,' '\n' | sort -n | tail -1)" local decode_max_num_seqs="${MAX_NUM_SEQS}" diff --git a/.github/scripts/atomesh/process_result.py b/.github/scripts/atomesh/process_result.py index 9528d216ca..d9dbc14506 100644 --- a/.github/scripts/atomesh/process_result.py +++ b/.github/scripts/atomesh/process_result.py @@ -27,6 +27,9 @@ ) TOPOLOGY_RE = re.compile(r"(?P

\d+)p(?P\d+)d", re.IGNORECASE) TP_RE = re.compile(r"tp(?P\d+)", re.IGNORECASE) +DUAL_TP_RE = re.compile(r"tp(?P\d+)-tp(?P\d+)", re.IGNORECASE) +CPP_PP_RE = re.compile(r"(?:cpp|pp)(?P\d+)", re.IGNORECASE) +PP_ARG_RE = re.compile(r"--pipeline-parallel-size(?:=|\s+)(\d+)", re.IGNORECASE) EVAL_CONC_RE = re.compile(r"(?:^|[_-])c(?P\d+)(?:$|[_-])", re.IGNORECASE) EVAL_TOPOLOGY_RE = re.compile( r"(?:^|[_-])(?P\d+p\d+d(?:[_-]dpa)?)(?:$|[_-])", @@ -106,6 +109,12 @@ def round_or_none(*values: Any, digits: int = 4) -> float | None: return round(parsed, digits) if parsed is not None else None +def divide_by_total_gpu(value: float | None, total_gpu: int | None) -> float | None: + if value and total_gpu: + return value / total_gpu + return None + + def interactivity_value(payload: dict[str, Any]) -> float | None: # An already-resolved value wins: apply_p90_e2e_interactivity() writes the # p90 e2e normalized number here, and without this branch perf_point() would @@ -202,7 +211,6 @@ def topology_resources( ) ) topology = TOPOLOGY_RE.search(text) - tp = TP_RE.search(text) prefill_workers = int_value( payload.get("prefill_workers"), payload.get("num_prefill_workers") ) @@ -219,14 +227,36 @@ def topology_resources( decode_tp = int_value( payload.get("decode_tp"), payload.get("decode_tensor_parallel_size") ) - if tp: - prefill_tp = prefill_tp or int(tp.group("tp")) - decode_tp = decode_tp or int(tp.group("tp")) + dual_tp = DUAL_TP_RE.search(text) + if dual_tp: + prefill_tp = prefill_tp or int(dual_tp.group("prefill_tp")) + decode_tp = decode_tp or int(dual_tp.group("decode_tp")) + else: + tp = TP_RE.search(text) + if tp: + tp_size = int(tp.group("tp")) + prefill_tp = prefill_tp or tp_size + decode_tp = decode_tp or tp_size + + prefill_pp = int_value( + payload.get("prefill_pp"), payload.get("prefill_pipeline_parallel_size") + ) + if prefill_pp is None: + cpp_pp = CPP_PP_RE.search(text) + if cpp_pp: + prefill_pp = int(cpp_pp.group("pp")) + if prefill_pp is None: + for key in ("prefill_extra_server_args",): + pp_match = PP_ARG_RE.search(string_value(payload.get(key))) + if pp_match: + prefill_pp = int(pp_match.group(1)) + break + prefill_pp = prefill_pp or 1 num_prefill_gpu = int_value(payload.get("num_prefill_gpu")) num_decode_gpu = int_value(payload.get("num_decode_gpu")) if num_prefill_gpu is None and prefill_workers and prefill_tp: - num_prefill_gpu = prefill_workers * prefill_tp + num_prefill_gpu = prefill_workers * prefill_tp * prefill_pp if num_decode_gpu is None and decode_workers and decode_tp: num_decode_gpu = decode_workers * decode_tp total_gpu = int_value(payload.get("total_gpu")) @@ -239,6 +269,7 @@ def topology_resources( "decode_workers": decode_workers, "prefill_tp": prefill_tp, "decode_tp": decode_tp, + "prefill_pp": prefill_pp, "num_prefill_gpu": num_prefill_gpu, "num_decode_gpu": num_decode_gpu, "total_gpu": total_gpu, @@ -339,6 +370,9 @@ def enrich_payload( enriched.setdefault("decode_workers", env.get("DECODE_WORKERS")) enriched.setdefault("prefill_tp", env.get("PREFILL_TP")) enriched.setdefault("decode_tp", env.get("DECODE_TP")) + enriched.setdefault( + "prefill_extra_server_args", env.get("PREFILL_EXTRA_SERVER_ARGS") + ) runner = env.get("SLURM_SUBMIT_RUNNER", "") if hardware: enriched["hardware"] = hardware @@ -373,28 +407,17 @@ def enrich_payload( enriched.setdefault("interactivity", interactivity_value(enriched)) resources = topology_resources(enriched, fields) total_gpu = resources["total_gpu"] - num_prefill_gpu = resources["num_prefill_gpu"] - num_decode_gpu = resources["num_decode_gpu"] input_tput = number(enriched.get("input_throughput")) output_tput = number(enriched.get("output_throughput")) total_tput = number( enriched.get("total_token_throughput"), enriched.get("total_throughput") ) + enriched.setdefault("tput_per_gpu", divide_by_total_gpu(total_tput, total_gpu)) enriched.setdefault( - "tput_per_gpu", total_tput / total_gpu if total_tput and total_gpu else None + "input_tput_per_gpu", divide_by_total_gpu(input_tput, total_gpu) ) enriched.setdefault( - "input_tput_per_gpu", - input_tput / num_prefill_gpu if input_tput and num_prefill_gpu else None, - ) - output_tput_denominator = num_decode_gpu or total_gpu - enriched.setdefault( - "output_tput_per_gpu", - ( - output_tput / output_tput_denominator - if output_tput and output_tput_denominator - else None - ), + "output_tput_per_gpu", divide_by_total_gpu(output_tput, total_gpu) ) return enriched @@ -519,18 +542,10 @@ def perf_point( "cache_hit_rate": round_or_none(payload.get("cache_hit_rate")), "cache_hit_tokens": int_value(payload.get("cache_hit_tokens")), "cache_total_tokens": int_value(payload.get("cache_total_tokens")), - "tput_per_gpu": round_or_none( - total_tput / total_gpu if total_tput and total_gpu else None - ), - "input_tput_per_gpu": round_or_none( - input_tput / resources["num_prefill_gpu"] - if input_tput and resources["num_prefill_gpu"] - else None - ), + "tput_per_gpu": round_or_none(divide_by_total_gpu(total_tput, total_gpu)), + "input_tput_per_gpu": round_or_none(divide_by_total_gpu(input_tput, total_gpu)), "output_tput_per_gpu": round_or_none( - output_tput / (resources["num_decode_gpu"] or total_gpu) - if output_tput and (resources["num_decode_gpu"] or total_gpu) - else None + divide_by_total_gpu(output_tput, total_gpu) ), "run_url": run_url or "", "image": string_value(payload.get("docker_image"), payload.get("image")), diff --git a/.github/workflows/atomesh-benchmark.yaml b/.github/workflows/atomesh-benchmark.yaml index c02afa1389..0b6351991f 100644 --- a/.github/workflows/atomesh-benchmark.yaml +++ b/.github/workflows/atomesh-benchmark.yaml @@ -59,7 +59,9 @@ on: glm-52-mxfp4-1p1d-tp4,glm-52-mxfp4-1p1d-tp8, glm-52-mxfp4-1p1d-tp4-agentic-1m-c1,glm-52-mxfp4-1p1d-tp4-agentic-1m-c2, glm-52-mxfp4-1p1d-tp4-agentic-1m-c4,glm-52-mxfp4-1p1d-tp4-agentic-1m-c8, - glm-52-mxfp4-1p1d-tp4-agentic-1m-c48, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c32, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c24, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c32, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c48 @@ -124,6 +126,10 @@ on: - mia1-p02-g42,mia1-p02-g44 - mia1-p02-g42,mia1-p02-g47 - mia1-p02-g44,mia1-p02-g47 + # single_node cases only use the first entry, so keep orderings that + # let any one of the three nodes be selected. + - mia1-p02-g47,mia1-p02-g44 + - mia1-p01-g43,mia1-p01-g36 - mia1-p01-g36,mia1-p01-g43 atomesh_2p1d_nodes: description: 'ATOMesh 2P1D nodes'