From 5814e2c6a3175567f783e292ef4249b2a02dca4e Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Thu, 3 Sep 2026 02:04:54 +0000 Subject: [PATCH 1/5] ci: add GLM-5.2 CPP4+DCP4 ATOMesh coverage Wire single-node PD CPP4/DCP4 jobs, isolate prefill/decode env so PP partitions do not leak across roles, and divide per-GPU throughput by total cluster GPUs. Co-authored-by: Cursor --- .github/benchmark/models_atomesh.yaml | 66 ++++++++++++++++++++ .github/scripts/atomesh/pd_matrix.py | 4 +- .github/scripts/atomesh/pd_server_atom.sh | 32 +++++++++- .github/scripts/atomesh/process_result.py | 73 +++++++++++++---------- .github/workflows/atomesh-benchmark.yaml | 7 +++ 5 files changed, 149 insertions(+), 33 deletions(-) diff --git a/.github/benchmark/models_atomesh.yaml b/.github/benchmark/models_atomesh.yaml index b6b3583c42..62e028bb77 100644 --- a/.github/benchmark/models_atomesh.yaml +++ b/.github/benchmark/models_atomesh.yaml @@ -325,6 +325,72 @@ models: server: common_args: *glm52_agentic_mtp_args + # CPP4 prefill (PP4 x TP1) paired with TP4+DCP4 decode on one node. + - &glm52_agentic_lmcache_cpp4_dcp4 + <<: *glm52_agentic_tp4_tp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48 + topology: 1p1d_cpp4_dcp4 + pd_worker_layout: single_node + concurrency: [48] + server: + common_args: + kv_cache_dtype: fp8 + block_size: 16 + max_num_seqs: 512 + enable_prefix_caching: true + decode_max_num_seqs: 512 + # Empty means "do not pass the flag", so the model config's own + # context length applies. The 16384 the model defaults carry would + # truncate this suite's 1M agentic traces. + max_model_len: "" + max_num_batched_tokens: "" + decode_max_num_batched_tokens: "" + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' + gpu_memory_utilization: 0.85 + extra_args: "--level 3 --method mtp --num-speculative-tokens 3 --spec-decode-acceptance-rate 0.6633" + prefill: + workers: 1 + tp: 1 + cudagraph: none + extra_args: "--pipeline-parallel-size 4 --enforce-eager --max-num-batched-tokens 8192" + decode: + workers: 1 + tp: 4 + cudagraph: >- + [1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256] + extra_args: "--decode-context-parallel-size 4 --cudagraph-mode FULL" + env: + common: + ATOM_MLA_PAGE_SIZE: "1" + ATOM_ONLINE_QUANT_STREAMING: "0" + ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB: "2047" + ATOM_USE_TRITON_MLA: "0" + MAX_JOBS: "16" + PYTHONHASHSEED: "0" + TOPK_FORCE_PATH: "one" + prefill: + HIP_VISIBLE_DEVICES: "0,1,2,3" + VLLM_PP_LAYER_PARTITION: "20,20,20,18" + LMCACHE_LOCAL_CPU: "True" + LMCACHE_MAX_LOCAL_CPU_SIZE: "256" + LMCACHE_CHUNK_SIZE: "256" + OFFLOAD_PROFILE: "1" + OFFLOAD_MIN_LOAD_TOKENS: "0" + PREFILL_KV_TRANSFER_CONFIG: >- + {"kv_connector":"multi","connectors":[{"kv_connector":"mooncake","kv_role":"kv_producer","proxy_ip":"${ROLE_IP}","handshake_port":${HANDSHAKE_PORT},"protocol":"rdma"},{"kv_connector":"lmcache_offload","kv_role":"offload"}]} + decode: + HIP_VISIBLE_DEVICES: "4,5,6,7" + DECODE_KV_TRANSFER_CONFIG: >- + {"kv_connector":"mooncake","kv_role":"kv_consumer","proxy_ip":"${ROLE_IP}","handshake_port":${HANDSHAKE_PORT},"protocol":"rdma"} + + - <<: *glm52_agentic_lmcache_cpp4_dcp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c32 + concurrency: [32] + + - <<: *glm52_agentic_lmcache_cpp4_dcp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40 + concurrency: [40] + - &glm52_agentic_lmcache_tp4_dpa <<: *glm52_agentic_tp4_tp4 name: glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c24 diff --git a/.github/scripts/atomesh/pd_matrix.py b/.github/scripts/atomesh/pd_matrix.py index b80efb095c..e4bfe92610 100644 --- a/.github/scripts/atomesh/pd_matrix.py +++ b/.github/scripts/atomesh/pd_matrix.py @@ -206,7 +206,9 @@ def role_env( model_cfg.get("env", {}).get(role, {}), suite_cfg.get("env", {}).get(role, {}), ) - env = resolve_env_refs_in_value(env, preserve_names={"ROLE_IP"}) + # ROLE_IP and HANDSHAKE_PORT are filled in by pd_server_atom.sh at + # launch time (host IP and 6301 + ATOMESH_SERVICE_PORT_OFFSET). + env = resolve_env_refs_in_value(env, preserve_names={"ROLE_IP", "HANDSHAKE_PORT"}) return {str(key): str(value) for key, value in env.items()} diff --git a/.github/scripts/atomesh/pd_server_atom.sh b/.github/scripts/atomesh/pd_server_atom.sh index b899eeb6a5..6d49485bf6 100644 --- a/.github/scripts/atomesh/pd_server_atom.sh +++ b/.github/scripts/atomesh/pd_server_atom.sh @@ -251,10 +251,38 @@ apply_prefixed_env() { while IFS='=' read -r name raw; do [[ "${name}" == "${prefix}"* ]] || continue value="${raw//\$\{ROLE_IP\}/${role_ip}}" + value="${value//\$\{HANDSHAKE_PORT\}/${HANDSHAKE_PORT}}" export "${name#${prefix}}=${value}" done < <(env) } +# Names the last apply_role_env() exported for a role. +ROLE_ENV_NAMES=() + +# Both servers are launched from this one shell, so a variable exported for one +# role stays in the environment the next role inherits. Only names the two roles +# both define get overwritten; a prefill-only name reaches decode unchanged -- +# VLLM_PP_LAYER_PARTITION from a pp4 prefill aborts a pp1 decode's model build +# with "len(partitions)=4 does not match pp_size=1". Drop the previous role's +# names before applying this one's. +apply_role_env() { + local prefix="$1" + local role_ip="$2" + local name + for name in ${ROLE_ENV_NAMES[@]+"${ROLE_ENV_NAMES[@]}"}; do + unset "${name}" + done + ROLE_ENV_NAMES=() + while IFS='=' read -r name _; do + [[ "${name}" == "${prefix}"* ]] || continue + ROLE_ENV_NAMES+=("${name#${prefix}}") + done < <(env) + # A name the common block also sets was just unset with the previous role's, + # so put the common value back before the role overrides it. + apply_prefixed_env "ATOMESH_ENV_" "${role_ip}" + apply_prefixed_env "${prefix}" "${role_ip}" +} + host_ip="$(echo "${IPADDRS}" | tr ',' '\n' | sed -n "$((NODE_RANK + 1))p")" if [[ -z "${host_ip}" ]]; then host_ip="$(hostname -I 2>/dev/null | awk '{print $1}')" @@ -610,7 +638,7 @@ start_prefill() { local handshake_port="${3:-${HANDSHAKE_PORT}}" local dp_master_port="${4:-${PREFILL_DP_MASTER_PORT}}" local dp_base_port="${5:-${PREFILL_DP_BASE_PORT}}" - apply_prefixed_env "ATOMESH_PREFILL_ENV_" "${host_ip}" + apply_role_env "ATOMESH_PREFILL_ENV_" "${host_ip}" local -a prefill_cache_env=() build_server_cache_env "prefill" "${server_port}" prefill_cache_env local -a prefill_dp_env=() @@ -647,7 +675,7 @@ start_decode() { local handshake_port="${3:-${HANDSHAKE_PORT}}" local dp_master_port="${4:-${DECODE_DP_MASTER_PORT}}" local dp_base_port="${5:-${DECODE_DP_BASE_PORT}}" - apply_prefixed_env "ATOMESH_DECODE_ENV_" "${host_ip}" + apply_role_env "ATOMESH_DECODE_ENV_" "${host_ip}" local max_conc max_conc="$(echo "${BENCH_MAX_CONCURRENCY}" | tr 'x,' '\n' | sort -n | tail -1)" local decode_max_num_seqs="${MAX_NUM_SEQS}" diff --git a/.github/scripts/atomesh/process_result.py b/.github/scripts/atomesh/process_result.py index 06142ce61a..be9f516a2d 100644 --- a/.github/scripts/atomesh/process_result.py +++ b/.github/scripts/atomesh/process_result.py @@ -35,6 +35,9 @@ TOPOLOGY_RE = re.compile(r"(?P

\d+)p(?P\d+)d", re.IGNORECASE) TP_RE = re.compile(r"tp(?P\d+)", re.IGNORECASE) DCP_RE = re.compile(r"dcp(?P\d+)", re.IGNORECASE) +DUAL_TP_RE = re.compile(r"tp(?P\d+)-tp(?P\d+)", re.IGNORECASE) +CPP_PP_RE = re.compile(r"(?:cpp|pp)(?P\d+)", re.IGNORECASE) +PP_ARG_RE = re.compile(r"--pipeline-parallel-size(?:=|\s+)(\d+)", re.IGNORECASE) EVAL_CONC_RE = re.compile(r"(?:^|[_-])c(?P\d+)(?:$|[_-])", re.IGNORECASE) EVAL_TOPOLOGY_RE = re.compile( r"(?:^|[_-])(?P\d+p\d+d(?:[_-]dpa)?)(?:$|[_-])", @@ -134,6 +137,10 @@ def pd_label(prefill: Any, decode: Any) -> str: return str(prefill) if prefill == decode else f"P{prefill}/D{decode}" value = prefill if prefill is not None else decode return "--" if value is None else str(value) +def divide_by_total_gpu(value: float | None, total_gpu: int | None) -> float | None: + if value and total_gpu: + return value / total_gpu + return None def interactivity_value(payload: dict[str, Any]) -> float | None: @@ -232,7 +239,6 @@ def topology_resources( ) ) topology = TOPOLOGY_RE.search(text) - tp = TP_RE.search(text) prefill_workers = int_value( payload.get("prefill_workers"), payload.get("num_prefill_workers") ) @@ -249,9 +255,31 @@ def topology_resources( decode_tp = int_value( payload.get("decode_tp"), payload.get("decode_tensor_parallel_size") ) - if tp: - prefill_tp = prefill_tp or int(tp.group("tp")) - decode_tp = decode_tp or int(tp.group("tp")) + dual_tp = DUAL_TP_RE.search(text) + if dual_tp: + prefill_tp = prefill_tp or int(dual_tp.group("prefill_tp")) + decode_tp = decode_tp or int(dual_tp.group("decode_tp")) + else: + tp = TP_RE.search(text) + if tp: + tp_size = int(tp.group("tp")) + prefill_tp = prefill_tp or tp_size + decode_tp = decode_tp or tp_size + + prefill_pp = int_value( + payload.get("prefill_pp"), payload.get("prefill_pipeline_parallel_size") + ) + if prefill_pp is None: + cpp_pp = CPP_PP_RE.search(text) + if cpp_pp: + prefill_pp = int(cpp_pp.group("pp")) + if prefill_pp is None: + for key in ("prefill_extra_server_args",): + pp_match = PP_ARG_RE.search(string_value(payload.get(key))) + if pp_match: + prefill_pp = int(pp_match.group(1)) + break + prefill_pp = prefill_pp or 1 prefill_dcp = int_value( payload.get("prefill_dcp"), payload.get("prefill_decode_context_parallel_size") @@ -267,7 +295,7 @@ def topology_resources( num_prefill_gpu = int_value(payload.get("num_prefill_gpu")) num_decode_gpu = int_value(payload.get("num_decode_gpu")) if num_prefill_gpu is None and prefill_workers and prefill_tp: - num_prefill_gpu = prefill_workers * prefill_tp + num_prefill_gpu = prefill_workers * prefill_tp * prefill_pp if num_decode_gpu is None and decode_workers and decode_tp: num_decode_gpu = decode_workers * decode_tp total_gpu = int_value(payload.get("total_gpu")) @@ -282,6 +310,7 @@ def topology_resources( "decode_tp": decode_tp, "prefill_dcp": prefill_dcp, "decode_dcp": decode_dcp, + "prefill_pp": prefill_pp, "num_prefill_gpu": num_prefill_gpu, "num_decode_gpu": num_decode_gpu, "total_gpu": total_gpu, @@ -386,6 +415,9 @@ def enrich_payload( enriched.setdefault("decode_dcp", env.get("DECODE_DCP_SIZE")) enriched.setdefault("speculative_method", env.get("SPEC_METHOD")) enriched.setdefault("num_speculative_tokens", env.get("NUM_SPEC_TOKENS")) + enriched.setdefault( + "prefill_extra_server_args", env.get("PREFILL_EXTRA_SERVER_ARGS") + ) runner = env.get("SLURM_SUBMIT_RUNNER", "") if hardware: enriched["hardware"] = hardware @@ -420,28 +452,17 @@ def enrich_payload( enriched.setdefault("interactivity", interactivity_value(enriched)) resources = topology_resources(enriched, fields) total_gpu = resources["total_gpu"] - num_prefill_gpu = resources["num_prefill_gpu"] - num_decode_gpu = resources["num_decode_gpu"] input_tput = number(enriched.get("input_throughput")) output_tput = number(enriched.get("output_throughput")) total_tput = number( enriched.get("total_token_throughput"), enriched.get("total_throughput") ) + enriched.setdefault("tput_per_gpu", divide_by_total_gpu(total_tput, total_gpu)) enriched.setdefault( - "tput_per_gpu", total_tput / total_gpu if total_tput and total_gpu else None + "input_tput_per_gpu", divide_by_total_gpu(input_tput, total_gpu) ) enriched.setdefault( - "input_tput_per_gpu", - input_tput / num_prefill_gpu if input_tput and num_prefill_gpu else None, - ) - output_tput_denominator = num_decode_gpu or total_gpu - enriched.setdefault( - "output_tput_per_gpu", - ( - output_tput / output_tput_denominator - if output_tput and output_tput_denominator - else None - ), + "output_tput_per_gpu", divide_by_total_gpu(output_tput, total_gpu) ) return enriched @@ -579,18 +600,10 @@ def perf_point( "cache_hit_rate": round_or_none(payload.get("cache_hit_rate")), "cache_hit_tokens": int_value(payload.get("cache_hit_tokens")), "cache_total_tokens": int_value(payload.get("cache_total_tokens")), - "tput_per_gpu": round_or_none( - total_tput / total_gpu if total_tput and total_gpu else None - ), - "input_tput_per_gpu": round_or_none( - input_tput / resources["num_prefill_gpu"] - if input_tput and resources["num_prefill_gpu"] - else None - ), + "tput_per_gpu": round_or_none(divide_by_total_gpu(total_tput, total_gpu)), + "input_tput_per_gpu": round_or_none(divide_by_total_gpu(input_tput, total_gpu)), "output_tput_per_gpu": round_or_none( - output_tput / (resources["num_decode_gpu"] or total_gpu) - if output_tput and (resources["num_decode_gpu"] or total_gpu) - else None + divide_by_total_gpu(output_tput, total_gpu) ), "run_url": run_url or "", "image": string_value(payload.get("docker_image"), payload.get("image")), diff --git a/.github/workflows/atomesh-benchmark.yaml b/.github/workflows/atomesh-benchmark.yaml index afcda86fe1..79296682b1 100644 --- a/.github/workflows/atomesh-benchmark.yaml +++ b/.github/workflows/atomesh-benchmark.yaml @@ -61,6 +61,9 @@ on: glm-52-mxfp4-1p1d-tp4,glm-52-mxfp4-1p1d-tp8, glm-52-mxfp4-1p1d-tp4-agentic-1m-c1,glm-52-mxfp4-1p1d-tp4-agentic-1m-c2, glm-52-mxfp4-1p1d-tp4-agentic-1m-c4,glm-52-mxfp4-1p1d-tp4-agentic-1m-c8, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c32, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c24, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c32, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c48 @@ -138,6 +141,10 @@ on: - mia1-p02-g42,mia1-p02-g44 - mia1-p02-g42,mia1-p02-g47 - mia1-p02-g44,mia1-p02-g47 + # single_node cases only use the first entry, so keep orderings that + # let any one of the three nodes be selected. + - mia1-p02-g47,mia1-p02-g44 + - mia1-p01-g43,mia1-p01-g36 - mia1-p01-g36,mia1-p01-g43 atomesh_2p1d_nodes: description: 'ATOMesh 2P1D nodes' From 2e0af78d7280b2c920974c95f422a5b67d79a1b9 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Sun, 6 Sep 2026 04:20:42 +0000 Subject: [PATCH 2/5] ci: add the LMCache NVMe tier to the GLM-5.2 CPP4+DCP4 case The prefill role already offloads to CPU DRAM, but the L3 tier that PR #1851 wired up was never enabled in the matrix, so a working set larger than the 256GiB hot cache simply fell out of the cache. LMCACHE_LOCAL_DISK and LMCACHE_MAX_LOCAL_DISK_SIZE are required together, and the size is per rank because scale_cpu_size_for_pp splits only the CPU budget across PP stages: 4 x 500GiB on a pp4 prefill. The tier lives on a host bind mount, so it outlives `docker run --rm` and would otherwise serve one concurrency's KV to the next while holding its disk. The launcher now empties the directory before the first prefill worker starts and removes it on exit. --- .github/benchmark/models_atomesh.yaml | 12 ++++++++++ .github/scripts/atomesh/pd_server_atom.sh | 27 +++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/.github/benchmark/models_atomesh.yaml b/.github/benchmark/models_atomesh.yaml index 62e028bb77..6af95c94e3 100644 --- a/.github/benchmark/models_atomesh.yaml +++ b/.github/benchmark/models_atomesh.yaml @@ -374,6 +374,18 @@ models: LMCACHE_LOCAL_CPU: "True" LMCACHE_MAX_LOCAL_CPU_SIZE: "256" LMCACHE_CHUNK_SIZE: "256" + # NVMe (L3) tier under the CPU hot cache. Both variables are + # required together; LMCACHE_MAX_LOCAL_CPU_SIZE must stay > 0 + # because LocalDiskBackend stages through the host allocator. + # scale_cpu_size_for_pp splits only the CPU budget across PP + # stages, so this size is per rank: 4 x 500GiB on a pp4 prefill. + # pd_server_atom.sh empties the directory around the run. + # The path follows the default atomesh-cicd fleet, where /mnt is + # local and already holds ATOMESH_MODEL_ROOT. The mi350 and crusoe + # fleets keep their models on /data and /shared_nfs instead, so a + # dispatch onto those needs this pointed at their own local disk. + LMCACHE_LOCAL_DISK: "/mnt/lmcache-job" + LMCACHE_MAX_LOCAL_DISK_SIZE: "500" OFFLOAD_PROFILE: "1" OFFLOAD_MIN_LOAD_TOKENS: "0" PREFILL_KV_TRANSFER_CONFIG: >- diff --git a/.github/scripts/atomesh/pd_server_atom.sh b/.github/scripts/atomesh/pd_server_atom.sh index 6d49485bf6..6eb15647f2 100644 --- a/.github/scripts/atomesh/pd_server_atom.sh +++ b/.github/scripts/atomesh/pd_server_atom.sh @@ -598,12 +598,38 @@ terminate_process_group() { wait "${pid}" 2>/dev/null || true } +# LMCache's NVMe tier lives on a host bind mount, so unlike the container's own +# /tmp it survives `docker run --rm`. Every concurrency runs as its own job, and +# a tier left behind would both serve the previous job's KV and hold its +# LMCACHE_MAX_LOCAL_DISK_SIZE of disk per rank. Start empty, leave nothing. +lmcache_disk_dir="" + +reset_lmcache_disk() { + local dir="${LMCACHE_LOCAL_DISK:-}" + [[ -n "${dir}" && "${dir}" != "/" ]] || return 0 + # Several prefill workers can share this shell, so only the first one empties + # the tier; a later one would delete a running worker's cache underneath it. + [[ "${lmcache_disk_dir}" != "${dir}" ]] || return 0 + lmcache_disk_dir="${dir}" + rm -rf -- "${dir}" + mkdir -p -- "${dir}" + echo "[lmcache] NVMe tier ${dir} reset (${LMCACHE_MAX_LOCAL_DISK_SIZE:-0}GiB per rank)" +} + +purge_lmcache_disk() { + [[ -n "${lmcache_disk_dir}" ]] || return 0 + rm -rf -- "${lmcache_disk_dir}" + echo "[lmcache] NVMe tier ${lmcache_disk_dir} removed" + lmcache_disk_dir="" +} + cleanup_processes() { local rc=$? local pid for pid in "$@"; do terminate_process_group "${pid}" done + purge_lmcache_disk return "${rc}" } @@ -639,6 +665,7 @@ start_prefill() { local dp_master_port="${4:-${PREFILL_DP_MASTER_PORT}}" local dp_base_port="${5:-${PREFILL_DP_BASE_PORT}}" apply_role_env "ATOMESH_PREFILL_ENV_" "${host_ip}" + reset_lmcache_disk local -a prefill_cache_env=() build_server_cache_env "prefill" "${server_port}" prefill_cache_env local -a prefill_dp_env=() From 267816d7fc99333eb3476ef9ce81f3b401a6c15e Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Mon, 7 Sep 2026 08:17:48 +0000 Subject: [PATCH 3/5] docs: add GLM-5.2 AgentX PD recipe --- recipes/Agentic-GLM-5.2.md | 232 ++++++++++++++++++++++++++++++++----- 1 file changed, 205 insertions(+), 27 deletions(-) diff --git a/recipes/Agentic-GLM-5.2.md b/recipes/Agentic-GLM-5.2.md index b13656e36b..9c165904f6 100644 --- a/recipes/Agentic-GLM-5.2.md +++ b/recipes/Agentic-GLM-5.2.md @@ -1,20 +1,177 @@ # GLM-5.2 AgentX Recipe on MI355X -This recipe runs the SemiAnalysis/Weka AgentX replay workload against GLM-5.2-MXFP4 with ATOM on 4×AMD MI355X GPUs. The validated configuration uses: +This recipe runs the SemiAnalysis/Weka AgentX replay workload against +GLM-5.2-MXFP4 with ATOM on AMD MI355X GPUs. It covers both a standalone TP4 +server on 4 GPUs and an optimized, single-node PD-disaggregated deployment on +8 GPUs. Both deployments use: - `amd/GLM-5.2-MXFP4` -- TP4 - FP8 KV cache - MTP with three speculative tokens - synthetic draft acceptance fixed to the InferenceX reference target -- native GPU prefix caching plus a 200 GiB LMCache CPU tier - the SemiAnalysis Weka AgentX workload -The workload uses the AIPerf scenario `inferencex-agentx-mvp` and public dataset `semianalysis_cc_traces_weka_062126`. It replays long-context, multi-turn coding traces with subagent fan-out rather than a fixed ISL/OSL workload. +Their parallelism and cache configurations differ and are documented in their +respective sections below. -For PD-disaggregated serving, see [`mesh/Agentic-GLM-5.2.md`](mesh/Agentic-GLM-5.2.md). This document covers a single standalone ATOM server. +The workload uses the AIPerf scenario `inferencex-agentx-mvp` and public dataset +`semianalysis_cc_traces_weka_062126`. It replays long-context, multi-turn coding +traces with subagent fan-out rather than a fixed ISL/OSL workload. -## Validated Configuration +The optimized single-node PD setup is documented below. For the general +multi-node PD workflow, see +[`mesh/Agentic-GLM-5.2.md`](mesh/Agentic-GLM-5.2.md). + +## 1. Start the ATOM Server + +### PD Disaggregated Deployment (1P1D) + +The optimized topology uses all 8 GPUs of one MI355X node: + +```text +AIPerf + | + v +atomesh router (:8000) + |-- Prefill (:8010), GPU 0-3: TP1 × PP4 + `-- Decode (:8020), GPU 4-7: TP4 × DCP4 +``` + +The validated prefill and decode configurations are: + +| Item | Prefill node | Decode node | +|---|---|---| +| GPU count | 4 | 4 | +| Parallelism | TP1 × PP4 | TP4 × DCP4 | +| PP partition | `20,20,20,18` | N/A | +| KV cache | FP8, block size 16 | FP8, block size 16 | +| GPU memory utilization | 0.85 | 0.85 | +| Maximum sequences | 512 | 512 | +| Compilation | Level 3, enforce eager | Level 3 | +| CUDAGraph | Disabled by enforce eager | Full mode, configured sizes up to 256 | +| Batched-token budget | 8192 | 16384 (default) | +| LMCache | 256 GiB CPU tier, 256-token chunks | Disabled | +| Native prefix caching | Enabled | Enabled | +| Speculative decoding | MTP3 | MTP3 | + +Both nodes use MXFP4 weights, online PTPC FP8 quantization, +`ATOM_MLA_PAGE_SIZE=1`, `ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB=2047`, +`ATOM_ONLINE_QUANT_STREAMING=0`, and `ATOM_USE_TRITON_MLA=0`. The prefill node +enables `OFFLOAD_PROFILE=1` and `OFFLOAD_MIN_LOAD_TOKENS=0`. + +#### Start the PD Deployment + +Cold-start the deployment for each concurrency point. The commands below write +`prefill.log`, `decode.log`, and `mesh.log` in the current directory: + +```bash +export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} + +rm -rf "${HOME}/.cache/atom/"* + +export PYTHONUNBUFFERED=1 +export PYTHONHASHSEED=0 +export AITER_LOG_LEVEL=WARNING +export AITER_QUICK_REDUCE_QUANTIZATION=INT4 +export AITER_USE_FLYDSL_MOE_SORTING=1 +export ATOM_MLA_PAGE_SIZE=1 +export ATOM_ONLINE_QUANT_STREAMING=0 +export ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB=2047 +export ATOM_USE_TRITON_MLA=0 +export MAX_JOBS=16 +export ATOM_HOST_IP=127.0.0.1 +export LD_LIBRARY_PATH="$(python3 -c \ + 'import sysconfig; print(sysconfig.get_path("purelib"))')/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-}" + +ONLINE_QUANT_CONFIG='{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' +PREFILL_KV_CONFIG='{"kv_connector":"multi","connectors":[{"kv_connector":"mooncake","kv_role":"kv_producer","proxy_ip":"127.0.0.1","handshake_port":6301,"protocol":"rdma"},{"kv_connector":"lmcache_offload","kv_role":"offload"}]}' +DECODE_KV_CONFIG='{"kv_connector":"mooncake","kv_role":"kv_consumer","proxy_ip":"127.0.0.1","handshake_port":6301,"protocol":"rdma"}' +DECODE_CUDAGRAPH='[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' + +COMMON_ARGS=( + --model "${MODEL_PATH}" + --host 0.0.0.0 + --trust-remote-code + --kv_cache_dtype fp8 + --block-size 16 + --gpu-memory-utilization 0.85 + --max-num-seqs 512 + --enable_prefix_caching + --online_quant_config "${ONLINE_QUANT_CONFIG}" + --level 3 + --method mtp + --num-speculative-tokens 3 +) + +env \ + HIP_VISIBLE_DEVICES=0,1,2,3 \ + VLLM_PP_LAYER_PARTITION=20,20,20,18 \ + LMCACHE_LOCAL_CPU=True \ + LMCACHE_MAX_LOCAL_CPU_SIZE=256 \ + LMCACHE_CHUNK_SIZE=256 \ + OFFLOAD_PROFILE=1 \ + OFFLOAD_MIN_LOAD_TOKENS=0 \ + nohup python3 -m atom.entrypoints.openai_server \ + "${COMMON_ARGS[@]}" \ + --server-port 8010 \ + --tensor-parallel-size 1 \ + --pipeline-parallel-size 4 \ + --enforce-eager \ + --max-num-batched-tokens 8192 \ + --kv-transfer-config "${PREFILL_KV_CONFIG}" \ + >prefill.log 2>&1 & +PREFILL_PID=$! + +env \ + HIP_VISIBLE_DEVICES=4,5,6,7 \ + nohup python3 -m atom.entrypoints.openai_server \ + "${COMMON_ARGS[@]}" \ + --server-port 8020 \ + --tensor-parallel-size 4 \ + --decode-context-parallel-size 4 \ + --cudagraph-mode FULL \ + --cudagraph-capture-sizes "${DECODE_CUDAGRAPH}" \ + --kv-transfer-config "${DECODE_KV_CONFIG}" \ + >decode.log 2>&1 & +DECODE_PID=$! + +wait_ready() { + local name=$1 port=$2 pid=$3 log=$4 + until curl -sf "http://127.0.0.1:${port}/v1/models" >/dev/null; do + kill -0 "${pid}" 2>/dev/null || { + echo "${name} failed; see ${log}" >&2 + tail -100 "${log}" >&2 + return 1 + } + sleep 10 + done +} + +wait_ready prefill 8010 "${PREFILL_PID}" prefill.log +wait_ready decode 8020 "${DECODE_PID}" decode.log + +nohup atomesh launch \ + --host 0.0.0.0 \ + --port 8000 \ + --pd-disaggregation \ + --prefill http://127.0.0.1:8010 6301 \ + --decode http://127.0.0.1:8020 \ + --policy random \ + --backend atom \ + --log-level info \ + --disable-circuit-breaker \ + --prometheus-port 29100 \ + >mesh.log 2>&1 & +MESH_PID=$! + +wait_ready mesh 8000 "${MESH_PID}" mesh.log +``` + +### PD Mixed Deployment (Standalone) + +Start a fresh server for each concurrency point. + +The validated standalone configuration is: | Item | Value | |---|---| @@ -31,14 +188,10 @@ For PD-disaggregated serving, see [`mesh/Agentic-GLM-5.2.md`](mesh/Agentic-GLM-5 | Warmup | 10 additional one-token requests per lane | | AIPerf | `0.12.0` (`agentx-v1.0.4`) | -## 1. Start the ATOM Server - -Start a fresh server for each concurrency point. - -### GLM-5.2 MXFP4 with MTP +#### GLM-5.2 MXFP4 with MTP ```bash -export MODEL_PATH=${MODEL_PATH:-models/GLM-5.2-MXFP4} +export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} export AITER_QUICK_REDUCE_QUANTIZATION=INT4 export AITER_USE_FLYDSL_MOE_SORTING=1 @@ -86,7 +239,7 @@ python -m atom.entrypoints.openai_server \ 2>&1 | tee "server-glm52-mtp3-synth-c${CONC}.log" ``` -#### Synthetic Acceptance Semantics +##### Synthetic Acceptance Semantics `--spec-decode-acceptance-rate 0.6633` fixes the mean draft-token acceptance ratio: @@ -99,7 +252,7 @@ The draft model and target verification still run. This override controls which This mode is **performance-only**. Disable `--spec-decode-acceptance-rate` for SWE-bench, GSM8K, or any correctness evaluation because synthetic acceptance does not preserve model accuracy. -#### Use GPU Prefix Caching Without LMCache +##### Use GPU Prefix Caching Without LMCache To use only the native GPU prefix cache, unset the LMCache-related environment variables before starting the server: @@ -113,10 +266,10 @@ unset OFFLOAD_MIN_LOAD_TOKENS Also remove the `--kv-transfer-config` argument from the server command. -### GLM-5.2 MXFP4 Without MTP +#### GLM-5.2 MXFP4 Without MTP ```bash -export MODEL_PATH=${MODEL_PATH:-models/GLM-5.2-MXFP4} +export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} export AITER_QUICK_REDUCE_QUANTIZATION=INT4 export AITER_USE_FLYDSL_MOE_SORTING=1 @@ -163,11 +316,13 @@ python -m atom.entrypoints.openai_server \ ## 2. Run the AgentX Profile +### PD Mixed Deployment (Standalone) + Run this once per concurrency point against a newly started server: ```bash export CONC=${CONC:-10} -export MODEL_PATH=${MODEL_PATH:-models/GLM-5.2-MXFP4} +export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} export OUTPUT_DIR=${OUTPUT_DIR:-results/glm52-agentx-c${CONC}} export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0 @@ -208,31 +363,54 @@ aiperf profile \ 2>&1 | tee "${OUTPUT_DIR}/aiperf.log" ``` +### PD Disaggregated Deployment (1P1D) + +Use the same AIPerf command. The client URL remains +`http://127.0.0.1:8000`, which is the atomesh endpoint. Replace the +single-server metrics argument with both ATOM endpoints: + +```bash +--server-metrics \ + http://127.0.0.1:8010/metrics \ + http://127.0.0.1:8020/metrics +``` ## Accuracy -Synthetic acceptance is performance-only. For accuracy evaluation, either use the non-MTP server command or use MTP without `--spec-decode-acceptance-rate`. Then run: +Synthetic acceptance is performance-only. For accuracy evaluation, either use +the non-MTP standalone command or use MTP without +`--spec-decode-acceptance-rate`. ```bash -export MODEL_PATH=${MODEL_PATH:-models/GLM-5.2-MXFP4} +export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} python3 -m lm_eval \ --model local-chat-completions \ - --apply_chat_template \ - --tasks gsm8k \ - --output_path ./eval_out-tta1J8 \ - --log_samples \ --model_args \ - "model=${MODEL_PATH},base_url=http://127.0.0.1:8000/v1/chat/completions,api_key=EMPTY,eos_string=,max_retries=5,num_concurrent=16,timeout=1800,tokenized_requests=False,max_length=1048576" \ - --gen_kwargs max_tokens=16384,temperature=0,top_p=1 + "model=${MODEL_PATH},base_url=http://127.0.0.1:8000/v1/chat/completions,num_concurrent=64,max_retries=3,tokenized_requests=False,trust_remote_code=True" \ + --tasks gsm8k \ + --num_fewshot 5 \ + --apply_chat_template \ + --fewshot_as_multiturn \ + --gen_kwargs max_gen_toks=16384,temperature=0,top_p=1 ``` -Validated GSM8K 5-shot result: +Validated standalone GSM8K 5-shot result: ```text -local-chat-completions ({'model': '/shared/data/amd_int/models/GLM-5.2-MXFP4', 'base_url': 'http://0.0.0.0:8000/v1/chat/completions', 'api_key': 'EMPTY', 'eos_string': '', 'max_retries': 5, 'num_concurrent': 16, 'timeout': 1800, 'tokenized_requests': False, 'max_length': 1048576}), gen_kwargs: ({'max_tokens': 16384, 'temperature': 0, 'top_p': 1}), limit: None, num_fewshot: None, batch_size: 1 +local-chat-completions ({'model': 'amd/GLM-5.2-MXFP4', 'base_url': 'http://0.0.0.0:8000/v1/chat/completions', 'api_key': 'EMPTY', 'eos_string': '', 'max_retries': 5, 'num_concurrent': 16, 'timeout': 1800, 'tokenized_requests': False, 'max_length': 1048576}), gen_kwargs: ({'max_tokens': 16384, 'temperature': 0, 'top_p': 1}), limit: None, num_fewshot: None, batch_size: 1 |Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr| |-----|------:|----------------|-----:|-----------|---|-----:|---|-----:| |gsm8k| 3|flexible-extract| 5|exact_match|↑ |0.9674|± |0.0049| | | |strict-match | 5|exact_match|↑ |0.9659|± |0.0050| ``` + +Validated GSM8K 5-shot accuracy for the PD-disaggregated deployment above +(full 1319 samples, real MTP3 without synthetic acceptance): + +```text +|Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr| +|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:| +|gsm8k| 3|flexible-extract| 5|exact_match|↑ |0.9613|± |0.0053| +| | |strict-match | 5|exact_match|↑ |0.9621|± |0.0053| +``` From 4e8727d70f2f05214acee539984b63dd1970c700 Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Fri, 11 Sep 2026 06:54:40 +0000 Subject: [PATCH 4/5] fix(ci): revert extra node orderings and satisfy Black Drop the workflow_dispatch 1P1D node permutations added for single-node cases, and fix Black formatting in process_result.py so Pre Checkin passes. Co-authored-by: Cursor --- .github/scripts/atomesh/process_result.py | 2 ++ .github/workflows/atomesh-benchmark.yaml | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/scripts/atomesh/process_result.py b/.github/scripts/atomesh/process_result.py index be9f516a2d..9d87612886 100644 --- a/.github/scripts/atomesh/process_result.py +++ b/.github/scripts/atomesh/process_result.py @@ -137,6 +137,8 @@ def pd_label(prefill: Any, decode: Any) -> str: return str(prefill) if prefill == decode else f"P{prefill}/D{decode}" value = prefill if prefill is not None else decode return "--" if value is None else str(value) + + def divide_by_total_gpu(value: float | None, total_gpu: int | None) -> float | None: if value and total_gpu: return value / total_gpu diff --git a/.github/workflows/atomesh-benchmark.yaml b/.github/workflows/atomesh-benchmark.yaml index 79296682b1..5ebd28c4d8 100644 --- a/.github/workflows/atomesh-benchmark.yaml +++ b/.github/workflows/atomesh-benchmark.yaml @@ -141,10 +141,6 @@ on: - mia1-p02-g42,mia1-p02-g44 - mia1-p02-g42,mia1-p02-g47 - mia1-p02-g44,mia1-p02-g47 - # single_node cases only use the first entry, so keep orderings that - # let any one of the three nodes be selected. - - mia1-p02-g47,mia1-p02-g44 - - mia1-p01-g43,mia1-p01-g36 - mia1-p01-g36,mia1-p01-g43 atomesh_2p1d_nodes: description: 'ATOMesh 2P1D nodes' From df2fde4cb9f7ed6291d30d0986bbc1e9f8992040 Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Fri, 11 Sep 2026 07:25:38 +0000 Subject: [PATCH 5/5] ci: add GLM-5.2 CPP4+DCP4 agentic c56 and tidy PD recipe Add the c56 ATOMesh case and split the Agentic-GLM-5.2 PD startup into self-contained prefill, decode, and atomesh scripts with inline args and curl readiness notes. Co-authored-by: Cursor --- .github/benchmark/models_atomesh.yaml | 4 + .github/workflows/atomesh-benchmark.yaml | 1 + recipes/Agentic-GLM-5.2.md | 124 ++++++++++++++--------- 3 files changed, 82 insertions(+), 47 deletions(-) diff --git a/.github/benchmark/models_atomesh.yaml b/.github/benchmark/models_atomesh.yaml index 6af95c94e3..95ad6ae0e6 100644 --- a/.github/benchmark/models_atomesh.yaml +++ b/.github/benchmark/models_atomesh.yaml @@ -403,6 +403,10 @@ models: name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40 concurrency: [40] + - <<: *glm52_agentic_lmcache_cpp4_dcp4 + name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c56 + concurrency: [56] + - &glm52_agentic_lmcache_tp4_dpa <<: *glm52_agentic_tp4_tp4 name: glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c24 diff --git a/.github/workflows/atomesh-benchmark.yaml b/.github/workflows/atomesh-benchmark.yaml index 5ebd28c4d8..f890639e45 100644 --- a/.github/workflows/atomesh-benchmark.yaml +++ b/.github/workflows/atomesh-benchmark.yaml @@ -64,6 +64,7 @@ on: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c32, glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40, glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48, + glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c56, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c24, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c32, glm-52-mxfp4-1p1d-tp4-dpa-agentic-lmcache-1m-c48 diff --git a/recipes/Agentic-GLM-5.2.md b/recipes/Agentic-GLM-5.2.md index 9c165904f6..85b4937aeb 100644 --- a/recipes/Agentic-GLM-5.2.md +++ b/recipes/Agentic-GLM-5.2.md @@ -61,8 +61,24 @@ enables `OFFLOAD_PROFILE=1` and `OFFLOAD_MIN_LOAD_TOKENS=0`. #### Start the PD Deployment -Cold-start the deployment for each concurrency point. The commands below write -`prefill.log`, `decode.log`, and `mesh.log` in the current directory: +Cold-start the deployment for each concurrency point. Each script below is +self-contained and can be run from a separate terminal. They write `prefill.log`, +`decode.log`, and `mesh.log` in the current directory. + +Start prefill and decode first. **Do not start atomesh until both workers are +ready** — otherwise the router may fail health checks or route traffic to servers +that are still loading weights. After each worker starts, wait until its endpoint +responds (model load can take several minutes): + +```bash +curl -sf http://127.0.0.1:8010/v1/models # prefill ready +curl -sf http://127.0.0.1:8020/v1/models # decode ready +``` + +If a curl fails, check the corresponding log (`prefill.log` or `decode.log`) and +retry once the server is up. + +##### Start Prefill ```bash export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} @@ -83,26 +99,6 @@ export ATOM_HOST_IP=127.0.0.1 export LD_LIBRARY_PATH="$(python3 -c \ 'import sysconfig; print(sysconfig.get_path("purelib"))')/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-}" -ONLINE_QUANT_CONFIG='{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' -PREFILL_KV_CONFIG='{"kv_connector":"multi","connectors":[{"kv_connector":"mooncake","kv_role":"kv_producer","proxy_ip":"127.0.0.1","handshake_port":6301,"protocol":"rdma"},{"kv_connector":"lmcache_offload","kv_role":"offload"}]}' -DECODE_KV_CONFIG='{"kv_connector":"mooncake","kv_role":"kv_consumer","proxy_ip":"127.0.0.1","handshake_port":6301,"protocol":"rdma"}' -DECODE_CUDAGRAPH='[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' - -COMMON_ARGS=( - --model "${MODEL_PATH}" - --host 0.0.0.0 - --trust-remote-code - --kv_cache_dtype fp8 - --block-size 16 - --gpu-memory-utilization 0.85 - --max-num-seqs 512 - --enable_prefix_caching - --online_quant_config "${ONLINE_QUANT_CONFIG}" - --level 3 - --method mtp - --num-speculative-tokens 3 -) - env \ HIP_VISIBLE_DEVICES=0,1,2,3 \ VLLM_PP_LAYER_PARTITION=20,20,20,18 \ @@ -112,44 +108,81 @@ env \ OFFLOAD_PROFILE=1 \ OFFLOAD_MIN_LOAD_TOKENS=0 \ nohup python3 -m atom.entrypoints.openai_server \ - "${COMMON_ARGS[@]}" \ + --model "${MODEL_PATH}" \ + --host 0.0.0.0 \ + --trust-remote-code \ + --kv_cache_dtype fp8 \ + --block-size 16 \ + --gpu-memory-utilization 0.85 \ + --max-num-seqs 512 \ + --enable_prefix_caching \ + --online_quant_config \ + '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' \ + --level 3 \ + --method mtp \ + --num-speculative-tokens 3 \ --server-port 8010 \ --tensor-parallel-size 1 \ --pipeline-parallel-size 4 \ --enforce-eager \ --max-num-batched-tokens 8192 \ - --kv-transfer-config "${PREFILL_KV_CONFIG}" \ + --kv-transfer-config \ + '{"kv_connector":"multi","connectors":[{"kv_connector":"mooncake","kv_role":"kv_producer","proxy_ip":"127.0.0.1","handshake_port":6301,"protocol":"rdma"},{"kv_connector":"lmcache_offload","kv_role":"offload"}]}' \ >prefill.log 2>&1 & -PREFILL_PID=$! +``` + +##### Start Decode + +```bash +export MODEL_PATH=${MODEL_PATH:-amd/GLM-5.2-MXFP4} + +export PYTHONUNBUFFERED=1 +export PYTHONHASHSEED=0 +export AITER_LOG_LEVEL=WARNING +export AITER_QUICK_REDUCE_QUANTIZATION=INT4 +export AITER_USE_FLYDSL_MOE_SORTING=1 +export ATOM_MLA_PAGE_SIZE=1 +export ATOM_ONLINE_QUANT_STREAMING=0 +export ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB=2047 +export ATOM_USE_TRITON_MLA=0 +export MAX_JOBS=16 +export ATOM_HOST_IP=127.0.0.1 +export LD_LIBRARY_PATH="$(python3 -c \ + 'import sysconfig; print(sysconfig.get_path("purelib"))')/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-}" env \ HIP_VISIBLE_DEVICES=4,5,6,7 \ nohup python3 -m atom.entrypoints.openai_server \ - "${COMMON_ARGS[@]}" \ + --model "${MODEL_PATH}" \ + --host 0.0.0.0 \ + --trust-remote-code \ + --kv_cache_dtype fp8 \ + --block-size 16 \ + --gpu-memory-utilization 0.85 \ + --max-num-seqs 512 \ + --enable_prefix_caching \ + --online_quant_config \ + '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}' \ + --level 3 \ + --method mtp \ + --num-speculative-tokens 3 \ --server-port 8020 \ --tensor-parallel-size 4 \ --decode-context-parallel-size 4 \ --cudagraph-mode FULL \ - --cudagraph-capture-sizes "${DECODE_CUDAGRAPH}" \ - --kv-transfer-config "${DECODE_KV_CONFIG}" \ + --cudagraph-capture-sizes \ + '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' \ + --kv-transfer-config \ + '{"kv_connector":"mooncake","kv_role":"kv_consumer","proxy_ip":"127.0.0.1","handshake_port":6301,"protocol":"rdma"}' \ >decode.log 2>&1 & -DECODE_PID=$! - -wait_ready() { - local name=$1 port=$2 pid=$3 log=$4 - until curl -sf "http://127.0.0.1:${port}/v1/models" >/dev/null; do - kill -0 "${pid}" 2>/dev/null || { - echo "${name} failed; see ${log}" >&2 - tail -100 "${log}" >&2 - return 1 - } - sleep 10 - done -} - -wait_ready prefill 8010 "${PREFILL_PID}" prefill.log -wait_ready decode 8020 "${DECODE_PID}" decode.log +``` + +##### Start ATOMesh +Run this only after both prefill (`:8010`) and decode (`:8020`) pass the curl checks +above. + +```bash nohup atomesh launch \ --host 0.0.0.0 \ --port 8000 \ @@ -162,9 +195,6 @@ nohup atomesh launch \ --disable-circuit-breaker \ --prometheus-port 29100 \ >mesh.log 2>&1 & -MESH_PID=$! - -wait_ready mesh 8000 "${MESH_PID}" mesh.log ``` ### PD Mixed Deployment (Standalone)