[ATOMesh][CI][benchmark] add GLM-5.2 CPP4+DCP4 ATOMesh coverage

.github/benchmark/models_atomesh.yaml

@@ -248,7 +248,7 @@ models:
             isl: [1048576]
             osl: 1024
             server:
-              common_args:
+              common_args: &glm52_agentic_args
                 kv_cache_dtype: fp8
                 block_size: 16
                 gpu_memory_utilization: 0.9
@@ -294,36 +294,106 @@ models:
             run_eval: false
           name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c1
           concurrency: [1]
-          accuracy:
-            task: swebench_lite
-            threshold: 0.50
-            agent_workers: 32
-            agent_step_limit: 150
-            case_timeout: 3600
-            agent_timeout: 21600
-            score_timeout: 7200
-            max_workers: 4
-            instance_timeout: 900
-          eval_concurrency: [1]
-          run_eval: true
-          runner:
-            time_limit: "12:00:00"
+          server:
+            common_args: &glm52_agentic_mtp_args
+              <<: *glm52_agentic_args
+              extra_args: "--method mtp --num-speculative-tokens 4 --spec-decode-acceptance-rate 0.5825"
 
         - <<: *glm52_agentic_tp4_tp4
           name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c2
           concurrency: [2]
+          server:
+            common_args: *glm52_agentic_mtp_args
 
         - <<: *glm52_agentic_tp4_tp4
           name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c4
           concurrency: [4]
+          server:
+            common_args: *glm52_agentic_mtp_args
 
         - <<: *glm52_agentic_tp4_tp4
           name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c8
           concurrency: [8]
+          server:
+            common_args: *glm52_agentic_mtp_args
 
-        - <<: *glm52_agentic_tp4_tp4
-          name: glm-52-mxfp4-1p1d-tp4-agentic-1m-c48
+        # CPP4 prefill (PP4 x TP1) paired with TP4+DCP4 decode on one node.
+        - &glm52_agentic_lmcache_cpp4_dcp4
+          <<: *glm52_agentic_tp4_tp4
+          name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48
+          topology: 1p1d_cpp4_dcp4
+          pd_worker_layout: single_node
           concurrency: [48]
+          server:
+            common_args:
+              kv_cache_dtype: fp8
+              block_size: 16
+              max_num_seqs: 512
+              enable_prefix_caching: true
+              decode_max_num_seqs: 512
+              # Empty means "do not pass the flag", so the model config's own
+              # context length applies. The 16384 the model defaults carry would
+              # truncate this suite's 1M agentic traces.
+              max_model_len: ""
+              max_num_batched_tokens: ""
+              decode_max_num_batched_tokens: ""
+              online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","*expert*"]}'
+              gpu_memory_utilization: 0.85
+              extra_args: "--level 3 --method mtp --num-speculative-tokens 3 --spec-decode-acceptance-rate 0.6633"
+          prefill:
+            workers: 1
+            tp: 1
+            cudagraph: none
+            extra_args: "--pipeline-parallel-size 4 --enforce-eager --max-num-batched-tokens 8192"
+          decode:
+            workers: 1
+            tp: 4
+            cudagraph: >-
+              [1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]
+            extra_args: "--decode-context-parallel-size 4 --cudagraph-mode FULL"
+          env:
+            common:
+              ATOM_MLA_PAGE_SIZE: "1"
+              ATOM_ONLINE_QUANT_STREAMING: "0"
+              ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB: "2047"
+              ATOM_USE_TRITON_MLA: "0"
+              MAX_JOBS: "16"
+              PYTHONHASHSEED: "0"
+              TOPK_FORCE_PATH: "one"
+            prefill:
+              HIP_VISIBLE_DEVICES: "0,1,2,3"
+              VLLM_PP_LAYER_PARTITION: "20,20,20,18"
+              LMCACHE_LOCAL_CPU: "True"
+              LMCACHE_MAX_LOCAL_CPU_SIZE: "256"
+              LMCACHE_CHUNK_SIZE: "256"
+              # NVMe (L3) tier under the CPU hot cache. Both variables are
+              # required together; LMCACHE_MAX_LOCAL_CPU_SIZE must stay > 0
+              # because LocalDiskBackend stages through the host allocator.
+              # scale_cpu_size_for_pp splits only the CPU budget across PP
+              # stages, so this size is per rank: 4 x 500GiB on a pp4 prefill.
+              # pd_server_atom.sh empties the directory around the run.
+              # The path follows the default atomesh-cicd fleet, where /mnt is
+              # local and already holds ATOMESH_MODEL_ROOT. The mi350 and crusoe
+              # fleets keep their models on /data and /shared_nfs instead, so a
+              # dispatch onto those needs this pointed at their own local disk.
+              LMCACHE_LOCAL_DISK: "/mnt/lmcache-job"
+              LMCACHE_MAX_LOCAL_DISK_SIZE: "500"
+              OFFLOAD_PROFILE: "1"
+              OFFLOAD_MIN_LOAD_TOKENS: "0"
+              PREFILL_KV_TRANSFER_CONFIG: >-
+                {"kv_connector":"multi","connectors":[{"kv_connector":"mooncake","kv_role":"kv_producer","proxy_ip":"${ROLE_IP}","handshake_port":${HANDSHAKE_PORT},"protocol":"rdma"},{"kv_connector":"lmcache_offload","kv_role":"offload"}]}
+            decode:
+              HIP_VISIBLE_DEVICES: "4,5,6,7"
+              DECODE_KV_TRANSFER_CONFIG: >-
+                {"kv_connector":"mooncake","kv_role":"kv_consumer","proxy_ip":"${ROLE_IP}","handshake_port":${HANDSHAKE_PORT},"protocol":"rdma"}
+
+        - <<: *glm52_agentic_lmcache_cpp4_dcp4
+          name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c32
+          concurrency: [32]
+
+        - <<: *glm52_agentic_lmcache_cpp4_dcp4
+          name: glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c40
+          concurrency: [40]
 
         - &glm52_agentic_lmcache_tp4_dpa
           <<: *glm52_agentic_tp4_tp4

.github/scripts/atomesh/pd_matrix.py

@@ -206,7 +206,9 @@ def role_env(
         model_cfg.get("env", {}).get(role, {}),
         suite_cfg.get("env", {}).get(role, {}),
     )
-    env = resolve_env_refs_in_value(env, preserve_names={"ROLE_IP"})
+    # ROLE_IP and HANDSHAKE_PORT are filled in by pd_server_atom.sh at
+    # launch time (host IP and 6301 + ATOMESH_SERVICE_PORT_OFFSET).
+    env = resolve_env_refs_in_value(env, preserve_names={"ROLE_IP", "HANDSHAKE_PORT"})
     return {str(key): str(value) for key, value in env.items()}
 
 

.github/scripts/atomesh/pd_server_atom.sh

@@ -246,10 +246,38 @@ apply_prefixed_env() {
   while IFS='=' read -r name raw; do
     [[ "${name}" == "${prefix}"* ]] || continue
     value="${raw//\$\{ROLE_IP\}/${role_ip}}"
+    value="${value//\$\{HANDSHAKE_PORT\}/${HANDSHAKE_PORT}}"
     export "${name#${prefix}}=${value}"
   done < <(env)
 }
 
+# Names the last apply_role_env() exported for a role.
+ROLE_ENV_NAMES=()
+
+# Both servers are launched from this one shell, so a variable exported for one
+# role stays in the environment the next role inherits. Only names the two roles
+# both define get overwritten; a prefill-only name reaches decode unchanged --
+# VLLM_PP_LAYER_PARTITION from a pp4 prefill aborts a pp1 decode's model build
+# with "len(partitions)=4 does not match pp_size=1". Drop the previous role's
+# names before applying this one's.
+apply_role_env() {
+  local prefix="$1"
+  local role_ip="$2"
+  local name
+  for name in ${ROLE_ENV_NAMES[@]+"${ROLE_ENV_NAMES[@]}"}; do
+    unset "${name}"
+  done
+  ROLE_ENV_NAMES=()
+  while IFS='=' read -r name _; do
+    [[ "${name}" == "${prefix}"* ]] || continue
+    ROLE_ENV_NAMES+=("${name#${prefix}}")
+  done < <(env)
+  # A name the common block also sets was just unset with the previous role's,
+  # so put the common value back before the role overrides it.
+  apply_prefixed_env "ATOMESH_ENV_" "${role_ip}"
+  apply_prefixed_env "${prefix}" "${role_ip}"
+}
+
 host_ip="$(echo "${IPADDRS}" | tr ',' '\n' | sed -n "$((NODE_RANK + 1))p")"
 if [[ -z "${host_ip}" ]]; then
   host_ip="$(hostname -I 2>/dev/null | awk '{print $1}')"
@@ -503,12 +531,38 @@ terminate_process_group() {
   wait "${pid}" 2>/dev/null || true
 }
 
+# LMCache's NVMe tier lives on a host bind mount, so unlike the container's own
+# /tmp it survives `docker run --rm`. Every concurrency runs as its own job, and
+# a tier left behind would both serve the previous job's KV and hold its
+# LMCACHE_MAX_LOCAL_DISK_SIZE of disk per rank. Start empty, leave nothing.
+lmcache_disk_dir=""
+
+reset_lmcache_disk() {
+  local dir="${LMCACHE_LOCAL_DISK:-}"
+  [[ -n "${dir}" && "${dir}" != "/" ]] || return 0
+  # Several prefill workers can share this shell, so only the first one empties
+  # the tier; a later one would delete a running worker's cache underneath it.
+  [[ "${lmcache_disk_dir}" != "${dir}" ]] || return 0
+  lmcache_disk_dir="${dir}"
+  rm -rf -- "${dir}"
+  mkdir -p -- "${dir}"
+  echo "[lmcache] NVMe tier ${dir} reset (${LMCACHE_MAX_LOCAL_DISK_SIZE:-0}GiB per rank)"
+}
+
+purge_lmcache_disk() {
+  [[ -n "${lmcache_disk_dir}" ]] || return 0
+  rm -rf -- "${lmcache_disk_dir}"
+  echo "[lmcache] NVMe tier ${lmcache_disk_dir} removed"
+  lmcache_disk_dir=""
+}
+
 cleanup_processes() {
   local rc=$?
   local pid
   for pid in "$@"; do
     terminate_process_group "${pid}"
   done
+  purge_lmcache_disk
   return "${rc}"
 }
 
@@ -543,7 +597,8 @@ start_prefill() {
   local handshake_port="${3:-${HANDSHAKE_PORT}}"
   local dp_master_port="${4:-${PREFILL_DP_MASTER_PORT}}"
   local dp_base_port="${5:-${PREFILL_DP_BASE_PORT}}"
-  apply_prefixed_env "ATOMESH_PREFILL_ENV_" "${host_ip}"
+  apply_role_env "ATOMESH_PREFILL_ENV_" "${host_ip}"
+  reset_lmcache_disk
   local -a prefill_cache_env=()
   build_server_cache_env "prefill" "${server_port}" prefill_cache_env
   local -a prefill_dp_env=()
@@ -580,7 +635,7 @@ start_decode() {
   local handshake_port="${3:-${HANDSHAKE_PORT}}"
   local dp_master_port="${4:-${DECODE_DP_MASTER_PORT}}"
   local dp_base_port="${5:-${DECODE_DP_BASE_PORT}}"
-  apply_prefixed_env "ATOMESH_DECODE_ENV_" "${host_ip}"
+  apply_role_env "ATOMESH_DECODE_ENV_" "${host_ip}"
   local max_conc
   max_conc="$(echo "${BENCH_MAX_CONCURRENCY}" | tr 'x,' '\n' | sort -n | tail -1)"
   local decode_max_num_seqs="${MAX_NUM_SEQS}"

.github/scripts/atomesh/process_result.py

@@ -27,6 +27,9 @@
 )
 TOPOLOGY_RE = re.compile(r"(?P<p>\d+)p(?P<d>\d+)d", re.IGNORECASE)
 TP_RE = re.compile(r"tp(?P<tp>\d+)", re.IGNORECASE)
+DUAL_TP_RE = re.compile(r"tp(?P<prefill_tp>\d+)-tp(?P<decode_tp>\d+)", re.IGNORECASE)
+CPP_PP_RE = re.compile(r"(?:cpp|pp)(?P<pp>\d+)", re.IGNORECASE)
+PP_ARG_RE = re.compile(r"--pipeline-parallel-size(?:=|\s+)(\d+)", re.IGNORECASE)
 EVAL_CONC_RE = re.compile(r"(?:^|[_-])c(?P<conc>\d+)(?:$|[_-])", re.IGNORECASE)
 EVAL_TOPOLOGY_RE = re.compile(
     r"(?:^|[_-])(?P<topology>\d+p\d+d(?:[_-]dpa)?)(?:$|[_-])",
@@ -106,6 +109,12 @@ def round_or_none(*values: Any, digits: int = 4) -> float | None:
     return round(parsed, digits) if parsed is not None else None
 
 
+def divide_by_total_gpu(value: float | None, total_gpu: int | None) -> float | None:
+    if value and total_gpu:
+        return value / total_gpu
+    return None
+
+
 def interactivity_value(payload: dict[str, Any]) -> float | None:
     # An already-resolved value wins: apply_p90_e2e_interactivity() writes the
     # p90 e2e normalized number here, and without this branch perf_point() would
@@ -202,7 +211,6 @@ def topology_resources(
         )
     )
     topology = TOPOLOGY_RE.search(text)
-    tp = TP_RE.search(text)
     prefill_workers = int_value(
         payload.get("prefill_workers"), payload.get("num_prefill_workers")
     )
@@ -219,14 +227,36 @@ def topology_resources(
     decode_tp = int_value(
         payload.get("decode_tp"), payload.get("decode_tensor_parallel_size")
     )
-    if tp:
-        prefill_tp = prefill_tp or int(tp.group("tp"))
-        decode_tp = decode_tp or int(tp.group("tp"))
+    dual_tp = DUAL_TP_RE.search(text)
+    if dual_tp:
+        prefill_tp = prefill_tp or int(dual_tp.group("prefill_tp"))
+        decode_tp = decode_tp or int(dual_tp.group("decode_tp"))
+    else:
+        tp = TP_RE.search(text)
+        if tp:
+            tp_size = int(tp.group("tp"))
+            prefill_tp = prefill_tp or tp_size
+            decode_tp = decode_tp or tp_size
+
+    prefill_pp = int_value(
+        payload.get("prefill_pp"), payload.get("prefill_pipeline_parallel_size")
+    )
+    if prefill_pp is None:
+        cpp_pp = CPP_PP_RE.search(text)
+        if cpp_pp:
+            prefill_pp = int(cpp_pp.group("pp"))
+    if prefill_pp is None:
+        for key in ("prefill_extra_server_args",):
+            pp_match = PP_ARG_RE.search(string_value(payload.get(key)))
+            if pp_match:
+                prefill_pp = int(pp_match.group(1))
+                break
+    prefill_pp = prefill_pp or 1
 
     num_prefill_gpu = int_value(payload.get("num_prefill_gpu"))
     num_decode_gpu = int_value(payload.get("num_decode_gpu"))
     if num_prefill_gpu is None and prefill_workers and prefill_tp:
-        num_prefill_gpu = prefill_workers * prefill_tp
+        num_prefill_gpu = prefill_workers * prefill_tp * prefill_pp
     if num_decode_gpu is None and decode_workers and decode_tp:
         num_decode_gpu = decode_workers * decode_tp
     total_gpu = int_value(payload.get("total_gpu"))
@@ -239,6 +269,7 @@ def topology_resources(
         "decode_workers": decode_workers,
         "prefill_tp": prefill_tp,
         "decode_tp": decode_tp,
+        "prefill_pp": prefill_pp,
         "num_prefill_gpu": num_prefill_gpu,
         "num_decode_gpu": num_decode_gpu,
         "total_gpu": total_gpu,
@@ -339,6 +370,9 @@ def enrich_payload(
     enriched.setdefault("decode_workers", env.get("DECODE_WORKERS"))
     enriched.setdefault("prefill_tp", env.get("PREFILL_TP"))
     enriched.setdefault("decode_tp", env.get("DECODE_TP"))
+    enriched.setdefault(
+        "prefill_extra_server_args", env.get("PREFILL_EXTRA_SERVER_ARGS")
+    )
     runner = env.get("SLURM_SUBMIT_RUNNER", "")
     if hardware:
         enriched["hardware"] = hardware
@@ -373,28 +407,17 @@ def enrich_payload(
     enriched.setdefault("interactivity", interactivity_value(enriched))
     resources = topology_resources(enriched, fields)
     total_gpu = resources["total_gpu"]
-    num_prefill_gpu = resources["num_prefill_gpu"]
-    num_decode_gpu = resources["num_decode_gpu"]
     input_tput = number(enriched.get("input_throughput"))
     output_tput = number(enriched.get("output_throughput"))
     total_tput = number(
         enriched.get("total_token_throughput"), enriched.get("total_throughput")
     )
+    enriched.setdefault("tput_per_gpu", divide_by_total_gpu(total_tput, total_gpu))
     enriched.setdefault(
-        "tput_per_gpu", total_tput / total_gpu if total_tput and total_gpu else None
+        "input_tput_per_gpu", divide_by_total_gpu(input_tput, total_gpu)
     )
     enriched.setdefault(
-        "input_tput_per_gpu",
-        input_tput / num_prefill_gpu if input_tput and num_prefill_gpu else None,
-    )
-    output_tput_denominator = num_decode_gpu or total_gpu
-    enriched.setdefault(
-        "output_tput_per_gpu",
-        (
-            output_tput / output_tput_denominator
-            if output_tput and output_tput_denominator
-            else None
-        ),
+        "output_tput_per_gpu", divide_by_total_gpu(output_tput, total_gpu)
     )
     return enriched
 
@@ -519,18 +542,10 @@ def perf_point(
         "cache_hit_rate": round_or_none(payload.get("cache_hit_rate")),
         "cache_hit_tokens": int_value(payload.get("cache_hit_tokens")),
         "cache_total_tokens": int_value(payload.get("cache_total_tokens")),
-        "tput_per_gpu": round_or_none(
-            total_tput / total_gpu if total_tput and total_gpu else None
-        ),
-        "input_tput_per_gpu": round_or_none(
-            input_tput / resources["num_prefill_gpu"]
-            if input_tput and resources["num_prefill_gpu"]
-            else None
-        ),
+        "tput_per_gpu": round_or_none(divide_by_total_gpu(total_tput, total_gpu)),
+        "input_tput_per_gpu": round_or_none(divide_by_total_gpu(input_tput, total_gpu)),
         "output_tput_per_gpu": round_or_none(
-            output_tput / (resources["num_decode_gpu"] or total_gpu)
-            if output_tput and (resources["num_decode_gpu"] or total_gpu)
-            else None
+            divide_by_total_gpu(output_tput, total_gpu)
         ),
         "run_url": run_url or "",
         "image": string_value(payload.get("docker_image"), payload.get("image")),