diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml new file mode 100644 index 000000000..87979921f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml @@ -0,0 +1,118 @@ +# Agentic-coding SGLang aggregated recipe for GLM-5.2-NVFP4 on GB300 +# (single aggregated worker, TP8, EAGLE MTP + hierarchical-cache KV). +# +# Flat single-variant schema the agentic CI flow expects: applied via +# CONFIG_FILE= with no zip_override selector, so it must NOT be +# base:-wrapped. Concurrency is not a recipe field; the GHA matrix fans out +# one job per concurrency from the master-config conc-list into agentic_srt.sh. +name: gb300-fp4-glm5.2-agentx-agg +model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 +frontend: + type: dynamo + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 +dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true +backend: + type: sglang + sglang_config: + aggregated: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 10 + cuda-graph-max-bs: 10 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + context-length: 1048576 + speculative-algorithm: EAGLE + speculative-num-steps: 4 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 5 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_trtllm + disable-shared-experts-fusion: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + enable-metrics: true + enable-cache-report: true + aggregated_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_SIMULATE_ACC_LEN: '3.33' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + PIP_BREAK_SYSTEM_PACKAGES: '1' + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none +health_check: + max_attempts: 1440 + interval_seconds: 10 +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml new file mode 100644 index 000000000..b04ea1752 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml @@ -0,0 +1,219 @@ +base: + name: gb300-fp4-glm5.2-agentx + model: + path: glm-5.2-fp4 + container: dynamo-sglang + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + router-session-affinity-ttl-secs: 3600 + dynamo: + hash: 71eb001e17fa73c742f0afe1a6ed96836cb135fd + install: true + backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1' + SGLANG_HICACHE_DEBUG_LOG: '1' + SGLANG_HICACHE_DEBUG_SAMPLE_RATE: '16384' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_ENABLE_THINKING: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + SGLANG_REASONING_EFFORT: max + PIP_BREAK_SYSTEM_PACKAGES: '1' + UCX_TLS: ib,cuda_copy,cuda_ipc,sm,self + NVSHMEM_REMOTE_TRANSPORT: none + SGLANG_DG_CACHE_DIR: /deepgemm_cache + FLASHINFER_WORKSPACE_BASE: /flashinfer_cache + SGLANG_SIMULATE_ACC_LEN: '2.5' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + prefill: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + max-running-requests: 16 + cuda-graph-max-bs: 16 + disable-cuda-graph: true + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_cutlass + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.8 + context-length: 1048576 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-size: 135 + hicache-io-backend: direct + speculative-algorithm: EAGLE + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 2 + enable-metrics: true + enable-cache-report: true + decode: + served-model-name: nvidia/GLM-5.2-NVFP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + max-running-requests: 16 + cuda-graph-max-bs: 16 + chunked-prefill-size: 64 + context-length: 1048576 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + skip-tokenizer-init: true + stream-interval: 30 + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + mem-fraction-static: 0.9 + disaggregation-decode-extra-slots: 0 + enable-metrics: true + enable-cache-report: true + health_check: + max_attempts: 1440 + interval_seconds: 10 + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 64 + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' +# ################# agentx ################# +zip_override_mtp_agentx_hightpt: + name: [agentx-2p1d_dep16-c128, agentx-2p1d_dep16-c192] + backend: + sglang_config: + decode: + cuda-graph-max-bs: 144 + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + max-running-requests: 144 + mem-fraction-static: 0.85 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + prefill: + data-parallel-size: 8 + expert-parallel-size: 8 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + resources: + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + gpus_per_prefill: 8 + prefill_nodes: 4 + prefill_workers: 2 +zip_override_mtp_agentx_lowlat: + name: [agentx-1p2d_tp4-c48, agentx-1p4d_tp4-c48, agentx-1p6d_tp4-c45] + resources: + decode_nodes: [2, 4, 6] + decode_workers: [2, 4, 6] + gpus_per_decode: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a60792bd6..62ba9cff3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7955,8 +7955,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: # eagle-topk 1, 4 draft tokens = 3 speculative tokens), with acceptance pinned # to the golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, # K=3) through SGLANG_SIMULATE_ACC_*. Image is bumped to v0.5.16-cu130: it is -# the first release that reads SGLANG_SIMULATE_ACC_TOKEN_MODE, and it is the -# tag the GLM-5.2 GB300 dynamo-sglang agentic recipes already run. +# the first release that reads SGLANG_SIMULATE_ACC_TOKEN_MODE. # # Cookbook low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) only, on the AgentX # MTP concurrency grid [1, 4, 8, 12, 16]: steps of at least 2 because @@ -7989,7 +7988,7 @@ glm5.2-fp4-b300-sglang-agentic-mtp: # acceptance pinned to the golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, # thinking_on, K=3) through SGLANG_SIMULATE_ACC_*. Image lmsysorg/sglang:v0.5.16-cu130 # is the first release that reads SGLANG_SIMULATE_ACC_TOKEN_MODE and is what the -# B300 sibling and the GLM-5.2 GB300 dynamo-sglang agentic recipes run. +# B300 sibling runs. # # Same single arm and concurrency grid as glm5.2-fp4-b300-sglang-agentic-mtp so # the two SKUs are directly comparable: cookbook low-latency TP8 with HiCache @@ -8010,6 +8009,140 @@ glm5.2-fp4-b200-sglang-agentic-mtp: search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } +glm5.2-fp4-gb300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # aggregated 1-worker TP8 (prefill+decode on the same GPUs) + - spec-decoding: mtp + conc-list: [2, 4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx-agg.yaml" + # Aggregated worker serves prefill and decode on the same GPUs. + # Keep decode at zero to avoid double-counting the TP8 allocation. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "71eb001e17fa73c742f0afe1a6ed96836cb135fd" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # low-latency 1P2D TP4-prefill / TP4-decode (2 decode workers) + - spec-decoding: mtp + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[0]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P4D TP4-prefill / TP4-decode (4 decode workers) + - spec-decoding: mtp + conc-list: [48] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[1]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # low-latency 1P6D TP4-prefill / TP4-decode (6 decode workers) + - spec-decoding: mtp + conc-list: [45] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_lowlat[2]" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # high-throughput 2P1D DEP8-prefill / DEP16-decode + - spec-decoding: mtp + conc-list: [128] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [192] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/gb300-fp4/agentic/glm5.2-agentx.yaml:zip_override_mtp_agentx_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp8-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 889d4db48..da3f8d6dd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5972,6 +5972,17 @@ - "Use KV-aware Dynamo routing with 4-hour correlation-ID affinity, authoritative vLLM KV events, KV-cache token metrics, and every logical vLLM server-metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 +- config-keys: + - glm5.2-fp4-gb300-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2-NVFP4 GB300 AgentX (agentic-coding) Dynamo-SGLang benchmarks over a 7-point sweep: aggregated single-worker TP8 (conc 2/4), disaggregated low-latency TP4-prefill/TP4-decode 1P2D (conc 48), 1P4D (conc 48), 1P6D (conc 45), and high-throughput 2P1D DEP8-prefill/DEP16-decode (conc 128/192)" + - "EAGLE speculative decoding, hierarchical KV cache (hicache-size 135), MTP, decode mem-fraction-static 0.9, nixl KV transfer with UCX_TLS=ib, and X-Dynamo-Session-ID session affinity; driven from the NVIDIA/srt-slurm base+zip_override recipes via CONFIG_FILE selectors" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642; runner: gb300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2608 + - config-keys: - minimaxm3-fp4-b300-vllm-agentic-mtp scenario-type: diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 54d208bc7..83252461d 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -65,6 +65,9 @@ elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then # in our GLM-5.1 sglang recipes. export MODEL_PATH=/scratch/models/GLM-5.1-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then + export MODEL_PATH=/scratch/models/GLM-5.2-NVFP4 + export SRT_SLURM_MODEL_PREFIX="glm-5.2-fp4" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/GLM-5-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" @@ -94,7 +97,7 @@ elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH=/scratch/models/Qwen3.5-397B-A17B-FP8 export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" else - echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" + echo "Unsupported model: $MODEL_PREFIX-$PRECISION. Supported models are: dsr1-fp4, dsr1-fp8, dsv4-fp4, glm5-fp4, glm5-fp8, glm5.2-fp4, minimaxm2.5-fp4, minimaxm2.5-fp8, kimik2.5-fp4, qwen3.5-fp4, qwen3.5-fp8" exit 1 fi @@ -221,6 +224,16 @@ elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX = mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then + # GLM-5.2 GB300 sglang AgentX: srt-slurm main has the agentx-mvp scenario, + # the zip_override sweep selectors, and the multi-frontend session-affinity + # schema these recipes need. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout main + mkdir -p recipes/sglang/glm5.2/gb300-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/gb300-fp4" \ + recipes/sglang/glm5.2/gb300-fp4 elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic recipes use NVIDIA/srt-slurm v1.0.36. This is the upstream # version validated in InferenceX PR #2302 and includes per-node DP,