diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b46c4c5901..33960c9331 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -249,7 +249,7 @@ jobs: --model 'claude-fable-5' --max-turns 8 --allowedTools "Read,Glob,Grep,Bash(git diff:*)" - --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' + --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","sgl-router","vllm-router","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik3","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' prompt: | Inspect this Actions run's change range. @@ -264,7 +264,7 @@ jobs: - eval-only: evaluation without throughput measurement - fp4: FP4 precision - mtp, eagle, eagle3: speculative decoding method - - sglang, vllm, dynamo-vllm: runtime framework + - sglang, vllm, sgl-router, vllm-router, dynamo-vllm: runtime framework - model criterion: matching configured model family - checklist-complete: PR checklist is satisfied - patchwork: modified upstream engine or runtime source diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..5e7c1a3ce8 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -32,7 +32,14 @@ install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="http://localhost:${PORT}/metrics" + local frontend_metrics_url="" + + # Dynamo exposes its own active-request gauge. Native static routers do + # not promise a metrics endpoint, so their drain signal comes entirely + # from the explicit logical worker endpoints supplied by srt-slurm. + case "$FRAMEWORK" in + dynamo-*) frontend_metrics_url="http://localhost:${PORT}/metrics" ;; + esac "$AIPERF_PYTHON" - \ "$timeout_seconds" \ @@ -70,13 +77,19 @@ def metric_sum(metrics: str, name: str) -> float: while time.monotonic() < deadline: try: - frontend_metrics = fetch_metrics(frontend_url) - frontend_active = metric_sum(frontend_metrics, "dynamo_frontend_active_requests") + frontend_active = 0.0 + if frontend_url: + frontend_metrics = fetch_metrics(frontend_url) + frontend_active = metric_sum( + frontend_metrics, "dynamo_frontend_active_requests" + ) worker_active = 0.0 for worker_url in worker_urls: worker_metrics = fetch_metrics(worker_url) worker_active += metric_sum(worker_metrics, "vllm:num_requests_running") worker_active += metric_sum(worker_metrics, "vllm:num_requests_waiting") + worker_active += metric_sum(worker_metrics, "sglang:num_requests_running") + worker_active += metric_sum(worker_metrics, "sglang:num_requests_waiting") print( f"Agentic drain status: frontend_active={frontend_active:g} " f"worker_running_or_waiting={worker_active:g}", diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-deps.sh new file mode 100644 index 0000000000..a9cccf9c1c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-deps.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +python3 -m pip install --no-cache-dir --upgrade "sglang-router==0.3.2" +python3 -c 'import sglang_router; print(sglang_router.__file__)' diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-pd-fix-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-pd-fix-deps.sh new file mode 100644 index 0000000000..be44533a07 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/native-sgl-router-pd-fix-deps.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Temporary exact-source pin for the upstream SGLang Model Gateway one-token +# P/D response fix. The shared wheel cache and flock make the source build +# happen once even though srt-slurm runs the recipe setup on every role. +SGLANG_REPOSITORY="https://github.com/cquil11/sglang.git" +SGLANG_REF="fix/pd-one-token-prefill-response" +SGLANG_COMMIT="9072e0dc5aaf1962b4e2c6f1a51094356b5d3324" +WHEEL_CACHE_ROOT="/router_wheels/sglang-router-${SGLANG_COMMIT}" +LOCK_FILE="${WHEEL_CACHE_ROOT}.lock" + +mkdir -p "${WHEEL_CACHE_ROOT}" +exec 9>"${LOCK_FILE}" +flock -w 3600 9 + +shopt -s nullglob +router_wheels=("${WHEEL_CACHE_ROOT}"/sglang_router-*.whl) +if (( ${#router_wheels[@]} == 0 )); then + build_dir=$(mktemp -d /tmp/sglang-router-build.XXXXXX) + trap 'rm -rf "${build_dir}"' EXIT + + git clone --filter=blob:none --no-checkout --single-branch \ + --branch "${SGLANG_REF}" "${SGLANG_REPOSITORY}" "${build_dir}" + git -C "${build_dir}" sparse-checkout init --cone + git -C "${build_dir}" sparse-checkout set sgl-model-gateway + git -C "${build_dir}" checkout --detach "${SGLANG_COMMIT}" + if [[ "$(git -C "${build_dir}" rev-parse HEAD)" != "${SGLANG_COMMIT}" ]]; then + echo "ERROR: SGLang checkout does not match ${SGLANG_COMMIT}" >&2 + exit 1 + fi + + python3 -m pip install --break-system-packages --no-cache-dir --upgrade \ + "maturin>=1.0,<2.0" + ( + cd "${build_dir}/sgl-model-gateway/bindings/python" + CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-16}" \ + maturin build --release --out "${WHEEL_CACHE_ROOT}" + ) + router_wheels=("${WHEEL_CACHE_ROOT}"/sglang_router-*.whl) +fi + +if (( ${#router_wheels[@]} != 1 )); then + echo "ERROR: expected exactly one cached SGLang Router wheel, found ${#router_wheels[@]}" >&2 + exit 1 +fi + +# Install the released package first to resolve its Python dependencies, then +# replace only the native package with the wheel built from the exact fix SHA. +python3 -m pip install --break-system-packages --no-cache-dir --upgrade \ + "sglang-router==0.3.2" +python3 -m pip install --break-system-packages --no-cache-dir \ + --force-reinstall --no-deps "${router_wheels[0]}" +python3 -c 'import sglang_router; print(sglang_router.__file__)' +echo "Installed SGLang Router from ${SGLANG_COMMIT}" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/native-vllm-router-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/native-vllm-router-deps.sh new file mode 100644 index 0000000000..286ebc7fb8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/native-vllm-router-deps.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +python3 -m pip install --no-cache-dir --upgrade "vllm-router==0.1.15" +command -v vllm-router >/dev/null +python3 - <<'PY' +from importlib.metadata import version + +assert version("vllm-router") == "0.1.15" +print(f"vllm-router {version('vllm-router')}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b200-2xdep8-mtp-sgl-router-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b200-2xdep8-mtp-sgl-router-agentic.yaml new file mode 100644 index 0000000000..efe272db5f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-b200-2xdep8-mtp-sgl-router-agentic.yaml @@ -0,0 +1,96 @@ +name: "dsv4-b200-sgl-router-agg-2xdep8-mtp-agentic" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:v0.5.16-cu130" + precision: "fp4" + +setup_script: native-sgl-router-deps.sh + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 2 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + +frontend: + type: sglang + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + +backend: + type: sglang + aggregated_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "high" + SGLANG_SIMULATE_ACC_LEN: "2.99" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + NCCL_CUMEM_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + sglang_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + tool-call-parser: "deepseekv4" + stream-interval: 10 + watchdog-timeout: 1800 + mem-fraction-static: 0.90 + chunked-prefill-size: 8192 + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + max-running-requests: 256 + cuda-graph-max-bs: 256 + scheduler-recv-interval: 30 + dp-size: 8 + tp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + speculative-algorithm: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp-sgl-router.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp-sgl-router.yaml new file mode 100644 index 0000000000..e602939b4f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp-sgl-router.yaml @@ -0,0 +1,152 @@ +name: "glm5.2-h200-sgl-router-disagg-2p2d-mtp-agentic" + +# Native SGL Router validation of the established GLM-5.2 H200 Mooncake +# topology. This deliberately changes only the request plane: model, worker +# layout, KV transport, MTP, and AgentX behavior match the Dynamo baseline. + +model: + path: "hf:zai-org/GLM-5.2-FP8" + container: "lmsysorg/sglang:v0.5.16-cu130" + precision: "fp8" + +identity: + model: + repo: "zai-org/GLM-5.2-FP8" + revision: "ba978f7d347eaf65d22f1a86833408afdb953541" + container: + image: "lmsysorg/sglang:v0.5.16-cu130" + frameworks: + sglang: "0.5.16" + +setup_script: native-sgl-router-pd-fix-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: h200 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + +frontend: + type: sglang + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + +backend: + type: sglang + prefill_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + decode_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + SGLANG_SIMULATE_ACC_LEN: "2.99" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + + sglang_config: + prefill: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + ep-size: 1 + attn-cp-size: 8 + enable-prefill-cp: true + cp-strategy: interleave + enable-dsa-cache-layer-split: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + kv-cache-dtype: fp8_e4m3 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + max-running-requests: 32 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + + decode: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + dp-size: 8 + ep-size: 1 + enable-dp-attention: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + kv-cache-dtype: fp8_e4m3 + dsa-decode-backend: flashmla_kv + context-length: 1048576 + max-total-tokens: 1048576 + mem-fraction-static: 0.85 + max-running-requests: 200 + page-size: 64 + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: + mem: "0" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-b200-2xdep8-mtp-vllm-router-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-b200-2xdep8-mtp-vllm-router-agentic.yaml new file mode 100644 index 0000000000..0bb0f9ba8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-b200-2xdep8-mtp-vllm-router-agentic.yaml @@ -0,0 +1,100 @@ +name: "dsv4-b200-vllm-router-agg-2xdep8-mtp-agentic" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" + precision: "fp4" + +setup_script: native-vllm-router-deps.sh + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 2 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + # The released B200 DSV4 DEP path uses PYNCCL for local DP/EP collectives. + # NCCL symmetric-memory window registration fails during dummy profiling on + # this cluster, before the router can receive traffic. + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,rc" + NCCL_P2P_LEVEL: "NVL" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-prompt-tokens-details: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.99}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-b200-1p1d-dep8-mtp-vllm-router-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-b200-1p1d-dep8-mtp-vllm-router-agentic.yaml new file mode 100644 index 0000000000..1ef9ff90fb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-b200-1p1d-dep8-mtp-vllm-router-agentic.yaml @@ -0,0 +1,105 @@ +name: "dsv4-b200-vllm-router-disagg-1p1d-dep8-mtp-agentic" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec" + precision: "fp4" + +setup_script: native-vllm-router-deps.sh + +slurm: + time_limit: "4:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + +frontend: + type: vllm-router + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + prefill_environment: &vllm_environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + # Match the known-green B200 DSV4 DEP communicator path. NCCL symmetric + # memory fails while registering its local DP/EP communication window. + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_TLS: "cuda_copy,rc" + NCCL_P2P_LEVEL: "NVL" + decode_environment: *vllm_environment + vllm_config: + prefill: &vllm_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-prompt-tokens-details: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.99}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + decode: *vllm_config + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index 34565e2810..11628c1b49 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -18,6 +18,9 @@ adjustments: eagle: 0.75 eagle3: 0.75 framework-prefix: + # Match the explicit router runtimes before their broader engine prefixes. + sgl-router: 0.5 + vllm-router: 0.5 sglang: 0.5 vllm: 0.5 dynamo-sglang: 0.5 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 393babf91b..0a8670bc8d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -966,6 +966,97 @@ dsv4-fp4-b200-vllm-agentic-mtp: # DEP Mooncake + MTP (num_speculative_tokens=3) - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, spec-decoding: mtp, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } +dsv4-fp4-b200-vllm-router-agentic-agg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm-router + router: { name: vllm-router, version: "0.1.15" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [16] + prefill: + num-worker: 2 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git" + - "SRT_SLURM_REF=ab98030f0993bdb1595acc25e883b59d76ad7791" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-b200-2xdep8-mtp-vllm-router-agentic.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 8 + dp-attn: true + +dsv4-fp4-b200-vllm-router-agentic-disagg: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm-router + router: { name: vllm-router, version: "0.1.15" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [16] + prefill: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git" + - "SRT_SLURM_REF=ab98030f0993bdb1595acc25e883b59d76ad7791" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-b200-1p1d-dep8-mtp-vllm-router-agentic.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + +dsv4-fp4-b200-sgl-router-agentic-agg: + image: lmsysorg/sglang:v0.5.16-cu130 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: sgl-router + router: { name: sgl-router, version: "0.3.2" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [16] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git" + - "SRT_SLURM_REF=ab98030f0993bdb1595acc25e883b59d76ad7791" + - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-b200-2xdep8-mtp-sgl-router-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 8 + dp-attn: true + dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro @@ -7676,6 +7767,44 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: ep: 1 dp-attn: true +# Request-plane sibling of the established Dynamo deployment above. The H200 +# hosts provide the peer-memory support required by the proven Mooncake data +# plane; this keeps native SGL Router validation independent of B200's missing +# GPUDirect registration module. +glm5.2-fp8-h200-sgl-router-agentic-mtp-2p2d: + image: lmsysorg/sglang:v0.5.16-cu130 + model: zai-org/GLM-5.2-FP8 + model-prefix: glm5.2 + runner: cluster:h200-dgxc + precision: fp8 + framework: sgl-router + router: { name: sgl-router, version: "0.3.2" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [8] + kv-offloading: none + prefill: + num-worker: 2 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git" + - "SRT_SLURM_REF=ab98030f0993bdb1595acc25e883b59d76ad7791" + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp-sgl-router.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + # GLM-5.2 B300 NVFP4 AgentX with EAGLE/MTP speculative decoding, following the # AgentX speculative-decoding policy in MODELS.md. SGLang EAGLE runs off # GLM-5.2's built-in nextn head (num-steps 3, @@ -7877,7 +8006,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 # framework stays dynamo-vllm for launcher routing, but this variant serves - # DIRECTLY with vllm serve (srt-slurm PR #278 frontend.type: vllm + the + # DIRECTLY with vllm serve (srt-slurm PR #278, frontend.type: vllm, plus the # InferenceX multinode patch) — no dynamo frontend/worker/router involved. framework: dynamo-vllm multinode: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 471981f60c..fa1879f5cd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5709,3 +5709,18 @@ - "Enable SGLang metrics on every aggregate, prefill, and decode engine." - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 + +- config-keys: + - dsv4-fp4-b200-vllm-router-agentic-agg + - dsv4-fp4-b200-vllm-router-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Validate first-class vLLM Router 0.1.15 through srt-slurm on B200 without a Dynamo request plane." + - "Exercise vLLM Router with two-worker aggregate DEP8 and one-prefill/one-decode DEP8 DeepSeek V4 FP4 deployments on B200. Both validations use MTP and the AgentX trace workload." + - "Pin the test deployment to exact SemiAnalysisAI/srt-slurm PR head ab98030f0993bdb1595acc25e883b59d76ad7791, preserve each logical backend Prometheus endpoint for AIPerf, and require the vllm: metric family before accepting results." + - "Enable vLLM prompt-token details on both native Router deployments so AgentX captures response-level prompt-cache usage in addition to each logical backend's Prometheus metrics." + - "Verify the terminal Slurm state and exit code after log streaming so a failed benchmark cannot be reported as a successful GitHub job merely because its log tail ended cleanly." + - "Classify sgl-router and vllm-router as explicit runtime criteria in the CI scheduler, matching the router-specific framework keys before their broader engine prefixes." + - "Drain sequential native-router AgentX points from all explicit logical worker metrics without requiring a Dynamo frontend gauge; recognize both vllm:num_requests_* and sglang:num_requests_* worker families." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2543 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 778eac1e51..ffad35ede4 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -4,6 +4,8 @@ SLURM_PARTITION="${SLURM_PARTITION:-gpu-2}" SLURM_ACCOUNT="${SLURM_ACCOUNT:-benchmark}" +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + set -x # MODEL_PATH: Override with pre-downloaded paths on cluster-accessible storage. @@ -135,14 +137,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # Validate framework - if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" && $FRAMEWORK != "dynamo-vllm" ]]; then - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm" + if [[ $FRAMEWORK != "dynamo-sglang" && $FRAMEWORK != "dynamo-trt" && $FRAMEWORK != "dynamo-vllm" && $FRAMEWORK != "sgl-router" && $FRAMEWORK != "vllm-router" ]]; then + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm, sgl-router, vllm-router" exit 1 fi - # Multinode dsv4 currently only ships with the dynamo-vllm recipe - if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK != "dynamo-vllm" ]]; then - echo "Unsupported framework for multinode dsv4: $FRAMEWORK (only dynamo-vllm)" + # Multinode dsv4 currently ships with Dynamo vLLM and the two native + # static-router integrations under test in this PR. + if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK != "dynamo-vllm" && $FRAMEWORK != "sgl-router" && $FRAMEWORK != "vllm-router" ]]; then + echo "Unsupported framework for multinode dsv4: $FRAMEWORK" exit 1 fi @@ -155,10 +158,29 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi + if [[ -n "${SRT_SLURM_REPOSITORY:-}" || -n "${SRT_SLURM_REF:-}" ]]; then + if [[ -z "${SRT_SLURM_REPOSITORY:-}" || -z "${SRT_SLURM_REF:-}" ]]; then + echo "SRT_SLURM_REPOSITORY and SRT_SLURM_REF must be set together" >&2 + exit 1 + fi + git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout --detach "$SRT_SLURM_REF" || exit 1 + if [[ "$(git rev-parse HEAD)" != "$SRT_SLURM_REF" ]]; then + echo "srt-slurm checkout does not match requested exact commit: $SRT_SLURM_REF" >&2 + exit 1 + fi + mkdir -p recipes/vllm/deepseek-v4 recipes/sglang/deepseek-v4 configs || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" \ + recipes/vllm/deepseek-v4 || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" \ + recipes/sglang/deepseek-v4 || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs" \ + configs || exit 1 # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then + elif [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 - # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm + # (kylliang/direct-aggregate-vllm, frontend.type: vllm) adds direct `vllm # serve` owns the OpenAI port itself, no Dynamo layer. The fork branch # carries PR #278 plus the multi-node extension (vLLM-native # --master-addr/--nnodes/--node-rank serve + headless non-leader @@ -322,6 +344,8 @@ containers: dynamo-trtllm: "${SQUASH_FILE}" dynamo-sglang: "${SQUASH_FILE}" dynamo-vllm: "${SQUASH_FILE}" + sgl-router: "${SQUASH_FILE}" + vllm-router: "${SQUASH_FILE}" sglang-v0.5.11-cu130: "${SQUASH_FILE}" "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" @@ -407,6 +431,12 @@ EOF wait $POLL_PID + # Native router validation is all-or-nothing. Other B200 workflows retain + # the launcher's historical result-collection behavior. + if [[ "$FRAMEWORK" == "sgl-router" || "$FRAMEWORK" == "vllm-router" ]]; then + wait_for_slurm_job_success "$JOB_ID" || exit 1 + fi + set -x echo "Job $JOB_ID completed!" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 8991e04182..4f353562cc 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -21,7 +21,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # MODEL_PATH: Override with pre-downloaded paths on H200 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H200 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + if [[ $FRAMEWORK == "dynamo-sglang" || $FRAMEWORK == "sgl-router" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" @@ -32,7 +32,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi export SRT_SLURM_MODEL_PREFIX="glm5.2-fp8" else - echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" + echo "Unsupported model prefix/precision for SGLang: $MODEL_PREFIX/$PRECISION" exit 1 fi elif [[ $FRAMEWORK == "dynamo-trt" ]]; then @@ -44,16 +44,16 @@ if [[ "$IS_MULTINODE" == "true" ]]; then echo "Unsupported model prefix/precision for dynamo-trt: $MODEL_PREFIX/$PRECISION" exit 1 fi - elif [[ $FRAMEWORK == "vllm" ]]; then + elif [[ $FRAMEWORK == "vllm" || $FRAMEWORK == "vllm-router" ]]; then if [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/models/gharunners/hf-hub-cache/Kimi-K3" export SRT_SLURM_MODEL_PREFIX="kimik3" else - echo "Unsupported model prefix/precision for vllm: $MODEL_PREFIX/$PRECISION" + echo "Unsupported model prefix/precision for vLLM: $MODEL_PREFIX/$PRECISION" exit 1 fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, vllm" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, vllm, sgl-router, vllm-router" exit 1 fi @@ -64,7 +64,21 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - if [[ $IS_AGENTIC == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then + if [[ -n "${SRT_SLURM_REPOSITORY:-}" || -n "${SRT_SLURM_REF:-}" ]]; then + if [[ -z "${SRT_SLURM_REPOSITORY:-}" || -z "${SRT_SLURM_REF:-}" ]]; then + echo "SRT_SLURM_REPOSITORY and SRT_SLURM_REF must be set together" >&2 + exit 1 + fi + git clone "$SRT_SLURM_REPOSITORY" "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout --detach "$SRT_SLURM_REF" + if [[ "$(git rev-parse HEAD)" != "$SRT_SLURM_REF" ]]; then + echo "srt-slurm checkout does not match requested exact commit: $SRT_SLURM_REF" >&2 + exit 1 + fi + mkdir -p configs + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs" configs + elif [[ $IS_AGENTIC == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then # v1.0.44 includes the AgentX custom benchmark integration and passes # every logical SGLang worker's Prometheus URL to AIPerf. git clone --branch v1.0.44 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" @@ -103,7 +117,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Map container images to local squash files based on framework NGINX_SQUASH_FILE="/data/containers/nginx+1.27.4.sqsh" - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + if [[ $FRAMEWORK == "dynamo-sglang" || $FRAMEWORK == "sgl-router" ]]; then # SGLang container mapping if [[ $MODEL_PREFIX == "glm5.2" ]]; then SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" @@ -115,7 +129,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" - elif [[ $FRAMEWORK == "vllm" ]]; then + elif [[ $FRAMEWORK == "vllm" || $FRAMEWORK == "vllm-router" ]]; then CONTAINER_KEY="$IMAGE" SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" fi @@ -150,10 +164,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache" HF_HUB_CACHE_HOST_PATH="/models/gharunners/hf-hub-cache" + ROUTER_WHEEL_CACHE_HOST_PATH="/home/sa-shared/gharunners/router-wheels" mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" + mkdir -p "$ROUTER_WHEEL_CACHE_HOST_PATH" DEFAULT_MOUNTS_BLOCK="default_mounts: ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache - ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache + ${ROUTER_WHEEL_CACHE_HOST_PATH}: /router_wheels" fi echo "Creating srtslurm.yaml configuration..." cat > srtslurm.yaml </dev/null || true; exit "$rc"' EXIT INT TERM HUP stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + # Native router validation is all-or-nothing. Keep the shared log helper's + # legacy behavior for unrelated launchers that intentionally retain + # partial results from timed-out allocations. + if [[ "$FRAMEWORK" == "sgl-router" || "$FRAMEWORK" == "vllm-router" ]]; then + wait_for_slurm_job_success "$JOB_ID" || exit 1 + fi set -x diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 60ce818eb5..3a0f876f8f 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -18,6 +18,49 @@ slurm_job_is_active() { squeue -j "$job_id" --noheader 2>/dev/null | grep -q "$job_id" } +wait_for_slurm_job_success() { + local job_id="$1" + local attempts="${SLURM_ACCOUNTING_ATTEMPTS:-24}" + local interval="${SLURM_ACCOUNTING_INTERVAL_SECONDS:-5}" + local attempt record state exit_code + + if ! command -v sacct >/dev/null 2>&1; then + echo "ERROR: sacct is required to verify Slurm job $job_id" >&2 + return 1 + fi + + for ((attempt = 1; attempt <= attempts; attempt++)); do + record=$(sacct -X -n -P -j "$job_id" -o JobIDRaw,State,ExitCode 2>/dev/null \ + | awk -F'|' -v job_id="$job_id" '$1 == job_id { print $2 "|" $3; exit }') + if [[ -n "$record" ]]; then + state="${record%%|*}" + exit_code="${record#*|}" + # Slurm may suffix terminal states with "+" when the display is + # truncated; compare the canonical state name. + state="${state%% *}" + state="${state%%+}" + + if [[ "$state" == "COMPLETED" && "$exit_code" == "0:0" ]]; then + echo "Slurm job $job_id completed successfully ($state, exit $exit_code)" + return 0 + fi + + case "$state" in + BOOT_FAIL|CANCELLED|DEADLINE|FAILED|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|REVOKED|SPECIAL_EXIT|TIMEOUT) + echo "ERROR: Slurm job $job_id ended in $state with exit $exit_code" >&2 + return 1 + ;; + esac + fi + + sleep "$interval" + done + + echo "ERROR: Slurm accounting did not report a successful terminal state for job $job_id" >&2 + [[ -z "${record:-}" ]] || echo "Last accounting record: $record" >&2 + return 1 +} + stream_slurm_job_log() { local job_id="$1" local log_file="$2" diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index c027cc9b2f..a76cd262f8 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,6 +1,8 @@ import subprocess from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" @@ -15,6 +17,42 @@ def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str ) +def write_executable(path: Path, body: str) -> None: + path.write_text(f"#!/usr/bin/env bash\n{body}\n") + path.chmod(0o755) + + +@pytest.mark.parametrize( + ("accounting_record", "expected_returncode", "expected_message"), + [ + ("4242|COMPLETED|0:0", 0, "completed successfully"), + ("4242|FAILED|1:0", 1, "ended in FAILED with exit 1:0"), + ], +) +def test_wait_for_slurm_job_success_reports_accounting_status( + tmp_path: Path, + accounting_record: str, + expected_returncode: int, + expected_message: str, +) -> None: + mock_bin = tmp_path / "bin" + mock_bin.mkdir() + write_executable(mock_bin / "sacct", 'printf \'%s\\n\' "$MOCK_SACCT_RECORD"') + + result = run_bash( + 'export PATH="$3:$PATH" MOCK_SACCT_RECORD="$4" ' + "SLURM_ACCOUNTING_ATTEMPTS=1 SLURM_ACCOUNTING_INTERVAL_SECONDS=0; " + 'source "$1"; wait_for_slurm_job_success 4242', + SLURM_UTILS, + tmp_path, + mock_bin, + accounting_record, + ) + + assert result.returncode == expected_returncode + assert expected_message in result.stdout + result.stderr + + def test_copy_agentic_results_stages_only_matching_points(tmp_path: Path) -> None: source = tmp_path / "source" workspace = tmp_path / "workspace" diff --git a/utils/agentic/aggregation/backends/sglang.py b/utils/agentic/aggregation/backends/sglang.py index acbbc12b02..0e7f8d6e18 100644 --- a/utils/agentic/aggregation/backends/sglang.py +++ b/utils/agentic/aggregation/backends/sglang.py @@ -27,7 +27,7 @@ class SglangBackend(ServerMetricsBackend): def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: metric_names = set(metrics) return any(name.startswith("sglang:") for name in metric_names) or ( - not metrics and framework.lower() == "sglang" + not metrics and framework.lower() in {"sglang", "sgl-router"} ) def populate( diff --git a/utils/agentic/aggregation/backends/vllm.py b/utils/agentic/aggregation/backends/vllm.py index 18ed30ece8..a39735c3ed 100644 --- a/utils/agentic/aggregation/backends/vllm.py +++ b/utils/agentic/aggregation/backends/vllm.py @@ -27,7 +27,7 @@ class VllmBackend(ServerMetricsBackend): def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: metric_names = set(metrics) return any(name.startswith("vllm:") for name in metric_names) or ( - not metrics and framework.lower() == "vllm" + not metrics and framework.lower() in {"vllm", "vllm-router"} ) def populate(