From a472c847209f5b8e2082a1544e645d3ba5956ca9 Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 7 Aug 2026 12:11:55 -0700
Subject: [PATCH 1/7] [NV] Add SpeedBench AL collectors for DSV4-Pro DSpark on
 B300

Adds the acceptance-length collectors for deepseek-ai/DeepSeek-V4-Pro-DSpark,
following the Kimi-K3 DSpark collector (#2352). DSpark ships the draft inside a
separate checkpoint, so unlike K3 there is no external draft head to download
and no "model" key in the speculative-config.

dsv4dspark_fp4_b300_vllm.sh is a copy of the DSV4 MTP collector with only the
speculative-config changed (method dspark + draft_sample_method); every serve
flag is left identical so the DSpark and MTP curves stay comparable, and it
already matches the published DSpark recipe.

Benjamin asked for greedy and probabilistic drafting to be characterized on
DSV4-Pro. speedbench-al.yml resolves the collector purely as
${model-prefix}_fp4_b300_vllm.sh, so the second arm needs its own dispatchable
file; dsv4dsparkprob_fp4_b300_vllm.sh delegates to the base collector with
DRAFT_SAMPLE_METHOD=probabilistic rather than duplicating it, so the two arms
cannot drift. rejection_sample_method is left at the vLLM default and exposed as
an opt-in env knob: the K3 variant bundled it with probabilistic drafting, but
the forced-AL config has to stay on a sampling method TRT-LLM also supports.

The golden YAML and its README row follow in a separate PR once the collection
run is reviewed.
---
 .../speedbench/dsv4dspark_fp4_b300_vllm.sh    | 423 ++++++++++++++++++
 .../dsv4dsparkprob_fp4_b300_vllm.sh           |  19 +
 runners/launch_b300-nv.sh                     |   1 +
 3 files changed, 443 insertions(+)
 create mode 100755 benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
 create mode 100755 benchmarks/single_node/speedbench/dsv4dsparkprob_fp4_b300_vllm.sh

diff --git a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
new file mode 100755
index 000000000..b5f7f0032
--- /dev/null
+++ b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
@@ -0,0 +1,423 @@
+#!/usr/bin/env bash
+
+# DSV4-Pro B300 vLLM SPEED-Bench AL matrix collector for DSpark speculative
+# decoding.
+#
+# Produces the golden acceptance-length (AL) reference matrix consumed by the
+# synthetic-acceptance framework: for each thinking mode (on/off) and each
+# DSpark speculative-token count, measure the REAL AL on a single SPEED-Bench
+# category (default: coding) and emit a YAML matrix identical in shape to the
+# other golden_al_distribution curves.
+#
+# DSpark is DeepSeek's own speculative-decoding scheme and ships as a separate
+# checkpoint (deepseek-ai/DeepSeek-V4-Pro-DSpark, 960 GB) with the draft baked
+# in — unlike the Kimi-K3 DSpark collector there is no external draft head to
+# download and no "model" key in the speculative-config.
+#
+# Differences vs the DSV4 MTP collector (dsv4_fp4_b300_vllm.sh), which this is
+# otherwise a copy of so the two AL curves stay directly comparable:
+#   - target model       DeepSeek-V4-Pro-DSpark (was DeepSeek-V4-Pro)
+#   - speculative-config method dspark + draft_sample_method (was method mtp)
+# Every serve flag is left byte-identical to the MTP collector: it already
+# matches the published DSpark recipe, and holding the serving config fixed is
+# what makes "DSpark vs MTP on DSV4-Pro" a like-for-like AL comparison.
+#
+# Usage (inside the vLLM container, on a B300 node):
+#   export MODEL=deepseek-ai/DeepSeek-V4-Pro-DSpark
+#   bash benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
+#
+# Tunables (env):
+#   MTP_LIST          space-separated DSpark spec-token counts (default "1 2 3 4 5 6 7 8")
+#   THINKING_MODES    space-separated: off|on       (default "off on")
+#   CATEGORY          SPEED-Bench category          (default coding)
+#   SPEEDBENCH_OUTPUT_LEN  per-request output len   (default 4096)
+#   OUT_YAML          output matrix path            (default $RESULTS_DIR/speedbench-reference-al.yaml)
+#   DRAFT_SAMPLE_METHOD    greedy|probabilistic     (default greedy)
+#   REJECTION_SAMPLE_METHOD  passed through to speculative-config when non-empty
+#                            (default: unset, i.e. the vLLM default)
+
+set -uo pipefail
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+MODEL="${MODEL:?MODEL env var required (e.g. deepseek-ai/DeepSeek-V4-Pro-DSpark)}"
+# Serve from the local weights dir resolved by the launcher (MODEL_PATH points
+# at the pre-staged copy, e.g. /scratch/models/DeepSeek-V4-Pro-DSpark). Falls
+# back to MODEL for a standalone local run where MODEL is itself a path.
+SERVE_MODEL="${MODEL_PATH:-$MODEL}"
+TP="${TP:-8}"
+DP_ATTENTION="${DP_ATTENTION:-false}"
+EP_SIZE="${EP_SIZE:-1}"
+PORT="${PORT:-8888}"
+
+MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}"
+THINKING_MODES="${THINKING_MODES:-off on}"
+CATEGORY="${CATEGORY:-coding}"
+# Top-level key in the emitted YAML matrix. Derived from the model by the
+# workflow (e.g. deepseek-v4-pro-dspark); falls back to the model basename,
+# lowercased.
+MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}"
+SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}"
+# Kept at 1 to match the DSV4 MTP collector that produced the existing
+# deepseek-v4-pro golden curve; the DSpark numbers are only meaningful against
+# MTP if both were measured the same way. Raise it if the 8h allocation is the
+# binding constraint (AL itself is concurrency-independent).
+CONCURRENCY="${CONCURRENCY:-1}"
+TEMPERATURE="${TEMPERATURE:-1.0}"
+# thinking-on chat_template_kwargs. MUST match the production/golden config:
+# the reference matrix (golden_al_distribution/dsv4_mtp.yaml) was measured with
+# reasoning_effort=high.
+DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"thinking": true, "reasoning_effort": "high"}'
+CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}"
+# The greedy/probabilistic knob Benjamin asked to characterize on DSV4-Pro. The
+# published recipe uses greedy; probabilistic won at every level on Kimi-K3
+# (golden_al_distribution/kimik3_dspark*.yaml). vLLM accepts exactly these two
+# values (vllm/config/speculative.py: DraftSampleMethod).
+DRAFT_SAMPLE_METHOD="${DRAFT_SAMPLE_METHOD:-greedy}"
+case "$DRAFT_SAMPLE_METHOD" in
+    greedy|probabilistic) ;;
+    *)
+        echo "CRITICAL: DRAFT_SAMPLE_METHOD must be 'greedy' or 'probabilistic' (got '$DRAFT_SAMPLE_METHOD')"
+        exit 1
+        ;;
+esac
+# Left unset by default. The K3 probabilistic variant also flipped this to
+# "block", but that bundles two variables into one measurement and the forced-AL
+# config has to stay on a sampling method TRT-LLM supports too, so it is opt-in
+# here rather than tied to draft_sample_method.
+REJECTION_SAMPLE_METHOD="${REJECTION_SAMPLE_METHOD:-}"
+
+SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}"
+RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}"
+OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}"
+
+export VLLM_ENGINE_READY_TIMEOUT_S=3600
+
+mkdir -p "$RESULTS_DIR"
+nvidia-smi
+
+# ---- Resolve target weights ----
+# DeepSeek-V4-Pro-DSpark is in the launcher's STAGED_MODELS, so MODEL_PATH
+# resolves to the read-only staged mount and this block is a no-op. It still
+# covers a standalone run where the weights are not staged.
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        if [[ ! -w "$(dirname "$MODEL_PATH")" ]]; then
+            echo "CRITICAL: $MODEL_PATH is empty and $(dirname "$MODEL_PATH") is not writable."
+            echo "The DSpark checkpoint is not staged on this node. Ask for it to be staged,"
+            echo "or drop DeepSeek-V4-Pro-DSpark from STAGED_MODELS so MODEL_PATH resolves to"
+            echo "the writable models dir and the download below can run."
+            exit 1
+        fi
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi
+fi
+
+# ---- Download SPEED-Bench dataset ----
+echo "=== Downloading SPEED-Bench dataset ==="
+pip install -q datasets tiktoken
+curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \
+  | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR"
+
+if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then
+    echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found"
+    exit 1
+fi
+
+# ---- Temporary shim: add a real --chat-template-kwargs CLI option ----
+# Upstream gap (until vllm-project/vllm#44244 lands): speed_bench/CustomDataset
+# pre-renders the chat template client-side WITHOUT chat_template_kwargs and
+# posts to /v1/completions, so thinking mode cannot be enabled via --extra-body
+# or --default-chat-template-kwargs. This wires a proper --chat-template-kwargs
+# option through get_samples into CustomDataset.sample's apply_chat_template.
+# TODO: delete this whole block once #44244 is released in the benchmark image;
+# the patch is idempotent (marker check) so it is safe to leave until then.
+apply_chat_template_kwargs_shim() {
+    echo "=== Patching vLLM benchmark to add --chat-template-kwargs (temporary shim) ==="
+    python3 - <<'PYEOF'
+import vllm.benchmarks.serve as S
+import vllm.benchmarks.datasets.datasets as D
+
+def patch(mod, edits, marker):
+    f = mod.__file__
+    src = open(f).read()
+    if marker in src:
+        print("already patched:", f)
+        return
+    for old, new in edits:
+        n = src.count(old)
+        assert n == 1, f"anchor matched {n} times in {f}, aborting:\n{old[:80]}..."
+        src = src.replace(old, new, 1)
+    open(f, "w").write(src)
+    print("patched OK ->", f)
+
+# Edit 1: serve.py -- declare the --chat-template-kwargs argument before --extra-body
+serve_old = '''    parser.add_argument(
+        "--extra-body",'''
+serve_new = '''    parser.add_argument(
+        "--chat-template-kwargs",
+        type=json.loads,
+        default=None,
+        help="JSON dict forwarded to apply_chat_template during "
+        "client-side prompt rendering, e.g. to enable reasoning mode.",
+    )
+    parser.add_argument(
+        "--extra-body",'''
+patch(S, [(serve_old, serve_new)], marker='"--chat-template-kwargs"')
+
+# Edit 2: datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call
+disp_old = '''                output_len=args.speed_bench_output_len,
+                enable_multimodal_chat=args.enable_multimodal_chat,'''
+disp_new = '''                output_len=args.speed_bench_output_len,
+                chat_template_kwargs=args.chat_template_kwargs,
+                enable_multimodal_chat=args.enable_multimodal_chat,'''
+
+# Edit 3: datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call
+samp_old = '''                # apply template
+                if not skip_chat_template:
+                    prompt = tokenizer.apply_chat_template(
+                        [{"role": "user", "content": prompt}],
+                        add_generation_prompt=True,
+                        tokenize=False,
+                    )
+
+                prompt_len = len(tokenizer(prompt).input_ids)'''
+samp_new = '''                # apply template
+                if not skip_chat_template:
+                    _ctk = kwargs.get("chat_template_kwargs") or {}
+                    prompt = tokenizer.apply_chat_template(
+                        [{"role": "user", "content": prompt}],
+                        add_generation_prompt=True,
+                        tokenize=False,
+                        **_ctk,
+                    )
+
+                prompt_len = len(tokenizer(prompt).input_ids)'''
+patch(D, [(disp_old, disp_new), (samp_old, samp_new)],
+      marker="chat_template_kwargs=args.chat_template_kwargs")
+PYEOF
+}
+
+# Apply the shim once if any thinking-on cell is requested.
+if [[ " $THINKING_MODES " == *" on "* ]]; then
+    if ! apply_chat_template_kwargs_shim; then
+        echo "CRITICAL: --chat-template-kwargs shim failed — aborting"
+        exit 1
+    fi
+fi
+
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
+if [ "${DP_ATTENTION}" = "true" ]; then
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
+fi
+EP_ARGS=()
+if [ "${EP_SIZE:-1}" -gt 1 ]; then
+    EP_ARGS=(--enable-expert-parallel)
+fi
+MOE_ARGS=()
+if [ "${DP_ATTENTION}" = "true" ]; then
+    MOE_ARGS=(--moe-backend deep_gemm_mega_moe)
+fi
+
+# Optional extra speculative-config keys, rendered once so run_cell only has to
+# interpolate num_speculative_tokens.
+SPEC_EXTRA=""
+if [[ -n "$REJECTION_SAMPLE_METHOD" ]]; then
+    SPEC_EXTRA=", \"rejection_sample_method\": \"$REJECTION_SAMPLE_METHOD\""
+fi
+
+fetch_metric() {
+    local port="$1" name="$2"
+    curl -s "http://localhost:${port}/metrics" \
+      | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0"
+}
+
+SERVER_PID=""
+# List all descendant PIDs of $1 recursively, matched by PARENT pid. This can
+# never include this script (the script is an ancestor of the server, not a
+# descendant), so it avoids the self-kill a name-based `pkill -f vllm` caused
+# (the script filename contains "vllm").
+_descendants() {
+    local pid="$1" child
+    for child in $(pgrep -P "$pid" 2>/dev/null || true); do
+        echo "$child"
+        _descendants "$child"
+    done
+}
+cleanup_server() {
+    if [[ -n "$SERVER_PID" ]]; then
+        # Snapshot the server's worker/EngineCore subprocesses BEFORE killing the
+        # parent: once the parent dies the children reparent to init and the tree
+        # link is lost. Killing the captured PIDs guarantees no orphaned worker
+        # survives to hold GPU memory and OOM the next server start.
+        local descendants
+        descendants=$(_descendants "$SERVER_PID")
+        kill "$SERVER_PID" 2>/dev/null || true
+        wait "$SERVER_PID" 2>/dev/null || true
+        local pid
+        for pid in $descendants; do
+            kill -9 "$pid" 2>/dev/null || true
+        done
+        # Wait for GPU memory to actually free before the next server starts.
+        local waited=0
+        while [[ $waited -lt 120 ]]; do
+            local used
+            used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1)
+            if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi
+            sleep 3; waited=$((waited + 3))
+        done
+        SERVER_PID=""
+    fi
+}
+trap 'cleanup_server' EXIT
+
+start_gpu_monitor
+
+# Per-cell AL is collected into associative arrays keyed by "mode_mtp".
+declare -A AL_RESULT
+
+run_cell() {
+    local mode="$1" mtp="$2"
+    local think_args=()
+    if [[ "$mode" == "on" ]]; then
+        think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON")
+    fi
+
+    echo ""
+    echo "=========================================="
+    echo "  Cell: thinking=$mode  DSPARK=$mtp  category=$CATEGORY"
+    echo "  draft_sample_method=$DRAFT_SAMPLE_METHOD"
+    echo "=========================================="
+
+    local serve_args=(
+        --host 0.0.0.0 --port "$PORT"
+        "${PARALLEL_ARGS[@]}"
+        --pipeline-parallel-size 1
+        --kv-cache-dtype fp8
+        --trust-remote-code
+        --block-size 256
+        --no-enable-prefix-caching
+        "${EP_ARGS[@]}"
+        "${MOE_ARGS[@]}"
+        --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+        --attention_config.use_fp4_indexer_cache True
+        --tokenizer-mode deepseek_v4
+        --tool-call-parser deepseek_v4
+        --enable-auto-tool-choice
+        --reasoning-parser deepseek_v4
+        --max-cudagraph-capture-size 2048
+        --max-model-len 16384
+        --speculative-config "{\"method\": \"dspark\", \"num_speculative_tokens\": $mtp, \"draft_sample_method\": \"$DRAFT_SAMPLE_METHOD\"$SPEC_EXTRA}"
+    )
+
+    local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log"
+    vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 &
+    SERVER_PID=$!
+
+    if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then
+        echo "  -> server failed to start (thinking=$mode dspark=$mtp), recording N/A"
+        AL_RESULT["${mode}_${mtp}"]="N/A"
+        cleanup_server
+        return
+    fi
+
+    local acc_before drf_before acc_after drf_after
+    acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total")
+    drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total")
+
+    vllm bench serve \
+        --model "$SERVE_MODEL" \
+        --port "$PORT" \
+        --dataset-name speed_bench \
+        --dataset-path "$SPEEDBENCH_DIR" \
+        --speed-bench-category "$CATEGORY" \
+        --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \
+        --num-prompts -1 \
+        --max-concurrency "$CONCURRENCY" \
+        --save-result \
+        --save-detailed \
+        --result-dir "$RESULTS_DIR" \
+        --result-filename "speedbench_${mode}_mtp${mtp}" \
+        --trust-remote-code \
+        --tokenizer-mode deepseek_v4 \
+        --temperature "$TEMPERATURE" \
+        "${think_args[@]}"
+
+    acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total")
+    drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total")
+
+    local delta_acc delta_drf al
+    delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}")
+    delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}")
+    if [[ "$delta_drf" -gt 0 ]]; then
+        al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}")
+    else
+        al="N/A"
+    fi
+    echo "  -> thinking=$mode DSPARK=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)"
+    AL_RESULT["${mode}_${mtp}"]="$al"
+
+    cleanup_server
+}
+
+for mode in $THINKING_MODES; do
+    for mtp in $MTP_LIST; do
+        run_cell "$mode" "$mtp"
+    done
+done
+
+stop_gpu_monitor
+
+# ---- Emit the YAML matrix ----
+emit_mode_block() {
+    local mode="$1"
+    for mtp in $MTP_LIST; do
+        echo "    $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}"
+    done
+}
+
+SPEC_SUMMARY="method=dspark | draft_sample_method=$DRAFT_SAMPLE_METHOD"
+if [[ -n "$REJECTION_SAMPLE_METHOD" ]]; then
+    SPEC_SUMMARY="$SPEC_SUMMARY | rejection_sample_method=$REJECTION_SAMPLE_METHOD"
+fi
+
+{
+    echo "# Acceptance Length (AL) reference values measured with SPEED-Bench."
+    echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | output_len: $SPEEDBENCH_OUTPUT_LEN"
+    echo "# thinking_on chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON"
+    echo "# speculative-config: $SPEC_SUMMARY"
+    echo "# Measured on $MODEL_KEY (B300, vLLM DSpark), per num_speculative_tokens."
+    echo "# Auto-generated by benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh (speedbench-al.yml)."
+    echo "#"
+    echo "# key = num_speculative_tokens (DSpark level); value = golden AL"
+    echo "${MODEL_KEY}:"
+    if [[ " $THINKING_MODES " == *" on "* ]]; then
+        echo "  thinking_on:"
+        emit_mode_block on
+    fi
+    if [[ " $THINKING_MODES " == *" off "* ]]; then
+        echo "  thinking_off:"
+        emit_mode_block off
+    fi
+} > "$OUT_YAML"
+
+echo ""
+echo "=========================================="
+echo "  SPEED-Bench AL matrix written to: $OUT_YAML"
+echo "=========================================="
+cat "$OUT_YAML"
+
+# A matrix where every cell is N/A is a failed collection, not a result: fail the
+# job so it is not mistaken for a curve worth reviewing.
+MEASURED=0
+for mode in $THINKING_MODES; do
+    for mtp in $MTP_LIST; do
+        [[ "${AL_RESULT[${mode}_${mtp}]:-N/A}" != "N/A" ]] && MEASURED=$((MEASURED + 1))
+    done
+done
+if [[ "$MEASURED" -eq 0 ]]; then
+    echo "CRITICAL: no cell produced an AL value — see the server logs and the"
+    echo "benchmark client output above."
+    exit 1
+fi
diff --git a/benchmarks/single_node/speedbench/dsv4dsparkprob_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dsparkprob_fp4_b300_vllm.sh
new file mode 100755
index 000000000..0a4c0ad5f
--- /dev/null
+++ b/benchmarks/single_node/speedbench/dsv4dsparkprob_fp4_b300_vllm.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Probabilistic-drafting arm of the DSV4-Pro DSpark AL collection.
+#
+# Identical to dsv4dspark_fp4_b300_vllm.sh in every respect except
+# draft_sample_method, which becomes "probabilistic" instead of the published
+# recipe's "greedy". Running both arms is what answers the open question on
+# DSV4-Pro (probabilistic beat greedy at every level on Kimi-K3, see
+# golden_al_distribution/kimik3_dspark*.yaml, but that has not been shown here).
+#
+# This exists as a separate file only because speedbench-al.yml resolves the
+# collector purely as ${model-prefix}_fp4_b300_vllm.sh, so a second dispatchable
+# entry point is the only way to launch the second arm. It delegates instead of
+# duplicating the collector so the two arms cannot drift apart.
+#
+# Dispatch with model-prefix=dsv4dsparkprob.
+
+exec env DRAFT_SAMPLE_METHOD=probabilistic \
+    bash "$(dirname "$0")/dsv4dspark_fp4_b300_vllm.sh" "$@"
diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh
index f39616801..414286d58 100644
--- a/runners/launch_b300-nv.sh
+++ b/runners/launch_b300-nv.sh
@@ -383,6 +383,7 @@ else
         DeepSeek-R1-0528-NVFP4-v2
         DeepSeek-V4-Flash
         DeepSeek-V4-Pro
+        DeepSeek-V4-Pro-DSpark
         GLM-5-FP8
         GLM-5-NVFP4
         GLM-5.1

From eb9c2c23328cf12a5990738687c22b502a8b3364 Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 7 Aug 2026 12:31:38 -0700
Subject: [PATCH 2/7] Download the DSpark checkpoint instead of expecting it
 staged; batch the pass
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The first collection run failed the preflight in 50s: DeepSeek-V4-Pro-DSpark is
not staged at /scratch/models on the B300 cluster, so listing it in the
launcher's STAGED_MODELS resolved MODEL_PATH to a read-only mount that does not
hold the weights. Drop it from the list so MODEL_PATH resolves to the writable
models dir and the collector's existing download block pulls the ~960 GB
checkpoint once, on the first run. Once the weights are staged, adding the
basename back is a one-line change that switches to the faster read-only mount.

Also raise CONCURRENCY from 1 to 32. AL is a per-draft accept/reject property
and is independent of batch size, and nothing here sets
speculative_disable_by_batch_size, so drafting stays on and the curve stays
comparable to the MTP one while the pass takes far less wall-clock — which
matters more now that the first run also pays for the download.
---
 .../speedbench/dsv4dspark_fp4_b300_vllm.sh    | 31 +++++++++++--------
 runners/launch_b300-nv.sh                     |  1 -
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
index b5f7f0032..e4e92bf21 100755
--- a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
@@ -41,8 +41,9 @@ source "$(dirname "$0")/../../benchmark_lib.sh"
 
 MODEL="${MODEL:?MODEL env var required (e.g. deepseek-ai/DeepSeek-V4-Pro-DSpark)}"
 # Serve from the local weights dir resolved by the launcher (MODEL_PATH points
-# at the pre-staged copy, e.g. /scratch/models/DeepSeek-V4-Pro-DSpark). Falls
-# back to MODEL for a standalone local run where MODEL is itself a path.
+# at the writable models dir, e.g. /data/models/DeepSeek-V4-Pro-DSpark, until the
+# checkpoint is staged; see the download block below). Falls back to MODEL for a
+# standalone local run where MODEL is itself a path.
 SERVE_MODEL="${MODEL_PATH:-$MODEL}"
 TP="${TP:-8}"
 DP_ATTENTION="${DP_ATTENTION:-false}"
@@ -57,11 +58,12 @@ CATEGORY="${CATEGORY:-coding}"
 # lowercased.
 MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}"
 SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}"
-# Kept at 1 to match the DSV4 MTP collector that produced the existing
-# deepseek-v4-pro golden curve; the DSpark numbers are only meaningful against
-# MTP if both were measured the same way. Raise it if the 8h allocation is the
-# binding constraint (AL itself is concurrency-independent).
-CONCURRENCY="${CONCURRENCY:-1}"
+# AL is a per-draft accept/reject property and is independent of batch size, so
+# the SPEED-Bench pass is batched to cut wall-clock. Note this differs from the
+# DSV4 MTP collector, which measured the existing deepseek-v4-pro curve at 1;
+# nothing here sets speculative_disable_by_batch_size, so drafting stays on at
+# this batch size and the curves remain comparable.
+CONCURRENCY="${CONCURRENCY:-32}"
 TEMPERATURE="${TEMPERATURE:-1.0}"
 # thinking-on chat_template_kwargs. MUST match the production/golden config:
 # the reference matrix (golden_al_distribution/dsv4_mtp.yaml) was measured with
@@ -96,18 +98,21 @@ mkdir -p "$RESULTS_DIR"
 nvidia-smi
 
 # ---- Resolve target weights ----
-# DeepSeek-V4-Pro-DSpark is in the launcher's STAGED_MODELS, so MODEL_PATH
-# resolves to the read-only staged mount and this block is a no-op. It still
-# covers a standalone run where the weights are not staged.
+# The DSpark checkpoint is NOT in the launcher's STAGED_MODELS (it is not staged
+# on the B300 cluster yet), so MODEL_PATH resolves to the writable models dir and
+# the ~960 GB download below runs once, on the first collection. Add the basename
+# back to STAGED_MODELS once the weights are staged to read them from the faster
+# read-only mount instead.
 if [[ -n "${MODEL_PATH:-}" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         if [[ ! -w "$(dirname "$MODEL_PATH")" ]]; then
             echo "CRITICAL: $MODEL_PATH is empty and $(dirname "$MODEL_PATH") is not writable."
-            echo "The DSpark checkpoint is not staged on this node. Ask for it to be staged,"
-            echo "or drop DeepSeek-V4-Pro-DSpark from STAGED_MODELS so MODEL_PATH resolves to"
-            echo "the writable models dir and the download below can run."
+            echo "This means the basename is listed in the launcher's STAGED_MODELS but the"
+            echo "weights were never staged. Either get them staged, or remove it from"
+            echo "STAGED_MODELS so MODEL_PATH resolves to the writable models dir instead."
             exit 1
         fi
+        echo "=== $MODEL_PATH is empty; downloading $MODEL (~960 GB, first run only) ==="
         hf download "$MODEL" --local-dir "$MODEL_PATH"
     fi
 else
diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh
index 414286d58..f39616801 100644
--- a/runners/launch_b300-nv.sh
+++ b/runners/launch_b300-nv.sh
@@ -383,7 +383,6 @@ else
         DeepSeek-R1-0528-NVFP4-v2
         DeepSeek-V4-Flash
         DeepSeek-V4-Pro
-        DeepSeek-V4-Pro-DSpark
         GLM-5-FP8
         GLM-5-NVFP4
         GLM-5.1

From b60e95d56bdc04c2b2d16aa2b05b0be1f68bf0c5 Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 7 Aug 2026 13:48:28 -0700
Subject: [PATCH 3/7] Probe for native --chat-template-kwargs instead of always
 patching

The collection run aborted in the shim: on v0.25.1 serve.py already declares
--chat-template-kwargs natively (vllm-project/vllm#44244 has landed), so the
marker check short-circuited there, but datasets.py no longer matches the
pre-#44244 anchor and the assertion killed the run before the first server start.

Replace the shim with the Kimi-K3 collector's version, which probes all three
pieces of native support (CLI option, speed_bench forward, CustomDataset unpack)
and no-ops when they are present, patching only what an older image is missing.
Verified against the current vLLM benchmark source: all three probes match, so
the shim reports native support and returns without touching anything.

Note dsv4_fp4_b300_vllm.sh still carries the old unconditional shim and would hit
the same assertion if the MTP curve were ever re-collected on a v0.25.x image.
---
 .../speedbench/dsv4dspark_fp4_b300_vllm.sh    | 100 +++++++++++-------
 1 file changed, 61 insertions(+), 39 deletions(-)

diff --git a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
index e4e92bf21..97d52a8c9 100755
--- a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
@@ -130,37 +130,58 @@ if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then
     exit 1
 fi
 
-# ---- Temporary shim: add a real --chat-template-kwargs CLI option ----
-# Upstream gap (until vllm-project/vllm#44244 lands): speed_bench/CustomDataset
-# pre-renders the chat template client-side WITHOUT chat_template_kwargs and
-# posts to /v1/completions, so thinking mode cannot be enabled via --extra-body
-# or --default-chat-template-kwargs. This wires a proper --chat-template-kwargs
-# option through get_samples into CustomDataset.sample's apply_chat_template.
-# TODO: delete this whole block once #44244 is released in the benchmark image;
-# the patch is idempotent (marker check) so it is safe to leave until then.
+# ---- Conditional shim: ensure --chat-template-kwargs reaches the chat template ----
+# speed_bench/CustomDataset pre-renders the chat template client-side and posts to
+# /v1/completions, so thinking mode cannot be enabled via --extra-body or
+# --default-chat-template-kwargs — the kwargs must reach apply_chat_template.
+# vllm-project/vllm#44244 added this natively (serve.py declares the CLI option,
+# the speed_bench dispatch forwards it, CustomDataset.sample unpacks it), and it
+# is present in the v0.25.x images this collector runs on. So: probe for native
+# support first and no-op when present; only patch the pieces an older image is
+# missing. Idempotent. Same shim as the Kimi-K3 collector.
 apply_chat_template_kwargs_shim() {
-    echo "=== Patching vLLM benchmark to add --chat-template-kwargs (temporary shim) ==="
+    echo "=== Checking vLLM benchmark --chat-template-kwargs support ==="
     python3 - <<'PYEOF'
+import sys
 import vllm.benchmarks.serve as S
 import vllm.benchmarks.datasets.datasets as D
 
-def patch(mod, edits, marker):
-    f = mod.__file__
-    src = open(f).read()
-    if marker in src:
-        print("already patched:", f)
-        return
+def read(mod):
+    with open(mod.__file__) as fh:
+        return fh.read()
+
+s_src, d_src = read(S), read(D)
+
+# Native (post-#44244) spellings, plus the ones this shim itself writes.
+have_cli = '"--chat-template-kwargs"' in s_src
+have_forward = ('chat_template_kwargs=getattr(args' in d_src
+                or 'chat_template_kwargs=args.chat_template_kwargs' in d_src)
+have_unpack = ('**(chat_template_kwargs or {})' in d_src
+               or '**_ctk' in d_src)
+
+if have_cli and have_forward and have_unpack:
+    print("native --chat-template-kwargs support present; no patching needed")
+    sys.exit(0)
+
+print(f"patching (cli={have_cli} forward={have_forward} unpack={have_unpack})")
+
+def apply(mod, src, edits):
     for old, new in edits:
         n = src.count(old)
-        assert n == 1, f"anchor matched {n} times in {f}, aborting:\n{old[:80]}..."
+        assert n == 1, (
+            f"anchor matched {n} times in {mod.__file__}, aborting. This image's "
+            f"benchmark source differs from both the pre-#44244 and post-#44244 "
+            f"layouts; update the shim.\n{old[:120]}..."
+        )
         src = src.replace(old, new, 1)
-    open(f, "w").write(src)
-    print("patched OK ->", f)
-
-# Edit 1: serve.py -- declare the --chat-template-kwargs argument before --extra-body
-serve_old = '''    parser.add_argument(
-        "--extra-body",'''
-serve_new = '''    parser.add_argument(
+    with open(mod.__file__, "w") as fh:
+        fh.write(src)
+    print("patched OK ->", mod.__file__)
+
+if not have_cli:
+    # serve.py -- declare the --chat-template-kwargs argument before --extra-body
+    apply(S, s_src, [('''    parser.add_argument(
+        "--extra-body",''', '''    parser.add_argument(
         "--chat-template-kwargs",
         type=json.loads,
         default=None,
@@ -168,18 +189,19 @@ serve_new = '''    parser.add_argument(
         "client-side prompt rendering, e.g. to enable reasoning mode.",
     )
     parser.add_argument(
-        "--extra-body",'''
-patch(S, [(serve_old, serve_new)], marker='"--chat-template-kwargs"')
-
-# Edit 2: datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call
-disp_old = '''                output_len=args.speed_bench_output_len,
-                enable_multimodal_chat=args.enable_multimodal_chat,'''
-disp_new = '''                output_len=args.speed_bench_output_len,
+        "--extra-body",''')])
+
+d_edits = []
+if not have_forward:
+    # datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call
+    d_edits.append(('''                output_len=args.speed_bench_output_len,
+                enable_multimodal_chat=args.enable_multimodal_chat,''',
+                    '''                output_len=args.speed_bench_output_len,
                 chat_template_kwargs=args.chat_template_kwargs,
-                enable_multimodal_chat=args.enable_multimodal_chat,'''
-
-# Edit 3: datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call
-samp_old = '''                # apply template
+                enable_multimodal_chat=args.enable_multimodal_chat,'''))
+if not have_unpack:
+    # datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call
+    d_edits.append(('''                # apply template
                 if not skip_chat_template:
                     prompt = tokenizer.apply_chat_template(
                         [{"role": "user", "content": prompt}],
@@ -187,8 +209,8 @@ samp_old = '''                # apply template
                         tokenize=False,
                     )
 
-                prompt_len = len(tokenizer(prompt).input_ids)'''
-samp_new = '''                # apply template
+                prompt_len = len(tokenizer(prompt).input_ids)''',
+                    '''                # apply template
                 if not skip_chat_template:
                     _ctk = kwargs.get("chat_template_kwargs") or {}
                     prompt = tokenizer.apply_chat_template(
@@ -198,9 +220,9 @@ samp_new = '''                # apply template
                         **_ctk,
                     )
 
-                prompt_len = len(tokenizer(prompt).input_ids)'''
-patch(D, [(disp_old, disp_new), (samp_old, samp_new)],
-      marker="chat_template_kwargs=args.chat_template_kwargs")
+                prompt_len = len(tokenizer(prompt).input_ids)'''))
+if d_edits:
+    apply(D, d_src, d_edits)
 PYEOF
 }
 

From b6cca89b707ec6c49061e0a41dd5e358a01c7dcd Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 7 Aug 2026 14:05:33 -0700
Subject: [PATCH 4/7] Assert native --chat-template-kwargs support instead of
 patching for it

vllm-project/vllm#44244 is in the images this collector runs on, so the
site-packages monkey-patching the older collectors carry is dead code. Drop it.

Keep a preflight assertion rather than assuming, because one direction fails
silently: if the CLI option exists but the speed_bench path does not forward it,
the flag is accepted and ignored, every thinking_on prompt renders without
thinking, and the cell writes a non-thinking AL under the thinking_on key. A
missing CLI option would instead fail loudly at argument parsing. The assertion
turns the silent case into an abort before the first server start.

Verified against the current vLLM benchmark source: all three pieces are present,
so the preflight passes and the run proceeds.
---
 .../speedbench/dsv4dspark_fp4_b300_vllm.sh    | 115 +++++-------------
 1 file changed, 33 insertions(+), 82 deletions(-)

diff --git a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
index 97d52a8c9..d696e1444 100755
--- a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
@@ -130,16 +130,20 @@ if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then
     exit 1
 fi
 
-# ---- Conditional shim: ensure --chat-template-kwargs reaches the chat template ----
+# ---- Preflight: --chat-template-kwargs must reach the chat template ----
 # speed_bench/CustomDataset pre-renders the chat template client-side and posts to
 # /v1/completions, so thinking mode cannot be enabled via --extra-body or
-# --default-chat-template-kwargs — the kwargs must reach apply_chat_template.
-# vllm-project/vllm#44244 added this natively (serve.py declares the CLI option,
-# the speed_bench dispatch forwards it, CustomDataset.sample unpacks it), and it
-# is present in the v0.25.x images this collector runs on. So: probe for native
-# support first and no-op when present; only patch the pieces an older image is
-# missing. Idempotent. Same shim as the Kimi-K3 collector.
-apply_chat_template_kwargs_shim() {
+# --default-chat-template-kwargs — the kwargs have to reach apply_chat_template.
+# vllm-project/vllm#44244 made that native and the images this collector runs on
+# carry it, so there is nothing to patch (the older collectors monkey-patched
+# site-packages here; that is what this replaces).
+#
+# It is still worth asserting rather than assuming, because the failure is silent
+# in one direction: if the CLI option exists but the speed_bench path does not
+# forward it, the flag is accepted and ignored, every thinking_on prompt renders
+# without thinking, and the cell reports a non-thinking AL under the thinking_on
+# key. A missing CLI option, by contrast, would fail loudly at argument parsing.
+assert_chat_template_kwargs_support() {
     echo "=== Checking vLLM benchmark --chat-template-kwargs support ==="
     python3 - <<'PYEOF'
 import sys
@@ -152,84 +156,31 @@ def read(mod):
 
 s_src, d_src = read(S), read(D)
 
-# Native (post-#44244) spellings, plus the ones this shim itself writes.
-have_cli = '"--chat-template-kwargs"' in s_src
-have_forward = ('chat_template_kwargs=getattr(args' in d_src
-                or 'chat_template_kwargs=args.chat_template_kwargs' in d_src)
-have_unpack = ('**(chat_template_kwargs or {})' in d_src
-               or '**_ctk' in d_src)
-
-if have_cli and have_forward and have_unpack:
-    print("native --chat-template-kwargs support present; no patching needed")
-    sys.exit(0)
-
-print(f"patching (cli={have_cli} forward={have_forward} unpack={have_unpack})")
-
-def apply(mod, src, edits):
-    for old, new in edits:
-        n = src.count(old)
-        assert n == 1, (
-            f"anchor matched {n} times in {mod.__file__}, aborting. This image's "
-            f"benchmark source differs from both the pre-#44244 and post-#44244 "
-            f"layouts; update the shim.\n{old[:120]}..."
-        )
-        src = src.replace(old, new, 1)
-    with open(mod.__file__, "w") as fh:
-        fh.write(src)
-    print("patched OK ->", mod.__file__)
-
-if not have_cli:
-    # serve.py -- declare the --chat-template-kwargs argument before --extra-body
-    apply(S, s_src, [('''    parser.add_argument(
-        "--extra-body",''', '''    parser.add_argument(
-        "--chat-template-kwargs",
-        type=json.loads,
-        default=None,
-        help="JSON dict forwarded to apply_chat_template during "
-        "client-side prompt rendering, e.g. to enable reasoning mode.",
-    )
-    parser.add_argument(
-        "--extra-body",''')])
-
-d_edits = []
-if not have_forward:
-    # datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call
-    d_edits.append(('''                output_len=args.speed_bench_output_len,
-                enable_multimodal_chat=args.enable_multimodal_chat,''',
-                    '''                output_len=args.speed_bench_output_len,
-                chat_template_kwargs=args.chat_template_kwargs,
-                enable_multimodal_chat=args.enable_multimodal_chat,'''))
-if not have_unpack:
-    # datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call
-    d_edits.append(('''                # apply template
-                if not skip_chat_template:
-                    prompt = tokenizer.apply_chat_template(
-                        [{"role": "user", "content": prompt}],
-                        add_generation_prompt=True,
-                        tokenize=False,
-                    )
-
-                prompt_len = len(tokenizer(prompt).input_ids)''',
-                    '''                # apply template
-                if not skip_chat_template:
-                    _ctk = kwargs.get("chat_template_kwargs") or {}
-                    prompt = tokenizer.apply_chat_template(
-                        [{"role": "user", "content": prompt}],
-                        add_generation_prompt=True,
-                        tokenize=False,
-                        **_ctk,
-                    )
-
-                prompt_len = len(tokenizer(prompt).input_ids)'''))
-if d_edits:
-    apply(D, d_src, d_edits)
+missing = []
+if '"--chat-template-kwargs"' not in s_src:
+    missing.append(f"CLI option in {S.__file__}")
+if ('chat_template_kwargs=getattr(args' not in d_src
+        and 'chat_template_kwargs=args.chat_template_kwargs' not in d_src):
+    missing.append(f"speed_bench forward in {D.__file__}")
+if '**(chat_template_kwargs or {})' not in d_src:
+    missing.append(f"apply_chat_template unpack in {D.__file__}")
+
+if missing:
+    print("CRITICAL: this image lacks native --chat-template-kwargs support:")
+    for item in missing:
+        print("  missing:", item)
+    print("thinking_on cells would silently measure a non-thinking AL. Use an")
+    print("image that contains vllm-project/vllm#44244.")
+    sys.exit(1)
+
+print("native --chat-template-kwargs support confirmed")
 PYEOF
 }
 
-# Apply the shim once if any thinking-on cell is requested.
+# Only thinking-on cells pass chat_template_kwargs, so only they need the support.
 if [[ " $THINKING_MODES " == *" on "* ]]; then
-    if ! apply_chat_template_kwargs_shim; then
-        echo "CRITICAL: --chat-template-kwargs shim failed — aborting"
+    if ! assert_chat_template_kwargs_support; then
+        echo "CRITICAL: --chat-template-kwargs preflight failed — aborting"
         exit 1
     fi
 fi

From 119551eb85ea3a611cee069c793afa5435b31db1 Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 7 Aug 2026 15:23:06 -0700
Subject: [PATCH 5/7] Cap max_num_seqs so DSpark cells past 3 tokens fit, and
 survive a dead cell

The DSpark=4 cell OOMed during warmup: the rejection sampler copies logits to
fp32 at max_num_seqs * (1 + num_speculative_tokens) * vocab, which the default
max_num_seqs of 1024 makes 2.47 GiB at 4 speculative tokens and 4.4 GiB at 8.
The memory profiler does not account for it, so the KV cache is already sized
before the allocation fails. The collection only drives 32 concurrent requests,
so pin max_num_seqs to 64 and expose gpu-memory-utilization to trade KV cache
for headroom if a higher level still runs out.

The run also aborted at that cell instead of recording N/A and continuing:
wait_for_server_ready exits rather than returns, so call it in a subshell.
---
 .../speedbench/dsv4dspark_fp4_b300_vllm.sh    | 31 ++++++++++++++++---
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
index d696e1444..0c7bcfea7 100755
--- a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
@@ -18,9 +18,12 @@
 # otherwise a copy of so the two AL curves stay directly comparable:
 #   - target model       DeepSeek-V4-Pro-DSpark (was DeepSeek-V4-Pro)
 #   - speculative-config method dspark + draft_sample_method (was method mtp)
-# Every serve flag is left byte-identical to the MTP collector: it already
-# matches the published DSpark recipe, and holding the serving config fixed is
-# what makes "DSpark vs MTP on DSV4-Pro" a like-for-like AL comparison.
+#   - --max-num-seqs and --gpu-memory-utilization pinned (MTP took the defaults)
+# The third one is a memory fix rather than a recipe change. AL is a per-draft
+# accept/reject property that does not depend on batch size or graph capture, so
+# pinning those two still leaves "DSpark vs MTP on DSV4-Pro" like-for-like; every
+# flag that does affect drafting is byte-identical to the MTP collector and to
+# the published DSpark recipe.
 #
 # Usage (inside the vLLM container, on a B300 node):
 #   export MODEL=deepseek-ai/DeepSeek-V4-Pro-DSpark
@@ -35,6 +38,8 @@
 #   DRAFT_SAMPLE_METHOD    greedy|probabilistic     (default greedy)
 #   REJECTION_SAMPLE_METHOD  passed through to speculative-config when non-empty
 #                            (default: unset, i.e. the vLLM default)
+#   MAX_NUM_SEQS      engine max batch size         (default 64)
+#   GPU_MEM_UTIL      --gpu-memory-utilization      (default 0.90)
 
 set -uo pipefail
 source "$(dirname "$0")/../../benchmark_lib.sh"
@@ -64,6 +69,18 @@ SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}"
 # nothing here sets speculative_disable_by_batch_size, so drafting stays on at
 # this batch size and the curves remain comparable.
 CONCURRENCY="${CONCURRENCY:-32}"
+# Engine batch size; must stay >= CONCURRENCY or the client's requests just queue.
+# Held far below the vLLM default of 1024 because that default sizes two
+# allocations the memory profiler never sees — the rejection sampler's fp32 logits
+# scratch, max_num_seqs * (1 + num_speculative_tokens) * vocab * 4B, which is
+# 2.5 GB at 4 speculative tokens and 4.4 GB at 8, and the spec-decode CUDA graphs,
+# which grow with the same product. DSV4-Pro has no room for either: 141.5 GiB of
+# weights plus a 100 GiB KV cache already fills 266 of the 268 GiB on each B300,
+# and warmup died asking for 2.47 GiB more at num_speculative_tokens=4.
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-64}"
+# vLLM's own default, exposed so the KV cache can be traded for headroom if some
+# higher num_speculative_tokens still runs out.
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
 TEMPERATURE="${TEMPERATURE:-1.0}"
 # thinking-on chat_template_kwargs. MUST match the production/golden config:
 # the reference matrix (golden_al_distribution/dsv4_mtp.yaml) was measured with
@@ -286,6 +303,8 @@ run_cell() {
         --reasoning-parser deepseek_v4
         --max-cudagraph-capture-size 2048
         --max-model-len 16384
+        --max-num-seqs "$MAX_NUM_SEQS"
+        --gpu-memory-utilization "$GPU_MEM_UTIL"
         --speculative-config "{\"method\": \"dspark\", \"num_speculative_tokens\": $mtp, \"draft_sample_method\": \"$DRAFT_SAMPLE_METHOD\"$SPEC_EXTRA}"
     )
 
@@ -293,7 +312,11 @@ run_cell() {
     vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 &
     SERVER_PID=$!
 
-    if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then
+    # wait_for_server_ready exits the shell rather than returning when the server
+    # dies, which would make the N/A branch below unreachable and let one bad cell
+    # abort the whole matrix. Running it in a subshell keeps that exit local, so a
+    # cell that cannot start its server costs one cell instead of the run.
+    if ! (wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"); then
         echo "  -> server failed to start (thinking=$mode dspark=$mtp), recording N/A"
         AL_RESULT["${mode}_${mtp}"]="N/A"
         cleanup_server

From f3a5ae2169b02988e87ad6fa82816a8efadfaac9 Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 7 Aug 2026 15:35:48 -0700
Subject: [PATCH 6/7] Follow the published B300 DSpark recipe: expert parallel
 + deep_gemm_mega_moe

The collector inherited EP_SIZE / DP_ATTENTION knobs from the MTP collector, but
speedbench-al.yml exports EP_SIZE=1 and DP_ATTENTION=false for the whole matrix,
so both recipe flags were silently dropped and the run was plain TP on the auto
MoE backend. TP-sharding the FP4 experts cost 141.53 GiB per GPU against an
831 GiB checkpoint (1132 GiB across 8 ranks), roughly 37 GiB per GPU of sharding
overhead on weights expert parallel keeps whole, which is most of why there was
nothing left for warmup after the KV cache.

Hard-code TEP8 instead of honoring knobs the workflow overrides. TP stays 8 to
match the MTP collector; AL does not depend on expert placement either way.
---
 .../speedbench/dsv4dspark_fp4_b300_vllm.sh    | 41 +++++++++++--------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
index 0c7bcfea7..59c138d82 100755
--- a/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
+++ b/benchmarks/single_node/speedbench/dsv4dspark_fp4_b300_vllm.sh
@@ -18,12 +18,13 @@
 # otherwise a copy of so the two AL curves stay directly comparable:
 #   - target model       DeepSeek-V4-Pro-DSpark (was DeepSeek-V4-Pro)
 #   - speculative-config method dspark + draft_sample_method (was method mtp)
+#   - expert parallel + deep_gemm_mega_moe, per the published B300 DSpark recipe
 #   - --max-num-seqs and --gpu-memory-utilization pinned (MTP took the defaults)
-# The third one is a memory fix rather than a recipe change. AL is a per-draft
-# accept/reject property that does not depend on batch size or graph capture, so
-# pinning those two still leaves "DSpark vs MTP on DSV4-Pro" like-for-like; every
-# flag that does affect drafting is byte-identical to the MTP collector and to
-# the published DSpark recipe.
+# The last two are about fitting in memory, not about drafting: see the TEP block
+# and MAX_NUM_SEQS below for what each one was fixing. AL is a per-draft
+# accept/reject property, independent of expert placement, batch size and graph
+# capture, so they leave "DSpark vs MTP on DSV4-Pro" like-for-like. Every flag
+# that does affect drafting is byte-identical to the MTP collector.
 #
 # Usage (inside the vLLM container, on a B300 node):
 #   export MODEL=deepseek-ai/DeepSeek-V4-Pro-DSpark
@@ -51,8 +52,6 @@ MODEL="${MODEL:?MODEL env var required (e.g. deepseek-ai/DeepSeek-V4-Pro-DSpark)
 # standalone local run where MODEL is itself a path.
 SERVE_MODEL="${MODEL_PATH:-$MODEL}"
 TP="${TP:-8}"
-DP_ATTENTION="${DP_ATTENTION:-false}"
-EP_SIZE="${EP_SIZE:-1}"
 PORT="${PORT:-8888}"
 
 MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}"
@@ -202,18 +201,24 @@ if [[ " $THINKING_MODES " == *" on "* ]]; then
     fi
 fi
 
+# TEP8, exactly as the published B300 DSpark recipe (vllm-project/recipes: TP 8 +
+# --enable-expert-parallel + --moe-backend deep_gemm_mega_moe).
+#
+# This is hard-coded rather than driven by the EP_SIZE / DP_ATTENTION knobs the
+# MTP collector carries, because speedbench-al.yml exports EP_SIZE=1 and
+# DP_ATTENTION=false for every model in the matrix, which silently turned the
+# recipe into plain TP. That cost real memory: TP-sharding the FP4 experts loaded
+# 141.53 GiB per GPU, 8x that being 1132 GiB against an 831 GiB checkpoint, so
+# roughly 37 GiB per GPU went to sharding overhead on weights that expert
+# parallel keeps whole. With a ~100 GiB KV cache on top, 266 of the 268 GiB were
+# gone before warmup, which is what made the num_speculative_tokens=4 cell OOM.
+#
+# TP stays 8 (not the DP+EP variant the recipe also lists) to match the MTP
+# collector's parallelism. Either way AL is unaffected: expert placement changes
+# where a matmul runs, not which draft tokens the target model accepts.
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
-if [ "${DP_ATTENTION}" = "true" ]; then
-    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
-fi
-EP_ARGS=()
-if [ "${EP_SIZE:-1}" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
-fi
-MOE_ARGS=()
-if [ "${DP_ATTENTION}" = "true" ]; then
-    MOE_ARGS=(--moe-backend deep_gemm_mega_moe)
-fi
+EP_ARGS=(--enable-expert-parallel)
+MOE_ARGS=(--moe-backend deep_gemm_mega_moe)
 
 # Optional extra speculative-config keys, rendered once so run_cell only has to
 # interpolate num_speculative_tokens.

From 6162a1374bd9dfca7b37e8208daa4de5074bf76b Mon Sep 17 00:00:00 2001
From: Albert Cheng <albecheng@nvidia.com>
Date: Fri, 14 Aug 2026 10:07:55 -0700
Subject: [PATCH 7/7] Read staged DeepSeek-V4-Pro-0813 from the read-only mount

0813 is now staged on the B300 shared FS (AIAOPS-853), so add its basename to
STAGED_MODELS. MODEL_PATH now resolves to /scratch/models instead of triggering
the one-off download to the writable models dir on the first collection.
---
 runners/launch_b300-nv.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh
index f39616801..47b15edfb 100644
--- a/runners/launch_b300-nv.sh
+++ b/runners/launch_b300-nv.sh
@@ -383,6 +383,7 @@ else
         DeepSeek-R1-0528-NVFP4-v2
         DeepSeek-V4-Flash
         DeepSeek-V4-Pro
+        DeepSeek-V4-Pro-0813
         GLM-5-FP8
         GLM-5-NVFP4
         GLM-5.1