diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh index 5dd3edb47..849b97490 100644 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh @@ -45,16 +45,12 @@ export SGLANG_TIMEOUT_KEEP_ALIVE=900 export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV41_REASONING_EFFORT=high -# TP4 has room for the original Engram tables in HBM: the STP baseline used -# 73.3 GiB for weights before the ~47.2 GiB tables. Host shards still had 0% -# huge-page backing on some ranks after model-local cache advice, so avoid -# that lookup bottleneck on TP4. TP2 retains the anonymous host-table layout. -if (( TP >= 4 )); then +# C1/C2 TP4 keep Engram on GPU; only C1 has matched local measurements. +# Other points retain host tables for prefill workspace and cached prefixes. +export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 +export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank +if (( TP == 4 && CONC <= 2 )); then export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=0 - unset SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT -else - export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 - export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank fi # The bundled Markov embedding/head weights are natively BF16. Preserve the @@ -134,6 +130,20 @@ if (( TP == 2 && CONC >= 64 )); then CHUNKED_PREFILL_SIZE=8192 fi +# Matched one-hour C64 runs favor 16K chunks without delaying prefills. +# C1/C2 use 4K/interval16; C1 keeps automatic SWA tails and C2 reserves 128. +# TP2 retains its recipe above; the caller selects EP in the master config. +if (( TP == 4 )); then + export SGLANG_RAGGED_VERIFY_MODE=static + if (( CONC > 2 )); then + CHUNKED_PREFILL_SIZE=16384 + SCHEDULING_ARGS=(--prefill-decode-interval 0) + CACHE_ARGS=(--swa-prefix-tails 4096) + MAX_RUNNING_REQUESTS=128 + CUDA_GRAPH_MAX_BS=128 + fi +fi + SGLANG_CMD=( python3 -m sglang.launch_server --model-path "$MODEL_PATH" --served-model-name "$MODEL" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7ee2ca0be..ea3fccf1c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9000,9 +9000,9 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } -# Official SGLang nightly; Engram weights in host DRAM, native DSpark draft. +# Official SGLang nightly; TP4 C1/C2 use GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40 + image: lmsysorg/sglang:dev-cu13-nightly-0924@sha256:d2c9929f8cc9889326203f6cbc46da48b69de42891b167592c9c76cc803d246e model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv @@ -9014,4 +9014,5 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2] } + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 32, 64, 128] } diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 9f083eae9..ab9db16fc 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -408,6 +408,10 @@ Source: [upstream recipe](https://github.com/vllm-project/recipes/blob/main/mode ### DeepSeek-V4.1-Flash DSpark on SGLang +The GB300 DeepSeek-V4.1-Flash SGLang curve uses `dev-cu13-nightly-0924`: +TP4/EP1 at C1/C2 with GPU Engram and 4K prefill chunks, and TP4/EP4 at C4+ +with host Engram and 16K chunks. TP2 is unchanged. Full sweep validation is pending. + The H100 SGLang candidate sweeps DSpark at concurrency 1/2/4/8/16/20. It retains 8 SWA prefix tails per concurrency at C1/C2 and 32 at C4 and above. A matched one-hour comparison rejected a blanket 128-tail floor: C2 throughput improved only 1.7% while interactivity fell 44.5%. Completed STP comparisons did not contribute a measured frontier point, so STP is excluded from the selected sweep. The recipe interleaves 16 decode steps between prefill chunks, preserving trace content and context limits. The same sweep also qualifies supported TP8/EP8/DP8 attention at C4/C8/C16/C20. DP uses a stock consistent-hash router with stable session keys, DP LM-head execution, and 64 SWA prefix tails per rank. Full C16 GSM8K passed on all 1,319 examples; its performance contribution remains under measurement. The native 1M context and the AgentX subagent/session semantics are preserved. @@ -418,8 +422,8 @@ The nightly candidate uses `nightly-dev-cu13-20260922-582389ce`, native MXFP4 Ma arms, one PR per SKU across h100, h200, b200, b300, gb200, gb300 and mi355x. They follow the [SGLang cookbook](https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1), which has no released SGLang version for this model yet. B200 pins the CUDA 13 nightly -`lmsysorg/sglang:nightly-dev-cu13-20260922-4cbf290f` by digest; the other NVIDIA arms use -`lmsysorg/sglang:dev-dsv41` and MI355X uses `lmsysorg/sglang:dev-dsv41-mi35x`. +`lmsysorg/sglang:nightly-dev-cu13-20260922-4cbf290f` by digest; other NVIDIA image pins +are recorded in `configs/nvidia-master.yaml`, and MI355X uses `lmsysorg/sglang:dev-dsv41-mi35x`. B200 uses shipped-default DSpark across TP4/EP4 C1–128 and TP2/EP2 C1–8. Engram stays in host DRAM with `SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank`. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 83fd0bee9..c7cdc20be 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -354,6 +354,10 @@ Maximum concurrency for 1,048,576 tokens per request: 6.70x ### SGLang 上的 DeepSeek-V4.1-Flash DSpark +GB300 DeepSeek-V4.1-Flash SGLang 曲线使用 `dev-cu13-nightly-0924`: +C1/C2 使用 TP4/EP1、GPU Engram 和 4K prefill chunk;C4+ 使用 TP4/EP4、 +主机 Engram 和 16K chunk。TP2 不变,仍待全量 sweep 验证。 + H100 SGLang 候选配方在并发 1/2/4/8/16/20 下测试 DSpark。C1/C2 按并发数的 8 倍保留 SWA 前缀尾部,C4 及以上按 32 倍保留。相同条件下的一小时对比否决了统一的 128 尾部下限:C2 吞吐量仅提高 1.7%,交互性能却下降 44.5%。已完成的 STP 对比没有贡献实测性能前沿点,因此所选 sweep 不包含 STP。配方在预填充分块之间插入 16 步解码,轨迹内容和上下文限制保持不变。 同一 sweep 还会在 C4/C8/C16/C20 下验证受支持的 TP8/EP8/DP8 attention。DP 使用原生一致性哈希路由器与稳定会话键、DP LM-head,以及每 rank 64 个 SWA 前缀尾部。C16 的完整 GSM8K 已通过全部 1,319 个样本;其性能贡献仍在测量中。原生 1M 上下文与 AgentX 子代理/会话语义保持不变。 @@ -365,8 +369,8 @@ nightly 候选配方使用 `nightly-dev-cu13-20260922-582389ce`、原生 MXFP4 M 与 mi355x 上的 SGLang 对应版本(每个 SKU 一个 PR),遵循 [SGLang cookbook](https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1)。 该模型尚无正式发布的 SGLang 版本。B200 通过 digest 固定 CUDA 13 nightly 镜像 -`lmsysorg/sglang:nightly-dev-cu13-20260922-4cbf290f`;其他 NVIDIA 配方使用 -`lmsysorg/sglang:dev-dsv41`,MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。 +`lmsysorg/sglang:nightly-dev-cu13-20260922-4cbf290f`;其他 NVIDIA 镜像固定版本见 +`configs/nvidia-master.yaml`,MI355X 使用 `lmsysorg/sglang:dev-dsv41-mi35x`。 B200 在 TP4/EP4 C1–128 与 TP2/EP2 C1–8 全部使用上游默认 DSpark。 Engram 保留在主机 DRAM,设置 `SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank`。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 89f152cac..afc97dc79 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8887,3 +8887,12 @@ - "Add DeepSeek-V4-Pro-0813 golden AL for draft lengths 4, 5, 7 and 8 (3.36 / 3.61 / 3.73 / 3.47)." - "Agentic PD router: pin --decode-policy round_robin so decode no longer inherits the prefill --policy (consistent_hashing)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3256 + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the GB300 DeepSeek-V4.1-Flash SGLang curve to dev-cu13-nightly-0924: pure TP4 at C1/C2 and tuned TP4/EP4 at C4+; keep TP2 unchanged." + - "更新 GB300 DeepSeek-V4.1-Flash SGLang 曲线至 dev-cu13-nightly-0924:C1/C2 使用纯 TP4,C4+ 使用调优后的 TP4/EP4;TP2 不变。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 83ecb73eb..5d2b0b0d5 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -91,6 +91,15 @@ NGINX_SQUASH_FILE="/data/home/sa-shared/gharunners/squash/$(echo "$NGINX_IMAGE" # The login node is x86_64 and the compute nodes aarch64, so import on a compute node. import_squash() { local squash="$1" image="$2" + # Enroot uses the digest as the manifest tag, not Docker's @ syntax. + if [[ "$image" == *@sha256:* ]]; then + local image_digest="${image##*@}" + image="${image%@*}" + if [[ "${image##*/}" == *:* ]]; then + image="${image%:*}" + fi + image="${image}:${image_digest}" + fi local lock="${squash}.lock" srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=180 bash -c " exec 9>\"$lock\"