From b87dc97085084575901c3949e68b03c875dde55f Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 28 Sep 2026 10:12:09 -0700 Subject: [PATCH 1/2] refactor(minimaxm3): align GB200 recipes with e2e layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GB200 配方迁入端到端项目目录，并更新相关配置引用。 Signed-off-by: Rohit Pujar Nagraj --- ...-tp4-vllm-simple-mns16-nightly-native.yaml | 109 ++++++++++ ...g-tp4-vllm-simple-mns8-nightly-native.yaml | 109 ++++++++++ .../agentx/disagg-1p4d-dep4-tp2-mns1-c15.yaml | 190 +++++++++++++++++ .../disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml | 190 +++++++++++++++++ .../agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml | 191 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 76 ++++--- inferencex-e2e/perf-changelog.yaml | 12 ++ 7 files changed, 852 insertions(+), 25 deletions(-) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns16-nightly-native.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns8-nightly-native.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns1-c15.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns16-nightly-native.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns16-nightly-native.yaml new file mode 100644 index 0000000000..5140ede6d0 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns16-nightly-native.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-mns16-agentic-nightly-native" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + +dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' + stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 16 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns8-nightly-native.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns8-nightly-native.yaml new file mode 100644 index 0000000000..52cbbcf493 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns8-nightly-native.yaml @@ -0,0 +1,109 @@ +schema: 2 +name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-mns8-agentic-nightly-native" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: "vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7"} + frameworks: {dynamo: "1.5.0.dev20260908"} + +dynamo: {install: true, source: {pypi: "1.5.0.dev20260908"}} +environment: {ETCD_LEASE_TTL: "7200"} +slurm: {time_limit: "12:00:00"} +health_check: {max_attempts: 2160, interval_seconds: 10} +resources: {gpu_type: "gb200", gpus_per_node: 4} +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: {DYN_TCP_REQUEST_TIMEOUT: "60"} + args: + trust-remote-code: true + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +engine: + type: vllm + connector: +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":777389080576,"cpu_bytes_to_use_per_rank":194347270144,"lazy_offload":true}}' + stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + max-num-seqs: 8 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + + kv_events: true +sbatch_directives: {cpus-per-task: "144", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns1-c15.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns1-c15.yaml new file mode 100644 index 0000000000..b7062ac005 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns1-c15.yaml @@ -0,0 +1,190 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp2-mns1-c15-agentic" + +model: + path: "minimax-m3-nvfp4" + container: &container "vllm/vllm-openai@sha256:18372a7224938643461b846fb64c5c9d3d6e9727e82caf2dc3043e620c9d4d7a" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: *container} + frameworks: {dynamo: "1.5.0.dev20260917"} + +dynamo: {install: true, source: {wheel: "1.5.0.dev20260917"}} +environment: + PYTHONHASHSEED: "0" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + SRT_MOONCAKE_RANK_LOCAL_HCA_MAP: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + WITH_NVIDIA_PEERMEM: "0" +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "100GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + +engine: + type: vllm + connector: + # Preserve the vLLM process layout used by these retained recipes. + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 512 + gpu-memory-utilization: 0.93 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: &numa_nodes [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + decode: + nodes: 2 + workers: 4 + gpus: 2 + env: *worker_environment + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + router-mode: "least-loaded" + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + # The GB200 TP2 custom communicator faults during startup profiling. + disable-custom-all-reduce: true + enable-expert-parallel: false + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 512 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-num-seqs: 1 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + numa-bind: true + numa-bind-nodes: *numa_nodes + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + +sbatch_directives: {cpus-per-task: "140", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "1800" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml new file mode 100644 index 0000000000..59724a7618 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml @@ -0,0 +1,190 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp2-mns2-c20-c30-agentic" + +model: + path: "minimax-m3-nvfp4" + container: &container "vllm/vllm-openai@sha256:18372a7224938643461b846fb64c5c9d3d6e9727e82caf2dc3043e620c9d4d7a" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: *container} + frameworks: {dynamo: "1.5.0.dev20260917"} + +dynamo: {install: true, source: {wheel: "1.5.0.dev20260917"}} +environment: + PYTHONHASHSEED: "0" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + SRT_MOONCAKE_RANK_LOCAL_HCA_MAP: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + WITH_NVIDIA_PEERMEM: "0" +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "100GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + +engine: + type: vllm + connector: + # Preserve the vLLM process layout used by these retained recipes. + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 512 + gpu-memory-utilization: 0.93 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: &numa_nodes [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + decode: + nodes: 2 + workers: 4 + gpus: 2 + env: *worker_environment + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + router-mode: "least-loaded" + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + # The GB200 TP2 custom communicator faults during startup profiling. + disable-custom-all-reduce: true + enable-expert-parallel: false + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 512 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-num-seqs: 2 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + numa-bind: true + numa-bind-nodes: *numa_nodes + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + +sbatch_directives: {cpus-per-task: "140", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "1800" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml new file mode 100644 index 0000000000..93760c87d2 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml @@ -0,0 +1,191 @@ +schema: 2 +name: "minimax-m3-vllm-disagg-gb200-1p5d-dep4-tp2-mns4-c50-agentic" + +model: + path: "minimax-m3-nvfp4" + container: &container "vllm/vllm-openai@sha256:18372a7224938643461b846fb64c5c9d3d6e9727e82caf2dc3043e620c9d4d7a" + precision: "fp4" + +identity: + model: {repo: "nvidia/MiniMax-M3-NVFP4"} + container: {image: *container} + frameworks: {dynamo: "1.5.0.dev20260917"} + +dynamo: {install: true, source: {wheel: "1.5.0.dev20260917"}} +environment: + PYTHONHASHSEED: "0" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + SRT_MOONCAKE_RANK_LOCAL_HCA_MAP: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + WITH_NVIDIA_PEERMEM: "0" +health_check: {max_attempts: 2160, interval_seconds: 10} + +resources: + gpu_type: "gb200" + gpus_per_node: 4 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "100GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + nginx_session_affinity: true + nginx_session_affinity_header: "X-Dynamo-Session-ID" + args: + dyn-chat-processor: "vllm" + trust-remote-code: true + tool-call-parser: "minimax_m3" + reasoning-parser: "minimax_m3" + enable-auto-tool-choice: true + router-mode: "least-loaded" + env: + DYN_LOG: "info" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_TOKENIZER: "fastokens" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + +engine: + type: vllm + connector: + # Preserve the vLLM process layout used by these retained recipes. + dp_launch_mode: per_gpu +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + DYN_TCP_CONNECT_TIMEOUT: "120" + DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + WITH_NVIDIA_PEERMEM: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "1" + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + router-mode: "least-loaded" + router-session-affinity-ttl-secs: 1800 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + all2all-backend: "flashinfer_nvlink_one_sided" + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 512 + gpu-memory-utilization: 0.93 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: &numa_nodes [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + decode: + nodes: 3 + workers: 5 + gpus: 2 + env: *worker_environment + args: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + router-mode: "least-loaded" + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + # The GB200 TP2 custom communicator faults during startup profiling. + disable-custom-all-reduce: true + enable-expert-parallel: false + trust-remote-code: true + enable-prefix-caching: true + language-model-only: true + kv-cache-dtype: "fp8" + block-size: 512 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-num-seqs: 4 + watermark: 0.10 + max-cudagraph-capture-size: 512 + stream-interval: 20 + no-enable-flashinfer-autotune: true + numa-bind: true + numa-bind-nodes: *numa_nodes + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + dyn-default-thinking-mode: "enabled" + +sbatch_directives: {cpus-per-task: "140", mem: "0"} +srun_options: {container-remap-root: ""} + +benchmark: + type: custom + command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AGENTIC_WARMUP_GRACE_PERIOD: "1800" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "1800" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index b0fc9ef451..aa0727662b 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5695,10 +5695,10 @@ minimaxm3-fp4-b200-trtllm-agentic-mtp: search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [5, 10, 15, 20, 25, 30, 35, 40, 45], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [1, 5], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml } -# Run the GB200 aggregate configuration with TP4 resident, TP8 resident across -# two NVL4 nodes, and TP4 lazy SimpleCPU KV offload variants. The pinned -# runtime uses Dynamo-native MiniMax parsing on the worker without the vLLM -# chat processor in the frontend. +# Keep the GB200 aggregate anchors at TP4 C1/C30 and TP8 C1, then fill the +# middle of the curve with the retained TP4 SimpleCPU scheduler-cap recipes. +# The pinned runtime is used as shipped; no setup hook or engine patch is +# applied. minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: image: vllm/vllm-openai:nightly-9ea8f3ffc354901b740f0b31988900897b7221d7 model: nvidia/MiniMax-M3-NVFP4 @@ -5715,7 +5715,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: search-space: - spec-decoding: mtp kv-offloading: none - conc-list: [1, 10] + conc-list: [1] num-nodes: 1 worker: num-worker: 1 @@ -5727,7 +5727,31 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } - conc-list: [15, 25, 30] + conc-list: [20] + num-nodes: 1 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns8-nightly-native.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [25] + num-nodes: 1 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-tp4-vllm-simple-mns16-nightly-native.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [30] num-nodes: 1 worker: num-worker: 1 @@ -5749,57 +5773,59 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/agg-nightly-native-variants.yaml:override_tp8" minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: - image: vllm/vllm-openai@sha256:b9104b7ef3048e42f79fba9ab5da06e5aff8164aca9968692ec2f569aaaf34c6 + image: vllm/vllm-openai@sha256:18372a7224938643461b846fb64c5c9d3d6e9727e82caf2dc3043e620c9d4d7a model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:gb200-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.5.0.dev20260819" } + router: { name: dynamo-router, version: "1.5.0.dev20260917" } kv-p2p-transfer: nixl multinode: true disagg: true scenarios: agentic-coding: - # Each active four-rank prefill node reserves 150 GB/rank in Mooncake. - - dram-utilization: 0.665 + # The matrix describes the four-GPU DEP4 prefill pool as TP4/EP4 with DP + # attention; each recipe is authoritative and launches TP1 x DP4/EP4. + # 0.443487 resolves to 400 GB for the prefill node, matching 100 GB/rank. + - dram-utilization: 0.443487 search-space: - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [1] + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } + conc-list: [15] prefill: num-worker: 1 tp: 4 ep: 4 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_tp8_c1" - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: false } + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns1-c15.yaml" + decode: { num-worker: 4, tp: 2, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [8, 16] + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } + conc-list: [20, 30] prefill: num-worker: 1 tp: 4 ep: 4 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p2d_c8_c16" - decode: { num-worker: 2, tp: 4, ep: 4, dp-attn: false } + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml" + decode: { num-worker: 4, tp: 2, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } - conc-list: [24] + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } + conc-list: [50] prefill: num-worker: 1 tp: 4 ep: 4 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-variants.yaml:override_1p1d_c24" - decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: false } + - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml" + decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e280ad359f..9a0c0ce97c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -8995,3 +8995,15 @@ - "Validate MiniMax-M3 AgentX on H200 with vLLM FP8 after moving the end-to-end project into inferencex-e2e/. Preserve the existing image, recipe, and benchmark settings." - "将端到端项目迁入 inferencex-e2e/ 后，验证 H200 上的 vLLM FP8 MiniMax-M3 AgentX 运行路径，沿用现有镜像、配方和基准测试设置。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3525 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 GB200 aggregate TP4 SimpleCPU recipe variants with separate scheduler caps, and replace the disaggregated search space with DEP4 prefill plus TP2 decode pools." + - "Pin each checked-in recipe to its declared vLLM, Dynamo, and Mooncake runtime, using the shipped EAGLE3 draft precision and real verification for evals." + - "新增 MiniMax-M3 GB200 聚合 TP4 SimpleCPU 配方变体，并分别设置调度上限；同时将分离部署搜索空间调整为 DEP4 prefill 与 TP2 decode 池。" + - "每个已检入配方固定其声明的 vLLM、Dynamo 与 Mooncake 运行时，使用 EAGLE3 草稿模型随附精度，并在评测中保留真实验证。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3424 From 0a90206f83051208364895957385bc2be69c1c4b Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 28 Sep 2026 10:51:28 -0700 Subject: [PATCH 2/2] fix(minimaxm3): remove GB200 disaggregated C50 point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除 MiniMax-M3 GB200 分离部署的 C50 配置点及其不再使用的配方。 Signed-off-by: Rohit Pujar Nagraj --- .../agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml | 191 ------------------ inferencex-e2e/configs/nvidia-master.yaml | 12 -- inferencex-e2e/perf-changelog.yaml | 9 + 3 files changed, 9 insertions(+), 203 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml deleted file mode 100644 index 93760c87d2..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml +++ /dev/null @@ -1,191 +0,0 @@ -schema: 2 -name: "minimax-m3-vllm-disagg-gb200-1p5d-dep4-tp2-mns4-c50-agentic" - -model: - path: "minimax-m3-nvfp4" - container: &container "vllm/vllm-openai@sha256:18372a7224938643461b846fb64c5c9d3d6e9727e82caf2dc3043e620c9d4d7a" - precision: "fp4" - -identity: - model: {repo: "nvidia/MiniMax-M3-NVFP4"} - container: {image: *container} - frameworks: {dynamo: "1.5.0.dev20260917"} - -dynamo: {install: true, source: {wheel: "1.5.0.dev20260917"}} -environment: - PYTHONHASHSEED: "0" - DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" - SRT_MOONCAKE_RANK_LOCAL_HCA_MAP: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - WITH_NVIDIA_PEERMEM: "0" -health_check: {max_attempts: 2160, interval_seconds: 10} - -resources: - gpu_type: "gb200" - gpus_per_node: 4 -services: - - name: etcd - type: etcd - placement: - node: infra - - name: nats - type: nats - placement: - node: infra - options: - max_payload_mb: 32 - - name: mooncake-master - type: mooncake-master - options: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "100GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_session_affinity: true - nginx_session_affinity_header: "X-Dynamo-Session-ID" - args: - dyn-chat-processor: "vllm" - trust-remote-code: true - tool-call-parser: "minimax_m3" - reasoning-parser: "minimax_m3" - enable-auto-tool-choice: true - router-mode: "least-loaded" - env: - DYN_LOG: "info" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_TOKENIZER: "fastokens" - DYN_TOKENIZER_CACHE_BYTES: "8589934592" - DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" - -engine: - type: vllm - connector: - # Preserve the vLLM process layout used by these retained recipes. - dp_launch_mode: per_gpu -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 4 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - DYN_TCP_CONNECT_TIMEOUT: "120" - DYN_ROUTER_LEAST_LOADED_TOKEN_AWARE: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - UCX_MEMTYPE_CACHE: "n" - UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - WITH_NVIDIA_PEERMEM: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "1" - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - router-mode: "least-loaded" - router-session-affinity-ttl-secs: 1800 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - all2all-backend: "flashinfer_nvlink_one_sided" - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 512 - gpu-memory-utilization: 0.93 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - enable-cumem-allocator: true - numa-bind: true - numa-bind-nodes: &numa_nodes [0, 0, 1, 1] - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"lookup_async":false,"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - decode: - nodes: 3 - workers: 5 - gpus: 2 - env: *worker_environment - args: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - router-mode: "least-loaded" - tensor-parallel-size: 2 - pipeline-parallel-size: 1 - # The GB200 TP2 custom communicator faults during startup profiling. - disable-custom-all-reduce: true - enable-expert-parallel: false - trust-remote-code: true - enable-prefix-caching: true - language-model-only: true - kv-cache-dtype: "fp8" - block-size: 512 - gpu-memory-utilization: 0.9 - max-num-batched-tokens: 16384 - max-num-seqs: 4 - watermark: 0.10 - max-cudagraph-capture-size: 512 - stream-interval: 20 - no-enable-flashinfer-autotune: true - numa-bind: true - numa-bind-nodes: *numa_nodes - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN","rejection_sample_method":"block"}' - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_connector_extra_config":{"backends":["UCX"],"num_threads":8}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"enable_lookup":false}}]}}' - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - dyn-default-thinking-mode: "enabled" - -sbatch_directives: {cpus-per-task: "140", mem: "0"} -srun_options: {container-remap-root: ""} - -benchmark: - type: custom - command: "bash /infmax-workspace/benchmarks/srt_agentic.sh" - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AGENTIC_WARMUP_GRACE_PERIOD: "1800" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "1800" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_SERVER_METRICS_COLLECTION_INTERVAL: "1.0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index aa0727662b..69c76848b1 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5814,18 +5814,6 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: additional-settings: - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p4d-dep4-tp2-mns2-c20-c30.yaml" decode: { num-worker: 4, tp: 2, ep: 1, dp-attn: false } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } - conc-list: [50] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/minimaxm3/vllm/gb200-fp4/agentx/disagg-1p5d-dep4-tp2-mns4-c50.yaml" - decode: { num-worker: 5, tp: 2, ep: 1, dp-attn: false } minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9a0c0ce97c..fe8cda9b8d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9007,3 +9007,12 @@ - "新增 MiniMax-M3 GB200 聚合 TP4 SimpleCPU 配方变体，并分别设置调度上限；同时将分离部署搜索空间调整为 DEP4 prefill 与 TP2 decode 池。" - "每个已检入配方固定其声明的 vLLM、Dynamo 与 Mooncake 运行时，使用 EAGLE3 草稿模型随附精度，并在评测中保留真实验证。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3424 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Remove the disaggregated C50 point and its unused recipe from the MiniMax-M3 GB200 configuration." + - "从 MiniMax-M3 GB200 分离部署配置中移除 C50 配置点及其不再使用的配方。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3424