diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 13456c2dcb..c6435db9cf 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -41,7 +41,8 @@ MiniMax-M2.5: MiniMax-M3-MXFP4: prefill_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" decode_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" + prefill_env: "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048" hf_dir: "models--amd--MiniMax-M3-MXFP4" gpt-oss-120b: diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh index f19ce8560b..55154cd015 100755 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ b/benchmarks/multi_node/amd_utils/server_vllm.sh @@ -145,10 +145,12 @@ pf = bash_escape(m.get('prefill_flags', '--tensor-parallel-size 8')) df = bash_escape(m.get('decode_flags', '--tensor-parallel-size 8')) ev = bash_escape(m.get('env', '')) dev = bash_escape(m.get('decode_env', '')) +pev = bash_escape(m.get('prefill_env', '')) print(f'PREFILL_SERVER_CONFIG=\"{pf}\"') print(f'DECODE_SERVER_CONFIG=\"{df}\"') print(f'MODEL_ENVS=\"{ev}\"') print(f'DECODE_MODEL_ENVS=\"{dev}\"') +print(f'PREFILL_MODEL_ENVS=\"{pev}\"') ")" echo "Loaded model configuration for: $MODEL_NAME" @@ -251,6 +253,11 @@ if [ "$NODE_RANK" -eq 0 ]; then setup_vllm_env + for env_pair in ${PREFILL_MODEL_ENVS}; do + export "$env_pair" + echo "[PREFILL_ENV] $env_pair" + done + # Router is started as an external container by job.slurm (VLLM_ROUTER_IMAGE) echo "Using external vllm-router container (started by job.slurm on this node)" @@ -420,6 +427,11 @@ elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then setup_vllm_env + for env_pair in ${PREFILL_MODEL_ENVS}; do + export "$env_pair" + echo "[PREFILL_ENV] $env_pair" + done + SERVED_MODEL="${MODEL_NAME}" PREFILL_CMD="vllm serve ${MODEL_PATH} \ --served-model-name ${SERVED_MODEL} \ diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9851e8d6d0..d14e1dc990 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2420,7 +2420,7 @@ minimaxm3-fp8-mi355x-vllm-mtp: # MiniMax-M3 MXFP4 MI355X vLLM disaggregated (prefill/decode) config. minimaxm3-fp4-mi355x-vllm-disagg: - image: rocm/vllm-dev:vllm-0.23.1-rocm723-mi35x-mori-0625 + image: vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: mi355x-disagg @@ -2433,9 +2433,9 @@ minimaxm3-fp4-mi355x-vllm-disagg: - isl: 8192 osl: 1024 search-space: - # 1P TP4 + 1D TP4 (2 nodes total), conc sweep 1..512 (single job, looped) + # 1P TP4 + 1D TP4 (2 nodes total), conc sweep 1..256 (single job, looped) - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] prefill: num-worker: 1 tp: 4 @@ -2450,23 +2450,6 @@ minimaxm3-fp4-mi355x-vllm-disagg: dp-attn: false additional-settings: - "DECODE_NODES=1" - # 2P TP4 + 1D TP4 (3 nodes total), conc 128/256/512 (single job, looped) - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" # MiniMax-M3 MXFP4 MI355X vLLM recipe. The pinned nightly includes upstream # MiniMax-M3 Quark MXFP4 support (vllm-project/vllm#45794). Use the text-only # language-model path and mirror the MXFP8 MI355X search space for a direct diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 336edc9a61..a3ee8c8ae3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4575,6 +4575,15 @@ - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 +- config-keys: + - minimaxm3-fp4-mi355x-vllm-disagg + description: + - "Update the MiniMax-M3 MXFP4 MI355X vLLM disagg image to vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 (from rocm/vllm-dev:vllm-0.23.1-rocm723-mi35x-mori-0625) for AITER MoE and shared-expert fusion support." + - "Export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 for MiniMax-M3-MXFP4 (both prefill and decode)." + - "Enable prefill-only INT4 quick-reduce: set VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 and VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 on the prefill workers via a new prefill_env channel (mirrors the existing decode_env path in server_vllm.sh)." + - "Cap the 1P1D TP4 concurrency sweep at 256 (was 512); drop the 2P1D TP4 layout (128/256/512) as it is CI-flaky with negligible curve impact." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1943 + - config-keys: - minimaxm3-fp8-mi355x-vllm-mtp description: