From 7f8786b0c44a92c168de9629932e3e131fed6a11 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 1 Sep 2025 07:37:08 +0800 Subject: [PATCH 1/5] rename fused_get_rope.cu --- custom_ops/gpu_ops/cpp_extensions.cc | 2 +- ...sed_get_rope.cu => fused_get_rotary_embedding.cu} | 0 .../{moe_ffn_wint2.cu => moe_expert_ffn_wint2.cu} | 0 .../gpu_ops/moe/{wintx_unzip.cu => winx_unzip.cu} | 0 ...lue_by_flags.cu => set_value_by_flags_and_idx.cu} | 0 ...gs.cu => speculate_set_value_by_flags_and_idx.cu} | 0 custom_ops/setup_ops.py | 12 ++++++------ 7 files changed, 7 insertions(+), 7 deletions(-) rename custom_ops/gpu_ops/{fused_get_rope.cu => fused_get_rotary_embedding.cu} (100%) rename custom_ops/gpu_ops/moe/{moe_ffn_wint2.cu => moe_expert_ffn_wint2.cu} (100%) rename custom_ops/gpu_ops/moe/{wintx_unzip.cu => winx_unzip.cu} (100%) rename custom_ops/gpu_ops/{set_value_by_flags.cu => set_value_by_flags_and_idx.cu} (100%) rename custom_ops/gpu_ops/speculate_decoding/{speculate_set_value_by_flags.cu => speculate_set_value_by_flags_and_idx.cu} (100%) diff --git a/custom_ops/gpu_ops/cpp_extensions.cc b/custom_ops/gpu_ops/cpp_extensions.cc index 027a33dc01a..a9b61f29065 100644 --- a/custom_ops/gpu_ops/cpp_extensions.cc +++ b/custom_ops/gpu_ops/cpp_extensions.cc @@ -1023,7 +1023,7 @@ PYBIND11_MODULE(fastdeploy_ops, m) { m.def("moe_expert_ffn", &MoeExpertFFNFunc, "moe export ffn function"); /** - * moe/fused_moe/moe_ffn_wint2.cu + * moe/fused_moe/moe_expert_ffn_wint2.cu * moe_expert_ffn_wint2 */ m.def("moe_expert_ffn_wint2", &MoeExpertFFNWint2Func, "moe export ffn wint2 function"); diff --git a/custom_ops/gpu_ops/fused_get_rope.cu b/custom_ops/gpu_ops/fused_get_rotary_embedding.cu similarity index 100% rename from custom_ops/gpu_ops/fused_get_rope.cu rename to custom_ops/gpu_ops/fused_get_rotary_embedding.cu diff --git a/custom_ops/gpu_ops/moe/moe_ffn_wint2.cu b/custom_ops/gpu_ops/moe/moe_expert_ffn_wint2.cu similarity index 100% rename from custom_ops/gpu_ops/moe/moe_ffn_wint2.cu rename to custom_ops/gpu_ops/moe/moe_expert_ffn_wint2.cu diff --git a/custom_ops/gpu_ops/moe/wintx_unzip.cu b/custom_ops/gpu_ops/moe/winx_unzip.cu similarity index 100% rename from custom_ops/gpu_ops/moe/wintx_unzip.cu rename to custom_ops/gpu_ops/moe/winx_unzip.cu diff --git a/custom_ops/gpu_ops/set_value_by_flags.cu b/custom_ops/gpu_ops/set_value_by_flags_and_idx.cu similarity index 100% rename from custom_ops/gpu_ops/set_value_by_flags.cu rename to custom_ops/gpu_ops/set_value_by_flags_and_idx.cu diff --git a/custom_ops/gpu_ops/speculate_decoding/speculate_set_value_by_flags.cu b/custom_ops/gpu_ops/speculate_decoding/speculate_set_value_by_flags_and_idx.cu similarity index 100% rename from custom_ops/gpu_ops/speculate_decoding/speculate_set_value_by_flags.cu rename to custom_ops/gpu_ops/speculate_decoding/speculate_set_value_by_flags_and_idx.cu diff --git a/custom_ops/setup_ops.py b/custom_ops/setup_ops.py index a0757d18063..9a57455e752 100644 --- a/custom_ops/setup_ops.py +++ b/custom_ops/setup_ops.py @@ -204,7 +204,7 @@ def find_end_files(directory, end_str): "gpu_ops/get_output_msg_with_topk.cc", "gpu_ops/save_output_msg_with_topk.cc", "gpu_ops/transfer_output.cc", - "gpu_ops/set_value_by_flags.cu", + "gpu_ops/set_value_by_flags_and_idx.cu", "gpu_ops/token_penalty_multi_scores.cu", "gpu_ops/stop_generation.cu", "gpu_ops/stop_generation_multi_ends.cu", @@ -223,7 +223,7 @@ def find_end_files(directory, end_str): "gpu_ops/speculate_decoding/speculate_get_output_padding_offset.cu", "gpu_ops/speculate_decoding/speculate_get_seq_lens_output.cu", "gpu_ops/speculate_decoding/speculate_save_output.cc", - "gpu_ops/speculate_decoding/speculate_set_value_by_flags.cu", + "gpu_ops/speculate_decoding/speculate_set_value_by_flags_and_idx.cu", "gpu_ops/speculate_decoding/speculate_step.cu", "gpu_ops/speculate_decoding/speculate_step_system_cache.cu", "gpu_ops/speculate_decoding/speculate_update_v3.cu", @@ -261,7 +261,7 @@ def find_end_files(directory, end_str): "gpu_ops/save_output_msg_with_topk.cc", "gpu_ops/transfer_output.cc", "gpu_ops/set_mask_value.cu", - "gpu_ops/set_value_by_flags.cu", + "gpu_ops/set_value_by_flags_and_idx.cu", "gpu_ops/ngram_mask.cu", "gpu_ops/gather_idx.cu", "gpu_ops/get_output_ep.cc", @@ -276,7 +276,7 @@ def find_end_files(directory, end_str): "gpu_ops/recover_decode_task.cu", "gpu_ops/step.cu", "gpu_ops/step_reschedule.cu", - "gpu_ops/fused_get_rope.cu", + "gpu_ops/fused_get_rotary_embedding.cu", "gpu_ops/get_padding_offset.cu", "gpu_ops/update_inputs.cu", "gpu_ops/update_inputs_beam.cu", @@ -560,7 +560,7 @@ def find_end_files(directory, end_str): "gpu_ops/save_output_msg_with_topk.cc", "gpu_ops/transfer_output.cc", "gpu_ops/get_padding_offset.cu", - "gpu_ops/set_value_by_flags.cu", + "gpu_ops/set_value_by_flags_and_idx.cu", "gpu_ops/rebuild_padding.cu", "gpu_ops/update_inputs.cu", "gpu_ops/stop_generation_multi_ends.cu", @@ -609,7 +609,7 @@ def find_end_files(directory, end_str): "gpu_ops/transfer_output.cc", "gpu_ops/save_with_output.cc", "gpu_ops/set_mask_value.cu", - "gpu_ops/set_value_by_flags.cu", + "gpu_ops/set_value_by_flags_and_idx.cu", "gpu_ops/ngram_mask.cu", "gpu_ops/gather_idx.cu", "gpu_ops/get_output_ep.cc", From e5f80ef5a3b31ad817343abe5e05366af30d6799 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 1 Sep 2025 17:34:53 +0800 Subject: [PATCH 2/5] fix --- custom_ops/setup_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_ops/setup_ops.py b/custom_ops/setup_ops.py index 9a57455e752..a65d84580b0 100644 --- a/custom_ops/setup_ops.py +++ b/custom_ops/setup_ops.py @@ -618,7 +618,7 @@ def find_end_files(directory, end_str): "gpu_ops/stop_generation.cu", "gpu_ops/stop_generation_multi_ends.cu", "gpu_ops/set_flags.cu", - "gpu_ops/fused_get_rope.cu", + "gpu_ops/fused_get_rotary_embedding.cu", "gpu_ops/get_padding_offset.cu", "gpu_ops/update_inputs.cu", "gpu_ops/update_inputs_beam.cu", From fc4277518c3856caf1d77e54316d0a83e9e30e21 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 1 Sep 2025 18:11:58 +0800 Subject: [PATCH 3/5] fix typos --- fastdeploy/model_executor/models/deepseek_v3.py | 2 +- fastdeploy/model_executor/models/ernie4_5_moe.py | 2 +- .../models/ernie4_5_vl/ernie4_5_vl_moe.py | 2 +- fastdeploy/model_executor/models/qwen2.py | 2 +- fastdeploy/model_executor/models/qwen3.py | 2 +- fastdeploy/model_executor/models/qwen3moe.py | 2 +- .../test_cuda_graph_dynamic_subgraph.py | 6 +++--- .../test_cuda_graph_recapture.py | 16 ++++++++-------- .../test_cuda_graph_spec_decode.py | 6 +++--- .../test_static_graph_cuda_graph_split.py | 6 +++--- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/fastdeploy/model_executor/models/deepseek_v3.py b/fastdeploy/model_executor/models/deepseek_v3.py index beb34877935..c9bef18444c 100644 --- a/fastdeploy/model_executor/models/deepseek_v3.py +++ b/fastdeploy/model_executor/models/deepseek_v3.py @@ -733,7 +733,7 @@ def forward( return hidden_states def clear_grpah_opt_backend(self): - """Clear graph optimization bakcend, the captured cuda graph will be cleaned""" + """Clear graph optimization backend, the captured cuda graph will be cleaned""" self.model.clear_grpah_opt_backend(fd_config=self.fd_config) diff --git a/fastdeploy/model_executor/models/ernie4_5_moe.py b/fastdeploy/model_executor/models/ernie4_5_moe.py index 932a4376c6e..2bf8a4bd197 100644 --- a/fastdeploy/model_executor/models/ernie4_5_moe.py +++ b/fastdeploy/model_executor/models/ernie4_5_moe.py @@ -620,7 +620,7 @@ def forward( return hidden_states def clear_grpah_opt_backend(self): - """Clear graph optimization bakcend, the captured cuda graph will be cleaned""" + """Clear graph optimization backend, the captured cuda graph will be cleaned""" self.ernie.clear_grpah_opt_backend(fd_config=self.fd_config) diff --git a/fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py b/fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py index fff67033414..b1ff9ea97fd 100644 --- a/fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py +++ b/fastdeploy/model_executor/models/ernie4_5_vl/ernie4_5_vl_moe.py @@ -721,7 +721,7 @@ def forward( return hidden_states def clear_grpah_opt_backend(self): - """Clear graph optimization bakcend, the captured cuda graph will be cleaned""" + """Clear graph optimization backend, the captured cuda graph will be cleaned""" self.ernie.clear_grpah_opt_backend(fd_config=self.fd_config) diff --git a/fastdeploy/model_executor/models/qwen2.py b/fastdeploy/model_executor/models/qwen2.py index c01bfbe2a7a..e86533f9440 100644 --- a/fastdeploy/model_executor/models/qwen2.py +++ b/fastdeploy/model_executor/models/qwen2.py @@ -395,7 +395,7 @@ def forward( return hidden_states def clear_grpah_opt_backend(self): - """Clear graph optimization bakcend, the captured cuda graph will be cleaned""" + """Clear graph optimization backend, the captured cuda graph will be cleaned""" self.qwen2.clear_grpah_opt_backend(fd_config=self.fd_config) diff --git a/fastdeploy/model_executor/models/qwen3.py b/fastdeploy/model_executor/models/qwen3.py index cb65f8f50d7..d0c532cfc5e 100644 --- a/fastdeploy/model_executor/models/qwen3.py +++ b/fastdeploy/model_executor/models/qwen3.py @@ -331,7 +331,7 @@ def forward( return hidden_states def clear_grpah_opt_backend(self): - """Clear graph optimization bakcend, the captured cuda graph will be cleaned""" + """Clear graph optimization backend, the captured cuda graph will be cleaned""" self.model.clear_grpah_opt_backend(fd_config=self.fd_config) diff --git a/fastdeploy/model_executor/models/qwen3moe.py b/fastdeploy/model_executor/models/qwen3moe.py index 4210f14446a..e28b5748f7e 100644 --- a/fastdeploy/model_executor/models/qwen3moe.py +++ b/fastdeploy/model_executor/models/qwen3moe.py @@ -452,7 +452,7 @@ def forward( return hidden_states def clear_grpah_opt_backend(self): - """Clear graph optimization bakcend, the captured cuda graph will be cleaned""" + """Clear graph optimization backend, the captured cuda graph will be cleaned""" self.model.clear_grpah_opt_backend(fd_config=self.fd_config) diff --git a/tests/graph_optimization/test_cuda_graph_dynamic_subgraph.py b/tests/graph_optimization/test_cuda_graph_dynamic_subgraph.py index 1b5039ade6d..77c154b77dd 100644 --- a/tests/graph_optimization/test_cuda_graph_dynamic_subgraph.py +++ b/tests/graph_optimization/test_cuda_graph_dynamic_subgraph.py @@ -170,14 +170,14 @@ def test_cuda_graph_subgraph(self): input_tensor1 = paddle.ones([8]) forward_meta1 = ForwardMeta(input_ids=input_tensor1, ids_remove_padding=input_tensor1, step_use_cudagraph=True) - # Triger Capture + # Trigger Capture _ = test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) - # Reaplay + # Replay _ = test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) output1 = test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) - # Corrent output + # Correct output output1_correct = test_model1.forward_correct(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) assert (output1 == output1_correct).all() diff --git a/tests/graph_optimization/test_cuda_graph_recapture.py b/tests/graph_optimization/test_cuda_graph_recapture.py index 6198e1eb401..126d231fe33 100644 --- a/tests/graph_optimization/test_cuda_graph_recapture.py +++ b/tests/graph_optimization/test_cuda_graph_recapture.py @@ -102,43 +102,43 @@ def test_cuda_graph_recapture(self): input_tensor1 = paddle.ones([1, 32768]) forward_meta1 = ForwardMeta(input_ids=input_tensor1, ids_remove_padding=input_tensor1, step_use_cudagraph=True) - # Corrent output + # Correct output self.output_correct = self.test_model1.forward_correct( ids_remove_padding=input_tensor1, forward_meta=forward_meta1 ) - # Capture and Destory + # Capture and Destroy self.capture_and_replay(input_tensor1, forward_meta1) self.recapture_and_replay(input_tensor1, forward_meta1) def capture_and_replay(self, input_tensor1, forward_meta1): """ """ - # Triger Capture + # Trigger Capture print_gpu_memory_use(0, "before capture") output1 = self.test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) print_gpu_memory_use(0, "after capture") - # Reaplay + # Replay output1 = self.test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) assert (output1 == self.output_correct).all() - # Destory + # Destroy print_gpu_memory_use(0, "before destory") self.test_model1.clear_grpah_opt_backend() print_gpu_memory_use(0, "after destory") def recapture_and_replay(self, input_tensor1, forward_meta1): """ """ - # Triger Capture + # Trigger Capture print_gpu_memory_use(0, "before recapture") output2 = self.test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) print_gpu_memory_use(0, "after recapture") - # Reaplay + # Replay output2 = self.test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) assert (output2 == self.output_correct).all() - # Destory + # Destroy print_gpu_memory_use(0, "before destory") self.test_model1.clear_grpah_opt_backend() print_gpu_memory_use(0, "after destory") diff --git a/tests/graph_optimization/test_cuda_graph_spec_decode.py b/tests/graph_optimization/test_cuda_graph_spec_decode.py index 2fc685bbf8d..3ea67d4b3e5 100644 --- a/tests/graph_optimization/test_cuda_graph_spec_decode.py +++ b/tests/graph_optimization/test_cuda_graph_spec_decode.py @@ -117,14 +117,14 @@ def test_cuda_graph_spec_decode(self): input_tensor1 = paddle.ones([1, 32768]) forward_meta1 = ForwardMeta(input_ids=input_tensor1, ids_remove_padding=input_tensor1, step_use_cudagraph=True) - # Triger Capture + # Trigger Capture _ = test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) - # Reaplay + # Replay _ = test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) output1 = test_model1(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) - # Corrent output + # Correct output output1_correct = test_model1.forward_correct(ids_remove_padding=input_tensor1, forward_meta=forward_meta1) assert (output1 == output1_correct).all() diff --git a/tests/graph_optimization/test_static_graph_cuda_graph_split.py b/tests/graph_optimization/test_static_graph_cuda_graph_split.py index 7421333a5d4..5bb721eba71 100644 --- a/tests/graph_optimization/test_static_graph_cuda_graph_split.py +++ b/tests/graph_optimization/test_static_graph_cuda_graph_split.py @@ -104,14 +104,14 @@ def test(self): x = paddle.randint(32, shape=[1, 8]) forward_meta1 = ForwardMeta(input_ids=x, ids_remove_padding=x, step_use_cudagraph=True) - # Triger Capture + # Trigger Capture _ = test_model1(x, forward_meta=forward_meta1) - # Reaplay + # Replay _ = test_model1(x, forward_meta=forward_meta1) output1 = test_model1(x, forward_meta=forward_meta1) - # Corrent output + # Correct output output1_correct = test_model1.forward_correct(x, forward_meta=forward_meta1) assert (output1 == output1_correct).all() From 10c95540f1a8622e3c085f61e0425dfb6eae60d3 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 1 Sep 2025 19:36:30 +0800 Subject: [PATCH 4/5] fix --- tests/model_loader/test_load_mtp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/model_loader/test_load_mtp.py b/tests/model_loader/test_load_mtp.py index 4469562465a..415cb469924 100644 --- a/tests/model_loader/test_load_mtp.py +++ b/tests/model_loader/test_load_mtp.py @@ -26,7 +26,7 @@ from fastdeploy.model_executor.models.ernie4_5_mtp import Ernie4_5_MTPForCausalLM sys.path.append("../") -from utils import get_default_test_fd_config +from tests.utils import get_default_test_fd_config strategy = fleet.DistributedStrategy() fleet.init(strategy=strategy) From fe4a8c91b2355dc9dac3f97935fca99ec9719464 Mon Sep 17 00:00:00 2001 From: co63oc Date: Mon, 1 Sep 2025 19:37:23 +0800 Subject: [PATCH 5/5] fix --- tests/model_loader/test_load_mtp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/model_loader/test_load_mtp.py b/tests/model_loader/test_load_mtp.py index 4469562465a..415cb469924 100644 --- a/tests/model_loader/test_load_mtp.py +++ b/tests/model_loader/test_load_mtp.py @@ -26,7 +26,7 @@ from fastdeploy.model_executor.models.ernie4_5_mtp import Ernie4_5_MTPForCausalLM sys.path.append("../") -from utils import get_default_test_fd_config +from tests.utils import get_default_test_fd_config strategy = fleet.DistributedStrategy() fleet.init(strategy=strategy)