[FDConfig]Remove total_block_num/dtype/block_size/enc_dec_block_num in ParallelConfig

fastdeploy/config.py

@@ -181,7 +181,7 @@ def __init__(
         self.model = ""
         self.is_quantized = False
         self.max_model_len = 0
-        self.dtype = ""
+        self.dtype = "bfloat16"
         self.enable_logprob = False
         self.enable_redundant_experts = False
         self.redundant_experts_num = 0
@@ -529,25 +529,10 @@ def __init__(
         self.data_parallel_size = 1  # DP degree
         self.enable_expert_parallel = False
         self.local_data_parallel_id = 0
-        # The embedding weight distributed on your gpu cards is divided by row or column.
-        # Defaults to False means divide by row. When vocab_size can not be divided by world_size
-        # but hidden_size can, we can consider split embedding weight by column.
-        """
-        From old wersion worker args
-        TODO(gongshaotian): Reclassify
-        """
-        # Set default block num for profile run
-        self.total_block_num: int = 2000
-        # block size
-        self.block_size: int = 64
         # Engine worker queue port
         self.engine_worker_queue_port: str = "9923"
         # cuda visible devices
         self.device_ids: str = "0"
-        # Input dtype
-        self.dtype: str = "bfloat16"
-        # Encoder's decoder num
-        self.enc_dec_block_num: int = 1
         # First token id
         self.first_token_id: int = 1
         # Process ID of engine

fastdeploy/model_executor/layers/attention/iluvatar_attn_backend.py

@@ -71,7 +71,7 @@ class IluvatarAttnBackend(AttentionBackend):
     def __init__(self, fd_config: FDConfig, kv_num_heads: int, num_heads: int, head_dim: int):
         super().__init__()
         self.attention_metadata = IluvatarAttentionMetadata()
-        self.block_size = fd_config.parallel_config.block_size
+        self.block_size = fd_config.cache_config.block_size
         assert self.block_size == 16, "Iluvatar paged attn requires block_size must be 16."
         self.max_context_len = fd_config.model_config.max_model_len
         self.causal = getattr(fd_config.model_config, "causal", True)

fastdeploy/model_executor/layers/attention/moba_attention_backend.py

@@ -75,7 +75,7 @@ def __init__(
         super().__init__()
         self.attention_metadata: PlasAttentionMetadata = None
         assert fd_config.plas_attention_config is not None, "plas_attention_config is None"
-        self.block_size = fd_config.parallel_config.block_size
+        self.block_size = fd_config.cache_config.block_size
         self.max_seq_len = fd_config.model_config.max_model_len
         self.max_num_seqs = fd_config.scheduler_config.max_num_seqs
         self.kv_num_heads = kv_num_heads

fastdeploy/model_executor/layers/backends/intel_hpu/attention/hpu_attn_backend.py

@@ -167,7 +167,7 @@ def __init__(self, llm_config: FDConfig, kv_num_heads: int, num_heads: int, head
         super().__init__()
         self.attention_metadata: HPUAttentionMetadata = None
         # TODO(gongshaotian): Use llm_config parameters in the correct location
-        self.block_size = llm_config.parallel_config.block_size
+        self.block_size = llm_config.cache_config.block_size
         self.max_seq_len = llm_config.model_config.max_model_len
         self.rope_theta = 10000.0 if llm_config.model_config.rope_theta is None else llm_config.model_config.rope_theta
         self.rope_3d = getattr(llm_config.model_config, "rope_3d", False)

fastdeploy/model_executor/layers/backends/metax/attention/flash_attn_backend.py

@@ -89,7 +89,7 @@ def __init__(
         super().__init__()
         self.attention_metadata: FlashAttentionMetadata = None
         self.record_block_table_metadata = {}
-        self.block_size: int = fd_config.parallel_config.block_size
+        self.block_size: int = fd_config.cache_config.block_size
         self.max_seq_len: int = fd_config.model_config.max_model_len
         self.rope_theta: float = (
             10000.0 if fd_config.model_config.rope_theta is None else fd_config.model_config.rope_theta

fastdeploy/spec_decode/mtp.py

@@ -148,7 +148,7 @@ def initialize_kv_cache(self, main_model_num_blocks, profile: bool = False):
         self.cache_kvs = {}
 
         # Get kv cache dtype
-        cache_type = self.parallel_config.dtype
+        cache_type = self.model_config.dtype
         kv_cache_quant_type = None
         if (
             self.quant_config
@@ -383,8 +383,8 @@ def _init_model_inputs(self):
 
         self.free_list = list(
             range(
-                self.parallel_config.total_block_num - 1,
-                int(self.parallel_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
+                self.cache_config.total_block_num - 1,
+                int(self.cache_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
                 -1,
             )
         )

fastdeploy/worker/dcu_worker.py

@@ -53,7 +53,7 @@ def init_device(self):
             self.device_ids = self.parallel_config.device_ids.split(",")
             self.device = f"gpu:{self.local_rank % self.max_chips_per_node}"
             paddle.device.set_device(self.device)
-            paddle.set_default_dtype(self.parallel_config.dtype)
+            paddle.set_default_dtype(self.model_config.dtype)
 
             gc.collect()
             paddle.device.cuda.empty_cache()
@@ -128,7 +128,7 @@ def determine_available_memory(self) -> int:
         available_kv_cache_memory = (
             total_gpu_memory * self.cache_config.gpu_memory_utilization - after_used_gpu_memory - paddle_peak_increase
         )
-        available_kv_cache_memory += model_block_memory_used * self.parallel_config.total_block_num
+        available_kv_cache_memory += model_block_memory_used * self.cache_config.total_block_num
 
         end_time = time.perf_counter()
         logger.info(

fastdeploy/worker/gcu_model_runner.py

@@ -455,8 +455,8 @@ def _init_share_inputs(self, max_num_seqs: int):
         # Initialize free list
         free_list = list(
             range(
-                self.parallel_config.total_block_num - 1,
-                int(self.parallel_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
+                self.cache_config.total_block_num - 1,
+                int(self.cache_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
                 -1,
             )
         )
@@ -632,7 +632,7 @@ def initialize_kv_cache(self, profile: bool = False) -> None:
         max_block_num = self.num_gcu_blocks
 
         # Get kv cache dtype
-        cache_type = self.parallel_config.dtype
+        cache_type = self.model_config.dtype
 
         kv_cache_quant_type = None
         if (
@@ -844,7 +844,7 @@ def _dummy_run(
                 sampler_output=sampler_output,
                 model_output=model_output_data,
                 share_inputs=self.share_inputs,
-                block_size=self.parallel_config.block_size,
+                block_size=self.cache_config.block_size,
                 speculative_decoding=self.speculative_decoding,
                 skip_save_output=True,
             )
@@ -1076,7 +1076,7 @@ class at the server level, which is too granular for ModelRunner.
             sampler_output=sampler_output,
             model_output=model_output_data,
             share_inputs=self.share_inputs,
-            block_size=self.parallel_config.block_size,
+            block_size=self.cache_config.block_size,
             save_each_rank=self.parallel_config.use_ep,
             speculative_decoding=self.speculative_decoding,
             skip_save_output=skip_save_output,
@@ -1132,7 +1132,7 @@ def profile_run(self) -> None:
         """Execute a forward pass with dummy inputs to profile the memory usage of the model"""
 
         # Initialize kv cache for profile run. After profile run kv cache will be reset.
-        self.num_gcu_blocks = self.parallel_config.total_block_num
+        self.num_gcu_blocks = self.cache_config.total_block_num
         self.initialize_kv_cache(profile=True)
 
         # 1. Profile with multimodal encoder & encoder cache

fastdeploy/worker/gcu_worker.py

@@ -53,7 +53,7 @@ def init_device(self):
             self.device_ids = self.parallel_config.device_ids.split(",")
             self.device = f"gcu:{self.local_rank}"
             paddle.device.set_device(self.device)
-            paddle.set_default_dtype(self.parallel_config.dtype)
+            paddle.set_default_dtype(self.model_config.dtype)
             logger.info(f"GcuWorker init_device:{self.device}, device_ids:{self.device_ids}")
 
             gc.collect()

fastdeploy/worker/gpu_model_runner.py

@@ -920,8 +920,8 @@ def _init_share_inputs(self, max_num_seqs: int):
         # Initialize free list
         free_list = list(
             range(
-                self.parallel_config.total_block_num - 1,
-                int(self.parallel_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
+                self.cache_config.total_block_num - 1,
+                int(self.cache_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
                 -1,
             )
         )
@@ -1168,7 +1168,7 @@ def initialize_kv_cache(self, profile: bool = False) -> None:
         max_block_num = self.num_gpu_blocks
 
         # Get kv cache dtype
-        cache_type = self.parallel_config.dtype
+        cache_type = self.model_config.dtype
         kv_cache_quant_type = None
         if (
             self.quant_config
@@ -1936,7 +1936,7 @@ def profile_run(self) -> None:
         """Execute a forward pass with dummy inputs to profile the memory usage of the model"""
         # Initialize kv cache for profile run. After profile run kv cache will be reset.
         # TODO(gongshaotian): Optimize the management logic of kvcache
-        self.num_gpu_blocks = self.parallel_config.total_block_num
+        self.num_gpu_blocks = self.cache_config.total_block_num
         self.initialize_kv_cache(profile=True)
         if self.speculative_method in ["mtp"]:
             self.proposer.initialize_kv_cache(main_model_num_blocks=self.num_gpu_blocks, profile=True)
@@ -2158,7 +2158,7 @@ def extract_vision_features_ernie(self, inputs: list[paddle.Tensor]) -> paddle.T
             custom_black_list=self.amp_black,
             custom_white_list=self.amp_white,
             level="O2",
-            dtype=self.parallel_config.dtype,
+            dtype=self.model_config.dtype,
         ):
             image_features = self.model.vision_model.extract_feature(images, grid_thw)
             if self.parallel_config.tensor_parallel_size > 1:
@@ -2185,7 +2185,7 @@ def extract_vision_features_qwen(self, inputs: list[paddle.Tensor]) -> paddle.Te
             custom_black_list=self.amp_black,
             custom_white_list=self.amp_white,
             level="O2",
-            dtype=self.parallel_config.dtype,
+            dtype=self.model_config.dtype,
         ):
             image_features = self.model.visual.extract_feature(images, grid_thw)
 

fastdeploy/worker/gpu_worker.py

@@ -64,7 +64,7 @@ def init_device(self):
             self.device_ids = self.parallel_config.device_ids.split(",")
             self.device = f"gpu:{self.local_rank % self.max_chips_per_node}"
             paddle.device.set_device(self.device)
-            paddle.set_default_dtype(self.parallel_config.dtype)
+            paddle.set_default_dtype(self.model_config.dtype)
 
             gc.collect()
             paddle.device.cuda.empty_cache()
@@ -153,7 +153,7 @@ def determine_available_memory(self) -> int:
             - after_run_meminfo.used
             - paddle_peak_increase
         )
-        available_kv_cache_memory += model_block_memory_used * self.parallel_config.total_block_num
+        available_kv_cache_memory += model_block_memory_used * self.cache_config.total_block_num
 
         end_time = time.perf_counter()
         logger.info(

fastdeploy/worker/hpu_model_runner.py

@@ -654,8 +654,8 @@ def _init_share_inputs(self, max_num_seqs: int):
         # Initialize free list
         free_list = list(
             range(
-                self.parallel_config.total_block_num - 2,
-                int(self.parallel_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
+                self.cache_config.total_block_num - 2,
+                int(self.cache_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
                 -1,
             )
         )
@@ -712,7 +712,7 @@ def _prepare_inputs(self) -> None:
             self.share_inputs["seq_lens_encoder"],
             self.share_inputs["seq_lens_decoder"],
             self.cache_config.block_size,
-            self.parallel_config.dtype,
+            self.model_config.dtype,
         )
         is_prompt = is_prompt.item() == 1 if is_prompt.item() > 0 else None
         if is_prompt is True:
@@ -857,7 +857,7 @@ def initialize_kv_cache(self) -> None:
         kv_cache_shape = self.attn_backends[0].get_kv_cache_shape(max_num_blocks=max_block_num)
 
         for i in range(self.model_config.num_hidden_layers):
-            cache_type = self.parallel_config.dtype
+            cache_type = self.model_config.dtype
             cache_kvs["key_caches_{}".format(i)] = paddle.full(
                 shape=kv_cache_shape,
                 fill_value=0,
@@ -1375,7 +1375,7 @@ def profile_run(self) -> None:
 
         # Initialize kv cache for profile run. After profile run kv cache will be reset.
         # TODO(gongshaotian): Optimize the management logic of kvcache
-        self.num_gpu_blocks = self.parallel_config.total_block_num
+        self.num_gpu_blocks = self.cache_config.total_block_num
         self.initialize_kv_cache()
 
         # 1. Profile with multimodal encoder & encoder cache

fastdeploy/worker/hpu_worker.py

@@ -76,7 +76,7 @@ def init_device(self):
             intel_hpus_module_id = int(self.device_ids[self.local_rank])
             self.device = f"intel_hpu:{intel_hpus_module_id}"
             paddle.device.set_device(self.device)
-            paddle.set_default_dtype(self.parallel_config.dtype)
+            paddle.set_default_dtype(self.model_config.dtype)
 
             gc.collect()
             paddle.device.cuda.empty_cache()

fastdeploy/worker/iluvatar_worker.py

@@ -54,7 +54,7 @@ def init_device(self):
             # Set environment variable
             self.device = f"iluvatar_gpu:{self.local_rank}"
             paddle.device.set_device(self.device)
-            paddle.set_default_dtype(self.parallel_config.dtype)
+            paddle.set_default_dtype(self.model_config.dtype)
             self.device_ids = self.parallel_config.device_ids.split(",")
 
             gc.collect()
@@ -167,7 +167,7 @@ def initialize_kv_cache(self) -> None:
 
             self.get_profile_block_num_signal.value[self.local_rank] = num_blocks_global
         else:
-            num_blocks_global = self.fd_config.parallel_config.total_block_num
+            num_blocks_global = self.fd_config.cache_config.total_block_num
         # 4. init kv_cache with accurate num_blocks
         logger.info(f"------- num_blocks_global:{num_blocks_global} --------")
         self.worker.initialize_cache(num_gpu_blocks=num_blocks_global)

fastdeploy/worker/metax_model_runner.py

@@ -711,8 +711,8 @@ def _init_share_inputs(self, max_num_seqs: int):
         # Initialize free list
         free_list = list(
             range(
-                self.parallel_config.total_block_num - 1,
-                int(self.parallel_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
+                self.cache_config.total_block_num - 1,
+                int(self.cache_config.total_block_num * self.cache_config.kv_cache_ratio) - 1,
                 -1,
             )
         )
@@ -930,7 +930,7 @@ def initialize_kv_cache(self, profile: bool = False) -> None:
         max_block_num = self.num_gpu_blocks
 
         # Get kv cache dtype
-        cache_type = self.parallel_config.dtype
+        cache_type = self.model_config.dtype
 
         kv_cache_quant_type = None
         if (
@@ -1484,7 +1484,7 @@ def profile_run(self) -> None:
 
         # Initialize kv cache for profile run. After profile run kv cache will be reset.
         # TODO(gongshaotian): Optimize the management logic of kvcache
-        self.num_gpu_blocks = self.parallel_config.total_block_num
+        self.num_gpu_blocks = self.cache_config.total_block_num
         self.initialize_kv_cache(profile=True)
 
         # 1. Profile with multimodal encoder & encoder cache
@@ -1673,7 +1673,7 @@ def extract_vision_features(self, inputs: list[paddle.Tensor]) -> paddle.Tensor:
             custom_black_list=self.amp_black,
             custom_white_list=self.amp_white,
             level="O2",
-            dtype=self.parallel_config.dtype,
+            dtype=self.model_config.dtype,
         ):
             image_features = self.model.vision_model.extract_feature(images, grid_thw)
             if self.parallel_config.tensor_parallel_size > 1:

fastdeploy/worker/metax_worker.py

@@ -58,7 +58,7 @@ def init_device(self):
             self.device_ids = self.parallel_config.device_ids.split(",")
             self.device = f"metax_gpu:{self.local_rank % self.max_chips_per_node}"
             paddle.device.set_device(self.device)
-            paddle.set_default_dtype(self.parallel_config.dtype)
+            paddle.set_default_dtype(self.model_config.dtype)
 
             gc.collect()
 
@@ -149,7 +149,7 @@ def determine_available_memory(self) -> int:
             available_kv_cache_memory = (
                 after_run_meminfo_free - paddle_peak_increase
             ) * self.cache_config.gpu_memory_utilization
-            available_kv_cache_memory += model_block_memory_used * self.parallel_config.total_block_num
+            available_kv_cache_memory += model_block_memory_used * self.cache_config.total_block_num
 
             end_time = time.perf_counter()
 

fastdeploy/worker/worker_process.py

@@ -426,7 +426,7 @@ def initialize_kv_cache(self) -> None:
                 )
                 self.get_profile_block_num_signal.value[0] = num_blocks_local
         else:
-            num_blocks_local = self.fd_config.parallel_config.total_block_num
+            num_blocks_local = self.fd_config.cache_config.total_block_num
         logger.info(f"------- num_blocks_global: {num_blocks_local} --------")
 
         # 4. init kv_cache with accurate num_blocks