Merge branch 'develop' into pre

Author: zeroRains

fastdeploy/config.py

@@ -542,8 +542,6 @@ def __init__(
         self.block_size: int = 64
         # Engine worker queue port
         self.engine_worker_queue_port: str = "9923"
-        # Max model len
-        self.max_model_len: int = 3072  # max_seq_len
         # cuda visible devices
         self.device_ids: str = "0"
         # Input dtype
@@ -1402,7 +1400,6 @@ def __init__(
         plas_attention_config: PlasAttentionConfig = None,
         speculative_config: SpeculativeConfig = None,
         tokenizer: str = None,
-        max_model_len: int = 8192,
         ips: str = None,
         use_warmup: bool = False,
         limit_mm_per_prompt: Optional[Dict[str, Any]] = None,
@@ -1470,7 +1467,6 @@ def __init__(
                 if ip == self.host_ip:
                     self.node_rank = idx
 
-        self.max_model_len = max_model_len
         self.limit_mm_per_prompt = limit_mm_per_prompt
         self.mm_processor_kwargs = mm_processor_kwargs
         self.use_warmup = use_warmup
@@ -1534,20 +1530,20 @@ def postprocess(self):
         if self.scheduler_config.max_num_batched_tokens is None:
             if int(envs.ENABLE_V1_KVCACHE_SCHEDULER):
                 if paddle.is_compiled_with_xpu():
-                    self.scheduler_config.max_num_batched_tokens = self.max_model_len
+                    self.scheduler_config.max_num_batched_tokens = self.model_config.max_model_len
                 else:
                     self.scheduler_config.max_num_batched_tokens = 8192  # if set to max_model_len, it's easy to be OOM
             else:
                 if self.cache_config.enable_chunked_prefill:
                     self.scheduler_config.max_num_batched_tokens = 2048
                 else:
-                    self.scheduler_config.max_num_batched_tokens = self.max_model_len
+                    self.scheduler_config.max_num_batched_tokens = self.model_config.max_model_len
 
         if self.long_prefill_token_threshold == 0:
-            self.long_prefill_token_threshold = int(self.max_model_len * 0.04)
+            self.long_prefill_token_threshold = int(self.model_config.max_model_len * 0.04)
 
         self.cache_config.postprocess(self.scheduler_config.max_num_batched_tokens, self.scheduler_config.max_num_seqs)
-        self.cache_config.max_block_num_per_seq = int(self.max_model_len // self.cache_config.block_size)
+        self.cache_config.max_block_num_per_seq = int(self.model_config.max_model_len // self.cache_config.block_size)
         if self.model_config is not None and self.model_config.enable_mm:
             self.cache_config.enable_prefix_caching = False
 
@@ -1576,7 +1572,9 @@ def check(self):
             f"but now it's {self.scheduler_config.max_num_seqs}."
         )
         assert self.nnode >= 1, f"nnode: {self.nnode} should no less than 1"
-        assert self.max_model_len >= 16, f"max_model_len: {self.max_model_len} should be larger than 16"
+        assert (
+            self.model_config.max_model_len >= 16
+        ), f"max_model_len: {self.model_config.max_model_len} should be larger than 16"
         assert (
             self.scheduler_config.max_num_seqs >= 1
         ), f"max_num_seqs: {self.scheduler_config.max_num_seqs} should be larger than 1"
@@ -1585,10 +1583,11 @@ def check(self):
             f"should be larger than or equal to max_num_seqs: {self.scheduler_config.max_num_seqs}"
         )
         assert (
-            self.scheduler_config.max_num_batched_tokens <= self.max_model_len * self.scheduler_config.max_num_seqs
+            self.scheduler_config.max_num_batched_tokens
+            <= self.model_config.max_model_len * self.scheduler_config.max_num_seqs
         ), (
             f"max_num_batched_tokens: {self.scheduler_config.max_num_batched_tokens} should be larger"
-            f"than or equal to max_num_seqs: {self.scheduler_config.max_num_seqs} * max_model_len: {self.max_model_len}"
+            f"than or equal to max_num_seqs: {self.scheduler_config.max_num_seqs} * max_model_len: {self.model_config.max_model_len}"
         )
         assert (
             self.max_num_partial_prefills >= 1
@@ -1609,9 +1608,9 @@ def check(self):
 
         if not self.cache_config.enable_chunked_prefill:
             if not envs.ENABLE_V1_KVCACHE_SCHEDULER:
-                assert self.scheduler_config.max_num_batched_tokens >= self.max_model_len, (
+                assert self.scheduler_config.max_num_batched_tokens >= self.model_config.max_model_len, (
                     f"max_num_batched_tokens: {self.scheduler_config.max_num_batched_tokens} "
-                    f"should be larger than or equal to max_model_len: {self.max_model_len}"
+                    f"should be larger than or equal to max_model_len: {self.model_config.max_model_len}"
                 )
         else:
             assert self.scheduler_config.max_num_batched_tokens >= self.cache_config.block_size, (
@@ -1623,9 +1622,9 @@ def check(self):
             assert (
                 self.cache_config.enable_chunked_prefill is True
             ), "Chunked prefill must be enabled to set max_num_partial_prefills > 1"
-            assert self.long_prefill_token_threshold < self.max_model_len, (
+            assert self.long_prefill_token_threshold < self.model_config.max_model_len, (
                 f"long_prefill_token_threshold: {self.long_prefill_token_threshold} should be less than"
-                f" max_model_len: {self.max_model_len}"
+                f" max_model_len: {self.model_config.max_model_len}"
             )
 
         if self.guided_decoding_backend is not None:

fastdeploy/engine/args_utils.py

@@ -1079,7 +1079,6 @@ def create_engine_config(self) -> FDConfig:
             cache_config=cache_cfg,
             load_config=load_cfg,
             parallel_config=parallel_cfg,
-            max_model_len=self.max_model_len,
             speculative_config=speculative_cfg,
             ips=self.ips,
             use_warmup=self.use_warmup,

fastdeploy/engine/common_engine.py

@@ -630,7 +630,7 @@ def _fetch_request():
                     available_blocks=available_blocks,
                     block_size=self.cfg.cache_config.block_size,
                     reserved_output_blocks=self.cfg.cache_config.enc_dec_block_num,
-                    max_num_batched_tokens=self.cfg.max_model_len,
+                    max_num_batched_tokens=self.cfg.model_config.max_model_len,
                     batch=num_prefill_batch,
                 )
                 if self.cfg.scheduler_config.splitwise_role != "mixed":

fastdeploy/engine/engine.py

@@ -187,15 +187,15 @@ def check_worker_initialize_status_func(res: dict):
             num_gpu_blocks = self.cfg.cache_config.num_gpu_blocks_override or self.cfg.cache_config.total_block_num
             num_cpu_blocks = self.cfg.cache_config.num_cpu_blocks
             max_running_requests = min(
-                (num_gpu_blocks + num_cpu_blocks) * block_size // self.cfg.max_model_len,
+                (num_gpu_blocks + num_cpu_blocks) * block_size // self.cfg.model_config.max_model_len,
                 self.cfg.scheduler_config.max_num_seqs,
             )
             console_logger.info(
                 f"Detected {num_gpu_blocks} gpu blocks and {num_cpu_blocks} cpu blocks in cache (block size: {block_size})."
             )
             console_logger.info(
                 f"FastDeploy will be serving {max_running_requests} running requests "
-                f"if each sequence reaches its maximum length: {self.cfg.max_model_len}"
+                f"if each sequence reaches its maximum length: {self.cfg.model_config.max_model_len}"
             )
 
         return True
@@ -248,30 +248,28 @@ def add_requests(self, task, sampling_params=None, **kwargs):
         chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
         chat_template_kwargs["chat_template"] = kwargs.get("chat_template")
         kwargs["chat_template_kwargs"] = chat_template_kwargs
-        request = self.data_processor.process_request(request, self.cfg.max_model_len, **kwargs)
+        request = self.data_processor.process_request(request, self.cfg.model_config.max_model_len, **kwargs)
         request.prompt_token_ids_len = len(request.prompt_token_ids)
         request.need_prefill_tokens = request.prompt_token_ids_len
         input_ids_len = request.prompt_token_ids_len
         request.set(
             "max_tokens",
             min(
-                self.cfg.max_model_len - input_ids_len,
+                self.cfg.model_config.max_model_len - input_ids_len,
                 request.get("max_tokens"),
             ),
         )
         min_tokens = request.get("min_tokens")
-        if input_ids_len + min_tokens >= self.cfg.max_model_len:
+        if input_ids_len + min_tokens >= self.cfg.model_config.max_model_len:
             error_msg = (
                 f"Input text is too long, length of prompt token({input_ids_len}) "
                 f"+ min_dec_len ({min_tokens}) >= max_model_len "
             )
             llm_logger.error(error_msg)
             raise EngineError(error_msg, error_code=400)
 
-        if input_ids_len > self.cfg.max_model_len:
-            error_msg = (
-                f"Length of input token({input_ids_len}) exceeds the limit max_model_len({self.cfg.max_model_len})."
-            )
+        if input_ids_len > self.cfg.model_config.max_model_len:
+            error_msg = f"Length of input token({input_ids_len}) exceeds the limit max_model_len({self.cfg.model_config.max_model_len})."
             llm_logger.error(error_msg)
             raise EngineError(error_msg, error_code=400)
 
@@ -506,7 +504,7 @@ def _start_worker_service(self):
             ips = ",".join(self.cfg.ips)
         arguments = (
             f" --devices {self.cfg.device_ids} {py_script}"
-            f" --max_num_seqs {self.cfg.scheduler_config.max_num_seqs} --max_model_len {self.cfg.max_model_len}"
+            f" --max_num_seqs {self.cfg.scheduler_config.max_num_seqs} --max_model_len {self.cfg.model_config.max_model_len}"
             f" --gpu_memory_utilization {self.cfg.cache_config.gpu_memory_utilization}"
             f" --model {self.cfg.model_config.model!s}"
             f" --device_ids {self.cfg.device_ids}"
@@ -587,7 +585,7 @@ def _format_and_add_data(self, prompts: dict):
                     prompts["prompt"] = query_list
 
         if "max_tokens" not in prompts:
-            prompts["max_tokens"] = self.cfg.max_model_len
+            prompts["max_tokens"] = self.cfg.model_config.max_model_len
 
         self.add_requests(prompts)
         return prompts["request_id"]

fastdeploy/entrypoints/llm.py

@@ -93,7 +93,7 @@ def __init__(
         # Create the Engine
         self.llm_engine = LLMEngine.from_engine_args(engine_args=engine_args)
 
-        self.default_sampling_params = SamplingParams(max_tokens=self.llm_engine.cfg.max_model_len)
+        self.default_sampling_params = SamplingParams(max_tokens=self.llm_engine.cfg.model_config.max_model_len)
 
         self.llm_engine.start()
 

fastdeploy/model_executor/layers/attention/append_attn_backend.py

@@ -85,7 +85,7 @@ def __init__(
         super().__init__()
         self.attention_metadata: AppendAttentionMetadata = None
         self.block_size: int = fd_config.cache_config.block_size
-        self.max_seq_len: int = fd_config.parallel_config.max_model_len
+        self.max_seq_len: int = fd_config.model_config.max_model_len
         self.rope_theta: float = (
             10000.0 if fd_config.model_config.rope_theta is None else fd_config.model_config.rope_theta
         )

fastdeploy/model_executor/layers/attention/block_multihead_attn_backend.py

@@ -81,7 +81,7 @@ def __init__(
         super().__init__()
         self.attention_metadata: BlockAttentionMetadata = None
         self.block_size = fd_config.cache_config.block_size
-        self.max_seq_len = fd_config.parallel_config.max_model_len
+        self.max_seq_len = fd_config.model_config.max_model_len
         self.rope_theta = 10000.0 if fd_config.model_config.rope_theta is None else fd_config.model_config.rope_theta
         self.rank = fd_config.parallel_config.tensor_parallel_rank
 

fastdeploy/model_executor/layers/attention/flash_attn_backend.py

@@ -110,7 +110,7 @@ def __init__(
         """
         super().__init__()
         self.attention_metadata: FlashAttentionMetadata = None
-        self.max_seq_len = fd_config.parallel_config.max_model_len
+        self.max_seq_len = fd_config.model_config.max_model_len
         self.causal = getattr(fd_config.model_config, "causal", True)
 
         self.kv_num_heads = kv_num_heads

fastdeploy/model_executor/layers/attention/iluvatar_attn_backend.py

@@ -73,7 +73,7 @@ def __init__(self, fd_config: FDConfig, kv_num_heads: int, num_heads: int, head_
         self.attention_metadata = IluvatarAttentionMetadata()
         self.block_size = fd_config.parallel_config.block_size
         assert self.block_size == 16, "Iluvatar paged attn requires block_size must be 16."
-        self.max_context_len = fd_config.parallel_config.max_model_len
+        self.max_context_len = fd_config.model_config.max_model_len
         self.causal = getattr(fd_config.model_config, "causal", True)
         self.speculate_method = getattr(fd_config.parallel_config, "speculate_method", None)
         self.use_speculate = self.speculate_method is not None

fastdeploy/model_executor/layers/attention/mla_attention_backend.py

@@ -111,7 +111,7 @@ def __init__(
 
         # 基础配置
         self.block_size: int = fd_config.cache_config.block_size
-        self.max_seq_len: int = fd_config.parallel_config.max_model_len
+        self.max_seq_len: int = fd_config.model_config.max_model_len
         self.rope_theta: float = (
             10000.0 if fd_config.model_config.rope_theta is None else fd_config.model_config.rope_theta
         )