diff --git a/fastdeploy/worker/gpu_model_runner.py b/fastdeploy/worker/gpu_model_runner.py index 7ff93415aad..c150f43e72e 100644 --- a/fastdeploy/worker/gpu_model_runner.py +++ b/fastdeploy/worker/gpu_model_runner.py @@ -3650,7 +3650,7 @@ def _get_prompt_logprobs_list( continue offset = self.share_inputs["cu_seqlens_q"][batch_id] prompt_hidden_states = hidden_states[offset : offset + num_logits] - logits = self.model.compute_logits(prompt_hidden_states) + logits = self.model.compute_logits(prompt_hidden_states, self.forward_meta) prompt_token_ids = request.prompt_token_ids[start_tok : start_tok + num_logits] prompt_token_ids_tensor = paddle.to_tensor(prompt_token_ids, dtype="int64") if logprobs_mode == "raw_logprobs": diff --git a/tests/worker/test_gpu_prompt_logprobs.py b/tests/worker/test_gpu_prompt_logprobs.py index f12bc4cf3dc..cc4e0120a8b 100644 --- a/tests/worker/test_gpu_prompt_logprobs.py +++ b/tests/worker/test_gpu_prompt_logprobs.py @@ -20,6 +20,7 @@ import numpy as np import paddle +from utils import MockForwardMeta from fastdeploy.config import ( CacheConfig, @@ -104,7 +105,7 @@ def __init__(self, vocab_size=128, hidden_size=128): self.hidden_size = hidden_size self.weight = paddle.rand([hidden_size, vocab_size], dtype="float32") - def compute_logits(self, x): + def compute_logits(self, x, forward_meta=None): return paddle.matmul(x.astype("float32"), self.weight) @@ -175,6 +176,7 @@ def setup_model_runner(self): model_runner.model = FakeModel(cfg.model_config.vocab_size, cfg.model_config.hidden_size) model_runner.in_progress_prompt_logprobs = {} + model_runner.forward_meta = MockForwardMeta() return model_runner