[None][fix] acceptance rate calculation fix in benchmark_serving (#6746)

Signed-off-by: Zero Zeng <[email protected]>

Author: zerollzeng

cpp/include/tensorrt_llm/executor/executor.h

@@ -860,6 +860,10 @@ struct Result
     /// one token can be generated per iteration. Used for speculative decoding statistics.
     SizeType32 decodingIter{0};
 
+    /// @brief The average number of decoded tokens per iteration. For standard model it is 1.
+    /// For speculative decoding model >= 1 -- number of draft tokens accepted per step + 1.
+    float avgDecodedTokensPerIter{0.0f};
+
     /// @brief The index of the output sequence of this result where 0 <= sequenceIndex < numReturnSequences.
     /// In beam search (beamWidth > 1), this index will be always zero because all beams to be returned are included
     /// in this result.

cpp/tensorrt_llm/batch_manager/llmRequest.cpp

@@ -200,6 +200,7 @@ std::optional<executor::Result> LlmRequest::createResult(bool useFastLogits, int
 
     result.finishReasons = sliceBeams(mFinishReasons);
     result.decodingIter = mDecodingIter;
+    result.avgDecodedTokensPerIter = getAvgDecodedTokensPerIter();
 
     if (hasAdditionalOutputs())
     {

cpp/tensorrt_llm/executor/serialization.cpp

@@ -895,6 +895,7 @@ Result Serialization::deserializeResult(std::istream& is)
     result.finishReasons = su::deserialize<std::vector<FinishReason>>(is);
     result.contextPhaseParams = su::deserialize<std::optional<ContextPhaseParams>>(is);
     result.decodingIter = su::deserialize<SizeType32>(is);
+    result.avgDecodedTokensPerIter = su::deserialize<float>(is);
     result.sequenceIndex = su::deserialize<SizeType32>(is);
     result.isSequenceFinal = su::deserialize<bool>(is);
     result.requestPerfMetrics = su::deserialize<std::optional<RequestPerfMetrics>>(is);
@@ -915,6 +916,7 @@ void Serialization::serialize(Result const& result, std::ostream& os)
     su::serialize(result.finishReasons, os);
     su::serialize(result.contextPhaseParams, os);
     su::serialize(result.decodingIter, os);
+    su::serialize(result.avgDecodedTokensPerIter, os);
     su::serialize(result.sequenceIndex, os);
     su::serialize(result.isSequenceFinal, os);
     su::serialize(result.requestPerfMetrics, os);
@@ -935,6 +937,7 @@ size_t Serialization::serializedSize(Result const& result)
     totalSize += su::serializedSize(result.finishReasons);
     totalSize += su::serializedSize(result.contextPhaseParams);
     totalSize += su::serializedSize(result.decodingIter);
+    totalSize += su::serializedSize(result.avgDecodedTokensPerIter);
     totalSize += su::serializedSize(result.sequenceIndex);
     totalSize += su::serializedSize(result.isSequenceFinal);
     totalSize += su::serializedSize(result.requestPerfMetrics);

cpp/tensorrt_llm/nanobind/executor/request.cpp

@@ -851,7 +851,7 @@ void initRequestBindings(nb::module_& m)
 
     auto resultSetstate = [](tle::Result& self, nb::tuple const& state)
     {
-        if (state.size() != 13)
+        if (state.size() != 14)
         {
             throw std::runtime_error("Invalid Request state!");
         }
@@ -867,16 +867,17 @@ void initRequestBindings(nb::module_& m)
         result.sequenceIndex = nb::cast<SizeType32>(state[8]);
         result.isSequenceFinal = nb::cast<bool>(state[9]);
         result.decodingIter = nb::cast<SizeType32>(state[10]);
-        result.contextPhaseParams = nb::cast<std::optional<tle::ContextPhaseParams>>(state[11]);
-        result.requestPerfMetrics = nb::cast<std::optional<tle::RequestPerfMetrics>>(state[12]);
+        result.avgDecodedTokensPerIter = nb::cast<float>(state[11]);
+        result.contextPhaseParams = nb::cast<std::optional<tle::ContextPhaseParams>>(state[12]);
+        result.requestPerfMetrics = nb::cast<std::optional<tle::RequestPerfMetrics>>(state[13]);
         new (&self) tle::Result(result);
     };
 
     auto resultGetstate = [](tle::Result const& self)
     {
         return nb::make_tuple(self.isFinal, self.outputTokenIds, self.cumLogProbs, self.logProbs, self.contextLogits,
             self.generationLogits, self.encoderOutput, self.finishReasons, self.sequenceIndex, self.isSequenceFinal,
-            self.decodingIter, self.contextPhaseParams, self.requestPerfMetrics);
+            self.decodingIter, self.avgDecodedTokensPerIter, self.contextPhaseParams, self.requestPerfMetrics);
     };
 
     nb::class_<tle::Result>(m, "Result")
@@ -893,6 +894,7 @@ void initRequestBindings(nb::module_& m)
         .def_rw("sequence_index", &tle::Result::sequenceIndex)
         .def_rw("is_sequence_final", &tle::Result::isSequenceFinal)
         .def_rw("decoding_iter", &tle::Result::decodingIter)
+        .def_rw("avg_decoded_tokens_per_iter", &tle::Result::avgDecodedTokensPerIter)
         .def_rw("context_phase_params", &tle::Result::contextPhaseParams)
         .def_rw("request_perf_metrics", &tle::Result::requestPerfMetrics)
         .def_rw("additional_outputs", &tle::Result::additionalOutputs)

cpp/tensorrt_llm/pybind/executor/request.cpp

@@ -795,7 +795,7 @@ void initRequestBindings(pybind11::module_& m)
 
     auto resultSetstate = [](py::tuple const& state)
     {
-        if (state.size() != 13)
+        if (state.size() != 14)
         {
             throw std::runtime_error("Invalid Request state!");
         }
@@ -811,16 +811,17 @@ void initRequestBindings(pybind11::module_& m)
         result.sequenceIndex = state[8].cast<SizeType32>();
         result.isSequenceFinal = state[9].cast<bool>();
         result.decodingIter = state[10].cast<SizeType32>();
-        result.contextPhaseParams = state[11].cast<std::optional<tle::ContextPhaseParams>>();
-        result.requestPerfMetrics = state[12].cast<std::optional<tle::RequestPerfMetrics>>();
+        result.avgDecodedTokensPerIter = state[11].cast<float>();
+        result.contextPhaseParams = state[12].cast<std::optional<tle::ContextPhaseParams>>();
+        result.requestPerfMetrics = state[13].cast<std::optional<tle::RequestPerfMetrics>>();
         return std::make_unique<tle::Result>(result);
     };
 
     auto resultGetstate = [](tle::Result const& self)
     {
         return py::make_tuple(self.isFinal, self.outputTokenIds, self.cumLogProbs, self.logProbs, self.contextLogits,
             self.generationLogits, self.encoderOutput, self.finishReasons, self.sequenceIndex, self.isSequenceFinal,
-            self.decodingIter, self.contextPhaseParams, self.requestPerfMetrics);
+            self.decodingIter, self.avgDecodedTokensPerIter, self.contextPhaseParams, self.requestPerfMetrics);
     };
 
     py::class_<tle::Result>(m, "Result")
@@ -837,6 +838,7 @@ void initRequestBindings(pybind11::module_& m)
         .def_readwrite("sequence_index", &tle::Result::sequenceIndex)
         .def_readwrite("is_sequence_final", &tle::Result::isSequenceFinal)
         .def_readwrite("decoding_iter", &tle::Result::decodingIter)
+        .def_readwrite("avg_decoded_tokens_per_iter", &tle::Result::avgDecodedTokensPerIter)
         .def_readwrite("context_phase_params", &tle::Result::contextPhaseParams)
         .def_readwrite("request_perf_metrics", &tle::Result::requestPerfMetrics)
         .def_readwrite("additional_outputs", &tle::Result::additionalOutputs)

tensorrt_llm/executor/result.py

@@ -158,6 +158,9 @@ def __init__(self,
         self.postproc_params = postproc_params
         self.disaggregated_params = None
         self.decoding_iter = 0
+        # Average decoded tokens per runtime iteration; set when the first LLM response arrives.
+        # None indicates not yet available (e.g., before first step/stream).
+        self.avg_decoded_tokens_per_iter: Optional[float] = None
         self._done = False
         self.metrics_dict = {}
 
@@ -331,6 +334,7 @@ def _handle_response(self,
             self._done = response_result.is_final
             context_phase_params = response_result.context_phase_params
             self.decoding_iter = response_result.decoding_iter
+            self.avg_decoded_tokens_per_iter = response_result.avg_decoded_tokens_per_iter
             if context_phase_params is not None:
                 self.disaggregated_params = DisaggregatedParams(
                     request_type="context_only",

tensorrt_llm/serve/openai_protocol.py

@@ -128,6 +128,7 @@ class CompletionResponseChoice(OpenAIBaseModel):
             "including encountering the EOS token"),
     )
     disaggregated_params: Optional[DisaggregatedParams] = Field(default=None)
+    avg_decoded_tokens_per_iter: Optional[float] = Field(default=None)
 
 
 class CompletionResponse(OpenAIBaseModel):
@@ -155,6 +156,7 @@ class CompletionResponseStreamChoice(OpenAIBaseModel):
             "to stop, None if the completion finished for some other reason "
             "including encountering the EOS token"),
     )
+    avg_decoded_tokens_per_iter: Optional[float] = Field(default=None)
 
 
 class CompletionStreamResponse(OpenAIBaseModel):
@@ -392,6 +394,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
     stop_reason: Optional[Union[int, str]] = None
 
     disaggregated_params: Optional[DisaggregatedParams] = Field(default=None)
+    avg_decoded_tokens_per_iter: Optional[float] = Field(default=None)
 
 
 class ChatCompletionResponse(OpenAIBaseModel):
@@ -419,6 +422,7 @@ class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
     logprobs: Optional[ChatCompletionLogProbs] = None
     finish_reason: Optional[str] = None
     stop_reason: Optional[Union[int, str]] = None
+    avg_decoded_tokens_per_iter: Optional[float] = Field(default=None)
 
 
 class ChatCompletionStreamResponse(OpenAIBaseModel):

tensorrt_llm/serve/postprocess_handlers.py

@@ -160,7 +160,8 @@ def yield_first_chat(num_tokens: int,
 
         choice = ChatCompletionResponseStreamChoice(index=i,
                                                     delta=delta_message,
-                                                    finish_reason=None)
+                                                    finish_reason=None,
+                                                    avg_decoded_tokens_per_iter=getattr(rsp, 'avg_decoded_tokens_per_iter', None))
         if args.return_logprobs:
             logprobs = output.logprobs_diff
             token_ids = output.token_ids_diff
@@ -224,6 +225,7 @@ def chat_response_post_processor(rsp: GenerationResultBase, args: ChatPostprocAr
             finish_reason=output.finish_reason,
             stop_reason=output.stop_reason,
             disaggregated_params=disaggregated_params,
+            avg_decoded_tokens_per_iter=getattr(rsp, 'avg_decoded_tokens_per_iter', None),
         )
 
         if args.return_logprobs:
@@ -293,6 +295,7 @@ def completion_stream_post_processor(rsp: DetokenizedGenerationResultBase, args:
             token_ids=None if args.detokenize else output.token_ids_diff,
             finish_reason = output.finish_reason,
             stop_reason = output.stop_reason,
+            avg_decoded_tokens_per_iter=getattr(rsp, 'avg_decoded_tokens_per_iter', None),
         )
         chunk = CompletionStreamResponse(model=args.model, choices=[choice])
         if include_continuous_usage:
@@ -337,6 +340,7 @@ def completion_response_post_processor(rsp: GenerationResult, args: CompletionPo
             context_logits=None if rsp.context_logits is None else rsp.context_logits.tolist(),
             stop_reason=output.stop_reason,
             finish_reason=output.finish_reason,
+            avg_decoded_tokens_per_iter=getattr(rsp, 'avg_decoded_tokens_per_iter', None),
         )
 
         completion_tokens += output.length

tensorrt_llm/serve/scripts/backend_request_func.py

@@ -45,7 +45,7 @@ class RequestFuncOutput:
     tpot: float = 0.0  # avg next-token latencies
     prompt_len: int = 0
     error: str = ""
-    decode_iteration: int = 0  # Number of decoding iterations
+    avg_decoded_tokens_per_iter: float = 0.0  # Average tokens decoded per iteration
 
 
 async def async_request_trt_llm(
@@ -82,7 +82,6 @@ async def async_request_trt_llm(
     ttft = 0.0
     st = time.perf_counter()
     most_recent_timestamp = st
-    decode_iteration_count = 0  # Track decoding iterations
     try:
         async with request_session.post(url=api_url, json=payload) as response:
             if response.status == 200:
@@ -108,22 +107,27 @@ async def async_request_trt_llm(
                         else:
                             output.itl.append(timestamp - most_recent_timestamp)
 
-                        # Increment decode iteration for each chunk
-                        decode_iteration_count += 1
                         most_recent_timestamp = timestamp
 
+                        # Extract avg_decoded_tokens_per_iter from TensorRT-LLM response
+                        if "avg_decoded_tokens_per_iter" in data:
+                            output.avg_decoded_tokens_per_iter = data[
+                                "avg_decoded_tokens_per_iter"]
+
                     output.latency = most_recent_timestamp - st
-                    output.decode_iteration = decode_iteration_count
+
                 else:
                     content = await response.content.read()
                     data = json.loads(content.decode())
                     output.ttft = -1
                     output.itl = []
                     output.generated_text = data["text_output"]
                     output.latency = time.perf_counter() - st
-                    # For non-streaming, estimate decode_iteration as number of output tokens
-                    output.decode_iteration = len(output.generated_text.split(
-                    )) if output.generated_text else 1
+
+                    # Extract avg_decoded_tokens_per_iter from non-streaming TensorRT-LLM response
+                    if "avg_decoded_tokens_per_iter" in data:
+                        output.avg_decoded_tokens_per_iter = data[
+                            "avg_decoded_tokens_per_iter"]
 
             else:
                 output.error = response.reason or ""
@@ -138,6 +142,7 @@ async def async_request_trt_llm(
 
     if pbar:
         pbar.update(1)
+
     return output
 
 
@@ -183,7 +188,6 @@ async def async_request_openai_completions(
     generated_text = ""
     st = time.perf_counter()
     most_recent_timestamp = st
-    decode_iteration_count = 0  # Track decoding iterations
     try:
         async with request_session.post(url=api_url,
                                         json=payload,
@@ -220,11 +224,13 @@ async def async_request_openai_completions(
                                     output.itl.append(timestamp -
                                                       most_recent_timestamp)
 
-                                # Increment decode iteration for each chunk with text
-                                if text is not None:
-                                    decode_iteration_count += 1
                                 most_recent_timestamp = timestamp
                                 generated_text += text or ""
+
+                                # Extract avg_decoded_tokens_per_iter from streaming response
+                                if "avg_decoded_tokens_per_iter" in choices[0]:
+                                    output.avg_decoded_tokens_per_iter = choices[
+                                        0]["avg_decoded_tokens_per_iter"]
                             elif usage := data.get("usage"):
                                 output.output_tokens = usage.get(
                                     "completion_tokens")
@@ -237,7 +243,6 @@ async def async_request_openai_completions(
                             "This response will be marked as failed!")
                     output.generated_text = generated_text
                     output.latency = most_recent_timestamp - st
-                    output.decode_iteration = decode_iteration_count
                 else:
                     content = await response.content.read()
                     data = json.loads(content.decode())
@@ -248,8 +253,11 @@ async def async_request_openai_completions(
                     output.ttft = -1
                     output.itl = []
                     output.output_tokens = data["usage"]["completion_tokens"]
-                    # For non-streaming, estimate decode_iteration as number of output tokens
-                    output.decode_iteration = output.output_tokens if output.output_tokens > 0 else 1
+                    # Extract avg_decoded_tokens_per_iter if available
+                    choice = data["choices"][0]
+                    if "avg_decoded_tokens_per_iter" in choice:
+                        output.avg_decoded_tokens_per_iter = choice[
+                            "avg_decoded_tokens_per_iter"]
             else:
                 output.error = response.reason or ""
                 output.success = False
@@ -263,6 +271,7 @@ async def async_request_openai_completions(
 
     if pbar:
         pbar.update(1)
+
     return output
 
 
@@ -322,7 +331,6 @@ async def async_request_openai_chat_completions(
     ttft = 0.0
     st = time.perf_counter()
     most_recent_timestamp = st
-    decode_iteration_count = 0  # Track decoding iterations
     try:
         async with request_session.post(url=api_url,
                                         json=payload,
@@ -353,10 +361,12 @@ async def async_request_openai_chat_completions(
                                     output.itl.append(timestamp -
                                                       most_recent_timestamp)
 
-                                # Increment decode iteration for each chunk with content
-                                if content is not None:
-                                    decode_iteration_count += 1
                                 generated_text += content or ""
+
+                                # Extract avg_decoded_tokens_per_iter from streaming chat response
+                                if "avg_decoded_tokens_per_iter" in choices[0]:
+                                    output.avg_decoded_tokens_per_iter = choices[
+                                        0]["avg_decoded_tokens_per_iter"]
                             elif usage := data.get("usage"):
                                 output.output_tokens = usage.get(
                                     "completion_tokens")
@@ -365,7 +375,6 @@ async def async_request_openai_chat_completions(
 
                     output.generated_text = generated_text
                     output.latency = most_recent_timestamp - st
-                    output.decode_iteration = decode_iteration_count
                 else:
                     content = await response.content.read()
                     data = json.loads(content.decode())
@@ -375,8 +384,12 @@ async def async_request_openai_chat_completions(
                     output.itl = []
                     output.latency = time.perf_counter() - st
                     output.ttft = -1
-                    # For non-streaming, estimate decode_iteration as number of output tokens
-                    output.decode_iteration = output.output_tokens if output.output_tokens > 0 else 1
+
+                    # Extract avg_decoded_tokens_per_iter if available
+                    choice = data["choices"][0]
+                    if "avg_decoded_tokens_per_iter" in choice:
+                        output.avg_decoded_tokens_per_iter = choice[
+                            "avg_decoded_tokens_per_iter"]
 
             else:
                 output.error = response.reason or ""
@@ -391,6 +404,7 @@ async def async_request_openai_chat_completions(
 
     if pbar:
         pbar.update(1)
+
     return output