From 3035c046d559546f5aa34ff7b66021b761f6083b Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 23 Oct 2024 12:47:20 -0400 Subject: [PATCH] [Misc] Separate total and output tokens in benchmark_throughput.py (#8914) Signed-off-by: Sumit Dubey --- benchmarks/benchmark_throughput.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/benchmark_throughput.py b/benchmarks/benchmark_throughput.py index 24eb54e7b73bc..ee41c8ea38382 100644 --- a/benchmarks/benchmark_throughput.py +++ b/benchmarks/benchmark_throughput.py @@ -272,8 +272,10 @@ def main(args: argparse.Namespace): raise ValueError(f"Unknown backend: {args.backend}") total_num_tokens = sum(prompt_len + output_len for _, prompt_len, output_len in requests) + total_output_tokens = sum(output_len for _, _, output_len in requests) print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, " - f"{total_num_tokens / elapsed_time:.2f} tokens/s") + f"{total_num_tokens / elapsed_time:.2f} total tokens/s, " + f"{total_output_tokens / elapsed_time:.2f} output tokens/s") # Output JSON results if specified if args.output_json: