From 7983364e3923654b526bc04a856dae6d400a117d Mon Sep 17 00:00:00 2001 From: Edwinhr716 Date: Tue, 3 Sep 2024 22:14:45 +0000 Subject: [PATCH] changes to support jetstream --- .../tools/profile-generator/container/benchmark_serving.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/benchmark/tools/profile-generator/container/benchmark_serving.py b/benchmarks/benchmark/tools/profile-generator/container/benchmark_serving.py index 6cefd2e56..8ae1109ee 100644 --- a/benchmarks/benchmark/tools/profile-generator/container/benchmark_serving.py +++ b/benchmarks/benchmark/tools/profile-generator/container/benchmark_serving.py @@ -181,7 +181,7 @@ async def send_request( elif backend == "jetstream": pload = { "prompt": prompt, - "max_tokens": 1, + "max_tokens": output_len, } else: raise ValueError(f"Unknown backend: {backend}")