diff --git a/llm/sglang/index.html b/llm/sglang/index.html
index 38538280e..225b41773 100644
--- a/llm/sglang/index.html
+++ b/llm/sglang/index.html
@@ -328,7 +328,7 @@
 </head>
 <body>
 <h1 id="title">index.html</h1>
-<p>Report generated on 23-Jan-2025 at 22:39:37 by <a href="https://pypi.python.org/pypi/pytest-html">pytest-html</a>
+<p>Report generated on 05-Feb-2025 at 16:00:46 by <a href="https://pypi.python.org/pypi/pytest-html">pytest-html</a>
         v4.1.1</p>
 <div id="environment-header">
 <h2>Environment</h2>
@@ -382,7 +382,7 @@ <h2>Environment</h2>
 <h2>Summary</h2>
 <div class="additional-summary prefix">
 </div>
-<p class="run-count">18 tests ran in 4766 seconds</p>
+<p class="run-count">18 tests ran in 2755 seconds</p>
 <p class="filter">(Un)check the boxes to filter the results.</p>
 <div class="summary__reload">
 <div class="summary__reload__button hidden" onclick="location.reload()">
@@ -429,7 +429,7 @@ <h2>Summary</h2>
 </table>
 </body>
 <footer>
-<div data-jsonblob='{"environment": {"Python": "3.11.10", "Platform": "Linux-5.15.0-130-generic-x86_64-with-glibc2.35", "Packages": {"pytest": "8.0.0", "pluggy": "1.5.0"}, "Plugins": {"timeout": "2.3.1", "anyio": "4.8.0", "html": "4.1.1", "xdist": "3.5.0", "metadata": "3.1.1"}, "CI": "true"}, "tests": {"reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-1-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-1-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:13:19", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-1-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:13:19&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "---------------------------- Captured stdout setup -----------------------------\nDownloading dataset llama3_8B_fp16\n  gguf: [PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/meta-llama-3.1-8b-instruct.f16.gguf&amp;#x27;)]\n  tokenizer_config.json: [PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/tokenizer_config.json&amp;#x27;), PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/tokenizer.json&amp;#x27;)]\nExporting prefill_bs1\nExporting decode_bs1\nExporting prefill_bs4\nExporting decode_bs4\nGENERATED!\nExporting\nSaving to &amp;#x27;/tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/model.mlir&amp;#x27;\n\n---------------------------- Captured stderr setup -----------------------------\n/home/nod/actions-runner-shark-ai/_work/shark-ai/shark-ai/sharktank/sharktank/types/gguf_interop/base.py:100: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.)\n  data_tensor = torch.as_tensor(data.reshape(logical_shape))\n\n------------------------------ Captured log setup ------------------------------\nINFO     sglang_benchmarks.conftest:conftest.py:34 Preparing model artifacts...\n::group::Preparing model artifacts\nINFO     integration_tests.llm.utils:utils.py:50 Download model llama3_8B_fp16 with `hf_datasets` to /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0...\nINFO     integration_tests.llm.utils:utils.py:62 Model llama3_8B_fp16 successfully downloaded.\nINFO     integration_tests.llm.utils:utils.py:82 Exporting model with following settings:\n  MLIR Path: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/model.mlir\n  Config Path: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/config.json\n  Batch Sizes: 1,4\nINFO     integration_tests.llm.utils:utils.py:101 Model successfully exported to /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/model.mlir\nINFO     integration_tests.llm.utils:utils.py:105 Compiling model to /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/model.vmfb\nINFO     integration_tests.llm.utils:utils.py:116 Model successfully compiled to /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/model.vmfb\nINFO     sglang_benchmarks.conftest:conftest.py:61 Model artifacts setup successfully\n::endgroup::\nINFO     sglang_benchmarks.conftest:conftest.py:96 Saving edited config to: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/1_4_none.json\n\nINFO     sglang_benchmarks.conftest:conftest.py:97 Config: {\n  &amp;quot;module_name&amp;quot;: &amp;quot;module&amp;quot;,\n  &amp;quot;module_abi_version&amp;quot;: 1,\n  &amp;quot;max_seq_len&amp;quot;: 131072,\n  &amp;quot;attn_head_dim&amp;quot;: 128,\n  &amp;quot;prefill_batch_sizes&amp;quot;: [\n    1,\n    4\n  ],\n  &amp;quot;decode_batch_sizes&amp;quot;: [\n    1,\n    4\n  ],\n  &amp;quot;transformer_block_count&amp;quot;: 32,\n  &amp;quot;paged_kv_cache&amp;quot;: {\n    &amp;quot;attention_head_count_kv&amp;quot;: 8,\n    &amp;quot;block_seq_stride&amp;quot;: 16,\n    &amp;quot;device_block_count&amp;quot;: 256,\n    &amp;quot;prefix_sharing_algorithm&amp;quot;: &amp;quot;none&amp;quot;\n  }\n}\n\n----------------------------- Captured stderr call -----------------------------\n[2025-01-23 21:31:17] Started server process [2263577]\n[2025-01-23 21:31:17] Waiting for application startup.\n[2025-01-23 21:31:17] Application startup complete.\n[2025-01-23 21:31:17] Uvicorn running on http://0.0.0.0:33037 (Press CTRL+C to quit)\n[2025-01-23 21:31:17] 127.0.0.1:43044 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 21:31:21] 127.0.0.1:55884 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 21:33:25] 127.0.0.1:40648 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:25] 127.0.0.1:40658 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:26] 127.0.0.1:40664 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:26] 127.0.0.1:40668 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:27] 127.0.0.1:40674 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:27] 127.0.0.1:40682 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:27] 127.0.0.1:40686 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:27] 127.0.0.1:40688 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:28] 127.0.0.1:40692 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:33:28] 127.0.0.1:40554 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:10&amp;lt;01:30, 10.05s/it]\r 20%|\u2588\u2588        | 2/10 [00:12&amp;lt;00:46,  5.78s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:19&amp;lt;03:53, 33.39s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:39&amp;lt;02:49, 28.28s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:53&amp;lt;01:56, 23.20s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [02:00&amp;lt;01:10, 17.69s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [02:11&amp;lt;00:45, 15.32s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:15&amp;lt;00:23, 11.85s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:16&amp;lt;00:08,  8.46s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:32&amp;lt;00:00, 10.91s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:32&amp;lt;00:00, 15.30s/it]\n[2025-01-23 21:35:58] Shutting down\n[2025-01-23 21:35:58] Waiting for application shutdown.\n[2025-01-23 21:35:58] Application shutdown complete.\n[2025-01-23 21:35:58] Finished server process [2263577]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 33037\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 33037...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:33037...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0026304721832275 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 2.004680633544922 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith none...\n::group::Benchmark run with none algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: none\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:33037\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 280.21321749687195 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 1\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 783\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 97177.62285199715\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 115412.30743749475\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 1638.2293080023373\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.004617002559825778\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 18.13449059701092\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 152.96817879501032\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-2-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-2-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:41", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-2-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:41&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 21:35:59] Started server process [2264487]\n[2025-01-23 21:35:59] Waiting for application startup.\n[2025-01-23 21:35:59] Application startup complete.\n[2025-01-23 21:35:59] Uvicorn running on http://0.0.0.0:34737 (Press CTRL+C to quit)\n[2025-01-23 21:36:00] 127.0.0.1:37504 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 21:36:03] 127.0.0.1:37508 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 21:38:07] 127.0.0.1:60042 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:07] 127.0.0.1:60054 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:08] 127.0.0.1:36156 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:08] 127.0.0.1:36162 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:08] 127.0.0.1:36164 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:08] 127.0.0.1:36180 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:08] 127.0.0.1:36182 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:09] 127.0.0.1:36194 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:09] 127.0.0.1:36198 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:38:09] 127.0.0.1:36206 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:08&amp;lt;01:18,  8.72s/it]\r 20%|\u2588\u2588        | 2/10 [00:14&amp;lt;00:58,  7.28s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:02&amp;lt;02:58, 25.51s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:29&amp;lt;02:37, 26.26s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:34&amp;lt;01:33, 18.65s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:36&amp;lt;00:52, 13.02s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [02:03&amp;lt;00:52, 17.39s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:03&amp;lt;00:23, 11.95s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:38&amp;lt;00:19, 19.02s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:31&amp;lt;00:00, 29.50s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:31&amp;lt;00:00, 21.11s/it]\n[2025-01-23 21:41:39] Shutting down\n[2025-01-23 21:41:39] Waiting for application shutdown.\n[2025-01-23 21:41:39] Application shutdown complete.\n[2025-01-23 21:41:39] Finished server process [2264487]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 34737\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 34737...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:34737...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0031967163085938 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith none...\n::group::Benchmark run with none algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: none\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34737\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 338.46793603897095 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 2\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 288\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 97287.2840572978\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 95014.80709549651\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 2132.283509497938\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.0052679970394819975\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 13.14178316413758\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 211.0824661580118\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-4-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-4-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:37", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-4-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:37&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 21:41:40] Started server process [2265371]\n[2025-01-23 21:41:40] Waiting for application startup.\n[2025-01-23 21:41:40] Application startup complete.\n[2025-01-23 21:41:40] Uvicorn running on http://0.0.0.0:55739 (Press CTRL+C to quit)\n[2025-01-23 21:41:41] 127.0.0.1:44082 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 21:41:44] 127.0.0.1:44086 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 21:43:48] 127.0.0.1:60576 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:48] 127.0.0.1:41966 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:48] 127.0.0.1:41972 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:48] 127.0.0.1:41980 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:49] 127.0.0.1:41990 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:49] 127.0.0.1:41994 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:49] 127.0.0.1:42008 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:49] 127.0.0.1:42022 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:49] 127.0.0.1:42038 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:43:49] 127.0.0.1:42044 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:07&amp;lt;01:06,  7.39s/it]\r 20%|\u2588\u2588        | 2/10 [00:16&amp;lt;01:05,  8.19s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:00&amp;lt;02:53, 24.81s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:37&amp;lt;02:56, 29.48s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:37&amp;lt;01:34, 18.99s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:42&amp;lt;00:56, 14.23s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [02:01&amp;lt;00:47, 15.83s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:17&amp;lt;00:31, 15.66s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:35&amp;lt;00:16, 16.55s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:27&amp;lt;00:00, 27.31s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:27&amp;lt;00:00, 20.71s/it]\n[2025-01-23 21:47:15] Shutting down\n[2025-01-23 21:47:15] Waiting for application shutdown.\n[2025-01-23 21:47:15] Application shutdown complete.\n[2025-01-23 21:47:15] Finished server process [2265371]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 55739\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 55739...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:55739...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0040998458862305 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith none...\n::group::Benchmark run with none algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: none\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:55739\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 334.49570298194885 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 4\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 669\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 99891.01795130264\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 100199.156841496\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3204.689010497532\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.0040660088416188955\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 13.394123127945914\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 207.10575627099024\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-8-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-8-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:49", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-8-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:49&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 21:47:17] Started server process [2266200]\n[2025-01-23 21:47:17] Waiting for application startup.\n[2025-01-23 21:47:17] Application startup complete.\n[2025-01-23 21:47:17] Uvicorn running on http://0.0.0.0:38567 (Press CTRL+C to quit)\n[2025-01-23 21:47:18] 127.0.0.1:34104 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 21:47:21] 127.0.0.1:37478 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 21:49:25] 127.0.0.1:42242 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42258 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42266 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42282 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42286 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42300 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42306 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42322 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42336 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:49:25] 127.0.0.1:42346 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:07&amp;lt;01:10,  7.82s/it]\r 20%|\u2588\u2588        | 2/10 [00:16&amp;lt;01:05,  8.24s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:16&amp;lt;03:44, 32.12s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:29&amp;lt;02:26, 24.34s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:42&amp;lt;01:42, 20.49s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:45&amp;lt;00:57, 14.44s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:58&amp;lt;00:41, 13.78s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:14&amp;lt;00:29, 14.75s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [03:05&amp;lt;00:26, 26.11s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:39&amp;lt;00:00, 28.36s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:39&amp;lt;00:00, 21.94s/it]\n[2025-01-23 21:53:04] Shutting down\n[2025-01-23 21:53:05] Waiting for application shutdown.\n[2025-01-23 21:53:05] Application shutdown complete.\n[2025-01-23 21:53:05] Finished server process [2266200]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 38567\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 38567...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:38567...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0031073093414307 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith none...\n::group::Benchmark run with none algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: none\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:38567\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 346.75700187683105 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 8\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 1262\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 105474.1795777998\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 104079.61580449773\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3428.6042930034455\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.005879002856090665\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 12.644388244709832\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 219.38586085100542\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-16-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-16-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:59", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-16-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:59&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 21:53:06] Started server process [2267140]\n[2025-01-23 21:53:06] Waiting for application startup.\n[2025-01-23 21:53:06] Application startup complete.\n[2025-01-23 21:53:06] Uvicorn running on http://0.0.0.0:52139 (Press CTRL+C to quit)\n[2025-01-23 21:53:07] 127.0.0.1:49390 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 21:53:10] 127.0.0.1:48148 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 21:55:14] 127.0.0.1:57812 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57814 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57816 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57824 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57840 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57854 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57858 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57864 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57878 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 21:55:14] 127.0.0.1:57888 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:07&amp;lt;01:07,  7.45s/it]\r 20%|\u2588\u2588        | 2/10 [00:12&amp;lt;00:46,  5.79s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:11&amp;lt;03:31, 30.28s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:13&amp;lt;01:55, 19.25s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:44&amp;lt;01:57, 23.42s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:47&amp;lt;01:06, 16.53s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:48&amp;lt;00:34, 11.44s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:34&amp;lt;00:44, 22.25s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [03:12&amp;lt;00:27, 27.19s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:49&amp;lt;00:00, 30.34s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:49&amp;lt;00:00, 22.96s/it]\n[2025-01-23 21:59:04] Shutting down\n[2025-01-23 21:59:04] Waiting for application shutdown.\n[2025-01-23 21:59:04] Application shutdown complete.\n[2025-01-23 21:59:04] Finished server process [2267140]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 52139\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 52139...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:52139...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0028071403503418 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith none...\n::group::Benchmark run with none algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: none\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:52139\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 356.99551153182983 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 16\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 513\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 106083.90161539719\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 106121.8441154997\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3484.9562124945805\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.004215995431877673\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 12.080794735125647\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 229.62065499999153\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-32-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-32-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:42", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model0-write_config0-32-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:42&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 21:59:06] Started server process [2268114]\n[2025-01-23 21:59:06] Waiting for application startup.\n[2025-01-23 21:59:06] Application startup complete.\n[2025-01-23 21:59:06] Uvicorn running on http://0.0.0.0:45851 (Press CTRL+C to quit)\n[2025-01-23 21:59:06] 127.0.0.1:40256 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 21:59:09] 127.0.0.1:40384 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:01:13] 127.0.0.1:34256 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34262 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34270 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34280 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34284 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34296 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34310 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34322 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34328 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:01:13] 127.0.0.1:34342 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:07&amp;lt;01:10,  7.88s/it]\r 20%|\u2588\u2588        | 2/10 [00:13&amp;lt;00:52,  6.60s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:16&amp;lt;03:45, 32.19s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:25&amp;lt;02:18, 23.02s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:29&amp;lt;01:21, 16.40s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:50&amp;lt;01:10, 17.71s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:52&amp;lt;00:38, 12.73s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:44&amp;lt;00:50, 25.19s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [03:07&amp;lt;00:24, 24.52s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:31&amp;lt;00:00, 24.48s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:31&amp;lt;00:00, 21.19s/it]\n[2025-01-23 22:04:45] Shutting down\n[2025-01-23 22:04:46] Waiting for application shutdown.\n[2025-01-23 22:04:46] Application shutdown complete.\n[2025-01-23 22:04:46] Finished server process [2268114]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 45851\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 45851...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:45851...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0030596256256104 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith none...\n::group::Benchmark run with none algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: none\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:45851\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 339.27814769744873 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 32\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 392\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 105857.901618701\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 99914.73780549859\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3533.721013001923\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.0050670059863477945\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 13.091711133221065\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 211.88979590000235\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-1-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-1-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:41", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-1-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:41&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "------------------------------ Captured log setup ------------------------------\nINFO     sglang_benchmarks.conftest:conftest.py:34 Preparing model artifacts...\n::group::Preparing model artifacts\nINFO     sglang_benchmarks.conftest:conftest.py:40 Reusing existing model artifacts directory: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\n::endgroup::\nINFO     sglang_benchmarks.conftest:conftest.py:96 Saving edited config to: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0/1_4_trie.json\n\nINFO     sglang_benchmarks.conftest:conftest.py:97 Config: {\n  &amp;quot;module_name&amp;quot;: &amp;quot;module&amp;quot;,\n  &amp;quot;module_abi_version&amp;quot;: 1,\n  &amp;quot;max_seq_len&amp;quot;: 131072,\n  &amp;quot;attn_head_dim&amp;quot;: 128,\n  &amp;quot;prefill_batch_sizes&amp;quot;: [\n    1,\n    4\n  ],\n  &amp;quot;decode_batch_sizes&amp;quot;: [\n    1,\n    4\n  ],\n  &amp;quot;transformer_block_count&amp;quot;: 32,\n  &amp;quot;paged_kv_cache&amp;quot;: {\n    &amp;quot;attention_head_count_kv&amp;quot;: 8,\n    &amp;quot;block_seq_stride&amp;quot;: 16,\n    &amp;quot;device_block_count&amp;quot;: 256,\n    &amp;quot;prefix_sharing_algorithm&amp;quot;: &amp;quot;trie&amp;quot;\n  }\n}\n\n----------------------------- Captured stderr call -----------------------------\n[2025-01-23 22:04:47] Started server process [2269124]\n[2025-01-23 22:04:47] Waiting for application startup.\n[2025-01-23 22:04:47] Application startup complete.\n[2025-01-23 22:04:47] Uvicorn running on http://0.0.0.0:54847 (Press CTRL+C to quit)\n[2025-01-23 22:04:48] 127.0.0.1:47958 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 22:04:51] 127.0.0.1:34012 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:06:55] 127.0.0.1:58834 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:55] 127.0.0.1:58836 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:57] 127.0.0.1:58846 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:57] 127.0.0.1:58852 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:57] 127.0.0.1:58856 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:57] 127.0.0.1:58858 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:57] 127.0.0.1:58860 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:58] 127.0.0.1:58874 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:58] 127.0.0.1:58884 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:06:58] 127.0.0.1:45868 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:07&amp;lt;01:07,  7.53s/it]\r 20%|\u2588\u2588        | 2/10 [00:16&amp;lt;01:08,  8.54s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:23&amp;lt;04:04, 34.99s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:27&amp;lt;02:16, 22.73s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:35&amp;lt;01:28, 17.70s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:37&amp;lt;00:48, 12.19s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:53&amp;lt;00:40, 13.51s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:16&amp;lt;00:32, 16.35s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:33&amp;lt;00:16, 16.70s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:31&amp;lt;00:00, 29.45s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:31&amp;lt;00:00, 21.16s/it]\n[2025-01-23 22:10:27] Shutting down\n[2025-01-23 22:10:27] Waiting for application shutdown.\n[2025-01-23 22:10:27] Application shutdown complete.\n[2025-01-23 22:10:27] Finished server process [2269124]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 54847\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 54847...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:54847...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0030343532562256 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith trie...\n::group::Benchmark run with trie algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: trie\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:54847\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 338.9017779827118 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 1\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 1713\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 98259.67320930067\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 95100.52514250128\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 1636.93795799918\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.015403013094328344\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 13.112159061131436\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 211.5593615869875\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-2-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-2-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:01", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-2-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:01&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 22:10:28] Started server process [2270092]\n[2025-01-23 22:10:28] Waiting for application startup.\n[2025-01-23 22:10:28] Application startup complete.\n[2025-01-23 22:10:28] Uvicorn running on http://0.0.0.0:48021 (Press CTRL+C to quit)\n[2025-01-23 22:10:29] 127.0.0.1:45442 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 22:10:32] 127.0.0.1:45454 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:12:36] 127.0.0.1:55208 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:36] 127.0.0.1:55218 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:37] 127.0.0.1:55232 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:37] 127.0.0.1:55244 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:37] 127.0.0.1:55256 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:37] 127.0.0.1:55272 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:37] 127.0.0.1:55276 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:37] 127.0.0.1:55282 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:38] 127.0.0.1:55286 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:12:38] 127.0.0.1:55288 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:08&amp;lt;01:16,  8.47s/it]\r 20%|\u2588\u2588        | 2/10 [00:14&amp;lt;00:57,  7.13s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:08&amp;lt;03:19, 28.48s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:16&amp;lt;02:02, 20.37s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:32&amp;lt;01:33, 18.77s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:34&amp;lt;00:52, 13.24s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:54&amp;lt;00:46, 15.37s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:16&amp;lt;00:34, 17.30s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:43&amp;lt;00:20, 20.48s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:51&amp;lt;00:00, 16.59s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:51&amp;lt;00:00, 17.15s/it]\n[2025-01-23 22:15:28] Shutting down\n[2025-01-23 22:15:28] Waiting for application shutdown.\n[2025-01-23 22:15:28] Application shutdown complete.\n[2025-01-23 22:15:28] Finished server process [2270092]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 48021\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 48021...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:48021...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0028419494628906 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith trie...\n::group::Benchmark run with trie algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: trie\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:48021\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 298.80367970466614 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 2\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 817\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 93109.19306159922\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 92985.39121950307\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 2132.973885498359\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.008502000127919018\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 16.177383153448535\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 171.473962981996\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-4-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-4-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:54", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-4-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:54&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 22:15:30] Started server process [2271019]\n[2025-01-23 22:15:30] Waiting for application startup.\n[2025-01-23 22:15:30] Application startup complete.\n[2025-01-23 22:15:30] Uvicorn running on http://0.0.0.0:41391 (Press CTRL+C to quit)\n[2025-01-23 22:15:30] 127.0.0.1:33626 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 22:15:33] 127.0.0.1:33640 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:17:37] 127.0.0.1:34258 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:37] 127.0.0.1:34264 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:34266 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:34276 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:34284 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:34290 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:34306 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:34308 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:58812 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:17:38] 127.0.0.1:58820 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:08&amp;lt;01:16,  8.49s/it]\r 20%|\u2588\u2588        | 2/10 [00:14&amp;lt;00:54,  6.76s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:17&amp;lt;03:49, 32.78s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:33&amp;lt;02:36, 26.14s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:42&amp;lt;01:39, 19.82s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:48&amp;lt;01:00, 15.11s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:57&amp;lt;00:39, 13.15s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:13&amp;lt;00:28, 14.13s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:20&amp;lt;00:11, 11.74s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:44&amp;lt;00:00, 34.19s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:44&amp;lt;00:00, 22.46s/it]\n[2025-01-23 22:21:22] Shutting down\n[2025-01-23 22:21:22] Waiting for application shutdown.\n[2025-01-23 22:21:22] Application shutdown complete.\n[2025-01-23 22:21:22] Finished server process [2271019]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 41391\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 41391...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:41391...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0030896663665771 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith trie...\n::group::Benchmark run with trie algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: trie\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:41391\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 351.9727318286896 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 4\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 700\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 101549.33688339661\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 104825.13219099928\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3207.0858549996046\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.008251998224295676\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 12.349306406454254\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 224.62800004299788\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-8-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-8-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:06:17", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-8-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:06:17&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 22:21:24] Started server process [2271978]\n[2025-01-23 22:21:24] Waiting for application startup.\n[2025-01-23 22:21:24] Application startup complete.\n[2025-01-23 22:21:24] Uvicorn running on http://0.0.0.0:42935 (Press CTRL+C to quit)\n[2025-01-23 22:21:24] 127.0.0.1:34390 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 22:21:28] 127.0.0.1:34402 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:23:32] 127.0.0.1:57556 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57568 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57576 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57588 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57600 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57602 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57606 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57612 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57628 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:23:32] 127.0.0.1:57638 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:07&amp;lt;01:04,  7.12s/it]\r 20%|\u2588\u2588        | 2/10 [00:13&amp;lt;00:53,  6.70s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:30&amp;lt;04:32, 38.90s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:42&amp;lt;02:47, 28.00s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:49&amp;lt;01:43, 20.74s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:52&amp;lt;00:58, 14.72s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:54&amp;lt;00:31, 10.46s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:23&amp;lt;00:32, 16.22s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:57&amp;lt;00:21, 21.79s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [04:07&amp;lt;00:00, 36.65s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [04:07&amp;lt;00:00, 24.71s/it]\n[2025-01-23 22:27:39] Shutting down\n[2025-01-23 22:27:39] Waiting for application shutdown.\n[2025-01-23 22:27:39] Application shutdown complete.\n[2025-01-23 22:27:39] Finished server process [2271978]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 42935\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 42935...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:42935...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0030450820922852 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith trie...\n::group::Benchmark run with trie algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: trie\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:42935\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 374.50862979888916 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 8\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 1230\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 111582.68981519941\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 111064.2297930026\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3468.1079930014675\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.0044470070861279964\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 11.22448250509124\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 247.13834234600654\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-16-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-16-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:06:01", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-16-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:06:01&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 22:27:41] Started server process [2272950]\n[2025-01-23 22:27:41] Waiting for application startup.\n[2025-01-23 22:27:41] Application startup complete.\n[2025-01-23 22:27:41] Uvicorn running on http://0.0.0.0:57191 (Press CTRL+C to quit)\n[2025-01-23 22:27:41] 127.0.0.1:43478 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 22:27:44] 127.0.0.1:43484 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:29:48] 127.0.0.1:40158 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:48] 127.0.0.1:40174 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40186 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40188 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40200 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40204 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40208 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40212 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40216 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:29:49] 127.0.0.1:40228 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:08&amp;lt;01:14,  8.23s/it]\r 20%|\u2588\u2588        | 2/10 [00:15&amp;lt;01:00,  7.52s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:15&amp;lt;00:29,  4.15s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:25&amp;lt;03:00, 30.16s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:33&amp;lt;01:51, 22.28s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:33&amp;lt;00:59, 14.76s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:41&amp;lt;00:37, 12.55s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:15&amp;lt;00:38, 19.39s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:27&amp;lt;00:16, 16.93s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:50&amp;lt;00:00, 37.47s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:50&amp;lt;00:00, 23.09s/it]\n[2025-01-23 22:33:40] Shutting down\n[2025-01-23 22:33:40] Waiting for application shutdown.\n[2025-01-23 22:33:40] Application shutdown complete.\n[2025-01-23 22:33:40] Finished server process [2272950]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 57191\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 57191...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:57191...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.003800392150879 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith trie...\n::group::Benchmark run with trie algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: trie\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:57191\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 358.2875771522522 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 16\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 473\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 92683.78119230329\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 93754.23635100015\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3521.3557185052196\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.0029949878808110952\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 12.013267842921307\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 230.9113586970052\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-32-meta-llama-3.1-8b-instruct.f16.gguf]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-32-meta-llama-3.1-8b-instruct.f16.gguf]", "duration": "00:05:57", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[pre_process_model1-write_config1-32-meta-llama-3.1-8b-instruct.f16.gguf]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:05:57&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-01-23 22:33:41] Started server process [2273916]\n[2025-01-23 22:33:41] Waiting for application startup.\n[2025-01-23 22:33:41] Application startup complete.\n[2025-01-23 22:33:41] Uvicorn running on http://0.0.0.0:47277 (Press CTRL+C to quit)\n[2025-01-23 22:33:42] 127.0.0.1:52720 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\nSpecial tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n[2025-01-23 22:33:45] 127.0.0.1:52734 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-01-23 22:35:49] 127.0.0.1:38410 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38412 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38428 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38430 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38438 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38450 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38452 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38460 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38466 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-01-23 22:35:49] 127.0.0.1:38468 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:08&amp;lt;01:12,  8.08s/it]\r 20%|\u2588\u2588        | 2/10 [00:17&amp;lt;01:11,  8.91s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [01:27&amp;lt;04:18, 36.90s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [01:42&amp;lt;02:49, 28.23s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [01:49&amp;lt;01:42, 20.57s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [01:51&amp;lt;00:57, 14.33s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [01:59&amp;lt;00:35, 11.98s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [02:20&amp;lt;00:29, 14.97s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:34&amp;lt;00:14, 14.67s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:47&amp;lt;00:00, 32.66s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [03:47&amp;lt;00:00, 22.74s/it]\n[2025-01-23 22:39:37] Shutting down\n[2025-01-23 22:39:37] Waiting for application shutdown.\n[2025-01-23 22:39:37] Application shutdown complete.\n[2025-01-23 22:39:37] Finished server process [2273916]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:123 Finding available port...\nINFO     integration_tests.llm.utils:utils.py:128 Found available port: 47277\nINFO     integration_tests.llm.utils:utils.py:181 Starting LLM server on port 47277...\nINFO     integration_tests.llm.utils:utils.py:209 Process started... waiting for server\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:47277...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0030663013458252 seconds; timeout: 30 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:109 Starting benchmark run with prefix sharing algorith trie...\n::group::Benchmark run with trie algorithm\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:113 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:114 Prefix sharing algorith: trie\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:115 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:47277\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-19/sglang_benchmark_test0\nRequest Rate: 32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:126 Benchmark run completed in 354.8037631511688 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:127 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 32\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 31\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 107846.08779090049\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 110803.83743900165\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 3523.0800684948917\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 0.0029940129024907947\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 12.199121698372581\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 227.39341967299697\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:129 Benchmark run successful\n::endgroup::\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[1-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[1-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:58", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[1-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:58&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:21,  2.36s/it]\r 20%|\u2588\u2588        | 2/10 [00:02&amp;lt;00:09,  1.13s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:04&amp;lt;00:09,  1.33s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:05&amp;lt;00:07,  1.20s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:05&amp;lt;00:05,  1.00s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:08&amp;lt;00:06,  1.67s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:09&amp;lt;00:03,  1.22s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:09&amp;lt;00:01,  1.11it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:10&amp;lt;00:01,  1.06s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:11&amp;lt;00:00,  1.07it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:11&amp;lt;00:00,  1.14s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:68 Preparing tokenizer_path: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test0/tokenizer.json...\nINFO     integration_tests.llm.utils:utils.py:70 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     integration_tests.llm.utils:utils.py:75 Tokenizer saved to /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test0/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Beginning SGLang benchmark test...\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:30000...\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 0 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 1.0025370121002197 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 2.004657745361328 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 3.0069961547851562 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 4.00969934463501 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 5.013218879699707 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 6.015554189682007 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 7.0184855461120605 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 8.021502494812012 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 9.023715496063232 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 10.025827646255493 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 11.029221534729004 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 12.031790256500244 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 13.03456974029541 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 14.037696123123169 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 15.040669679641724 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 16.043336868286133 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 17.046671628952026 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 18.049577951431274 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 19.051741123199463 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 20.054457426071167 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 21.05715012550354 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 22.058915376663208 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 23.062137126922607 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 24.065064907073975 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 25.06788992881775 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 26.070701837539673 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 27.073627948760986 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 28.076531648635864 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 29.079780340194702 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 30.082648992538452 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 31.085580348968506 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 32.08847498893738 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 33.091328144073486 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 34.09422779083252 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:142 Server has not started yet; waited 35.097623348236084 seconds; timeout: 600 seconds.\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:55 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test0\nRequest Rate: 1\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test0&amp;#x27;, request_rate=1, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test0/sglang_10_1.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    1         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  11.38     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              0.88      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          172.20    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         243.71    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          415.90    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   4915.22   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 5447.59   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          57.84     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        44.11     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           117.89    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          17.58     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        17.64     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           18.46     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           17.57     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         17.79     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            36.78     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:63 Benchmark run completed in 21.1195867061615 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:64 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 1\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 4915.222553801141\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 5447.593403499923\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 44.10843150981236\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 17.792589002056047\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 243.70884888498173\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 11.382434461003868\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[2-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[2-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:19", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[2-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:19&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:01&amp;lt;00:12,  1.41s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:04&amp;lt;00:09,  1.43s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:04&amp;lt;00:06,  1.12s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:04&amp;lt;00:03,  1.26it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:07&amp;lt;00:05,  1.41s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:07&amp;lt;00:03,  1.00s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:09&amp;lt;00:00,  1.20it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:10&amp;lt;00:00,  1.11it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:10&amp;lt;00:00,  1.01s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:68 Preparing tokenizer_path: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test1/tokenizer.json...\nINFO     integration_tests.llm.utils:utils.py:70 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     integration_tests.llm.utils:utils.py:75 Tokenizer saved to /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test1/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Beginning SGLang benchmark test...\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:30000...\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:55 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test1\nRequest Rate: 2\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test1&amp;#x27;, request_rate=2, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test1/sglang_10_2.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    2         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  10.15     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              0.99      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          193.15    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         273.37    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          466.52    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   4909.05   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 5468.32   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          29.92     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        31.71     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           35.80     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          18.10     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        17.94     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           19.96     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           17.65     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         17.89     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            33.64     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:63 Benchmark run completed in 18.27428889274597 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:64 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 2\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 4909.046602700255\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 5468.31933100475\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 31.71354349615285\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 17.890700495627243\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 273.36852024610357\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 10.147474176992546\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[4-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[4-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:18", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[4-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:18&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:07,  1.24it/s]\r 20%|\u2588\u2588        | 2/10 [00:01&amp;lt;00:03,  2.20it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:04&amp;lt;00:12,  1.78s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:04&amp;lt;00:04,  1.08it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:07&amp;lt;00:05,  1.28s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:08&amp;lt;00:00,  1.28it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.11it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.05it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:68 Preparing tokenizer_path: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test2/tokenizer.json...\nINFO     integration_tests.llm.utils:utils.py:70 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     integration_tests.llm.utils:utils.py:75 Tokenizer saved to /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test2/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Beginning SGLang benchmark test...\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:30000...\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:55 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test2\nRequest Rate: 4\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test2&amp;#x27;, request_rate=4, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test2/sglang_10_4.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    4         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  9.56      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.05      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          205.05    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         290.21    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          495.26    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   4949.92   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 5543.78   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          29.91     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        29.44     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           46.73     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          18.40     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        18.04     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           21.03     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           17.80     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         17.93     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            33.36     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:63 Benchmark run completed in 17.685285806655884 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:64 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 4\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 4949.921798599826\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 5543.7764119997155\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 29.437801502353977\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 17.928850997122936\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 290.20736142162116\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 9.558682406990556\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[8-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[8-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:18", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[8-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:18&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:04,  1.91it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:02,  2.70it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:04&amp;lt;00:12,  1.74s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:04&amp;lt;00:07,  1.18s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:04&amp;lt;00:04,  1.11it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:06&amp;lt;00:04,  1.21s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:06&amp;lt;00:02,  1.18it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:07&amp;lt;00:00,  1.47it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.14it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.08it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:68 Preparing tokenizer_path: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test3/tokenizer.json...\nINFO     integration_tests.llm.utils:utils.py:70 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     integration_tests.llm.utils:utils.py:75 Tokenizer saved to /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test3/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Beginning SGLang benchmark test...\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:30000...\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:55 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test3\nRequest Rate: 8\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test3&amp;#x27;, request_rate=8, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test3/sglang_10_8.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    8         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  9.25      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.08      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          211.90    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         299.91    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          511.81    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   4958.41   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 5555.42   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          27.76     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        27.57     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           34.88     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          18.82     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        18.16     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           23.53     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           17.84     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         17.94     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            33.40     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:63 Benchmark run completed in 17.307039737701416 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:64 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 8\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 4958.408982500259\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 5555.41742600326\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 27.572318496822845\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 17.942186997970566\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 299.9053687635496\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 9.249584332006634\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[16-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[16-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:18", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[16-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:18&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:03,  2.66it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:02,  3.03it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:04&amp;lt;00:11,  1.71s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:04&amp;lt;00:07,  1.22s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:04&amp;lt;00:04,  1.09it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:06&amp;lt;00:04,  1.15s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:06&amp;lt;00:02,  1.22it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:07&amp;lt;00:00,  1.54it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.14it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.10it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:68 Preparing tokenizer_path: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test4/tokenizer.json...\nINFO     integration_tests.llm.utils:utils.py:70 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     integration_tests.llm.utils:utils.py:75 Tokenizer saved to /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test4/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Beginning SGLang benchmark test...\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:30000...\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:55 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test4\nRequest Rate: 16\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test4&amp;#x27;, request_rate=16, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test4/sglang_10_16.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    16        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  9.10      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.10      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          215.49    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         304.99    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          520.48    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   4962.35   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 5573.55   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          30.05     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        31.20     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           39.74     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          18.84     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        18.16     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           23.46     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           17.84     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         17.94     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            18.57     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:63 Benchmark run completed in 17.105153799057007 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:64 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 16\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 4962.354662797588\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 5573.554371505452\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 31.198875505651813\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 17.939693003427237\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 304.9858310867143\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 9.095504502998665\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[32-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[32-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:18", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[32-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:18&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:02,  3.44it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:02,  3.39it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:03&amp;lt;00:11,  1.70s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:04&amp;lt;00:07,  1.24s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:04&amp;lt;00:04,  1.09it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:06&amp;lt;00:04,  1.14s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:06&amp;lt;00:02,  1.24it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:07&amp;lt;00:00,  1.55it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.14it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:09&amp;lt;00:00,  1.11it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     integration_tests.llm.utils:utils.py:68 Preparing tokenizer_path: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test5/tokenizer.json...\nINFO     integration_tests.llm.utils:utils.py:70 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     integration_tests.llm.utils:utils.py:75 Tokenizer saved to /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test5/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Beginning SGLang benchmark test...\nINFO     integration_tests.llm.utils:utils.py:133 Waiting for server to start at http://localhost:30000...\nINFO     integration_tests.llm.utils:utils.py:139 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:55 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /tmp/pytest-of-nod/pytest-14/sglang_benchmark_test5\nRequest Rate: 32\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test5&amp;#x27;, request_rate=32, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/tmp/pytest-of-nod/pytest-14/sglang_benchmark_test5/sglang_10_32.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    32        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  9.01      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.11      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          217.57    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         307.93    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          525.51    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   4947.75   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 5554.19   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          29.35     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        30.00     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           41.49     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          18.49     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        18.07     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           21.55     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           17.79     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         17.94     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            18.44     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:63 Benchmark run completed in 16.994997262954712 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:64 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:71 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:71 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:71 REQUEST_RATE: 32\nINFO     sglang_benchmarks.utils:utils.py:71 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:71 MEAN_E2E_LATENCY_MS: 4947.747132100631\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_E2E_LATENCY_MS: 5554.189205504372\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_TTFT_MS: 30.004358501173556\nINFO     sglang_benchmarks.utils:utils.py:71 MEDIAN_ITL_MS: 17.93893249850953\nINFO     sglang_benchmarks.utils:utils.py:71 OUTPUT_THROUGHPUT: 307.93290334970726\nINFO     sglang_benchmarks.utils:utils.py:71 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:71 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:71 DURATION: 9.008455964998575\nINFO     sglang_benchmarks.utils:utils.py:71 COMPLETED: 10\n\n"}]}, "renderCollapsed": ["passed"], "initialSort": "result", "title": "shortfin_index.html"}' id="data-container"></div>
+<div data-jsonblob='{"environment": {"Python": "3.11.11", "Platform": "Linux-5.15.0-131-generic-x86_64-with-glibc2.35", "Packages": {"pytest": "8.0.0", "pluggy": "1.5.0"}, "Plugins": {"metadata": "3.1.1", "timeout": "2.3.1", "html": "4.1.1", "anyio": "4.8.0", "xdist": "3.5.0"}, "CI": "true"}, "tests": {"reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-1]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-1]", "duration": "00:18:34", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-1]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:18:34&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "---------------------------- Captured stdout setup -----------------------------\nExporting prefill_bs1\nExporting decode_bs1\nExporting prefill_bs4\nExporting decode_bs4\nGENERATED!\nExporting\nSaving to &amp;#x27;/shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF/model.mlir&amp;#x27;\n\n---------------------------- Captured stderr setup -----------------------------\n/shark-dev/actions-runner-shark-ai-3/_work/shark-ai/shark-ai/sharktank/sharktank/types/gguf_interop/base.py:100: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.)\n  data_tensor = torch.as_tensor(data.reshape(logical_shape))\n[2025-02-05 15:32:10] Started server process [1111148]\n[2025-02-05 15:32:10] Waiting for application startup.\n[2025-02-05 15:32:12] Application startup complete.\n[2025-02-05 15:32:12] Uvicorn running on http://0.0.0.0:34563 (Press CTRL+C to quit)\n[2025-02-05 15:32:13] 127.0.0.1:53300 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\n\n------------------------------ Captured log setup ------------------------------\nINFO     integration_tests.llm.model_management:model_management.py:119 Downloading model SanctumAI/Meta-Llama-3.1-8B-Instruct-GGUF from HuggingFace\nINFO     integration_tests.llm.model_management:model_management.py:186 Downloading tokenizer NousResearch/Meta-Llama-3.1-8B using transformers\nINFO     integration_tests.llm.model_management:model_management.py:202 Exporting model with following settings:\n  MLIR Path: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF/model.mlir\n  Config Path: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF/config.json\n  Batch Sizes: 1,4\nINFO     integration_tests.llm.model_management:model_management.py:223 Model successfully exported to /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF/model.mlir\nINFO     integration_tests.llm.model_management:model_management.py:229 Compiling model to /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF/model.vmfb\nINFO     integration_tests.llm.model_management:model_management.py:240 Model successfully compiled to /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF/model.vmfb\n\n----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:32:16] 127.0.0.1:53304 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:32:33] 127.0.0.1:34024 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:34] 127.0.0.1:34030 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:35] 127.0.0.1:34042 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:35] 127.0.0.1:34056 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:35] 127.0.0.1:34066 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:35] 127.0.0.1:34082 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:35] 127.0.0.1:34094 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:36] 127.0.0.1:34106 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:36] 127.0.0.1:34122 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:32:37] 127.0.0.1:34130 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:04&amp;lt;00:37,  4.12s/it]\r 20%|\u2588\u2588        | 2/10 [00:05&amp;lt;00:20,  2.59s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:35&amp;lt;01:44, 14.88s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:36&amp;lt;00:57,  9.61s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:39&amp;lt;00:35,  7.02s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:39&amp;lt;00:10,  3.52s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [01:49&amp;lt;00:42, 21.21s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [02:15&amp;lt;00:22, 22.54s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:50&amp;lt;00:00, 26.06s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:50&amp;lt;00:00, 17.04s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_none-1...\n::group::Benchmark run on llama31_8b_none-1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_none-1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34563\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 190.94339036941528 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 1\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 170.41955596901244\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 230\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.058678711742555595\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 11.501027501540896\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 16.27747463738492\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 59394.74308669742\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 36541.000608485774\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 53267.88359556635\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 164816.66756709022\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 194.4619328045519\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 136.97240999317728\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 134.54578240445883\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 397.1443428931525\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 192.83984200626952\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 159.58125959161745\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 81.82449769984346\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 318.7702200906243\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 213.40469628585979\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 141.59594048396684\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 165.90280333295345\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 524.193467500736\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.4852070086074645\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-2]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-2]", "duration": "00:02:19", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-2]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:19&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:35:27] 127.0.0.1:50392 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:35:44] 127.0.0.1:59606 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:44] 127.0.0.1:59618 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59630 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59638 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59650 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59658 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59662 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59666 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59678 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:35:45] 127.0.0.1:59680 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:23,  2.64s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:16,  2.09s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:24&amp;lt;01:13, 10.46s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:26&amp;lt;00:41,  6.98s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:28&amp;lt;00:26,  5.38s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:36&amp;lt;00:24,  6.11s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:38&amp;lt;00:14,  4.68s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [01:01&amp;lt;00:20, 10.46s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:33&amp;lt;00:17, 17.36s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:59&amp;lt;00:00, 19.88s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:59&amp;lt;00:00, 11.91s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_none-2...\n::group::Benchmark run on llama31_8b_none-2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_none-2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34563\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 139.37390398979187 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 2\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 119.0917125089909\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 222\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.0839688991729382\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 16.45790423789589\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 23.292972630573058\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 42529.252636103774\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 31812.90353850636\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 35374.26274144369\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 115568.82960671443\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 257.36049899715\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 241.54974249540828\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 122.8739448629008\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 439.1904081674875\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 140.71947180403694\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 116.84192793727149\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 44.87435195903104\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 223.01780099322443\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 152.38336978586648\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 110.12498200580012\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 128.72002852300315\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 502.863037227653\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.5711345265014143\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-4]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-4]", "duration": "00:02:27", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-4]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:27&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:37:46] 127.0.0.1:48870 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:38:03] 127.0.0.1:45640 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:03] 127.0.0.1:45652 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45666 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45682 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45684 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45690 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45698 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45708 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45722 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:38:04] 127.0.0.1:45738 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:25,  2.81s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:16,  2.08s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:22&amp;lt;01:06,  9.54s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:30&amp;lt;00:53,  8.99s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:39&amp;lt;00:43,  8.72s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:40&amp;lt;00:24,  6.24s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:46&amp;lt;00:18,  6.26s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [01:04&amp;lt;00:19,  9.83s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:26&amp;lt;00:13, 13.80s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:06&amp;lt;00:00, 21.89s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:06&amp;lt;00:00, 12.69s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_none-4...\n::group::Benchmark run on llama31_8b_none-4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_none-4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34563\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 147.42578887939453 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 4\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 126.9456260559964\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 235\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.07877388383266506\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 15.439681231202353\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 21.85187537518129\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 46082.9477135936\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 39447.13300149306\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 35990.732402397494\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 122750.68649666267\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 455.51314509357326\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 450.02366448170505\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 226.41312478179762\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 772.1808718793909\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 159.43819975595005\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 147.94108660539084\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 49.3554200821531\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 240.6038173449948\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 164.48048524332637\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 120.9237074945122\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 140.1670190343055\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 511.9830371209536\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.6301327698574006\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-8]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-8]", "duration": "00:01:59", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-8]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:01:59&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:40:14] 127.0.0.1:39164 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:40:39] 127.0.0.1:34088 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34096 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34104 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34116 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34122 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34130 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34132 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34140 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34144 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:40:39] 127.0.0.1:34160 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:23,  2.57s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:15,  1.92s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:24&amp;lt;01:12, 10.38s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:28&amp;lt;00:46,  7.72s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:33&amp;lt;00:33,  6.75s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:36&amp;lt;00:22,  5.65s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:39&amp;lt;00:14,  4.72s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:55&amp;lt;00:16,  8.31s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:23&amp;lt;00:14, 14.47s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:30&amp;lt;00:00, 12.16s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:30&amp;lt;00:00,  9.05s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_none-8...\n::group::Benchmark run on llama31_8b_none-8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_none-8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34563\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 118.98630166053772 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 8\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 90.47319212497678\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 227\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.11052997871663818\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 21.663875828461084\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 30.661016095995432\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 39543.00889910082\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 34690.40310100536\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 27902.22069584111\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 89535.28844025423\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 592.6666169980308\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 655.3590514959069\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 250.49889620242143\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 909.9135958874831\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 140.40010182673885\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 130.31004588634923\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 33.08116968126523\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 195.86176021670448\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 140.40973567160532\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 111.42337150522508\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 110.05433854086436\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 501.704525629757\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 4.3706879320094485\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-16]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-16]", "duration": "00:02:41", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-16]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:41&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:42:13] 127.0.0.1:34654 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:42:29] 127.0.0.1:40428 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:29] 127.0.0.1:40436 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40440 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40446 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40450 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40458 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40464 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40468 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40474 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:42:30] 127.0.0.1:40488 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:01&amp;lt;00:16,  1.80s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:16,  2.04s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:24&amp;lt;01:14, 10.63s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:27&amp;lt;00:45,  7.50s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:28&amp;lt;00:26,  5.31s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:33&amp;lt;00:20,  5.20s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:37&amp;lt;00:13,  4.59s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:52&amp;lt;00:15,  7.90s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:04&amp;lt;00:09,  9.18s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:20&amp;lt;00:00, 29.82s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:20&amp;lt;00:00, 14.03s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_none-16...\n::group::Benchmark run on llama31_8b_none-16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_none-16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34563\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 160.61767435073853 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 16\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 140.33601672502118\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 220\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.07125754480828907\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 13.966478782424655\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 19.766842929819386\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 41412.59078050207\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 31382.664225515327\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 37513.86708002592\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 133334.09731315277\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 605.9228640049696\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 696.5276240080129\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 241.25496961784498\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 865.1533227710752\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 133.4233696986571\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 115.7362984833423\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 47.46855522419147\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 255.28043642638752\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 147.09984298305542\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 109.33046501304489\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 123.39421856493576\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 502.01081668667024\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 2.9509595431689646\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-32]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-32]", "duration": "00:02:44", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_none-32]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:44&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:44:53] 127.0.0.1:47084 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:45:10] 127.0.0.1:58528 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58540 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58552 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58564 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58576 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58590 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58592 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58596 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58608 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:45:10] 127.0.0.1:58620 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:01&amp;lt;00:15,  1.71s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:19,  2.46s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:24&amp;lt;01:11, 10.20s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:28&amp;lt;00:47,  7.90s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:31&amp;lt;00:30,  6.14s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:37&amp;lt;00:23,  5.94s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:39&amp;lt;00:14,  4.68s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:42&amp;lt;00:08,  4.11s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:58&amp;lt;00:07,  7.99s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:23&amp;lt;00:00, 31.79s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:23&amp;lt;00:00, 14.36s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_none-32...\n::group::Benchmark run on llama31_8b_none-32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_none-32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:34563\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 163.8120219707489 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 32\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 143.63546776599833\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 231\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.0696206873938083\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 13.645654729186429\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 19.312778683042424\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 41022.776429596706\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 34220.74842049915\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 37749.779020109265\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 135899.64922179555\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 609.5357034966582\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 733.3407794940285\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 228.79014885909237\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 769.1571468723123\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 135.23856131398105\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 113.56366131681692\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 49.70345847954422\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 261.6388791299071\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 145.68158661677313\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 110.23468000348657\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 115.88399938204341\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 503.8049189682351\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 2.8560338938310403\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-1]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-1]", "duration": "00:02:23", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-1]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:23&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "---------------------------- Captured stderr setup -----------------------------\n[2025-02-05 15:47:34] Shutting down\n[2025-02-05 15:47:34] Waiting for application shutdown.\n[2025-02-05 15:47:34] Application shutdown complete.\n[2025-02-05 15:47:34] Finished server process [1111148]\n[2025-02-05 15:47:36] Started server process [1361045]\n[2025-02-05 15:47:36] Waiting for application startup.\n[2025-02-05 15:47:39] Application startup complete.\n[2025-02-05 15:47:39] Uvicorn running on http://0.0.0.0:39645 (Press CTRL+C to quit)\n[2025-02-05 15:47:40] 127.0.0.1:48898 - &amp;quot;GET /health HTTP/1.1&amp;quot; 200\n\n----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:47:44] 127.0.0.1:41288 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:48:01] 127.0.0.1:56976 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:02] 127.0.0.1:56986 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:03] 127.0.0.1:40898 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:03] 127.0.0.1:40904 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:03] 127.0.0.1:40912 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:03] 127.0.0.1:40922 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:03] 127.0.0.1:40936 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:04] 127.0.0.1:40938 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:04] 127.0.0.1:40942 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:48:05] 127.0.0.1:40948 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:03&amp;lt;00:33,  3.76s/it]\r 20%|\u2588\u2588        | 2/10 [00:05&amp;lt;00:18,  2.34s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:30&amp;lt;01:29, 12.77s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:32&amp;lt;00:51,  8.65s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:35&amp;lt;00:33,  6.67s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:39&amp;lt;00:22,  5.52s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:39&amp;lt;00:11,  3.91s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [01:15&amp;lt;00:27, 13.93s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:17&amp;lt;00:10, 10.35s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:56&amp;lt;00:00, 19.05s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:56&amp;lt;00:00, 11.61s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_trie-1...\n::group::Benchmark run on llama31_8b_trie-1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_trie-1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:39645\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 1\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 137.56899571418762 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 1\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 116.05045018100645\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 226\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.08616942014789929\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 16.88920634898826\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 23.903397149027263\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 43454.9856490019\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 36105.01189000206\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 32508.35653340082\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 110137.40995678236\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 198.12668070371728\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 152.0729250041768\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 106.85953926715756\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 362.044016913278\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 146.7733377642843\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 139.5678537402816\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 38.54366982912771\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 215.04794458793992\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 155.93016549890572\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 111.51933850487694\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 129.5239212745198\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 507.87133009987883\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.744490915909779\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-2]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-2]", "duration": "00:01:52", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-2]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:01:52&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:50:01] 127.0.0.1:52976 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:50:18] 127.0.0.1:43930 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:18] 127.0.0.1:43932 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:18] 127.0.0.1:43944 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:18] 127.0.0.1:43946 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:19] 127.0.0.1:43958 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:19] 127.0.0.1:43970 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:19] 127.0.0.1:43984 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:19] 127.0.0.1:43986 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:19] 127.0.0.1:43996 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:50:19] 127.0.0.1:44004 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:25,  2.86s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:16,  2.06s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:25&amp;lt;01:14, 10.64s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:26&amp;lt;00:41,  6.99s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:30&amp;lt;00:29,  5.99s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:38&amp;lt;00:25,  6.44s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:40&amp;lt;00:15,  5.26s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:45&amp;lt;00:09,  4.95s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:47&amp;lt;00:04,  4.03s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:31&amp;lt;00:00, 16.38s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:31&amp;lt;00:00,  9.13s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_trie-2...\n::group::Benchmark run on llama31_8b_trie-2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_trie-2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:39645\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 2\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 111.7774658203125 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 2\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 91.28525483899284\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 222\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.1095467172396879\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 21.471156578978828\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 30.38825936228942\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 34249.681045199395\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 33565.773092996096\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 23600.875388699005\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 86156.1568472942\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 261.39611969992984\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 266.92902500508353\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 117.54996509276613\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 441.2411970947869\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 124.27465887561073\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 118.85418964039182\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 23.773826372714726\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 170.2491906740083\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 122.5209640562573\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 105.70212500169873\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 86.35964056325314\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 501.1406297664507\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.7519401250079563\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-4]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-4]", "duration": "00:02:30", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-4]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:30&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:51:52] 127.0.0.1:37878 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:52:09] 127.0.0.1:60670 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:09] 127.0.0.1:60672 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60686 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60696 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60710 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60726 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60734 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60742 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60746 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:52:10] 127.0.0.1:60752 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:24,  2.78s/it]\r 20%|\u2588\u2588        | 2/10 [00:05&amp;lt;00:22,  2.84s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:28&amp;lt;01:23, 11.97s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:30&amp;lt;00:47,  7.93s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:30&amp;lt;00:26,  5.32s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:33&amp;lt;00:17,  4.46s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:39&amp;lt;00:14,  4.93s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:49&amp;lt;00:12,  6.49s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:15&amp;lt;00:12, 12.51s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:10&amp;lt;00:00, 25.64s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:10&amp;lt;00:00, 13.02s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_trie-4...\n::group::Benchmark run on llama31_8b_trie-4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_trie-4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:39645\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 4\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 150.43805861473083 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 4\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 130.24024797900347\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 229\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.07678118058875424\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 15.049111395395832\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 21.299099495320426\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 42109.113225800684\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 32244.788669995614\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 35020.48032201268\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 124668.46396506242\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 451.96010660147294\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 452.72947251214646\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 223.14462327359558\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 775.025058654719\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 150.9333835893978\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 134.0313303514014\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 45.89838162103943\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 240.51542389549556\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 150.16786086302275\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 114.0831469965633\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 141.219896260481\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 502.11496058094787\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.233187427022502\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-8]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-8]", "duration": "00:02:22", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-8]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:22&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:54:23] 127.0.0.1:37112 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:54:40] 127.0.0.1:36592 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36594 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36604 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36618 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36630 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36634 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36648 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36664 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36670 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:54:40] 127.0.0.1:36682 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:26,  2.98s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:17,  2.13s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:21&amp;lt;01:01,  8.85s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:26&amp;lt;00:45,  7.52s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:31&amp;lt;00:33,  6.63s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:36&amp;lt;00:24,  6.05s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:40&amp;lt;00:15,  5.31s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:56&amp;lt;00:17,  8.53s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:01&amp;lt;00:07,  7.43s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:01&amp;lt;00:00, 23.82s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [02:01&amp;lt;00:00, 12.16s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_trie-8...\n::group::Benchmark run on llama31_8b_trie-8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_trie-8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:39645\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 8\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 141.88902759552002 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 8\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 121.56148310299614\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 235\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.08226289894412722\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 16.123528193048937\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 22.81972816710089\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 40104.898013800266\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 34072.563436988275\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 32482.556984168008\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 115808.10929485451\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 574.889738301863\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 632.4853370024357\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 206.61027495066187\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 776.3973312385497\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 146.96331862188111\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 143.0848640218028\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 53.18875294103405\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 245.3639635564074\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 142.4997629469869\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 108.02235802111682\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 116.92969093084415\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 501.65656326018507\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.29914517247378\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-16]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-16]", "duration": "00:01:59", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-16]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:01:59&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:56:45] 127.0.0.1:36558 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:57:01] 127.0.0.1:38354 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:01] 127.0.0.1:38370 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38384 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38386 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38388 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38402 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38418 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38422 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38436 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:57:02] 127.0.0.1:38442 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:24,  2.67s/it]\r 20%|\u2588\u2588        | 2/10 [00:04&amp;lt;00:19,  2.42s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:25&amp;lt;01:15, 10.73s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:27&amp;lt;00:43,  7.19s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:30&amp;lt;00:27,  5.60s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:34&amp;lt;00:20,  5.23s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:40&amp;lt;00:15,  5.31s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:41&amp;lt;00:08,  4.08s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:59&amp;lt;00:08,  8.28s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:38&amp;lt;00:00, 17.92s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:38&amp;lt;00:00,  9.85s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_trie-16...\n::group::Benchmark run on llama31_8b_trie-16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_trie-16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:39645\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 16\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 118.80619144439697 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 16\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 98.54306925699348\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 226\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.101478471042146\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 19.889780324260617\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 28.1501278670913\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 36290.707125593326\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 32248.39327899099\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 26029.7276033414\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 94825.45089612278\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 613.9958217943786\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 726.5308150090277\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 246.76767010499512\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 833.5408247529995\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 141.5350902166957\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 116.3258070697363\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 47.54099350209567\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 248.83867568978593\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 128.6083895955196\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 104.7832144977292\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 99.61668265508888\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 501.7250573099591\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.682725472143524\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n"}], "reports/shortfin_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-32]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-32]", "duration": "00:02:06", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/shortfin_benchmark_test.py::test_shortfin_benchmark[llama31_8b_trie-32]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:02:06&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\n[2025-02-05 15:58:43] 127.0.0.1:35390 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s][2025-02-05 15:59:00] 127.0.0.1:36272 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36278 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36284 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36298 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36302 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36318 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36330 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36346 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36352 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n[2025-02-05 15:59:00] 127.0.0.1:36366 - &amp;quot;POST /generate HTTP/1.1&amp;quot; 200\n\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:23,  2.66s/it]\r 20%|\u2588\u2588        | 2/10 [00:03&amp;lt;00:13,  1.69s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:24&amp;lt;01:14, 10.59s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:29&amp;lt;00:49,  8.28s/it]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:30&amp;lt;00:28,  5.61s/it]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:38&amp;lt;00:25,  6.38s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:39&amp;lt;00:13,  4.57s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [01:12&amp;lt;00:27, 13.82s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [01:41&amp;lt;00:18, 18.42s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:45&amp;lt;00:00, 13.96s/it]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [01:45&amp;lt;00:00, 10.53s/it]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:87 Starting benchmark run on llama31_8b_trie-32...\n::group::Benchmark run on llama31_8b_trie-32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:91 Running SGLang Benchmark with the following settings:\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:92 Test parameterization: llama31_8b_trie-32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:93 Benchmark Args: Backend: shortfin\nBase URL: http://localhost:39645\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-787/model_cache0/SanctumAI_Meta-Llama-3.1-8B-Instruct-GGUF\nRequest Rate: 32\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:104 Benchmark run completed in 125.52126741409302 seconds\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:105 \n\n======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: shortfin\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 32\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 105.30859574800706\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 232\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 0.09495901003113744\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 18.611965966102936\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 26.341629382637525\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 44749.20414389635\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 34336.23252149846\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 34654.02349149112\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 104869.70349395735\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 629.5038835000014\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 749.887102007051\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 251.498117963953\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 856.0684450663393\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 155.22852137737635\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 125.74550245680749\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 50.19779674547357\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 237.7822115797438\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 159.04403268134033\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 114.52329500752967\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 205.9546275949179\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 502.8681076873909\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 4.249340125185671\nINFO     sglang_benchmarks.shortfin_benchmark_test:shortfin_benchmark_test.py:107 Benchmark run successful\n::endgroup::\n\n--------------------------- Captured stderr teardown ---------------------------\n[2025-02-05 16:00:46] Shutting down\n[2025-02-05 16:00:46] Waiting for application shutdown.\n[2025-02-05 16:00:46] Application shutdown complete.\n[2025-02-05 16:00:46] Finished server process [1361045]\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[1-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[1-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:56", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[1-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:56&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\nCould not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:02&amp;lt;00:18,  2.10s/it]\r 20%|\u2588\u2588        | 2/10 [00:02&amp;lt;00:07,  1.01it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:02&amp;lt;00:04,  1.53it/s]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:03&amp;lt;00:04,  1.42it/s]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:04&amp;lt;00:04,  1.14it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:06&amp;lt;00:05,  1.40s/it]\r 70%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588   | 7/10 [00:07&amp;lt;00:03,  1.06s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:07&amp;lt;00:01,  1.28it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:08&amp;lt;00:00,  1.18it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:08&amp;lt;00:00,  1.61it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:08&amp;lt;00:00,  1.16it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Preparing tokenizer_path: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test0/tokenizer.json...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:34 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:39 Tokenizer saved to /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test0/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:71 Beginning SGLang benchmark test...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:45 Waiting for server to start at http://localhost:30000...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 0 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 1.0020861625671387 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 2.0040180683135986 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 3.0058681964874268 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 4.00773811340332 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 5.009791135787964 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 6.012682676315308 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 7.015617370605469 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 8.01838493347168 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 9.021151304244995 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 10.022801637649536 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 11.024739742279053 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 12.026344299316406 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 13.028058290481567 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 14.029765367507935 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 15.031472444534302 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 16.03307294845581 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 17.03638505935669 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 18.038103342056274 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 19.04052495956421 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 20.043206930160522 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 21.044754028320312 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 22.046297311782837 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 23.048229932785034 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 24.049895524978638 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 25.051570177078247 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 26.053257703781128 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 27.054922342300415 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 28.05670690536499 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 29.058990001678467 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 30.060951948165894 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 31.062747478485107 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 32.06452512741089 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 33.066102504730225 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 34.0676543712616 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 35.06957006454468 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 36.07119393348694 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 37.072898626327515 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 38.07448482513428 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:54 Server has not started yet; waited 39.07611155509949 seconds; timeout: 600 seconds.\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:51 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:93 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:94 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test0\nRequest Rate: 1\nERROR    huggingface_hub.file_download:file_download.py:1387 Could not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test0&amp;#x27;, request_rate=1, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test0/sglang_10_1.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, sharegpt_context_len=None, apply_chat_template=False, return_logprob=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    1         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  8.60      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.16      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          228.00    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         322.69    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          550.69    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Concurrency:                             3.87      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   3328.33   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 3616.49   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          30.78     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        24.09     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           56.55     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          11.75     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        12.21     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           13.46     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           11.93     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         13.11     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            16.50     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:102 Benchmark run completed in 15.635726690292358 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:103 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 1\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 8.596545824984787\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 1.1632579181903793\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 227.99855196531433\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 322.6877465060112\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 3328.3344815979945\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 3616.4940085000126\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 1931.6749100642105\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 6138.447787179903\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 30.776787002105266\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 24.092781502986327\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 13.601361887149041\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 56.554866467777174\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 11.752074749577393\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 12.208471717904875\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 1.45229517540248\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 13.456993066468579\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 11.930356352381875\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 13.107530001434498\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 2.9155848476299675\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 16.504459557763763\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 3.871711440104938\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[2-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[2-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:13", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[2-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:13&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\nCould not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:01&amp;lt;00:11,  1.28s/it]\r 30%|\u2588\u2588\u2588       | 3/10 [00:02&amp;lt;00:06,  1.05it/s]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:03&amp;lt;00:05,  1.19it/s]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:03&amp;lt;00:03,  1.57it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:05&amp;lt;00:04,  1.13s/it]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:06&amp;lt;00:01,  1.60it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:06&amp;lt;00:00,  1.49it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:07&amp;lt;00:00,  1.55it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:07&amp;lt;00:00,  1.33it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Preparing tokenizer_path: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test1/tokenizer.json...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:34 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:39 Tokenizer saved to /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test1/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:71 Beginning SGLang benchmark test...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:45 Waiting for server to start at http://localhost:30000...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:51 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:93 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:94 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test1\nRequest Rate: 2\nERROR    huggingface_hub.file_download:file_download.py:1387 Could not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test1&amp;#x27;, request_rate=2, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test1/sglang_10_2.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, sharegpt_context_len=None, apply_chat_template=False, return_logprob=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    2         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  7.52      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.33      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          260.76    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         369.06    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          629.82    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Concurrency:                             4.71      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   3542.05   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 3951.72   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          31.09     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        28.07     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           51.78     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          13.09     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        13.16     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           15.28     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           12.70     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         13.15     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            28.03     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:102 Benchmark run completed in 12.397382259368896 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:103 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 2\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 7.516427864000434\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 1.330419207226682\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 260.7621646164297\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 369.0582880846816\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 3542.053556701285\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 3951.720820987248\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 1927.6646302208983\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 6239.600631733774\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 31.094804999884218\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 28.072797998902388\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 9.271340258039437\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 51.783227179257665\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 13.092278126215644\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 13.155353054476114\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 0.97559828496976\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 15.276190136905644\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 12.702441047755352\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 13.152656494639814\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 3.0340062373093906\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 28.031199074757684\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 4.712416084860973\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[4-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[4-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:13", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[4-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:13&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\nCould not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:06,  1.31it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:03,  2.51it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:03&amp;lt;00:09,  1.32s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:03&amp;lt;00:05,  1.12it/s]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:03&amp;lt;00:03,  1.49it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:05&amp;lt;00:04,  1.17s/it]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:06&amp;lt;00:00,  1.60it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:07&amp;lt;00:00,  1.52it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:07&amp;lt;00:00,  1.33it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Preparing tokenizer_path: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test2/tokenizer.json...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:34 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:39 Tokenizer saved to /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test2/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:71 Beginning SGLang benchmark test...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:45 Waiting for server to start at http://localhost:30000...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:51 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:93 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:94 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test2\nRequest Rate: 4\nERROR    huggingface_hub.file_download:file_download.py:1387 Could not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test2&amp;#x27;, request_rate=4, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test2/sglang_10_4.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, sharegpt_context_len=None, apply_chat_template=False, return_logprob=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    4         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  7.50      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.33      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          261.26    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         369.76    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          631.01    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Concurrency:                             5.26      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   3944.05   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 4449.89   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          50.24     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        26.61     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           229.00    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          14.43     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        14.69     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           15.86     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           14.09     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         13.27     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            39.68     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:102 Benchmark run completed in 12.426339626312256 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:103 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 4\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 7.502231040009065\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 1.3329368219494233\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 261.255617102087\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 369.75667440877004\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 3944.0511326945852\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 4449.887998503982\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 2140.858602808857\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 6817.57385136385\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 50.23789359256625\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 26.608628482790664\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 65.99005671666077\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 228.9971389883431\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 14.429913406964669\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 14.686280576944725\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 0.8749021147727248\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 15.86351094365964\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 14.087586348764995\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 13.266175505123101\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 5.8413020961596755\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 39.67887636710656\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 5.257170982419944\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[8-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[8-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:12", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[8-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:12&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\nCould not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:04,  2.00it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:02,  3.06it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:03&amp;lt;00:09,  1.31s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:03&amp;lt;00:05,  1.19it/s]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:03&amp;lt;00:03,  1.53it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:05&amp;lt;00:03,  1.10it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:05&amp;lt;00:00,  1.97it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:06&amp;lt;00:00,  1.69it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:06&amp;lt;00:00,  1.51it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Preparing tokenizer_path: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test3/tokenizer.json...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:34 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:39 Tokenizer saved to /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test3/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:71 Beginning SGLang benchmark test...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:45 Waiting for server to start at http://localhost:30000...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:51 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:93 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:94 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test3\nRequest Rate: 8\nERROR    huggingface_hub.file_download:file_download.py:1387 Could not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test3&amp;#x27;, request_rate=8, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test3/sglang_10_8.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, sharegpt_context_len=None, apply_chat_template=False, return_logprob=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    8         \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  6.62      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.51      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          296.13    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         419.12    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          715.25    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Concurrency:                             5.49      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   3634.17   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 4106.87   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          29.62     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        23.81     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           53.02     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          14.19     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        13.40     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           19.70     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           13.04     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         13.18     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            24.54     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:102 Benchmark run completed in 11.365898132324219 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:103 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 8\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 6.618671122996602\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 1.5108773066628065\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 296.1319521059101\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 419.11736486826254\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 3634.1660848032916\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 4106.869311493938\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 1900.3122683507702\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 6228.65769140888\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 29.62200190522708\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 23.8129920035135\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 12.424088291313582\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 53.01803818234475\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 14.194036187710365\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 13.404325060341803\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 2.293904466943341\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 19.697748371817582\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 13.04102284298169\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 13.175340500310995\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 3.406765014294953\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 24.53644106979434\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 5.490779066172914\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[16-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[16-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:12", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[16-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:12&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\nCould not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:03,  2.90it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:02,  3.68it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:03&amp;lt;00:08,  1.28s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:03&amp;lt;00:05,  1.13it/s]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:03&amp;lt;00:03,  1.50it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:04&amp;lt;00:03,  1.16it/s]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:04&amp;lt;00:00,  2.14it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:05&amp;lt;00:00,  2.05it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:06&amp;lt;00:00,  1.64it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:06&amp;lt;00:00,  1.55it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Preparing tokenizer_path: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test4/tokenizer.json...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:34 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:39 Tokenizer saved to /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test4/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:71 Beginning SGLang benchmark test...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:45 Waiting for server to start at http://localhost:30000...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:51 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:93 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:94 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test4\nRequest Rate: 16\nERROR    huggingface_hub.file_download:file_download.py:1387 Could not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test4&amp;#x27;, request_rate=16, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test4/sglang_10_16.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, sharegpt_context_len=None, apply_chat_template=False, return_logprob=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    16        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  6.45      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.55      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          303.75    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         429.90    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          733.65    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Concurrency:                             5.63      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   3629.96   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 4115.93   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          33.73     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        32.95     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           47.78     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          14.08     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        13.40     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           19.56     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           13.01     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         13.19     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            16.23     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:102 Benchmark run completed in 11.55259656906128 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:103 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 16\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 6.452656374021899\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 1.5497493466813985\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 303.7508719495541\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 429.90046876941994\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 3629.9630500987405\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 4115.934913003002\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 1903.2037240534246\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 6211.25403786602\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 33.72621079615783\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 32.95455548504833\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 11.306355324781203\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 47.78268943715375\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 14.084884791379922\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 13.397187276788125\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 2.18841052464557\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 19.556648979798776\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 13.010972801361495\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 13.18767899647355\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 3.517678469463846\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 16.231544022448357\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 5.62553286536814\n\n"}], "reports/sglang_index.html:app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[32-NousResearch/Meta-Llama-3-8B]": [{"extras": [], "result": "Passed", "testId": "app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[32-NousResearch/Meta-Llama-3-8B]", "duration": "00:00:12", "resultsTableRow": ["&lt;td class=\"col-result\"&gt;Passed&lt;/td&gt;", "&lt;td class=\"col-testId\"&gt;app_tests/benchmark_tests/llm/sglang_benchmarks/sglang_benchmark_test.py::test_sglang_benchmark[32-NousResearch/Meta-Llama-3-8B]&lt;/td&gt;", "&lt;td class=\"col-duration\"&gt;00:00:12&lt;/td&gt;", "&lt;td class=\"col-links\"&gt;&lt;/td&gt;"], "log": "----------------------------- Captured stderr call -----------------------------\nCould not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\n\r  0%|          | 0/10 [00:00&amp;lt;?, ?it/s]\r 10%|\u2588         | 1/10 [00:00&amp;lt;00:02,  3.65it/s]\r 20%|\u2588\u2588        | 2/10 [00:00&amp;lt;00:01,  4.11it/s]\r 30%|\u2588\u2588\u2588       | 3/10 [00:02&amp;lt;00:08,  1.26s/it]\r 40%|\u2588\u2588\u2588\u2588      | 4/10 [00:03&amp;lt;00:05,  1.09it/s]\r 50%|\u2588\u2588\u2588\u2588\u2588     | 5/10 [00:03&amp;lt;00:03,  1.48it/s]\r 60%|\u2588\u2588\u2588\u2588\u2588\u2588    | 6/10 [00:04&amp;lt;00:03,  1.18it/s]\r 80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 8/10 [00:04&amp;lt;00:00,  2.18it/s]\r 90%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588 | 9/10 [00:05&amp;lt;00:00,  2.08it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:06&amp;lt;00:00,  1.65it/s]\r100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:06&amp;lt;00:00,  1.57it/s]\n\n------------------------------ Captured log call -------------------------------\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:32 Preparing tokenizer_path: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test5/tokenizer.json...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:34 Downloading tokenizer NousResearch/Meta-Llama-3-8B from Hugging Face...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:39 Tokenizer saved to /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test5/tokenizer.json\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:71 Beginning SGLang benchmark test...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:45 Waiting for server to start at http://localhost:30000...\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:51 Server successfully started\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:93 Running SGLang Benchmark with the following args:\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:94 Backend: sglang\nBase URL: http://localhost:30000\nNum Prompt: 10\nTokenizer: /shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test5\nRequest Rate: 32\nERROR    huggingface_hub.file_download:file_download.py:1387 Could not cache non-existence of file. Will ignore error and continue. Error: [Errno 13] Permission denied: &amp;#x27;/home/nod/.cache/huggingface/hub/models--meta-llama--Llama-3.1-8B-Instruct/.no_exist/0e9e39f249a16976918f6564b8830bc894c89659/chat_template.jinja&amp;#x27;\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Namespace(num_prompts=10, base_url=&amp;#x27;http://localhost:30000&amp;#x27;, tokenizer=&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test5&amp;#x27;, request_rate=32, backend=&amp;#x27;sglang&amp;#x27;, output_file=PosixPath(&amp;#x27;/shark-dev/pytest-of-nod/pytest-786/sglang_benchmark_test5/sglang_10_32.jsonl&amp;#x27;), seed=1, extra_request_body=None, port=8000, model=&amp;#x27;meta-llama/Llama-3.1-8B-Instruct&amp;#x27;, dataset_name=&amp;#x27;sharegpt&amp;#x27;, random_input_len=None, random_output_len=None, random_range_ratio=0.0, dataset_path=&amp;#x27;&amp;#x27;, sharegpt_output_len=None, multi=False, disable_tqdm=False, disable_stream=False, disable_ignore_eos=False, lora_name=None, profile=False, sharegpt_context_len=None, apply_chat_template=False, return_logprob=False, max_concurrency=None)\n\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Input tokens: 1960\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 #Output tokens: 2774\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Starting initial single prompt test run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Initial test run completed. Starting main benchmark run...\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 \n============ Serving Benchmark Result ============\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Backend:                                 sglang    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Traffic request rate:                    32        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Max reqeuest concurrency:                not set   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Successful requests:                     10        \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Benchmark duration (s):                  6.38      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total input tokens:                      1960      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens:                  2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total generated tokens (retokenized):    2774      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Request throughput (req/s):              1.57      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Input token throughput (tok/s):          307.01    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Output token throughput (tok/s):         434.51    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Total token throughput (tok/s):          741.52    \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Concurrency:                             5.71      \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ----------------End-to-End Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean E2E Latency (ms):                   3642.28   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median E2E Latency (ms):                 4138.37   \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Time to First Token----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TTFT (ms):                          45.21     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TTFT (ms):                        49.04     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TTFT (ms):                           74.10     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 -----Time per Output Token (excl. 1st token)------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean TPOT (ms):                          13.53     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median TPOT (ms):                        13.45     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 TPOT (ms):                           15.18     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ---------------Inter-token Latency----------------\nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Mean ITL (ms):                           13.01     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 Median ITL (ms):                         13.22     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 P99 ITL (ms):                            14.08     \nINFO     sglang_benchmarks.sglang_benchmark_test:mock.py:1189 ==================================================\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:102 Benchmark run completed in 11.273217678070068 seconds\nINFO     sglang_benchmarks.sglang_benchmark_test:sglang_benchmark_test.py:103 ======== RESULTS ========\nINFO     sglang_benchmarks.utils:utils.py:74 BACKEND: sglang\nINFO     sglang_benchmarks.utils:utils.py:74 DATASET_NAME: sharegpt\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_RATE: 32\nINFO     sglang_benchmarks.utils:utils.py:74 MAX_CONCURRENCY: None\nINFO     sglang_benchmarks.utils:utils.py:74 SHAREGPT_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_INPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_OUTPUT_LEN: None\nINFO     sglang_benchmarks.utils:utils.py:74 RANDOM_RANGE_RATIO: 0.0\nINFO     sglang_benchmarks.utils:utils.py:74 DURATION: 6.384181888977764\nINFO     sglang_benchmarks.utils:utils.py:74 COMPLETED: 10\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_INPUT_TOKENS: 1960\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 TOTAL_OUTPUT_TOKENS_RETOKENIZED: 2774\nINFO     sglang_benchmarks.utils:utils.py:74 REQUEST_THROUGHPUT: 1.566371412015205\nINFO     sglang_benchmarks.utils:utils.py:74 INPUT_THROUGHPUT: 307.0087967549802\nINFO     sglang_benchmarks.utils:utils.py:74 OUTPUT_THROUGHPUT: 434.5114296930179\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_E2E_LATENCY_MS: 3642.2810099989874\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_E2E_LATENCY_MS: 4138.368875996093\nINFO     sglang_benchmarks.utils:utils.py:74 STD_E2E_LATENCY_MS: 1910.8916845126002\nINFO     sglang_benchmarks.utils:utils.py:74 P99_E2E_LATENCY_MS: 6219.060044316284\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TTFT_MS: 45.20975229679607\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TTFT_MS: 49.035676987841725\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TTFT_MS: 18.852383271533796\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TTFT_MS: 74.09534197446192\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_TPOT_MS: 13.527607250304518\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_TPOT_MS: 13.450782652307169\nINFO     sglang_benchmarks.utils:utils.py:74 STD_TPOT_MS: 0.9283311873283568\nINFO     sglang_benchmarks.utils:utils.py:74 P99_TPOT_MS: 15.1802504500316\nINFO     sglang_benchmarks.utils:utils.py:74 MEAN_ITL_MS: 13.013990884247912\nINFO     sglang_benchmarks.utils:utils.py:74 MEDIAN_ITL_MS: 13.217618005000986\nINFO     sglang_benchmarks.utils:utils.py:74 STD_ITL_MS: 4.078968661083562\nINFO     sglang_benchmarks.utils:utils.py:74 P99_ITL_MS: 14.084607198601587\nINFO     sglang_benchmarks.utils:utils.py:74 CONCURRENCY: 5.705164848588281\n\n"}]}, "renderCollapsed": ["passed"], "initialSort": "result", "title": "shortfin_index.html"}' id="data-container"></div>
 <script>
       (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
 const { getCollapsedCategory, setCollapsedIds } = require('./storage.js')
