Draft: Schedule File GenAI Perf frontend

genai-perf/genai_perf/inputs/converters/openai_completions_converter.py

@@ -29,7 +29,7 @@
 from genai_perf.inputs.converters.base_converter import BaseConverter
 from genai_perf.inputs.input_constants import DEFAULT_OUTPUT_TOKENS_MEAN
 from genai_perf.inputs.inputs_config import InputsConfig
-from genai_perf.inputs.retrievers.generic_dataset import GenericDataset
+from genai_perf.inputs.retrievers.generic_dataset import DataRow, GenericDataset
 from genai_perf.utils import sample_bounded_normal
 
 
@@ -50,6 +50,7 @@ def convert(
                     "prompt": prompt,
                 }
                 self._add_request_params(payload, config)
+                self._override_extra(payload, row)
                 request_body["data"].append({"payload": [payload]})
 
         return request_body
@@ -67,3 +68,7 @@ def _add_request_params(self, payload: Dict, config: InputsConfig) -> None:
             )
         for key, value in config.extra_inputs.items():
             payload[key] = value
+
+    def _override_extra(self, payload: Dict, row: DataRow) -> None:
+        for key, value in row.extra_args.items():
+            payload[key] = value

genai-perf/genai_perf/inputs/inputs_config.py

@@ -77,6 +77,12 @@ class InputsConfig:
     # The filename where the input data is available
     input_filename: Optional[Path] = Path("")
 
+    # TODO
+    schedule_file: Optional[Path] = None
+
+    # TODO
+    block_size: int = 512
+
     # The filenames used for synthetic data generation
     synthetic_input_filenames: Optional[List[str]] = field(default_factory=list)
 

genai-perf/genai_perf/inputs/retrievers/generic_dataset.py

@@ -25,7 +25,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from dataclasses import dataclass, field
-from typing import Dict, List, TypeAlias
+from typing import Any, Dict, List
+
+from typing_extensions import TypeAlias
 
 Filename: TypeAlias = str
 TypeOfData: TypeAlias = str
@@ -38,6 +40,7 @@
 class DataRow:
     texts: List[str] = field(default_factory=list)
     images: List[str] = field(default_factory=list)
+    extra_args: Dict[str, Any] = {}
 
     def to_dict(self) -> DataRowDict:
         """

genai-perf/genai_perf/inputs/retrievers/synthetic_data_retriever.py

@@ -25,6 +25,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import json
 from typing import List
 
 from genai_perf.inputs.input_constants import DEFAULT_SYNTHETIC_FILENAME
@@ -51,16 +52,39 @@ def retrieve_data(self) -> GenericDataset:
         files = self.config.synthetic_input_filenames or [DEFAULT_SYNTHETIC_FILENAME]
         synthetic_dataset = GenericDataset(files_data={})
 
+        prompt_desc = []
+        if self.config.schedule_file is not None:
+            with open(self.config.schedule_file, "r") as f:
+                for j, line in enumerate(f):
+                    if j == self.config.num_prompts:
+                        break
+                    prompt_desc.append(json.loads(line))
+
         for file in files:
             data_rows: List[DataRow] = []
 
-            for _ in range(self.config.num_prompts):
+            for i in range(self.config.num_prompts):
                 row = DataRow(texts=[], images=[])
-                prompt = SyntheticPromptGenerator.create_synthetic_prompt(
-                    self.config.tokenizer,
-                    self.config.prompt_tokens_mean,
-                    self.config.prompt_tokens_stddev,
-                )
+                if prompt_desc:
+                    prompt = SyntheticPromptGenerator.create_synthetic_prompt(
+                        self.config.tokenizer,
+                        prompt_desc[i]["input_length"],
+                        0,
+                        prompt_desc[i].get("hash_ids", None),
+                        self.config.block_size,
+                    )
+                    # Generic processing needed here probably
+                    row.extra_args["max_tokens"] = prompt_desc[i].get(
+                        "output_length", None
+                    )
+                    row.extra_args["model"] = prompt_desc[i].get("model", None)
+                else:
+                    prompt = SyntheticPromptGenerator.create_synthetic_prompt(
+                        self.config.tokenizer,
+                        self.config.prompt_tokens_mean,
+                        self.config.prompt_tokens_stddev,
+                    )
+
                 for _ in range(self.config.batch_size_text):
                     row.texts.append(prompt)
 

genai-perf/genai_perf/inputs/retrievers/synthetic_prompt_generator.py

@@ -17,18 +17,22 @@
 import pathlib
 import random
 import re
-from typing import List
+from typing import Dict, List, Optional
 
 from genai_perf.tokenizer import Tokenizer
 
 
 class SyntheticPromptGenerator:
+    cache: Dict[int, str] = {}
+
     @classmethod
     def create_synthetic_prompt(
         cls,
         tokenizer: Tokenizer,
         prompt_tokens_mean: int = 550,
         prompt_tokens_stddev: int = 250,
+        prompt_hash_list: Optional[List[int]] = None,
+        block_size: Optional[int] = None,
     ) -> str:
         """
         Generate a prompt that randomly samples lines from
@@ -49,9 +53,26 @@ def create_synthetic_prompt(
         )
 
         farewell_lines = SyntheticPromptGenerator._create_farewell_lines()
-        prompt = SyntheticPromptGenerator._create_prompt_from_lines(
-            num_prompt_tokens, farewell_lines, tokenizer
-        )
+        if block_size is not None:
+            assert prompt_hash_list, "Need hash of prompt list to continue"
+            final_prompt = []
+            size_to_use = block_size
+            for j, hash_index in enumerate(prompt_hash_list):
+                if j == len(prompt_hash_list) - 1:
+                    size_to_use = prompt_tokens_mean - (j * block_size)
+                if hash_index not in cls.cache:
+                    prompt = SyntheticPromptGenerator._create_prompt_from_lines(
+                        size_to_use, farewell_lines, tokenizer
+                    )
+                    cls.cache[hash_index] = prompt
+
+                final_prompt.append(cls.cache[hash_index])
+            prompt = " ".join(final_prompt)
+
+        else:
+            prompt = SyntheticPromptGenerator._create_prompt_from_lines(
+                num_prompt_tokens, farewell_lines, tokenizer
+            )
 
         return prompt
 
@@ -110,9 +131,10 @@ def word_generator():
         prompt += final_line
 
         # Final tweaks
-        diff = requested_prompt_tokens - get_token_length(prompt)
-        for _ in range(diff):
-            prompt = "hi " + prompt
+        for _ in range(2):
+            diff = requested_prompt_tokens - get_token_length(prompt)
+            for _ in range(diff):
+                prompt = "hi " + prompt
 
         return prompt
 

genai-perf/genai_perf/main.py

@@ -74,6 +74,8 @@ def create_config_options(args: Namespace) -> InputsConfig:
         model_name=args.model,
         model_selection_strategy=args.model_selection_strategy,
         input_filename=args.input_file,
+        schedule_file=args.schedule_file,
+        block_size=args.block_size,
         synthetic_input_filenames=args.synthetic_input_files,
         starting_index=DEFAULT_STARTING_INDEX,
         length=args.num_prompts,

genai-perf/genai_perf/parser.py

@@ -448,6 +448,22 @@ def _add_input_args(parser):
         'in JSONL format. Example: {"text": "Your prompt here"}',
     )
 
+    prompt_source_group.add_argument(
+        "--schedule-file",
+        type=file_or_directory,
+        default=None,
+        required=False,
+        help="Fixed Schedule TODO",
+    )
+
+    prompt_source_group.add_argument(
+        "--block-size",
+        type=int,
+        default=512,
+        required=False,
+        help="Fixed Schedule TODO",
+    )
+
     input_group.add_argument(
         "--num-prompts",
         type=positive_integer,

genai-perf/genai_perf/wrapper.py

@@ -24,6 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import json
 import subprocess
 from argparse import Namespace
 from typing import List, Optional
@@ -61,6 +62,16 @@ def add_inference_load_args(args: Namespace) -> List[str]:
             cmd += ["--concurrency-range", f"{args.concurrency}"]
         elif args.request_rate:
             cmd += ["--request-rate-range", f"{args.request_rate}"]
+
+        if args.schedule_file is not None:
+            assert args.request_rate, "Must use request rate with fixed schedule"
+            timings = []
+            with open(args.schedule_file, "r") as f:
+                for j, line in enumerate(f):
+                    if j == args.num_prompts:
+                        break
+                    timings.append(float(json.loads(line)["timestamp"]) / 1000)
+            cmd += ["--schedule", ",".join(map(str, timings))]
         return cmd
 
     @staticmethod