Add function to set end_id and apply chat template for GAP Triton In-Process

genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py

@@ -52,7 +52,10 @@ def convert(
 
         for file_data in generic_dataset.files_data.values():
             for row in file_data.rows:
-                token_ids = config.tokenizer.encode(row.texts[0])
+                if not config.apply_chat_template:
+                    token_ids = config.tokenizer.encode(row.texts[0])
+                else:
+                    token_ids = config.tokenizer.apply_chat_template([{"role": "user", "content": row.texts[0]}])
                 payload = {
                     "input_ids": {
                         "content": token_ids,
@@ -80,6 +83,8 @@ def _add_request_params(self, payload: Dict, config: InputsConfig) -> None:
             payload["request_output_len"] = [num_tokens]
             if config.output_tokens_deterministic:
                 payload["min_length"] = [num_tokens]
+        if config.set_end_id:
+            payload["end_id"] = [config.tokenizer._tokenizer.eos_token_id]
 
         for key, value in config.extra_inputs.items():
             payload[key] = [value]

genai-perf/genai_perf/inputs/inputs_config.py

@@ -142,3 +142,9 @@ class InputsConfig:
 
     # Seed used to generate random values
     random_seed: int = DEFAULT_RANDOM_SEED
+
+    # whether to set end_id in triton converter
+    set_end_id: bool = False
+
+    # whether to apply chat template in triton converter
+    apply_chat_template: bool = False

genai-perf/genai_perf/main.py

@@ -97,6 +97,8 @@ def create_config_options(args: Namespace) -> InputsConfig:
         batch_size_image=args.batch_size_image,
         batch_size_text=args.batch_size_text,
         output_dir=args.artifact_dir,
+        set_end_id=args.triton_converter_set_end_id,
+        apply_chat_template=args.triton_converter_apply_chat_template,
     )
 
 

genai-perf/genai_perf/parser.py

@@ -571,6 +571,21 @@ def _add_image_input_args(parser):
         "If format is not selected, format of generated image is selected at random",
     )
 
+    input_group.add_argument(
+        "--triton-converter-set-end-id",
+        action="store_true",
+        required=False,
+        help="If specified, the input to trtllm engines in triton server will "
+        "contain end_id set to EOS token."
+    )
+
+    input_group.add_argument(
+        "--triton-converter-apply-chat-template",
+        action="store_true",
+        required=False,
+        help="If specified, the input to trtllm engines in triton server will "
+        "be wrapped with chat template."
+    )
 
 def _add_profile_args(parser):
     profile_group = parser.add_argument_group("Profiling")

genai-perf/genai_perf/tokenizer.py

@@ -68,6 +68,9 @@ def __call__(self, text, **kwargs) -> "BatchEncoding":
     def encode(self, text, **kwargs) -> List[int]:
         self._encode_args.update(kwargs)
         return self._tokenizer.encode(text, **self._encode_args)
+
+    def apply_chat_template(self, text, **kwargs) -> List[int]:
+        return self._tokenizer.apply_chat_template(text, **kwargs)
 
     def decode(self, token_ids, **kwargs) -> str:
         self._decode_args.update(kwargs)

genai-perf/genai_perf/wrapper.py

@@ -108,6 +108,8 @@ def build_cmd(args: Namespace, extra_args: Optional[List[str]] = None) -> List[s
             "tokenizer",
             "tokenizer_trust_remote_code",
             "tokenizer_revision",
+            "triton_converter_set_end_id",
+            "triton_converter_apply_chat_template"
         ]
 
         utils.remove_file(args.profile_export_file)