Support payload input in GAP

genai-perf/docs/customizable_frontends.md

@@ -75,8 +75,9 @@ class NewConverter(BaseConverter):
                     "input": row.texts,
                 }
 
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row)
+                )
 
         return request_body
 ```

genai-perf/genai_perf/export_data/json_exporter.py

@@ -63,10 +63,13 @@ def export(self) -> None:
         with open(str(filename), "w") as f:
             f.write(json.dumps(self._stats_and_args, indent=2))
 
+    def _exclude_args(self, args_to_exclude) -> None:
+        for arg in args_to_exclude:
+            self._args.pop(arg, None)
+
     def _prepare_args_for_export(self) -> None:
-        self._args.pop("func", None)
-        self._args.pop("output_format", None)
-        self._args.pop("input_file", None)
+        args_to_exclude = ["func", "output_format", "input_file", "payload_input_file"]
+        self._exclude_args(args_to_exclude)
         self._args["profile_export_file"] = str(self._args["profile_export_file"])
         self._args["artifact_dir"] = str(self._args["artifact_dir"])
         for k, v in self._args.items():

genai-perf/genai_perf/inputs/converters/base_converter.py

@@ -30,7 +30,7 @@
 from genai_perf.exceptions import GenAIPerfException
 from genai_perf.inputs.input_constants import ModelSelectionStrategy
 from genai_perf.inputs.inputs_config import InputsConfig
-from genai_perf.inputs.retrievers.generic_dataset import GenericDataset
+from genai_perf.inputs.retrievers.generic_dataset import DataRow, GenericDataset
 
 
 class BaseConverter:
@@ -70,3 +70,26 @@ def _add_request_params(
     ) -> None:
         for key, value in config.extra_inputs.items():
             payload[key] = value
+
+    def _add_payload_params(self, payload: Dict[Any, Any], optional_data) -> None:
+        for key, value in optional_data.items():
+            payload[key] = value
+
+    def _finalize_payload(
+        self,
+        payload: Dict[Any, Any],
+        config: InputsConfig,
+        row: DataRow,
+        triton_format=False,
+    ) -> Dict[str, Any]:
+        self._add_request_params(payload, config)
+        self._add_payload_params(payload, row.optional_data)
+        record: Dict[str, Any] = {}
+        if not triton_format:
+            record["payload"] = [payload]
+        else:
+            record.update(payload)
+        if row.timestamp:
+            record["timestamp"] = [row.timestamp]
+
+        return record

genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py

@@ -56,6 +56,9 @@ def convert(
                 payload = {
                     "input": [{"type": "image_url", "url": img} for img in row.images]
                 }
-                request_body["data"].append({"payload": [payload]})
+
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row)
+                )
 
         return request_body

genai-perf/genai_perf/inputs/converters/nvclip_converter.py

@@ -60,7 +60,8 @@ def convert(
                     "input": input_items,
                 }
 
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row)
+                )
 
         return request_body

genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py

@@ -62,7 +62,9 @@ def convert(
         for file_data in generic_dataset.files_data.values():
             for index, row in enumerate(file_data.rows):
                 payload = self._create_payload(index, row, config)
-                request_body["data"].append({"payload": [payload]})
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row)
+                )
 
         return request_body
 
@@ -82,7 +84,6 @@ def _create_payload(
             ],
         }
 
-        self._add_request_params(payload, config)
         return payload
 
     def _retrieve_content(

genai-perf/genai_perf/inputs/converters/openai_completions_converter.py

@@ -49,8 +49,9 @@ def convert(
                     "model": model_name,
                     "prompt": prompt,
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py

@@ -53,7 +53,9 @@ def convert(
                     "model": model_name,
                     "input": row.texts,
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row)
+                )
 
         return request_body

genai-perf/genai_perf/inputs/converters/rankings_converter.py

@@ -79,8 +79,9 @@ def convert(
                     "model": model_name,
                 }
 
-            self._add_request_params(payload, config)
-            request_body["data"].append({"payload": [payload]})
+            request_body["data"].append(
+                self._finalize_payload(payload, config, passage_entry)
+            )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py

@@ -61,8 +61,10 @@ def convert(
                     "text_input": [text],
                     "max_tokens": [DEFAULT_TENSORRTLLM_MAX_TOKENS],  # default
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append(payload)
+
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row, triton_format=True)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py

@@ -61,8 +61,10 @@ def convert(
                     "input_lengths": [len(token_ids)],
                     "request_output_len": [DEFAULT_TENSORRTLLM_MAX_TOKENS],
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append(payload)
+
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row, triton_format=True)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/triton_generate_converter.py

@@ -55,8 +55,13 @@ def convert(
                 payload = {
                     "text_input": prompt,
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append({"payload": [payload]})
+                request_body["data"].append(
+                    self._finalize_payload(
+                        payload,
+                        config,
+                        row,
+                    )
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/converters/vllm_converter.py

@@ -61,8 +61,9 @@ def convert(
                     "text_input": text,
                     "exclude_input_in_output": [True],  # default
                 }
-                self._add_request_params(payload, config)
-                request_body["data"].append(payload)
+                request_body["data"].append(
+                    self._finalize_payload(payload, config, row, triton_format=True)
+                )
 
         return request_body
 

genai-perf/genai_perf/inputs/input_constants.py

@@ -25,7 +25,6 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from enum import Enum, auto
-from typing import Dict
 
 
 class ModelSelectionStrategy(Enum):

genai-perf/genai_perf/inputs/inputs_config.py

@@ -82,6 +82,9 @@ class InputsConfig:
     # The filenames used for synthetic data generation
     synthetic_input_filenames: Optional[List[str]] = field(default_factory=list)
 
+    # The filename where payload input data is available
+    payload_input_filename: Optional[Path] = Path("")
+
     # The compression format of the images.
     image_format: ImageFormat = ImageFormat.PNG
 

genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py

@@ -0,0 +1,79 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from pathlib import Path
+from typing import Any, Dict, List, Tuple, Union
+
+from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever
+from genai_perf.inputs.retrievers.generic_dataset import FileData, GenericDataset
+
+
+class BaseFileInputRetriever(BaseInputRetriever):
+    """
+    A base input retriever class that defines file input methods.
+    """
+
+    def _verify_file(self, filename: Path) -> None:
+        """
+        Verifies that the file exists.
+
+        Args
+        ----------
+        filename : Path
+            The file path to verify.
+
+        Raises
+        ------
+        FileNotFoundError
+            If the file does not exist.
+        """
+        if not filename.exists():
+            raise FileNotFoundError(f"The file '{filename}' does not exist.")
+
+    def _get_content_from_input_file(self, filename: Path) -> Union[
+        Tuple[List[str], List[str]],
+        Tuple[List[str], List[str], List[Dict[Any, Any]]],
+    ]:
+        """
+        Reads the content from a JSONL file and returns lists of each content type.
+
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")
+
+    def _get_input_dataset_from_file(self, filename: Path) -> FileData:
+        """
+        Retrieves the dataset from a specific JSONL file.
+
+        """
+
+        raise NotImplementedError("This method should be implemented by subclasses.")
+
+    def retrieve_data(self) -> GenericDataset:
+        """
+        Retrieves the dataset from a file or directory.
+        """
+        raise NotImplementedError("This method should be implemented by subclasses.")

genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py

@@ -31,8 +31,9 @@
 from genai_perf import utils
 from genai_perf.exceptions import GenAIPerfException
 from genai_perf.inputs.input_constants import DEFAULT_BATCH_SIZE
-from genai_perf.inputs.inputs_config import InputsConfig
-from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever
+from genai_perf.inputs.retrievers.base_file_input_retriever import (
+    BaseFileInputRetriever,
+)
 from genai_perf.inputs.retrievers.generic_dataset import (
     DataRow,
     FileData,
@@ -46,7 +47,7 @@
 from PIL import Image
 
 
-class FileInputRetriever(BaseInputRetriever):
+class FileInputRetriever(BaseFileInputRetriever):
     """
     A input retriever class that handles input data provided by the user through
     file and directories.
@@ -118,24 +119,7 @@ def _get_input_dataset_from_file(self, filename: Path) -> FileData:
         """
         self._verify_file(filename)
         prompts, images = self._get_content_from_input_file(filename)
-        return self._convert_content_to_data_file(prompts, images, filename)
-
-    def _verify_file(self, filename: Path) -> None:
-        """
-        Verifies that the file exists.
-
-        Args
-        ----------
-        filename : Path
-            The file path to verify.
-
-        Raises
-        ------
-        FileNotFoundError
-            If the file does not exist.
-        """
-        if not filename.exists():
-            raise FileNotFoundError(f"The file '{filename}' does not exist.")
+        return self._convert_content_to_data_file(prompts, filename, images)
 
     def _get_content_from_input_file(
         self, filename: Path
@@ -224,7 +208,7 @@ def _encode_image(self, filename: str) -> str:
         return payload
 
     def _convert_content_to_data_file(
-        self, prompts: List[str], images: List[str], filename: Path
+        self, prompts: List[str], filename: Path, images: List[str] = []
     ) -> FileData:
         """
         Converts the content to a DataFile.