diff --git a/genai-perf/genai_perf/export_data/json_exporter.py b/genai-perf/genai_perf/export_data/json_exporter.py index 33ed30da..b17cda71 100644 --- a/genai-perf/genai_perf/export_data/json_exporter.py +++ b/genai-perf/genai_perf/export_data/json_exporter.py @@ -59,7 +59,6 @@ def export(self) -> None: 0 ] filename = self._output_dir / f"{prefix}_genai_perf.json" - logger.info(f"Generating {filename}") with open(str(filename), "w") as f: f.write(json.dumps(self._stats_and_args, indent=2)) @@ -67,6 +66,7 @@ def _prepare_args_for_export(self) -> None: self._args.pop("func", None) self._args.pop("output_format", None) self._args.pop("input_file", None) + self._args.pop("payload_input_file", None) self._args["profile_export_file"] = str(self._args["profile_export_file"]) self._args["artifact_dir"] = str(self._args["artifact_dir"]) for k, v in self._args.items(): diff --git a/genai-perf/genai_perf/inputs/converters/base_converter.py b/genai-perf/genai_perf/inputs/converters/base_converter.py index c1713bb6..62f57a5e 100644 --- a/genai-perf/genai_perf/inputs/converters/base_converter.py +++ b/genai-perf/genai_perf/inputs/converters/base_converter.py @@ -70,3 +70,9 @@ def _add_request_params( ) -> None: for key, value in config.extra_inputs.items(): payload[key] = value + + def _add_payload_params( + self, payload: Dict[Any, Any], optional_data: Dict[Any, Any] + ) -> None: + for key, value in optional_data.items(): + payload[key] = value diff --git a/genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py b/genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py index 5485cd04..673fef8f 100644 --- a/genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py +++ b/genai-perf/genai_perf/inputs/converters/image_retrieval_converter.py @@ -56,6 +56,7 @@ def convert( payload = { "input": [{"type": "image_url", "url": img} for img in row.images] } + self._add_payload_params(payload, row.optional_data) request_body["data"].append({"payload": [payload]}) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/nvclip_converter.py b/genai-perf/genai_perf/inputs/converters/nvclip_converter.py index 1e66b6b8..868af9e3 100644 --- a/genai-perf/genai_perf/inputs/converters/nvclip_converter.py +++ b/genai-perf/genai_perf/inputs/converters/nvclip_converter.py @@ -61,6 +61,7 @@ def convert( } self._add_request_params(payload, config) + self._add_payload_params(payload, row.optional_data) request_body["data"].append({"payload": [payload]}) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py b/genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py index 0c6e960d..36970f1d 100644 --- a/genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py +++ b/genai-perf/genai_perf/inputs/converters/openai_chat_completions_converter.py @@ -83,6 +83,7 @@ def _create_payload( } self._add_request_params(payload, config) + self._add_payload_params(payload, row.optional_data) return payload def _retrieve_content( diff --git a/genai-perf/genai_perf/inputs/converters/openai_completions_converter.py b/genai-perf/genai_perf/inputs/converters/openai_completions_converter.py index 29813ae4..5fae06f1 100644 --- a/genai-perf/genai_perf/inputs/converters/openai_completions_converter.py +++ b/genai-perf/genai_perf/inputs/converters/openai_completions_converter.py @@ -50,6 +50,7 @@ def convert( "prompt": prompt, } self._add_request_params(payload, config) + self._add_payload_params(payload, row.optional_data) request_body["data"].append({"payload": [payload]}) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py b/genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py index 5cb7c2a3..71175be1 100644 --- a/genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py +++ b/genai-perf/genai_perf/inputs/converters/openai_embeddings_converter.py @@ -53,7 +53,9 @@ def convert( "model": model_name, "input": row.texts, } + self._add_request_params(payload, config) + self._add_payload_params(payload, row.optional_data) request_body["data"].append({"payload": [payload]}) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/rankings_converter.py b/genai-perf/genai_perf/inputs/converters/rankings_converter.py index f3613108..7e2966e2 100644 --- a/genai-perf/genai_perf/inputs/converters/rankings_converter.py +++ b/genai-perf/genai_perf/inputs/converters/rankings_converter.py @@ -80,6 +80,7 @@ def convert( } self._add_request_params(payload, config) + self._add_payload_params(payload, passage_entry.optional_data) request_body["data"].append({"payload": [payload]}) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py b/genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py index 551b05d7..0c0e0a1f 100644 --- a/genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py +++ b/genai-perf/genai_perf/inputs/converters/tensorrtllm_converter.py @@ -61,7 +61,9 @@ def convert( "text_input": [text], "max_tokens": [DEFAULT_TENSORRTLLM_MAX_TOKENS], # default } + self._add_request_params(payload, config) + self._add_payload_params(payload, row.optional_data) request_body["data"].append(payload) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py b/genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py index a6f74063..3c85ca68 100644 --- a/genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py +++ b/genai-perf/genai_perf/inputs/converters/tensorrtllm_engine_converter.py @@ -61,7 +61,9 @@ def convert( "input_lengths": [len(token_ids)], "request_output_len": [DEFAULT_TENSORRTLLM_MAX_TOKENS], } + self._add_request_params(payload, config) + self._add_payload_params(payload, row.optional_data) request_body["data"].append(payload) return request_body diff --git a/genai-perf/genai_perf/inputs/converters/vllm_converter.py b/genai-perf/genai_perf/inputs/converters/vllm_converter.py index bd17aa4f..c4f7e159 100644 --- a/genai-perf/genai_perf/inputs/converters/vllm_converter.py +++ b/genai-perf/genai_perf/inputs/converters/vllm_converter.py @@ -61,7 +61,9 @@ def convert( "text_input": text, "exclude_input_in_output": [True], # default } + optional_data = row.optional_data self._add_request_params(payload, config) + self._add_payload_params(payload, optional_data) request_body["data"].append(payload) return request_body diff --git a/genai-perf/genai_perf/inputs/input_constants.py b/genai-perf/genai_perf/inputs/input_constants.py index bf088fe5..4377d149 100644 --- a/genai-perf/genai_perf/inputs/input_constants.py +++ b/genai-perf/genai_perf/inputs/input_constants.py @@ -25,7 +25,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from enum import Enum, auto -from typing import Dict class ModelSelectionStrategy(Enum): diff --git a/genai-perf/genai_perf/inputs/inputs_config.py b/genai-perf/genai_perf/inputs/inputs_config.py index 5b1429f5..e8df47a4 100644 --- a/genai-perf/genai_perf/inputs/inputs_config.py +++ b/genai-perf/genai_perf/inputs/inputs_config.py @@ -82,6 +82,9 @@ class InputsConfig: # The filenames used for synthetic data generation synthetic_input_filenames: Optional[List[str]] = field(default_factory=list) + # The filename where payload input data is available + payload_input_filename: Optional[Path] = Path("") + # The compression format of the images. image_format: ImageFormat = ImageFormat.PNG diff --git a/genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py b/genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py new file mode 100644 index 00000000..205f9215 --- /dev/null +++ b/genai-perf/genai_perf/inputs/retrievers/base_file_input_retriever.py @@ -0,0 +1,78 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from pathlib import Path +from typing import Any, Dict, List, Tuple, Union + +from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever +from genai_perf.inputs.retrievers.generic_dataset import FileData, GenericDataset + + +class BaseFileInputRetriever(BaseInputRetriever): + """ + A base input retriever class that defines file input methods. + """ + + def retrieve_data(self) -> GenericDataset: + """ + Retrieves the dataset from a file or directory. + """ + raise NotImplementedError("This method should be implemented by subclasses.") + + def _get_input_dataset_from_file(self, filename: Path) -> FileData: + """ + Retrieves the dataset from a specific JSONL file. + + """ + + raise NotImplementedError("This method should be implemented by subclasses.") + + def _verify_file(self, filename: Path) -> None: + """ + Verifies that the file exists. + + Args + ---------- + filename : Path + The file path to verify. + + Raises + ------ + FileNotFoundError + If the file does not exist. + """ + if not filename.exists(): + raise FileNotFoundError(f"The file '{filename}' does not exist.") + + def _get_content_from_input_file( + self, filename: Path + ) -> Union[Tuple[List[str], List[str]], Tuple[List[str], List[Dict[Any, Any]]]]: + """ + Reads the content from a JSONL file and returns lists of each content type. + + """ + raise NotImplementedError("This method should be implemented by subclasses.") diff --git a/genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py b/genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py index a307e123..bd337c49 100644 --- a/genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py +++ b/genai-perf/genai_perf/inputs/retrievers/file_input_retriever.py @@ -31,8 +31,9 @@ from genai_perf import utils from genai_perf.exceptions import GenAIPerfException from genai_perf.inputs.input_constants import DEFAULT_BATCH_SIZE -from genai_perf.inputs.inputs_config import InputsConfig -from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever +from genai_perf.inputs.retrievers.base_file_input_retriever import ( + BaseFileInputRetriever, +) from genai_perf.inputs.retrievers.generic_dataset import ( DataRow, FileData, @@ -46,7 +47,7 @@ from PIL import Image -class FileInputRetriever(BaseInputRetriever): +class FileInputRetriever(BaseFileInputRetriever): """ A input retriever class that handles input data provided by the user through file and directories. @@ -118,24 +119,7 @@ def _get_input_dataset_from_file(self, filename: Path) -> FileData: """ self._verify_file(filename) prompts, images = self._get_content_from_input_file(filename) - return self._convert_content_to_data_file(prompts, images, filename) - - def _verify_file(self, filename: Path) -> None: - """ - Verifies that the file exists. - - Args - ---------- - filename : Path - The file path to verify. - - Raises - ------ - FileNotFoundError - If the file does not exist. - """ - if not filename.exists(): - raise FileNotFoundError(f"The file '{filename}' does not exist.") + return self._convert_content_to_data_file(prompts, filename, images) def _get_content_from_input_file( self, filename: Path @@ -224,7 +208,7 @@ def _encode_image(self, filename: str) -> str: return payload def _convert_content_to_data_file( - self, prompts: List[str], images: List[str], filename: Path + self, prompts: List[str], filename: Path, images: List[str] = [] ) -> FileData: """ Converts the content to a DataFile. diff --git a/genai-perf/genai_perf/inputs/retrievers/generic_dataset.py b/genai-perf/genai_perf/inputs/retrievers/generic_dataset.py index a38b2b25..bfeaae6a 100644 --- a/genai-perf/genai_perf/inputs/retrievers/generic_dataset.py +++ b/genai-perf/genai_perf/inputs/retrievers/generic_dataset.py @@ -25,12 +25,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from dataclasses import dataclass, field -from typing import Dict, List, TypeAlias +from typing import Any, Dict, List, TypeAlias, Union Filename: TypeAlias = str TypeOfData: TypeAlias = str ListOfData: TypeAlias = List[str] -DataRowDict: TypeAlias = Dict[TypeOfData, ListOfData] +DataRowDict: TypeAlias = Dict[str, Union[List[str], Dict[str, Any], str]] GenericDatasetDict: TypeAlias = Dict[Filename, List[DataRowDict]] @@ -38,12 +38,21 @@ class DataRow: texts: List[str] = field(default_factory=list) images: List[str] = field(default_factory=list) + optional_data: Dict[str, Any] = field(default_factory=dict) def to_dict(self) -> DataRowDict: """ Converts the DataRow object to a dictionary. """ - return {"texts": self.texts, "images": self.images} + datarow_dict: DataRowDict = {} + + if self.texts: + datarow_dict["texts"] = self.texts + if self.images: + datarow_dict["images"] = self.images + if self.optional_data: + datarow_dict["optional_data"] = self.optional_data + return datarow_dict @dataclass @@ -55,8 +64,8 @@ def to_list(self) -> List[DataRowDict]: Converts the FileData object to a list. Output format example for two payloads from a file: [ - {'texts': ['text1', 'text2'], 'images': ['image1', 'image2']}, - {'texts': ['text3', 'text4'], 'images': ['image3', 'image4']} + {'texts': ['text1', 'text2'], 'images': ['image1', 'image2'], 'optional_data': {}, 'session_id': 'session_id1'}, + {'texts': ['text3', 'text4'], 'images': ['image3', 'image4'], 'optional_data': {}, 'session_id': 'session_id2'}, ] """ return [row.to_dict() for row in self.rows] @@ -71,8 +80,8 @@ def to_dict(self) -> GenericDatasetDict: Converts the entire DataStructure object to a dictionary. Output format example for one payload from two files: { - 'file_0': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2']}], - 'file_1': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2']}] + 'file_0': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2'], 'optional_data': {}, 'session_id': 'session_id1'}], + 'file_1': [{'texts': ['text1', 'text2'], 'images': ['image1', 'image2'], 'optional_data': {}, 'session_id': 'session_id2'}], } """ return { diff --git a/genai-perf/genai_perf/inputs/retrievers/input_retriever_factory.py b/genai-perf/genai_perf/inputs/retrievers/input_retriever_factory.py index e7cca0de..73348569 100644 --- a/genai-perf/genai_perf/inputs/retrievers/input_retriever_factory.py +++ b/genai-perf/genai_perf/inputs/retrievers/input_retriever_factory.py @@ -29,6 +29,7 @@ from genai_perf.inputs.inputs_config import InputsConfig from genai_perf.inputs.retrievers.base_input_retriever import BaseInputRetriever from genai_perf.inputs.retrievers.file_input_retriever import FileInputRetriever +from genai_perf.inputs.retrievers.payload_input_retriever import PayloadInputRetriever from genai_perf.inputs.retrievers.synthetic_data_retriever import SyntheticDataRetriever @@ -43,6 +44,7 @@ def create(config: InputsConfig) -> BaseInputRetriever: retrievers = { PromptSource.SYNTHETIC: SyntheticDataRetriever, PromptSource.FILE: FileInputRetriever, + PromptSource.PAYLOAD: PayloadInputRetriever, } input_type = config.input_type if input_type not in retrievers: diff --git a/genai-perf/genai_perf/inputs/retrievers/payload_input_retriever.py b/genai-perf/genai_perf/inputs/retrievers/payload_input_retriever.py new file mode 100644 index 00000000..463278d9 --- /dev/null +++ b/genai-perf/genai_perf/inputs/retrievers/payload_input_retriever.py @@ -0,0 +1,170 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from pathlib import Path +from typing import Any, Dict, List, Tuple, cast + +from genai_perf.exceptions import GenAIPerfException +from genai_perf.inputs.retrievers.base_file_input_retriever import ( + BaseFileInputRetriever, +) +from genai_perf.inputs.retrievers.generic_dataset import ( + DataRow, + FileData, + GenericDataset, +) +from genai_perf.inputs.retrievers.synthetic_prompt_generator import ( + SyntheticPromptGenerator, +) +from genai_perf.utils import load_json_str + + +class PayloadInputRetriever(BaseFileInputRetriever): + """ + A input retriever class that handles payload level input data provided by the user + through a file. + """ + + def retrieve_data(self) -> GenericDataset: + """ + Retrieves the dataset from a file. + + Returns + ------- + GenericDataset + The dataset containing file data. + """ + + files_data: Dict[str, FileData] = {} + input_file = cast(Path, self.config.payload_input_filename) + file_data = self._get_input_dataset_from_file(input_file) + files_data = {str(input_file): file_data} + + return GenericDataset(files_data) + + def _get_input_dataset_from_file(self, filename: Path) -> FileData: + """ + Retrieves the dataset from a specific JSONL file. + + Args + ---------- + filename : Path + The path of the file to process. + + Returns + ------- + Dict + The dataset in the required format with the content + read from the file. + """ + self._verify_file(filename) + prompts, optional_datas = self._get_content_from_input_file(filename) + return self._convert_content_to_data_file(prompts, optional_datas) + + def _get_content_from_input_file( + self, filename: Path + ) -> Tuple[List[str], List[Dict[Any, Any]]]: + """ + Reads the content from a JSONL file and returns lists of each content type. + + Args + ---------- + filename : Path + The file path from which to read the content. + + Returns + ------- + Tuple[List[str], Dict, str] + A list of prompts, and optional data. + """ + prompts = [] + optional_datas = [] + with open(filename, mode="r", newline=None) as file: + for line in file: + if line.strip(): + data = load_json_str(line) + # None if not provided + prompt = data.get("text") + prompt_alt = data.get("text_input") + # Check if only one of the keys is provided + if prompt and prompt_alt: + raise ValueError( + "Each data entry must have only one of 'text_input' or 'text' key name." + ) + # If none of the keys are provided, generate a synthetic prompt + if not prompt and not prompt_alt: + prompt = SyntheticPromptGenerator.create_synthetic_prompt( + self.config.tokenizer, + self.config.prompt_tokens_mean, + self.config.prompt_tokens_stddev, + ) + prompt = prompt if prompt else prompt_alt + prompts.append(prompt.strip() if prompt else prompt) + optional_data = self._check_for_optional_data(data) + optional_datas.append(optional_data) + return prompts, optional_datas + + def _check_for_optional_data(self, data: Dict[str, Any]) -> Dict[Any, Any]: + """ + Checks if there is any optional data in the file to pass in the payload. + """ + optional_data = {} + for k, v in data.items(): + if k not in ["text", "text_input"]: + optional_data[k] = v + return optional_data + + def _convert_content_to_data_file( + self, + prompts: List[str], + optional_datas: List[Dict[Any, Any]] = [{}], + ) -> FileData: + """ + Converts the content to a DataFile. + + Args + ---------- + prompts : List[str] + The list of prompts to convert. + optional_data : Dict + The optional data included in every payload. + + Returns + ------- + FileData + The DataFile containing the converted data. + """ + data_rows: List[DataRow] = [] + + if prompts: + for index, prompt in enumerate(prompts): + data_rows.append( + DataRow( + texts=[prompt], + optional_data=optional_datas[index], + ) + ) + return FileData(data_rows) diff --git a/genai-perf/genai_perf/parser.py b/genai-perf/genai_perf/parser.py index 450499d5..2c3dd151 100644 --- a/genai-perf/genai_perf/parser.py +++ b/genai-perf/genai_perf/parser.py @@ -478,12 +478,12 @@ def _infer_prompt_source(args: argparse.Namespace) -> argparse.Namespace: ) elif input_file_str.startswith("payload:"): args.prompt_source = ic.PromptSource.PAYLOAD - payload_input_file_str = input_file_str.split(":", 1)[1] - if not payload_input_file_str: + input_file_str = input_file_str.split(":", 1)[1] + if not input_file_str: raise ValueError( f"Invalid payload input: '{input_file_str}' is missing the file path" ) - args.payload_input_file = payload_input_file_str.split(",") + args.payload_input_file = Path(f"{input_file_str}.jsonl") logger.debug( f"Input source is a payload file with timing information in the following path: {args.payload_input_file}" ) diff --git a/genai-perf/genai_perf/subcommand/common.py b/genai-perf/genai_perf/subcommand/common.py index 21a2e8db..95c0c517 100644 --- a/genai-perf/genai_perf/subcommand/common.py +++ b/genai-perf/genai_perf/subcommand/common.py @@ -160,6 +160,7 @@ def create_config_options(args: Namespace) -> InputsConfig: model_name=args.model, model_selection_strategy=args.model_selection_strategy, input_filename=args.input_file, + payload_input_filename=args.payload_input_file, synthetic_input_filenames=args.synthetic_input_files, starting_index=DEFAULT_STARTING_INDEX, length=args.num_dataset_entries, diff --git a/genai-perf/genai_perf/wrapper.py b/genai-perf/genai_perf/wrapper.py index 2b47c730..854db4bf 100644 --- a/genai-perf/genai_perf/wrapper.py +++ b/genai-perf/genai_perf/wrapper.py @@ -24,13 +24,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json from argparse import Namespace from typing import List, Optional import genai_perf.logging as logging import genai_perf.utils as utils from genai_perf.constants import DEFAULT_GRPC_URL -from genai_perf.inputs.input_constants import DEFAULT_INPUT_DATA_JSON +from genai_perf.inputs.input_constants import DEFAULT_INPUT_DATA_JSON, PromptSource from genai_perf.inputs.inputs import OutputFormat from genai_perf.telemetry_data.triton_telemetry_data_collector import ( TelemetryDataCollector, @@ -55,13 +56,38 @@ def add_protocol_args(args: Namespace) -> List[str]: @staticmethod def add_inference_load_args(args: Namespace) -> List[str]: - cmd = [] + cmd: list[str] = [] + if args.prompt_source == PromptSource.PAYLOAD: + return cmd if args.concurrency: cmd += ["--concurrency-range", f"{args.concurrency}"] elif args.request_rate: cmd += ["--request-rate-range", f"{args.request_rate}"] return cmd + @staticmethod + def add_payload_args(args: Namespace) -> List[str]: + cmd = [] + timings = [] + + if args.prompt_source == PromptSource.PAYLOAD: + try: + with open(args.payload_input_file, "r") as file: + for line in file: + try: + timestamp = float(json.loads(line)["timestamp"]) + timings.append(timestamp) + except (KeyError, ValueError) as e: + raise ValueError( + f"Invalid line in payload file: {line.strip()}. Details: {e}" + ) + cmd += ["--schedule", ",".join(map(str, timings))] + except FileNotFoundError: + raise FileNotFoundError( + f"Payload input file not found: {args.payload_input_file}" + ) + return cmd + @staticmethod def build_cmd(args: Namespace, extra_args: Optional[List[str]] = None) -> List[str]: skip_args = [ @@ -112,6 +138,14 @@ def build_cmd(args: Namespace, extra_args: Optional[List[str]] = None) -> List[s "tokenizer_revision", ] + if args.prompt_source == PromptSource.PAYLOAD: + skip_args += [ + "request_count", + "measurement_interval", + "stability_percentage", + "warmup_request_count", + ] + utils.remove_file(args.profile_export_file) cmd = [ @@ -124,6 +158,7 @@ def build_cmd(args: Namespace, extra_args: Optional[List[str]] = None) -> List[s ] cmd += Profiler.add_protocol_args(args) cmd += Profiler.add_inference_load_args(args) + cmd += Profiler.add_payload_args(args) for arg, value in vars(args).items(): if arg in skip_args: diff --git a/genai-perf/tests/test_cli.py b/genai-perf/tests/test_cli.py index 44dee315..b7aa77bb 100644 --- a/genai-perf/tests/test_cli.py +++ b/genai-perf/tests/test_cli.py @@ -844,7 +844,7 @@ def test_goodput_args_warning(self, monkeypatch, args, expected_error): assert str(exc_info.value) == expected_error @pytest.mark.parametrize( - "args, expected_prompt_source, expected_payload_input_file, expect_error", + "args, expected_prompt_source, expected_input_file, expect_error", [ ([], PromptSource.SYNTHETIC, None, False), (["--input-file", "prompt.txt"], PromptSource.FILE, None, False), @@ -855,14 +855,14 @@ def test_goodput_args_warning(self, monkeypatch, args, expected_error): False, ), ( - ["--input-file", "payload:test.jsonl"], + ["--input-file", "payload:test"], PromptSource.PAYLOAD, - ["test.jsonl"], + Path("test.jsonl"), False, ), (["--input-file", "payload:"], PromptSource.PAYLOAD, [], True), ( - ["--input-file", "synthetic:test.jsonl"], + ["--input-file", "synthetic:test"], PromptSource.SYNTHETIC, None, False, @@ -876,7 +876,7 @@ def test_inferred_prompt_source( mocker, args, expected_prompt_source, - expected_payload_input_file, + expected_input_file, expect_error, ): mocker.patch.object(Path, "is_file", return_value=True) @@ -891,8 +891,8 @@ def test_inferred_prompt_source( assert args.prompt_source == expected_prompt_source - if expected_payload_input_file is not None: - assert args.payload_input_file == expected_payload_input_file + if expected_input_file is not None: + assert args.payload_input_file == expected_input_file @pytest.mark.parametrize( "args", diff --git a/genai-perf/tests/test_converters/test_embeddings_converter.py b/genai-perf/tests/test_converters/test_embeddings_converter.py index 9e22ab14..394de443 100644 --- a/genai-perf/tests/test_converters/test_embeddings_converter.py +++ b/genai-perf/tests/test_converters/test_embeddings_converter.py @@ -213,3 +213,63 @@ def test_convert_empty_dataset(self): expected_result = {"data": []} assert result == expected_result + + def test_convert_with_payload_parameters(self): + optional_data_1 = { + "timestamp": "0", + "session_id": "abcd", + "additional_key": "additional_value", + } + optional_data_2 = { + "timestamp": "3047", + "session_id": "cdef", + } + + generic_dataset = GenericDataset( + files_data={ + "file1": FileData( + rows=[ + DataRow(texts=["text_1"], optional_data=optional_data_1), + DataRow(texts=["text_2"], optional_data=optional_data_2), + ], + ) + } + ) + + config = InputsConfig( + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + output_format=OutputFormat.OPENAI_EMBEDDINGS, + tokenizer=get_empty_tokenizer(), + ) + + converter = OpenAIEmbeddingsConverter() + result = converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "payload": [ + { + "input": ["text_1"], + "model": "test_model", + "timestamp": "0", + "session_id": "abcd", + "additional_key": "additional_value", + } + ] + }, + { + "payload": [ + { + "input": ["text_2"], + "model": "test_model", + "timestamp": "3047", + "session_id": "cdef", + } + ] + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_image_retrieval_converter.py b/genai-perf/tests/test_converters/test_image_retrieval_converter.py index fa59e4ca..606f68eb 100644 --- a/genai-perf/tests/test_converters/test_image_retrieval_converter.py +++ b/genai-perf/tests/test_converters/test_image_retrieval_converter.py @@ -37,7 +37,6 @@ class TestImageRetrievalConverter: - @staticmethod def create_generic_dataset() -> GenericDataset: return GenericDataset( @@ -50,7 +49,23 @@ def create_generic_dataset() -> GenericDataset: } ) - def test_convert_default(self) -> None: + @staticmethod + def create_generic_dataset_with_payload_parameters() -> GenericDataset: + optional_data = {"timestamp": "0", "session_id": "abcd"} + return GenericDataset( + files_data={ + "file1": FileData( + rows=[ + DataRow( + images=["test_image_1", "test_image_2"], + optional_data=optional_data, + ) + ] + ) + } + ) + + def test_convert_default(self): """ Test Image Retrieval request payload """ @@ -81,3 +96,34 @@ def test_convert_default(self) -> None: } assert result == expected_result + + def test_convert_with_payload_parameters(self): + generic_dataset = self.create_generic_dataset_with_payload_parameters() + + config = InputsConfig( + extra_inputs={}, + output_format=OutputFormat.IMAGE_RETRIEVAL, + tokenizer=get_empty_tokenizer(), + ) + + image_retrieval_converter = ImageRetrievalConverter() + result = image_retrieval_converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "payload": [ + { + "input": [ + {"type": "image_url", "url": "test_image_1"}, + {"type": "image_url", "url": "test_image_2"}, + ], + "timestamp": "0", + "session_id": "abcd", + } + ] + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_nvclip_converter.py b/genai-perf/tests/test_converters/test_nvclip_converter.py index db87924f..c33da9cf 100644 --- a/genai-perf/tests/test_converters/test_nvclip_converter.py +++ b/genai-perf/tests/test_converters/test_nvclip_converter.py @@ -24,6 +24,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import copy + import pytest from genai_perf.exceptions import GenAIPerfException from genai_perf.inputs.converters import NVClipConverter @@ -43,7 +45,16 @@ class TestNVClipConverter: def create_generic_dataset(rows) -> GenericDataset: return GenericDataset(files_data={"file1": FileData(rows)}) - def test_convert_default(self): + @pytest.fixture + def default_config(self): + yield InputsConfig( + extra_inputs={}, + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + tokenizer=get_empty_tokenizer(), + ) + + def test_convert_default(self, default_config): generic_dataset = self.create_generic_dataset( [ DataRow(texts=["text1"], images=["image1"]), @@ -52,15 +63,8 @@ def test_convert_default(self): ] ) - config = InputsConfig( - extra_inputs={}, - model_name=["test_model"], - model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, - tokenizer=get_empty_tokenizer(), - ) - nv_clip_converter = NVClipConverter() - result = nv_clip_converter.convert(generic_dataset, config) + result = nv_clip_converter.convert(generic_dataset, default_config) expected_result = { "data": [ @@ -93,22 +97,15 @@ def test_convert_default(self): assert result == expected_result - def test_convert_batched(self): + def test_convert_batched(self, default_config): generic_dataset = self.create_generic_dataset( [ DataRow(texts=["text1", "text2"], images=["image1", "image2"]), ] ) - config = InputsConfig( - extra_inputs={}, - model_name=["test_model"], - model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, - tokenizer=get_empty_tokenizer(), - ) - nv_clip_converter = NVClipConverter() - result = nv_clip_converter.convert(generic_dataset, config) + result = nv_clip_converter.convert(generic_dataset, default_config) expected_result = { "data": [ @@ -125,7 +122,7 @@ def test_convert_batched(self): assert result == expected_result - def test_convert_with_request_parameters(self): + def test_convert_with_request_parameters(self, default_config): generic_dataset = self.create_generic_dataset( [ DataRow(texts=["text1"], images=["image1"]), @@ -133,13 +130,8 @@ def test_convert_with_request_parameters(self): ) extra_inputs = {"encoding_format": "base64"} - - config = InputsConfig( - extra_inputs=extra_inputs, - model_name=["test_model"], - model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, - tokenizer=get_empty_tokenizer(), - ) + config = copy.deepcopy(default_config) + config.extra_inputs.update(extra_inputs) nv_clip_converter = NVClipConverter() result = nv_clip_converter.convert(generic_dataset, config) @@ -169,3 +161,32 @@ def test_check_config_raises_exception_for_streaming(self): GenAIPerfException, match="The --streaming option is not supported" ): nv_clip_converter.check_config(config) + + def test_convert_with_payload_parameters(self, default_config): + optional_data = {"timestamp": "0", "session_id": "abcd"} + generic_dataset = self.create_generic_dataset( + [ + DataRow( + texts=["text1"], images=["image1"], optional_data=optional_data + ), + ] + ) + + nv_clip_converter = NVClipConverter() + result = nv_clip_converter.convert(generic_dataset, default_config) + expected_result = { + "data": [ + { + "payload": [ + { + "model": "test_model", + "input": ["text1", "image1"], + "timestamp": "0", + "session_id": "abcd", + } + ] + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_openai_chat_converter.py b/genai-perf/tests/test_converters/test_openai_chat_converter.py index baa0261d..0f0d327f 100644 --- a/genai-perf/tests/test_converters/test_openai_chat_converter.py +++ b/genai-perf/tests/test_converters/test_openai_chat_converter.py @@ -58,11 +58,21 @@ def clean_image(row): return [image] return [] + def clean_optional_data(row): + optional_data = row.get("optional_data", {}) + if isinstance(optional_data, Dict): + return optional_data + return {} + return GenericDataset( files_data={ "file1": FileData( rows=[ - DataRow(texts=clean_text(row), images=clean_image(row)) + DataRow( + texts=clean_text(row), + images=clean_image(row), + optional_data=clean_optional_data(row), + ) for row in rows ], ) @@ -303,3 +313,43 @@ def test_convert_multi_modal( } assert result == expected_result + + def test_convert_with_payload_parameters(self): + optional_data = {"timestamp": "0", "session_id": "abcd"} + generic_dataset = self.create_generic_dataset( + [{"text": "text input one", "optional_data": optional_data}] + ) + + config = InputsConfig( + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + output_format=OutputFormat.OPENAI_CHAT_COMPLETIONS, + tokenizer=get_empty_tokenizer(), + add_stream=True, + ) + + chat_converter = OpenAIChatCompletionsConverter() + result = chat_converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "payload": [ + { + "messages": [ + { + "role": "user", + "content": "text input one", + } + ], + "model": "test_model", + "stream": True, + "timestamp": "0", + "session_id": "abcd", + } + ] + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_openai_completions_converter.py b/genai-perf/tests/test_converters/test_openai_completions_converter.py index 52b08ddb..c4ca8105 100644 --- a/genai-perf/tests/test_converters/test_openai_completions_converter.py +++ b/genai-perf/tests/test_converters/test_openai_completions_converter.py @@ -55,6 +55,30 @@ def create_generic_dataset() -> GenericDataset: } ) + @staticmethod + def create_generic_dataset_with_payload_parameters() -> GenericDataset: + optional_data_1 = {"timestamp": "0", "session_id": "abcd"} + optional_data_2 = { + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + return GenericDataset( + files_data={ + "file1": FileData( + rows=[ + DataRow( + texts=["text input one"], optional_data=optional_data_1 + ), + DataRow( + texts=["text input two"], optional_data=optional_data_2 + ), + ], + ) + } + ) + def test_convert_default(self): generic_dataset = self.create_generic_dataset() @@ -258,3 +282,46 @@ def test_convert_with_multiple_models(self): } assert result == expected_result + + def test_convert_with_payload_parameters(self): + generic_dataset = self.create_generic_dataset_with_payload_parameters() + + config = InputsConfig( + extra_inputs={}, + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + output_format=OutputFormat.OPENAI_COMPLETIONS, + tokenizer=get_empty_tokenizer(), + ) + + completions_converter = OpenAICompletionsConverter() + result = completions_converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "payload": [ + { + "prompt": ["text input one"], + "model": "test_model", + "timestamp": "0", + "session_id": "abcd", + } + ] + }, + { + "payload": [ + { + "prompt": ["text input two"], + "model": "test_model", + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + ] + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_rankings_converter.py b/genai-perf/tests/test_converters/test_rankings_converter.py index ad9cb7cc..5f57bc9b 100644 --- a/genai-perf/tests/test_converters/test_rankings_converter.py +++ b/genai-perf/tests/test_converters/test_rankings_converter.py @@ -24,7 +24,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from typing import List, Optional +from typing import Any, Dict, List, Optional import pytest from genai_perf.inputs.converters import RankingsConverter @@ -59,6 +59,36 @@ def create_generic_dataset( return GenericDataset(files_data=files_data) + @staticmethod + def create_generic_dataset_payload_parameters( + queries_data: Optional[List[List[str]]] = None, + passages_data: Optional[List[List[str]]] = None, + optional_data: Optional[List[Dict[Any, Any]]] = None, + ) -> GenericDataset: + files_data = {} + + if queries_data is not None: + files_data["queries"] = FileData( + rows=[DataRow(texts=query) for query in queries_data], + ) + + if passages_data is not None: + files_data["passages"] = FileData( + rows=[ + DataRow( + texts=passage, + optional_data=( + optional_data[index] + if optional_data and index < len(optional_data) + else {} + ), + ) + for index, passage in enumerate(passages_data) + ], + ) + + return GenericDataset(files_data=files_data) + def test_convert_default(self): generic_dataset = self.create_generic_dataset( queries_data=[["query 1"], ["query 2"]], @@ -324,3 +354,65 @@ def test_convert_mismatched_queries_and_passages( result = rankings_converter.convert(generic_dataset, config) assert result == expected_result + + def test_convert_with_payload_parameters(self): + optional_data_1 = {"timestamp": "0", "session_id": "abcd"} + optional_data_2 = { + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + generic_dataset = self.create_generic_dataset_payload_parameters( + queries_data=[["query 1"], ["query 2"]], + passages_data=[["passage 1", "passage 2"], ["passage 3", "passage 4"]], + optional_data=[optional_data_1, optional_data_2], + ) + + config = InputsConfig( + extra_inputs={}, + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + output_format=OutputFormat.RANKINGS, + tokenizer=get_empty_tokenizer(), + ) + + rankings_converter = RankingsConverter() + result = rankings_converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "payload": [ + { + "query": {"text": "query 1"}, + "passages": [ + {"text": "passage 1"}, + {"text": "passage 2"}, + ], + "model": "test_model", + "timestamp": "0", + "session_id": "abcd", + } + ] + }, + { + "payload": [ + { + "query": {"text": "query 2"}, + "passages": [ + {"text": "passage 3"}, + {"text": "passage 4"}, + ], + "model": "test_model", + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + ] + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_tensorrtllm_engine_converter.py b/genai-perf/tests/test_converters/test_tensorrtllm_engine_converter.py index 1de20d13..ccd8b59f 100644 --- a/genai-perf/tests/test_converters/test_tensorrtllm_engine_converter.py +++ b/genai-perf/tests/test_converters/test_tensorrtllm_engine_converter.py @@ -57,6 +57,30 @@ def create_generic_dataset() -> GenericDataset: } ) + @staticmethod + def create_generic_dataset_with_payload_parameters() -> GenericDataset: + optional_data_1 = {"timestamp": "0", "session_id": "abcd"} + optional_data_2 = { + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + return GenericDataset( + files_data={ + "file1": FileData( + rows=[ + DataRow( + texts=["text input one"], optional_data=optional_data_1 + ), + DataRow( + texts=["text input two"], optional_data=optional_data_2 + ), + ], + ) + } + ) + def test_convert_default(self): generic_dataset = self.create_generic_dataset() @@ -160,3 +184,46 @@ def test_check_config_invalid_batch_size(self): assert str(exc_info.value) == ( "The --batch-size-text flag is not supported for tensorrtllm_engine." ) + + def test_convert_with_payload_parameters(self): + generic_dataset = self.create_generic_dataset_with_payload_parameters() + + config = InputsConfig( + extra_inputs={}, + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + output_format=OutputFormat.TENSORRTLLM_ENGINE, + tokenizer=get_tokenizer(DEFAULT_TOKENIZER), + ) + + trtllm_engine_converter = TensorRTLLMEngineConverter() + result = trtllm_engine_converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "input_ids": { + "content": [1426, 1881, 697], + "shape": [3], + }, + "input_lengths": [3], + "request_output_len": [DEFAULT_TENSORRTLLM_MAX_TOKENS], + "timestamp": "0", + "session_id": "abcd", + }, + { + "input_ids": { + "content": [1426, 1881, 1023], + "shape": [3], + }, + "input_lengths": [3], + "request_output_len": [DEFAULT_TENSORRTLLM_MAX_TOKENS], + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_triton_tensorrtllm_converter.py b/genai-perf/tests/test_converters/test_triton_tensorrtllm_converter.py index 95d3315a..caad3faa 100644 --- a/genai-perf/tests/test_converters/test_triton_tensorrtllm_converter.py +++ b/genai-perf/tests/test_converters/test_triton_tensorrtllm_converter.py @@ -24,6 +24,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import copy + +import pytest from genai_perf.inputs.converters import TensorRTLLMConverter from genai_perf.inputs.input_constants import ( DEFAULT_TENSORRTLLM_MAX_TOKENS, @@ -55,10 +58,33 @@ def create_generic_dataset(): } ) - def test_convert_default(self): - generic_dataset = self.create_generic_dataset() + @staticmethod + def create_generic_dataset_with_payload_parameters(): + optional_data_1 = {"timestamp": "0", "session_id": "abcd"} + optional_data_2 = { + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + return GenericDataset( + files_data={ + "file1": FileData( + rows=[ + DataRow( + texts=["text input one"], optional_data=optional_data_1 + ), + DataRow( + texts=["text input two"], optional_data=optional_data_2 + ), + ], + ) + } + ) - config = InputsConfig( + @pytest.fixture + def default_config(self): + yield InputsConfig( extra_inputs={}, model_name=["test_model"], model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, @@ -66,8 +92,11 @@ def test_convert_default(self): tokenizer=get_empty_tokenizer(), ) + def test_convert_default(self, default_config): + generic_dataset = self.create_generic_dataset() + trtllm_converter = TensorRTLLMConverter() - result = trtllm_converter.convert(generic_dataset, config) + result = trtllm_converter.convert(generic_dataset, default_config) expected_result = { "data": [ @@ -86,7 +115,7 @@ def test_convert_default(self): assert result == expected_result - def test_convert_with_request_parameters(self): + def test_convert_with_request_parameters(self, default_config): generic_dataset = self.create_generic_dataset() extra_inputs = { @@ -95,14 +124,9 @@ def test_convert_with_request_parameters(self): "additional_key": "additional_value", } - config = InputsConfig( - extra_inputs=extra_inputs, - model_name=["test_model"], - model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, - output_format=OutputFormat.TENSORRTLLM, - add_stream=True, - tokenizer=get_empty_tokenizer(), - ) + config = copy.deepcopy(default_config) + config.add_stream = True + config.extra_inputs.update(extra_inputs) trtllm_converter = TensorRTLLMConverter() result = trtllm_converter.convert(generic_dataset, config) @@ -130,19 +154,40 @@ def test_convert_with_request_parameters(self): assert result == expected_result - def test_convert_empty_dataset(self): + def test_convert_empty_dataset(self, default_config): generic_dataset = GenericDataset(files_data={}) - config = InputsConfig( - extra_inputs={}, - model_name=["test_model"], - model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, - output_format=OutputFormat.TENSORRTLLM, - tokenizer=get_empty_tokenizer(), - ) - trtllm_converter = TensorRTLLMConverter() - result = trtllm_converter.convert(generic_dataset, config) + result = trtllm_converter.convert(generic_dataset, default_config) expected_result = {"data": []} assert result == expected_result + + def test_convert_with_payload_parameters(self, default_config): + generic_dataset = self.create_generic_dataset_with_payload_parameters() + + trtllm_converter = TensorRTLLMConverter() + result = trtllm_converter.convert(generic_dataset, default_config) + + expected_result = { + "data": [ + { + "model": "test_model", + "text_input": ["text input one"], + "max_tokens": [DEFAULT_TENSORRTLLM_MAX_TOKENS], + "timestamp": "0", + "session_id": "abcd", + }, + { + "model": "test_model", + "text_input": ["text input two"], + "max_tokens": [DEFAULT_TENSORRTLLM_MAX_TOKENS], + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_converters/test_triton_vllm_converter.py b/genai-perf/tests/test_converters/test_triton_vllm_converter.py index 1231c5d7..8d385541 100644 --- a/genai-perf/tests/test_converters/test_triton_vllm_converter.py +++ b/genai-perf/tests/test_converters/test_triton_vllm_converter.py @@ -53,6 +53,30 @@ def create_generic_dataset(): } ) + @staticmethod + def create_generic_dataset_with_payload_parameters(): + optional_data_1 = {"timestamp": "0", "session_id": "abcd"} + optional_data_2 = { + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + } + return GenericDataset( + files_data={ + "file1": FileData( + rows=[ + DataRow( + texts=["text input one"], optional_data=optional_data_1 + ), + DataRow( + texts=["text input two"], optional_data=optional_data_2 + ), + ], + ) + } + ) + def test_convert_default(self): generic_dataset = self.create_generic_dataset() @@ -205,3 +229,40 @@ def test_convert_empty_dataset(self): expected_result = {"data": []} assert result == expected_result + + def test_convert_with_payload_parameters(self): + generic_dataset = self.create_generic_dataset_with_payload_parameters() + + config = InputsConfig( + extra_inputs={}, + model_name=["test_model"], + model_selection_strategy=ModelSelectionStrategy.ROUND_ROBIN, + output_format=OutputFormat.VLLM, + tokenizer=get_empty_tokenizer(), + ) + + vllm_converter = VLLMConverter() + result = vllm_converter.convert(generic_dataset, config) + + expected_result = { + "data": [ + { + "model": "test_model", + "text_input": ["text input one"], + "exclude_input_in_output": [True], + "timestamp": "0", + "session_id": "abcd", + }, + { + "model": "test_model", + "text_input": ["text input two"], + "exclude_input_in_output": [True], + "timestamp": "2345", + "session_id": "dfwe", + "input_length": "6755", + "output_length": "500", + }, + ] + } + + assert result == expected_result diff --git a/genai-perf/tests/test_exporters/test_json_exporter.py b/genai-perf/tests/test_exporters/test_json_exporter.py index 9a2f4c5d..62d40dcb 100644 --- a/genai-perf/tests/test_exporters/test_json_exporter.py +++ b/genai-perf/tests/test_exporters/test_json_exporter.py @@ -230,7 +230,6 @@ class TestJsonExporter: "output_tokens_mean": -1, "output_tokens_mean_deterministic": false, "output_tokens_stddev": 0, - "payload_input_file": null, "random_seed": 0, "request_count": 0, "synthetic_input_files": null, diff --git a/genai-perf/tests/test_retrievers/test_input_retriever_factory.py b/genai-perf/tests/test_retrievers/test_input_retriever_factory.py index 377722d5..a2bc4c05 100644 --- a/genai-perf/tests/test_retrievers/test_input_retriever_factory.py +++ b/genai-perf/tests/test_retrievers/test_input_retriever_factory.py @@ -19,13 +19,14 @@ from genai_perf.inputs.inputs_config import InputsConfig from genai_perf.inputs.retrievers.file_input_retriever import FileInputRetriever from genai_perf.inputs.retrievers.input_retriever_factory import InputRetrieverFactory +from genai_perf.inputs.retrievers.payload_input_retriever import PayloadInputRetriever from genai_perf.inputs.retrievers.synthetic_data_retriever import SyntheticDataRetriever from genai_perf.tokenizer import get_empty_tokenizer class TestInputRetrieverFactory: - def test_create_file_retrieverg(self): + def test_create_file_retriever(self): config = InputsConfig( input_type=PromptSource.FILE, input_filename=Path("input_data.jsonl"), @@ -59,3 +60,22 @@ def test_create_synthetic_retriever(self): assert isinstance( retriever, SyntheticDataRetriever ), "Should return a SyntheticDataRetriever" + + def test_create_payload_retriever(self): + """ + Test that PayloadInputRetriever is created and passed the correct config. + """ + config = InputsConfig( + input_type=PromptSource.PAYLOAD, + payload_input_filename="test_payload_data.jsonl", + tokenizer=get_empty_tokenizer(), + ) + with patch( + "genai_perf.inputs.retrievers.payload_input_retriever.PayloadInputRetriever.__init__", + return_value=None, + ) as mock_init: + retriever = InputRetrieverFactory.create(config) + mock_init.assert_called_once_with(config) + assert isinstance( + retriever, PayloadInputRetriever + ), "Should return a PayloadInputRetriever" diff --git a/genai-perf/tests/test_retrievers/test_payload_input_retriever.py b/genai-perf/tests/test_retrievers/test_payload_input_retriever.py new file mode 100644 index 00000000..b7c27f00 --- /dev/null +++ b/genai-perf/tests/test_retrievers/test_payload_input_retriever.py @@ -0,0 +1,152 @@ +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import io +from pathlib import Path +from unittest.mock import mock_open, patch + +import pytest +from genai_perf.inputs.retrievers.generic_dataset import ( + DataRow, + FileData, + GenericDataset, +) +from genai_perf.inputs.retrievers.payload_input_retriever import PayloadInputRetriever +from genai_perf.inputs.retrievers.synthetic_prompt_generator import ( + SyntheticPromptGenerator, +) +from genai_perf.tokenizer import get_empty_tokenizer + + +class TestPayloadInputRetriever: + + @pytest.fixture + def mock_config(self): + class MockConfig: + def __init__(self): + self.tokenizer = get_empty_tokenizer() + self.model_name = ["test_model"] + self.model_selection_strategy = "round_robin" + self.payload_input_filename = Path("test_input.jsonl") + self.prompt_tokens_mean = 10 + self.prompt_tokens_stddev = 2 + + return MockConfig() + + @pytest.fixture + def retriever(self, mock_config): + return PayloadInputRetriever(mock_config) + + @pytest.mark.parametrize( + "input_data, expected_prompts, expected_optional_data", + [ + ( + '{"text": "What is AI?", "timestamp": "123", "session_id": "abc"}\n' + '{"text": "How does ML work?", "custom_field": "value"}\n', + ["What is AI?", "How does ML work?"], + [{"timestamp": "123", "session_id": "abc"}, {"custom_field": "value"}], + ), + ( + '{"text_input": "Legacy prompt", "timestamp": "456"}\n' + '{"text": "New prompt", "session_id": "def"}\n', + ["Legacy prompt", "New prompt"], + [{"timestamp": "456"}, {"session_id": "def"}], + ), + ( + '{"text": "What is AI?", "timestamp": "123", "session_id": "abc"}', + ["What is AI?"], + [{"timestamp": "123", "session_id": "abc"}], + ), + ( + '{"text_input": "Legacy prompt", "timestamp": "456"}', + ["Legacy prompt"], + [{"timestamp": "456"}], + ), + ('{"timestamp": "789"}\n', ["Synthetic prompt"], [{"timestamp": "789"}]), + ], + ) + @patch("builtins.open") + @patch.object(SyntheticPromptGenerator, "create_synthetic_prompt") + def test_get_content_from_input_file( + self, + mock_synthetic_prompt, + mock_file, + retriever, + input_data, + expected_prompts, + expected_optional_data, + ): + mock_file.return_value = io.StringIO(input_data) + mock_synthetic_prompt.return_value = "Synthetic prompt" + + prompts, optional_data = retriever._get_content_from_input_file( + Path("test_input.jsonl") + ) + + assert prompts == expected_prompts + assert optional_data == expected_optional_data + + if "text" not in input_data and "text_input" not in input_data: + mock_synthetic_prompt.assert_called_once() + + def test_convert_content_to_data_file(self, retriever): + prompts = ["Prompt 1", "Prompt 2"] + optional_data = [{"timestamp": "0"}, {"session_id": "3425"}] + + file_data = retriever._convert_content_to_data_file(prompts, optional_data) + + assert len(file_data.rows) == 2 + assert file_data.rows[0].texts == ["Prompt 1"] + assert file_data.rows[0].optional_data == {"timestamp": "0"} + assert file_data.rows[1].texts == ["Prompt 2"] + assert file_data.rows[1].optional_data == {"session_id": "3425"} + + @patch.object(PayloadInputRetriever, "_get_input_dataset_from_file") + def test_retrieve_data(self, mock_get_input, retriever): + mock_file_data = FileData( + [DataRow(texts=["Test prompt"], optional_data={"key": "value"})] + ) + mock_get_input.return_value = mock_file_data + + dataset = retriever.retrieve_data() + + assert isinstance(dataset, GenericDataset) + assert str(retriever.config.payload_input_filename) in dataset.files_data + assert ( + dataset.files_data[str(retriever.config.payload_input_filename)] + == mock_file_data + ) + + @patch("builtins.open", new_callable=mock_open) + def test_conflicting_keys_error(self, mock_file, retriever): + conflicting_data = '{"text": "Prompt", "text_input": "Conflicting prompt"}\n' + mock_file.return_value = io.StringIO(conflicting_data) + + with pytest.raises( + ValueError, + match="Each data entry must have only one of 'text_input' or 'text' key name.", + ): + retriever._get_content_from_input_file(Path("test_input.jsonl")) diff --git a/genai-perf/tests/test_wrapper.py b/genai-perf/tests/test_wrapper.py index 4749bad9..2c754ed7 100644 --- a/genai-perf/tests/test_wrapper.py +++ b/genai-perf/tests/test_wrapper.py @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import subprocess -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, mock_open import pytest from genai_perf import parser @@ -229,3 +229,44 @@ def test_headers_passed_correctly( assert ( False ), f"Missing expected header flag: {expected_flag} or value: {expected_value}" + + def test_build_cmd_for_payload(self, monkeypatch): + mock_file_content = ( + '{"timestamp": 0, "input_length": 6755, "output_length": 500}\n' + '{"timestamp": 1, "input_length": 7319, "output_length": 490}\n' + ) + args = MagicMock() + + with patch("genai_perf.wrapper.open", mock_open(read_data=mock_file_content)): + base_args = [ + "genai-perf", + "profile", + "-m", + "test_model", + "--service-kind", + "openai", + "--endpoint-type", + "chat", + "--input-file", + "payload:test_file", + ] + monkeypatch.setattr("sys.argv", base_args) + parsed_args, extra_args = parser.parse_args() + for key, value in vars(parsed_args).items(): + setattr(args, key, value) + + cmd = Profiler.build_cmd(args, extra_args) + cmd_string = " ".join(cmd) + + args_to_be_excluded = [ + "--concurrency", + "--request-rate-range", + "--request-count", + "--warmup-request-count", + "measurement-interval", + "--stability-percentage", + ] + for arg in args_to_be_excluded: + assert arg not in cmd_string + + assert "--schedule 0.0,1.0" in cmd_string