diff --git a/examples/model_configs/base_model.yaml b/examples/model_configs/base_model.yaml index 4954150c..925d3e50 100644 --- a/examples/model_configs/base_model.yaml +++ b/examples/model_configs/base_model.yaml @@ -9,5 +9,4 @@ model: adapter_weights: false # set to True of your model has been trained with peft, also need to provide the base model name base_model: null # path to the base_model generation: - multichoice_continuations_start_space: false # Whether to force multiple choice continuations to start with a space - no_multichoice_continuations_start_space: false # Whether to force multiple choice continuations to not start with a space + multichoice_continuations_start_space: null # If true/false, will force multiple choice continuations to start/not start with a space. If none, will do nothing diff --git a/examples/nanotron/lighteval_config_override_template.yaml b/examples/nanotron/lighteval_config_override_template.yaml index 390a81ec..6ffb4886 100644 --- a/examples/nanotron/lighteval_config_override_template.yaml +++ b/examples/nanotron/lighteval_config_override_template.yaml @@ -24,7 +24,6 @@ lighteval: dataset_loading_processes: 8 max_samples: 10 multichoice_continuations_start_space: null - no_multichoice_continuations_start_space: null num_fewshot_seeds: null tasks: early-signal # tasks: custom|hellaswag|0 diff --git a/src/lighteval/models/base_model.py b/src/lighteval/models/base_model.py index e4ae5b5f..666c0131 100644 --- a/src/lighteval/models/base_model.py +++ b/src/lighteval/models/base_model.py @@ -296,10 +296,11 @@ def _check_continuations_start_space(self, continuation: str) -> str: - None (Don't touch - default) Todo: find a way to add this back WITHOUT breaking compatibility with the harness """ - if self.multichoice_continuations_start_space is True and continuation[0] != " ": - continuation = " " + continuation - if self.multichoice_continuations_start_space is False and continuation[0] == " ": - continuation = continuation.lstrip() + if self.multichoice_continuations_start_space is not None: + if self.multichoice_continuations_start_space and continuation[0] != " ": + continuation = " " + continuation + if not self.multichoice_continuations_start_space and continuation[0] == " ": + continuation = continuation.lstrip() return continuation def _model_call(self, inputs: torch.Tensor) -> torch.Tensor: @@ -949,7 +950,7 @@ def loglikelihood_single_token( if any(len(c) > 1 for c in continuations_enc): raise ValueError( f"Trying to do single token multiple choice but one choice has several tokens: {continuations_enc}. " - "If the additional pre-token is a space, try to set --no_multichoice_continuations_start_space " + "If the additional pre-token is a space, try to set `multichoice_continuations_start_space=False` in the model parameters " ) request.tokenized_continuation = continuations_enc diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py index eb8320c0..d1e5ab38 100644 --- a/src/lighteval/models/model_config.py +++ b/src/lighteval/models/model_config.py @@ -34,6 +34,7 @@ NO_AUTOGPTQ_ERROR_MSG, NO_BNB_ERROR_MSG, NO_PEFT_ERROR_MSG, + boolstring_to_bool, is_accelerate_available, is_autogptq_available, is_bnb_available, @@ -76,6 +77,7 @@ class BaseModelConfig: space at the start of each continuation in multichoice generation. For example, context: "What is the capital of France?" and choices: "Paris", "London". Will be tokenized as: "What is the capital of France? Paris" and "What is the capital of France? London". + True adds a space, False strips a space, None does nothing subfolder (Optional[str]): The subfolder within the model repository. revision (str): The revision of the model. batch_size (int): The batch size for model training. @@ -127,6 +129,9 @@ class BaseModelConfig: compile: bool = False def __post_init__(self): + # Making sure this parameter is a boolean + self.multichoice_continuations_start_space = boolstring_to_bool(self.multichoice_continuations_start_space) + if self.quantization_config is not None and not is_bnb_available(): raise ImportError(NO_BNB_ERROR_MSG) @@ -350,15 +355,21 @@ def create_model_config( # noqa: C901 return InferenceModelConfig(model=config["base_params"]["endpoint_name"]) if config["type"] == "base": - # Tests on the multichoice space parameters - multichoice_continuations_start_space = config["generation"]["multichoice_continuations_start_space"] - no_multichoice_continuations_start_space = config["generation"]["no_multichoice_continuations_start_space"] - if not multichoice_continuations_start_space and not no_multichoice_continuations_start_space: - multichoice_continuations_start_space = None - if multichoice_continuations_start_space and no_multichoice_continuations_start_space: - raise ValueError( - "You cannot force both the multichoice continuations to start with a space and not to start with a space" - ) + # Creating the multichoice space parameters + # We need to take into account possible conversion issues from our different input formats + multichoice_continuations_start_space = boolstring_to_bool( + config["generation"]["multichoice_continuations_start_space"] + ) + + if multichoice_continuations_start_space is not None: + if multichoice_continuations_start_space: + hlog( + "You set `multichoice_continuations_start_space` to true. This will force multichoice continuations to use a starting space" + ) + else: + hlog( + "You set `multichoice_continuations_start_space` to false. This will remove a leading space from multichoice continuations, if present." + ) # Creating optional quantization configuration if config["base_params"]["dtype"] == "4bit": diff --git a/src/lighteval/models/nanotron_model.py b/src/lighteval/models/nanotron_model.py index 560b29b3..d2d6a07f 100644 --- a/src/lighteval/models/nanotron_model.py +++ b/src/lighteval/models/nanotron_model.py @@ -63,7 +63,7 @@ LoglikelihoodRequest, LoglikelihoodRollingRequest, ) -from lighteval.utils import as_list +from lighteval.utils import as_list, boolstring_to_bool from lighteval.utils_parallelism import find_executable_batch_size @@ -113,20 +113,10 @@ def __init__( # To implement PP parallelism we need to think about how we want to sync the output for the PP ranks without outputs raise ValueError("PP parallelism is not supported yet") - multichoice_continuations_start_space = lighteval_config.tasks.multichoice_continuations_start_space - if ( - not multichoice_continuations_start_space - and not lighteval_config.tasks.no_multichoice_continuations_start_space - ): - multichoice_continuations_start_space = None - # multichoice_continuations_start_space can be True (forcing space), False (forcing no space) or None (no forcing) - if ( + # multichoice_continuations_start_space can be True (forcing space), False (forcing no space) or None (no forcing) + multichoice_continuations_start_space = boolstring_to_bool( lighteval_config.tasks.multichoice_continuations_start_space - and lighteval_config.tasks.no_multichoice_continuations_start_space - ): - raise ValueError( - "You cannot force both the multichoice continuations to start with a space and not to start with a space" - ) + ) self.generation_config = lighteval_config.generation if isinstance(self.generation_config, dict): @@ -409,10 +399,11 @@ def _check_continuations_start_space(self, continuation: str) -> str: - None (Don't touch - default) Todo: find a way to add this back WITHOUT breaking compatibility with the harness """ - if self.multichoice_continuations_start_space is True and continuation[0] != " ": - continuation = " " + continuation - if self.multichoice_continuations_start_space is False and continuation[0] == " ": - continuation = continuation.lstrip() + if self.multichoice_continuations_start_space is not None: + if self.multichoice_continuations_start_space and continuation[0] != " ": + continuation = " " + continuation + if not self.multichoice_continuations_start_space and continuation[0] == " ": + continuation = continuation.lstrip() return continuation def loglikelihood_single_token( @@ -443,7 +434,7 @@ def loglikelihood_single_token( if any(len(c) > 1 for c in continuations_enc): raise ValueError( f"Trying to do single token multiple choice but one choice has several tokens: {continuations_enc}. " - "If the additional pre-token is a space, try to set --no_multichoice_continuations_start_space " + "If the additional pre-token is a space, try to set multichoice_continuations_start_space=False in the model parameters " ) request.tokenized_continuation = continuations_enc diff --git a/src/lighteval/utils.py b/src/lighteval/utils.py index 3e032d1f..5602a17f 100644 --- a/src/lighteval/utils.py +++ b/src/lighteval/utils.py @@ -145,6 +145,24 @@ def flatten(item: list[Union[list, str]]) -> list[str]: return flat_item +def boolstring_to_bool(x: Union[str, bool, int]) -> Union[bool, None]: + """Allows to manage string or bool to bool conversion, in case a configuration input is badly formatted. + + Args: + x (str): A string (true, false, True, False, ...) + + Returns: + Union[bool, None]: the corresponding boolean + """ + if x in [None, "None", "null", ""]: + return None + if x in ["True", "true", True, 1]: + return True + if x in ["False", "false", False, 0]: + return False + raise ValueError(f"You tried to convert {x} to a boolean but it's not possible.") + + def is_accelerate_available() -> bool: return importlib.util.find_spec("accelerate") is not None