diff --git a/dbgpt_hub/llm_base/config_parser.py b/dbgpt_hub/llm_base/config_parser.py index be12508..0129b69 100644 --- a/dbgpt_hub/llm_base/config_parser.py +++ b/dbgpt_hub/llm_base/config_parser.py @@ -93,7 +93,8 @@ def parse_infer_args( def get_train_args( - args: Optional[Dict[str, Any]] = None + args: Optional[Dict[str, Any]] = None, + data_args_init: bool = True ) -> Tuple[ ModelArguments, DataArguments, @@ -121,7 +122,8 @@ def get_train_args( transformers.utils.logging.enable_explicit_format() # Check arguments (do not check finetuning_args since it may be loaded from checkpoints) - data_args.init_for_training() + if data_args_init: + data_args.init_for_training() if training_args.max_steps == -1 and data_args.streaming: raise ValueError("Please specify `max_steps` in streaming mode.") diff --git a/dbgpt_hub/llm_base/model_trainer.py b/dbgpt_hub/llm_base/model_trainer.py index fd51f69..228be35 100644 --- a/dbgpt_hub/llm_base/model_trainer.py +++ b/dbgpt_hub/llm_base/model_trainer.py @@ -401,7 +401,7 @@ def plot_loss( def export_model( args: Optional[Dict[str, Any]] = None, max_shard_size: Optional[str] = "10GB" ): - model_args, _, training_args, finetuning_args, _ = get_train_args(args) + model_args, _, training_args, finetuning_args, _ = get_train_args(args, data_args_init=False) model, tokenizer = load_model_and_tokenizer(model_args, finetuning_args) model.save_pretrained(training_args.output_dir, max_shard_size=max_shard_size) try: