From 49fc55d4e4c01e79c9f535d7ef4feace62203f7c Mon Sep 17 00:00:00 2001 From: shibing624 Date: Sun, 13 Oct 2024 13:32:33 +0800 Subject: [PATCH] update eval --- README.md | 1 - examples/evaluate_models/evaluate_models.py | 18 +++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9b300a4e..9dbfca5a 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,6 @@ python examples/macbert/gradio_demo.py | Model Name | Model Link | Base Model | SIGHAN-2015 | EC-LAW | MCSC | GPU/CPU | QPS | |:-----------------|:--------------------------------------------------------------------------------------------------------------------|:---------------------------|:------------|:-------|:-------|:-----------|:--------| | Kenlm-CSC | [shibing624/chinese-kenlm-klm](https://huggingface.co/shibing624/chinese-kenlm-klm) | kenlm | 0.3147 | 0.3763 | 0.3317 | CPU | 9 | -| BART-CSC | [shibing624/bart4csc-base-chinese](https://huggingface.co/shibing624/bart4csc-base-chinese) | fnlp/bart-base-chinese | 0.6654 | - | - | GPU | 58 | | Mengzi-T5-CSC | [shibing624/mengzi-t5-base-chinese-correction](https://huggingface.co/shibing624/mengzi-t5-base-chinese-correction) | mengzi-t5-base | 0.7758 | 0.3156 | 0.1039 | GPU | 214 | | ERNIE-CSC | [ernie-csc](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/legacy/examples/text_correction/ernie-csc) | PaddlePaddle/ernie-1.0-base-zh | 0.8383 | 0.3357 | 0.1318 | GPU | 114 | | MacBERT-CSC | [shibing624/macbert4csc-base-chinese](https://huggingface.co/shibing624/macbert4csc-base-chinese) | hfl/chinese-macbert-base | 0.8314 | 0.1610 | 0.2055 | GPU | **224** | diff --git a/examples/evaluate_models/evaluate_models.py b/examples/evaluate_models/evaluate_models.py index d4aaad22..32c4f4e7 100644 --- a/examples/evaluate_models/evaluate_models.py +++ b/examples/evaluate_models/evaluate_models.py @@ -80,11 +80,19 @@ def main(args): # Sentence Level: acc:0.4600, precision:0.2971, recall:0.0847, f1:0.1318, cost time:18.69 s, total num: 1000 elif args.model == 'chatglm': from pycorrector.gpt.gpt_corrector import GptCorrector - model = GptCorrector(model_name_or_path="THUDM/chatglm3-6b", - model_type='chatglm', - peft_name="shibing624/chatglm3-6b-csc-chinese-lora") - eval_model_batch(model.correct_batch) - # chatglm3-6b-csc: Sentence Level: acc:0.5564, precision:0.5574, recall:0.4917, f1:0.5225, cost time:1572.49 s, total num: 1100 + m = GptCorrector(model_name_or_path="THUDM/chatglm3-6b", + model_type='chatglm', + peft_name="shibing624/chatglm3-6b-csc-chinese-lora") + if args.data == 'sighan': + eval_model_batch(m.correct_batch) + # Sentence Level: acc:0.5564, precision:0.5574, recall:0.4917, f1:0.5225, cost time:1572.49 s, total num: 1100 + # + elif args.data == 'ec_law': + eval_model_batch(m.correct_batch, input_tsv_file=os.path.join(pwd_path, "../data/ec_law_test.tsv")) + # + elif args.data == 'mcsc': + eval_model_batch(m.correct_batch, input_tsv_file=os.path.join(pwd_path, "../data/mcsc_test.tsv")) + # elif args.model == 'qwen1.5b': from pycorrector.gpt.gpt_corrector import GptCorrector m = GptCorrector(model_name_or_path="shibing624/chinese-text-correction-1.5b")