update eval

shibing624 · Oct 13, 2024 · 49fc55d · 49fc55d
1 parent f004460
commit 49fc55d
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -89,7 +89,6 @@ python examples/macbert/gradio_demo.py
 | Model Name       | Model Link                                                                                                          | Base Model                 | SIGHAN-2015 | EC-LAW | MCSC   | GPU/CPU | QPS     |
 |:-----------------|:--------------------------------------------------------------------------------------------------------------------|:---------------------------|:------------|:-------|:-------|:-----------|:--------|
 | Kenlm-CSC        | [shibing624/chinese-kenlm-klm](https://huggingface.co/shibing624/chinese-kenlm-klm)                                 | kenlm                      | 0.3147      | 0.3763 | 0.3317 | CPU | 9       |
-| BART-CSC         | [shibing624/bart4csc-base-chinese](https://huggingface.co/shibing624/bart4csc-base-chinese)                         | fnlp/bart-base-chinese     | 0.6654      | -      | -      | GPU | 58      |
 | Mengzi-T5-CSC    | [shibing624/mengzi-t5-base-chinese-correction](https://huggingface.co/shibing624/mengzi-t5-base-chinese-correction) | mengzi-t5-base             | 0.7758      | 0.3156 | 0.1039 | GPU | 214     |
 | ERNIE-CSC        | [ernie-csc](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/legacy/examples/text_correction/ernie-csc) | PaddlePaddle/ernie-1.0-base-zh             | 0.8383      | 0.3357 | 0.1318 | GPU | 114     |
 | MacBERT-CSC      | [shibing624/macbert4csc-base-chinese](https://huggingface.co/shibing624/macbert4csc-base-chinese)                   | hfl/chinese-macbert-base   | 0.8314      | 0.1610 | 0.2055 | GPU | **224** |

diff --git a/examples/evaluate_models/evaluate_models.py b/examples/evaluate_models/evaluate_models.py
@@ -80,11 +80,19 @@ def main(args):
             # Sentence Level: acc:0.4600, precision:0.2971, recall:0.0847, f1:0.1318, cost time:18.69 s, total num: 1000
     elif args.model == 'chatglm':
         from pycorrector.gpt.gpt_corrector import GptCorrector
-        model = GptCorrector(model_name_or_path="THUDM/chatglm3-6b",
-                             model_type='chatglm',
-                             peft_name="shibing624/chatglm3-6b-csc-chinese-lora")
-        eval_model_batch(model.correct_batch)
-        # chatglm3-6b-csc: Sentence Level: acc:0.5564, precision:0.5574, recall:0.4917, f1:0.5225, cost time:1572.49 s, total num: 1100
+        m = GptCorrector(model_name_or_path="THUDM/chatglm3-6b",
+                         model_type='chatglm',
+                         peft_name="shibing624/chatglm3-6b-csc-chinese-lora")
+        if args.data == 'sighan':
+            eval_model_batch(m.correct_batch)
+            # Sentence Level: acc:0.5564, precision:0.5574, recall:0.4917, f1:0.5225, cost time:1572.49 s, total num: 1100
+            #
+        elif args.data == 'ec_law':
+            eval_model_batch(m.correct_batch, input_tsv_file=os.path.join(pwd_path, "../data/ec_law_test.tsv"))
+            #
+        elif args.data == 'mcsc':
+            eval_model_batch(m.correct_batch, input_tsv_file=os.path.join(pwd_path, "../data/mcsc_test.tsv"))
+            #
     elif args.model == 'qwen1.5b':
         from pycorrector.gpt.gpt_corrector import GptCorrector
         m = GptCorrector(model_name_or_path="shibing624/chinese-text-correction-1.5b")