From 45d2c5564d94ec16972b287665e1488a8a4fedf9 Mon Sep 17 00:00:00 2001
From: junewgl <1965259211@qq.com>
Date: Mon, 25 Dec 2023 17:34:30 +0800
Subject: [PATCH] docs: update baseline result
---
README.md | 261 +++++++++++++++++++++++++++++++++++++++++++++++++++
README.zh.md | 261 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 522 insertions(+)
diff --git a/README.md b/README.md
index 6d098a9..d5fd01f 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,267 @@
[**简体中文**](README.zh.md) | [**Discord**](https://discord.gg/7uQnPuveTY) | [**Wechat**](https://github.com/eosphoros-ai/DB-GPT/blob/main/README.zh.md#%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC) | [**Huggingface**](https://huggingface.co/eosphoros) | [**Community**](https://github.com/eosphoros-ai/community)
+## Baseline
+- update time: 2023/12/08
+- metric: execution accuracy (ex)
+- more details refer to [docs/eval-llm-result.md](https://github.com/eosphoros-ai/DB-GPT-Hub/blob/main/docs/eval_llm_result.md)
+
+
+
+ | Model |
+ Method |
+ Easy |
+ Medium |
+ Hard |
+ Extra |
+ All |
+
+
+ |
+ base |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+
+
+ | Llama2-7B-Chat |
+ lora |
+ 0.887 |
+ 0.641 |
+ 0.489 |
+ 0.331 |
+ 0.626 |
+
+
+ |
+ qlora |
+ 0.847 |
+ 0.623 |
+ 0.466 |
+ 0.361 |
+ 0.608 |
+
+
+ |
+ base |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+
+
+ | Llama2-13B-Chat |
+ lora |
+ 0.907 |
+ 0.729 |
+ 0.552 |
+ 0.343 |
+ 0.68 |
+
+
+ |
+ qlora |
+ 0.911 |
+ 0.7 |
+ 0.552 |
+ 0.319 |
+ 0.664 |
+
+
+ |
+ base |
+ 0.214 |
+ 0.177 |
+ 0.092 |
+ 0.036 |
+ 0.149 |
+
+
+ | CodeLlama-7B-Instruct |
+ lora |
+ 0.923 |
+ 0.756 |
+ 0.586 |
+ 0.349 |
+ 0.702 |
+
+
+ |
+ qlora |
+ 0.911 |
+ 0.751 |
+ 0.598 |
+ 0.331 |
+ 0.696 |
+
+
+ |
+ base |
+ 0.698 |
+ 0.601 |
+ 0.408 |
+ 0.271 |
+ 0.539 |
+
+
+ | CodeLlama-13B-Instruct |
+ lora |
+ 0.94 |
+ 0.789 |
+ 0.684 |
+ 0.404 |
+ 0.746 |
+
+
+ |
+ qlora |
+ 0.94 |
+ 0.774 |
+ 0.626 |
+ 0.392 |
+ 0.727 |
+
+
+ |
+ base |
+ 0.577 |
+ 0.352 |
+ 0.201 |
+ 0.066 |
+ 335 |
+
+
+ | Baichuan2-7B-Chat |
+ lora |
+ 0.871 |
+ 0.63 |
+ 0.448 |
+ 0.295 |
+ 0.603 |
+
+
+ |
+ qlora |
+ 0.891 |
+ 0.637 |
+ 0.489 |
+ 0.331 |
+ 0.624 |
+
+
+ |
+ base |
+ 0.581 |
+ 0.413 |
+ 0.264 |
+ 0.187 |
+ 0.392 |
+
+
+ | Baichuan2-13B-Chat |
+ lora |
+ 0.903 |
+ 0.702 |
+ 0.569 |
+ 0.392 |
+ 0.678 |
+
+
+ |
+ qlora |
+ 0.895 |
+ 0.675 |
+ 0.58 |
+ 0.343 |
+ 0.659 |
+
+
+ |
+ base |
+ 0.395 |
+ 0.256 |
+ 0.138 |
+ 0.042 |
+ 0.235 |
+
+
+| Qwen-7B-Chat |
+ lora |
+ 0.855 |
+ 0.688 |
+ 0.575 |
+ 0.331 |
+ 0.652 |
+
+
+ |
+ qlora |
+ 0.911 |
+ 0.675 |
+ 0.575 |
+ 0.343 |
+ 0.662 |
+
+
+ |
+ base |
+ 0.871 |
+ 0.632 |
+ 0.368 |
+ 0.181 |
+ 0.573 |
+
+
+ | Qwen-14B-Chat |
+ lora |
+ 0.895 |
+ 0.702 |
+ 0.552 |
+ 0.331 |
+ 0.663 |
+
+
+ |
+ qlora |
+ 0.919 |
+ 0.744 |
+ 0.598 |
+ 0.367 |
+ 0.701 |
+
+
+ |
+ base |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+
+
+ | ChatGLM3-6b |
+ lora |
+ 0.855 |
+ 0.605 |
+ 0.477 |
+ 0.271 |
+ 0.59 |
+
+
+ |
+ qlora |
+ 0.843 |
+ 0.603 |
+ 0.506 |
+ 0.211 |
+ 0.581 |
+
+
+
+
## Contents
- [DB-GPT-Hub: Text-to-SQL parsing with LLMs](#db-gpt-hub-text-to-sql-parsing-with-llms)
- [Contents](#contents)
diff --git a/README.zh.md b/README.zh.md
index 7dbe21d..4657a1c 100644
--- a/README.zh.md
+++ b/README.zh.md
@@ -26,6 +26,267 @@
[**英文**](README.md) | [**Discord**](https://discord.gg/7uQnPuveTY) | [**Wechat**](https://github.com/eosphoros-ai/DB-GPT/blob/main/README.zh.md#%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC) | [**Huggingface**](https://huggingface.co/eosphoros) | [**Community**](https://github.com/eosphoros-ai/community)
+
+## Baseline
+- 更新日期: 2023/12/08
+- 评价指标: execution accuracy (ex)
+- 详情参考[docs/eval-llm-result.md](https://github.com/eosphoros-ai/DB-GPT-Hub/blob/main/docs/eval_llm_result.md)
+
+
+
+ | Model |
+ Method |
+ Easy |
+ Medium |
+ Hard |
+ Extra |
+ All |
+
+
+ |
+ base |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+
+
+ | Llama2-7B-Chat |
+ lora |
+ 0.887 |
+ 0.641 |
+ 0.489 |
+ 0.331 |
+ 0.626 |
+
+
+ |
+ qlora |
+ 0.847 |
+ 0.623 |
+ 0.466 |
+ 0.361 |
+ 0.608 |
+
+
+ |
+ base |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+
+
+ | Llama2-13B-Chat |
+ lora |
+ 0.907 |
+ 0.729 |
+ 0.552 |
+ 0.343 |
+ 0.68 |
+
+
+ |
+ qlora |
+ 0.911 |
+ 0.7 |
+ 0.552 |
+ 0.319 |
+ 0.664 |
+
+
+ |
+ base |
+ 0.214 |
+ 0.177 |
+ 0.092 |
+ 0.036 |
+ 0.149 |
+
+
+ | CodeLlama-7B-Instruct |
+ lora |
+ 0.923 |
+ 0.756 |
+ 0.586 |
+ 0.349 |
+ 0.702 |
+
+
+ |
+ qlora |
+ 0.911 |
+ 0.751 |
+ 0.598 |
+ 0.331 |
+ 0.696 |
+
+
+ |
+ base |
+ 0.698 |
+ 0.601 |
+ 0.408 |
+ 0.271 |
+ 0.539 |
+
+
+ | CodeLlama-13B-Instruct |
+ lora |
+ 0.94 |
+ 0.789 |
+ 0.684 |
+ 0.404 |
+ 0.746 |
+
+
+ |
+ qlora |
+ 0.94 |
+ 0.774 |
+ 0.626 |
+ 0.392 |
+ 0.727 |
+
+
+ |
+ base |
+ 0.577 |
+ 0.352 |
+ 0.201 |
+ 0.066 |
+ 335 |
+
+
+ | Baichuan2-7B-Chat |
+ lora |
+ 0.871 |
+ 0.63 |
+ 0.448 |
+ 0.295 |
+ 0.603 |
+
+
+ |
+ qlora |
+ 0.891 |
+ 0.637 |
+ 0.489 |
+ 0.331 |
+ 0.624 |
+
+
+ |
+ base |
+ 0.581 |
+ 0.413 |
+ 0.264 |
+ 0.187 |
+ 0.392 |
+
+
+ | Baichuan2-13B-Chat |
+ lora |
+ 0.903 |
+ 0.702 |
+ 0.569 |
+ 0.392 |
+ 0.678 |
+
+
+ |
+ qlora |
+ 0.895 |
+ 0.675 |
+ 0.58 |
+ 0.343 |
+ 0.659 |
+
+
+ |
+ base |
+ 0.395 |
+ 0.256 |
+ 0.138 |
+ 0.042 |
+ 0.235 |
+
+
+| Qwen-7B-Chat |
+ lora |
+ 0.855 |
+ 0.688 |
+ 0.575 |
+ 0.331 |
+ 0.652 |
+
+
+ |
+ qlora |
+ 0.911 |
+ 0.675 |
+ 0.575 |
+ 0.343 |
+ 0.662 |
+
+
+ |
+ base |
+ 0.871 |
+ 0.632 |
+ 0.368 |
+ 0.181 |
+ 0.573 |
+
+
+ | Qwen-14B-Chat |
+ lora |
+ 0.895 |
+ 0.702 |
+ 0.552 |
+ 0.331 |
+ 0.663 |
+
+
+ |
+ qlora |
+ 0.919 |
+ 0.744 |
+ 0.598 |
+ 0.367 |
+ 0.701 |
+
+
+ |
+ base |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+ 0 |
+
+
+ | ChatGLM3-6b |
+ lora |
+ 0.855 |
+ 0.605 |
+ 0.477 |
+ 0.271 |
+ 0.59 |
+
+
+ |
+ qlora |
+ 0.843 |
+ 0.603 |
+ 0.506 |
+ 0.211 |
+ 0.581 |
+
+
+
## Contents
- [DB-GPT-Hub:利用LLMs实现Text-to-SQL](#db-gpt-hub利用llms实现text-to-sql)
- [Contents](#contents)