repo templete

.gitignore

@@ -118,4 +118,7 @@ work_dirs/
 # Pytorch
 *.pth
 *.py~
-*.sh~
+*.sh~
+
+# mim
+*.mim

.pre-commit-config-zh-cn.yaml

@@ -47,4 +47,4 @@ repos:
     rev: v3.0.0
     hooks:
       - id: pyupgrade
-        args: ["--py36-plus"]
+        args: ["--py36-plus"]

README.md

@@ -0,0 +1,32 @@
+# MMChat
+
+### Installation
+
+1. 安装 Pytorch (https://pytorch.org/get-started/locally/)
+2. 安装 MMChat
+
+```
+pip install -r requirements.txt
+pip install -e .
+```
+
+### TODO
+
+##### 基础功能
+
+- [x] MMEngine 加载 HF Dataset
+- [x] MMEngine 加载 HF Model & Tokenizer
+- [ ] DeepSpeed Config
+- [ ] FSDP Config
+- [ ] MMLU 评测
+- [ ] PTH -> HF Checkpoint 转换脚本
+- [ ] Attention Score 可视化
+
+##### 算法
+
+- [x] SFT
+- [ ] Alpaca SFT Setting & 精度对齐
+- [x] QLora SFT
+- [ ] QLora SFT Setting & 精度对齐
+- [ ] QLora RLHF
+- [ ] Distill Finetune

mmchat/registry.py

@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-"""mmchat provides 21 registry nodes to support using modules across
-projects. Each node is a child of the root registry in MMEngine.
+"""mmchat provides 21 registry nodes to support using modules across projects.
+Each node is a child of the root registry in MMEngine.
 
 More details can be found at
 https://mmengine.readthedocs.io/en/latest/tutorials/registry.html.
@@ -160,6 +160,12 @@
     locations=['mmchat.models'],
 )
 
+# Large Language Model
+LLM = Registry(
+    'llm',
+    locations=['mmchat.models'],
+)
+
 #######################################################################
 #                       mmchat.evaluation                         #
 #######################################################################
@@ -192,4 +198,4 @@
     'vis_backend',
     parent=MMENGINE_VISBACKENDS,
     locations=['mmchat.visualization'],
-)
+)

mmchat/version.py

@@ -22,4 +22,4 @@ def parse_version_info(version_str):
     return tuple(version_info)
 
 
-version_info = parse_version_info(__version__)
+version_info = parse_version_info(__version__)

requirements.txt

@@ -0,0 +1,4 @@
+bitsandbytes==0.39.1
+mmengine==0.8.1
+peft==0.4.0.dev0
+transformers==4.30.2