diff --git a/.github/workflows/onnx.yml b/.github/workflows/onnx.yml new file mode 100644 index 00000000..acb5bec2 --- /dev/null +++ b/.github/workflows/onnx.yml @@ -0,0 +1,57 @@ +name: 'onnx docker' + +on: + workflow_dispatch: + push: + branches: ['master'] + paths: + - 'onnx/Dockerfile' + - 'onnx/export/**' + - 'onnx/misc/**' + - 'onnx/version.txt' + - '.github/workflow/onnx.yml' + pull_request: + branches: ['master'] + paths: + - 'onnx/Dockerfile' + - 'onnx/export/**' + - 'onnx/misc/**' + - 'onnx/version.txt' + - '.github/workflow/onnx.yml' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set Env + run: | + VERSION=$(cat onnx/version.txt) + echo "VERSION=$VERSION" >> $GITHUB_ENV + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v4 + with: + context: ./onnx/ + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{secrets.DOCKER_ORG}}/altclip-onnx:latest, ${{secrets.DOCKER_ORG}}/altclip-onnx:${{env.VERSION}} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + GIT=https://github.com/${{github.repository}}.git diff --git a/README_zh.md b/README_zh.md index 96439785..d09cc41c 100644 --- a/README_zh.md +++ b/README_zh.md @@ -17,19 +17,19 @@ 1. **可通过 API 快速下载模型** - + 提供 API 方便你快速下载模型，并在给定（中/英文）文本上使用这些预训练模型，在从[SuperGLUE](https://super.gluebenchmark.com/)和[CLUE](https://github.com/CLUEbenchmark/CLUE) benchmarks收集的广泛使用的数据集上对它们进行微调。 - + FlagAI 现已支持 30+ 主流模型，包括多模态模型 [**AltCLIP**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/AltCLIP) 、文生图模型 [**AltDiffusion**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/AltDiffusion) [![Huggingface space](https://img.shields.io/badge/🤗-Huggingface%20Space-cyan.svg)](https://huggingface.co/spaces/BAAI/bilingual_stable_diffusion)、最高百亿参数的 **[悟道GLM](/doc_zh/GLM.md)**，[**EVA-CLIP**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/EVA_CLIP)、**[Galactica](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/galactica)**、**OPT**、**BERT**、**RoBERTa**、**GPT2**、**T5**、**ALM**、**Huggingface Transformers** 等。 - + 2. **仅用十行代码即可进行并行训练** 飞智由四个最流行的数据/模型并行库（[PyTorch](https://pytorch.org/)/[Deepspeed](https://www.deepspeed.ai/)/[Megatron-LM](https://github.com/NVIDIA/Megatron-LM)/[BMTrain](https://github.com/OpenBMB/BMTrain)）提供支持，它们之间实现了无缝集成。 你可以用不到十行代码来并行你的训练/测试过程。 - + 3. **提供提示学习工具包** FlagAI 提供了提示学习（[prompt-learning](https://github.com/FlagAI-Open/FlagAI/blob/master/docs/TUTORIAL_7_PROMPT_LEARNING.md)）的工具包，用于少样本学习(few-shot learning)任务。 - + 4. **尤其擅长中文任务** FlagAI 目前支持的模型可以应用于文本分类、信息提取、问答、摘要、文本生成等任务，尤其擅长中文任务。 @@ -51,40 +51,40 @@ ### 模型 -| 模型名称 | 任务 | 训练 | 微调 | 推理 | 样例 | +| 模型名称 | 任务 | 训练 | 微调 | 推理 | 样例 | | :---------------- | :------- | :-- |:-- | :-- | :--------------------------------------------- | -| ALM | 阿拉伯语文本生成 | ✅ | ❌ | ✅ | [README.md](/examples/ALM/README.md) | -| AltCLIP | 文图匹配 | ✅ | ✅ | ✅ | [README.md](/examples/AltCLIP/README.md) | -| AltCLIP-m18 | 文图匹配 | ✅ | ✅ | ✅ | [README.md](examples/AltCLIP-m18/README.md) | +| ALM | 阿拉伯语文本生成 | ✅ | ❌ | ✅ | [README.md](/examples/ALM/README.md) | +| AltCLIP | 文图匹配 | ✅ | ✅ | ✅ | [README.md](/examples/AltCLIP/README.md) | +| AltCLIP-m18 | 文图匹配 | ✅ | ✅ | ✅ | [README.md](examples/AltCLIP-m18/README.md) | | AltDiffusion | 文生图 | ❌ | ❌ | ✅ | [README.md](/examples/AltDiffusion/README.md) | | AltDiffusion-m18 | 文生图，支持 18 种语言 | ❌ | ❌ | ✅ | [README.md](/examples/AltDiffusion-m18/README.md) | | BERT-title-generation-english | 英文标题生成 | ✅ | ❌ | ✅ | [README.md](/examples/bert_title_generation_english/README.md) | -| CLIP | 图文匹配 | ✅ | ❌ | ✅ | —— | -| CPM3-finetune | 文本续写 | ❌ | ✅ | ❌ | —— | -| CPM3-generate | 文本续写 | ❌ | ❌ | ✅ | —— | +| CLIP | 图文匹配 | ✅ | ❌ | ✅ | —— | +| CPM3-finetune | 文本续写 | ❌ | ✅ | ❌ | —— | +| CPM3-generate | 文本续写 | ❌ | ❌ | ✅ | —— | | CPM3_pretrain | 文本续写 | ✅ | ❌ | ❌ | —— | | CPM_1 | 文本续写 | ❌ | ❌ | ✅ | [README.md](/examples/cpm_1/README.md) | | EVA-CLIP | 图文匹配 | ✅ | ✅ | ✅ | [README.md](/examples/EVA_CLIP/README.md) | -| Galactica | 文本续写 | ❌ | ❌ | ✅ | —— | +| Galactica | 文本续写 | ❌ | ❌ | ✅ | —— | | GLM-large-ch-blank-filling | 完形填空问答 | ❌ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_11_GLM_BLANK_FILLING_QA.md) | | GLM-large-ch-poetry-generation | 诗歌生成 | ✅ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md) | | GLM-large-ch-title-generation | 标题生成 | ✅ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md) | -| GLM-pretrain | 预训练 | ✅ | ❌ | ❌ | —— | -| GLM-seq2seq | 生成任务 | ✅ | ❌ | ✅ | —— | -| GLM-superglue | 判别任务 | ✅ | ❌ | ❌ | —— | +| GLM-pretrain | 预训练 | ✅ | ❌ | ❌ | —— | +| GLM-seq2seq | 生成任务 | ✅ | ❌ | ✅ | —— | +| GLM-superglue | 判别任务 | ✅ | ❌ | ❌ | —— | | GPT-2-text-writting | 文本续写 | ❌ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_18_GPT2_WRITING.md) | -| GPT2-text-writting | 文本续写 | ❌ | ❌ | ✅ | —— | -| GPT2-title-generation | 标题生成 | ❌ | ❌ | ✅ | —— | -| OPT | 文本续写 | ❌ | ❌ | ✅ | [README.md](/examples/opt/README.md) | +| GPT2-text-writting | 文本续写 | ❌ | ❌ | ✅ | —— | +| GPT2-title-generation | 标题生成 | ❌ | ❌ | ✅ | —— | +| OPT | 文本续写 | ❌ | ❌ | ✅ | [README.md](/examples/opt/README.md) | | RoBERTa-base-ch-ner | 命名实体识别 | ✅ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_17_BERT_EXAMPLE_NER.md) | | RoBERTa-base-ch-semantic-matching | 语义相似度匹配 | ✅ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_16_BERT_EXAMPLE_SEMANTIC_MATCHING.md) | | RoBERTa-base-ch-title-generation | 标题生成 | ✅ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_15_BERT_EXAMPLE_TITLE_GENERATION.md) | -| RoBERTa-faq | 问答 | ❌ | ❌ | ✅ | [README.md](/examples/roberta_faq/README.md) | -| Swinv1 | 图片分类 | ✅ | ❌ | ✅ | —— | -| Swinv2 | 图片分类 | ✅ | ❌ | ✅ | —— | +| RoBERTa-faq | 问答 | ❌ | ❌ | ✅ | [README.md](/examples/roberta_faq/README.md) | +| Swinv1 | 图片分类 | ✅ | ❌ | ✅ | —— | +| Swinv2 | 图片分类 | ✅ | ❌ | ✅ | —— | | T5-huggingface-11b | 训练 | ✅ | ❌ | ❌ | [TUTORIAL](/doc_zh/TUTORIAL_14_HUGGINGFACE_T5.md) | | T5-title-generation | 标题生成 | ❌ | ❌ | ✅ | [TUTORIAL](/doc_zh/TUTORIAL_19_T5_EXAMPLE_TITLE_GENERATION.md) | -| T5-flagai-11b | 预训练 | ✅ | ❌ | ❌ | —— | +| T5-flagai-11b | 预训练 | ✅ | ❌ | ❌ | —— | | ViT-cifar100 | 预训练 | ✅ | ❌ | ❌ | —— | @@ -144,7 +144,7 @@ ds_report # 检查deepspeed的状态 ``` git clone https://github.com/OpenBMB/BMTrain cd BMTrain -python setup.py install +python setup.py install ``` - [可选] 镜像构建，请参照 [Dockerfile](https://github.com/FlagAI-Open/FlagAI/blob/master/Dockerfile) - [提示] 单节点docker环境下，运行多卡数据并行需要设置host。 例如，docker节点 root@127.0.0.1，其端口 7110。 @@ -167,7 +167,7 @@ Host 127.0.0.1 from flagai.auto_model.auto_loader import AutoLoader auto_loader = AutoLoader( task_name="title-generation", - model_name="RoBERTa-base-ch" + model_name="RoBERTa-base-ch" ) model = auto_loader.get_model() tokenizer = auto_loader.get_tokenizer() @@ -261,12 +261,15 @@ for text_pair in test_data: ``` +### 模型部署 + +* AltCLIP 部署见 [AltCLIP 的 ONNX 模型导出](./onnx/README.md) ## 动态 -- [31 Mar 2023] 支持v1.6.3版本, 增加AltCLIP-m18模型 [#303](https://github.com/FlagAI-Open/FlagAI/pull/303) 以及 AltDiffusion-m18模型 [#302](https://github.com/FlagAI-Open/FlagAI/pull/302); -- [17 Mar 2023] 支持v1.6.2版本, 可以使用新的优化器 [#266](https://github.com/FlagAI-Open/FlagAI/pull/266), 并增加了英文gpt模型GPT2-base-en; +- [31 Mar 2023] 支持v1.6.3版本, 增加AltCLIP-m18模型 [#303](https://github.com/FlagAI-Open/FlagAI/pull/303) 以及 AltDiffusion-m18模型 [#302](https://github.com/FlagAI-Open/FlagAI/pull/302); +- [17 Mar 2023] 支持v1.6.2版本, 可以使用新的优化器 [#266](https://github.com/FlagAI-Open/FlagAI/pull/266), 并增加了英文gpt模型GPT2-base-en; - [2 Mar 2023] 支持v1.6.1版本, 增加Galactica模型 [#234](https://github.com/FlagAI-Open/FlagAI/pull/234), 大模型推理的低资源工具包BMInf [#238](https://github.com/FlagAI-Open/FlagAI/pull/238), 以及P-tuning样例 [#227](https://github.com/FlagAI-Open/FlagAI/pull/238) -- [12 Jan 2023] 发布v1.6.0版本, 新增支持并行训练库 [**BMTrain**](https://github.com/OpenBMB/BMTrain) 以及集成 [**Flash Attention**](https://github.com/HazyResearch/flash-attention) 到 Bert 和 Vit 模型提速端到端训练, 示例见 [FlashAttentionBERT](https://github.com/FlagAI-Open/FlagAI/blob/master/examples/bert_title_generation_english/train_flash_atten.py)和 [FlashAttentionViT](https://github.com/FlagAI-Open/FlagAI/blob/master/examples/vit_cifar100/train_single_gpu_flash_atten.py). 同时增加了基于对比搜索的文本生成方法 [**SimCTG**](https://github.com/yxuansu/SimCTG) 以及基于 AltDiffusion 进行 DreamBooth 个性化微调, 示例见 [AltDiffusionNaruto](https://github.com/FlagAI-Open/FlagAI/blob/master/examples/AltDiffusion/dreambooth.py). +- [12 Jan 2023] 发布v1.6.0版本, 新增支持并行训练库 [**BMTrain**](https://github.com/OpenBMB/BMTrain) 以及集成 [**Flash Attention**](https://github.com/HazyResearch/flash-attention) 到 Bert 和 Vit 模型提速端到端训练, 示例见 [FlashAttentionBERT](https://github.com/FlagAI-Open/FlagAI/blob/master/examples/bert_title_generation_english/train_flash_atten.py)和 [FlashAttentionViT](https://github.com/FlagAI-Open/FlagAI/blob/master/examples/vit_cifar100/train_single_gpu_flash_atten.py). 同时增加了基于对比搜索的文本生成方法 [**SimCTG**](https://github.com/yxuansu/SimCTG) 以及基于 AltDiffusion 进行 DreamBooth 个性化微调, 示例见 [AltDiffusionNaruto](https://github.com/FlagAI-Open/FlagAI/blob/master/examples/AltDiffusion/dreambooth.py). - [28 Nov 2022] 发布v1.5.0版本, 支持1.1B参数的 [**EVA-CLIP**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/EVA_CLIP) 以及[ALM: 基于GLM的阿拉伯语大模型], 示例见[**ALM**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/ALM) - [10 Nov 2022] 发布v1.4.0版本, 支持[AltCLIP: 更改CLIP中的语言编码器以扩展语言功能](https://arxiv.org/abs/2211.06679v1), 示例见[**AltCLIP**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/AltCLIP)以及[**AltDiffusion**](https://github.com/FlagAI-Open/FlagAI/tree/master/examples/AltDiffusion) - [29 Aug 2022] 支持v1.3.0版本, 增加CLIP模块以及重新设计了tokenizer的API: [#81](https://github.com/FlagAI-Open/FlagAI/pull/81) @@ -274,7 +277,7 @@ for text_pair in test_data: - [29 Jun 2022] 支持v1.1.0版本, 支持OPT的加载，微调和推理[#63](https://github.com/FlagAI-Open/FlagAI/pull/63) - [17 May 2022] 做出了我们的第一份贡献[#1](https://github.com/FlagAI-Open/FlagAI/pull/1) -## 许可 LICENSE +## 许可 LICENSE FlagAI飞智大部分项目基于 [Apache 2.0 license](LICENSE)，但是请注意部分项目代码基于其他协议： @@ -299,4 +302,4 @@ FlagAI飞智大部分项目基于 [Apache 2.0 license](LICENSE)，但是请注 ![Star History Chart](https://api.star-history.com/svg?repos=FlagAI-Open/FlagAI&type=Date)] - \ No newline at end of file + diff --git a/onnx/.envrc b/onnx/.envrc new file mode 100644 index 00000000..9a427020 --- /dev/null +++ b/onnx/.envrc @@ -0,0 +1,8 @@ +export PYTHONPATH=`pwd`:$PYTHONPATH + +if [ -f ".env" ]; then + set -o allexport + source .env + set +o allexport +fi + diff --git a/onnx/.gitignore b/onnx/.gitignore new file mode 100644 index 00000000..acbd3db9 --- /dev/null +++ b/onnx/.gitignore @@ -0,0 +1,9 @@ +.env +.DS_Store +__pycache__/ + +/onnx/* +!onnx/.keep + +dist/ +model/ diff --git a/onnx/.tool-versions b/onnx/.tool-versions new file mode 100644 index 00000000..28b9ff63 --- /dev/null +++ b/onnx/.tool-versions @@ -0,0 +1 @@ +python 3.10.11 diff --git a/onnx/Dockerfile b/onnx/Dockerfile new file mode 100755 index 00000000..1920076d --- /dev/null +++ b/onnx/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:23.10 + +WORKDIR app + +ADD ./version.txt version + +# sed -i s/ports.ubuntu.com/mirrors.aliyun.com/g /etc/apt/sources.list &&\ +# sed -i s/archive.ubuntu.com/mirrors.aliyun.com/g /etc/apt/sources.list &&\ +# sed -i s/security.ubuntu.com/mirrors.aliyun.com/g /etc/apt/sources.list &&\ +# pip install -i https://mirrors.aliyun.com/pypi/simple/ \ + +RUN \ +apt-get update && \ +apt-get install -y git pkg-config bash \ +python3-full python3-pip python3-aiohttp &&\ +update-alternatives --install /usr/bin/python python /usr/bin/python3 1 &&\ +pip install --break-system-packages \ +setuptools==66.0.0 \ +urllib3==1.26.16 \ +scipy transformers huggingface_hub packaging \ +tqdm requests cython \ +torch onnx && apt-get clean -y + +ARG GIT +ENV GIT=$GIT + +RUN git clone --depth=1 $GIT + +ADD os/ / + +RUN cd FlagAI &&\ + rm setup.cfg &&\ + python setup.py install + +RUN pip uninstall -y google-auth + +ENV PYTHONPATH /app + diff --git a/onnx/README.md b/onnx/README.md new file mode 100644 index 00000000..9e399b66 --- /dev/null +++ b/onnx/README.md @@ -0,0 +1,95 @@ +# AltCLIP 的 ONNX 模型导出 + +## ONNX 是什么？ + +ONNX(Open Neural Network Exchange)，开放神经网络交换，用于在各种深度学习训练和推理框架转换的一个中间表示格式。 + +在实际业务中，可以使用 Pytorch 或者 TensorFlow 训练模型，导出成 ONNX 格式，然后用 ONNX Runtime 直接运行 ONNX。 + +使用 ONNX 可以减少模型的依赖，降低部署成本。 + +也可以进一步借助 ONNX 转换成目标设备上运行时支撑的模型格式，比如 [TensorRT Engine](https://developer.nvidia.com/tensorrt)、[NCNN](https://github.com/Tencent/ncnn)、[MNN](https://github.com/alibaba/MNN) 等格式， 优化性能。 + +## 下载 AltCLIP 的 ONNX + +可以从[xxai/AltCLIP](https://huggingface.co/xxai/AltCLIP/tree/main)下载打包好的 onnx，并解压到 `FlagAI/onnx/onnx/` 下。 + +如此就可以直接运行 onnx 的测试，而无需下载原始模型运行导出。 + +## 文件说明 + +因为 flagai 的依赖复杂，所以构建容器便于导出。 + +### 脚本 + +* `./build.sh` 在本地构建容器 + +可设置环境变量 `ORG=xxai` 使用 [hub.docker.com 上的已构建的镜像](https://hub.docker.com/repository/docker/xxai/altclip-onnx)。 + +比如，运行 `ORG=xxai ./bash.sh` 。 + +* `./bash.sh` 在本地进入容器的 bash，方便调试 + +* `./export.sh` 运行容器，导出 onnx + +设置环境变量 MODEL ，可以配置导出、测试脚本运行的模型 。 + +默认导出的模型是 AltCLIP-XLMR-L-m18 。 + +其他可选的模型有： + + - AltCLIP-XLMR-L + - AltCLIP-XLMR-L-m9 + +* `./dist.sh` 运行容器，导出以上 3 个模型的 onnx，并打包放到 dist 目录下。 + +### 目录 + +* model/ 存放下载的模型 +* onnx/ 存放导出的 onnx，下载的 onnx 也请解压到这里 + +### 测试 + +#### onnx 模型的依赖安装 + +test/onnx 下面的依赖很简单，只有 transformers 和 onnxruntime，不依赖于 flagai。 + +onnxruntime 有很多版本可以选择，见[onnxruntime](https://onnxruntime.ai/)。 + +对于 python 而言，常见的运行时推荐如下： + +* 显卡 `pip install onnxruntime-gpu` +* ARM 架构的 MAC `pip install onnxruntime-silicon` (目前还不支持 python3.11) +* INTEL 的 CPU `pip install onnxruntime-openvino` +* 其他 CPU `pip install onnxruntime` + +运行 [./test/onnx/setup.sh](./test/onnx/setup.sh) 会自动判断环境，选择安装合适的 onnxruntime 版本和 transformers。 + +#### onnx 模型的测试脚本 + +请先安装 [direnv](https://github.com/direnv/direnv/blob/master/README.md) 并在本目录下 `direnv allow` 或者手工 `source .envrc` 来设置 PYTHONPATH 环境变量。 + +* [./test/onnx/onnx_img.py](./test/onnx/onnx_img.py) 生成图片向量 +* [./test/onnx/onnx_txt.py](./test/onnx/onnx_txt.py) 生成文本向量 +* [./test/onnx/onnx_test.py](./test/onnx/onnx_test.py) 匹配图片向量和文本向量，进行零样本分类 + +如果想把生成的文本向量和图片向量存入数据库，进行相似性搜索，请先对特征进行归一化。 + +```python +image_features /= image_features.norm(dim=-1, keepdim=True) +text_features /= text_features.norm(dim=-1, keepdim=True) +``` + +可借助向量数据库，提升零样本分类的准确性，参见[ECCV 2022 | 无需下游训练，Tip-Adapter 大幅提升 CLIP 图像分类准确率](https://cloud.tencent.com/developer/article/2126102)。 + +#### pytorch 模型 + +用于对比 onnx 模型的向量输出，查看是否一致。 + +因为用到了 flagai，请如下图所示运行 [./bash.sh ](./bash.sh) 进入容器运行调试。 + +![](https://pub-b8db533c86124200a9d799bf3ba88099.r2.dev/2023/06/ei64CNo.webp) + +* [./test/clip/clip_img.py](./test/clip/clip_img.py) 生成图片向量 +* [./test/clip/clip_txt.py](./test/clip/clip_txt.py) 生成文本向量 +* [./test/clip/clip_test.py](./test/clip/clip_test.py) 匹配图片向量和文本向量，进行零样本分类 diff --git a/onnx/bash.sh b/onnx/bash.sh new file mode 100755 index 00000000..bb0654fa --- /dev/null +++ b/onnx/bash.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -ex +mkdir -p out dist model + +NAME=altclip-onnx + +if ! [ -z $ORG ]; then + NAME=$ORG/$NAME +fi + +if [ -z "$1" ]; then + exe="exec bash" + it="-it" +else + exe="exec bash -c \"$@\"" + it="" +fi + +docker run \ + -v $DIR/misc:/app/misc \ + -v $DIR/img:/app/img \ + -v $DIR/export:/app/export \ + -v $DIR/onnx:/app/onnx \ + -v $DIR/dist:/app/dist \ + -v $DIR/model:/app/model \ + -v $DIR/test:/app/test \ + $it --rm $NAME bash -c "$exe" diff --git a/onnx/build.sh b/onnx/build.sh new file mode 100755 index 00000000..ae7bc70c --- /dev/null +++ b/onnx/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -ex + +GIT=$(cat ../.git/config | grep url | awk -F= '{print $2}' | sed -e 's/[ ]*//g') + +# 替换 : 为 / +GIT=${GIT//://} + +# 替换 git@ 为 https:// +GIT=${GIT//git@/https:\/\/} + +docker build --build-arg GIT=$GIT . -t altclip-onnx diff --git a/onnx/dist.sh b/onnx/dist.sh new file mode 100755 index 00000000..5d99c6ba --- /dev/null +++ b/onnx/dist.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -ex + +LI="L L-m9 L-m18" + +VERSION=$(cat version.txt) + +mkdir -p dist + +for model in $LI; do + export MODEL="AltCLIP-XLMR-${model}" + ./export.sh + ./export/tar.bz2.py + mv onnx/$MODEL.tar.bz2 dist/$MODEL-$VERSION.tar.bz2 +done diff --git a/onnx/export.sh b/onnx/export.sh new file mode 100755 index 00000000..4470d614 --- /dev/null +++ b/onnx/export.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -ex + +cmd="export MODEL=$MODEL;export/onnx.sh" + +if ! [ -z $1 ]; then + cmd="$cmd && $@" +fi + +./bash.sh $cmd diff --git a/onnx/export/down.py b/onnx/export/down.py new file mode 100755 index 00000000..e1f721a2 --- /dev/null +++ b/onnx/export/down.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from misc.config import MODEL_NAME, MODEL_DIR +from flagai.auto_model.auto_loader import AutoLoader + +loader = AutoLoader(task_name="txt_img_matching", + model_name=MODEL_NAME, + model_dir=MODEL_DIR) + +loader.get_model() diff --git a/onnx/export/onnx.sh b/onnx/export/onnx.sh new file mode 100755 index 00000000..d5f08f9a --- /dev/null +++ b/onnx/export/onnx.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -ex + +./down.py +./onnx_export_processor.py +./onnx_export.py diff --git a/onnx/export/onnx_export.py b/onnx/export/onnx_export.py new file mode 100755 index 00000000..b87e215d --- /dev/null +++ b/onnx/export/onnx_export.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +from PIL import Image +from os import makedirs +from os.path import join +from misc.clip_model import TXT, IMG +from misc.config import ONNX_FP, opset_version, IMG_DIR +from misc.proc import transform, tokenizer +import torch + +makedirs(ONNX_FP, exist_ok=True) + +JPG = join(IMG_DIR, 'cat.jpg') + +image = Image.open(JPG) +image = transform(image) +image = torch.tensor(image) + + +def onnx_export(outdir, model, args, **kwds): + name = f'{outdir}.onnx' + fp = join(ONNX_FP, name) + torch.onnx.export( + model, + args, + fp, + export_params=True, + # verbose=True, + opset_version=opset_version, + do_constant_folding=False, + output_names=['output'], + **kwds) + print(name, "DONE\n") + # rename(fp, join(ONNX_DIR, name)) + + +# 参考 https://github.com/OFA-Sys/Chinese-CLIP/blob/master/cn_clip/deploy/pytorch_to_onnx.py + +onnx_export('txt', + TXT, + tokenizer(['a photo of cat', 'a image of cat'], ), + input_names=['input', 'attention_mask'], + dynamic_axes={ + 'input': { + 0: 'batch', + 1: 'batch', + }, + 'attention_mask': { + 0: 'batch', + 1: 'batch', + } + }) + +onnx_export('img', + IMG, + image, + input_names=['input'], + dynamic_axes={'input': { + 0: 'batch' + }}) diff --git a/onnx/export/onnx_export_processor.py b/onnx/export/onnx_export_processor.py new file mode 100755 index 00000000..a1bc8640 --- /dev/null +++ b/onnx/export/onnx_export_processor.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +from flagai.model.mm.AltCLIP import AltCLIPProcess +from misc.config import MODEL_FP, PROCESS_DIR +from os import makedirs + +makedirs(PROCESS_DIR, exist_ok=True) +proc = AltCLIPProcess.from_pretrained(MODEL_FP) +proc.save_pretrained(PROCESS_DIR) diff --git a/onnx/export/tar.bz2.py b/onnx/export/tar.bz2.py new file mode 100755 index 00000000..896b2cbc --- /dev/null +++ b/onnx/export/tar.bz2.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +from misc.config import ONNX_DIR + +import bz2 +import tarfile + + +def txz(folder_path, output_path): + # 创建.tar文件流 + tar_stream = bz2.BZ2File(output_path, 'w') + + # 将文件夹压缩为.tar文件流 + with tarfile.TarFile(fileobj=tar_stream, mode='w') as tar: + tar.add(folder_path, arcname='') + + # 关闭.tar文件流 + tar_stream.close() + + +txz(ONNX_DIR, ONNX_DIR + '.tar.bz2') diff --git a/onnx/github.action.md b/onnx/github.action.md new file mode 100644 index 00000000..3c8512e5 --- /dev/null +++ b/onnx/github.action.md @@ -0,0 +1,16 @@ +# github action 上的容器自动构建 + +在 github action 中配置秘钥，就可以实现自动构建推送容器 + +* DOCKER_ORG + 组织名 + +* DOCKER_USERNAME + hub.docker.com 登录的用户 + +* DOCKER_PASSWORD + hub.docker.com 登录的密码 + +详情见[../.github/workflows/onnx.yml](../.github/workflows/onnx.yml)。 + +修改 [./version.txt](./version.txt) 可以设置推送容器的版本号。 diff --git a/onnx/img/build.jpg b/onnx/img/build.jpg new file mode 100644 index 00000000..a959e074 Binary files /dev/null and b/onnx/img/build.jpg differ diff --git a/onnx/img/cat.jpg b/onnx/img/cat.jpg new file mode 100644 index 00000000..887a369e Binary files /dev/null and b/onnx/img/cat.jpg differ diff --git a/onnx/img/dog.jpg b/onnx/img/dog.jpg new file mode 100644 index 00000000..dcea1f89 Binary files /dev/null and b/onnx/img/dog.jpg differ diff --git a/onnx/img/man.jpg b/onnx/img/man.jpg new file mode 100644 index 00000000..ea688c54 Binary files /dev/null and b/onnx/img/man.jpg differ diff --git a/onnx/img/rat.jpg b/onnx/img/rat.jpg new file mode 100644 index 00000000..460eb6a4 Binary files /dev/null and b/onnx/img/rat.jpg differ diff --git a/onnx/img/woman.jpg b/onnx/img/woman.jpg new file mode 100644 index 00000000..50fc137b Binary files /dev/null and b/onnx/img/woman.jpg differ diff --git a/onnx/misc/__init__.py b/onnx/misc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/onnx/misc/clip_model.py b/onnx/misc/clip_model.py new file mode 100755 index 00000000..6bd41a19 --- /dev/null +++ b/onnx/misc/clip_model.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import torch +import torch.nn as nn +from .device import DEVICE +from .config import MODEL_FP +from flagai.model.mm.AltCLIP import CLIPHF + +MODEL = CLIPHF.from_pretrained(MODEL_FP) + +MODEL.eval() +MODEL.to(DEVICE) + + +class ImgModel(nn.Module): + + def __init__(self): + super(ImgModel, self).__init__() + self.model = MODEL + + def forward(self, image): + with torch.no_grad(): + image = image.to(DEVICE) + return self.model.get_image_features(image) + + +class TxtModel(nn.Module): + + def __init__(self): + super(TxtModel, self).__init__() + self.model = MODEL + + def forward(self, text, attention_mask): + with torch.no_grad(): + text = text.to(DEVICE) + attention_mask = attention_mask.to(DEVICE) + return self.model.get_text_features(text, attention_mask=attention_mask) + + +IMG = ImgModel() +TXT = TxtModel() diff --git a/onnx/misc/config.py b/onnx/misc/config.py new file mode 100644 index 00000000..ebb76cbf --- /dev/null +++ b/onnx/misc/config.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +from os.path import abspath, dirname, join +import os + +MODEL_NAME = os.getenv('MODEL') or 'AltCLIP-XLMR-L-m18' + +ROOT = dirname(dirname(abspath(__file__))) + +IMG_DIR = join(ROOT, 'img') +MODEL_DIR = join(ROOT, 'model') + +MODEL_FP = join(MODEL_DIR, MODEL_NAME) + +ONNX_DIR = join(ROOT, 'onnx', MODEL_NAME) + +ONNX_FP = join(ONNX_DIR, 'onnx') +PROCESS_DIR = join(ONNX_DIR, 'process') + +opset_version = 17 diff --git a/onnx/misc/device.py b/onnx/misc/device.py new file mode 100644 index 00000000..3f43e279 --- /dev/null +++ b/onnx/misc/device.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import torch +import os + +device = os.getenv('DEVICE') +if not device: + if torch.cuda.is_available(): + device = 'cuda' + elif torch.backends.mps.is_available(): + device = 'mps' + else: + device = 'cpu' + +DEVICE = torch.device(device) diff --git a/onnx/misc/proc.py b/onnx/misc/proc.py new file mode 100755 index 00000000..04b36e9f --- /dev/null +++ b/onnx/misc/proc.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +from .config import PROCESS_DIR +from transformers import CLIPProcessor + + +class AltCLIPProcess(CLIPProcessor): + tokenizer_class = ("XLMRobertaTokenizer", "XLMRobertaTokenizerFast") + + def __init__(self, feature_extractor, tokenizer): + super().__init__(feature_extractor, tokenizer) + + +proc = AltCLIPProcess.from_pretrained(PROCESS_DIR) + +_tokenizer = proc.tokenizer +_transform = proc.feature_extractor + + +def transform(img): + img = _transform(img) + return img['pixel_values'] + + +def tokenizer(li): + tokenizer_out = _tokenizer(li, + padding=True, + truncation=True, + max_length=77, + return_tensors='pt') + text = tokenizer_out["input_ids"] + attention_mask = tokenizer_out["attention_mask"] + return text, attention_mask diff --git a/onnx/onnx/.keep b/onnx/onnx/.keep new file mode 100644 index 00000000..e69de29b diff --git a/onnx/os/root/.config/pip/pip.conf b/onnx/os/root/.config/pip/pip.conf new file mode 100644 index 00000000..5b7fab99 --- /dev/null +++ b/onnx/os/root/.config/pip/pip.conf @@ -0,0 +1,3 @@ +[global] +break-system-packages=true + diff --git a/onnx/test/clip/clip_img.py b/onnx/test/clip/clip_img.py new file mode 100755 index 00000000..aceeb6b7 --- /dev/null +++ b/onnx/test/clip/clip_img.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +from misc.proc import transform +from misc.clip_model import IMG +import torch + + +def img2vec(img): + img = transform(img) + img = torch.tensor(img) + return IMG.forward(img) + + +if __name__ == "__main__": + from misc.config import IMG_DIR + from os.path import join + fp = join(IMG_DIR, 'cat.jpg') + from PIL import Image + img = Image.open(fp) + print(img2vec(img)) diff --git a/onnx/test/clip/clip_test.py b/onnx/test/clip/clip_test.py new file mode 100755 index 00000000..c868c890 --- /dev/null +++ b/onnx/test/clip/clip_test.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import torch +from PIL import Image +from os.path import basename, join +from time import time +from clip_txt import txt2vec +from clip_img import img2vec + +COST = None + + +def inference(img, tmpl_kind_li): + global COST + image = Image.open(img) + begin = time() + image_features = img2vec(image) + if COST is not None: + COST += (time() - begin) + + for tmpl, kind_li in tmpl_kind_li: + li = [] + for i in kind_li: + li.append(tmpl % i) + + begin = time() + text_features = txt2vec(li) + if COST is not None: + COST += (time() - begin) + + with torch.no_grad(): + text_probs = (image_features @ text_features.T).softmax(dim=-1) + + for kind, p in zip(kind_li, text_probs.cpu().numpy()[0].tolist()): + p = round(p * 10000) + if p: + print(" %s %.1f%%" % (kind, p / 100)) + return + + +if __name__ == "__main__": + from misc.config import IMG_DIR + from glob import glob + li = glob(join(IMG_DIR, '*.jpg')) + # 预热，py.compile 要第一次运行才编译 + inference(li[0], + (('a photo of %s', ('cat', 'rat', 'dog', 'man', 'woman')), )) + COST = 0 + for i in li: + print("\n* " + basename(i)) + inference(i, (('a photo of %s', ('cat', 'rat', 'dog', 'man', 'woman')), + ('一张%s的图片', ('猫', '老鼠', '狗', '男人', '女人')))) + print('\ncost %2.fms' % (1000 * COST)) diff --git a/onnx/test/clip/clip_txt.py b/onnx/test/clip/clip_txt.py new file mode 100755 index 00000000..9473a76b --- /dev/null +++ b/onnx/test/clip/clip_txt.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +from misc.proc import tokenizer +from misc.clip_model import TXT + + +def txt2vec(li): + return TXT.forward(*tokenizer(li)) + + +if __name__ == "__main__": + from os.path import join + from glob import glob + from misc.config import ROOT + from test_txt import TEST_TXT + + li = glob(join(ROOT, 'jpg/*.jpg')) + for li in TEST_TXT: + r = txt2vec(li) + for txt, i in zip(li, r): + print(txt) + print(i) + print('\n') diff --git a/onnx/test/clip/test_txt.py b/onnx/test/clip/test_txt.py new file mode 120000 index 00000000..6e5c26bf --- /dev/null +++ b/onnx/test/clip/test_txt.py @@ -0,0 +1 @@ +../onnx/test_txt.py \ No newline at end of file diff --git a/onnx/test/onnx/onnx_img.py b/onnx/test/onnx/onnx_img.py new file mode 100755 index 00000000..8b350b20 --- /dev/null +++ b/onnx/test/onnx/onnx_img.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +from misc.proc import transform +from PIL import Image +from onnx_load import onnx_load + +MODEL = onnx_load('img') + + +def img2vec(img): + return MODEL.run(None, {'input': transform(img)})[0] + + +if __name__ == '__main__': + from misc.config import IMG_DIR + from os.path import join + img = Image.open(join(IMG_DIR, 'cat.jpg')) + vec = img2vec(img) + print(vec) diff --git a/onnx/test/onnx/onnx_load.py b/onnx/test/onnx/onnx_load.py new file mode 100644 index 00000000..491e64fa --- /dev/null +++ b/onnx/test/onnx/onnx_load.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import onnxruntime +from misc.config import ONNX_FP +from os.path import join + +session = onnxruntime.SessionOptions() +option = onnxruntime.RunOptions() +option.log_severity_level = 2 + + +def onnx_load(kind): + fp = join(ONNX_FP, f'{kind}.onnx') + + sess = onnxruntime.InferenceSession(fp, + sess_options=session, + providers=[ + 'TensorrtExecutionProvider', + 'CUDAExecutionProvider', + 'CoreMLExecutionProvider', + 'CPUExecutionProvider' + ]) + return sess diff --git a/onnx/test/onnx/onnx_test.py b/onnx/test/onnx/onnx_test.py new file mode 100755 index 00000000..4dfac789 --- /dev/null +++ b/onnx/test/onnx/onnx_test.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +from torch import tensor +import torch +from PIL import Image +from os.path import basename, join +from time import time +from onnx_txt import txt2vec +from onnx_img import img2vec + +COST = None + + +def inference(img, tmpl_kind_li): + global COST + image = Image.open(img) + begin = time() + image_features = img2vec(image) + if COST is not None: + COST += (time() - begin) + + for tmpl, kind_li in tmpl_kind_li: + li = [] + for i in kind_li: + li.append(tmpl % i) + + begin = time() + text_features = txt2vec(li) + if COST is not None: + COST += (time() - begin) + + with torch.no_grad(): + image_features = tensor(image_features) + text_features = tensor(text_features) + text_probs = (image_features @ text_features.T).softmax(dim=-1) + + for kind, p in zip(kind_li, text_probs.cpu().numpy()[0].tolist()): + p = round(p * 10000) + if p: + print(" %s %.1f%%" % (kind, p / 100)) + return + + +if __name__ == "__main__": + from misc.config import IMG_DIR + from glob import glob + li = glob(join(IMG_DIR, '*.jpg')) + # 预热，py.compile 要第一次运行才编译 + inference(li[0], + (('a photo of %s', ('cat', 'rat', 'dog', 'man', 'woman')), )) + COST = 0 + for i in li: + print("\n* " + basename(i)) + inference(i, (('a photo of %s', ('cat', 'rat', 'dog', 'man', 'woman')), + ('一张%s的图片', ('猫', '老鼠', '狗', '男人', '女人')))) + print('\ncost %2.fms' % (1000 * COST)) diff --git a/onnx/test/onnx/onnx_txt.py b/onnx/test/onnx/onnx_txt.py new file mode 100755 index 00000000..9a99077e --- /dev/null +++ b/onnx/test/onnx/onnx_txt.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +from misc.proc import tokenizer +from onnx_load import onnx_load + +MODEL = onnx_load('txt') + + +def txt2vec(li): + text, attention_mask = tokenizer(li) + text = text.numpy() + attention_mask = attention_mask.numpy() + output = MODEL.run(None, {'input': text, 'attention_mask': attention_mask}) + return output[0] + + +if __name__ == '__main__': + from test_txt import TEST_TXT + for li in TEST_TXT: + r = txt2vec(li) + for txt, i in zip(li, r): + print(txt) + print(i) + print('\n') diff --git a/onnx/test/onnx/requirements.txt b/onnx/test/onnx/requirements.txt new file mode 100644 index 00000000..976a2b1f --- /dev/null +++ b/onnx/test/onnx/requirements.txt @@ -0,0 +1 @@ +transformers diff --git a/onnx/test/onnx/setup.sh b/onnx/test/onnx/setup.sh new file mode 100755 index 00000000..d508ff3c --- /dev/null +++ b/onnx/test/onnx/setup.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +DIR=$(realpath $0) && DIR=${DIR%/*} +cd $DIR +set -ex + +pip install -r ./requirements.txt +if [ -x "$(command -v nvidia-smi)" ]; then + pip install onnxruntime-gpu +else + system=$(uname) + arch=$(uname -m) + if [[ $system == *"Darwin"* ]] && [[ $arch == "arm64" ]]; then + pip install onnxruntime-silicon + else + grep 'vendor_id' /proc/cpuinfo | awk '{print $3}' | grep 'GenuineIntel' >/dev/null + if [ $? -eq 0 ]; then + echo 'Intel CPU' + pip install onnxruntime-openvino + else + echo 'Non-Intel CPU' + pip install onnxruntime + fi + fi +fi diff --git a/onnx/test/onnx/test_txt.py b/onnx/test/onnx/test_txt.py new file mode 100644 index 00000000..0fa6707b --- /dev/null +++ b/onnx/test/onnx/test_txt.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +TEST_TXT = ( + ('a photo of dog', 'a photo for chinese woman'), + ('a photo of dog', 'a photo for cat'), +) diff --git a/onnx/version.txt b/onnx/version.txt new file mode 100644 index 00000000..04c21ffb --- /dev/null +++ b/onnx/version.txt @@ -0,0 +1 @@ +2023-06-02 diff --git a/setup.py b/setup.py index 9c460556..be25acc1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,8 @@ setup( name="flagai", version="v1.6.3", - description="FlagAI aims to help researchers and developers to freely train and test large-scale models for NLP/CV/VL tasks.", + description= + "FlagAI aims to help researchers and developers to freely train and test large-scale models for NLP/CV/VL tasks.", long_description=open("README.md", encoding="utf-8").read(), long_description_content_type="text/markdown", author="FlagAI-Open", @@ -17,24 +18,23 @@ include_package_data=True, python_requires=">=3.8", install_requires=[ - 'nltk==3.6.7', - 'sentencepiece==0.1.96', - 'boto3==1.21.42', - 'pandas==1.3.5', + 'nltk==3.8.1', + 'sentencepiece==0.1.99', + 'boto3==1.26.144', + 'pandas==2.0.2', 'jieba==0.42.1', - 'scikit-learn==1.0.2', - 'tensorboard==2.9.0', - 'transformers==4.20.1', + 'scikit-learn==1.2.2', + 'tensorboard==2.13.0', + 'transformers==4.29.2', 'datasets==2.0.0', - 'setuptools==59.5.0', - 'protobuf==3.19.6', + 'setuptools==66.0.0', + 'protobuf==4.23.2', 'ftfy == 6.1.1', - 'Pillow == 9.3.0', - 'einops == 0.3.0', - 'diffusers == 0.7.2', - 'pytorch-lightning == 1.6.5', + 'pillow == 9.5.0', + 'einops == 0.6.1', + 'diffusers == 0.16.1', + 'pytorch-lightning == 2.0.2', 'taming-transformers-rom1504 == 0.0.6', 'rouge-score == 0.1.2', 'sacrebleu == 2.3.1', - ] -) + ])