7.Prompt/mindprompt_emotect.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "基于GPT-2预训练模型的prompt learning：通过人工定义prompt template与verbalizer，进行句子情感分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/utils/download.py:26: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import mindspore.ops as ops\n",
    "\n",
    "from mindprompt.plms import load_plm\n",
    "from mindprompt import PromptDataLoader\n",
    "from mindprompt import PromptForClassification\n",
    "from mindprompt.prompts import ManualTemplate\n",
    "from mindprompt.prompts import ManualVerbalizer\n",
    "from mindprompt.data_utils import InputExample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第一步：定义情感分析任务\n",
    "classes = [\"negative\", \"positive\"] # label\n",
    "dataset = [\n",
    "    InputExample(\n",
    "        guid=0,\n",
    "        text_a=\"Albert Einstein was one of the greatest intellects of his time.\",\n",
    "    ),\n",
    "    InputExample(\n",
    "        guid=1,\n",
    "        text_a=\"The film was badly made.\",\n",
    "    ),\n",
    "    InputExample(\n",
    "        guid=2,\n",
    "        text_a=\"Kevin is a fantastic boy.\",\n",
    "    ),\n",
    "    InputExample(\n",
    "        guid=3,\n",
    "        text_a=\"kitty is a bad boy.\",\n",
    "    ),\n",
    "] # inputs [x] e.g., i love this movie "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"guid\": 0,\n",
      "  \"label\": null,\n",
      "  \"meta\": {},\n",
      "  \"text_a\": \"Albert Einstein was one of the greatest intellects of his time.\",\n",
      "  \"text_b\": \"\",\n",
      "  \"tgt_text\": null\n",
      "}\n",
      "\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "print(dataset[0])\n",
    "print(len(dataset))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 523M/523M [00:11<00:00, 46.0MB/s] \n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.269.655 [mindspore/train/serialization.py:1139] For 'load_param_into_net', 148 parameters in the 'net' are not loaded, because they are not in the 'parameter_dict', please check whether the network structure is consistent when training and loading checkpoint.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.271.966 [mindspore/train/serialization.py:1141] transformer.wte.embedding_table is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.272.441 [mindspore/train/serialization.py:1141] transformer.wpe.embedding_table is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.272.795 [mindspore/train/serialization.py:1141] transformer.h.0.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.273.139 [mindspore/train/serialization.py:1141] transformer.h.0.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.273.500 [mindspore/train/serialization.py:1141] transformer.h.0.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.274.303 [mindspore/train/serialization.py:1141] transformer.h.0.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.274.609 [mindspore/train/serialization.py:1141] transformer.h.0.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.274.951 [mindspore/train/serialization.py:1141] transformer.h.0.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.275.257 [mindspore/train/serialization.py:1141] transformer.h.0.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.275.559 [mindspore/train/serialization.py:1141] transformer.h.0.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.275.840 [mindspore/train/serialization.py:1141] transformer.h.0.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.276.167 [mindspore/train/serialization.py:1141] transformer.h.0.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.276.478 [mindspore/train/serialization.py:1141] transformer.h.0.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.276.778 [mindspore/train/serialization.py:1141] transformer.h.0.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.277.068 [mindspore/train/serialization.py:1141] transformer.h.1.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.277.396 [mindspore/train/serialization.py:1141] transformer.h.1.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.277.699 [mindspore/train/serialization.py:1141] transformer.h.1.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.278.001 [mindspore/train/serialization.py:1141] transformer.h.1.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.278.339 [mindspore/train/serialization.py:1141] transformer.h.1.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.278.702 [mindspore/train/serialization.py:1141] transformer.h.1.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.278.988 [mindspore/train/serialization.py:1141] transformer.h.1.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.279.315 [mindspore/train/serialization.py:1141] transformer.h.1.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.280.798 [mindspore/train/serialization.py:1141] transformer.h.1.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.281.153 [mindspore/train/serialization.py:1141] transformer.h.1.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.281.457 [mindspore/train/serialization.py:1141] transformer.h.1.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.281.757 [mindspore/train/serialization.py:1141] transformer.h.1.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.282.042 [mindspore/train/serialization.py:1141] transformer.h.2.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.282.362 [mindspore/train/serialization.py:1141] transformer.h.2.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.282.661 [mindspore/train/serialization.py:1141] transformer.h.2.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.282.964 [mindspore/train/serialization.py:1141] transformer.h.2.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.283.264 [mindspore/train/serialization.py:1141] transformer.h.2.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.283.562 [mindspore/train/serialization.py:1141] transformer.h.2.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.283.859 [mindspore/train/serialization.py:1141] transformer.h.2.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.284.155 [mindspore/train/serialization.py:1141] transformer.h.2.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.284.458 [mindspore/train/serialization.py:1141] transformer.h.2.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.284.764 [mindspore/train/serialization.py:1141] transformer.h.2.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.285.087 [mindspore/train/serialization.py:1141] transformer.h.2.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.285.389 [mindspore/train/serialization.py:1141] transformer.h.2.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.285.692 [mindspore/train/serialization.py:1141] transformer.h.3.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.285.990 [mindspore/train/serialization.py:1141] transformer.h.3.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.287.590 [mindspore/train/serialization.py:1141] transformer.h.3.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.287.883 [mindspore/train/serialization.py:1141] transformer.h.3.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.288.213 [mindspore/train/serialization.py:1141] transformer.h.3.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.288.515 [mindspore/train/serialization.py:1141] transformer.h.3.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.289.091 [mindspore/train/serialization.py:1141] transformer.h.3.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.289.414 [mindspore/train/serialization.py:1141] transformer.h.3.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.289.713 [mindspore/train/serialization.py:1141] transformer.h.3.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.290.014 [mindspore/train/serialization.py:1141] transformer.h.3.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.290.316 [mindspore/train/serialization.py:1141] transformer.h.3.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.290.615 [mindspore/train/serialization.py:1141] transformer.h.3.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.290.913 [mindspore/train/serialization.py:1141] transformer.h.4.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.291.209 [mindspore/train/serialization.py:1141] transformer.h.4.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.291.503 [mindspore/train/serialization.py:1141] transformer.h.4.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.291.796 [mindspore/train/serialization.py:1141] transformer.h.4.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.292.092 [mindspore/train/serialization.py:1141] transformer.h.4.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.292.386 [mindspore/train/serialization.py:1141] transformer.h.4.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.292.657 [mindspore/train/serialization.py:1141] transformer.h.4.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.292.976 [mindspore/train/serialization.py:1141] transformer.h.4.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.293.297 [mindspore/train/serialization.py:1141] transformer.h.4.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.293.599 [mindspore/train/serialization.py:1141] transformer.h.4.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.293.897 [mindspore/train/serialization.py:1141] transformer.h.4.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.294.194 [mindspore/train/serialization.py:1141] transformer.h.4.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.294.489 [mindspore/train/serialization.py:1141] transformer.h.5.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.294.784 [mindspore/train/serialization.py:1141] transformer.h.5.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.295.085 [mindspore/train/serialization.py:1141] transformer.h.5.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.296.912 [mindspore/train/serialization.py:1141] transformer.h.5.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.297.203 [mindspore/train/serialization.py:1141] transformer.h.5.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.297.528 [mindspore/train/serialization.py:1141] transformer.h.5.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.297.827 [mindspore/train/serialization.py:1141] transformer.h.5.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.298.125 [mindspore/train/serialization.py:1141] transformer.h.5.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.298.424 [mindspore/train/serialization.py:1141] transformer.h.5.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.298.720 [mindspore/train/serialization.py:1141] transformer.h.5.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.299.017 [mindspore/train/serialization.py:1141] transformer.h.5.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.299.314 [mindspore/train/serialization.py:1141] transformer.h.5.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.299.613 [mindspore/train/serialization.py:1141] transformer.h.6.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.299.907 [mindspore/train/serialization.py:1141] transformer.h.6.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.300.202 [mindspore/train/serialization.py:1141] transformer.h.6.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.300.498 [mindspore/train/serialization.py:1141] transformer.h.6.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.300.797 [mindspore/train/serialization.py:1141] transformer.h.6.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.301.100 [mindspore/train/serialization.py:1141] transformer.h.6.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.301.397 [mindspore/train/serialization.py:1141] transformer.h.6.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.301.700 [mindspore/train/serialization.py:1141] transformer.h.6.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.301.998 [mindspore/train/serialization.py:1141] transformer.h.6.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.302.293 [mindspore/train/serialization.py:1141] transformer.h.6.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.302.589 [mindspore/train/serialization.py:1141] transformer.h.6.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.302.889 [mindspore/train/serialization.py:1141] transformer.h.6.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.303.163 [mindspore/train/serialization.py:1141] transformer.h.7.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.303.485 [mindspore/train/serialization.py:1141] transformer.h.7.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.303.787 [mindspore/train/serialization.py:1141] transformer.h.7.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.304.086 [mindspore/train/serialization.py:1141] transformer.h.7.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.304.387 [mindspore/train/serialization.py:1141] transformer.h.7.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.304.682 [mindspore/train/serialization.py:1141] transformer.h.7.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.304.992 [mindspore/train/serialization.py:1141] transformer.h.7.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.305.290 [mindspore/train/serialization.py:1141] transformer.h.7.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.305.591 [mindspore/train/serialization.py:1141] transformer.h.7.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.305.888 [mindspore/train/serialization.py:1141] transformer.h.7.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.306.185 [mindspore/train/serialization.py:1141] transformer.h.7.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.306.481 [mindspore/train/serialization.py:1141] transformer.h.7.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.306.779 [mindspore/train/serialization.py:1141] transformer.h.8.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.307.078 [mindspore/train/serialization.py:1141] transformer.h.8.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.307.374 [mindspore/train/serialization.py:1141] transformer.h.8.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.307.670 [mindspore/train/serialization.py:1141] transformer.h.8.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.307.966 [mindspore/train/serialization.py:1141] transformer.h.8.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.308.264 [mindspore/train/serialization.py:1141] transformer.h.8.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.308.561 [mindspore/train/serialization.py:1141] transformer.h.8.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.308.870 [mindspore/train/serialization.py:1141] transformer.h.8.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.309.172 [mindspore/train/serialization.py:1141] transformer.h.8.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.309.471 [mindspore/train/serialization.py:1141] transformer.h.8.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.309.769 [mindspore/train/serialization.py:1141] transformer.h.8.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.310.068 [mindspore/train/serialization.py:1141] transformer.h.8.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.310.369 [mindspore/train/serialization.py:1141] transformer.h.9.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.310.667 [mindspore/train/serialization.py:1141] transformer.h.9.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.310.966 [mindspore/train/serialization.py:1141] transformer.h.9.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.314.631 [mindspore/train/serialization.py:1141] transformer.h.9.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.314.941 [mindspore/train/serialization.py:1141] transformer.h.9.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.315.273 [mindspore/train/serialization.py:1141] transformer.h.9.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.315.574 [mindspore/train/serialization.py:1141] transformer.h.9.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.315.873 [mindspore/train/serialization.py:1141] transformer.h.9.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.316.175 [mindspore/train/serialization.py:1141] transformer.h.9.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.316.476 [mindspore/train/serialization.py:1141] transformer.h.9.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.316.778 [mindspore/train/serialization.py:1141] transformer.h.9.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.317.084 [mindspore/train/serialization.py:1141] transformer.h.9.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.317.386 [mindspore/train/serialization.py:1141] transformer.h.10.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.317.683 [mindspore/train/serialization.py:1141] transformer.h.10.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.318.090 [mindspore/train/serialization.py:1141] transformer.h.10.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.318.406 [mindspore/train/serialization.py:1141] transformer.h.10.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.319.667 [mindspore/train/serialization.py:1141] transformer.h.10.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.319.953 [mindspore/train/serialization.py:1141] transformer.h.10.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.320.281 [mindspore/train/serialization.py:1141] transformer.h.10.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.320.583 [mindspore/train/serialization.py:1141] transformer.h.10.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.320.899 [mindspore/train/serialization.py:1141] transformer.h.10.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.321.208 [mindspore/train/serialization.py:1141] transformer.h.10.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.321.506 [mindspore/train/serialization.py:1141] transformer.h.10.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.322.343 [mindspore/train/serialization.py:1141] transformer.h.10.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.322.627 [mindspore/train/serialization.py:1141] transformer.h.11.ln_1.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.322.956 [mindspore/train/serialization.py:1141] transformer.h.11.ln_1.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.323.255 [mindspore/train/serialization.py:1141] transformer.h.11.attn.c_attn.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.323.554 [mindspore/train/serialization.py:1141] transformer.h.11.attn.c_attn.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.323.854 [mindspore/train/serialization.py:1141] transformer.h.11.attn.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.324.150 [mindspore/train/serialization.py:1141] transformer.h.11.attn.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.324.450 [mindspore/train/serialization.py:1141] transformer.h.11.ln_2.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.324.748 [mindspore/train/serialization.py:1141] transformer.h.11.ln_2.beta is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.325.055 [mindspore/train/serialization.py:1141] transformer.h.11.mlp.c_fc.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.325.352 [mindspore/train/serialization.py:1141] transformer.h.11.mlp.c_fc.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.326.412 [mindspore/train/serialization.py:1141] transformer.h.11.mlp.c_proj.weight is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.326.740 [mindspore/train/serialization.py:1141] transformer.h.11.mlp.c_proj.bias is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.327.041 [mindspore/train/serialization.py:1141] transformer.ln_f.gamma is not loaded.\n",
      "[WARNING] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.327.341 [mindspore/train/serialization.py:1141] transformer.ln_f.beta is not loaded.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:05:47.810.405 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:304] Using pad_token, but it is not set yet.\n"
     ]
    }
   ],
   "source": [
    "# 第二步：定义Pre-trained Language Models (PLMs)作为backbone.\n",
    "plm, tokenizer, model_config, WrapperClass = load_plm(\"gpt2\", \"gpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第三步：定义模板Template\n",
    "promptTemplate = ManualTemplate(\n",
    "    text='{\"placeholder\":\"text_a\"} It was {\"mask\"}',\n",
    "    tokenizer=tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第四步：定义标签词映射Verbalizer\n",
    "# classses： labels ['positive', 'negative']\n",
    "# label_words: mapping from prediction to label\n",
    "promptVerbalizer = ManualVerbalizer(\n",
    "    classes=classes,\n",
    "    label_words={\n",
    "        \"negative\": [\"bad\"],\n",
    "        \"positive\": [\"good\", \"wonderful\", \"great\"],\n",
    "    },\n",
    "    tokenizer=tokenizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input example: \n",
      " {\n",
      "  \"guid\": 0,\n",
      "  \"label\": null,\n",
      "  \"meta\": {},\n",
      "  \"text_a\": \"Albert Einstein was one of the greatest intellects of his time.\",\n",
      "  \"text_b\": \"\",\n",
      "  \"tgt_text\": null\n",
      "}\n",
      "\n",
      "wrapped example:\n",
      "{'text': 'Albert Einstein was one of the greatest intellects of his time.', 'loss_ids': 0, 'shortenable_ids': 1}\n",
      "{'text': ' It was', 'loss_ids': 0, 'shortenable_ids': 0}\n",
      "{'text': '<mask>', 'loss_ids': 1, 'shortenable_ids': 0}\n"
     ]
    }
   ],
   "source": [
    "# 补充：wrap 操作演示\n",
    "# 将prompt依照slot与template内容进行切分\n",
    "# loss_ids: 是否计算损失（是：1， 否：0）\n",
    "# shortenable_ids: 是否对内容序列进行长度裁剪\n",
    "\n",
    "print(f'input example: \\n {dataset[0]}')\n",
    "\n",
    "wrapped_example = promptTemplate.wrap_one_example(dataset[0])\n",
    "\n",
    "print(f'wrapped example:')\n",
    "for ele in wrapped_example[0]:\n",
    "    print(ele)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:17:53.776.034 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:328] Using mask_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:17:53.777.046 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:328] Using mask_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:17:53.777.392 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:293] Using sep_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:17:53.777.741 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:293] Using sep_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:17:53.778.060 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:316] Using cls_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:17:53.778.378 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:316] Using cls_token, but it is not set yet.\n"
     ]
    }
   ],
   "source": [
    "# 补充： tokenize 操作演示\n",
    "wrap_tokenizer = WrapperClass(max_seq_length=32, tokenizer=tokenizer)\n",
    "tokenized_example = wrap_tokenizer.tokenize_one_example(wrapped_example, teacher_forcing=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenized example: \n",
      " {'input_ids': [42590, 24572, 373, 530, 286, 262, 6000, 7654, 82, 286, 465, 640, 13, 632, 373, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257], 'loss_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'input_ids_len': 15}\n",
      "\n",
      "printing each key-value pair:\n",
      "input_ids:[42590, 24572, 373, 530, 286, 262, 6000, 7654, 82, 286, 465, 640, 13, 632, 373, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257, 50257]\n",
      "loss_ids:[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "attention_mask:[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "input_ids_len:15\n",
      "\n",
      "input_ids to tokens:\n",
      "Albert ĠEinstein Ġwas Ġone Ġof Ġthe Ġgreatest Ġintellect s Ġof Ġhis Ġtime . ĠIt Ġwas <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n"
     ]
    }
   ],
   "source": [
    "print(f'tokenized example: \\n {tokenized_example}\\n')\n",
    "\n",
    "# tokenized exmaple 中的信息：\n",
    "# input_ids: prompt进行分词，补齐长度，映射为数字索引后的结果\n",
    "# loss_ids: input_ids中每个元素对应的loss_id，即是否对该词计算loss\n",
    "# attention_mask: 标记input_ids中padding的位置\n",
    "# input_ids_len：带有实际意义的序列长度，即添加padding前的句子长度\n",
    "print('printing each key-value pair:')\n",
    "for key, value in tokenized_example.items():\n",
    "    print(f'{key}:{value}')\n",
    "\n",
    "# input_ids 变回 tokens\n",
    "print('\\ninput_ids to tokens:')\n",
    "tokens = []   \n",
    "for id in tokenized_example['input_ids']:\n",
    "    tokens.append(tokenizer.id_to_token(id))\n",
    "\n",
    "print(' '.join(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第五步：封装PromptModel\n",
    "promptModel = PromptForClassification(\n",
    "    template=promptTemplate,\n",
    "    plm=plm,\n",
    "    verbalizer=promptVerbalizer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:20:34.705.517 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:328] Using mask_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:20:34.706.888 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:328] Using mask_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:20:34.707.348 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:293] Using sep_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:20:34.707.760 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:293] Using sep_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:20:34.708.122 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:316] Using cls_token, but it is not set yet.\n",
      "[ERROR] ME(3116130:139764658304832,MainProcess):2023-06-07-21:20:34.708.491 [/home/daiyuxin/anaconda3/envs/xyr/lib/python3.7/site-packages/mindnlp/abc/mixins/special_tokens_mixin.py:316] Using cls_token, but it is not set yet.\n",
      "tokenizing: 4it [00:00, 1093.48it/s]\n"
     ]
    }
   ],
   "source": [
    "# 第六步：定义dataloader\n",
    "data_loader = PromptDataLoader(\n",
    "    dataset=dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    template=promptTemplate,\n",
    "    tokenizer_wrapper_class=WrapperClass,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive\n",
      "positive\n",
      "positive\n",
      "positive\n"
     ]
    }
   ],
   "source": [
    "# 第七步：训练与推理\n",
    "promptModel.set_train(False)\n",
    "for batch in data_loader.dataloader.create_tuple_iterator():\n",
    "    logits = promptModel(batch)\n",
    "    preds = ops.argmax(logits, dim=-1)\n",
    "    print(classes[preds])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "xyr",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}