fastai_NLP/BERT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XqLEbsvyOjJ5"
   },
   "source": [
    "# 在Fastai框架下基于BERT模型的中文文本分类\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "# 多行输出\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\" "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KdRDycJLO1qW"
   },
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Yea4tylQORLi"
   },
   "outputs": [],
   "source": [
    "#安装pytorch下包含BERT预训练模型的库\n",
    "# !pip install pytorch-pretrained-bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hmikQXQAPenZ"
   },
   "outputs": [],
   "source": [
    "#引入必要的依赖包\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from pathlib import Path\n",
    "from typing import *\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "\n",
    "from fastai import *\n",
    "from fastai.vision import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uFPURKkfQATE"
   },
   "outputs": [],
   "source": [
    "#设置一个config类，便于参数配置与更改\n",
    "class Config1(dict):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        for k, v in kwargs.items():\n",
    "            setattr(self, k, v)\n",
    "    \n",
    "    def set(self, key, val):\n",
    "        self[key] = val\n",
    "        setattr(self, key, val)\n",
    "\n",
    "config = Config1(\n",
    "    testing=False,\n",
    "    bert_model_name=\"bert-base-chinese\", \n",
    "    #选用中文预训练模型：Chinese Simplified and Traditional, 12-layer, 768-hidden, 12-heads, 110M parameters\n",
    "    max_lr=3e-5,\n",
    "    epochs=5,\n",
    "    use_fp16=False, #fastai里可以方便地调整精度，加快训练速度：learner.to_fp16()\n",
    "    bs=8,\n",
    "    max_seq_len=128, #选取合适的seq_length，较大的值可能导致训练极慢报错等\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3dS1_jNESiEv"
   },
   "outputs": [],
   "source": [
    "root = Path('~/.fastai/data/xiecheng/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fastai.datasets.Config"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bYXEtepaSilw"
   },
   "source": [
    "## Customize Tokenizer and Vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "JiKPDWViStZS",
    "outputId": "3e7057eb-4b85-41a6-c79d-30c2fa61735a"
   },
   "outputs": [],
   "source": [
    "#pytorch_bert包提供了BertTokenizer类，从选取的模型中提取tok\n",
    "from pytorch_pretrained_bert import BertTokenizer\n",
    "bert_tok = BertTokenizer.from_pretrained(\n",
    "    config.bert_model_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8WHx6utjUCVs"
   },
   "source": [
    "定制Tokenizer需要继承fastai的BaseTokenizer类，注意BERT会在sequence的首尾加上特殊token[CLS]和[SEP]，与fastai里的bos和eos有重合，所以在之后会有include_eos=Fasle的设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "09bYVoWBT0wr"
   },
   "outputs": [],
   "source": [
    "class FastAiBertTokenizer(BaseTokenizer):\n",
    "    \"\"\"Wrapper around a BertTokenizer to be a BaseTokenizer in fastai\"\"\"\n",
    "    def __init__(self, tokenizer: BertTokenizer, max_seq_len: int=128, **kwargs):\n",
    "        self._pretrained_tokenizer = tokenizer\n",
    "        self.max_seq_len = max_seq_len\n",
    "\n",
    "    def __call__(self, *args, **kwargs):\n",
    "        return self\n",
    "\n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        \"\"\"Limits the maximum sequence length\"\"\"\n",
    "        return [\"[CLS]\"] + self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2] + [\"[SEP]\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0HD1qCGCVMMe"
   },
   "outputs": [],
   "source": [
    "#最终的tokenizer\n",
    "fastai_tokenizer = Tokenizer(\n",
    "    tok_func=FastAiBertTokenizer(bert_tok, max_seq_len=config.max_seq_len), \n",
    "    pre_rules=[], \n",
    "    post_rules=[]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 369
    },
    "colab_type": "code",
    "id": "LxXk9yB1V0nB",
    "outputId": "cb24389b-849a-4396-e2af-2fc8b60b0475"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('傲', 1000),\n",
       " ('傳', 1001),\n",
       " ('債', 1002),\n",
       " ('傷', 1003),\n",
       " ('傻', 1004),\n",
       " ('傾', 1005),\n",
       " ('僅', 1006),\n",
       " ('働', 1007),\n",
       " ('像', 1008),\n",
       " ('僑', 1009)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check the vocab\n",
    "list(bert_tok.vocab.items())[1000:1010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NYIrjtkHW2F2"
   },
   "outputs": [],
   "source": [
    "#设置vocab\n",
    "fastai_bert_vocab = Vocab(list(bert_tok.vocab.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DUU3lFXrXAUw"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KHfJODECXEVp"
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rwkcOhG3Uc_x"
   },
   "source": [
    "本次使用的数据集信息：\n",
    "\n",
    "*   名称及出处：[ChnSentiCorp_htl_all](https://github.com/ZeroLeon/ChineseNlpCorpus/blob/master/datasets/ChnSentiCorp_htl_all/intro.ipynb)\n",
    "*   数据概览： 7000 多条酒店评论数据，5000 多条正向评论，2000 多条负向评论\n",
    "*   推荐实验： 情感/观点/评论 倾向性分析\n",
    "*   数据来源： 携程网\n",
    "*   原数据集： ChnSentiCorp_htl，由谭松波老师整理的一份数据集\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5gU8EtWUXRjS"
   },
   "outputs": [],
   "source": [
    "# !wget https://github.com/SophonPlus/ChineseNlpCorpus/raw/master/datasets/ChnSentiCorp_htl_all/ChnSentiCorp_htl_all.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xk7iS5FpXngc"
   },
   "outputs": [],
   "source": [
    "df_htl = pd.read_csv('data/ChnSentiCorp_htl_all.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 343
    },
    "colab_type": "code",
    "id": "E7EE0A_hYS4E",
    "outputId": "33fdd402-59e3-4588-eca5-ffc0c0ad85df"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6124</th>\n",
       "      <td>0</td>\n",
       "      <td>酒店服务还可以，就是设施有点旧了，抽水马桶坏了（应该是自动上水装置出现问题），每次冲过水后，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6791</th>\n",
       "      <td>0</td>\n",
       "      <td>我把该酒店的一些设施用相机照了下来,作为附件不知怎么发给你。（我现在不记得是住317还是31...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>647</th>\n",
       "      <td>1</td>\n",
       "      <td>这次去是在香港挂黑色暴雨警报那天，本想取消的，但是之前给携程打电话想取消，携程和酒店沟通后说...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4871</th>\n",
       "      <td>1</td>\n",
       "      <td>从庐山下来住了一夜,第二天打出租车到长途车站乘车去黄山。这里就在火车站旁边，如果坐火车倒是近...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5249</th>\n",
       "      <td>1</td>\n",
       "      <td>5月底入住的。房间装修感觉还不错，就是房间小了点，大床居然还是两张床拼起来的，有点奇怪。地理...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7426</th>\n",
       "      <td>0</td>\n",
       "      <td>除了地段可以，其他是一塌糊涂，惨不忍睹。和招待所差不多。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7553</th>\n",
       "      <td>0</td>\n",
       "      <td>1.客房服务态度较好2.早餐还算不错.3.就是房间的灯光,太不满意~~</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5954</th>\n",
       "      <td>0</td>\n",
       "      <td>服务差，怎有一酒店水呢?我之前看到是一月的点评说没热方不到现去还是没有，投诉说要换另一家他们...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7330</th>\n",
       "      <td>0</td>\n",
       "      <td>竟然有如此差的酒店,楼下的居然说满意,还有哪有什么接机,送机免费,我真怀疑你是媒子.四星级酒...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6313</th>\n",
       "      <td>0</td>\n",
       "      <td>房间冷的受不了，让服务员那条被子足足半小时，电话打了好几个，那床比木板还硬洗澡的水没有热气，...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "6124      0  酒店服务还可以，就是设施有点旧了，抽水马桶坏了（应该是自动上水装置出现问题），每次冲过水后，...\n",
       "6791      0  我把该酒店的一些设施用相机照了下来,作为附件不知怎么发给你。（我现在不记得是住317还是31...\n",
       "647       1  这次去是在香港挂黑色暴雨警报那天，本想取消的，但是之前给携程打电话想取消，携程和酒店沟通后说...\n",
       "4871      1  从庐山下来住了一夜,第二天打出租车到长途车站乘车去黄山。这里就在火车站旁边，如果坐火车倒是近...\n",
       "5249      1  5月底入住的。房间装修感觉还不错，就是房间小了点，大床居然还是两张床拼起来的，有点奇怪。地理...\n",
       "7426      0                       除了地段可以，其他是一塌糊涂，惨不忍睹。和招待所差不多。\n",
       "7553      0                1.客房服务态度较好2.早餐还算不错.3.就是房间的灯光,太不满意~~\n",
       "5954      0  服务差，怎有一酒店水呢?我之前看到是一月的点评说没热方不到现去还是没有，投诉说要换另一家他们...\n",
       "7330      0  竟然有如此差的酒店,楼下的居然说满意,还有哪有什么接机,送机免费,我真怀疑你是媒子.四星级酒...\n",
       "6313      0  房间冷的受不了，让服务员那条被子足足半小时，电话打了好几个，那床比木板还硬洗澡的水没有热气，..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#标签涵义：1代表正向评论，0代表负向评论\n",
    "label_denotation = {1:'positive',0:'negative'}\n",
    "df_htl.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 正负向语料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "评论数目（总体）：7766\n",
      "评论数目（正向）：5322\n",
      "评论数目（负向）：2444\n"
     ]
    }
   ],
   "source": [
    "print('评论数目（总体）：%d' % df_htl.shape[0])\n",
    "print('评论数目（正向）：%d' % df_htl[df_htl.label==1].shape[0])\n",
    "print('评论数目（负向）：%d' % df_htl[df_htl.label==0].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_htl.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 构造平衡语料\n",
    "- 用随机抽样的方法，很容易构造出类似的平衡语料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_positive = df_htl[df_htl.label==1]\n",
    "pd_negative = df_htl[df_htl.label==0]\n",
    "\n",
    "def get_balance_corpus(corpus_size, corpus_pos, corpus_neg):\n",
    "    sample_size = corpus_size // 2\n",
    "    # 是否有放回抽样\n",
    "    pd_corpus_balance = pd.concat([corpus_pos.sample(sample_size, random_state=123, replace=corpus_pos.shape[0]<sample_size), \\\n",
    "                                   corpus_neg.sample(sample_size, random_state=123, replace=corpus_neg.shape[0]<sample_size)])\n",
    "    \n",
    "    print('评论数目（总体）：%d' % pd_corpus_balance.shape[0])\n",
    "    print('评论数目（正向）：%d' % pd_corpus_balance[pd_corpus_balance.label==1].shape[0])\n",
    "    print('评论数目（负向）：%d' % pd_corpus_balance[pd_corpus_balance.label==0].shape[0])    \n",
    "    \n",
    "    return pd_corpus_balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "评论数目（总体）：10000\n",
      "评论数目（正向）：5000\n",
      "评论数目（负向）：5000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2454</th>\n",
       "      <td>1</td>\n",
       "      <td>1、酒店还算可以，设施有点旧，四星级勉强2、地处陈家坪，没有大型商场超市，背后有个美食街。送...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3048</th>\n",
       "      <td>1</td>\n",
       "      <td>酒店的外部环境很好,绿树成荫,楼与楼之间有点远,不过这才是别墅的特色,去餐厅有一点点远(我们...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7597</th>\n",
       "      <td>0</td>\n",
       "      <td>位置不错,在市中心,而且离周边几家4,5星酒店都只有10分钟路程.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7593</th>\n",
       "      <td>0</td>\n",
       "      <td>客观的评价，酒店的位置不错，但是周围的环境不好，门的位置较偏，感觉酒店不旺，但是生意较好，都...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5628</th>\n",
       "      <td>0</td>\n",
       "      <td>本来预定的就是别墅标间,前台小姐很热情的给我们推荐了\"超超豪华的普通标间\"给我们入住,说是比...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6829</th>\n",
       "      <td>0</td>\n",
       "      <td>招待所改装,确实是不行。以后再也不会住这里了。建议携程取消该定点，否则可能连带损害携程的声誉。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>1</td>\n",
       "      <td>环境不错，8月18号入住，8月16号安装好了电梯，预约人员对此信息不了解，这几月出行最好的酒...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4738</th>\n",
       "      <td>1</td>\n",
       "      <td>我于10月28日入住酒店.白天游览黄龙后,晚上7点登记入住,结果房间门卡有问题,以至我楼上楼...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3906</th>\n",
       "      <td>1</td>\n",
       "      <td>地理位置非常优越,机场大巴\\地铁和公车几乎就在门口,但靠街的房间晚上可能会有点吵.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4931</th>\n",
       "      <td>1</td>\n",
       "      <td>端午节期间去厦门自助游，选择入住了这里，酒店挂牌4星，我个人感觉是一家质量相当好的酒店，房价...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "2454      1  1、酒店还算可以，设施有点旧，四星级勉强2、地处陈家坪，没有大型商场超市，背后有个美食街。送...\n",
       "3048      1  酒店的外部环境很好,绿树成荫,楼与楼之间有点远,不过这才是别墅的特色,去餐厅有一点点远(我们...\n",
       "7597      0                  位置不错,在市中心,而且离周边几家4,5星酒店都只有10分钟路程.\n",
       "7593      0  客观的评价，酒店的位置不错，但是周围的环境不好，门的位置较偏，感觉酒店不旺，但是生意较好，都...\n",
       "5628      0  本来预定的就是别墅标间,前台小姐很热情的给我们推荐了\"超超豪华的普通标间\"给我们入住,说是比...\n",
       "6829      0    招待所改装,确实是不行。以后再也不会住这里了。建议携程取消该定点，否则可能连带损害携程的声誉。\n",
       "137       1  环境不错，8月18号入住，8月16号安装好了电梯，预约人员对此信息不了解，这几月出行最好的酒...\n",
       "4738      1  我于10月28日入住酒店.白天游览黄龙后,晚上7点登记入住,结果房间门卡有问题,以至我楼上楼...\n",
       "3906      1          地理位置非常优越,机场大巴\\地铁和公车几乎就在门口,但靠街的房间晚上可能会有点吵.\n",
       "4931      1  端午节期间去厦门自助游，选择入住了这里，酒店挂牌4星，我个人感觉是一家质量相当好的酒店，房价..."
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ChnSentiCorp_htl_ba_10000 = get_balance_corpus(10000, pd_positive, pd_negative)\n",
    "ChnSentiCorp_htl_ba_10000.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LKBd5hlzYUkr"
   },
   "outputs": [],
   "source": [
    "#分割训练集train与验证集val\n",
    "from sklearn.model_selection import train_test_split\n",
    "# 按分类分层抽样\n",
    "train, val = train_test_split(ChnSentiCorp_htl_ba_10000, test_size=0.1, stratify=ChnSentiCorp_htl_ba_10000.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "N69Q-7GjZzxq",
    "outputId": "45f15c1c-9bd8-4f1f-8af1-a729e78f3f8b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9000, 2) (1000, 2)\n"
     ]
    }
   ],
   "source": [
    "#check their size\n",
    "print(train.shape,val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4500, 4500)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(500, 500)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[train.label==1].shape[0], train[train.label==0].shape[0]\n",
    "val[val.label==1].shape[0], val[val.label==0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5785</th>\n",
       "      <td>0</td>\n",
       "      <td>这个酒店在苏州的园区，位置相当远，房间也一般，好象原先这个楼不是设计为酒店的，因为在园区内这...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2797</th>\n",
       "      <td>1</td>\n",
       "      <td>房间很大，可以和家里的媲美。进门就有种家的感觉，有衣物间，还有厨房。房间不大，但是客厅很大，...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3187</th>\n",
       "      <td>1</td>\n",
       "      <td>周边环境优美,交通便利.早餐种类较少,前台服务态度一般,如多一点热情微笑则更佳.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1189</th>\n",
       "      <td>1</td>\n",
       "      <td>环境还可以，房间不大但也不是太小，够住了。写字桌前的落地灯离得太近了，每次移椅子都会碰到，另...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7366</th>\n",
       "      <td>0</td>\n",
       "      <td>办入住的时候，说没有收到携程的预定，但还是给了携程的价格。问了早餐的事情才给了餐卷，早餐很差...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                             review\n",
       "5785      0  这个酒店在苏州的园区，位置相当远，房间也一般，好象原先这个楼不是设计为酒店的，因为在园区内这...\n",
       "2797      1  房间很大，可以和家里的媲美。进门就有种家的感觉，有衣物间，还有厨房。房间不大，但是客厅很大，...\n",
       "3187      1           周边环境优美,交通便利.早餐种类较少,前台服务态度一般,如多一点热情微笑则更佳.\n",
       "1189      1  环境还可以，房间不大但也不是太小，够住了。写字桌前的落地灯离得太近了，每次移椅子都会碰到，另...\n",
       "7366      0  办入住的时候，说没有收到携程的预定，但还是给了携程的价格。问了早餐的事情才给了餐卷，早餐很差..."
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['label', 'review'], dtype='object')"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.reset_index(inplace=True, drop=True)\n",
    "val.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv(root/'train.csv', index=False)\n",
    "val.to_csv(root/'val.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>优点:1.房间装修真的不错,属于常见的经济型酒店.(最大的优点)2.早餐也很丰富(每天有换品...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>可以说酒店很差。强烈建议CTRIP在郴州多准备几家酒店供选择。房间有霉味，床很大，但床垫很不...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>看到门口的样子，就知道不是真正的凯莱了，门厅零落要的商务房，对马路，跟没有玻璃一样，晚上吵闹...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>服务态度比较好，愿意听取客户所提出意见，290元的房间设施还可以，房间空间稍微小了些，早餐比...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>地处市中心,价格有竞争力,服务可以.规模不太大,前段时间正在重新装修.总体影响不错,值得一住.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                             review\n",
       "0      1  优点:1.房间装修真的不错,属于常见的经济型酒店.(最大的优点)2.早餐也很丰富(每天有换品...\n",
       "1      0  可以说酒店很差。强烈建议CTRIP在郴州多准备几家酒店供选择。房间有霉味，床很大，但床垫很不...\n",
       "2      0  看到门口的样子，就知道不是真正的凯莱了，门厅零落要的商务房，对马路，跟没有玻璃一样，晚上吵闹...\n",
       "3      1  服务态度比较好，愿意听取客户所提出意见，290元的房间设施还可以，房间空间稍微小了些，早餐比...\n",
       "4      1    地处市中心,价格有竞争力,服务可以.规模不太大,前段时间正在重新装修.总体影响不错,值得一住."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GWq0P4LnaL9M"
   },
   "outputs": [],
   "source": [
    "#建立TextDataBunch\n",
    "databunch = TextClasDataBunch.from_df(\".\", train, val,\n",
    "                  tokenizer=fastai_tokenizer,\n",
    "                  vocab=fastai_bert_vocab,\n",
    "                  include_bos=False,\n",
    "                  include_eos=False,\n",
    "                  text_cols=\"review\",\n",
    "                  label_cols='label',\n",
    "                  bs=config.bs,\n",
    "                  collate_fn=partial(pad_collate, pad_first=False, pad_idx=0)\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4 id=\"pad_collate\" class=\"doc_header\"><code>pad_collate</code><a href=\"https://github.com/fastai/fastai/blob/master/fastai/text/data.py#L128\" class=\"source_link\" style=\"float:right\">[source]</a><a class=\"source_link\" data-toggle=\"collapse\" data-target=\"#pad_collate-pytest\" style=\"float:right; padding-right:10px\">[test]</a></h4><blockquote><p><code>pad_collate</code>(<strong><code>samples</code></strong>:<code>BatchSamples</code>, <strong><code>pad_idx</code></strong>:<code>int</code>=<strong><em><code>1</code></em></strong>, <strong><code>pad_first</code></strong>:<code>bool</code>=<strong><em><code>True</code></em></strong>, <strong><code>backwards</code></strong>:<code>bool</code>=<strong><em><code>False</code></em></strong>) → <code>Tuple</code>[<code>LongTensor</code>, <code>LongTensor</code>]</p>\n",
       "</blockquote>\n",
       "<div class=\"collapse\" id=\"pad_collate-pytest\"><div class=\"card card-body pytest_card\"><a type=\"button\" data-toggle=\"collapse\" data-target=\"#pad_collate-pytest\" class=\"close\" aria-label=\"Close\"><span aria-hidden=\"true\">&times;</span></a><p>No tests found for <code>pad_collate</code>. To contribute a test please refer to <a href=\"/dev/test.html\">this guide</a> and <a href=\"https://forums.fast.ai/t/improving-expanding-functional-tests/32929\">this discussion</a>.</p></div></div><p>Function that collect samples and adds padding. Flips token order if needed</p>\n",
       "<p><a href=\"https://docs.fast.ai/text.data.html#pad_collate\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(pad_collate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 279
    },
    "colab_type": "code",
    "id": "2EtfF2hrfFQw",
    "outputId": "c27831b3-a7c9-452b-dcfa-e469cd361f73"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>[CLS] 非 常 非 常 一 般 的 酒 店 ， 无 法 跟 如 家 或 者 莫 泰 比 ， 居 然 还 称 之 为 是 环 保 经 济 型 酒 店 ， 汗 啊 。 第 一 次 见 到 房 间 里 有 燃 气 热 水 器 ， 还 是 [UNK] 前 锋 [UNK] 的 ， 是 那 种 开 了 之 后</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>[CLS] 如 果 一 个 人 住 ， 挑 18 号 的 大 床 房 ， 两 面 有 窗 户 ， 光 线 好 。 房 间 的 环 境 不 过 不 失 ， 就 是 空 调 声 稍 微 大 些 ， 但 上 网 速 度 奇 快 ！ down 一 部 400 多 兆 的 电 影 用 了 约 4 分 钟 ！</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>[CLS] 呼 和 浩 特 的 holiday ##in ##n 绝 对 超 过 我 的 预 期 , 原 先 以 为 内 陆 城 市 条 件 会 差 些 呢 . 酒 店 感 觉 挺 新 的 , 住 的 是 豪 华 房 , 它 有 专 门 的 楼 层 , 好 象 是 六 楼 和 七 楼 . 八 楼 和 九 楼</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>[CLS] 优 点 ： 位 置 极 佳 ， 服 务 很 好 ， 尤 其 专 设 了 一 名 电 梯 操 作 员 ， 很 好 ！ 早 餐 品 种 、 口 味 都 不 错 。 部 分 房 间 有 电 脑 ， 非 常 方 便 ！ 缺 点 ： 1 、 因 临 近 马 路 ， 房 间 有 些 嘈</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>[CLS] 刚 住 过 这 里 ， 感 觉 不 好 ， 房 间 味 道 太 大 ， 一 股 子 霉 味 ， 房 间 各 种 设 施 都 很 陈 旧 ， 我 所 预 订 的 大 床 间 ， 却 没 有 宽 带 ， 只 能 一 个 住 在 双 人 标 间 。 因 为 去 的 时 候 较 晚</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "databunch.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EWFDIeoJbgfb"
   },
   "outputs": [],
   "source": [
    "x, y = databunch.train_ds[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] 房 间 设 施 陈 旧 ， 屋 子 里 全 是 霉 味 。 网 络 收 费 为 每 分 钟 1 元 ， 每 天 100 元 ， 简 直 是 抢 。 顶 层 餐 厅 非 常 棒 ， 黄 浦 江 美 景 尽 收 眼 底 ， 漂 亮 。 。 [SEP]'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Category 1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(x)\n",
    "y\n",
    "y.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "129"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text [CLS] 酒 店 所 处 环 境 不 错 , 尤 其 是 下 午 五 点 半 所 有 喷 泉 一 起 启 动 , 真 是 很 壮 观 ! 晚 上 出 来 散 步 , 感 受 的 是 静 谧 和 清 新 , 一 排 排 整 齐 的 小 灯 , 仿 佛 带 你 走 进 梦 幻 的 世 界 . 如 果 情 侣 度 假 , 选 择 这 个 酒 店 是 很 好 的 . 遗 憾 的 是 没 看 到 桃 花 , 说 是 花 期 过 了 . 早 餐 太 简 单 了 , 和 酒 店 的 标 准 不 符 . 退 房 的 时 候 [SEP]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Category 1"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1, y1 = databunch.train_ds[2]\n",
    "x1\n",
    "y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "263"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(str(x1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "databunch.save('zh_databunch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv zh_databunch {root.as_posix()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CZ4lw8TadPMS"
   },
   "source": [
    "## Model and Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "6CLYlVWYdQlH",
    "outputId": "b58a66da-0dd6-4d17-e51d-ad16bd0cf1c1"
   },
   "outputs": [],
   "source": [
    "from pytorch_pretrained_bert.modeling import BertConfig, BertForSequenceClassification\n",
    "bert_model = BertForSequenceClassification.from_pretrained(config.bert_model_name, num_labels=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xyaJ7ooIdhvq"
   },
   "outputs": [],
   "source": [
    "#二分类问题选用CrossEntrypyLoss作为损失函数\n",
    "loss_func = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oN47kdZ-d69i"
   },
   "outputs": [],
   "source": [
    "#建立Learner(数据,预训练模型,损失函数)\n",
    "learner = Learner(databunch, bert_model, loss_func=loss_func, metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "VhCBr5YyeYQz",
    "outputId": "cbf4de65-dc42-479e-f258-3d19af52a24e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "#尝试寻找合适的最大学习率，这里使用了BERT原论文推荐的学习率3e-5作为默认值\n",
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "qj9hzFd0efEx",
    "outputId": "dee9db99-5783-400a-c20f-a33f098af923"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min numerical gradient: 5.75E-04\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VOXZ+PHvPVnJSgJhTSAJJOyIEjYVxKUKbtjWBbRq1UpxqUtb35+2b/Wtb7e3trXaai1Fq611XypaiksVBWULyo7ITsKakBCyr/fvj5mEEBIyk5mTyXJ/risXzJnnnPPkEObOs92PqCrGGGNMa1zBroAxxpjOwQKGMcYYr1jAMMYY4xULGMYYY7xiAcMYY4xXLGAYY4zxigUMY4wxXrGAYYwxxisWMIwxxnglNNgVCKTevXtrampqsKthjDGdxpo1a/JVNcmbsl0qYKSmppKdnR3sahhjTKchInu8LWtdUsYYY7xiAcMYY4xXLGAYY4zxigUMY4wxXnE0YIjIDBHZKiLbReT+Zt6/T0TWer42ikitiCSKSIqIfCQiW0Rkk4jc7WQ9jTHGtM6xgCEiIcATwExgJDBHREY2LqOqj6jqOFUdBzwAfKyqBUAN8ANVHQFMBu5oeq4xxpj25WQLYyKwXVV3qmoV8BIw6xTl5wAvAqjqAVX93PP3YmALMNDBuhpjjGmFkwFjIJDT6HUuLXzoi0gUMAN4vZn3UoHTgZUBr6Ef/r3hAOtzjwa7GsYY026cXLgnzRxraQPxy4BPPd1Rxy8gEoM7iNyjqseavYnIXGAuwKBBg9peWx/kl1Ry2z8+B+Cy0wZw34XDGNQrql3ubYwxweJkCyMXSGn0OhnY30LZ2Xi6o+qJSBjuYPEPVX2jpZuo6nxVzVLVrKQkr1a3+y17dyEAV4wbwAebD3H+75bw07c3UVBa1S73N8aYYHAyYKwGMkQkTUTCcQeFhU0LiUg8cA7wVqNjAjwNbFHV3zlYxzZZs6eA8FAX/3flWJbcN50rxyfz3Ge7OefXH/Hkku1UVNcGu4rGGBNwjgUMVa0B7gTexT1o/YqqbhKReSIyr1HRrwPvqWppo2NnAdcD5zWadnuxU3X1VfaeQsYOjCciNIS+cZH88htjefeeaUxKT+TXi7dy8WNLKamsCXY1jTEmoES1pWGFzicrK0udTj5YUV3LmP95l5vPTuOBmSNOev/DLw9xy3PZ3Dgllf+5fJSjdTHGGH+JyBpVzfKmrK309tH63CKqa5WswYnNvn/e8L5cP3kwzy3fzRd7C9u3csYY4yALGD7K3uOeyDV+cEKLZe67aBh9YyN54I0NVNfWtVfVjDHGURYwfLRmdyHpSdEkRoe3WCY2MoyHZ43iy4PFzP9kZzvWzhhjnGMBwwd1dcqavYVknaJ1Ue/CUf2YObofj/1nG7vyS1stb4wxHZ0FDB/szC/haFl1i+MXTf3P5aOICHXx4zc30JUmFxhjuicLGD6oX7A3PrX1FgZA37hI7p85nM92HOHVNblOVs0YYxzX7QNGeVUtj77/FUu2Hm61bPaeQhKjw0nvHe319edMGMSE1AR+/q8t5JdU+lNVY4wJqm4fMMJChFeyc3h62a5Wy67ZU8gZgxJwL0T3jssl/PIbYyirquHhtzf7U1VjjAmqbh8wQkNcXDMhhaXb8tl7pKzFcvkllezKLyXLy+6oxob2ieX26UNZuG4/72466E91jTEmaLp9wAC4ZkIKLoGXVu9tscyaPe7xC29mSDXn9nOHMHpgHPe+vJZN+4vadA1jjAkmCxhA//genDe8D69k57a40G7NnkLCQ1yMHhjfpntEhIbw9I0TiO8Rxs3PruZAUbk/VTbGmHZnAcPj2kmDyC+p5IPNh5p9P3t3AWOS44kMC2nzPfrGRfLMtydQWlnLLc9mW4JCY0ynYgHD45zMPgyIj+SFVSd3S1VU17Jx37E2d0c1NqJ/HE9cdwZbDxXzvRc+p8ZShxhjOgkLGB4hLuGaCYOaHfzesK+Iqtq6U+aP8sU5mUk8PGsUH23N4+F3NtuiPmNMp2ABo5H6we8Xmwx+NyzYC1DAALhu0mDmTkvnb8v38MynuwN2XWOMcYoFjEb6xUdy3vC+vJqdQ1XN8a6iNXsKSO8dTa+YiIDe7/4Zw5kxqh8/+9dm3rPptsaYDs4CRhPXTRpEfkkVH2xxD36rKmv2FAa0dVHP5RIevWYcYwfG84NX11FcUR3wexhjTKBYwGhiWmYSA3v24IWV7m6pHXmlFJZVt2nBnjd6hIfwv1eMpriihpdX5zhyD2OMCQQLGE24B79TWLY9nz1HSlnTsGGSdxlq22Jsck8mpyfyzLJdtuGSMabDsoDRjKuzUghxCS+uyiF7dyEJUWEMSfI+4WBbzJ2Wzv6iChZtOODofYwxpq0cDRgiMkNEtorIdhG5v5n37xORtZ6vjSJSKyKJnveeEZHDIrLRyTo2xz343YfX1uSwclcB4wf7lnCwLaZn9mFonxjmf7LTptkaYzokxwKGiIQATwAzgZHAHBEZ2biMqj6iquNUdRzwAPCxqhZ43n4WmOFU/VpzrWfwe29BGWc4MODdlMsl3Do1jU37j7F8xxHH72eMMb5ysoUxEdiuqjtVtQp4CZh1ivJzgBfrX6jqJ0BBy8WdNS3DPfgNeL3Dnr9mjRtI75gI5i+1fcCNMR2PkwFjINB42k+u59hJRCQKd2vidQfr45MQl3DTWakkRoczNrltCQd9FRkWwrfPHMySrXlsPVjcLvc0xhhvORkwmuv0b6lz/jLg00bdUd7fRGSuiGSLSHZeXp6vp5/SLWensfyB8/xKOOir6yYNpkdYCH+xVoYxpoNxMmDkAimNXicD+1soO5tG3VG+UNX5qpqlqllJSUltuUSLRISI0PYLFgAJ0eFcnZXMW2v3cehYRbve2xhjTsXJgLEayBCRNBEJxx0UFjYtJCLxwDnAWw7WpVO5+ew0auuUZz/bHeyqGGNMA8cChqrWAHcC7wJbgFdUdZOIzBOReY2Kfh14T1VLG58vIi8Cy4FhIpIrIrc4VdeOZnCvaGaM7sc/VuyxPTOMMaeUvbuAJ5dsb5d7hTp5cVVdBCxqcuypJq+fxT2Ftum5c5ysW0d369R0Fm04yCurc7j57LRgV8cY0wF9tiOf7zyXTb+4SG6YkkpMhKMf6bbSu6M6fVACE1ITeHrZLttkyZh2cv3TK3nwrY2d4v/cx1/lcdNfV5Oc0IOXvjvZ8WABFjA6tFunprPvaDnvrLd0IcY4raK6lqXb8vnb8j3Me/5zKqprg12lFn2w+RC3PpfNkKQYXpo7hT6xke1yXwsYHdgFI/oyon8cv3v/qxP25zDGBF5uYTngzlj9ny8Pcd2ClRwtqwpyrU62aMMB5j2/hhH9Y3nx1skkRoe3270tYHRgLpfwXzOGsbegjBeb2WvcGBM4OYXurZnvOm8oT1x7Bhtyi7jqqeXsP1re4jmVNbWs3HkkIIGltk7Zf7ScHXklVNY037r55xf7uPOFzxmX0pPnvzOJ+Kgwv+/rC+c7vYxfpmcmMTk9kT98uI1vjk9ul35KY7qj3AJ3wEhJjCIrNZGEqHDm/i2bb/7pM567eSKZfWMB9wf78h1HWLhuH4s3HuRYRQ0piT34282TSOvdelZrVWXJ1jw27isit7CcnMIycgvL2X+0nJo699pml8CAnj1I6x1Naq9oUntHU1Fdy2/e28rktF4suDGL6CB8FkhXyoyalZWl2dnZwa5GwK3NOcoVT3zKPRdkcM8FmcGujjFd0i8WbeHZz3bz5cMzcLnciSo27z/GjX9dRWV1LQ9dNor1uUf514aD5JdUEhMRyoWj+jIpLZFf/ftLXCI8d/NERg9sOZVQaWUNP35zA/9c617DnBQbQXJCD5ITojx/9iAyNIQ9BWXszi9l95FSduWXUlzhnl5/TmYSf75+fECzT4jIGlXN8qas/braCYxL6cnFY/rxl092ct2kwSTFBnZvcWMM5BSUkZzQoyFYAIwcEMcbt53Jjc+s4gevriMi1MX5I/pw2dgBnDu8T8MHd1ZqIjc8vYrZ81cw//rxnDm090nX33aomNv+8Tk780r4wdcyuXVaulcf/KpKYVk1B4sqyOwbQ2hI8EYSrIXRSezMK+Frj37CtyYN4qezRge7OsZ0OZc8vpTeMRE8d/PEk94rKqvmi5xCxg9OIDay+XGDg0UV3PjMKnbll/L72eO4eEz/hvfe+DyXH7+5keiIEB6ffXqzASVYfGlh2KB3J5GeFMPsCSm8sGove46Utn6CMcYnOQVlDEqMava9+Kgwpg/r02KwAPfGa698dwpjk+O544XP+fuKPVRU1/LAG+v5/ivrGJMcz6K7pnaoYOErCxidyN3nZxDqcvHb974KdlWM6VKKyqsbBq/9ER8Vxt9vmcR5w/rwk39u5LzfLOHFVTncPn0IL3xnEn3i2me9hFMsYHQifeIiueXsNBau28/GfUXBro4xXUZO/QyphOZbGL7oER7Cn68fz+wJKVTV1vHXb0/gv2YMD+rYQ6B0/u+gm5l7TjoJUWH83+Ivg10VY7qMnEZTagMhNMTFr745llU/uoBzh/cJyDU7AgsYnUxcZBh3npfB0m35LNuWH+zqGNMl1C/aC1TAqNd4xlVXYNNqO6FvTR7EM8t2cfOzq4kMOznmR4WH8rurT+vUg2vGtKecgnLiIkOJ79G+K6c7GwsYnVBEaAjzbxjPa2tyaW5W9IdfHua+19bz/venERVu/8TGtCansCzgrYuuyD5NOqlRA+IZNaD5FaWXjO3PVU8t59H3v+LHl4x05P51dcpHWw+z50gZh4orOHyskkPHKjh0zP33pLgILj9tAJefNoD0pBhH6mBMoOQUlJHRJzbY1ejwLGB0QRNSE5kzcRDPfLqbWeMGnjJVQVv98t9b+MvSXQCEh7roGxdB39hIhvWLZWpGElsPFvPYf7bx+w+2MWZgPLPGDeDSsQPoF9+5pxWarqeuTskpLOe8LjQ47RQLGF3U/TOG8/7mQ/zozQ28eftZhARw8O3l1Xv5y9JdfGvyIH544TDie4QhcvL1DxZV8M76/Sxct5+f/WsLP1+0hbOG9OZ315zWbvn7jWlNXkklVTV1LS7aM8fZLKkuKj4qjIcuG8n63CKe+2z3KcvW1imfbs9vMaVyY8t3HOHHb25kakZv/ueyUfSMCm82WIB75et3pqaz8M6z+fAH53D3+Rlk7yng7hfXUlvXdVLSmM6tfkptsgWMVlnA6MIuHduf6cOS+M17W9nXQk7//UfLmfOXFVy3YCXfePIzduaVtHi93fml3PaPNQzuFcUfrz3Dp4VI6Ukx3HNBJj+7YgzLdx7h0fdttbrpGBqm1AZg0V5XZwGjCxMR/nfWaFThobc20jTR5OKNB5n52FI27iviznOHsv9oOZf+YRmvr8k96VpF5dXc8txqAJ6+cUKbpx9eOT6Za7JS+ONH2/noy8NtuoYxgZRT4P5lKjnBv7Qg3YGjAUNEZojIVhHZLiL3N/P+fSKy1vO1UURqRSTRm3ONd1ISo/j+1zL5YMthFm88CLj3Lv7vf25g3vNrGJQYxb/umsoPLxrGv++expiB8fzg1XXc+/JaSirdOfhrauu484XP2VtQxlPfGk+qF5vEnMpPZ41iRP847n1lLbme3+6MCZa9BWX0jYsI6B4TXZVjAUNEQoAngJnASGCOiJwwx1NVH1HVcao6DngA+FhVC7w513jvprNSGTUgjocWbiJ7dwGX/3EZz6/Yy9xp6bx+25kNu4T1i4/khVsnc+8Fmby1dh+XPr6UDblFPPzOZpZuy+fnV4xhcnovv+sTGRbCn647g9pa5Y4XvrD9yk1Q5RSUWXeUl5xsYUwEtqvqTlWtAl4CZp2i/BzgxTaea04hNMTFL78xhvySSq58ajkFpdU8d/NEfnTxCMJDT/wRCHEJd1+QwUtzp1BZU8cVT37K35bvYe60dK6ekBKwOqX2juaRq8ayLucov1i0JWDXNcZXuYXltmjPS05Oqx0I5DR6nQtMaq6giEQBM4A7fT3XeGdsck/uu2g4m/YX8dBlo1rdtW9iWiL/vnsqDy3cRKjLxf+bMTzgdZoxuj/fOTuNBct2MX5wApedNiDg9/DW0m15/PTtzWT0iWFMcjxjB/ZkzMB44qMsVURXVlVTx4GiclJs/MIrTgaM5uZatjSX8jLgU1Ut8PVcEZkLzAUYNGiQr3XsVm6bPsSn8j2jwnls9ukO1cbt/80czhc5R7n/9fWMHBDHkCCtCn9tTS77j5ZTXVvHvz1jPQCpvaIYk9yTu8/PYGgfW7He1ew/Wk6d2pRabznZJZULNO7DSAb2t1B2Nse7o3w6V1Xnq2qWqmYlJSX5UV0TDGEhLv547elEhIVw7V9W8MXewnavQ12dsmxbPheN6sfH953L2ge/xvO3TOK+i4YxvF8cH315mB+/uaHd62WcVz+l1hbtecfJgLEayBCRNBEJxx0UFjYtJCLxwDnAW76ea7qG/vE9eOHWSYSHurjmzyt4JTun9ZMCaPOBYxwprWJqhju7b8+ocM7O6M0d5w7lqevHc88FGazcVcCGXNu0qqupn1JrYxjecSxgqGoN7jGJd4EtwCuquklE5onIvEZFvw68p6qlrZ3rVF1N8A3vF8fCO85mYloi//Xaeh56ayPVte0ze2qpZ1+Rs1tIB3/1hBRiIkJ5etnOdqmPaT85hWWEhQj9OvnWqe3F0XUYqrpIVTNVdYiq/txz7ClVfapRmWdVdbY355quLSE6nGdvmsCtU9N4bvkerluwkvySSsfvu2x7HsP7xba433JcZBjXTEjhnfUHOFDU/Ip50znlFJQxoGePgOZa68pspbfpUEJDXPz4kpH8/ppxrMs5yuV/WOZoV1B5VS2rdxU2dEe15NtnplKnynOf7XGsLqb92RoM31jAMB3SFacP5PXbzgTg6j8v59CxCkfus3LXEapq65iaceoJEymJUcwc3Z8XVu6h1LMC3nR+ObYGwycWMEyHNXpgPPNvyKK8utax/cuXbssnPNTFxLTEVsveMjWNYxU1vNrOg/LGGaWVNRSUVpGSaGswvGUBw3RoI/vHEd8jjJW7jjhy/WXb8pmYmuhVHqEzBiVwxqCePPPpbkvP3gVYllrfWcAwHZrLJUxITWTVroLWC/vo0LEKth4qbnX8orHvTE1nb0EZ728+FPD6mPZlU2p9ZwHDdHiT0xPZfaQs4OMY9dNpWxu/aOzCkX1JTuhhU2y7gL0FtmjPVxYwTIdXP76wYmdgu6WWbcujd0w4w/vFen1OaIiLm85KY/XuQtbmHA1ofUz7yikoIzo8hATLF+Y1CximwxvZP46YiNCAdkvV1SnLtudz9tDeuHycg3/NhBRiI0J5etmugNXHtL/cwjJSEqNa3GLYnMwChunwQkNcZKUmsDKAAWPLwWPkl1T51B1VLyYilDmTBrFow4EWt741HV9OQTnJNuDtEwsYplOYmJbI9sMlAVv5fXz8wvsB78ZuPDMVgGc/tVZGZ6Sq7C0osym1PrKAYTqFSWnunf5WB6iVsWxbPsP6tpwOpDUDe/bg4jH9eWlVDkfLqgJSp65EVdlzpLT1gkFypLSK8upaG/D2kQUM0ymMGRhPZJgrIN1S5VW1rNpd0ObWRb3bpw+hpKqGP39iM6aaWp9bxDmPLOGttfuCXZVm5RTYGoy2sIBhOoXwUBfjBwdmHGPV7gKqauqYmunf/ikj+sdx2dgB/PXTXRwudiZ1SWd10DMF+heLtnTIVCo5hbYGoy0sYJhOY2JqL748eIyismq/rrNsWx7hIS4mpraeDqQ13/9aJtW1yhMfbvf7Wl1JfZA4dKySpz7eEeTanKy+hZFsW7P6xAKG6TQmpSeiCqt3+9fKWLotnwlpCfQIbz0dSGtSe0dzdVYKL6za2/AhZKDEEzDOHtqbP3+ys8M9m5yCMnrHhBMd4eQu1V2PBQzTaYxL6Ul4iMuvvFKHj1Xw5cHiNk2nbcld5w9FRHjsP9sCds3Orj5g/OyK0YSI8Mt/bwlyjU6UU1hmU2rbwAKG6TQiw0IYl9LTr3EMf6fTNqd/fA9umDyYNz7PZfvh4oBdtzMrqagh1CUM7hXFbdOHsGjDQZbvcCaBZFvkFFha87awgGE6lUnpiWzcV9TwG6yvlm3Pp1d0OCP6xQW0XrdNH0KPsBB++95XAb1uZ1VaWUNMZCgiwtxp6Qzs2YOfvr2pQ2T5ra1T9h8tJ8XGL3xmAcN0KhPTEqlTyPZhHONAUTnPfbaba/+ygrfW7mNaZpLP6UBa0ysmgu9MTeffGw+yPtdyTBVX1hAd7h4fiAwL4YGLh/PlwWJeWr03yDVz/zzU1Km1MNrAAobpVMYPTiDUJa3mldqRV8KTS7Yz64/LmPLLD3lo4SYOF1dy2/Qh/OTSkY7U7TtT0+gZFcZvrJVBSUUNsZHHB5QvGdOfiamJ/Pa9rygq92+Wm78sS23bORowRGSGiGwVke0icn8LZaaLyFoR2SQiHzc6freIbPQcv8fJeprOIyo8lDHJ8accx/jTkh2c/9uP+fXirQDcd9EwPvj+OXzw/XO476LhJEaHO1K32Mgwbp8+hE++ygt4Zt3OprSq5oQZSCLCg5eNpLCsiseDPDkg17MGw6bU+s6xgCEiIcATwExgJDBHREY2KdMTeBK4XFVHAVd5jo8GbgUmAqcBl4pIhlN1NZ3LxLRE1ucepbyq9qT3Fq7bz/8t/pJLxvTns/vP4607z+aOc4cytE9Mu9Tthimp9I2L4DfvbkU1+P31wVJSUUNMkymrowfGM3tCCs99tpvth0uCVDM4UuJO5dIntm1pYbozJ1sYE4HtqrpTVauAl4BZTcpcC7yhqnsBVPWw5/gIYIWqlqlqDfAx8HUH62o6kclpvaiuVb7YW3jC8ezdBfzw1XVMSE3gd9ecxoCe7f8bZGRYCN87L4PsPYV8+OXh1k/ookoqTw4YAD+4cBg9wkL48ZsbgjYAXlhWRUSoKyDrcLobJwPGQCCn0etcz7HGMoEEEVkiImtE5AbP8Y3ANBHpJSJRwMVAioN1NZ3I+NQEXAIrGnVL7c4v5da/ZTOwZw/mX59FRGjwPgyumZBCWu9ofrFoC9W1dUGrRzC1FDB6x0Tw4GUjWbmrgEffD85YT2FplWPdkl2dkwGjuWkoTX+lCAXGA5cAFwE/EZFMVd0C/B/wPrAYWAc0O49SROaKSLaIZOfl5QWs8qbjiosMY+SAOFZ6xgkKS6u46dnVAPz12xNICPKHQViIi/++ZAQ78kr5+/I9Qa1LsJRW1ra4ivqqrBSuzkrmjx9t58Mv239v9MKyKhKiLGC0hZMBI5cTWwXJwP5myixW1VJVzQc+wT1mgao+rapnqOo0oABodqRMVeerapaqZiUlBW71runYJqX14oucoxRXVPPd59ewr7Cc+Tdkkdo7OthVA+C84X2YlpnEox98xZEA7eHRWdTVqbuFEdly2o2HZ41mRP847n15XbunDSkorSIh2rZlbQuvAoaIDBGRCM/fp4vIXZ4B61NZDWSISJqIhAOzgYVNyrwFTBWRUE/X0yRgi+c+fTx/DgK+Abzo7Tdlur6JaYlU1dRx3YKVrNpVwCNXjWVCAJIJBoqI8OClIyirquV3Qep6CZayavdkhNhT5GmKDAvhT9edQV2dcscLn1NZc/IEBqccLau2FkYbedvCeB2oFZGhwNNAGvDCqU7wDFbfCbyLOwi8oqqbRGSeiMzzlNmCu8tpPbAKWKCqG+vvKSKbgbeBO1S18KSbmG6rPtPs+twifnhhJrPGNR0eC76hfWK5fvJgXly1l837jwW7Ou2mpMLde9xaYr/U3tH85urTWJ9bxM/eab9cUwVlNobRVt6maqxT1RoR+Trwe1X9g4h80dpJqroIWNTk2FNNXj8CPNLMuVO9rJvphhKiw7lgRF8G9IzkjnOHBrs6Lbr3gkzeWruPh9/ZxIu3TkYksCvMO6L6tC2n6pKqd9Gofsydls78T3aSlZrgeOCvrVOKyqvpaS2MNvG2hVEtInOAG4F3PMesE9AE1YIbs3h41ugO/SEcHxXG9y8cxoqdBSzeeDDY1WkXDQEjwruZavddNIyJqYnc//oGth1yNnljUXk1qpAYZR9fbeFtwLgJmAL8XFV3iUga8Lxz1TKm65gzIYXh/WL5+aItVFQ331dfXFHNq9k5fPJVHmVVHW+HOl+UNgQM7z6Uw0Jc/OHa04mOCGXe82sc3aGvoNS9aC/YM+k6K6+6pFR1M3AXgIgkALGq+isnK2ZMVxEa4uLBS0dy7YKVLFi6kzvPO5604NCxCp75dBcvrNhLseeDMixEOD0lgTOH9uLMIb3d+4CEdp60b8UNYxjer4XpGxfJ43PGcd2ClfzPwk08ctVpjtStsMwTMKxLqk28ChgisgS43FN+LZAnIh+r6vcdrJsxXcaZQ3szY1Q/nvhoB1eOT6G4opr5n+zkn2v3UVunzBzdn5vPTqWkspbPduSzfMcRHvvPNn7/wTZ6hIXwzfED+d8O3v1Wr75LKtbLFka9M4f05nvnDuXxD7dzdkZvR8YzCj0tDBv0bhtvB73jVfWYiHwH+KuqPiQi652smDFdzY8uHsGHWw/z9Sc/5UBRBZFhLuZMHMQtZ6cxuNfx9SPnZLrXExWVVbNi1xEWrt3P8yv2ctnYAUxK7xWs6nutvkvJlxZGvbvOz+DTHUf47zc3cnpKAoN6BTajbEMLwwJGm3jbzg0Vkf7A1Rwf9DbG+GBQryjuPj+Dmjrlngsy+Oz+83l41ugTgkVj8VFhXDSqH7+56jQSo8N56uMd7VzjtvFlllRToSEuHps9DgTueumLgKdWKSh1p1ZPsEHvNvE2YDyMez3FDlVdLSLptLDy2hjTsjvOHcrqH1/APRdket0t0iM8hJvOTOWjrXlsOdDx13OUVNYQFiJtzueVnBDFr74xlrU5RwOeb+pofeLBMEs82BZeBQxVfVVVx6rqbZ7XO1X1m85WzRhT74YpqUSHh/DnTtDKaC61ua8uGduf2RNS+NPHO/h0e36AauaeJZUYHd4pxoI6Im9TgySLyJsiclhEDonI6yJVz7w4AAAa4UlEQVSS7HTljDFu8VFhzJk4iLfXH2j33Eu+Kq2saXWVtzcevGwk6b2juffltQHLx1VYVmWL9vzgbZfUX3HngRqAO0X5255jxph2csvUNFwCf1m6M9hVOaXiFlKb+yoqPJQ/zDmDo2XV3Pfa+oBsSFVYVk2iJR5sM28DRpKq/lVVazxfzwKWGtaYdtQ/vgdXjBvIy6tzyO/AGXBLK0/cz9sfIwfE8aOLh/Phl4d5foX/qeILSy21uT+8DRj5IvItEQnxfH0L6N6bFhsTBN89J52q2jqe+2x3sKvSopIAdUnVu/HMVEYNiOPtdQf8vlaB7YXhF28Dxs24p9QeBA4AV+JOF2KMaUdD+8Ry4ci+/G35nobpqx1NS7vttZWIMDY5nq8OF/vVLVWfeNDWYLSdt7Ok9qrq5aqapKp9VPUK3HtUGGPa2bxzhlBUXs1Lq/YGuyrNCsQsqaYy+8ZytKyaPD+64izxoP/8SVBjaUGMCYLTByUwOT2RBUt3UVXT8fYMLw1wCwPcAQPgq4Mlbb6GJR70nz8BwyYyGxMk884ZwsFjFfxz7b5gV+UEtXVKaVXL+3m3VUbfGAC+8iP9uSUe9J8/AcP/OW7GmDY5JzOJEf3jeOrjHdTVdZz/iqWe1OyBmiVVLykmgoSoMP8ChiUe9NspA4aIFIvIsWa+inGvyTDGBIGIcNv0IezMK+WjrYeDXZ0GxxMPBjZgiAgZfWMD08KwgNFmpwwYqhqrqnHNfMWqamB/IowxPpk5uh/R4SEs2ZoX7Ko0qN/PO9BjGADD+say7VBJm2dKWeJB/3WeXVmMMScIC3GRlZrI8p0dZ0nU8e1ZAx8wMvvGUFxZw4Giijadb4kH/WcBw5hObMqQXmw/XMLh4rZ9iAaaP6nNW9MwU6qN3VKWeNB/jgYMEZkhIltFZLuI3N9CmekislZENonIx42O3+s5tlFEXhSRSCfrakxnNMWzodKKnQVBrolbwxhGeMcLGJZ40H+OBQwRCQGeAGYCI4E5IjKySZmewJPA5ao6CrjKc3wg7j3Es1R1NBACzHaqrsZ0VqMGxBEbEcryHR2jW6p+P+9Az5IC92B1UmwEXx1q21oMSzzoPydbGBOB7Z69M6qAl4BZTcpcC7yhqnsBVLXxdI9QoIeIhAJRwH4H62pMpxQa4mJCWiIrO8g4RqmDYxjgHsdocwvDEg/6zcmAMRDIafQ613OssUwgQUSWiMgaEbkBQFX3Ab8B9uLOXVWkqu85WFdjOq0p6b3YmV/KoWPBH8cocWhabb1Mz0yptqw9scSD/nMyYDQ3stT0XzkUGA9cAlwE/EREMkUkAXdrJA33eo9oT4bck28iMldEskUkOy+v40wvNKa9TBniHsfoCN1SJZW1hIe6CA915qMls28s5dW15BaW+3SeJR4MDCcDRi6Q0uh1Mid3K+UCi1W1VFXzgU+A04ALgF2qmqeq1cAbwJnN3URV56tqlqpmJSXZFh2m+xnRP464yI4xjlFSWe1YdxS4u6TA94FvSzwYGE4GjNVAhoikiUg47kHrhU3KvAVMFZFQEYkCJgFbcHdFTRaRKHHPgTvfc9wY00SIS5iU3qtDrMcorax1NGBk1M+UOuxbwLDEg4HhWMBQ1RrgTuBd3B/2r6jqJhGZJyLzPGW2AIuB9cAqYIGqblTVlcBrwOfABk895ztVV2M6uynpvdhbUMa+o7511QRacUVgN09qKi4yjP7xkXx10LeAcdQSDwaEo+k9VHURsKjJsaeavH4EeKSZcx8CHnKyfsZ0FfXjGCt2HOGb45ODVo+SympiHQwY4B7H8HVqbYElHgwIW+ltTBcwrG8sCVFhQe+WKq2sJTrC2dQbmX1j2J5XQq0PM6XqEw/2tDEMv1jAMKYLcLmESWm9gj7wXVJZQ0yksx/KmX1jqaqpY8+RUq/PKSxzJx60FoZ/LGAY00VMGdKLfUfLySkoC1od3Pt5O93C8D1FSGGpJR4MBAsYxnQRHWE9hhP7eTd1fPc978cxCjyrvC3xoH8sYBjTRWT0iaF3THjQxjFq65Ty6sBvz9pUVHgoKYk9fGthlNmivUCwgGFMFyHiWY+x40ibNxnyh5N7YTQ1zMfd9wrLqizxYABYwDCmC5mS3ouDxyrYc6T9xzHqEw86kam2qYy+sezMK6Wqps6r8oWllto8ECxgGNOFNIxjBKFbyunEg40N6xtLTZ2y28uZUgVlVSRawPCbBQxjupD03tEkxUYEZeC7PbukMnzIKWWJBwPHAoYxXYiIMMWTV6q9xzFKKtovYAxJisEl3s2Uqk88mGCL9vxmAcOYLmbKkF7kFVeyI8/7hW2BUOrgft5NRYaFkNor2qucUpYWJHAsYBjTxdTv893e4xjFDu7n3ZyMvjFeZa21xIOBYwHDmC5mcK8o+sdHsqKdxzFKHNzPuznD+sayO7+UiuraU5ZrSG1uAcNvFjCM6WJEhOnDknh300HeXtd0zzLnlLbjLClwT62tU9jZStdbfeLBBFuH4TcLGMZ0QT+6eARnDE7grpe+4MVVe9vlniWVNUSEuggLaYePlR07mPr7B9nw6FWMSO4JcXFw++2wY8dJRS3xYOBYwDCmC4qNDOO5myYyLSOJB97YwIKlOx2/pzvxYDu0Lv79bxg7lvh//I3YqnJEFYqLYcECGDvW/X4jhaVVhFviwYCwgGFMF9UjPIS/3JDFJWP687N/beF373/l6FRbd2pzhwPGjh1w5ZVQVoZUV5/4XnU1lJW532/U0igodS/as8SD/rOAYUwXFh7q4vE5p3N1VjKP/2cbD7+zmTofNh7yRWlljfMzpH77W3dgOJXqanj00YaXlngwcCxgGNPFhbiEX31jLDedlcpfP93N/3t9vU+71XmruKIdWhjPP+9dwPj73xteWuLBwLGAYUw34HIJD146krvOG8qra3J5NTsn4PcorapxfD9vSrzcA6NROUs8GDiOBgwRmSEiW0Vku4jc30KZ6SKyVkQ2icjHnmPDPMfqv46JyD1O1tWYrk5EuPdrmWT0ieHF1YEPGCUVNc5PqY2J8blcoSUeDBjHAoaIhABPADOBkcAcERnZpExP4EngclUdBVwFoKpbVXWcqo4DxgNlwJtO1dWY7kJEuGZCCutyjrLVi7QaviiprHW+S+pb34KwVrqXwsLg+usBd+LBo5Z4MGCcbGFMBLar6k5VrQJeAmY1KXMt8Iaq7gVQ1cPNXOd8YIeq7nGwrsZ0G984I5mwEOHlALcySiqrnZ9W+4MfeBcw7r0XsMSDgeZkwBgINP6JzPUcaywTSBCRJSKyRkRuaOY6s4EXHaqjMd1OYnQ4XxvZlze/yKWy5tRpNbxVU1tHRXWd8wFjyBB47TWIijopcNSFhrmPv/aauxzHV3nbor3AcDJgNDfpuenUjFDcXU6XABcBPxGRzIYLiIQDlwOvtngTkbkiki0i2Xl5ef7X2phu4OqsFArLqvlgc3ONet+VVroDT7ukBZk5E9avh7lzIS4OFaE4PIotl13jPj5zZkPRQssjFVBOBoxcIKXR62SgaWKbXGCxqpaqaj7wCXBao/dnAp+r6qGWbqKq81U1S1WzkpKSAlR1Y7q2qRlJDIiP5OUAzZYqrnRPdXV8llS9IUPgj3+EoiKkro5Lf/YOj11xd0PLop4lHgwsJwPGaiBDRNI8LYXZwMImZd4CpopIqIhEAZOALY3en4N1RxkTcCEu4crxySzdlse+o+V+X69dWxjNmJiayOrdBSctSjzqySNliQcDw7GAoao1wJ3Au7iDwCuquklE5onIPE+ZLcBiYD2wCligqhsBPAHka8AbTtXRmO7sqqwUVOG17Fy/r1XiaWG0x+ZJzZmYlkhhWTXb805cp1FgYxgB5ei/rqouAhY1OfZUk9ePAI80c24Z0MvJ+hnTnaUkRnHW0F68uiaH7503FJer7bmWSjwtjJiI4CT4m5Tm/qhYuauAzL6xDcct8WBg2UpvY7qxq7NSyC0s5zM/N1s6vp93cLp+UhJ70C8uklW7Ck44Xr9ozxIPBoYFDGO6sYtG9SO+R5jfg9/HN08Kzm/yIsLEtERW7TpyQkbeglJbtBdIFjCM6cYiw0K4YtwA3t10sGHv67ao3887NkgtDHCPYxw6VsnegrKGY4VlVbZoL4AsYBjTzV09IYWqmjr++cW+Nl8j2C0MgElpiYB7HKNeYVmVtTACyAKGMd3cqAHxjB4Yx8vZuW3eYKmksobIMBeh7bE9awuG9okhMTr8hHGMwlJLPBhIFjCMMVyTlcKWA8fYuO9Ym853b88a3K4fEWFCakJDwGhIPGhdUgFjAcMYw+XjBhIR6uLl7L1tOr+koiZoU2obm5jWi70FZRwoKj+eeNC6pALGAoYxhvgeYVw6dgCvr9nH4eIKn88vbY/9vL1QP46xaleBJR50gAUMYwwAd543lKraOp78aIfP5xa3x37eXhjRP46YiFB3wPDkkbLd9gLHAoYxBoC03tFcNT6ZF1bu9Tm/VElFDbEdoIUR4hKyPOMY9YkHbdA7cCxgGGMafO/8DAAe/2CbT+eVVrXD9qxempiWyLbDJezIKwUs8WAgWcAwxjQY2LMH100exGuf57KzSSK/U3EPeneMgFE/jvHe5oOApTYPJAsYxpgT3D59KOEhLh71oZXhnlbbMQLGmIE9iQh18cXeo4SHuogKD/7sra7CAoYx5gRJsRHcdFYqb6/bz5YDra/LqK6to7KmHbZn9VJ4qIszBiUAWOLBALOAYYw5yXenDSE2MpTfvvdVq2WPpwXpGAED3OMYAD1t0V5AWcAwxpwkPiqM705L54Mth/h8b+EpyxbXpzbvALOk6tWPY9gajMCygGGMadZNZ6XRKzqc37639ZTlSqvq98LoOAHj9EEJhLrEVnkHmAUMY0yzoiNCuf3coXy6/Qif7chvsdzxzZM6TsDoER7CfRcN48rxycGuSpdiAcMY06LrJg2if3wkv3l3a4uZbEsqO16XFMB3zxnCucP6BLsaXYoFDGNMiyLDQvjeeRl8vvcoK3YWNFumIWB0oBaGcYYFDGPMKV0+bgAhLmHZ9rxm3y+1gNFtOBowRGSGiGwVke0icn8LZaaLyFoR2SQiHzc63lNEXhORL0Vki4hMcbKuxpjmxUSEMmZgfIstjPpZUh1pWq1xhmMBQ0RCgCeAmcBIYI6IjGxSpifwJHC5qo4Crmr09mPAYlUdDpwGbHGqrsaYU5uUnsj63KOUV9We9J51SXUfTrYwJgLbVXWnqlYBLwGzmpS5FnhDVfcCqOphABGJA6YBT3uOV6nqUQfraow5hclpvaiu1WbXZJRW1tAjLIQQl62o7uqcDBgDgZxGr3M9xxrLBBJEZImIrBGRGzzH04E84K8i8oWILBCRaAfraow5hazUBFwCK3ceOem9kg6yeZJxnpMBo7lfN5rOywsFxgOXABcBPxGRTM/xM4A/qerpQCnQ0hjIXBHJFpHsvLzmB+WMMf6JjQxj1IB4Vuw6eRyjpLLWuqO6CScDRi6Q0uh1MrC/mTKLVbVUVfOBT3CPV+QCuaq60lPuNdwB5CSqOl9Vs1Q1KykpKaDfgDHmuMnpiazNOUpF9YnjGCUV1RYwugknA8ZqIENE0kQkHJgNLGxS5i1gqoiEikgUMAnYoqoHgRwRGeYpdz6w2cG6GmNaMSmtF1U1dXyx98ThxNLKWqIjLIV4d+DYrwWqWiMidwLvAiHAM6q6SUTmed5/SlW3iMhiYD1QByxQ1Y2eS3wP+Icn2OwEbnKqrsaY1k1IS0QEVu46wpQhvRqOF1fWMLBnjyDWzLQXR9uRqroIWNTk2FNNXj8CPNLMuWuBLCfrZ4zxXnyPMEb0i2Nlk/UYpZU1xFgLo1uwld7GGK9NSk/k872FVNYcH8ewWVLdhwUMY4zXJqf3orKmjvW5RQ3H3Nuz2kZF3YEFDGOM1yamujcmql+PUVVTR1VNnXVJdRMWMIwxXkuIDmd4v9iGvFKWeLB7sYBhjPHJpLRE1uwppLq2riGPlCUe7B4sYBhjfDIpvRfl1bWszy1qyFQba4Pe3YIFDGOMTyamecYxdh1p2M/bWhjdgwUMY4xPesdEkNEnhpU7Czrkft7GORYwjDE+m5SeSPbuAorKqwELGN2FBQxjjM8mpfWitKqWlbvc02tt4V73YAHDGOOzSenucYz/bDkM2BhGd2EBwxjjsz6xkaQnRXO4uBKA6HALGN2BBQxjTJtMSnNnrI0Kt+1ZuwsLGMaYNpns6ZayAe/uwwKGMaZN6lsYNuDdfVjAMMa0Sb/4SAb3irIWRjdi/9LGmDZ7YOYI6lSDXQ3TTixgGGPabMbofsGugmlH1iVljDHGKxYwjDHGeMXRgCEiM0Rkq4hsF5H7WygzXUTWisgmEfm40fHdIrLB8162k/U0xhjTOsfGMEQkBHgC+BqQC6wWkYWqurlRmZ7Ak8AMVd0rIn2aXOZcVc13qo7GGGO852QLYyKwXVV3qmoV8BIwq0mZa4E3VHUvgKoedrA+xhhj/OBkwBgI5DR6nes51lgmkCAiS0RkjYjc0Og9Bd7zHJ/rYD2NMcZ4wclptc0ll2k6YTsUGA+cD/QAlovIClX9CjhLVfd7uqneF5EvVfWTk27iDiZzAQYNGhTQb8AYY8xxTrYwcoGURq+Tgf3NlFmsqqWesYpPgNMAVHW/58/DwJu4u7hOoqrzVTVLVbOSkpIC/C0YY4ypJ+rQKk0RCQW+wt162AesBq5V1U2NyowA/ghcBIQDq4DZwC7AparFIhINvA88rKqLW7lnHrCnhbfjgSIvj3tzrOnr3oCTA/Qt1T+Q57ZWzp6h/+XsGfpXztf3utIz9OU8X57TYFX17rdtVXXsC7gYd9DYAfzYc2weMK9RmfuAzcBG4B7PsXRgnedrU/25ftZlvrfHvTnWzOtsh59ls/UP5LmtlbNnaM8w2M/Q1/e60jP05by2PsPWvhxNDaKqi4BFTY491eT1I8AjTY7txNM1FUBv+3Dcm2MtXc8p/tzP23NbK2fP0P9y9gz9K+fre13pGfpyXluf4Sk51iXV3YhItqpmBbsenZk9Q//ZM/SfPcOWWWqQwJkf7Ap0AfYM/WfP0H/2DFtgLQxjjDFesRaGMcYYr1jAaIaIPCMih0VkYxvOHe9JmrhdRB4XEWn03tUistmTaPGFwNa6Y3HiGYrIt0Ukz5OQcq2IfCfwNe84nPo59Lx/pYioiHTpvnqHfg7nNUqMukxERga+5h2TBYzmPQvMaOO5f8K98jzD8zUDQEQygAdwr2AfBdzjfzU7tGcJ8DP0eFlVx3m+FvhXxQ7vWRx4hiISC9wFrPSzfp3BswT+Gb6gqmNUdRzwa+B3/lays7CA0Qx1pyApaHxMRIaIyGJPbqulIjK86Xki0h+IU9Xl6h4c+htwheftW4EnVLXQc48unWjRoWfYrTj4DP8X9wddhYPV7xCceIaqeqxR0WhOTnnUZVnA8N584HuqOh74Ie607E0NxJ3upF7jhIuZQKaIfCoiK0Skrb/1dGb+PkOAb4rIehF5TURS6H78eoYicjqQoqrvOF3RDszvn0MRuUNEduAOvHc5WNcOxfb09oKIxABnAq826gqOaK5oM8fqf/sIxd2snY47r9ZSERmtqkcDW9uOKUDP8G3gRVWtFJF5wHPAeYGua0fl7zMUERfwKPBtRyrYCQTo5xBVfQJ4QkSuBf4buDHAVe2QLGB4xwUc9fRZNvBsErXG83Ih7j7P5EZFGidczAVWqGo1sEtEtuIOIKudrHgH4vczVNUjjY7/Bfg/x2rbMfn7DGOB0cASz4dlP2ChiFyuqt1lV8tA/F9u7CVP2W7BuqS84Omz3CUiVwGI22mqWttoAPZBVT0AFIvIZM+MihuAtzyX+Sdwruf83ri7qHa2/3cTHIF4hp5+5XqXA1va+/sIJn+foaoWqWpvVU1V1VRgBdCdgkWgfg4zGl3yEmBbe38fQdPWJFRd+Qt4ETgAVONuGdwCpAGLcSdE3Aw82MK5WbgTKe7AnYm3fnGk4J5NsRnYAMwO9vfZCZ/hL3Eno1wHfAQMD/b32dmeYZMyS4CsYH+fne0ZAo95fg7Xen4ORwX7+2yvL1vpbYwxxivWJWWMMcYrFjCMMcZ4xQKGMcYYr1jAMMYY4xULGMYYY7xiAcN0aSJS0s73WxCo7KUiUuvJiLpRRN4WkZ6tlO8pIrcH4t7GNMem1ZouTURKVDUmgNcLVdWaQF2vlXs11F1EngO+UtWfn6J8KvCOqo5uj/qZ7sdaGKbbEZEkEXldRFZ7vs7yHJ8oIp+JyBeeP4d5jn9bRF4VkbeB90Rkuogs8SRA/FJE/uFZDYzneJbn7yUi8nMRWedJONnXc3yI5/VqEXnYy1bQco4nEIwRkf+IyOfi3pdhlqfMr4AhnlbJI56y93nus15EfhrAx2i6IQsYpjt6DHhUVScA3wTq99X4EpimqqcDDwK/aHTOFOBGVa1Pdng67j1NRgLpwFnN3Ccad/6w04BPcKe4r7//Y577N5ef6ASePEfn485xBO605F9X1TNwp5v5rSdg3Q/sUHd6i/tE5ELc+comAuOA8SIyrbX7GdMSSz5ouqMLgJGNspXGiXtToXjgOU+uIAXCGp3zvqo23ldhlarmAojIWiAVWNbkPlVAfRrxNcDXPH+fwvH9KV4AftNCPXs0uvYa4H3PcQF+4fnwr8Pd8ujbzPkXer6+8LyOwR1APmnhfsackgUM0x25gCmqWt74oIj8AfhIVb/uGQ9Y0ujt0ibXqGz091qa/79UrccHCVsqcyrlqjpOROJxB547gMeB64AkYLyqVovIbiCymfMF+KWq/tnH+xrTLOuSMt3Re8Cd9S9EpD7VdTywz/P3bzt4/xW4u8IAZrdWWFWLcG/S80MRCcNdz8OeYHEuMNhTtBh3CvN67wI3e/aAQEQGikifAH0PphuygGG6uigRyW309X3cH75ZnoHgzcA8T9lfA78UkU+BEAfrdA/wfRFZBfQHilo7QVW/wJ1ddTbwD9z1z8bd2vjSU+YI8KlnGu4jqvoe7i6v5SKyAXiNEwOKMT6xabXGtDMRicLd3aQiMhuYo6qzWjvPmGCzMQxj2t944I+emU1HgZuDXB9jvGItDGOMMV6xMQxjjDFesYBhjDHGKxYwjDHGeMUChjHGGK9YwDDGGOMVCxjGGGO88v8B1zYvVgIXrrcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot(suggestion=True, skip_end=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 77
    },
    "colab_type": "code",
    "id": "2C9Jwn7KemDB",
    "outputId": "a1fef34d-5ae6-4083-d81c-467015066ad0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 15:45 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.286651</td>\n",
       "      <td>0.210489</td>\n",
       "      <td>0.908000</td>\n",
       "      <td>03:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.187557</td>\n",
       "      <td>0.171379</td>\n",
       "      <td>0.930000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.127004</td>\n",
       "      <td>0.140221</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.038747</td>\n",
       "      <td>0.172435</td>\n",
       "      <td>0.952000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.008313</td>\n",
       "      <td>0.159150</td>\n",
       "      <td>0.959000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#开始训练\n",
    "learner.fit_one_cycle(config.epochs, slice(3e-5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('htl_stage1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(21128, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): BertLayerNorm()\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Embedding            [128, 768]           16,226,304 True      \n",
       "______________________________________________________________________\n",
       "Embedding            [128, 768]           393,216    True      \n",
       "______________________________________________________________________\n",
       "Embedding            [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 128, 128]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           590,592    True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [128, 3072]          2,362,368  True      \n",
       "______________________________________________________________________\n",
       "Linear               [128, 768]           2,360,064  True      \n",
       "______________________________________________________________________\n",
       "BertLayerNorm        [128, 768]           1,536      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [128, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [768]                590,592    True      \n",
       "______________________________________________________________________\n",
       "Tanh                 [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [2]                  1,538      True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 102,269,186\n",
       "Total trainable params: 102,269,186\n",
       "Total non-trainable params: 0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 09:27 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.034001</td>\n",
       "      <td>0.352218</td>\n",
       "      <td>0.929000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.014278</td>\n",
       "      <td>0.243409</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.001612</td>\n",
       "      <td>0.281989</td>\n",
       "      <td>0.947000</td>\n",
       "      <td>03:09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(3, slice(3e-5/2.6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('htl_stage2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Category 0, tensor(0), tensor([9.9972e-01, 2.8313e-04]))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#用样例测试下\n",
    "learner.predict(\"设施老化，紧靠马路噪音太大，晚上楼上卫生间的水流声和空调噪音非常大，无法入眠\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Category 1, tensor(1), tensor([3.6966e-05, 9.9996e-01]))"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.predict(\"位置真是好的没话讲，距解放碑步行只要两分钟不到，周边吃的、买的、看的都很多\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Chinese_htlData_Classification_with_BERT.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}