P2P_Bert.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d21517b5",
   "metadata": {
    "id": "e460cbb5",
    "papermill": {
     "duration": 0.043282,
     "end_time": "2023-02-20T12:32:43.209451",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.166169",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# About this notebook\n",
    "- inspired from https://www.kaggle.com/code/yasufuminakama/pppm-deberta-v3-large-baseline-w-w-b-train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6104027a",
   "metadata": {
    "papermill": {
     "duration": 0.036378,
     "end_time": "2023-02-20T12:32:43.317282",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.280904",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Directory settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "566b2a9b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:32:43.391974Z",
     "iopub.status.busy": "2023-02-20T12:32:43.390978Z",
     "iopub.status.idle": "2023-02-20T12:32:43.398865Z",
     "shell.execute_reply": "2023-02-20T12:32:43.398228Z",
     "shell.execute_reply.started": "2023-02-20T12:25:47.515939Z"
    },
    "id": "fa3b873b",
    "papermill": {
     "duration": 0.048873,
     "end_time": "2023-02-20T12:32:43.399043",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.350170",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Directory settings\n",
    "# ====================================================\n",
    "import os\n",
    "\n",
    "INPUT_DIR = '../input/us-patent-phrase-to-phrase-matching/'\n",
    "OUTPUT_DIR = './'\n",
    "if not os.path.exists(OUTPUT_DIR):\n",
    "    os.makedirs(OUTPUT_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "493b17cb",
   "metadata": {
    "id": "1d0c4430",
    "papermill": {
     "duration": 0.032375,
     "end_time": "2023-02-20T12:32:43.464878",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.432503",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# CFG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54488510",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:32:43.542860Z",
     "iopub.status.busy": "2023-02-20T12:32:43.541845Z",
     "iopub.status.idle": "2023-02-20T12:32:43.544171Z",
     "shell.execute_reply": "2023-02-20T12:32:43.544622Z",
     "shell.execute_reply.started": "2023-02-20T12:25:59.329047Z"
    },
    "id": "48dd82bb",
    "papermill": {
     "duration": 0.045562,
     "end_time": "2023-02-20T12:32:43.544814",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.499252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# CFG\n",
    "# ====================================================\n",
    "class CFG:\n",
    "    wandb=False\n",
    "    debug=False\n",
    "    apex=True\n",
    "    print_freq=100\n",
    "    num_workers=4\n",
    "    model=\"anferico/bert-for-patents\" #\"microsoft/deberta-v3-small\" #\"anferico/bert-for-patents\"\n",
    "    scheduler='cosine' # ['linear', 'cosine']\n",
    "    batch_scheduler=True\n",
    "    num_cycles=0.5\n",
    "    num_warmup_steps=0\n",
    "    epochs=6\n",
    "    encoder_lr=2e-5\n",
    "    decoder_lr=2e-5\n",
    "    min_lr=1e-6\n",
    "    eps=1e-6\n",
    "    betas=(0.9, 0.999)\n",
    "    batch_size=16\n",
    "    fc_dropout=0.2\n",
    "    target_size=1\n",
    "    max_len=512\n",
    "    weight_decay=0.01\n",
    "    gradient_accumulation_steps=1\n",
    "    max_grad_norm=1000\n",
    "    seed=42\n",
    "    n_fold=4\n",
    "    trn_fold=[0, 1, 2, 3]\n",
    "    train=True\n",
    "    \n",
    "if CFG.debug:\n",
    "    CFG.epochs = 2\n",
    "    CFG.trn_fold = [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aaf22b2b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:32:43.619709Z",
     "iopub.status.busy": "2023-02-20T12:32:43.618641Z",
     "iopub.status.idle": "2023-02-20T12:32:43.620789Z",
     "shell.execute_reply": "2023-02-20T12:32:43.621284Z",
     "shell.execute_reply.started": "2023-02-20T12:25:59.820635Z"
    },
    "id": "b88c983e",
    "papermill": {
     "duration": 0.043145,
     "end_time": "2023-02-20T12:32:43.621455",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.578310",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# wandb\n",
    "# ====================================================\n",
    "if CFG.wandb:\n",
    "    \n",
    "    import wandb\n",
    "\n",
    "    try:\n",
    "        from kaggle_secrets import UserSecretsClient\n",
    "        user_secrets = UserSecretsClient()\n",
    "        secret_value_0 = user_secrets.get_secret(\"wandb_api\")\n",
    "        wandb.login(key=secret_value_0)\n",
    "        anony = None\n",
    "    except:\n",
    "        anony = \"must\"\n",
    "        print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')\n",
    "\n",
    "\n",
    "    def class2dict(f):\n",
    "        return dict((name, getattr(f, name)) for name in dir(f) if not name.startswith('__'))\n",
    "\n",
    "    run = wandb.init(project='PPPM-Public', \n",
    "                     name=CFG.model,\n",
    "                     config=class2dict(CFG),\n",
    "                     group=CFG.model,\n",
    "                     job_type=\"train\",\n",
    "                     anonymous=anony)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa8c598a",
   "metadata": {
    "id": "f2ed8ef2",
    "papermill": {
     "duration": 0.032267,
     "end_time": "2023-02-20T12:32:43.686407",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.654140",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a42f711c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:32:43.766728Z",
     "iopub.status.busy": "2023-02-20T12:32:43.765910Z",
     "iopub.status.idle": "2023-02-20T12:33:16.787037Z",
     "shell.execute_reply": "2023-02-20T12:33:16.787981Z",
     "shell.execute_reply.started": "2023-02-20T12:26:01.993476Z"
    },
    "executionInfo": {
     "elapsed": 20123,
     "status": "ok",
     "timestamp": 1644920080956,
     "user": {
      "displayName": "Yasufumi Nakama",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "17486303986134302670"
     },
     "user_tz": -540
    },
    "id": "35916341",
    "outputId": "06fa0ab8-a380-4f54-a98d-b7015b79d9e2",
    "papermill": {
     "duration": 33.068958,
     "end_time": "2023-02-20T12:33:16.788262",
     "exception": false,
     "start_time": "2023-02-20T12:32:43.719304",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.__version__: 1.9.1\n",
      "Found existing installation: transformers 4.16.2\n",
      "Uninstalling transformers-4.16.2:\n",
      "  Successfully uninstalled transformers-4.16.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing installation: tokenizers 0.11.6\n",
      "Uninstalling tokenizers-0.11.6:\n",
      "  Successfully uninstalled tokenizers-0.11.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in links: ../input/pppm-pip-wheels\n",
      "Processing /kaggle/input/pppm-pip-wheels/transformers-4.18.0-py3-none-any.whl\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.49)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2021.11.10)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.62.3)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers) (4.11.3)\n",
      "Processing /kaggle/input/pppm-pip-wheels/tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.26.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (1.20.3)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.1.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->transformers) (3.0.6)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.6.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.26.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.0.9)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.1.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (8.0.3)\n",
      "Installing collected packages: tokenizers, transformers\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "allennlp 2.9.1 requires transformers<4.17,>=4.1, but you have transformers 4.18.0 which is incompatible.\n",
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully installed tokenizers-0.12.1 transformers-4.18.0\n",
      "Looking in links: ../input/pppm-pip-wheels\n",
      "Requirement already satisfied: tokenizers in /opt/conda/lib/python3.7/site-packages (0.12.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenizers.__version__: 0.12.1\n",
      "transformers.__version__: 4.18.0\n",
      "env: TOKENIZERS_PARALLELISM=true\n"
     ]
    }
   ],
   "source": [
    "# ====================================================\n",
    "# Library\n",
    "# ====================================================\n",
    "import os\n",
    "import gc\n",
    "import re\n",
    "import ast\n",
    "import sys\n",
    "import copy\n",
    "import json\n",
    "import time\n",
    "import math\n",
    "import shutil\n",
    "import string\n",
    "import pickle\n",
    "import random\n",
    "import joblib\n",
    "import itertools\n",
    "from pathlib import Path\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import scipy as sp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n",
    "\n",
    "import torch\n",
    "print(f\"torch.__version__: {torch.__version__}\")\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import Adam, SGD, AdamW\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "os.system('pip uninstall -y transformers')\n",
    "os.system('pip uninstall -y tokenizers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels transformers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels tokenizers')\n",
    "import tokenizers\n",
    "import transformers\n",
    "print(f\"tokenizers.__version__: {tokenizers.__version__}\")\n",
    "print(f\"transformers.__version__: {transformers.__version__}\")\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
    "from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n",
    "%env TOKENIZERS_PARALLELISM=true\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58dc52b0",
   "metadata": {
    "id": "fd586614",
    "papermill": {
     "duration": 0.041303,
     "end_time": "2023-02-20T12:33:16.887019",
     "exception": false,
     "start_time": "2023-02-20T12:33:16.845716",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22bb4cf5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:16.973736Z",
     "iopub.status.busy": "2023-02-20T12:33:16.972935Z",
     "iopub.status.idle": "2023-02-20T12:33:16.978712Z",
     "shell.execute_reply": "2023-02-20T12:33:16.978180Z",
     "shell.execute_reply.started": "2023-02-20T12:26:29.766512Z"
    },
    "id": "d5c0ccc6",
    "papermill": {
     "duration": 0.053542,
     "end_time": "2023-02-20T12:33:16.978864",
     "exception": false,
     "start_time": "2023-02-20T12:33:16.925322",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Utils\n",
    "# ====================================================\n",
    "def get_score(y_true, y_pred):\n",
    "    score = sp.stats.pearsonr(y_true, y_pred)[0]\n",
    "    return score\n",
    "\n",
    "\n",
    "def get_logger(filename=OUTPUT_DIR+'train'):\n",
    "    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n",
    "    logger = getLogger(__name__)\n",
    "    logger.setLevel(INFO)\n",
    "    handler1 = StreamHandler()\n",
    "    handler1.setFormatter(Formatter(\"%(message)s\"))\n",
    "    handler2 = FileHandler(filename=f\"{filename}.log\")\n",
    "    handler2.setFormatter(Formatter(\"%(message)s\"))\n",
    "    logger.addHandler(handler1)\n",
    "    logger.addHandler(handler2)\n",
    "    return logger\n",
    "\n",
    "LOGGER = get_logger()\n",
    "\n",
    "def seed_everything(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "seed_everything(seed=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9d5ca94",
   "metadata": {
    "id": "cb3d8e1e",
    "papermill": {
     "duration": 0.038724,
     "end_time": "2023-02-20T12:33:17.056094",
     "exception": false,
     "start_time": "2023-02-20T12:33:17.017370",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "384cfe94",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:17.139466Z",
     "iopub.status.busy": "2023-02-20T12:33:17.138305Z",
     "iopub.status.idle": "2023-02-20T12:33:17.294264Z",
     "shell.execute_reply": "2023-02-20T12:33:17.293747Z",
     "shell.execute_reply.started": "2023-02-20T12:26:29.779611Z"
    },
    "executionInfo": {
     "elapsed": 2627,
     "status": "ok",
     "timestamp": 1644920084001,
     "user": {
      "displayName": "Yasufumi Nakama",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "17486303986134302670"
     },
     "user_tz": -540
    },
    "id": "bef012d3",
    "outputId": "d4d60dbc-510c-4f34-8d64-dd1d88c4808c",
    "papermill": {
     "duration": 0.200373,
     "end_time": "2023-02-20T12:33:17.294409",
     "exception": false,
     "start_time": "2023-02-20T12:33:17.094036",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train.shape: (36473, 5)\n",
      "test.shape: (36, 4)\n",
      "submission.shape: (36, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>anchor</th>\n",
       "      <th>target</th>\n",
       "      <th>context</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>37d61fd2272659b1</td>\n",
       "      <td>abatement</td>\n",
       "      <td>abatement of pollution</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7b9652b17b68b7a4</td>\n",
       "      <td>abatement</td>\n",
       "      <td>act of abating</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36d72442aefd8232</td>\n",
       "      <td>abatement</td>\n",
       "      <td>active catalyst</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5296b0c19e1ce60e</td>\n",
       "      <td>abatement</td>\n",
       "      <td>eliminating process</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54c1e3b9184cb5b6</td>\n",
       "      <td>abatement</td>\n",
       "      <td>forest region</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id     anchor                  target context  score\n",
       "0  37d61fd2272659b1  abatement  abatement of pollution     A47   0.50\n",
       "1  7b9652b17b68b7a4  abatement          act of abating     A47   0.75\n",
       "2  36d72442aefd8232  abatement         active catalyst     A47   0.25\n",
       "3  5296b0c19e1ce60e  abatement     eliminating process     A47   0.50\n",
       "4  54c1e3b9184cb5b6  abatement           forest region     A47   0.00"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>anchor</th>\n",
       "      <th>target</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4112d61851461f60</td>\n",
       "      <td>opc drum</td>\n",
       "      <td>inorganic photoconductor drum</td>\n",
       "      <td>G02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>09e418c93a776564</td>\n",
       "      <td>adjust gas flow</td>\n",
       "      <td>altering gas flow</td>\n",
       "      <td>F23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36baf228038e314b</td>\n",
       "      <td>lower trunnion</td>\n",
       "      <td>lower locating</td>\n",
       "      <td>B60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1f37ead645e7f0c8</td>\n",
       "      <td>cap component</td>\n",
       "      <td>upper portion</td>\n",
       "      <td>D06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>71a5b6ad068d531f</td>\n",
       "      <td>neural stimulation</td>\n",
       "      <td>artificial neural network</td>\n",
       "      <td>H04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id              anchor                         target context\n",
       "0  4112d61851461f60            opc drum  inorganic photoconductor drum     G02\n",
       "1  09e418c93a776564     adjust gas flow              altering gas flow     F23\n",
       "2  36baf228038e314b      lower trunnion                 lower locating     B60\n",
       "3  1f37ead645e7f0c8       cap component                  upper portion     D06\n",
       "4  71a5b6ad068d531f  neural stimulation      artificial neural network     H04"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4112d61851461f60</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>09e418c93a776564</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36baf228038e314b</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1f37ead645e7f0c8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>71a5b6ad068d531f</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id  score\n",
       "0  4112d61851461f60      0\n",
       "1  09e418c93a776564      0\n",
       "2  36baf228038e314b      0\n",
       "3  1f37ead645e7f0c8      0\n",
       "4  71a5b6ad068d531f      0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ====================================================\n",
    "# Data Loading\n",
    "# ====================================================\n",
    "train = pd.read_csv(INPUT_DIR+'train.csv')\n",
    "test = pd.read_csv(INPUT_DIR+'test.csv')\n",
    "submission = pd.read_csv(INPUT_DIR+'sample_submission.csv')\n",
    "print(f\"train.shape: {train.shape}\")\n",
    "print(f\"test.shape: {test.shape}\")\n",
    "print(f\"submission.shape: {submission.shape}\")\n",
    "display(train.head())\n",
    "display(test.head())\n",
    "display(submission.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d3e0e11f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:17.389447Z",
     "iopub.status.busy": "2023-02-20T12:33:17.388729Z",
     "iopub.status.idle": "2023-02-20T12:33:18.302958Z",
     "shell.execute_reply": "2023-02-20T12:33:18.303442Z",
     "shell.execute_reply.started": "2023-02-20T12:26:29.929728Z"
    },
    "papermill": {
     "duration": 0.967167,
     "end_time": "2023-02-20T12:33:18.303632",
     "exception": false,
     "start_time": "2023-02-20T12:33:17.336465",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>anchor</th>\n",
       "      <th>target</th>\n",
       "      <th>context</th>\n",
       "      <th>score</th>\n",
       "      <th>context_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>37d61fd2272659b1</td>\n",
       "      <td>abatement</td>\n",
       "      <td>abatement of pollution</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.50</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7b9652b17b68b7a4</td>\n",
       "      <td>abatement</td>\n",
       "      <td>act of abating</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.75</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36d72442aefd8232</td>\n",
       "      <td>abatement</td>\n",
       "      <td>active catalyst</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.25</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5296b0c19e1ce60e</td>\n",
       "      <td>abatement</td>\n",
       "      <td>eliminating process</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.50</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54c1e3b9184cb5b6</td>\n",
       "      <td>abatement</td>\n",
       "      <td>forest region</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id     anchor                  target context  score                                       context_text\n",
       "0  37d61fd2272659b1  abatement  abatement of pollution     A47   0.50  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...\n",
       "1  7b9652b17b68b7a4  abatement          act of abating     A47   0.75  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...\n",
       "2  36d72442aefd8232  abatement         active catalyst     A47   0.25  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...\n",
       "3  5296b0c19e1ce60e  abatement     eliminating process     A47   0.50  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...\n",
       "4  54c1e3b9184cb5b6  abatement           forest region     A47   0.00  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>anchor</th>\n",
       "      <th>target</th>\n",
       "      <th>context</th>\n",
       "      <th>context_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4112d61851461f60</td>\n",
       "      <td>opc drum</td>\n",
       "      <td>inorganic photoconductor drum</td>\n",
       "      <td>G02</td>\n",
       "      <td>PHYSICS. OPTICS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>09e418c93a776564</td>\n",
       "      <td>adjust gas flow</td>\n",
       "      <td>altering gas flow</td>\n",
       "      <td>F23</td>\n",
       "      <td>MECHANICAL ENGINEERING; LIGHTING; HEATING; WEA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36baf228038e314b</td>\n",
       "      <td>lower trunnion</td>\n",
       "      <td>lower locating</td>\n",
       "      <td>B60</td>\n",
       "      <td>PERFORMING OPERATIONS; TRANSPORTING. VEHICLES ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1f37ead645e7f0c8</td>\n",
       "      <td>cap component</td>\n",
       "      <td>upper portion</td>\n",
       "      <td>D06</td>\n",
       "      <td>TEXTILES; PAPER. TREATMENT OF TEXTILES OR THE ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>71a5b6ad068d531f</td>\n",
       "      <td>neural stimulation</td>\n",
       "      <td>artificial neural network</td>\n",
       "      <td>H04</td>\n",
       "      <td>ELECTRICITY. ELECTRIC COMMUNICATION TECHNIQUE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id              anchor                         target context                                       context_text\n",
       "0  4112d61851461f60            opc drum  inorganic photoconductor drum     G02                                    PHYSICS. OPTICS\n",
       "1  09e418c93a776564     adjust gas flow              altering gas flow     F23  MECHANICAL ENGINEERING; LIGHTING; HEATING; WEA...\n",
       "2  36baf228038e314b      lower trunnion                 lower locating     B60  PERFORMING OPERATIONS; TRANSPORTING. VEHICLES ...\n",
       "3  1f37ead645e7f0c8       cap component                  upper portion     D06  TEXTILES; PAPER. TREATMENT OF TEXTILES OR THE ...\n",
       "4  71a5b6ad068d531f  neural stimulation      artificial neural network     H04      ELECTRICITY. ELECTRIC COMMUNICATION TECHNIQUE"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ====================================================\n",
    "# CPC Data\n",
    "# ====================================================\n",
    "def get_cpc_texts():\n",
    "    contexts = []\n",
    "    pattern = '[A-Z]\\d+'\n",
    "    for file_name in os.listdir('../input/cpc-data/CPCSchemeXML202105'):\n",
    "        result = re.findall(pattern, file_name)\n",
    "        if result:\n",
    "            contexts.append(result)\n",
    "    contexts = sorted(set(sum(contexts, [])))\n",
    "    results = {}\n",
    "    for cpc in ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'Y']:\n",
    "        with open(f'../input/cpc-data/CPCTitleList202202/cpc-section-{cpc}_20220201.txt') as f:\n",
    "            s = f.read()\n",
    "        pattern = f'{cpc}\\t\\t.+'\n",
    "        result = re.findall(pattern, s)\n",
    "        cpc_result = result[0].lstrip(pattern)\n",
    "        for context in [c for c in contexts if c[0] == cpc]:\n",
    "            pattern = f'{context}\\t\\t.+'\n",
    "            result = re.findall(pattern, s)\n",
    "            results[context] = cpc_result + \". \" + result[0].lstrip(pattern)\n",
    "    return results\n",
    "\n",
    "\n",
    "cpc_texts = get_cpc_texts()\n",
    "torch.save(cpc_texts, OUTPUT_DIR+\"cpc_texts.pth\")\n",
    "train['context_text'] = train['context'].map(cpc_texts)\n",
    "test['context_text'] = test['context'].map(cpc_texts)\n",
    "display(train.head())\n",
    "display(test.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "da5cafd6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:18.464074Z",
     "iopub.status.busy": "2023-02-20T12:33:18.463140Z",
     "iopub.status.idle": "2023-02-20T12:33:18.503143Z",
     "shell.execute_reply": "2023-02-20T12:33:18.504722Z",
     "shell.execute_reply.started": "2023-02-20T12:26:30.879588Z"
    },
    "papermill": {
     "duration": 0.118897,
     "end_time": "2023-02-20T12:33:18.504962",
     "exception": false,
     "start_time": "2023-02-20T12:33:18.386065",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>anchor</th>\n",
       "      <th>target</th>\n",
       "      <th>context</th>\n",
       "      <th>score</th>\n",
       "      <th>context_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>37d61fd2272659b1</td>\n",
       "      <td>abatement</td>\n",
       "      <td>abatement of pollution</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.50</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "      <td>abatement[SEP]abatement of pollution</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7b9652b17b68b7a4</td>\n",
       "      <td>abatement</td>\n",
       "      <td>act of abating</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.75</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "      <td>abatement[SEP]act of abating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36d72442aefd8232</td>\n",
       "      <td>abatement</td>\n",
       "      <td>active catalyst</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.25</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "      <td>abatement[SEP]active catalyst</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5296b0c19e1ce60e</td>\n",
       "      <td>abatement</td>\n",
       "      <td>eliminating process</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.50</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "      <td>abatement[SEP]eliminating process</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54c1e3b9184cb5b6</td>\n",
       "      <td>abatement</td>\n",
       "      <td>forest region</td>\n",
       "      <td>A47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...</td>\n",
       "      <td>abatement[SEP]forest region</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id     anchor                  target context  score                                       context_text                                  text\n",
       "0  37d61fd2272659b1  abatement  abatement of pollution     A47   0.50  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...  abatement[SEP]abatement of pollution\n",
       "1  7b9652b17b68b7a4  abatement          act of abating     A47   0.75  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...          abatement[SEP]act of abating\n",
       "2  36d72442aefd8232  abatement         active catalyst     A47   0.25  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...         abatement[SEP]active catalyst\n",
       "3  5296b0c19e1ce60e  abatement     eliminating process     A47   0.50  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...     abatement[SEP]eliminating process\n",
       "4  54c1e3b9184cb5b6  abatement           forest region     A47   0.00  HUMAN NECESSITIES. FURNITURE; DOMESTIC ARTICLE...           abatement[SEP]forest region"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>anchor</th>\n",
       "      <th>target</th>\n",
       "      <th>context</th>\n",
       "      <th>context_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4112d61851461f60</td>\n",
       "      <td>opc drum</td>\n",
       "      <td>inorganic photoconductor drum</td>\n",
       "      <td>G02</td>\n",
       "      <td>PHYSICS. OPTICS</td>\n",
       "      <td>opc drum[SEP]inorganic photoconductor drum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>09e418c93a776564</td>\n",
       "      <td>adjust gas flow</td>\n",
       "      <td>altering gas flow</td>\n",
       "      <td>F23</td>\n",
       "      <td>MECHANICAL ENGINEERING; LIGHTING; HEATING; WEA...</td>\n",
       "      <td>adjust gas flow[SEP]altering gas flow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36baf228038e314b</td>\n",
       "      <td>lower trunnion</td>\n",
       "      <td>lower locating</td>\n",
       "      <td>B60</td>\n",
       "      <td>PERFORMING OPERATIONS; TRANSPORTING. VEHICLES ...</td>\n",
       "      <td>lower trunnion[SEP]lower locating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1f37ead645e7f0c8</td>\n",
       "      <td>cap component</td>\n",
       "      <td>upper portion</td>\n",
       "      <td>D06</td>\n",
       "      <td>TEXTILES; PAPER. TREATMENT OF TEXTILES OR THE ...</td>\n",
       "      <td>cap component[SEP]upper portion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>71a5b6ad068d531f</td>\n",
       "      <td>neural stimulation</td>\n",
       "      <td>artificial neural network</td>\n",
       "      <td>H04</td>\n",
       "      <td>ELECTRICITY. ELECTRIC COMMUNICATION TECHNIQUE</td>\n",
       "      <td>neural stimulation[SEP]artificial neural network</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id              anchor                         target context                                       context_text                                              text\n",
       "0  4112d61851461f60            opc drum  inorganic photoconductor drum     G02                                    PHYSICS. OPTICS        opc drum[SEP]inorganic photoconductor drum\n",
       "1  09e418c93a776564     adjust gas flow              altering gas flow     F23  MECHANICAL ENGINEERING; LIGHTING; HEATING; WEA...             adjust gas flow[SEP]altering gas flow\n",
       "2  36baf228038e314b      lower trunnion                 lower locating     B60  PERFORMING OPERATIONS; TRANSPORTING. VEHICLES ...                 lower trunnion[SEP]lower locating\n",
       "3  1f37ead645e7f0c8       cap component                  upper portion     D06  TEXTILES; PAPER. TREATMENT OF TEXTILES OR THE ...                   cap component[SEP]upper portion\n",
       "4  71a5b6ad068d531f  neural stimulation      artificial neural network     H04      ELECTRICITY. ELECTRIC COMMUNICATION TECHNIQUE  neural stimulation[SEP]artificial neural network"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train['text'] = train['anchor'] + '[SEP]' + train['target'] #+ '[SEP]'  + train['context_text']\n",
    "test['text'] = test['anchor'] + '[SEP]' + test['target'] #+ '[SEP]'  + test['context_text']\n",
    "display(train.head())\n",
    "display(test.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39b2e94e",
   "metadata": {
    "papermill": {
     "duration": 0.063706,
     "end_time": "2023-02-20T12:33:18.635309",
     "exception": false,
     "start_time": "2023-02-20T12:33:18.571603",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "430ed298",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:18.769012Z",
     "iopub.status.busy": "2023-02-20T12:33:18.767937Z",
     "iopub.status.idle": "2023-02-20T12:33:19.108253Z",
     "shell.execute_reply": "2023-02-20T12:33:19.107745Z",
     "shell.execute_reply.started": "2023-02-20T12:26:30.912220Z"
    },
    "papermill": {
     "duration": 0.40945,
     "end_time": "2023-02-20T12:33:19.108406",
     "exception": false,
     "start_time": "2023-02-20T12:33:18.698956",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAT6klEQVR4nO3cf6zd9X3f8eerdkkIJJiE7iqyWe0pbjYHVo1eAVWk7iauwJAKI5VGIFrszKullmRZi9aYVRNTEiSilrLA8qPe8GwiGkNZN1uFllqEK7SpJkDJMD9KuQMC9kicxsadQ37U2Xt/nM9tT10b33vOvef4+jwf0tX9fj/fz/f7/bzPOfbrfn+cb6oKSdJo+5FhD0CSNHyGgSTJMJAkGQaSJAwDSRKweNgD6NU555xTy5cv72nd73znO5xxxhlzO6CTnDWPhlGredTqhf5rfuKJJ/6yqn7s6PYFGwbLly/n8ccf72ndyclJJiYm5nZAJzlrHg2jVvOo1Qv915zk68dq9zSRJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJJYwN9Alk5WyzfdP7R9b10zWo9m0NzxyECSdOIwSLIlyf4kT3e1/VaSP0/yVJL/lmRJ17Ibk0wleT7JpV3ta1rbVJJNXe0rkjza2u9Jctoc1idJmoGZHBlsBdYc1bYLOK+q/inwF8CNAElWAVcD72vrfD7JoiSLgM8BlwGrgGtaX4DPALdV1XuAg8CGviqSJM3aCcOgqh4BDhzV9idVdaTN7gaWtem1wPaq+n5VvQRMARe2n6mqerGqfgBsB9YmCfBB4L62/jbgyv5KkiTN1lxcQP4XwD1teimdcJi2t7UBvHpU+0XAu4DXu4Klu//fk2QjsBFgbGyMycnJngZ8+PDhntddqKx5cG44/8iJO82TUXufR61emL+a+wqDJL8JHAHunpvhvLmq2gxsBhgfH69en+ntM9BHw7BqXj/ku4lG6X32cz13eg6DJOuBnwNWV1W15n3AuV3dlrU2jtP+bWBJksXt6KC7vyRpQHq6tTTJGuA3gCuq6o2uRTuBq5O8JckKYCXwVeAxYGW7c+g0OheZd7YQeRi4qq2/DtjRWymSpF7N5NbSLwN/Crw3yd4kG4D/CLwd2JXka0m+CFBVzwD3As8CfwxcX1U/bH/1fxR4EHgOuLf1BfgE8OtJpuhcQ7hzTiuUJJ3QCU8TVdU1x2g+7n/YVXUzcPMx2h8AHjhG+4t07jaSJA2J30CWJBkGkiQfVDcy9uw7NJRbHl++5UMD36ek2fPIQJJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAksQMwiDJliT7kzzd1fbOJLuSvNB+n93ak+T2JFNJnkpyQdc661r/F5Ks62r/qSR72jq3J8lcFylJenMzOTLYCqw5qm0T8FBVrQQeavMAlwEr289G4AvQCQ/gJuAi4ELgpukAaX1+uWu9o/clSZpnJwyDqnoEOHBU81pgW5veBlzZ1X5XdewGliR5N3ApsKuqDlTVQWAXsKYte0dV7a6qAu7q2pYkaUAW97jeWFW91qa/AYy16aXAq1399ra2N2vfe4z2Y0qykc4RB2NjY0xOTvY0+MOHD/e87kI1djrccP6Rge93mK/zsN7nYbzO00btsz1q9cL81dxrGPyNqqokNReDmcG+NgObAcbHx2tiYqKn7UxOTtLrugvVHXfv4NY9fb/ds/bytRMD3+e0Yb3P6zfdP/B9Ttu65oyR+myP4r/l+aq517uJvtlO8dB+72/t+4Bzu/ota21v1r7sGO2SpAHqNQx2AtN3BK0DdnS1X9fuKroYONROJz0IXJLk7Hbh+BLgwbbsr5Jc3O4iuq5rW5KkATnheYMkXwYmgHOS7KVzV9AtwL1JNgBfBz7cuj8AXA5MAW8AHwGoqgNJPgU81vp9sqqmL0r/Kp07lk4H/qj9SJIG6IRhUFXXHGfR6mP0LeD642xnC7DlGO2PA+edaBySpPnjN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSRJ9hkOTXkjyT5OkkX07y1iQrkjyaZCrJPUlOa33f0uan2vLlXdu5sbU/n+TSPmuSJM1Sz2GQZCnwr4DxqjoPWARcDXwGuK2q3gMcBDa0VTYAB1v7ba0fSVa19d4HrAE+n2RRr+OSJM1ev6eJFgOnJ1kMvA14DfggcF9bvg24sk2vbfO05auTpLVvr6rvV9VLwBRwYZ/jkiTNwuJeV6yqfUl+G3gF+C7wJ8ATwOtVdaR12wssbdNLgVfbukeSHALe1dp3d226e52/I8lGYCPA2NgYk5OTPY398OHDPa+7UI2dDjecf+TEHefYMF/nYb3Pw3idp43aZ3vU6oX5q7nnMEhyNp2/6lcArwO/T+c0z7ypqs3AZoDx8fGamJjoaTuTk5P0uu5CdcfdO7h1T89vd89evnZi4PucNqz3ef2m+we+z2lb15wxUp/tUfy3PF8193Oa6GeBl6rqW1X118AfAO8HlrTTRgDLgH1teh9wLkBbfhbw7e72Y6wjSRqAfsLgFeDiJG9r5/5XA88CDwNXtT7rgB1temebpy3/SlVVa7+63W20AlgJfLWPcUmSZqmfawaPJrkP+DPgCPAknVM49wPbk3y6td3ZVrkT+FKSKeAAnTuIqKpnktxLJ0iOANdX1Q97HZckafb6OolcVTcBNx3V/CLHuBuoqr4H/MJxtnMzcHM/Y5Ek9c5vIEuSDANJkmEgSaLPawYL1Z59h4ZyL/jLt3xo4PuUpJnwyECSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJJEn2GQZEmS+5L8eZLnkvx0kncm2ZXkhfb77NY3SW5PMpXkqSQXdG1nXev/QpJ1/RYlSZqdfo8MPgv8cVX9Y+AngeeATcBDVbUSeKjNA1wGrGw/G4EvACR5J3ATcBFwIXDTdIBIkgaj5zBIchbwM8CdAFX1g6p6HVgLbGvdtgFXtum1wF3VsRtYkuTdwKXArqo6UFUHgV3Aml7HJUmavcV9rLsC+BbwX5L8JPAE8HFgrKpea32+AYy16aXAq13r721tx2v/e5JspHNUwdjYGJOTkz0NfOx0uOH8Iz2t249exzsXRrHmw4cPD2X/w3idpw2r5mEZtXph/mruJwwWAxcAH6uqR5N8lr89JQRAVVWS6meAR21vM7AZYHx8vCYmJnrazh137+DWPf2U3puXr50Y+D6njWLNk5OT9PoZ6cf6TfcPfJ/Ttq45Yyg1D8uw3uNhmq+a+7lmsBfYW1WPtvn76ITDN9vpH9rv/W35PuDcrvWXtbbjtUuSBqTnMKiqbwCvJnlva1oNPAvsBKbvCFoH7GjTO4Hr2l1FFwOH2umkB4FLkpzdLhxf0tokSQPS73mDjwF3JzkNeBH4CJ2AuTfJBuDrwIdb3weAy4Ep4I3Wl6o6kORTwGOt3yer6kCf45IkzUJfYVBVXwPGj7Fo9TH6FnD9cbazBdjSz1gkSb3zG8iSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJLEHIRBkkVJnkzyh21+RZJHk0wluSfJaa39LW1+qi1f3rWNG1v780ku7XdMkqTZmYsjg48Dz3XNfwa4rareAxwENrT2DcDB1n5b60eSVcDVwPuANcDnkyyag3FJkmaorzBIsgz4EPCf23yADwL3tS7bgCvb9No2T1u+uvVfC2yvqu9X1UvAFHBhP+OSJM3O4j7X/w/AbwBvb/PvAl6vqiNtfi+wtE0vBV4FqKojSQ61/kuB3V3b7F7n70iyEdgIMDY2xuTkZE+DHjsdbjj/yIk7zrFexzsXRrHmw4cPD2X/w3idpw2r5j37Dg18nwArzlo01M/YMMzXe9xzGCT5OWB/VT2RZGLORvQmqmozsBlgfHy8JiZ62+0dd+/g1j395uDsvXztxMD3OW0Ua56cnKTXz0g/1m+6f+D7nLZ1zRkjVfOw6h2m+fpc9/O/w/uBK5JcDrwVeAfwWWBJksXt6GAZsK/13wecC+xNshg4C/h2V/u07nUkSQPQ8zWDqrqxqpZV1XI6F4C/UlXXAg8DV7Vu64AdbXpnm6ct/0pVVWu/ut1ttAJYCXy113FJkmZvPs4bfALYnuTTwJPAna39TuBLSaaAA3QChKp6Jsm9wLPAEeD6qvrhPIxLknQccxIGVTUJTLbpFznG3UBV9T3gF46z/s3AzXMxFknS7PkNZEmSYSBJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCTRRxgkOTfJw0meTfJMko+39ncm2ZXkhfb77NaeJLcnmUryVJILura1rvV/Icm6/suSJM1GP0cGR4AbqmoVcDFwfZJVwCbgoapaCTzU5gEuA1a2n43AF6ATHsBNwEXAhcBN0wEiSRqMnsOgql6rqj9r0/8XeA5YCqwFtrVu24Ar2/Ra4K7q2A0sSfJu4FJgV1UdqKqDwC5gTa/jkiTNXqqq/40ky4FHgPOAV6pqSWsPcLCqliT5Q+CWqvofbdlDwCeACeCtVfXp1v7vgO9W1W8fYz8b6RxVMDY29lPbt2/vabz7Dxzim9/tadW+nL/0rMHvtBnFmg8fPsyZZ5458P3u2Xdo4PuctuKsRSNV87DqHaZ+P9cf+MAHnqiq8aPbF/c1KiDJmcB/Bf51Vf1V5///jqqqJP2nzd9ubzOwGWB8fLwmJiZ62s4dd+/g1j19lz5rL187MfB9ThvFmicnJ+n1M9KP9ZvuH/g+p21dc8ZI1Tyseodpvj7Xfd1NlORH6QTB3VX1B635m+30D+33/ta+Dzi3a/Vlre147ZKkAennbqIAdwLPVdXvdC3aCUzfEbQO2NHVfl27q+hi4FBVvQY8CFyS5Ox24fiS1iZJGpB+zhu8H/glYE+Sr7W2fwvcAtybZAPwdeDDbdkDwOXAFPAG8BGAqjqQ5FPAY63fJ6vqQB/jkiTNUs9h0C4E5ziLVx+jfwHXH2dbW4AtvY5FktQfv4EsSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSczBg+okaRQtH+LD+eaDRwaSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAksRJFAZJ1iR5PslUkk3DHo8kjZKTIgySLAI+B1wGrAKuSbJquKOSpNFxUoQBcCEwVVUvVtUPgO3A2iGPSZJGRqpq2GMgyVXAmqr6l23+l4CLquqjR/XbCGxss+8Fnu9xl+cAf9njuguVNY+GUat51OqF/mv+8ar6saMbF/exwYGrqs3A5n63k+TxqhqfgyEtGNY8Gkat5lGrF+av5pPlNNE+4Nyu+WWtTZI0ACdLGDwGrEyyIslpwNXAziGPSZJGxklxmqiqjiT5KPAgsAjYUlXPzOMu+z7VtABZ82gYtZpHrV6Yp5pPigvIkqThOllOE0mShsgwkCSd2mFwokdcJHlLknva8keTLB/CMOfMDOr99STPJnkqyUNJfnwY45xLM32MSZKfT1JJFvxtiDOpOcmH23v9TJLfG/QY59oMPtv/MMnDSZ5sn+/LhzHOuZJkS5L9SZ4+zvIkub29Hk8luaDvnVbVKflD50L0/wb+EXAa8L+AVUf1+VXgi236auCeYY97nuv9APC2Nv0rC7nemdbc+r0deATYDYwPe9wDeJ9XAk8CZ7f5fzDscQ+g5s3Ar7TpVcDLwx53nzX/DHAB8PRxll8O/BEQ4GLg0X73eSofGczkERdrgW1t+j5gdZIMcIxz6YT1VtXDVfVGm91N5/scC9lMH2PyKeAzwPcGObh5MpOafxn4XFUdBKiq/QMe41ybSc0FvKNNnwX8nwGOb85V1SPAgTfpsha4qzp2A0uSvLuffZ7KYbAUeLVrfm9rO2afqjoCHALeNZDRzb2Z1NttA52/LBayE9bcDp/Prar7BzmweTST9/kngJ9I8j+T7E6yZmCjmx8zqfnfA7+YZC/wAPCxwQxtaGb77/2ETorvGWiwkvwiMA7882GPZT4l+RHgd4D1Qx7KoC2mc6pogs7R3yNJzq+q14c5qHl2DbC1qm5N8tPAl5KcV1X/b9gDWyhO5SODmTzi4m/6JFlM5/Dy2wMZ3dyb0SM9kvws8JvAFVX1/QGNbb6cqOa3A+cBk0lepnNudecCv4g8k/d5L7Czqv66ql4C/oJOOCxUM6l5A3AvQFX9KfBWOg90O1XN+SN8TuUwmMkjLnYC69r0VcBXql2dWYBOWG+Sfwb8Lp0gWOjnkeEENVfVoao6p6qWV9VyOtdJrqiqx4cz3Dkxk8/1f6dzVECSc+icNnpxgGOcazOp+RVgNUCSf0InDL410FEO1k7gunZX0cXAoap6rZ8NnrKnieo4j7hI8kng8araCdxJ53Byis7FmquHN+L+zLDe3wLOBH6/XSd/paquGNqg+zTDmk8pM6z5QeCSJM8CPwT+TVUt1CPemdZ8A/CfkvwanYvJ6xfwH3Yk+TKdQD+nXQe5CfhRgKr6Ip3rIpcDU8AbwEf63ucCfr0kSXPkVD5NJEmaIcNAkmQYSJIMA0kShoEkCcNAkoRhIEkC/j/+hJpxR/8AwwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train['score'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4136696b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:19.203446Z",
     "iopub.status.busy": "2023-02-20T12:33:19.202586Z",
     "iopub.status.idle": "2023-02-20T12:33:19.218090Z",
     "shell.execute_reply": "2023-02-20T12:33:19.218539Z",
     "shell.execute_reply.started": "2023-02-20T12:26:31.171215Z"
    },
    "papermill": {
     "duration": 0.065573,
     "end_time": "2023-02-20T12:33:19.218732",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.153159",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "B    8019\n",
       "H    6195\n",
       "G    6013\n",
       "C    5288\n",
       "A    4094\n",
       "F    4054\n",
       "E    1531\n",
       "D    1279\n",
       "Name: context, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train['context'].apply(lambda x: x[0]).value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8c27aa6",
   "metadata": {
    "papermill": {
     "duration": 0.044877,
     "end_time": "2023-02-20T12:33:19.309306",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.264429",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "- Y is not in training data, but may be in test data?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ba06594",
   "metadata": {
    "id": "9e05b6c4",
    "papermill": {
     "duration": 0.044941,
     "end_time": "2023-02-20T12:33:19.399768",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.354827",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# CV split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "61287386",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:19.499711Z",
     "iopub.status.busy": "2023-02-20T12:33:19.498703Z",
     "iopub.status.idle": "2023-02-20T12:33:19.522652Z",
     "shell.execute_reply": "2023-02-20T12:33:19.523156Z",
     "shell.execute_reply.started": "2023-02-20T12:26:31.188910Z"
    },
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1644920084528,
     "user": {
      "displayName": "Yasufumi Nakama",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "17486303986134302670"
     },
     "user_tz": -540
    },
    "id": "3ba287c4",
    "outputId": "307dc0e2-17d6-4bfe-9e95-fdd9b6303974",
    "papermill": {
     "duration": 0.078223,
     "end_time": "2023-02-20T12:33:19.523330",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.445107",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fold\n",
       "0    9119\n",
       "1    9118\n",
       "2    9118\n",
       "3    9118\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ====================================================\n",
    "# CV split\n",
    "# ====================================================\n",
    "train['score_map'] = train['score'].map({0.00: 0, 0.25: 1, 0.50: 2, 0.75: 3, 1.00: 4})\n",
    "Fold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\n",
    "for n, (train_index, val_index) in enumerate(Fold.split(train, train['score_map'])):\n",
    "    train.loc[val_index, 'fold'] = int(n)\n",
    "train['fold'] = train['fold'].astype(int)\n",
    "display(train.groupby('fold').size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "091515ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:19.618903Z",
     "iopub.status.busy": "2023-02-20T12:33:19.617968Z",
     "iopub.status.idle": "2023-02-20T12:33:19.623721Z",
     "shell.execute_reply": "2023-02-20T12:33:19.623194Z",
     "shell.execute_reply.started": "2023-02-20T12:26:31.217761Z"
    },
    "id": "4c3ce877",
    "papermill": {
     "duration": 0.054441,
     "end_time": "2023-02-20T12:33:19.623868",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.569427",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if CFG.debug:\n",
    "    display(train.groupby('fold').size())\n",
    "    train = train.sample(n=1000, random_state=0).reset_index(drop=True)\n",
    "    display(train.groupby('fold').size())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eebc486",
   "metadata": {
    "id": "918a28aa",
    "papermill": {
     "duration": 0.046248,
     "end_time": "2023-02-20T12:33:19.715296",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.669048",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a8e595e0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:19.817447Z",
     "iopub.status.busy": "2023-02-20T12:33:19.816792Z",
     "iopub.status.idle": "2023-02-20T12:33:33.038185Z",
     "shell.execute_reply": "2023-02-20T12:33:33.037573Z",
     "shell.execute_reply.started": "2023-02-20T12:26:31.224810Z"
    },
    "papermill": {
     "duration": 13.272443,
     "end_time": "2023-02-20T12:33:33.038397",
     "exception": false,
     "start_time": "2023-02-20T12:33:19.765954",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14db1fdfdef0443c9430f99f9123a2c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/327 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe7cefa7a8444f81a1003172f66c7433",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/322k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ====================================================\n",
    "# tokenizer\n",
    "# ====================================================\n",
    "tokenizer = AutoTokenizer.from_pretrained(CFG.model)\n",
    "tokenizer.save_pretrained(OUTPUT_DIR+'tokenizer/')\n",
    "CFG.tokenizer = tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3adaca7d",
   "metadata": {
    "id": "14da40cf",
    "papermill": {
     "duration": 0.047171,
     "end_time": "2023-02-20T12:33:33.133025",
     "exception": false,
     "start_time": "2023-02-20T12:33:33.085854",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "67245562",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:33.294118Z",
     "iopub.status.busy": "2023-02-20T12:33:33.262791Z",
     "iopub.status.idle": "2023-02-20T12:33:38.528999Z",
     "shell.execute_reply": "2023-02-20T12:33:38.526297Z",
     "shell.execute_reply.started": "2023-02-20T12:26:43.073722Z"
    },
    "executionInfo": {
     "elapsed": 32827,
     "status": "ok",
     "timestamp": 1644920122500,
     "user": {
      "displayName": "Yasufumi Nakama",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "17486303986134302670"
     },
     "user_tz": -540
    },
    "id": "c00327b0",
    "outputId": "26e947da-b73a-494d-e776-906b037ac08a",
    "papermill": {
     "duration": 5.348439,
     "end_time": "2023-02-20T12:33:38.529163",
     "exception": false,
     "start_time": "2023-02-20T12:33:33.180724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec7b4f9f05f649c7a97dbcc7ade4b0a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/136 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51adf977b4774b9a9d1bcb0212a5eec8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/36473 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5f92b69534e4f28872123d0263f24a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/36473 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "max_len: 60\n"
     ]
    }
   ],
   "source": [
    "# ====================================================\n",
    "# Define max_len\n",
    "# ====================================================\n",
    "lengths_dict = {}\n",
    "\n",
    "lengths = []\n",
    "tk0 = tqdm(cpc_texts.values(), total=len(cpc_texts))\n",
    "for text in tk0:\n",
    "    length = len(tokenizer(text, add_special_tokens=False)['input_ids'])\n",
    "    lengths.append(length)\n",
    "lengths_dict['context_text'] = lengths\n",
    "\n",
    "for text_col in ['anchor', 'target']:\n",
    "    lengths = []\n",
    "    tk0 = tqdm(train[text_col].fillna(\"\").values, total=len(train))\n",
    "    for text in tk0:\n",
    "        length = len(tokenizer(text, add_special_tokens=False)['input_ids'])\n",
    "        lengths.append(length)\n",
    "    lengths_dict[text_col] = lengths\n",
    "    \n",
    "CFG.max_len = max(lengths_dict['anchor']) + max(lengths_dict['target']) + 3 + 3\n",
    "#                 + max(lengths_dict['context_text']) + 4 # CLS + SEP + SEP + SEP\n",
    "LOGGER.info(f\"max_len: {CFG.max_len}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "14fd62d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:38.638575Z",
     "iopub.status.busy": "2023-02-20T12:33:38.637230Z",
     "iopub.status.idle": "2023-02-20T12:33:38.639729Z",
     "shell.execute_reply": "2023-02-20T12:33:38.640232Z",
     "shell.execute_reply.started": "2023-02-20T12:27:21.542401Z"
    },
    "id": "9f791a19",
    "papermill": {
     "duration": 0.060999,
     "end_time": "2023-02-20T12:33:38.640435",
     "exception": false,
     "start_time": "2023-02-20T12:33:38.579436",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Dataset\n",
    "# ====================================================\n",
    "def prepare_input(cfg, text):\n",
    "    inputs = cfg.tokenizer(text,\n",
    "                           add_special_tokens=True,\n",
    "                           max_length=cfg.max_len,\n",
    "                           padding=\"max_length\",\n",
    "                           return_offsets_mapping=False)\n",
    "    for k, v in inputs.items():\n",
    "        inputs[k] = torch.tensor(v, dtype=torch.long)\n",
    "    return inputs\n",
    "\n",
    "\n",
    "class TrainDataset(Dataset):\n",
    "    def __init__(self, cfg, df):\n",
    "        self.cfg = cfg\n",
    "        self.texts = df['text'].values\n",
    "        self.labels = df['score'].values\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        inputs = prepare_input(self.cfg, self.texts[item])\n",
    "        label = torch.tensor(self.labels[item], dtype=torch.float)\n",
    "        return inputs, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ac36273c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:38.745290Z",
     "iopub.status.busy": "2023-02-20T12:33:38.744352Z",
     "iopub.status.idle": "2023-02-20T12:33:38.802558Z",
     "shell.execute_reply": "2023-02-20T12:33:38.803183Z",
     "shell.execute_reply.started": "2023-02-20T12:27:56.704781Z"
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1644920122808,
     "user": {
      "displayName": "Yasufumi Nakama",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "17486303986134302670"
     },
     "user_tz": -540
    },
    "id": "a200bd5b",
    "outputId": "a30fde2b-86f9-4ebd-ae81-33f467b69836",
    "papermill": {
     "duration": 0.113844,
     "end_time": "2023-02-20T12:33:38.803364",
     "exception": false,
     "start_time": "2023-02-20T12:33:38.689520",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([    2, 35736,     3, 35736,  1662, 10461,     3,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
      "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0]), 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}\n",
      "tensor(0.5000)\n"
     ]
    }
   ],
   "source": [
    "train_dataset = TrainDataset(CFG, train)\n",
    "inputs, label = train_dataset[0]\n",
    "print(inputs)\n",
    "print(label)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25f5142b",
   "metadata": {
    "id": "e04d6363",
    "papermill": {
     "duration": 0.050373,
     "end_time": "2023-02-20T12:33:38.904949",
     "exception": false,
     "start_time": "2023-02-20T12:33:38.854576",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "03ac0231",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:39.020839Z",
     "iopub.status.busy": "2023-02-20T12:33:39.019789Z",
     "iopub.status.idle": "2023-02-20T12:33:39.022109Z",
     "shell.execute_reply": "2023-02-20T12:33:39.022595Z",
     "shell.execute_reply.started": "2023-02-20T12:27:58.093526Z"
    },
    "id": "4c5bab44",
    "papermill": {
     "duration": 0.066878,
     "end_time": "2023-02-20T12:33:39.022793",
     "exception": false,
     "start_time": "2023-02-20T12:33:38.955915",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Model\n",
    "# ====================================================\n",
    "class CustomModel(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg\n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        if pretrained:\n",
    "            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "        else:\n",
    "            self.model = AutoModel.from_config(self.config)\n",
    "        self.fc_dropout = nn.Dropout(cfg.fc_dropout)\n",
    "        self.fc = nn.Linear(self.config.hidden_size, self.cfg.target_size)\n",
    "        self._init_weights(self.fc)\n",
    "        self.attention = nn.Sequential(\n",
    "            nn.Linear(self.config.hidden_size, 512),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(512, 1),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "        self._init_weights(self.attention)\n",
    "        \n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "        \n",
    "    def feature(self, inputs):\n",
    "        outputs = self.model(**inputs)\n",
    "        last_hidden_states = outputs[0]\n",
    "        # feature = torch.mean(last_hidden_states, 1)\n",
    "        weights = self.attention(last_hidden_states)\n",
    "        feature = torch.sum(weights * last_hidden_states, dim=1)\n",
    "        return feature\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        feature = self.feature(inputs)\n",
    "        output = self.fc(self.fc_dropout(feature))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea75f598",
   "metadata": {
    "id": "deee9675",
    "papermill": {
     "duration": 0.049672,
     "end_time": "2023-02-20T12:33:39.122696",
     "exception": false,
     "start_time": "2023-02-20T12:33:39.073024",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Helpler functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2b1ed405",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:39.253478Z",
     "iopub.status.busy": "2023-02-20T12:33:39.252356Z",
     "iopub.status.idle": "2023-02-20T12:33:39.254598Z",
     "shell.execute_reply": "2023-02-20T12:33:39.255269Z",
     "shell.execute_reply.started": "2023-02-20T12:28:04.554839Z"
    },
    "id": "c8263b0c",
    "papermill": {
     "duration": 0.08271,
     "end_time": "2023-02-20T12:33:39.255457",
     "exception": false,
     "start_time": "2023-02-20T12:33:39.172747",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Helper functions\n",
    "# ====================================================\n",
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "\n",
    "\n",
    "def asMinutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "\n",
    "def timeSince(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n",
    "\n",
    "\n",
    "def train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device):\n",
    "    model.train()\n",
    "    scaler = torch.cuda.amp.GradScaler(enabled=CFG.apex)\n",
    "    losses = AverageMeter()\n",
    "    start = end = time.time()\n",
    "    global_step = 0\n",
    "    for step, (inputs, labels) in enumerate(train_loader):\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        labels = labels.to(device)\n",
    "        batch_size = labels.size(0)\n",
    "        with torch.cuda.amp.autocast(enabled=CFG.apex):\n",
    "            y_preds = model(inputs)\n",
    "        loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))\n",
    "        if CFG.gradient_accumulation_steps > 1:\n",
    "            loss = loss / CFG.gradient_accumulation_steps\n",
    "        losses.update(loss.item(), batch_size)\n",
    "        scaler.scale(loss).backward()\n",
    "        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n",
    "        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n",
    "            scaler.step(optimizer)\n",
    "            scaler.update()\n",
    "            optimizer.zero_grad()\n",
    "            global_step += 1\n",
    "            if CFG.batch_scheduler:\n",
    "                scheduler.step()\n",
    "        end = time.time()\n",
    "        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n",
    "            print('Epoch: [{0}][{1}/{2}] '\n",
    "                  'Elapsed {remain:s} '\n",
    "                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n",
    "                  'Grad: {grad_norm:.4f}  '\n",
    "                  'LR: {lr:.8f}  '\n",
    "                  .format(epoch+1, step, len(train_loader), \n",
    "                          remain=timeSince(start, float(step+1)/len(train_loader)),\n",
    "                          loss=losses,\n",
    "                          grad_norm=grad_norm,\n",
    "                          lr=scheduler.get_lr()[0]))\n",
    "        if CFG.wandb:\n",
    "            wandb.log({f\"[fold{fold}] loss\": losses.val,\n",
    "                       f\"[fold{fold}] lr\": scheduler.get_lr()[0]})\n",
    "    return losses.avg\n",
    "\n",
    "\n",
    "def valid_fn(valid_loader, model, criterion, device):\n",
    "    losses = AverageMeter()\n",
    "    model.eval()\n",
    "    preds = []\n",
    "    start = end = time.time()\n",
    "    for step, (inputs, labels) in enumerate(valid_loader):\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        labels = labels.to(device)\n",
    "        batch_size = labels.size(0)\n",
    "        with torch.no_grad():\n",
    "            y_preds = model(inputs)\n",
    "        loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))\n",
    "        if CFG.gradient_accumulation_steps > 1:\n",
    "            loss = loss / CFG.gradient_accumulation_steps\n",
    "        losses.update(loss.item(), batch_size)\n",
    "        preds.append(y_preds.sigmoid().to('cpu').numpy())\n",
    "        end = time.time()\n",
    "        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n",
    "            print('EVAL: [{0}/{1}] '\n",
    "                  'Elapsed {remain:s} '\n",
    "                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n",
    "                  .format(step, len(valid_loader),\n",
    "                          loss=losses,\n",
    "                          remain=timeSince(start, float(step+1)/len(valid_loader))))\n",
    "    predictions = np.concatenate(preds)\n",
    "    predictions = np.concatenate(predictions)\n",
    "    return losses.avg, predictions\n",
    "\n",
    "\n",
    "def inference_fn(test_loader, model, device):\n",
    "    preds = []\n",
    "    model.eval()\n",
    "    model.to(device)\n",
    "    tk0 = tqdm(test_loader, total=len(test_loader))\n",
    "    for inputs in tk0:\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        with torch.no_grad():\n",
    "            y_preds = model(inputs)\n",
    "        preds.append(y_preds.sigmoid().to('cpu').numpy())\n",
    "    predictions = np.concatenate(preds)\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3500e617",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:39.360152Z",
     "iopub.status.busy": "2023-02-20T12:33:39.359100Z",
     "iopub.status.idle": "2023-02-20T12:33:39.381473Z",
     "shell.execute_reply": "2023-02-20T12:33:39.380862Z",
     "shell.execute_reply.started": "2023-02-20T12:28:06.147447Z"
    },
    "id": "bed940e1",
    "papermill": {
     "duration": 0.076211,
     "end_time": "2023-02-20T12:33:39.381617",
     "exception": false,
     "start_time": "2023-02-20T12:33:39.305406",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# train loop\n",
    "# ====================================================\n",
    "def train_loop(folds, fold):\n",
    "    \n",
    "    LOGGER.info(f\"========== fold: {fold} training ==========\")\n",
    "\n",
    "    # ====================================================\n",
    "    # loader\n",
    "    # ====================================================\n",
    "    train_folds = folds[folds['fold'] != fold].reset_index(drop=True)\n",
    "    valid_folds = folds[folds['fold'] == fold].reset_index(drop=True)\n",
    "    valid_labels = valid_folds['score'].values\n",
    "    \n",
    "    train_dataset = TrainDataset(CFG, train_folds)\n",
    "    valid_dataset = TrainDataset(CFG, valid_folds)\n",
    "\n",
    "    train_loader = DataLoader(train_dataset,\n",
    "                              batch_size=CFG.batch_size,\n",
    "                              shuffle=True,\n",
    "                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n",
    "    valid_loader = DataLoader(valid_dataset,\n",
    "                              batch_size=CFG.batch_size,\n",
    "                              shuffle=False,\n",
    "                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n",
    "\n",
    "    # ====================================================\n",
    "    # model & optimizer\n",
    "    # ====================================================\n",
    "    model = CustomModel(CFG, config_path=None, pretrained=True)\n",
    "    torch.save(model.config, OUTPUT_DIR+'config.pth')\n",
    "    model.to(device)\n",
    "    \n",
    "    def get_optimizer_params(model, encoder_lr, decoder_lr, weight_decay=0.0):\n",
    "        param_optimizer = list(model.named_parameters())\n",
    "        no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n",
    "        optimizer_parameters = [\n",
    "            {'params': [p for n, p in model.model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "             'lr': encoder_lr, 'weight_decay': weight_decay},\n",
    "            {'params': [p for n, p in model.model.named_parameters() if any(nd in n for nd in no_decay)],\n",
    "             'lr': encoder_lr, 'weight_decay': 0.0},\n",
    "            {'params': [p for n, p in model.named_parameters() if \"model\" not in n],\n",
    "             'lr': decoder_lr, 'weight_decay': 0.0}\n",
    "        ]\n",
    "        return optimizer_parameters\n",
    "\n",
    "    optimizer_parameters = get_optimizer_params(model,\n",
    "                                                encoder_lr=CFG.encoder_lr, \n",
    "                                                decoder_lr=CFG.decoder_lr,\n",
    "                                                weight_decay=CFG.weight_decay)\n",
    "    optimizer = AdamW(optimizer_parameters, lr=CFG.encoder_lr, eps=CFG.eps, betas=CFG.betas)\n",
    "    \n",
    "    # ====================================================\n",
    "    # scheduler\n",
    "    # ====================================================\n",
    "    def get_scheduler(cfg, optimizer, num_train_steps):\n",
    "        if cfg.scheduler == 'linear':\n",
    "            scheduler = get_linear_schedule_with_warmup(\n",
    "                optimizer, num_warmup_steps=cfg.num_warmup_steps, num_training_steps=num_train_steps\n",
    "            )\n",
    "        elif cfg.scheduler == 'cosine':\n",
    "            scheduler = get_cosine_schedule_with_warmup(\n",
    "                optimizer, num_warmup_steps=cfg.num_warmup_steps, num_training_steps=num_train_steps, num_cycles=cfg.num_cycles\n",
    "            )\n",
    "        return scheduler\n",
    "    \n",
    "    num_train_steps = int(len(train_folds) / CFG.batch_size * CFG.epochs)\n",
    "    scheduler = get_scheduler(CFG, optimizer, num_train_steps)\n",
    "\n",
    "    # ====================================================\n",
    "    # loop\n",
    "    # ====================================================\n",
    "    criterion = nn.BCEWithLogitsLoss(reduction=\"mean\")\n",
    "    \n",
    "    best_score = 0.\n",
    "\n",
    "    for epoch in range(CFG.epochs):\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "        # train\n",
    "        avg_loss = train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device)\n",
    "\n",
    "        # eval\n",
    "        avg_val_loss, predictions = valid_fn(valid_loader, model, criterion, device)\n",
    "        \n",
    "        # scoring\n",
    "        score = get_score(valid_labels, predictions)\n",
    "\n",
    "        elapsed = time.time() - start_time\n",
    "\n",
    "        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n",
    "        LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}')\n",
    "        if CFG.wandb:\n",
    "            wandb.log({f\"[fold{fold}] epoch\": epoch+1, \n",
    "                       f\"[fold{fold}] avg_train_loss\": avg_loss, \n",
    "                       f\"[fold{fold}] avg_val_loss\": avg_val_loss,\n",
    "                       f\"[fold{fold}] score\": score})\n",
    "        \n",
    "        if best_score < score:\n",
    "            best_score = score\n",
    "            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n",
    "            torch.save({'model': model.state_dict(),\n",
    "                        'predictions': predictions},\n",
    "                        OUTPUT_DIR+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\")\n",
    "\n",
    "    predictions = torch.load(OUTPUT_DIR+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\", \n",
    "                             map_location=torch.device('cpu'))['predictions']\n",
    "    valid_folds['pred'] = predictions\n",
    "\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "    \n",
    "    return valid_folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3800f7a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-20T12:33:39.486065Z",
     "iopub.status.busy": "2023-02-20T12:33:39.485051Z",
     "iopub.status.idle": "2023-02-20T17:46:56.310512Z",
     "shell.execute_reply": "2023-02-20T17:46:56.311899Z"
    },
    "id": "6cc76b1e",
    "papermill": {
     "duration": 18796.880771,
     "end_time": "2023-02-20T17:46:56.312274",
     "exception": false,
     "start_time": "2023-02-20T12:33:39.431503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== fold: 0 training ==========\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb9fd523a0ec48b48b0b1a4863d8ed03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.29G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at anferico/bert-for-patents were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [1][0/1709] Elapsed 0m 1s (remain 47m 3s) Loss: 0.7587(0.7587) Grad: inf  LR: 0.00002000  \n",
      "Epoch: [1][100/1709] Elapsed 0m 43s (remain 11m 37s) Loss: 0.6720(0.6628) Grad: 113597.6641  LR: 0.00002000  \n",
      "Epoch: [1][200/1709] Elapsed 1m 26s (remain 10m 45s) Loss: 0.6720(0.6408) Grad: 111407.9766  LR: 0.00001998  \n",
      "Epoch: [1][300/1709] Elapsed 2m 8s (remain 9m 59s) Loss: 0.6199(0.6249) Grad: 146739.5156  LR: 0.00001996  \n",
      "Epoch: [1][400/1709] Elapsed 2m 50s (remain 9m 15s) Loss: 0.6213(0.6142) Grad: 163034.7031  LR: 0.00001992  \n",
      "Epoch: [1][500/1709] Elapsed 3m 32s (remain 8m 32s) Loss: 0.6247(0.6063) Grad: 143194.1562  LR: 0.00001988  \n",
      "Epoch: [1][600/1709] Elapsed 4m 14s (remain 7m 49s) Loss: 0.4041(0.5987) Grad: 106071.7188  LR: 0.00001983  \n",
      "Epoch: [1][700/1709] Elapsed 4m 56s (remain 7m 6s) Loss: 0.6063(0.5942) Grad: 100318.4062  LR: 0.00001977  \n",
      "Epoch: [1][800/1709] Elapsed 5m 38s (remain 6m 23s) Loss: 0.5589(0.5894) Grad: 156251.4688  LR: 0.00001970  \n",
      "Epoch: [1][900/1709] Elapsed 6m 19s (remain 5m 40s) Loss: 0.5096(0.5875) Grad: 62403.8281  LR: 0.00001962  \n",
      "Epoch: [1][1000/1709] Elapsed 7m 1s (remain 4m 58s) Loss: 0.5897(0.5853) Grad: 47498.7461  LR: 0.00001953  \n",
      "Epoch: [1][1100/1709] Elapsed 7m 43s (remain 4m 15s) Loss: 0.4861(0.5834) Grad: 34347.1641  LR: 0.00001944  \n",
      "Epoch: [1][1200/1709] Elapsed 8m 25s (remain 3m 33s) Loss: 0.3607(0.5822) Grad: 55740.8594  LR: 0.00001933  \n",
      "Epoch: [1][1300/1709] Elapsed 9m 6s (remain 2m 51s) Loss: 0.5599(0.5805) Grad: 39788.9336  LR: 0.00001922  \n",
      "Epoch: [1][1400/1709] Elapsed 9m 48s (remain 2m 9s) Loss: 0.4342(0.5786) Grad: 33709.1953  LR: 0.00001909  \n",
      "Epoch: [1][1500/1709] Elapsed 10m 30s (remain 1m 27s) Loss: 0.5755(0.5774) Grad: 26676.2637  LR: 0.00001896  \n",
      "Epoch: [1][1600/1709] Elapsed 11m 12s (remain 0m 45s) Loss: 0.6032(0.5757) Grad: 63612.1016  LR: 0.00001882  \n",
      "Epoch: [1][1700/1709] Elapsed 11m 53s (remain 0m 3s) Loss: 0.4085(0.5746) Grad: 23702.0645  LR: 0.00001867  \n",
      "Epoch: [1][1708/1709] Elapsed 11m 57s (remain 0m 0s) Loss: 0.5084(0.5746) Grad: 55299.9297  LR: 0.00001866  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 2m 57s) Loss: 0.5365(0.5365) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.4697(0.5540) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5930(0.5514) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.6810(0.5525) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.6734(0.5519) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5538(0.5522) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1 - avg_train_loss: 0.5746  avg_val_loss: 0.5509  time: 779s\n",
      "Epoch 1 - Score: 0.8202\n",
      "Epoch 1 - Save Best Score: 0.8202 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5109(0.5509) \n",
      "Epoch: [2][0/1709] Elapsed 0m 0s (remain 23m 17s) Loss: 0.6244(0.6244) Grad: 166188.5156  LR: 0.00001866  \n",
      "Epoch: [2][100/1709] Elapsed 0m 42s (remain 11m 19s) Loss: 0.5647(0.5335) Grad: 184128.6719  LR: 0.00001850  \n",
      "Epoch: [2][200/1709] Elapsed 1m 24s (remain 10m 33s) Loss: 0.5958(0.5347) Grad: 305677.9375  LR: 0.00001834  \n",
      "Epoch: [2][300/1709] Elapsed 2m 6s (remain 9m 49s) Loss: 0.5049(0.5315) Grad: 175323.2344  LR: 0.00001816  \n",
      "Epoch: [2][400/1709] Elapsed 2m 47s (remain 9m 7s) Loss: 0.5529(0.5316) Grad: 52264.7812  LR: 0.00001798  \n",
      "Epoch: [2][500/1709] Elapsed 3m 29s (remain 8m 25s) Loss: 0.3834(0.5319) Grad: 122630.8984  LR: 0.00001780  \n",
      "Epoch: [2][600/1709] Elapsed 4m 11s (remain 7m 42s) Loss: 0.3968(0.5312) Grad: 60987.0859  LR: 0.00001760  \n",
      "Epoch: [2][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5748(0.5312) Grad: 92694.1172  LR: 0.00001740  \n",
      "Epoch: [2][800/1709] Elapsed 5m 34s (remain 6m 19s) Loss: 0.4460(0.5305) Grad: 35167.7383  LR: 0.00001719  \n",
      "Epoch: [2][900/1709] Elapsed 6m 16s (remain 5m 37s) Loss: 0.4572(0.5298) Grad: 50594.0469  LR: 0.00001697  \n",
      "Epoch: [2][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4328(0.5297) Grad: 100845.6953  LR: 0.00001675  \n",
      "Epoch: [2][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.4658(0.5292) Grad: 53562.0039  LR: 0.00001652  \n",
      "Epoch: [2][1200/1709] Elapsed 8m 21s (remain 3m 31s) Loss: 0.4240(0.5292) Grad: 67023.3047  LR: 0.00001628  \n",
      "Epoch: [2][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5115(0.5293) Grad: 197731.8125  LR: 0.00001604  \n",
      "Epoch: [2][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.4197(0.5282) Grad: 19878.0488  LR: 0.00001580  \n",
      "Epoch: [2][1500/1709] Elapsed 10m 26s (remain 1m 26s) Loss: 0.4167(0.5278) Grad: 38143.9648  LR: 0.00001554  \n",
      "Epoch: [2][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.6039(0.5284) Grad: 28150.2715  LR: 0.00001529  \n",
      "Epoch: [2][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.5060(0.5280) Grad: 17088.2109  LR: 0.00001502  \n",
      "Epoch: [2][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.4658(0.5278) Grad: 17814.2754  LR: 0.00001500  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 2m 37s) Loss: 0.5252(0.5252) \n",
      "EVAL: [100/570] Elapsed 0m 10s (remain 0m 50s) Loss: 0.5008(0.5582) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5504(0.5510) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.8654(0.5497) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.6612(0.5484) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5398(0.5484) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2 - avg_train_loss: 0.5278  avg_val_loss: 0.5467  time: 774s\n",
      "Epoch 2 - Score: 0.8320\n",
      "Epoch 2 - Save Best Score: 0.8320 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4959(0.5467) \n",
      "Epoch: [3][0/1709] Elapsed 0m 0s (remain 18m 17s) Loss: 0.4797(0.4797) Grad: 85980.1406  LR: 0.00001500  \n",
      "Epoch: [3][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.5646(0.5122) Grad: 151112.6719  LR: 0.00001473  \n",
      "Epoch: [3][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.4939(0.5123) Grad: 112119.2734  LR: 0.00001446  \n",
      "Epoch: [3][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.5467(0.5139) Grad: 53800.1367  LR: 0.00001418  \n",
      "Epoch: [3][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.6054(0.5149) Grad: 70254.4844  LR: 0.00001390  \n",
      "Epoch: [3][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5154(0.5131) Grad: 65851.9766  LR: 0.00001362  \n",
      "Epoch: [3][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.5993(0.5121) Grad: 102580.2188  LR: 0.00001333  \n",
      "Epoch: [3][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5397(0.5097) Grad: 93915.1484  LR: 0.00001304  \n",
      "Epoch: [3][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.5173(0.5094) Grad: 82900.9609  LR: 0.00001275  \n",
      "Epoch: [3][900/1709] Elapsed 6m 15s (remain 5m 37s) Loss: 0.5048(0.5089) Grad: 102091.4844  LR: 0.00001245  \n",
      "Epoch: [3][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5231(0.5095) Grad: 44217.7617  LR: 0.00001216  \n",
      "Epoch: [3][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.4593(0.5085) Grad: 88968.0859  LR: 0.00001186  \n",
      "Epoch: [3][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4830(0.5091) Grad: 63373.3945  LR: 0.00001155  \n",
      "Epoch: [3][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5528(0.5093) Grad: 79405.2578  LR: 0.00001125  \n",
      "Epoch: [3][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.4651(0.5087) Grad: 77660.4844  LR: 0.00001095  \n",
      "Epoch: [3][1500/1709] Elapsed 10m 26s (remain 1m 26s) Loss: 0.6167(0.5091) Grad: 78633.5859  LR: 0.00001064  \n",
      "Epoch: [3][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.5138(0.5097) Grad: 55790.3594  LR: 0.00001034  \n",
      "Epoch: [3][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.4702(0.5095) Grad: 62146.4766  LR: 0.00001003  \n",
      "Epoch: [3][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.4663(0.5095) Grad: 56529.0273  LR: 0.00001000  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 2m 49s) Loss: 0.5242(0.5242) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.4410(0.5621) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5670(0.5544) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.8031(0.5520) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.6861(0.5505) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5332(0.5503) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3 - avg_train_loss: 0.5095  avg_val_loss: 0.5486  time: 774s\n",
      "Epoch 3 - Score: 0.8470\n",
      "Epoch 3 - Save Best Score: 0.8470 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4922(0.5486) \n",
      "Epoch: [4][0/1709] Elapsed 0m 0s (remain 18m 14s) Loss: 0.5017(0.5017) Grad: 90584.3906  LR: 0.00001000  \n",
      "Epoch: [4][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.4213(0.5038) Grad: 325444.5312  LR: 0.00000970  \n",
      "Epoch: [4][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.5225(0.5059) Grad: 53326.7383  LR: 0.00000939  \n",
      "Epoch: [4][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.5623(0.5026) Grad: 40436.5312  LR: 0.00000908  \n",
      "Epoch: [4][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.5831(0.5014) Grad: 50749.3945  LR: 0.00000878  \n",
      "Epoch: [4][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.4374(0.5003) Grad: 92106.4844  LR: 0.00000848  \n",
      "Epoch: [4][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.4005(0.5014) Grad: 41977.6016  LR: 0.00000817  \n",
      "Epoch: [4][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4408(0.5014) Grad: 34137.1875  LR: 0.00000787  \n",
      "Epoch: [4][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.4794(0.5013) Grad: 30610.8359  LR: 0.00000758  \n",
      "Epoch: [4][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.6330(0.5012) Grad: 66221.6719  LR: 0.00000728  \n",
      "Epoch: [4][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5938(0.5007) Grad: 68689.5469  LR: 0.00000699  \n",
      "Epoch: [4][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.4767(0.5001) Grad: 72006.4609  LR: 0.00000670  \n",
      "Epoch: [4][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4642(0.5002) Grad: 38263.8281  LR: 0.00000641  \n",
      "Epoch: [4][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5018(0.5008) Grad: 121562.3828  LR: 0.00000612  \n",
      "Epoch: [4][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5747(0.5001) Grad: 67863.1484  LR: 0.00000584  \n",
      "Epoch: [4][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.4912(0.4997) Grad: 115477.8203  LR: 0.00000557  \n",
      "Epoch: [4][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.3486(0.5002) Grad: 40178.8633  LR: 0.00000529  \n",
      "Epoch: [4][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.4432(0.4998) Grad: 99325.0938  LR: 0.00000503  \n",
      "Epoch: [4][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5506(0.4997) Grad: 68820.7578  LR: 0.00000501  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 2m 49s) Loss: 0.5483(0.5483) \n",
      "EVAL: [100/570] Elapsed 0m 10s (remain 0m 50s) Loss: 0.4424(0.5650) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5539(0.5577) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.7961(0.5552) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.7005(0.5537) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5313(0.5530) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4 - avg_train_loss: 0.4997  avg_val_loss: 0.5514  time: 774s\n",
      "Epoch 4 - Score: 0.8481\n",
      "Epoch 4 - Save Best Score: 0.8481 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5125(0.5514) \n",
      "Epoch: [5][0/1709] Elapsed 0m 0s (remain 18m 8s) Loss: 0.5079(0.5079) Grad: 55949.2305  LR: 0.00000500  \n",
      "Epoch: [5][100/1709] Elapsed 0m 42s (remain 11m 13s) Loss: 0.4866(0.4924) Grad: 69200.8047  LR: 0.00000474  \n",
      "Epoch: [5][200/1709] Elapsed 1m 23s (remain 10m 29s) Loss: 0.5556(0.4981) Grad: 47441.8672  LR: 0.00000448  \n",
      "Epoch: [5][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.5481(0.5016) Grad: 38334.0586  LR: 0.00000423  \n",
      "Epoch: [5][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.5365(0.5001) Grad: 41723.5273  LR: 0.00000398  \n",
      "Epoch: [5][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.4418(0.4962) Grad: 33791.5000  LR: 0.00000374  \n",
      "Epoch: [5][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.4622(0.4955) Grad: 50348.0117  LR: 0.00000350  \n",
      "Epoch: [5][700/1709] Elapsed 4m 52s (remain 6m 59s) Loss: 0.5334(0.4960) Grad: 37684.8164  LR: 0.00000327  \n",
      "Epoch: [5][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.4385(0.4953) Grad: 26867.9453  LR: 0.00000305  \n",
      "Epoch: [5][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.5279(0.4947) Grad: 120638.3125  LR: 0.00000283  \n",
      "Epoch: [5][1000/1709] Elapsed 6m 57s (remain 4m 54s) Loss: 0.4694(0.4937) Grad: 28454.6992  LR: 0.00000262  \n",
      "Epoch: [5][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5557(0.4937) Grad: 16092.0127  LR: 0.00000242  \n",
      "Epoch: [5][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.3910(0.4937) Grad: 27214.5605  LR: 0.00000222  \n",
      "Epoch: [5][1300/1709] Elapsed 9m 1s (remain 2m 49s) Loss: 0.4042(0.4939) Grad: 27370.5391  LR: 0.00000204  \n",
      "Epoch: [5][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.4059(0.4946) Grad: 19627.8047  LR: 0.00000185  \n",
      "Epoch: [5][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.3844(0.4952) Grad: 50304.2383  LR: 0.00000168  \n",
      "Epoch: [5][1600/1709] Elapsed 11m 6s (remain 0m 44s) Loss: 0.4440(0.4945) Grad: 15996.2930  LR: 0.00000151  \n",
      "Epoch: [5][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.4795(0.4940) Grad: 26596.8984  LR: 0.00000136  \n",
      "Epoch: [5][1708/1709] Elapsed 11m 51s (remain 0m 0s) Loss: 0.4611(0.4939) Grad: 45813.7227  LR: 0.00000134  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 12s) Loss: 0.5173(0.5173) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.4428(0.5763) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5809(0.5655) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.8409(0.5622) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.7083(0.5607) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5335(0.5592) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5 - avg_train_loss: 0.4939  avg_val_loss: 0.5578  time: 773s\n",
      "Epoch 5 - Score: 0.8487\n",
      "Epoch 5 - Save Best Score: 0.8487 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5121(0.5578) \n",
      "Epoch: [6][0/1709] Elapsed 0m 0s (remain 18m 54s) Loss: 0.5486(0.5486) Grad: 47265.6094  LR: 0.00000134  \n",
      "Epoch: [6][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.5440(0.4884) Grad: 49288.6758  LR: 0.00000119  \n",
      "Epoch: [6][200/1709] Elapsed 1m 23s (remain 10m 29s) Loss: 0.4612(0.4926) Grad: 35721.3828  LR: 0.00000105  \n",
      "Epoch: [6][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.3523(0.4942) Grad: 67277.0938  LR: 0.00000092  \n",
      "Epoch: [6][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.3956(0.4899) Grad: 22261.3359  LR: 0.00000080  \n",
      "Epoch: [6][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.3903(0.4891) Grad: 66745.8828  LR: 0.00000068  \n",
      "Epoch: [6][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.5676(0.4899) Grad: 70179.1641  LR: 0.00000057  \n",
      "Epoch: [6][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4946(0.4895) Grad: 30306.1602  LR: 0.00000048  \n",
      "Epoch: [6][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.5007(0.4891) Grad: 46985.9570  LR: 0.00000039  \n",
      "Epoch: [6][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4100(0.4895) Grad: 110822.8828  LR: 0.00000031  \n",
      "Epoch: [6][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5535(0.4891) Grad: 496962.5625  LR: 0.00000024  \n",
      "Epoch: [6][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5399(0.4891) Grad: 40992.6172  LR: 0.00000017  \n",
      "Epoch: [6][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.5189(0.4889) Grad: 55021.9219  LR: 0.00000012  \n",
      "Epoch: [6][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4166(0.4895) Grad: 415181.8125  LR: 0.00000008  \n",
      "Epoch: [6][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.3915(0.4900) Grad: 14044.2451  LR: 0.00000005  \n",
      "Epoch: [6][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.3756(0.4909) Grad: 21036.8418  LR: 0.00000002  \n",
      "Epoch: [6][1600/1709] Elapsed 11m 7s (remain 0m 44s) Loss: 0.4008(0.4913) Grad: 18265.4863  LR: 0.00000001  \n",
      "Epoch: [6][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.4977(0.4908) Grad: 26102.7422  LR: 0.00000000  \n",
      "Epoch: [6][1708/1709] Elapsed 11m 51s (remain 0m 0s) Loss: 0.4313(0.4906) Grad: 29760.9805  LR: 0.00000000  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 2m 34s) Loss: 0.5178(0.5178) \n",
      "EVAL: [100/570] Elapsed 0m 10s (remain 0m 50s) Loss: 0.4439(0.5761) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5737(0.5650) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.8339(0.5618) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.7108(0.5603) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5300(0.5589) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6 - avg_train_loss: 0.4906  avg_val_loss: 0.5576  time: 773s\n",
      "Epoch 6 - Score: 0.8486\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5090(0.5576) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "========== fold: 0 result ==========\n",
      "Score: 0.8487\n",
      "========== fold: 1 training ==========\n",
      "Some weights of the model checkpoint at anferico/bert-for-patents were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [1][0/1709] Elapsed 0m 0s (remain 16m 55s) Loss: 0.8361(0.8361) Grad: inf  LR: 0.00002000  \n",
      "Epoch: [1][100/1709] Elapsed 0m 42s (remain 11m 11s) Loss: 0.5954(0.6567) Grad: 66789.2344  LR: 0.00002000  \n",
      "Epoch: [1][200/1709] Elapsed 1m 23s (remain 10m 28s) Loss: 0.5744(0.6282) Grad: 58133.5742  LR: 0.00001998  \n",
      "Epoch: [1][300/1709] Elapsed 2m 5s (remain 9m 46s) Loss: 0.5989(0.6132) Grad: 51433.4727  LR: 0.00001996  \n",
      "Epoch: [1][400/1709] Elapsed 2m 47s (remain 9m 4s) Loss: 0.4097(0.6047) Grad: 39034.0781  LR: 0.00001992  \n",
      "Epoch: [1][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.5356(0.5986) Grad: 26558.6562  LR: 0.00001988  \n",
      "Epoch: [1][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.5878(0.5940) Grad: 57345.6523  LR: 0.00001983  \n",
      "Epoch: [1][700/1709] Elapsed 4m 51s (remain 6m 59s) Loss: 0.6073(0.5910) Grad: 28287.8828  LR: 0.00001977  \n",
      "Epoch: [1][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.4757(0.5878) Grad: 25396.8926  LR: 0.00001970  \n",
      "Epoch: [1][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.5601(0.5849) Grad: 45153.6992  LR: 0.00001962  \n",
      "Epoch: [1][1000/1709] Elapsed 6m 56s (remain 4m 54s) Loss: 0.4726(0.5830) Grad: 21008.8848  LR: 0.00001953  \n",
      "Epoch: [1][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5571(0.5809) Grad: 25240.5371  LR: 0.00001944  \n",
      "Epoch: [1][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4679(0.5784) Grad: 72696.6484  LR: 0.00001933  \n",
      "Epoch: [1][1300/1709] Elapsed 9m 1s (remain 2m 49s) Loss: 0.5207(0.5757) Grad: 35195.5117  LR: 0.00001922  \n",
      "Epoch: [1][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.5196(0.5738) Grad: 39402.1523  LR: 0.00001909  \n",
      "Epoch: [1][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5725(0.5728) Grad: 39853.3984  LR: 0.00001896  \n",
      "Epoch: [1][1600/1709] Elapsed 11m 6s (remain 0m 44s) Loss: 0.4522(0.5718) Grad: 31947.2832  LR: 0.00001882  \n",
      "Epoch: [1][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.5677(0.5711) Grad: 55883.8828  LR: 0.00001867  \n",
      "Epoch: [1][1708/1709] Elapsed 11m 51s (remain 0m 0s) Loss: 0.4871(0.5712) Grad: 41306.6719  LR: 0.00001866  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 45s) Loss: 0.5367(0.5367) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.6429(0.5700) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.5002(0.5691) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.6095(0.5688) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.5658(0.5695) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4853(0.5671) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1 - avg_train_loss: 0.5712  avg_val_loss: 0.5664  time: 774s\n",
      "Epoch 1 - Score: 0.8157\n",
      "Epoch 1 - Save Best Score: 0.8157 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.3951(0.5664) \n",
      "Epoch: [2][0/1709] Elapsed 0m 1s (remain 30m 39s) Loss: 0.5669(0.5669) Grad: 98820.7031  LR: 0.00001866  \n",
      "Epoch: [2][100/1709] Elapsed 0m 42s (remain 11m 20s) Loss: 0.6171(0.5262) Grad: 69336.1328  LR: 0.00001850  \n",
      "Epoch: [2][200/1709] Elapsed 1m 24s (remain 10m 33s) Loss: 0.4144(0.5320) Grad: 39163.6328  LR: 0.00001834  \n",
      "Epoch: [2][300/1709] Elapsed 2m 6s (remain 9m 49s) Loss: 0.5588(0.5265) Grad: 53864.1484  LR: 0.00001816  \n",
      "Epoch: [2][400/1709] Elapsed 2m 47s (remain 9m 7s) Loss: 0.4251(0.5267) Grad: 44774.3594  LR: 0.00001798  \n",
      "Epoch: [2][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5585(0.5263) Grad: 57923.4766  LR: 0.00001780  \n",
      "Epoch: [2][600/1709] Elapsed 4m 11s (remain 7m 42s) Loss: 0.5203(0.5264) Grad: 37675.6289  LR: 0.00001760  \n",
      "Epoch: [2][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5159(0.5251) Grad: 76263.2500  LR: 0.00001740  \n",
      "Epoch: [2][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.6090(0.5244) Grad: 87746.6641  LR: 0.00001719  \n",
      "Epoch: [2][900/1709] Elapsed 6m 16s (remain 5m 37s) Loss: 0.6026(0.5249) Grad: 75018.1719  LR: 0.00001697  \n",
      "Epoch: [2][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5301(0.5259) Grad: 23698.2598  LR: 0.00001675  \n",
      "Epoch: [2][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5959(0.5250) Grad: 66336.9766  LR: 0.00001652  \n",
      "Epoch: [2][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.5405(0.5256) Grad: 45407.7227  LR: 0.00001628  \n",
      "Epoch: [2][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5141(0.5253) Grad: 28630.3125  LR: 0.00001604  \n",
      "Epoch: [2][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5908(0.5248) Grad: 202628.9062  LR: 0.00001580  \n",
      "Epoch: [2][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5239(0.5248) Grad: 74581.6328  LR: 0.00001554  \n",
      "Epoch: [2][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.3403(0.5249) Grad: 12268.5479  LR: 0.00001529  \n",
      "Epoch: [2][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.5337(0.5243) Grad: 60770.6289  LR: 0.00001502  \n",
      "Epoch: [2][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5683(0.5243) Grad: 29790.1875  LR: 0.00001500  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 23s) Loss: 0.4800(0.4800) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5668(0.5514) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4022(0.5569) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.6107(0.5555) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4834(0.5567) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4264(0.5546) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2 - avg_train_loss: 0.5243  avg_val_loss: 0.5550  time: 774s\n",
      "Epoch 2 - Score: 0.8370\n",
      "Epoch 2 - Save Best Score: 0.8370 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4786(0.5550) \n",
      "Epoch: [3][0/1709] Elapsed 0m 0s (remain 20m 43s) Loss: 0.5934(0.5934) Grad: 65158.2578  LR: 0.00001500  \n",
      "Epoch: [3][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.5878(0.5070) Grad: 29439.3984  LR: 0.00001473  \n",
      "Epoch: [3][200/1709] Elapsed 1m 23s (remain 10m 29s) Loss: 0.4127(0.5086) Grad: 47190.0312  LR: 0.00001446  \n",
      "Epoch: [3][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.4483(0.5062) Grad: 45402.8945  LR: 0.00001419  \n",
      "Epoch: [3][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.6453(0.5080) Grad: 55214.6836  LR: 0.00001391  \n",
      "Epoch: [3][500/1709] Elapsed 3m 29s (remain 8m 23s) Loss: 0.5375(0.5067) Grad: 85388.1406  LR: 0.00001362  \n",
      "Epoch: [3][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.6278(0.5081) Grad: 39346.6797  LR: 0.00001333  \n",
      "Epoch: [3][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4220(0.5081) Grad: 30028.2520  LR: 0.00001304  \n",
      "Epoch: [3][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.5557(0.5080) Grad: 23685.2988  LR: 0.00001275  \n",
      "Epoch: [3][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.5679(0.5071) Grad: 60224.8164  LR: 0.00001246  \n",
      "Epoch: [3][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.3803(0.5073) Grad: 94141.7891  LR: 0.00001216  \n",
      "Epoch: [3][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.7788(0.5077) Grad: 107565.8125  LR: 0.00001186  \n",
      "Epoch: [3][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4459(0.5080) Grad: 49181.0000  LR: 0.00001156  \n",
      "Epoch: [3][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5812(0.5080) Grad: 59319.6641  LR: 0.00001125  \n",
      "Epoch: [3][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.5546(0.5080) Grad: 96482.7109  LR: 0.00001095  \n",
      "Epoch: [3][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.4263(0.5084) Grad: 24746.1074  LR: 0.00001064  \n",
      "Epoch: [3][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.4891(0.5083) Grad: 25322.6738  LR: 0.00001034  \n",
      "Epoch: [3][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.6111(0.5084) Grad: 23406.3789  LR: 0.00001003  \n",
      "Epoch: [3][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.4969(0.5086) Grad: 59817.2344  LR: 0.00001001  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 23s) Loss: 0.4796(0.4796) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5655(0.5417) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.4304(0.5473) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.5793(0.5471) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4987(0.5470) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4394(0.5456) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3 - avg_train_loss: 0.5086  avg_val_loss: 0.5443  time: 774s\n",
      "Epoch 3 - Score: 0.8476\n",
      "Epoch 3 - Save Best Score: 0.8476 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.3857(0.5443) \n",
      "Epoch: [4][0/1709] Elapsed 0m 0s (remain 21m 6s) Loss: 0.5487(0.5487) Grad: 39539.0742  LR: 0.00001000  \n",
      "Epoch: [4][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.4652(0.4958) Grad: 244394.7344  LR: 0.00000970  \n",
      "Epoch: [4][200/1709] Elapsed 1m 24s (remain 10m 31s) Loss: 0.5600(0.4966) Grad: 41737.7148  LR: 0.00000939  \n",
      "Epoch: [4][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.4620(0.4971) Grad: 60115.8984  LR: 0.00000909  \n",
      "Epoch: [4][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.4666(0.5010) Grad: 56145.1211  LR: 0.00000878  \n",
      "Epoch: [4][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5856(0.5017) Grad: 77017.8672  LR: 0.00000848  \n",
      "Epoch: [4][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.5078(0.5004) Grad: 46993.8438  LR: 0.00000818  \n",
      "Epoch: [4][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5308(0.4993) Grad: 71884.3906  LR: 0.00000788  \n",
      "Epoch: [4][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.3960(0.4988) Grad: 46667.4375  LR: 0.00000758  \n",
      "Epoch: [4][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4819(0.4984) Grad: 45309.6172  LR: 0.00000728  \n",
      "Epoch: [4][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5195(0.4991) Grad: 60279.6758  LR: 0.00000699  \n",
      "Epoch: [4][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.4821(0.4986) Grad: 62604.2070  LR: 0.00000670  \n",
      "Epoch: [4][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.6238(0.4985) Grad: 359093.2188  LR: 0.00000641  \n",
      "Epoch: [4][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5042(0.4980) Grad: 68720.3359  LR: 0.00000613  \n",
      "Epoch: [4][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5208(0.4972) Grad: 66455.0000  LR: 0.00000585  \n",
      "Epoch: [4][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.4863(0.4972) Grad: 29494.0684  LR: 0.00000557  \n",
      "Epoch: [4][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.3435(0.4964) Grad: 10861.6367  LR: 0.00000530  \n",
      "Epoch: [4][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.5165(0.4965) Grad: 60571.4961  LR: 0.00000503  \n",
      "Epoch: [4][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5807(0.4963) Grad: 24175.6250  LR: 0.00000501  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 8s) Loss: 0.4798(0.4798) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5594(0.5629) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4456(0.5674) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.5793(0.5664) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4806(0.5676) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4419(0.5661) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4 - avg_train_loss: 0.4963  avg_val_loss: 0.5643  time: 774s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4495(0.5643) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4 - Score: 0.8470\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [5][0/1709] Elapsed 0m 0s (remain 20m 55s) Loss: 0.3652(0.3652) Grad: 66943.4297  LR: 0.00000500  \n",
      "Epoch: [5][100/1709] Elapsed 0m 42s (remain 11m 17s) Loss: 0.5042(0.4854) Grad: 30850.5527  LR: 0.00000474  \n",
      "Epoch: [5][200/1709] Elapsed 1m 24s (remain 10m 32s) Loss: 0.5294(0.4873) Grad: 62977.7656  LR: 0.00000448  \n",
      "Epoch: [5][300/1709] Elapsed 2m 5s (remain 9m 49s) Loss: 0.4640(0.4887) Grad: 229239.7344  LR: 0.00000423  \n",
      "Epoch: [5][400/1709] Elapsed 2m 47s (remain 9m 7s) Loss: 0.4287(0.4863) Grad: 53033.2461  LR: 0.00000398  \n",
      "Epoch: [5][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.4344(0.4865) Grad: 355203.5000  LR: 0.00000374  \n",
      "Epoch: [5][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.4235(0.4901) Grad: 58390.1719  LR: 0.00000351  \n",
      "Epoch: [5][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4897(0.4908) Grad: 68161.3203  LR: 0.00000328  \n",
      "Epoch: [5][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.5581(0.4906) Grad: 51310.9375  LR: 0.00000305  \n",
      "Epoch: [5][900/1709] Elapsed 6m 15s (remain 5m 37s) Loss: 0.4552(0.4914) Grad: 58513.4062  LR: 0.00000284  \n",
      "Epoch: [5][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4420(0.4907) Grad: 78651.8047  LR: 0.00000263  \n",
      "Epoch: [5][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.4657(0.4912) Grad: 125730.6406  LR: 0.00000242  \n",
      "Epoch: [5][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4204(0.4918) Grad: 40978.5820  LR: 0.00000223  \n",
      "Epoch: [5][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4521(0.4915) Grad: 47871.9609  LR: 0.00000204  \n",
      "Epoch: [5][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5479(0.4915) Grad: 74183.5391  LR: 0.00000186  \n",
      "Epoch: [5][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5470(0.4921) Grad: 43157.3203  LR: 0.00000168  \n",
      "Epoch: [5][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.4136(0.4919) Grad: 40134.1992  LR: 0.00000152  \n",
      "Epoch: [5][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.5922(0.4912) Grad: 61393.4141  LR: 0.00000136  \n",
      "Epoch: [5][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.6145(0.4913) Grad: 102609.2891  LR: 0.00000134  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 16s) Loss: 0.4852(0.4852) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5590(0.5546) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.4379(0.5590) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.5774(0.5578) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4824(0.5587) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4467(0.5574) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5 - avg_train_loss: 0.4913  avg_val_loss: 0.5559  time: 774s\n",
      "Epoch 5 - Score: 0.8506\n",
      "Epoch 5 - Save Best Score: 0.8506 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.3997(0.5559) \n",
      "Epoch: [6][0/1709] Elapsed 0m 0s (remain 19m 46s) Loss: 0.5516(0.5516) Grad: 41411.8320  LR: 0.00000134  \n",
      "Epoch: [6][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.5445(0.4926) Grad: 43836.0625  LR: 0.00000119  \n",
      "Epoch: [6][200/1709] Elapsed 1m 24s (remain 10m 31s) Loss: 0.4947(0.4862) Grad: 29375.5488  LR: 0.00000105  \n",
      "Epoch: [6][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.5121(0.4875) Grad: 30942.9082  LR: 0.00000092  \n",
      "Epoch: [6][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.5297(0.4865) Grad: 47131.8086  LR: 0.00000080  \n",
      "Epoch: [6][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.4900(0.4871) Grad: 99519.2812  LR: 0.00000068  \n",
      "Epoch: [6][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.4076(0.4860) Grad: 28871.6348  LR: 0.00000057  \n",
      "Epoch: [6][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4670(0.4861) Grad: 66736.0547  LR: 0.00000048  \n",
      "Epoch: [6][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.5007(0.4866) Grad: 54174.5078  LR: 0.00000039  \n",
      "Epoch: [6][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4605(0.4870) Grad: 58823.7969  LR: 0.00000031  \n",
      "Epoch: [6][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4434(0.4872) Grad: 390775.3750  LR: 0.00000024  \n",
      "Epoch: [6][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5943(0.4870) Grad: 46657.7344  LR: 0.00000018  \n",
      "Epoch: [6][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4570(0.4873) Grad: 57350.9531  LR: 0.00000012  \n",
      "Epoch: [6][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5679(0.4880) Grad: 106122.5312  LR: 0.00000008  \n",
      "Epoch: [6][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.5315(0.4881) Grad: 69320.1250  LR: 0.00000005  \n",
      "Epoch: [6][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5690(0.4882) Grad: 40115.2852  LR: 0.00000002  \n",
      "Epoch: [6][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.3984(0.4882) Grad: 16933.0352  LR: 0.00000001  \n",
      "Epoch: [6][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.5287(0.4884) Grad: 20096.3418  LR: 0.00000000  \n",
      "Epoch: [6][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.4348(0.4884) Grad: 33547.8359  LR: 0.00000000  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 20s) Loss: 0.4852(0.4852) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5590(0.5559) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4298(0.5599) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.5761(0.5585) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4874(0.5594) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4479(0.5581) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.3952(0.5566) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6 - avg_train_loss: 0.4884  avg_val_loss: 0.5566  time: 774s\n",
      "Epoch 6 - Score: 0.8505\n",
      "========== fold: 1 result ==========\n",
      "Score: 0.8506\n",
      "========== fold: 2 training ==========\n",
      "Some weights of the model checkpoint at anferico/bert-for-patents were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [1][0/1709] Elapsed 0m 0s (remain 18m 28s) Loss: 0.7799(0.7799) Grad: inf  LR: 0.00002000  \n",
      "Epoch: [1][100/1709] Elapsed 0m 42s (remain 11m 11s) Loss: 0.6574(0.6545) Grad: 95958.4766  LR: 0.00002000  \n",
      "Epoch: [1][200/1709] Elapsed 1m 23s (remain 10m 29s) Loss: 0.5620(0.6353) Grad: 71753.8906  LR: 0.00001998  \n",
      "Epoch: [1][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.6190(0.6206) Grad: 37985.4492  LR: 0.00001996  \n",
      "Epoch: [1][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.5274(0.6102) Grad: 44362.8281  LR: 0.00001992  \n",
      "Epoch: [1][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.6474(0.6031) Grad: 77664.8125  LR: 0.00001988  \n",
      "Epoch: [1][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.6705(0.5975) Grad: 61406.2188  LR: 0.00001983  \n",
      "Epoch: [1][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5831(0.5919) Grad: 23564.3984  LR: 0.00001977  \n",
      "Epoch: [1][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.5888(0.5873) Grad: 89802.9609  LR: 0.00001970  \n",
      "Epoch: [1][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.6145(0.5851) Grad: 50121.3594  LR: 0.00001962  \n",
      "Epoch: [1][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4803(0.5827) Grad: 66412.4688  LR: 0.00001953  \n",
      "Epoch: [1][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5736(0.5802) Grad: 36555.2812  LR: 0.00001944  \n",
      "Epoch: [1][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.6039(0.5784) Grad: 41609.2500  LR: 0.00001933  \n",
      "Epoch: [1][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4511(0.5760) Grad: 31443.8203  LR: 0.00001922  \n",
      "Epoch: [1][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.5860(0.5749) Grad: 53461.4102  LR: 0.00001909  \n",
      "Epoch: [1][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5621(0.5740) Grad: 20011.7012  LR: 0.00001896  \n",
      "Epoch: [1][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.5633(0.5734) Grad: 20341.4863  LR: 0.00001882  \n",
      "Epoch: [1][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.4856(0.5724) Grad: 22069.5664  LR: 0.00001867  \n",
      "Epoch: [1][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5934(0.5722) Grad: 18408.0801  LR: 0.00001866  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 17s) Loss: 0.6183(0.6183) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5916(0.5618) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.5519(0.5727) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.4532(0.5727) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.5073(0.5721) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5378(0.5731) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5838(0.5717) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1 - avg_train_loss: 0.5722  avg_val_loss: 0.5717  time: 774s\n",
      "Epoch 1 - Score: 0.8025\n",
      "Epoch 1 - Save Best Score: 0.8025 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [2][0/1709] Elapsed 0m 1s (remain 28m 44s) Loss: 0.5213(0.5213) Grad: 544455.8125  LR: 0.00001866  \n",
      "Epoch: [2][100/1709] Elapsed 0m 42s (remain 11m 18s) Loss: 0.5828(0.5388) Grad: 157698.2812  LR: 0.00001850  \n",
      "Epoch: [2][200/1709] Elapsed 1m 24s (remain 10m 31s) Loss: 0.6684(0.5361) Grad: 24254.3809  LR: 0.00001834  \n",
      "Epoch: [2][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.5483(0.5350) Grad: 14838.0811  LR: 0.00001816  \n",
      "Epoch: [2][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.5213(0.5349) Grad: 19780.5957  LR: 0.00001798  \n",
      "Epoch: [2][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5464(0.5350) Grad: 24879.2578  LR: 0.00001780  \n",
      "Epoch: [2][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.5929(0.5347) Grad: 164006.5625  LR: 0.00001760  \n",
      "Epoch: [2][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4062(0.5346) Grad: 14701.1484  LR: 0.00001740  \n",
      "Epoch: [2][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.6519(0.5359) Grad: 18869.1582  LR: 0.00001719  \n",
      "Epoch: [2][900/1709] Elapsed 6m 15s (remain 5m 37s) Loss: 0.5653(0.5366) Grad: 9234.4639  LR: 0.00001697  \n",
      "Epoch: [2][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5714(0.5363) Grad: 4944.6724  LR: 0.00001675  \n",
      "Epoch: [2][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5657(0.5364) Grad: 8887.6396  LR: 0.00001652  \n",
      "Epoch: [2][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.5192(0.5361) Grad: 13473.7246  LR: 0.00001628  \n",
      "Epoch: [2][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4207(0.5364) Grad: 22398.9551  LR: 0.00001604  \n",
      "Epoch: [2][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5102(0.5358) Grad: 14506.2891  LR: 0.00001580  \n",
      "Epoch: [2][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5025(0.5353) Grad: 37401.0508  LR: 0.00001554  \n",
      "Epoch: [2][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.5506(0.5352) Grad: 20879.2188  LR: 0.00001529  \n",
      "Epoch: [2][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.5559(0.5339) Grad: 21728.3652  LR: 0.00001502  \n",
      "Epoch: [2][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5772(0.5340) Grad: 25766.5078  LR: 0.00001500  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 24s) Loss: 0.5610(0.5610) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5876(0.5281) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5380(0.5417) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.4544(0.5391) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4831(0.5406) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4913(0.5417) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4500(0.5412) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2 - avg_train_loss: 0.5340  avg_val_loss: 0.5412  time: 774s\n",
      "Epoch 2 - Score: 0.8356\n",
      "Epoch 2 - Save Best Score: 0.8356 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [3][0/1709] Elapsed 0m 0s (remain 20m 10s) Loss: 0.5383(0.5383) Grad: 44284.5820  LR: 0.00001500  \n",
      "Epoch: [3][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.4277(0.5157) Grad: 18679.6953  LR: 0.00001473  \n",
      "Epoch: [3][200/1709] Elapsed 1m 24s (remain 10m 31s) Loss: 0.5284(0.5163) Grad: 26699.2539  LR: 0.00001446  \n",
      "Epoch: [3][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.4080(0.5134) Grad: 32901.7812  LR: 0.00001419  \n",
      "Epoch: [3][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.5908(0.5092) Grad: 35390.1719  LR: 0.00001391  \n",
      "Epoch: [3][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.4751(0.5115) Grad: 27291.9238  LR: 0.00001362  \n",
      "Epoch: [3][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.5688(0.5118) Grad: 57976.8477  LR: 0.00001333  \n",
      "Epoch: [3][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.6328(0.5118) Grad: 39350.0039  LR: 0.00001304  \n",
      "Epoch: [3][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.4967(0.5105) Grad: 31326.9531  LR: 0.00001275  \n",
      "Epoch: [3][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.5268(0.5087) Grad: 32817.4727  LR: 0.00001246  \n",
      "Epoch: [3][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4605(0.5087) Grad: 25791.2305  LR: 0.00001216  \n",
      "Epoch: [3][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5754(0.5088) Grad: 88598.0547  LR: 0.00001186  \n",
      "Epoch: [3][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4196(0.5082) Grad: 16859.5879  LR: 0.00001156  \n",
      "Epoch: [3][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.3818(0.5075) Grad: 70778.9062  LR: 0.00001125  \n",
      "Epoch: [3][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.4285(0.5066) Grad: 15527.4834  LR: 0.00001095  \n",
      "Epoch: [3][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.6507(0.5070) Grad: 28900.0781  LR: 0.00001064  \n",
      "Epoch: [3][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.4894(0.5061) Grad: 46675.8594  LR: 0.00001034  \n",
      "Epoch: [3][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.5453(0.5052) Grad: 12770.7295  LR: 0.00001003  \n",
      "Epoch: [3][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5790(0.5052) Grad: 22424.4727  LR: 0.00001001  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 4m 38s) Loss: 0.5785(0.5785) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 52s) Loss: 0.5709(0.5371) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.5454(0.5518) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.4784(0.5477) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4790(0.5486) \n",
      "EVAL: [500/570] Elapsed 0m 54s (remain 0m 7s) Loss: 0.4436(0.5492) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4604(0.5483) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3 - avg_train_loss: 0.5052  avg_val_loss: 0.5483  time: 774s\n",
      "Epoch 3 - Score: 0.8437\n",
      "Epoch 3 - Save Best Score: 0.8437 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [4][0/1709] Elapsed 0m 0s (remain 21m 31s) Loss: 0.5234(0.5234) Grad: 50664.6367  LR: 0.00001000  \n",
      "Epoch: [4][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.6085(0.4831) Grad: 42223.9492  LR: 0.00000970  \n",
      "Epoch: [4][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.3924(0.4913) Grad: 41968.6484  LR: 0.00000939  \n",
      "Epoch: [4][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.6023(0.4934) Grad: 49592.0078  LR: 0.00000909  \n",
      "Epoch: [4][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.4277(0.4931) Grad: 77085.8047  LR: 0.00000878  \n",
      "Epoch: [4][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5668(0.4936) Grad: 58415.5430  LR: 0.00000848  \n",
      "Epoch: [4][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.5334(0.4952) Grad: 48928.1094  LR: 0.00000818  \n",
      "Epoch: [4][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5581(0.4963) Grad: 126142.7422  LR: 0.00000788  \n",
      "Epoch: [4][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.5650(0.4957) Grad: 58459.2188  LR: 0.00000758  \n",
      "Epoch: [4][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.6517(0.4963) Grad: 327470.8750  LR: 0.00000728  \n",
      "Epoch: [4][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5722(0.4967) Grad: 46977.1562  LR: 0.00000699  \n",
      "Epoch: [4][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5722(0.4957) Grad: 50778.3672  LR: 0.00000670  \n",
      "Epoch: [4][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4144(0.4952) Grad: 10700.2520  LR: 0.00000641  \n",
      "Epoch: [4][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4241(0.4956) Grad: 20795.3984  LR: 0.00000613  \n",
      "Epoch: [4][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.4369(0.4960) Grad: 32938.6719  LR: 0.00000585  \n",
      "Epoch: [4][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.4279(0.4959) Grad: 22024.7344  LR: 0.00000557  \n",
      "Epoch: [4][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.5164(0.4964) Grad: 23736.7168  LR: 0.00000530  \n",
      "Epoch: [4][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.6067(0.4961) Grad: 99672.2969  LR: 0.00000503  \n",
      "Epoch: [4][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5677(0.4961) Grad: 17980.3574  LR: 0.00000501  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 17s) Loss: 0.6203(0.6203) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5549(0.5411) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5397(0.5568) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.4820(0.5516) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.5135(0.5529) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4756(0.5537) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5065(0.5529) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4 - avg_train_loss: 0.4961  avg_val_loss: 0.5529  time: 774s\n",
      "Epoch 4 - Score: 0.8485\n",
      "Epoch 4 - Save Best Score: 0.8485 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [5][0/1709] Elapsed 0m 0s (remain 20m 22s) Loss: 0.4942(0.4942) Grad: 34551.0977  LR: 0.00000500  \n",
      "Epoch: [5][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.5372(0.4884) Grad: 34537.5156  LR: 0.00000474  \n",
      "Epoch: [5][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.4458(0.4905) Grad: 63215.3320  LR: 0.00000448  \n",
      "Epoch: [5][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.4394(0.4915) Grad: 40958.1797  LR: 0.00000423  \n",
      "Epoch: [5][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.5360(0.4963) Grad: 46464.7969  LR: 0.00000398  \n",
      "Epoch: [5][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.3537(0.4947) Grad: 19091.9453  LR: 0.00000374  \n",
      "Epoch: [5][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.4667(0.4926) Grad: 52841.8516  LR: 0.00000351  \n",
      "Epoch: [5][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5203(0.4929) Grad: 38057.5352  LR: 0.00000328  \n",
      "Epoch: [5][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.4800(0.4910) Grad: 56141.1836  LR: 0.00000305  \n",
      "Epoch: [5][900/1709] Elapsed 6m 15s (remain 5m 37s) Loss: 0.5626(0.4910) Grad: 41667.7461  LR: 0.00000284  \n",
      "Epoch: [5][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5422(0.4914) Grad: 39819.2930  LR: 0.00000263  \n",
      "Epoch: [5][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.4854(0.4916) Grad: 52979.5508  LR: 0.00000242  \n",
      "Epoch: [5][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.5574(0.4920) Grad: 32241.5664  LR: 0.00000223  \n",
      "Epoch: [5][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5266(0.4918) Grad: 54434.1797  LR: 0.00000204  \n",
      "Epoch: [5][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.4763(0.4918) Grad: 40931.8555  LR: 0.00000186  \n",
      "Epoch: [5][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.4958(0.4915) Grad: 40604.9453  LR: 0.00000168  \n",
      "Epoch: [5][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.4693(0.4914) Grad: 35786.4453  LR: 0.00000152  \n",
      "Epoch: [5][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.4215(0.4915) Grad: 37215.4922  LR: 0.00000136  \n",
      "Epoch: [5][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.4198(0.4915) Grad: 22521.8945  LR: 0.00000134  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 26s) Loss: 0.6138(0.6138) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5596(0.5470) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.5471(0.5619) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.4902(0.5564) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.5061(0.5577) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.4714(0.5585) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4950(0.5581) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5 - avg_train_loss: 0.4915  avg_val_loss: 0.5581  time: 774s\n",
      "Epoch 5 - Score: 0.8473\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [6][0/1709] Elapsed 0m 0s (remain 19m 27s) Loss: 0.4841(0.4841) Grad: 33265.9688  LR: 0.00000134  \n",
      "Epoch: [6][100/1709] Elapsed 0m 42s (remain 11m 13s) Loss: 0.3378(0.4897) Grad: 48017.0586  LR: 0.00000119  \n",
      "Epoch: [6][200/1709] Elapsed 1m 23s (remain 10m 29s) Loss: 0.5298(0.4877) Grad: 451305.3750  LR: 0.00000105  \n",
      "Epoch: [6][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.5810(0.4915) Grad: 63607.1914  LR: 0.00000092  \n",
      "Epoch: [6][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.4383(0.4923) Grad: 64322.8320  LR: 0.00000080  \n",
      "Epoch: [6][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.5790(0.4907) Grad: 543806.0000  LR: 0.00000068  \n",
      "Epoch: [6][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.4810(0.4932) Grad: 199272.0625  LR: 0.00000057  \n",
      "Epoch: [6][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5324(0.4932) Grad: 38767.9258  LR: 0.00000048  \n",
      "Epoch: [6][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.3436(0.4919) Grad: 30297.1816  LR: 0.00000039  \n",
      "Epoch: [6][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4396(0.4904) Grad: 66983.3828  LR: 0.00000031  \n",
      "Epoch: [6][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4660(0.4891) Grad: 25171.1055  LR: 0.00000024  \n",
      "Epoch: [6][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5476(0.4895) Grad: 40164.4844  LR: 0.00000018  \n",
      "Epoch: [6][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4262(0.4900) Grad: 49140.9453  LR: 0.00000012  \n",
      "Epoch: [6][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4749(0.4901) Grad: 56494.8750  LR: 0.00000008  \n",
      "Epoch: [6][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5709(0.4892) Grad: 26857.9922  LR: 0.00000005  \n",
      "Epoch: [6][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.3583(0.4896) Grad: 30304.1289  LR: 0.00000002  \n",
      "Epoch: [6][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.4663(0.4895) Grad: 99093.2266  LR: 0.00000001  \n",
      "Epoch: [6][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.4251(0.4895) Grad: 35017.9180  LR: 0.00000000  \n",
      "Epoch: [6][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5567(0.4893) Grad: 28941.9766  LR: 0.00000000  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 42s) Loss: 0.6366(0.6366) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.5588(0.5502) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.5477(0.5653) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.4930(0.5598) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.5192(0.5607) \n",
      "EVAL: [500/570] Elapsed 0m 54s (remain 0m 7s) Loss: 0.4832(0.5618) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5023(0.5615) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6 - avg_train_loss: 0.4893  avg_val_loss: 0.5615  time: 774s\n",
      "Epoch 6 - Score: 0.8469\n",
      "========== fold: 2 result ==========\n",
      "Score: 0.8485\n",
      "========== fold: 3 training ==========\n",
      "Some weights of the model checkpoint at anferico/bert-for-patents were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [1][0/1709] Elapsed 0m 0s (remain 20m 23s) Loss: 0.6604(0.6604) Grad: 183819.9375  LR: 0.00002000  \n",
      "Epoch: [1][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.7207(0.6502) Grad: 147470.3438  LR: 0.00002000  \n",
      "Epoch: [1][200/1709] Elapsed 1m 23s (remain 10m 29s) Loss: 0.5913(0.6247) Grad: 58722.3008  LR: 0.00001998  \n",
      "Epoch: [1][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.5051(0.6131) Grad: 87306.5000  LR: 0.00001996  \n",
      "Epoch: [1][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.4980(0.6049) Grad: 46530.8398  LR: 0.00001992  \n",
      "Epoch: [1][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.5861(0.5999) Grad: 44165.5195  LR: 0.00001988  \n",
      "Epoch: [1][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.6210(0.5971) Grad: 37351.9844  LR: 0.00001983  \n",
      "Epoch: [1][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.6132(0.5926) Grad: 31261.4531  LR: 0.00001977  \n",
      "Epoch: [1][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.5321(0.5906) Grad: 18496.5273  LR: 0.00001970  \n",
      "Epoch: [1][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.5594(0.5893) Grad: 16806.4395  LR: 0.00001962  \n",
      "Epoch: [1][1000/1709] Elapsed 6m 57s (remain 4m 54s) Loss: 0.4871(0.5871) Grad: 27498.9375  LR: 0.00001953  \n",
      "Epoch: [1][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5725(0.5851) Grad: 27364.7559  LR: 0.00001944  \n",
      "Epoch: [1][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.5893(0.5835) Grad: 31581.3047  LR: 0.00001933  \n",
      "Epoch: [1][1300/1709] Elapsed 9m 1s (remain 2m 49s) Loss: 0.5291(0.5828) Grad: 18537.6191  LR: 0.00001922  \n",
      "Epoch: [1][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.4535(0.5818) Grad: 19995.5723  LR: 0.00001909  \n",
      "Epoch: [1][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5803(0.5804) Grad: 10027.5283  LR: 0.00001896  \n",
      "Epoch: [1][1600/1709] Elapsed 11m 6s (remain 0m 44s) Loss: 0.4531(0.5786) Grad: 13903.2275  LR: 0.00001882  \n",
      "Epoch: [1][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.5368(0.5770) Grad: 8929.6650  LR: 0.00001867  \n",
      "Epoch: [1][1708/1709] Elapsed 11m 51s (remain 0m 0s) Loss: 0.7640(0.5772) Grad: 29857.3242  LR: 0.00001866  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 26s) Loss: 0.6078(0.6078) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.7405(0.5684) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.4758(0.5587) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.6404(0.5574) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4353(0.5569) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5584(0.5556) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4941(0.5565) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1 - avg_train_loss: 0.5772  avg_val_loss: 0.5565  time: 774s\n",
      "Epoch 1 - Score: 0.7955\n",
      "Epoch 1 - Save Best Score: 0.7955 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [2][0/1709] Elapsed 0m 0s (remain 20m 55s) Loss: 0.5260(0.5260) Grad: 67759.7422  LR: 0.00001866  \n",
      "Epoch: [2][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.6141(0.5322) Grad: 207558.0312  LR: 0.00001850  \n",
      "Epoch: [2][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.4814(0.5368) Grad: 53157.8438  LR: 0.00001834  \n",
      "Epoch: [2][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.5915(0.5341) Grad: 69995.0234  LR: 0.00001816  \n",
      "Epoch: [2][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.6916(0.5329) Grad: 118492.6562  LR: 0.00001798  \n",
      "Epoch: [2][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.4679(0.5316) Grad: 21790.6113  LR: 0.00001780  \n",
      "Epoch: [2][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.6198(0.5322) Grad: 51064.3203  LR: 0.00001760  \n",
      "Epoch: [2][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4078(0.5309) Grad: 25825.8008  LR: 0.00001740  \n",
      "Epoch: [2][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.5666(0.5302) Grad: 57880.5781  LR: 0.00001719  \n",
      "Epoch: [2][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4755(0.5297) Grad: 22430.0684  LR: 0.00001697  \n",
      "Epoch: [2][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.5532(0.5294) Grad: 16543.0586  LR: 0.00001675  \n",
      "Epoch: [2][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.4360(0.5283) Grad: 10782.4844  LR: 0.00001652  \n",
      "Epoch: [2][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4993(0.5267) Grad: 18371.1719  LR: 0.00001628  \n",
      "Epoch: [2][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5505(0.5269) Grad: 28643.4121  LR: 0.00001604  \n",
      "Epoch: [2][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.6126(0.5268) Grad: 28794.0391  LR: 0.00001580  \n",
      "Epoch: [2][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.6175(0.5263) Grad: 39915.3594  LR: 0.00001554  \n",
      "Epoch: [2][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.3924(0.5260) Grad: 12423.0459  LR: 0.00001529  \n",
      "Epoch: [2][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.6139(0.5255) Grad: 41960.0977  LR: 0.00001502  \n",
      "Epoch: [2][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.6198(0.5255) Grad: 22982.3105  LR: 0.00001500  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 34s) Loss: 0.5895(0.5895) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.6862(0.5554) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4593(0.5506) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.5596(0.5449) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4342(0.5471) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5477(0.5468) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.5152(0.5492) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2 - avg_train_loss: 0.5255  avg_val_loss: 0.5492  time: 774s\n",
      "Epoch 2 - Score: 0.8349\n",
      "Epoch 2 - Save Best Score: 0.8349 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [3][0/1709] Elapsed 0m 0s (remain 22m 6s) Loss: 0.3878(0.3878) Grad: 79290.6094  LR: 0.00001500  \n",
      "Epoch: [3][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.6755(0.5059) Grad: 151916.8125  LR: 0.00001473  \n",
      "Epoch: [3][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.4860(0.5058) Grad: 29112.8867  LR: 0.00001446  \n",
      "Epoch: [3][300/1709] Elapsed 2m 5s (remain 9m 47s) Loss: 0.4846(0.5044) Grad: 42310.4688  LR: 0.00001419  \n",
      "Epoch: [3][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.5982(0.5075) Grad: 31747.3672  LR: 0.00001391  \n",
      "Epoch: [3][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.5034(0.5063) Grad: 32902.7891  LR: 0.00001362  \n",
      "Epoch: [3][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.5141(0.5063) Grad: 100206.1406  LR: 0.00001333  \n",
      "Epoch: [3][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.6295(0.5070) Grad: 134748.7031  LR: 0.00001304  \n",
      "Epoch: [3][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.4934(0.5071) Grad: 80765.4844  LR: 0.00001275  \n",
      "Epoch: [3][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.5281(0.5080) Grad: 25453.2441  LR: 0.00001246  \n",
      "Epoch: [3][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4911(0.5083) Grad: 65461.0156  LR: 0.00001216  \n",
      "Epoch: [3][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.5261(0.5086) Grad: 173514.2969  LR: 0.00001186  \n",
      "Epoch: [3][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4856(0.5089) Grad: 25166.6211  LR: 0.00001156  \n",
      "Epoch: [3][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4216(0.5085) Grad: 69523.6953  LR: 0.00001125  \n",
      "Epoch: [3][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.5188(0.5083) Grad: 76293.3594  LR: 0.00001095  \n",
      "Epoch: [3][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5406(0.5088) Grad: 37377.3672  LR: 0.00001064  \n",
      "Epoch: [3][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.3969(0.5087) Grad: 60623.3164  LR: 0.00001034  \n",
      "Epoch: [3][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.4858(0.5084) Grad: 87329.5156  LR: 0.00001003  \n",
      "Epoch: [3][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.4876(0.5082) Grad: 60042.7383  LR: 0.00001001  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 28s) Loss: 0.5817(0.5817) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.6880(0.5544) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 40s) Loss: 0.4665(0.5489) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.5806(0.5447) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4223(0.5462) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.5964(0.5453) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4299(0.5472) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3 - avg_train_loss: 0.5082  avg_val_loss: 0.5472  time: 774s\n",
      "Epoch 3 - Score: 0.8437\n",
      "Epoch 3 - Save Best Score: 0.8437 Model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [4][0/1709] Elapsed 0m 0s (remain 22m 50s) Loss: 0.4157(0.4157) Grad: 35974.0078  LR: 0.00001000  \n",
      "Epoch: [4][100/1709] Elapsed 0m 42s (remain 11m 15s) Loss: 0.4014(0.4946) Grad: 37412.4492  LR: 0.00000970  \n",
      "Epoch: [4][200/1709] Elapsed 1m 24s (remain 10m 31s) Loss: 0.4936(0.5003) Grad: 88823.9688  LR: 0.00000939  \n",
      "Epoch: [4][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.5312(0.4968) Grad: 40120.7617  LR: 0.00000909  \n",
      "Epoch: [4][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.4484(0.4964) Grad: 96227.1562  LR: 0.00000878  \n",
      "Epoch: [4][500/1709] Elapsed 3m 28s (remain 8m 23s) Loss: 0.4797(0.4983) Grad: 34140.3906  LR: 0.00000848  \n",
      "Epoch: [4][600/1709] Elapsed 4m 10s (remain 7m 41s) Loss: 0.5149(0.4987) Grad: 16822.0020  LR: 0.00000818  \n",
      "Epoch: [4][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.5042(0.4990) Grad: 50810.6562  LR: 0.00000788  \n",
      "Epoch: [4][800/1709] Elapsed 5m 33s (remain 6m 18s) Loss: 0.4775(0.4982) Grad: 16410.1602  LR: 0.00000758  \n",
      "Epoch: [4][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4528(0.4988) Grad: 81874.1562  LR: 0.00000728  \n",
      "Epoch: [4][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4879(0.4990) Grad: 28111.0039  LR: 0.00000699  \n",
      "Epoch: [4][1100/1709] Elapsed 7m 38s (remain 4m 13s) Loss: 0.3167(0.4989) Grad: 28480.3770  LR: 0.00000670  \n",
      "Epoch: [4][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4999(0.4990) Grad: 38664.0781  LR: 0.00000641  \n",
      "Epoch: [4][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4417(0.4983) Grad: 55815.2891  LR: 0.00000613  \n",
      "Epoch: [4][1400/1709] Elapsed 9m 43s (remain 2m 8s) Loss: 0.4462(0.4991) Grad: 26623.7852  LR: 0.00000585  \n",
      "Epoch: [4][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5210(0.4987) Grad: 106713.6562  LR: 0.00000557  \n",
      "Epoch: [4][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.5997(0.4987) Grad: 63315.1562  LR: 0.00000530  \n",
      "Epoch: [4][1700/1709] Elapsed 11m 48s (remain 0m 3s) Loss: 0.5089(0.4985) Grad: 66755.8906  LR: 0.00000503  \n",
      "Epoch: [4][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.6329(0.4985) Grad: 31865.5820  LR: 0.00000501  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 26s) Loss: 0.5784(0.5784) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.6949(0.5694) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4580(0.5597) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 28s) Loss: 0.5981(0.5555) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4240(0.5563) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.6177(0.5556) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4245(0.5583) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4 - avg_train_loss: 0.4985  avg_val_loss: 0.5583  time: 774s\n",
      "Epoch 4 - Score: 0.8425\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [5][0/1709] Elapsed 0m 0s (remain 23m 34s) Loss: 0.3929(0.3929) Grad: 78818.8828  LR: 0.00000500  \n",
      "Epoch: [5][100/1709] Elapsed 0m 42s (remain 11m 16s) Loss: 0.6426(0.4967) Grad: 35394.4609  LR: 0.00000474  \n",
      "Epoch: [5][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.3743(0.4941) Grad: 68240.7500  LR: 0.00000448  \n",
      "Epoch: [5][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.4346(0.4937) Grad: 133242.3438  LR: 0.00000423  \n",
      "Epoch: [5][400/1709] Elapsed 2m 47s (remain 9m 6s) Loss: 0.5441(0.4937) Grad: 48070.1289  LR: 0.00000398  \n",
      "Epoch: [5][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5227(0.4945) Grad: 34720.7578  LR: 0.00000374  \n",
      "Epoch: [5][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.4407(0.4943) Grad: 32378.1797  LR: 0.00000351  \n",
      "Epoch: [5][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.3898(0.4927) Grad: 22855.5898  LR: 0.00000328  \n",
      "Epoch: [5][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.4805(0.4930) Grad: 62580.5586  LR: 0.00000305  \n",
      "Epoch: [5][900/1709] Elapsed 6m 15s (remain 5m 37s) Loss: 0.5493(0.4937) Grad: 22356.9258  LR: 0.00000284  \n",
      "Epoch: [5][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.6341(0.4941) Grad: 35429.6172  LR: 0.00000263  \n",
      "Epoch: [5][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5131(0.4943) Grad: 32000.1074  LR: 0.00000242  \n",
      "Epoch: [5][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.5020(0.4945) Grad: 10068.4775  LR: 0.00000223  \n",
      "Epoch: [5][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.4047(0.4935) Grad: 34434.7266  LR: 0.00000204  \n",
      "Epoch: [5][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.4461(0.4933) Grad: 37135.1875  LR: 0.00000186  \n",
      "Epoch: [5][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.6092(0.4933) Grad: 1553459.5000  LR: 0.00000168  \n",
      "Epoch: [5][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.5412(0.4922) Grad: 18505.3906  LR: 0.00000152  \n",
      "Epoch: [5][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.5720(0.4926) Grad: 67778.2656  LR: 0.00000136  \n",
      "Epoch: [5][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5607(0.4926) Grad: 24161.6133  LR: 0.00000134  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 18s) Loss: 0.5797(0.5797) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.6891(0.5732) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4566(0.5638) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.6064(0.5594) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4237(0.5612) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.6331(0.5596) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4309(0.5625) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5 - avg_train_loss: 0.4926  avg_val_loss: 0.5625  time: 774s\n",
      "Epoch 5 - Score: 0.8430\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [6][0/1709] Elapsed 0m 0s (remain 19m 32s) Loss: 0.5616(0.5616) Grad: 43145.9062  LR: 0.00000134  \n",
      "Epoch: [6][100/1709] Elapsed 0m 42s (remain 11m 14s) Loss: 0.6083(0.4951) Grad: 82612.3125  LR: 0.00000119  \n",
      "Epoch: [6][200/1709] Elapsed 1m 24s (remain 10m 30s) Loss: 0.5644(0.4935) Grad: 32268.6680  LR: 0.00000105  \n",
      "Epoch: [6][300/1709] Elapsed 2m 5s (remain 9m 48s) Loss: 0.5098(0.4907) Grad: 42328.7070  LR: 0.00000092  \n",
      "Epoch: [6][400/1709] Elapsed 2m 47s (remain 9m 5s) Loss: 0.5560(0.4928) Grad: 52264.0391  LR: 0.00000080  \n",
      "Epoch: [6][500/1709] Elapsed 3m 29s (remain 8m 24s) Loss: 0.5690(0.4923) Grad: 69719.6172  LR: 0.00000068  \n",
      "Epoch: [6][600/1709] Elapsed 4m 10s (remain 7m 42s) Loss: 0.5169(0.4927) Grad: 51472.2969  LR: 0.00000057  \n",
      "Epoch: [6][700/1709] Elapsed 4m 52s (remain 7m 0s) Loss: 0.4268(0.4924) Grad: 39954.2891  LR: 0.00000048  \n",
      "Epoch: [6][800/1709] Elapsed 5m 34s (remain 6m 18s) Loss: 0.4857(0.4924) Grad: 46564.8086  LR: 0.00000039  \n",
      "Epoch: [6][900/1709] Elapsed 6m 15s (remain 5m 36s) Loss: 0.4675(0.4916) Grad: 52409.1562  LR: 0.00000031  \n",
      "Epoch: [6][1000/1709] Elapsed 6m 57s (remain 4m 55s) Loss: 0.4779(0.4903) Grad: 225847.7188  LR: 0.00000024  \n",
      "Epoch: [6][1100/1709] Elapsed 7m 39s (remain 4m 13s) Loss: 0.5254(0.4900) Grad: 40471.1992  LR: 0.00000018  \n",
      "Epoch: [6][1200/1709] Elapsed 8m 20s (remain 3m 31s) Loss: 0.4734(0.4907) Grad: 681153.3750  LR: 0.00000012  \n",
      "Epoch: [6][1300/1709] Elapsed 9m 2s (remain 2m 50s) Loss: 0.5054(0.4905) Grad: 39486.3984  LR: 0.00000008  \n",
      "Epoch: [6][1400/1709] Elapsed 9m 44s (remain 2m 8s) Loss: 0.5447(0.4903) Grad: 24291.0938  LR: 0.00000005  \n",
      "Epoch: [6][1500/1709] Elapsed 10m 25s (remain 1m 26s) Loss: 0.5863(0.4899) Grad: 26577.6211  LR: 0.00000002  \n",
      "Epoch: [6][1600/1709] Elapsed 11m 7s (remain 0m 45s) Loss: 0.4852(0.4901) Grad: 13378.0205  LR: 0.00000001  \n",
      "Epoch: [6][1700/1709] Elapsed 11m 49s (remain 0m 3s) Loss: 0.2175(0.4895) Grad: 10923.1729  LR: 0.00000000  \n",
      "Epoch: [6][1708/1709] Elapsed 11m 52s (remain 0m 0s) Loss: 0.5670(0.4895) Grad: 8892.5957  LR: 0.00000000  \n",
      "EVAL: [0/570] Elapsed 0m 0s (remain 3m 27s) Loss: 0.5802(0.5802) \n",
      "EVAL: [100/570] Elapsed 0m 11s (remain 0m 51s) Loss: 0.6913(0.5770) \n",
      "EVAL: [200/570] Elapsed 0m 21s (remain 0m 39s) Loss: 0.4554(0.5667) \n",
      "EVAL: [300/570] Elapsed 0m 32s (remain 0m 29s) Loss: 0.6082(0.5619) \n",
      "EVAL: [400/570] Elapsed 0m 43s (remain 0m 18s) Loss: 0.4240(0.5638) \n",
      "EVAL: [500/570] Elapsed 0m 53s (remain 0m 7s) Loss: 0.6351(0.5621) \n",
      "EVAL: [569/570] Elapsed 1m 1s (remain 0m 0s) Loss: 0.4271(0.5649) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6 - avg_train_loss: 0.4895  avg_val_loss: 0.5649  time: 774s\n",
      "Epoch 6 - Score: 0.8429\n",
      "========== fold: 3 result ==========\n",
      "Score: 0.8437\n",
      "========== CV ==========\n",
      "Score: 0.8476\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    \n",
    "    def get_result(oof_df):\n",
    "        labels = oof_df['score'].values\n",
    "        preds = oof_df['pred'].values\n",
    "        score = get_score(labels, preds)\n",
    "        LOGGER.info(f'Score: {score:<.4f}')\n",
    "    \n",
    "    if CFG.train:\n",
    "        oof_df = pd.DataFrame()\n",
    "        for fold in range(CFG.n_fold):\n",
    "            if fold in CFG.trn_fold:\n",
    "                _oof_df = train_loop(train, fold)\n",
    "                oof_df = pd.concat([oof_df, _oof_df])\n",
    "                LOGGER.info(f\"========== fold: {fold} result ==========\")\n",
    "                get_result(_oof_df)\n",
    "        oof_df = oof_df.reset_index(drop=True)\n",
    "        LOGGER.info(f\"========== CV ==========\")\n",
    "        get_result(oof_df)\n",
    "        oof_df.to_pickle(OUTPUT_DIR+'oof_df.pkl')\n",
    "        \n",
    "    if CFG.wandb:\n",
    "        wandb.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66bc6204",
   "metadata": {
    "papermill": {
     "duration": 0.239606,
     "end_time": "2023-02-20T17:46:56.991361",
     "exception": false,
     "start_time": "2023-02-20T17:46:56.751755",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e676d98",
   "metadata": {
    "papermill": {
     "duration": 0.24127,
     "end_time": "2023-02-20T17:46:57.474397",
     "exception": false,
     "start_time": "2023-02-20T17:46:57.233127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "723c4689",
   "metadata": {
    "papermill": {
     "duration": 0.242233,
     "end_time": "2023-02-20T17:46:57.957994",
     "exception": false,
     "start_time": "2023-02-20T17:46:57.715761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d91a9329",
   "metadata": {
    "papermill": {
     "duration": 0.239859,
     "end_time": "2023-02-20T17:46:58.440500",
     "exception": false,
     "start_time": "2023-02-20T17:46:58.200641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "560bda07",
   "metadata": {
    "papermill": {
     "duration": 0.240638,
     "end_time": "2023-02-20T17:46:58.924860",
     "exception": false,
     "start_time": "2023-02-20T17:46:58.684222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70ac7d14",
   "metadata": {
    "papermill": {
     "duration": 0.241205,
     "end_time": "2023-02-20T17:46:59.407467",
     "exception": false,
     "start_time": "2023-02-20T17:46:59.166262",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7c74d6",
   "metadata": {
    "papermill": {
     "duration": 0.246451,
     "end_time": "2023-02-20T17:46:59.893581",
     "exception": false,
     "start_time": "2023-02-20T17:46:59.647130",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 18869.754751,
   "end_time": "2023-02-20T17:47:03.138826",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-02-20T12:32:33.384075",
   "version": "2.3.3"
  },
  "vscode": {
   "interpreter": {
    "hash": "b683bd4b87c8b16609ac9a3ef3aab015b96a8503b853fa71d3fe1e7ebcd29d95"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0541f06fd6384742a7e7e45cc7d213b5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "05b46798da3b461dac25076251c8bba8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "075bc08b4c3042f5a3e41b81380f7f80": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1248bc8d9bb64ec999fd8be2933752eb",
       "placeholder": "​",
       "style": "IPY_MODEL_0541f06fd6384742a7e7e45cc7d213b5",
       "value": "Downloading: 100%"
      }
     },
     "0d36a89075ea4c1fa1144a6e024e04bb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1248bc8d9bb64ec999fd8be2933752eb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "14db1fdfdef0443c9430f99f9123a2c6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_7192d6d600244e11ab46338bade2a69c",
        "IPY_MODEL_da74ea1c39324558a8d0e53cfb496378",
        "IPY_MODEL_ec0cb49620534c528405cb7145c88473"
       ],
       "layout": "IPY_MODEL_5f1cc27e440a44eb83fd522025dc70c7"
      }
     },
     "20b04defaceb4a33a14b8452471d2e48": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2ac04d32386a4804ac49e792bc050f91": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_0d36a89075ea4c1fa1144a6e024e04bb",
       "placeholder": "​",
       "style": "IPY_MODEL_f0f60209c8234b65b25f08e39f078cd1",
       "value": "100%"
      }
     },
     "3362d0800d3944778e01929ab0e32a60": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_77d3a5f33a2545569f6a6e68a2fc2c25",
       "max": 1383349812,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_f289d95e8a1e4523b88177628aad6419",
       "value": 1383349812
      }
     },
     "3373b8c35d3f427c9e6268c2b96899c7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dcc8c89288dc4d5394bcc657b3e760ab",
       "placeholder": "​",
       "style": "IPY_MODEL_7d9cbe63cd19495a9e20e16942de0f96",
       "value": " 36473/36473 [00:02&lt;00:00, 14476.74it/s]"
      }
     },
     "3e847d2060ae4d7e9321e685c91d7989": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4065accb9b3e4654ab790f67c61cbeb4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_878f089d6e2349e688f64393788d006a",
       "max": 36473,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ecca039606df45069b0c7f4f6ff8abfd",
       "value": 36473
      }
     },
     "415e91e01dd64c5e8e341dda1ae32c34": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_db717c83b0164c5ebf6d0452444e5a45",
       "placeholder": "​",
       "style": "IPY_MODEL_ca2efd70d40f4e1198eba0bab6caeed1",
       "value": "Downloading: 100%"
      }
     },
     "4415137afa694476adc934ead0afc515": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "51adf977b4774b9a9d1bcb0212a5eec8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_2ac04d32386a4804ac49e792bc050f91",
        "IPY_MODEL_dc2be51bf4d5404ebbe840ad442f3e56",
        "IPY_MODEL_3373b8c35d3f427c9e6268c2b96899c7"
       ],
       "layout": "IPY_MODEL_cc8fcc2215504c48be05e64fadc3b5ce"
      }
     },
     "54ac0b1d828f48988ad5900db00903d7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5b00bd4bbadb446dbb1c8b3cbbc2c401": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5f1cc27e440a44eb83fd522025dc70c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "61f7ee2a84bb4fd1889fdf6680b20d1c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "664c29255ec640908b6160edb28d064c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "6855892baa664a28a7104194000c3777": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ef0037ed3fa947a4be6cbded05b84ab0",
       "placeholder": "​",
       "style": "IPY_MODEL_5b00bd4bbadb446dbb1c8b3cbbc2c401",
       "value": " 36473/36473 [00:02&lt;00:00, 14110.41it/s]"
      }
     },
     "6a0c360d71c14a65a7ec95f5b7cb4771": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_83ed8654d27f4fd38ee1b5215607ea06",
       "placeholder": "​",
       "style": "IPY_MODEL_b98789fa80b747fab8512a343b861417",
       "value": " 1.29G/1.29G [02:01&lt;00:00, 11.6MB/s]"
      }
     },
     "6ab93bf37bd245a59b721c71b5fe02ef": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "7192d6d600244e11ab46338bade2a69c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e13378ae7341465a99faf2ac6da18d86",
       "placeholder": "​",
       "style": "IPY_MODEL_7f13fcee7db14d999a4e148fc0d5f372",
       "value": "Downloading: 100%"
      }
     },
     "72551cead8eb4f5ea90c4476ae2030e1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "72a552bd0f2247fabda7c6450dc2168c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "77d3a5f33a2545569f6a6e68a2fc2c25": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7d9cbe63cd19495a9e20e16942de0f96": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7f13fcee7db14d999a4e148fc0d5f372": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "83ed8654d27f4fd38ee1b5215607ea06": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "878f089d6e2349e688f64393788d006a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "87abd0881075445e97c8db2e54390a18": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8c5108990c774594a46aae2120f8b8c1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_54ac0b1d828f48988ad5900db00903d7",
       "max": 329241,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_72551cead8eb4f5ea90c4476ae2030e1",
       "value": 329241
      }
     },
     "8c5d9d65011340c381bc0b8445c636c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8defb43fa6ee4a959b736d5deadd0687": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a2e878a9d8e142bc9e7459468b2e3e85": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b98789fa80b747fab8512a343b861417": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "badb96d5349f420291c4c9620834a08e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "bb9fd523a0ec48b48b0b1a4863d8ed03": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_415e91e01dd64c5e8e341dda1ae32c34",
        "IPY_MODEL_3362d0800d3944778e01929ab0e32a60",
        "IPY_MODEL_6a0c360d71c14a65a7ec95f5b7cb4771"
       ],
       "layout": "IPY_MODEL_f921cd99f01f4d4899fcbaf35af0edf5"
      }
     },
     "bf8f2b4a2f1d4f3cb8172e08e348c9fe": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c5e34c371cd04636b26eaee758cc880e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "ca2efd70d40f4e1198eba0bab6caeed1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "cc8fcc2215504c48be05e64fadc3b5ce": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cdafde6750c54365a575dd2267d7b409": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ceec0367b31745e5972e1268416b133b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_05b46798da3b461dac25076251c8bba8",
       "placeholder": "​",
       "style": "IPY_MODEL_cdafde6750c54365a575dd2267d7b409",
       "value": " 136/136 [00:00&lt;00:00, 2749.59it/s]"
      }
     },
     "da74ea1c39324558a8d0e53cfb496378": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ec25f24f247b4dfe94401cdf5134d0ac",
       "max": 327,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6ab93bf37bd245a59b721c71b5fe02ef",
       "value": 327
      }
     },
     "db717c83b0164c5ebf6d0452444e5a45": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dc2be51bf4d5404ebbe840ad442f3e56": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bf8f2b4a2f1d4f3cb8172e08e348c9fe",
       "max": 36473,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_20b04defaceb4a33a14b8452471d2e48",
       "value": 36473
      }
     },
     "dcc8c89288dc4d5394bcc657b3e760ab": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e13378ae7341465a99faf2ac6da18d86": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e3c079a7df2340f5ba2104711ac33bf2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ec0cb49620534c528405cb7145c88473": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e3c079a7df2340f5ba2104711ac33bf2",
       "placeholder": "​",
       "style": "IPY_MODEL_a2e878a9d8e142bc9e7459468b2e3e85",
       "value": " 327/327 [00:00&lt;00:00, 10.8kB/s]"
      }
     },
     "ec25f24f247b4dfe94401cdf5134d0ac": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ec40bfeaecd04975b98825e93b0c8e8a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4415137afa694476adc934ead0afc515",
       "placeholder": "​",
       "style": "IPY_MODEL_664c29255ec640908b6160edb28d064c",
       "value": " 322k/322k [00:00&lt;00:00, 300kB/s]"
      }
     },
     "ec7b4f9f05f649c7a97dbcc7ade4b0a0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ff71890a67bc4caa9706ac9c246393ab",
        "IPY_MODEL_f0d9eee0329e40fcb2ab3c722df9e179",
        "IPY_MODEL_ceec0367b31745e5972e1268416b133b"
       ],
       "layout": "IPY_MODEL_8c5d9d65011340c381bc0b8445c636c3"
      }
     },
     "ecca039606df45069b0c7f4f6ff8abfd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "ef0037ed3fa947a4be6cbded05b84ab0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ef942263fbf74279b42c055ccb0c855f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f0d9eee0329e40fcb2ab3c722df9e179": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_72a552bd0f2247fabda7c6450dc2168c",
       "max": 136,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_c5e34c371cd04636b26eaee758cc880e",
       "value": 136
      }
     },
     "f0f60209c8234b65b25f08e39f078cd1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f289d95e8a1e4523b88177628aad6419": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "f4f432f85cb7461bb9a27bb90edca16d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_61f7ee2a84bb4fd1889fdf6680b20d1c",
       "placeholder": "​",
       "style": "IPY_MODEL_badb96d5349f420291c4c9620834a08e",
       "value": "100%"
      }
     },
     "f5f92b69534e4f28872123d0263f24a8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_f4f432f85cb7461bb9a27bb90edca16d",
        "IPY_MODEL_4065accb9b3e4654ab790f67c61cbeb4",
        "IPY_MODEL_6855892baa664a28a7104194000c3777"
       ],
       "layout": "IPY_MODEL_87abd0881075445e97c8db2e54390a18"
      }
     },
     "f921cd99f01f4d4899fcbaf35af0edf5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fe7cefa7a8444f81a1003172f66c7433": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_075bc08b4c3042f5a3e41b81380f7f80",
        "IPY_MODEL_8c5108990c774594a46aae2120f8b8c1",
        "IPY_MODEL_ec40bfeaecd04975b98825e93b0c8e8a"
       ],
       "layout": "IPY_MODEL_ef942263fbf74279b42c055ccb0c855f"
      }
     },
     "ff71890a67bc4caa9706ac9c246393ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3e847d2060ae4d7e9321e685c91d7989",
       "placeholder": "​",
       "style": "IPY_MODEL_8defb43fa6ee4a959b736d5deadd0687",
       "value": "100%"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}