fine-tune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "557172ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e205e0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "026b8e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Seq2SeqTrainer\n",
    "from transformers import Seq2SeqTrainingArguments\n",
    "from transformers import T5Config\n",
    "from transformers import T5ForConditionalGeneration\n",
    "from transformers import T5Tokenizer\n",
    "from transformers import set_seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a101993c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import argparse\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append(\"./hf_transformers/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0bbf88ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from data_reader import GetDataAsPython\n",
    "from prepare_data import create_data\n",
    "from prepare_data import create_dataset\n",
    "from prepare_data import extract_warning_types\n",
    "from utils import boolean_string\n",
    "from utils import get_current_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "82e078ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from data_reader import GetDataAsPython\n",
    "from prepare_data import create_data\n",
    "from prepare_data import create_dataset\n",
    "from prepare_data import extract_warning_types\n",
    "from utils import boolean_string\n",
    "from utils import get_current_time\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2cc8e6c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_directory = './storage/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8b7a2bd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e6178f1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "985"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exec_number = random.randint(0, 1000)\n",
    "exec_number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "39260e34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/data/all/data/svgdotjs/svg.js'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo = '/data/all/data/svgdotjs/svg.js'\n",
    "repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5df74c5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_percent = 0.3\n",
    "sample_percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2f4c2265",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tuned'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "name='tuned'\n",
    "name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c1e840d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read and prepare data\n",
    "data = GetDataAsPython(f\"{storage_directory}/data_and_models/data/data_autofix_tracking_repo_specific_final.json\")\n",
    "data_eslint = GetDataAsPython(f\"{storage_directory}/data_and_models/data/data_autofix_tracking_eslint_final.json\")\n",
    "data += data_eslint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a1a5fbaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "104804"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "31c8c56a",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_warning_types = extract_warning_types(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "5e85c0cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "splitting by : repo-based-included\n",
      "train size: 105\n",
      "val size: 7\n",
      "test size: 37\n"
     ]
    }
   ],
   "source": [
    "(train_inputs, train_labels, val_inputs, val_labels, test_inputs, test_labels, train_info, val_info, test_info, ) =\\\n",
    "    create_data(data, all_warning_types, include_warning=True, design='repo-based-included', select_repo=repo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "850a57c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_model = f'./{storage_directory}/checkpoint-37375'\n",
    "tokenizer = T5Tokenizer.from_pretrained(load_model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "077034c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "105"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c9aa42f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105 31 0.3\n"
     ]
    }
   ],
   "source": [
    "# Create dataset required by pytorch\n",
    "samples = int(sample_percent * len(train_inputs))\n",
    "print(f'{len(train_inputs)} {samples} {sample_percent}')\n",
    "train_dataset = create_dataset(train_inputs[:samples], train_labels[:samples], tokenizer, pad_truncate=True, max_length=128)\n",
    "val_dataset = create_dataset(val_inputs[:samples], val_labels[:samples], tokenizer, pad_truncate=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "62a42ffc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./storage//tmp/tuned_985_svg.js_0.3'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "now = datetime.now()\n",
    "test_result_directory = f'{storage_directory}/fine-tune-result'\n",
    "full_name = f'{name}_{exec_number}_{repo.rsplit(\"/\", 1)[1][-20:]}_{sample_percent}'\n",
    "model_directory = f'{storage_directory}/tmp/{full_name}'\n",
    "model_directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "22f31e82",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define an objective function to be minimized.\n",
    "def objective(trial):\n",
    "    \n",
    "    clear_output()\n",
    "    # Invoke suggest methods of a Trial object to generate hyperparameters.\n",
    "    lr = trial.suggest_float('lr', 1e-6, 1e-2, log=True)\n",
    "    wd = trial.suggest_float('wd', 0, 0.9)\n",
    "    ws = trial.suggest_float('ws', 0, 2000)\n",
    "    \n",
    "    model = T5ForConditionalGeneration.from_pretrained(load_model)\n",
    "    \n",
    "    model.resize_token_embeddings(len(tokenizer))\n",
    "    \n",
    "    training_args = Seq2SeqTrainingArguments(\n",
    "        output_dir=model_directory,\n",
    "        num_train_epochs=15,\n",
    "        per_device_train_batch_size=16,\n",
    "        per_device_eval_batch_size=16,\n",
    "        warmup_steps=ws,\n",
    "        weight_decay=wd,\n",
    "        logging_dir=model_directory,\n",
    "        logging_steps=100,\n",
    "        do_eval=True,\n",
    "        evaluation_strategy=\"epoch\",\n",
    "        learning_rate=lr,\n",
    "        load_best_model_at_end=True,\n",
    "        metric_for_best_model=\"eval_loss\",\n",
    "        greater_is_better=False,\n",
    "        save_total_limit=1,\n",
    "        eval_accumulation_steps=1,  # set this lower, if testing or validation crashes\n",
    "        disable_tqdm=False,\n",
    "        predict_with_generate=True,  # never set this to false.\n",
    "        seed=42,  # default value\n",
    "    )\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=train_dataset,\n",
    "        eval_dataset=val_dataset,\n",
    "        optimizers=[torch.optim.Adam(params=model.parameters(), lr=lr), None],\n",
    "        tokenizer=tokenizer,\n",
    "    )\n",
    "    \n",
    "    trainer.train()\n",
    "    \n",
    "    return trainer.evaluate()['eval_loss']  # An objective value linked with the Trial object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "fc7adcab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-16 12:43:26,671]\u001b[0m A new study created in memory with name: no-name-9e828f08-dd0b-45bb-b186-29a461b82c15\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [30/30 00:20, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Samples Per Second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062873</td>\n",
       "      <td>0.023700</td>\n",
       "      <td>295.713000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062862</td>\n",
       "      <td>0.023400</td>\n",
       "      <td>298.527000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062844</td>\n",
       "      <td>0.022800</td>\n",
       "      <td>307.388000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062816</td>\n",
       "      <td>0.023300</td>\n",
       "      <td>300.818000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062781</td>\n",
       "      <td>0.024100</td>\n",
       "      <td>290.588000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062738</td>\n",
       "      <td>0.023100</td>\n",
       "      <td>303.580000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062687</td>\n",
       "      <td>0.022900</td>\n",
       "      <td>305.392000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062627</td>\n",
       "      <td>0.023400</td>\n",
       "      <td>298.779000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062558</td>\n",
       "      <td>0.022500</td>\n",
       "      <td>311.523000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062483</td>\n",
       "      <td>0.022700</td>\n",
       "      <td>307.807000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062398</td>\n",
       "      <td>0.023000</td>\n",
       "      <td>304.083000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062305</td>\n",
       "      <td>0.022800</td>\n",
       "      <td>307.217000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062204</td>\n",
       "      <td>0.022600</td>\n",
       "      <td>310.285000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062096</td>\n",
       "      <td>0.022600</td>\n",
       "      <td>309.327000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.061976</td>\n",
       "      <td>0.022700</td>\n",
       "      <td>308.823000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1/1 : < :]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-16 12:43:48,563]\u001b[0m Trial 0 finished with value: 0.0619756281375885 and parameters: {'lr': 2.2647922891592956e-05, 'wd': 0.373997251974964, 'ws': 1116.428316311118}. Best is trial 0 with value: 0.0619756281375885.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [30/30 00:19, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Samples Per Second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062875</td>\n",
       "      <td>0.023900</td>\n",
       "      <td>293.033000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062874</td>\n",
       "      <td>0.022400</td>\n",
       "      <td>311.834000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062873</td>\n",
       "      <td>0.023200</td>\n",
       "      <td>301.566000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062872</td>\n",
       "      <td>0.022800</td>\n",
       "      <td>306.986000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062871</td>\n",
       "      <td>0.022000</td>\n",
       "      <td>317.757000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062869</td>\n",
       "      <td>0.022900</td>\n",
       "      <td>306.316000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062866</td>\n",
       "      <td>0.022800</td>\n",
       "      <td>307.246000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062863</td>\n",
       "      <td>0.022700</td>\n",
       "      <td>307.974000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062860</td>\n",
       "      <td>0.022400</td>\n",
       "      <td>311.871000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062856</td>\n",
       "      <td>0.039300</td>\n",
       "      <td>178.130000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062852</td>\n",
       "      <td>0.021900</td>\n",
       "      <td>319.291000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062847</td>\n",
       "      <td>0.022000</td>\n",
       "      <td>318.198000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062842</td>\n",
       "      <td>0.022700</td>\n",
       "      <td>307.855000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062836</td>\n",
       "      <td>0.023400</td>\n",
       "      <td>299.633000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062830</td>\n",
       "      <td>0.022800</td>\n",
       "      <td>307.076000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1/1 : < :]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-16 12:44:09,589]\u001b[0m Trial 1 finished with value: 0.06282969564199448 and parameters: {'lr': 1.9204962126438614e-06, 'wd': 0.5472255947178908, 'ws': 1688.974805502949}. Best is trial 0 with value: 0.0619756281375885.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [30/30 00:19, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Samples Per Second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062836</td>\n",
       "      <td>0.024700</td>\n",
       "      <td>283.659000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062616</td>\n",
       "      <td>0.022300</td>\n",
       "      <td>313.522000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062234</td>\n",
       "      <td>0.023700</td>\n",
       "      <td>294.807000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.061688</td>\n",
       "      <td>0.023100</td>\n",
       "      <td>302.922000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.060988</td>\n",
       "      <td>0.022700</td>\n",
       "      <td>308.651000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.060147</td>\n",
       "      <td>0.022900</td>\n",
       "      <td>306.278000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.059163</td>\n",
       "      <td>0.022700</td>\n",
       "      <td>307.881000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.058039</td>\n",
       "      <td>0.023100</td>\n",
       "      <td>303.317000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.056800</td>\n",
       "      <td>0.024100</td>\n",
       "      <td>290.177000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.055522</td>\n",
       "      <td>0.024800</td>\n",
       "      <td>281.791000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.054155</td>\n",
       "      <td>0.026800</td>\n",
       "      <td>261.609000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.052738</td>\n",
       "      <td>0.022400</td>\n",
       "      <td>312.954000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.051284</td>\n",
       "      <td>0.023900</td>\n",
       "      <td>292.437000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.049847</td>\n",
       "      <td>0.022900</td>\n",
       "      <td>306.224000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.048345</td>\n",
       "      <td>0.021700</td>\n",
       "      <td>321.984000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1/1 : < :]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-11-16 12:44:30,930]\u001b[0m Trial 2 finished with value: 0.04834530130028725 and parameters: {'lr': 0.00044404090754716056, 'wd': 0.5818183811812224, 'ws': 1069.8045009257423}. Best is trial 2 with value: 0.04834530130028725.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "study = optuna.create_study() \n",
    "study.optimize(objective, n_trials=100) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "ce789f87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.00044404090754716056, 0.5818183811812224, 1069.8045009257423)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = study.best_params['lr']\n",
    "wd = study.best_params['wd']\n",
    "ws = study.best_params['ws']\n",
    "lr, wd, ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e8ed18b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(32104, 512)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained(load_model)\n",
    "model = T5ForConditionalGeneration.from_pretrained(load_model)\n",
    "model.resize_token_embeddings(len(tokenizer))\n",
    "# model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "c7272675",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=model_directory,\n",
    "    num_train_epochs=30,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    warmup_steps=ws,\n",
    "    weight_decay=wd,\n",
    "    logging_dir=model_directory,\n",
    "    logging_steps=100,\n",
    "    do_eval=True,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    learning_rate=lr,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"eval_loss\",\n",
    "    greater_is_better=False,\n",
    "    save_total_limit=1,\n",
    "    eval_accumulation_steps=1,  # set this lower, if testing or validation crashes\n",
    "    disable_tqdm=False,\n",
    "    predict_with_generate=True,  # never set this to false.\n",
    "    seed=42,  # default value\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "a9618af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "import numpy as np\n",
    "def compute_metrics(p):\n",
    "    target_max_length = 256\n",
    "    predictions, labels = p.predictions, p.label_ids\n",
    "    \n",
    "    predictions = np.pad(predictions, ((0, 0), (0, target_max_length - predictions.shape[1])), mode=\"constant\")\n",
    "    predictions = np.delete(predictions, 0, axis=1)\n",
    "    predictions = np.insert(predictions, target_max_length - 1, 0, axis=1)\n",
    "\n",
    "    \n",
    "\n",
    "    labels = np.array(labels)\n",
    "    labels = np.pad(labels, ((0, 0), (0, target_max_length - labels.shape[1])), mode=\"constant\")\n",
    "    labels = np.delete(labels, 0, axis=1)\n",
    "    labels = np.insert(labels, target_max_length - 1, 0, axis=1)\n",
    "    \n",
    "\n",
    "    correct_counter = np.sum(np.all(np.equal(labels, predictions), axis=1))\n",
    "    return {'acc': int(correct_counter)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "74605553",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    optimizers=[torch.optim.Adam(params=model.parameters(), lr=lr), None],\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "be664a15",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='60' max='60' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [60/60 00:43, Epoch 30/30]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Acc</th>\n",
       "      <th>Runtime</th>\n",
       "      <th>Samples Per Second</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062836</td>\n",
       "      <td>0</td>\n",
       "      <td>0.199700</td>\n",
       "      <td>35.061000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062616</td>\n",
       "      <td>0</td>\n",
       "      <td>0.197000</td>\n",
       "      <td>35.526000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.062234</td>\n",
       "      <td>0</td>\n",
       "      <td>0.192600</td>\n",
       "      <td>36.349000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.061688</td>\n",
       "      <td>0</td>\n",
       "      <td>0.197000</td>\n",
       "      <td>35.525000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.060988</td>\n",
       "      <td>0</td>\n",
       "      <td>0.195100</td>\n",
       "      <td>35.872000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.060147</td>\n",
       "      <td>0</td>\n",
       "      <td>0.195800</td>\n",
       "      <td>35.745000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.059163</td>\n",
       "      <td>0</td>\n",
       "      <td>0.194300</td>\n",
       "      <td>36.020000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.058039</td>\n",
       "      <td>0</td>\n",
       "      <td>0.192300</td>\n",
       "      <td>36.405000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.056800</td>\n",
       "      <td>0</td>\n",
       "      <td>0.192400</td>\n",
       "      <td>36.382000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.055522</td>\n",
       "      <td>0</td>\n",
       "      <td>0.191600</td>\n",
       "      <td>36.531000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.054155</td>\n",
       "      <td>0</td>\n",
       "      <td>0.194600</td>\n",
       "      <td>35.967000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.052738</td>\n",
       "      <td>0</td>\n",
       "      <td>0.190000</td>\n",
       "      <td>36.848000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.051284</td>\n",
       "      <td>0</td>\n",
       "      <td>0.195700</td>\n",
       "      <td>35.771000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.049847</td>\n",
       "      <td>0</td>\n",
       "      <td>0.194800</td>\n",
       "      <td>35.940000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.048345</td>\n",
       "      <td>0</td>\n",
       "      <td>0.191000</td>\n",
       "      <td>36.652000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.046897</td>\n",
       "      <td>0</td>\n",
       "      <td>0.192700</td>\n",
       "      <td>36.318000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.045499</td>\n",
       "      <td>0</td>\n",
       "      <td>0.197100</td>\n",
       "      <td>35.523000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.044122</td>\n",
       "      <td>0</td>\n",
       "      <td>0.201700</td>\n",
       "      <td>34.713000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.042730</td>\n",
       "      <td>0</td>\n",
       "      <td>0.199000</td>\n",
       "      <td>35.174000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.041469</td>\n",
       "      <td>0</td>\n",
       "      <td>0.218400</td>\n",
       "      <td>32.044000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.040310</td>\n",
       "      <td>0</td>\n",
       "      <td>0.195300</td>\n",
       "      <td>35.838000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.039359</td>\n",
       "      <td>0</td>\n",
       "      <td>0.199700</td>\n",
       "      <td>35.059000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.038436</td>\n",
       "      <td>0</td>\n",
       "      <td>0.194100</td>\n",
       "      <td>36.058000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.037541</td>\n",
       "      <td>0</td>\n",
       "      <td>0.191900</td>\n",
       "      <td>36.486000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.036761</td>\n",
       "      <td>0</td>\n",
       "      <td>0.213500</td>\n",
       "      <td>32.791000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.035998</td>\n",
       "      <td>0</td>\n",
       "      <td>0.214400</td>\n",
       "      <td>32.652000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.035372</td>\n",
       "      <td>0</td>\n",
       "      <td>0.209300</td>\n",
       "      <td>33.453000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.034960</td>\n",
       "      <td>0</td>\n",
       "      <td>0.191400</td>\n",
       "      <td>36.576000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.034571</td>\n",
       "      <td>0</td>\n",
       "      <td>0.194100</td>\n",
       "      <td>36.063000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.034245</td>\n",
       "      <td>0</td>\n",
       "      <td>0.199100</td>\n",
       "      <td>35.151000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=60, training_loss=0.10599960486094157, metrics={'train_runtime': 43.5513, 'train_samples_per_second': 1.378, 'total_flos': 25991996405760, 'epoch': 30.0})"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "9d799cd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "        </style>\n",
       "      \n",
       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1/1 : < :]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.03424501046538353"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()['eval_loss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "e8d7c520",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model_dir = f'{model_directory}/best'\n",
    "trainer.save_model(best_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "2c6d027e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start time:  12:57:11\n",
      "['no-invalid-this', 'no-throw-literal', 'no-new-wrappers', 'guard-for-in', 'no-new-object', 'comma-style', 'prefer-spread', 'no-caller', 'no-extra-bind', 'no-array-constructor', 'prefer-rest-params', 'generator-star-spacing', 'no-this-before-super', 'no-extend-native', 'no-undef', 'no-useless-escape', 'no-dupe-keys', 'no-console', 'no-constant-condition', 'no-duplicate-case', 'no-empty', 'no-extra-semi', 'no-redeclare', 'no-cond-assign', 'no-extra-boolean-cast', 'no-fallthrough', 'no-unreachable', 'valid-typeof', 'no-unsafe-finally', 'no-unused-vars', 'no-debugger', 'no-unsafe-negation', 'no-case-declarations', 'no-self-assign', 'no-process-exit', 'no-inner-declarations', 'for-direction', 'no-compare-neg-zero', 'no-sparse-arrays', 'no-func-assign', 'no-const-assign', 'no-global-assign', 'use-isnan', 'no-unused-labels', 'require-yield', 'getter-return', 'no-dupe-class-members', 'no-ex-assign', 'constructor-super', 'no-new-symbol', 'no-empty-pattern', 'no-class-assign']\n",
      "splitting by : repo-based-included\n",
      "train size: 105\n",
      "val size: 7\n",
      "test size: 37\n",
      "Loaded tokenizer from directory ./storage//tmp/tuned_985_svg.js_0.3/best\n",
      "Loaded model from directory ./storage//tmp/tuned_985_svg.js_0.3/best\n",
      "cuda:0\n",
      "/home/armin/TFix/env/lib/python3.8/site-packages/transformers/models/t5/tokenization_t5.py:181: UserWarning: This sequence already has </s>. In future versions this behavior may lead to duplicated eos tokens being added.\n",
      "  warnings.warn(\n",
      "Testing has started\n",
      "Number of testing samples:  37\n",
      "rule 0: no-invalid-this, # 1\n",
      "  0%|                                                     | 0/1 [00:00<?, ?it/s]rule 0 acc: 0.0\n",
      "rule 5: comma-style, # 32\n",
      "3it [00:02,  1.42it/s]                                                          rule 5 acc: 0.78125\n",
      "rule 14: no-undef, # 2\n",
      "4it [00:04,  1.10s/it]rule 14 acc: 0.0\n",
      "rule 23: no-cond-assign, # 1\n",
      "5it [00:04,  1.06it/s]rule 23 acc: 1.0\n",
      "rule 26: no-unreachable, # 1\n",
      "6it [00:05,  1.30it/s]rule 26 acc: 1.0\n",
      "score average 0.7297297297297297\n",
      "result : ./storage//16/per-repo/t5-small_test_svg.js_16-11-2021_12-57-11\n",
      "end time:  12:57:25\n",
      "6it [00:05,  1.09it/s]\n"
     ]
    }
   ],
   "source": [
    "!python hf_transformers/tfix_testing.py --load-model $best_model_dir -bs 16 --model-name t5-small -d repo-based-included -r $repo\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ef3e347",
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}