LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM (IMDB sentiment classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "import time\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "RANDOM_SEED = 7\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "\n",
    "\n",
    "## Hyper-parameters\n",
    "VOCABULARY_SIZE = 20000\n",
    "LEARNING_RATE = 1e-4\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 10\n",
    "\n",
    "## Model Architecture\n",
    "DEVICE = DEVICE = torch.device('cuda:3' if torch.cuda.is_available() else 'cpu')\n",
    "EMBEDDING_DIM = 128\n",
    "HIDDEN_DIM = 256\n",
    "OUTPUT_DIM = 1\n",
    "BIDIRECTIONAL = True\n",
    "NUM_LAYERS = 2\n",
    "DROPOUT = 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/yelp_review_polarity_csv/train.csv\", header=None, index_col=None)\n",
    "df.columns = ['classlabel', 'content']\n",
    "df['classlabel'] = df['classlabel']-1\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(df[\"classlabel\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.bincount(df['classlabel'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[0:5000,['classlabel', 'content']].to_csv('data/yelp_review_polarity_csv/train_prepocessed.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/yelp_review_polarity_csv/test.csv', header=None, index_col=None)\n",
    "df.columns = ['classlabel', 'content']\n",
    "df['classlabel'] = df['classlabel']-1\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(df['classlabel'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.bincount(df['classlabel'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[:1000,['classlabel', 'content']].to_csv('data/yelp_review_polarity_csv/test_prepocessed.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = data.Field(sequential=True,\n",
    "                  tokenize='spacy',\n",
    "                  include_lengths=True) # necessary for packed_padded_sequence\n",
    "\n",
    "LABEL = data.LabelField(dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = [('classlabel', LABEL), ('content', TEXT)]\n",
    "\n",
    "train_dataset = data.TabularDataset(\n",
    "    path = \"data/yelp_review_polarity_csv/train_prepocessed.csv\",\n",
    "    format = \"CSV\",\n",
    "    fields = fields,\n",
    "    skip_header = True)\n",
    "\n",
    "test_dataset = data.TabularDataset(\n",
    "    path=\"data/yelp_review_polarity_csv/test_prepocessed.csv\", \n",
    "    format='csv',\n",
    "    fields = fields,\n",
    "    skip_header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, valid_data = train_dataset.split(\n",
    "    split_ratio=[0.8, 0.2],\n",
    "    random_state=random.seed(RANDOM_SEED))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Num Train: {len(train_data)}')\n",
    "print(f'Num Valid: {len(valid_data)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT.build_vocab(train_data, \n",
    "                 max_size = VOCABULARY_SIZE)\n",
    "LABEL.build_vocab(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(LABEL.vocab.freqs)[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader, valid_loader, test_loader = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_dataset), \n",
    "    batch_size=BATCH_SIZE,\n",
    "    sort_within_batch=True, # necessary for packed_padded_sequence\n",
    "    sort_key=lambda x: len(x.content),\n",
    "    device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Train')\n",
    "for batch in train_loader:\n",
    "    print(f'Text matrix size: {batch.content[0].size()}')\n",
    "    print(f'Target vector size: {batch.classlabel.size()}')\n",
    "    break\n",
    "    \n",
    "print('\\nValid:')\n",
    "for batch in valid_loader:\n",
    "    print(f'Text matrix size: {batch.content[0].size()}')\n",
    "    print(f'Target vector size: {batch.classlabel.size()}')\n",
    "    break\n",
    "    \n",
    "print('\\nTest:')\n",
    "for batch in test_loader:\n",
    "    print(f'Text matrix size: {batch.content[0].size()}')\n",
    "    print(f'Target vector size: {batch.classlabel.size()}')\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNZrc(torch.nn.Module):\n",
    "    def __init__(self, input_dim, \n",
    "                 embedding_dim, \n",
    "                 bidirectional, \n",
    "                 hidden_dim, \n",
    "                 num_layers, \n",
    "                 output_dim, \n",
    "                 dropout, \n",
    "                 pad_idx):\n",
    "        \n",
    "        super(RNNZrc, self).__init__()\n",
    "        self.embedding = torch.nn.Embedding(input_dim, embedding_dim, padding_idx=pad_idx)\n",
    "        self.rnn = torch.nn.LSTM(embedding_dim, \n",
    "                           hidden_dim,\n",
    "                           num_layers=num_layers,\n",
    "                           bidirectional=bidirectional, \n",
    "                           dropout=dropout)\n",
    "        self.fc1 = torch.nn.Linear(hidden_dim * num_layers, 64)\n",
    "        self.fc2 = torch.nn.Linear(64, output_dim)\n",
    "        self.dropout = torch.nn.Dropout(dropout)\n",
    "    \n",
    "    def forward(self, text, text_length):\n",
    "\n",
    "        embedded = self.dropout(self.embedding(text))\n",
    "        packed_embedded = torch.nn.utils.rnn.pack_padded_sequence(embedded, text_length)\n",
    "        packed_output, (hidden, cell) = self.rnn(packed_embedded)\n",
    "        output, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(packed_output)\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))\n",
    "        hidden = self.fc1(hidden)\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIM = len(TEXT.vocab)\n",
    "\n",
    "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "model = RNNZrc(INPUT_DIM, EMBEDDING_DIM, BIDIRECTIONAL, HIDDEN_DIM, NUM_LAYERS, OUTPUT_DIM, DROPOUT, PAD_IDX)\n",
    "model = model.to(DEVICE)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(model, data_loader, device):\n",
    "    model.eval()\n",
    "    correct_pred, num_examples = 0, 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, batch_data in enumerate(data_loader):\n",
    "            text, text_lengths = batch_data.content\n",
    "            logits = model(text, text_lengths).squeeze(1)\n",
    "            _, predicted_labels = torch.max(logits, 1)\n",
    "            num_examples += batch_data.classlabel.size(0)\n",
    "            correct_pred += (predicted_labels.long() == batch_data.classlabel.long()).sum()\n",
    "        return correct_pred.float()/num_examples * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "for epoch in range(NUM_EPOCHS):\n",
    "    model.train()\n",
    "    for batch_idx, batch_data in enumerate(train_loader):\n",
    "        \n",
    "        text, text_lengths = batch_data.content\n",
    "        \n",
    "        ### FORWARD AND BACK PROP\n",
    "        logits = model(text, text_lengths).squeeze(1)\n",
    "        cost = F.cross_entropy(logits, batch_data.classlabel.long())\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        cost.backward()\n",
    "        \n",
    "        ### UPDATE MODEL PARAMETERS\n",
    "        optimizer.step()\n",
    "        \n",
    "        ### LOGGING\n",
    "        if not batch_idx % 1000:\n",
    "            print (f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d} | '\n",
    "                   f'Batch {batch_idx:03d}/{len(train_loader):03d} | '\n",
    "                   f'Cost: {cost:.4f}')\n",
    "\n",
    "    with torch.set_grad_enabled(False):\n",
    "        print(f'training accuracy: '\n",
    "              f'{compute_accuracy(model, train_loader, DEVICE):.2f}%'\n",
    "              f'\\nvalid accuracy: '\n",
    "              f'{compute_accuracy(model, valid_loader, DEVICE):.2f}%')\n",
    "        \n",
    "    print(f'Time elapsed: {(time.time() - start_time)/60:.2f} min')\n",
    "    \n",
    "print(f'Total Training Time: {(time.time() - start_time)/60:.2f} min')\n",
    "print(f'Test accuracy: {compute_accuracy(model, test_loader, DEVICE):.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tryit",
   "language": "python",
   "name": "tryit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}