Final_Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### English to Marathi Neural Machine Translation using Encoder-Decoder Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "from string import digits\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import re\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.layers import Input, LSTM, Embedding, Dense\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "lines= pd.read_table('mar.txt', names=['eng', 'mar'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lowercase all characters\n",
    "lines.eng=lines.eng.apply(lambda x: x.lower())\n",
    "lines.mar=lines.mar.apply(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove quotes\n",
    "lines.eng=lines.eng.apply(lambda x: re.sub(\"'\", '', x))\n",
    "lines.mar=lines.mar.apply(lambda x: re.sub(\"'\", '', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude = set(string.punctuation) # Set of all special characters\n",
    "# Remove all the special characters\n",
    "lines.eng=lines.eng.apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n",
    "lines.mar=lines.mar.apply(lambda x: ''.join(ch for ch in x if ch not in exclude))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove all numbers from text\n",
    "remove_digits = str.maketrans('', '', digits)\n",
    "lines.eng=lines.eng.apply(lambda x: x.translate(remove_digits))\n",
    "lines.mar = lines.mar.apply(lambda x: re.sub(\"[२३०८१५७९४६]\", \"\", x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove extra spaces\n",
    "lines.eng=lines.eng.apply(lambda x: x.strip())\n",
    "lines.mar=lines.mar.apply(lambda x: x.strip())\n",
    "lines.eng=lines.eng.apply(lambda x: re.sub(\" +\", \" \", x))\n",
    "lines.mar=lines.mar.apply(lambda x: re.sub(\" +\", \" \", x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add start and end tokens to target sequences\n",
    "lines.mar = lines.mar.apply(lambda x : 'START_ '+ x + ' _END')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Computation the vocabulary for both English and Marathi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vocabulary of English\n",
    "all_eng_words=set()\n",
    "for eng in lines.eng:\n",
    "    for word in eng.split():\n",
    "        if word not in all_eng_words:\n",
    "            all_eng_words.add(word)\n",
    "\n",
    "# Vocabulary of French \n",
    "all_marathi_words=set()\n",
    "for mar in lines.mar:\n",
    "    for word in mar.split():\n",
    "        if word not in all_marathi_words:\n",
    "            all_marathi_words.add(word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Length of maximum sequence for both the languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Max Length of source sequence\n",
    "lenght_list=[]\n",
    "for l in lines.eng:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "max_length_src = np.max(lenght_list)\n",
    "max_length_src\n",
    "\n",
    "# Max Length of target sequence\n",
    "lenght_list=[]\n",
    "for l in lines.mar:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "max_length_tar = np.max(lenght_list)\n",
    "max_length_tar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Computation of vocabulary sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_words = sorted(list(all_eng_words))\n",
    "target_words = sorted(list(all_marathi_words))\n",
    "# Create Vocab Size for both source and target\n",
    "num_encoder_tokens = len(all_eng_words)\n",
    "num_decoder_tokens = len(all_marathi_words)\n",
    "num_encoder_tokens, num_decoder_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_decoder_tokens += 1 # For zero padding\n",
    "num_decoder_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Finally we create 4 Python dictionaries (two for each language) to convert a given token into an integer index and vice-versa."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create word to token dictionary for both source and target\n",
    "input_token_index = dict([(word, i+1) for i, word in enumerate(input_words)])\n",
    "target_token_index = dict([(word, i+1) for i, word in enumerate(target_words)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create token to word dictionary for both source and target\n",
    "reverse_input_char_index = dict((i, word) for word, i in input_token_index.items())\n",
    "reverse_target_char_index = dict((i, word) for word, i in target_token_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = shuffle(lines)\n",
    "lines.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We make a 90–10 train and test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train - Test Split\n",
    "X, y = lines.eng, lines.mar\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save the train and test dataframes for reproducing the results later, as they are shuffled."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_pickle('X_train.pkl')\n",
    "X_test.to_pickle('X_test.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Python generator function to load the data in batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(X = X_train, y = y_train, batch_size = 128):\n",
    "    ''' Generate a batch of data '''\n",
    "    while True:\n",
    "        for j in range(0, len(X), batch_size):\n",
    "            encoder_input_data = np.zeros((batch_size, max_length_src),dtype='float32')\n",
    "            decoder_input_data = np.zeros((batch_size, max_length_tar),dtype='float32')\n",
    "            decoder_target_data = np.zeros((batch_size, max_length_tar, num_decoder_tokens),dtype='float32')\n",
    "            for i, (input_text, target_text) in enumerate(zip(X[j:j+batch_size], y[j:j+batch_size])):\n",
    "                for t, word in enumerate(input_text.split()):\n",
    "                    encoder_input_data[i, t] = input_token_index[word] # encoder input seq\n",
    "                for t, word in enumerate(target_text.split()):\n",
    "                    if t<len(target_text.split())-1:\n",
    "                        decoder_input_data[i, t] = target_token_index[word] # decoder input seq\n",
    "                    if t>0:\n",
    "                        # decoder target sequence (one hot encoded)\n",
    "                        # does not include the START_ token\n",
    "                        # Offset by one timestep\n",
    "                        decoder_target_data[i, t - 1, target_token_index[word]] = 1.\n",
    "            yield([encoder_input_data, decoder_input_data], decoder_target_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder - Decoder Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_dim = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "enc_emb =  Embedding(num_encoder_tokens, latent_dim, mask_zero = True)(encoder_inputs)\n",
    "encoder_lstm = LSTM(latent_dim, return_state=True)\n",
    "encoder_outputs, state_h, state_c = encoder_lstm(enc_emb)\n",
    "# We discard `encoder_outputs` and only keep the states.\n",
    "encoder_states = [state_h, state_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up the decoder, using `encoder_states` as initial state.\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "dec_emb_layer = Embedding(num_decoder_tokens, latent_dim, mask_zero = True)\n",
    "dec_emb = dec_emb_layer(decoder_inputs)\n",
    "# We set up our decoder to return full output sequences,\n",
    "# and to return internal states as well. We don't use the\n",
    "# return states in the training model, but we will use them in inference.\n",
    "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
    "decoder_outputs, _, _ = decoder_lstm(dec_emb,\n",
    "                                     initial_state=encoder_states)\n",
    "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "# Define the model that will turn\n",
    "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_samples = len(X_train)\n",
    "val_samples = len(X_test)\n",
    "batch_size = 128\n",
    "epochs = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit_generator(generator = generate_batch(X_train, y_train, batch_size = batch_size),\n",
    "                    steps_per_epoch = train_samples//batch_size,\n",
    "                    epochs=epochs,\n",
    "                    validation_data = generate_batch(X_test, y_test, batch_size = batch_size),\n",
    "                    validation_steps = val_samples//batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights('nmt_weights.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the weights (if you close the application)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('nmt_weights.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode the input sequence to get the \"thought vectors\"\n",
    "encoder_model = Model(encoder_inputs, encoder_states)\n",
    "\n",
    "# Decoder setup\n",
    "# Below tensors will hold the states of the previous time step\n",
    "decoder_state_input_h = Input(shape=(latent_dim,))\n",
    "decoder_state_input_c = Input(shape=(latent_dim,))\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "dec_emb2= dec_emb_layer(decoder_inputs) # Get the embeddings of the decoder sequence\n",
    "\n",
    "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
    "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=decoder_states_inputs)\n",
    "decoder_states2 = [state_h2, state_c2]\n",
    "decoder_outputs2 = decoder_dense(decoder_outputs2) # A dense softmax layer to generate prob dist. over the target vocabulary\n",
    "\n",
    "# Final decoder model\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + decoder_states_inputs,\n",
    "    [decoder_outputs2] + decoder_states2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Finally, we generate the output sequence by invoking the above setup in a loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1,1))\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    target_seq[0, 0] = target_token_index['START_']\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += ' '+sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if (sampled_char == '_END' or\n",
    "           len(decoded_sentence) > 50):\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1,1))\n",
    "        target_seq[0, 0] = sampled_token_index\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation on Train Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_gen = generate_batch(X_train, y_train, batch_size = 1)\n",
    "k=-1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The code given below can be run in as many cells as you want to decode more number of sample sentences from train dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k+=1\n",
    "(input_seq, actual_output), _ = next(train_gen)\n",
    "decoded_sentence = decode_sequence(input_seq)\n",
    "print('Input English sentence:', X_train[k:k+1].values[0])\n",
    "print('Actual Marathi Translation:', y_train[k:k+1].values[0][6:-4])\n",
    "print('Predicted Marathi Translation:', decoded_sentence[:-4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation on Validation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_gen = generate_batch(X_test, y_test, batch_size = 1)\n",
    "k=-1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The code given below can be run in as many cells as you want to decode more number of sample sentences from test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k+=1\n",
    "(input_seq, actual_output), _ = next(val_gen)\n",
    "decoded_sentence = decode_sequence(input_seq)\n",
    "print('Input English sentence:', X_test[k:k+1].values[0])\n",
    "print('Actual Marathi Translation:', y_test[k:k+1].values[0][6:-4])\n",
    "print('Predicted Marathi Translation:', decoded_sentence[:-4])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}