Guido.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras in /opt/conda/envs/tensorflow/lib/python3.5/site-packages\n",
      "Requirement already satisfied: scipy>=0.14 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from keras)\n",
      "Requirement already satisfied: numpy>=1.9.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from keras)\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from keras)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from keras)\n",
      "Requirement already satisfied: matplotlib in /opt/conda/envs/tensorflow/lib/python3.5/site-packages\n",
      "Requirement already satisfied: pytz in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: six>=1.10 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: numpy>=1.7.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from matplotlib)\n",
      "Requirement already satisfied: setuptools in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from kiwisolver>=1.0.1->matplotlib)\n",
      "Requirement already satisfied: sklearn in /opt/conda/envs/tensorflow/lib/python3.5/site-packages\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from sklearn)\n",
      "Requirement already satisfied: imblearn in /opt/conda/envs/tensorflow/lib/python3.5/site-packages\n",
      "Requirement already satisfied: imbalanced-learn in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from imblearn)\n",
      "Requirement already satisfied: numpy in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: scipy in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: python-resize-image in /opt/conda/envs/tensorflow/lib/python3.5/site-packages\n",
      "Requirement already satisfied: pillow in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from python-resize-image)\n",
      "Requirement already satisfied: requests in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from python-resize-image)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from requests->python-resize-image)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from requests->python-resize-image)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from requests->python-resize-image)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from requests->python-resize-image)\n",
      "Requirement already satisfied: opencv-python in /opt/conda/envs/tensorflow/lib/python3.5/site-packages\n",
      "Requirement already satisfied: numpy>=1.11.1 in /opt/conda/envs/tensorflow/lib/python3.5/site-packages (from opencv-python)\n"
     ]
    }
   ],
   "source": [
    "!pip install keras\n",
    "!pip install matplotlib\n",
    "!pip install sklearn\n",
    "!pip install imblearn\n",
    "!pip install python-resize-image\n",
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, I focus just on MNIST data to find a good model that works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/20\n",
      "60000/60000 [==============================] - 64s 1ms/step - loss: 0.2294 - acc: 0.9297 - val_loss: 0.0587 - val_acc: 0.9806\n",
      "Epoch 2/20\n",
      "60000/60000 [==============================] - 63s 1ms/step - loss: 0.0815 - acc: 0.9755 - val_loss: 0.0388 - val_acc: 0.9871\n",
      "Epoch 3/20\n",
      "60000/60000 [==============================] - 65s 1ms/step - loss: 0.0625 - acc: 0.9812 - val_loss: 0.0332 - val_acc: 0.9881\n",
      "Epoch 4/20\n",
      "60000/60000 [==============================] - 63s 1ms/step - loss: 0.0530 - acc: 0.9847 - val_loss: 0.0304 - val_acc: 0.9895\n",
      "Epoch 5/20\n",
      "60000/60000 [==============================] - 63s 1ms/step - loss: 0.0441 - acc: 0.9865 - val_loss: 0.0320 - val_acc: 0.9898\n",
      "Epoch 6/20\n",
      "60000/60000 [==============================] - 62s 1ms/step - loss: 0.0416 - acc: 0.9872 - val_loss: 0.0330 - val_acc: 0.9893\n",
      "Epoch 7/20\n",
      "60000/60000 [==============================] - 63s 1ms/step - loss: 0.0362 - acc: 0.9889 - val_loss: 0.0291 - val_acc: 0.9910\n",
      "Epoch 8/20\n",
      "60000/60000 [==============================] - 64s 1ms/step - loss: 0.0342 - acc: 0.9893 - val_loss: 0.0282 - val_acc: 0.9912\n",
      "Epoch 9/20\n",
      "60000/60000 [==============================] - 63s 1ms/step - loss: 0.0322 - acc: 0.9904 - val_loss: 0.0282 - val_acc: 0.9913\n",
      "Epoch 10/20\n",
      "60000/60000 [==============================] - 63s 1ms/step - loss: 0.0305 - acc: 0.9904 - val_loss: 0.0294 - val_acc: 0.9909\n",
      "Epoch 11/20\n",
      "60000/60000 [==============================] - 64s 1ms/step - loss: 0.0282 - acc: 0.9915 - val_loss: 0.0260 - val_acc: 0.9923\n",
      "Epoch 12/20\n",
      "38900/60000 [==================>...........] - ETA: 21s - loss: 0.0253 - acc: 0.9921"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-171-f6679e5444b1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     58\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m           \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m           validation_data=(x_test_mnist, y_test_mnist))\n\u001b[0m\u001b[1;32m     61\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwritten_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test_mnist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test_mnist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Test loss:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1703\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1704\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1233\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1236\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2476\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2478\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2479\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1321\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Flatten, Input\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "\n",
    "batch_size = 128\n",
    "num_classes = 10\n",
    "epochs = 12\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# the data, split between train and test sets\n",
    "(x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = mnist.load_data()\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train_mnist = x_train_mnist.reshape(x_train_mnist.shape[0], 1, img_rows, img_cols)\n",
    "    x_test_mnist = x_test_mnist.reshape(x_test_mnist.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train_mnist = x_train_mnist.reshape(x_train_mnist.shape[0], img_rows, img_cols, 1)\n",
    "    x_test_mnist = x_test_mnist.reshape(x_test_mnist.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "x_train_mnist = x_train_mnist.astype('float32')\n",
    "x_test_mnist = x_test_mnist.astype('float32')\n",
    "x_train_mnist /= 255\n",
    "x_test_mnist /= 255\n",
    "print('x_train shape:', x_train_mnist.shape)\n",
    "print(x_train_mnist.shape[0], 'train samples')\n",
    "print(x_test_mnist.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train_mnist = keras.utils.to_categorical(y_train_mnist, num_classes)\n",
    "y_test_mnist = keras.utils.to_categorical(y_test_mnist, num_classes)\n",
    "\n",
    "my_input = Input(shape=input_shape, name='written_input')\n",
    "x = Conv2D(32, kernel_size=(3, 3), activation='relu')(my_input)\n",
    "x = Conv2D(64, (3, 3), activation='relu')(x)\n",
    "x = MaxPooling2D(pool_size=(2,2))(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(128, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "output = Dense(10, activation='softmax')(x)\n",
    "\n",
    "written_model = Model(inputs=[my_input], outputs=[output])\n",
    "\n",
    "written_model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "written_model.fit(x_train_mnist, y_train_mnist,\n",
    "          batch_size=100,\n",
    "          epochs=20,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test_mnist, y_test_mnist))\n",
    "\n",
    "score = written_model.evaluate(x_test_mnist, y_test_mnist, verbose=0)\n",
    "\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "written_model.save_weights('weights_written.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ARABIC DATASET"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now I focus just on a labeled set with same format as the given data. Using this data I try to find a good model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 685,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 686,
   "metadata": {},
   "outputs": [],
   "source": [
    "fragment_length = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 687,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('Train_Arabic_Digit.txt','r')\n",
    "x_train = []\n",
    "block = []\n",
    "for line in file:\n",
    "    if '    ' in line:\n",
    "        x_train.append(np.array(block))\n",
    "        block = []\n",
    "        continue\n",
    "    block.append(np.array([-float(i) for i in line.strip('\\n').split(' ')]))\n",
    "x_train.append(np.array(block))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 688,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38, 13)"
      ]
     },
     "execution_count": 688,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 689,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.array([])\n",
    "for i in range(10):\n",
    "    y_train = np.concatenate((y_train,i*np.ones((660))),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 690,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('Test_Arabic_Digit.txt','r')\n",
    "x_test = []\n",
    "block = []\n",
    "for line in file:\n",
    "    if '    ' in line:\n",
    "        x_test.append(np.array(block))\n",
    "        block = []\n",
    "        continue\n",
    "    block.append(np.array([-float(i) for i in line.strip('\\n').split(' ')]))\n",
    "x_test.append(np.array(block))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 691,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = np.array([])\n",
    "for i in range(10):\n",
    "    y_test = np.concatenate((y_test,i*np.ones((220))),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 792,
   "metadata": {},
   "outputs": [],
   "source": [
    "def annoying_teacher_transformer(mat):\n",
    "    new_mat = np.zeros(mat.shape) \n",
    "    new_mat[:,1] = -mat[:,0]\n",
    "    new_mat[:,2] = -mat[:,1]\n",
    "    new_mat[:,9] = -mat[:,2]\n",
    "    new_mat[:,3] = -mat[:,3]\n",
    "    new_mat[:,12] = -mat[:,4]\n",
    "    new_mat[:,0] = -mat[:,5]\n",
    "    new_mat[:,10] = -mat[:,6]\n",
    "    new_mat[:,6] = -mat[:,7]\n",
    "    new_mat[:,11] = -mat[:,8]  \n",
    "    new_mat[:,4] = -mat[:,9]\n",
    "    new_mat[:,8] = -mat[:,10]  \n",
    "    new_mat[:,7] = -mat[:,11] \n",
    "    new_mat[:,5] = -mat[:,12]\n",
    "    return new_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 693,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = [annoying_teacher_transformer(x) for x in x_train]\n",
    "x_test  = [annoying_teacher_transformer(x) for x in x_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 694,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_data(x_old,new_shape):\n",
    "    x_new = np.zeros((len(x_old),new_shape[0],new_shape[1],new_shape[2]))\n",
    "    for idx,im in enumerate(x_old):\n",
    "        im = cv2.cvtColor(im.astype('float32'), cv2.COLOR_GRAY2BGR)\n",
    "        new_im = cv2.resize(im,(new_shape[1],new_shape[0]), interpolation = cv2.INTER_CUBIC)\n",
    "        x_new[idx,:,:,:] = new_im[:,:,0].reshape(new_shape[0],new_shape[1],new_shape[2])\n",
    "    return x_new "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 695,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_new = transform_data(x_train,(fragment_length,13,1))\n",
    "x_test_new = transform_data(x_test,(fragment_length,13,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 696,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "X_tr, y_tr = shuffle(x_train_new, y_train)\n",
    "X_te, y_te = shuffle(x_test_new, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 697,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6600 train samples\n",
      "2200 test samples\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Concatenate, Dense, LSTM, Input, concatenate\n",
    "from keras.optimizers import Adagrad\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 20\n",
    "num_classes = 10\n",
    "\n",
    "X_tr /= 8\n",
    "X_te /= 8\n",
    "\n",
    "y_tr = keras.utils.to_categorical(y_tr, num_classes)\n",
    "y_te = keras.utils.to_categorical(y_te, num_classes)\n",
    "s\n",
    "\n",
    "print(X_tr.shape[0], 'train samples')\n",
    "print(X_te.shape[0], 'test samples')\n",
    "\n",
    "\n",
    "my_input = Input(shape=(fragment_length,13,1), name='spoken_input')\n",
    "x = Conv2D(32, kernel_size=(3, 3), activation='tanh')(my_input)\n",
    "x = Conv2D(64, (3, 3), activation='tanh')(x)\n",
    "x = MaxPooling2D(pool_size=(2,2))(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(100, activation='relu',kernel_initializer='uniform')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "output = Dense(10, activation='softmax')(x)\n",
    "\n",
    "spoken_model = Model(inputs=[my_input], outputs=[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 698,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6600 samples, validate on 2200 samples\n",
      "Epoch 1/20\n",
      "6600/6600 [==============================] - 3s 480us/step - loss: 1.1499 - acc: 0.6436 - val_loss: 0.3277 - val_acc: 0.8982\n",
      "Epoch 2/20\n",
      "6600/6600 [==============================] - 2s 360us/step - loss: 0.3681 - acc: 0.8862 - val_loss: 0.2326 - val_acc: 0.9200\n",
      "Epoch 3/20\n",
      "6600/6600 [==============================] - 2s 351us/step - loss: 0.2498 - acc: 0.9221 - val_loss: 0.1609 - val_acc: 0.9441\n",
      "Epoch 4/20\n",
      "6600/6600 [==============================] - 3s 389us/step - loss: 0.1929 - acc: 0.9394 - val_loss: 0.1467 - val_acc: 0.9477\n",
      "Epoch 5/20\n",
      "6600/6600 [==============================] - 2s 372us/step - loss: 0.1592 - acc: 0.9489 - val_loss: 0.1338 - val_acc: 0.9486\n",
      "Epoch 6/20\n",
      "6600/6600 [==============================] - 2s 375us/step - loss: 0.1329 - acc: 0.9574 - val_loss: 0.1220 - val_acc: 0.9550\n",
      "Epoch 7/20\n",
      "6600/6600 [==============================] - 2s 356us/step - loss: 0.1075 - acc: 0.9676 - val_loss: 0.1095 - val_acc: 0.9568\n",
      "Epoch 8/20\n",
      "6600/6600 [==============================] - 2s 367us/step - loss: 0.1025 - acc: 0.9691 - val_loss: 0.0951 - val_acc: 0.9632\n",
      "Epoch 9/20\n",
      "6600/6600 [==============================] - 2s 345us/step - loss: 0.0909 - acc: 0.9705 - val_loss: 0.1007 - val_acc: 0.9609\n",
      "Epoch 10/20\n",
      "6600/6600 [==============================] - 2s 358us/step - loss: 0.0714 - acc: 0.9788 - val_loss: 0.0827 - val_acc: 0.9695\n",
      "Epoch 11/20\n",
      "6600/6600 [==============================] - 2s 367us/step - loss: 0.0718 - acc: 0.9762 - val_loss: 0.0905 - val_acc: 0.9650\n",
      "Epoch 12/20\n",
      "6600/6600 [==============================] - 2s 366us/step - loss: 0.0650 - acc: 0.9758 - val_loss: 0.0835 - val_acc: 0.9695\n",
      "Epoch 13/20\n",
      "6600/6600 [==============================] - 2s 347us/step - loss: 0.0595 - acc: 0.9820 - val_loss: 0.0741 - val_acc: 0.9723\n",
      "Epoch 14/20\n",
      "6600/6600 [==============================] - 2s 364us/step - loss: 0.0499 - acc: 0.9845 - val_loss: 0.0905 - val_acc: 0.9673\n",
      "Epoch 15/20\n",
      "6600/6600 [==============================] - 2s 364us/step - loss: 0.0484 - acc: 0.9838 - val_loss: 0.1043 - val_acc: 0.9586\n",
      "Epoch 16/20\n",
      "6600/6600 [==============================] - 2s 337us/step - loss: 0.0497 - acc: 0.9859 - val_loss: 0.0773 - val_acc: 0.9727\n",
      "Epoch 17/20\n",
      "6600/6600 [==============================] - 3s 416us/step - loss: 0.0446 - acc: 0.9876 - val_loss: 0.0841 - val_acc: 0.9709\n",
      "Epoch 18/20\n",
      "6600/6600 [==============================] - 2s 352us/step - loss: 0.0407 - acc: 0.9861 - val_loss: 0.1037 - val_acc: 0.9650\n",
      "Epoch 19/20\n",
      "6600/6600 [==============================] - 2s 350us/step - loss: 0.0367 - acc: 0.9889 - val_loss: 0.0916 - val_acc: 0.9686\n",
      "Epoch 20/20\n",
      "6600/6600 [==============================] - 2s 356us/step - loss: 0.0384 - acc: 0.9870 - val_loss: 0.0809 - val_acc: 0.9732\n",
      "Test loss: 0.08086870738712605\n",
      "Test accuracy: 0.9731818181818181\n"
     ]
    }
   ],
   "source": [
    "spoken_model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "spoken_model.fit(X_tr, y_tr,\n",
    "          batch_size=200,\n",
    "          epochs=20,\n",
    "          verbose=1,\n",
    "          validation_data=(X_te, y_te))\n",
    "score = spoken_model.evaluate(X_te, y_te, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "spoken_model.save_weights('spoken_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 699,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8, 0, 3, 9, 9, 7, 0, 6, 3, 4])"
      ]
     },
     "execution_count": 699,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_tr[:10].argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 700,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_spoken_results = spoken_model.predict(X_tr[:10]).argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 701,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8, 0, 3, 9, 9, 7, 0, 6, 3, 4])"
      ]
     },
     "execution_count": 701,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_spoken_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 765,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "df_train_spoken = np.load('spoken_train.npy')\n",
    "df_train_written = np.load('written_train.npy')\n",
    "df_train_y = np.load('match_train.npy')\n",
    "df_assignment_spoken = np.load('spoken_test.npy')\n",
    "df_assignment_written = np.load('written_test.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First all dataframe to same format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 766,
   "metadata": {},
   "outputs": [],
   "source": [
    "fragment_length = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 767,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_written = np.array([x.astype('float32') for x in df_train_written])\n",
    "assignment_written = np.array([x.astype('float32') for x in df_assignment_written])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 768,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_spoken = np.zeros((len(df_train_spoken),13*fragment_length))\n",
    "for idx,im in enumerate(df_train_spoken):\n",
    "    im = cv2.cvtColor(im.astype('float32'), cv2.COLOR_GRAY2BGR)\n",
    "    new_im = cv2.resize(im,(13,fragment_length), interpolation = cv2.INTER_CUBIC)\n",
    "    train_spoken[idx,:] = new_im[:,:,0].reshape(13*fragment_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 769,
   "metadata": {},
   "outputs": [],
   "source": [
    "assignment_spoken = np.zeros((len(df_assignment_spoken),13*fragment_length))\n",
    "for idx,im in enumerate(df_assignment_spoken):\n",
    "    im = cv2.cvtColor(im.astype('float32'), cv2.COLOR_GRAY2BGR)\n",
    "    new_im = cv2.resize(im,(13,fragment_length), interpolation = cv2.INTER_CUBIC)\n",
    "    assignment_spoken[idx,:] = new_im[:,:,0].reshape(13*fragment_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 796,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd3ee12d1d0>"
      ]
     },
     "execution_count": 796,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJYAAAD8CAYAAACVbl3eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADfJJREFUeJzt3V2MXGUZB/D/M7Mzs59tqdCCpSpqRXtDiQ0xkQsISgo34IUGLkxjMHghCSZ6QbxBveJGjBfEWLWhMXzERBt60QBNJSEmxlgIkSKYNnzY1u0Xle1+dGZnzjxe7Gmzrez7PJ2ZZ87O7v+XNLs75+x73tn975ndp+85j6gqiHqtVPQEaGVisCgEg0UhGCwKwWBRCAaLQjBYFILBohAMFoUY6uvB1oxqZcO65D7ttpjjDP+nndze2GT/vPTqPxxKpfRArsOo/ZyHypm5TysrJ7eL2LMpGfs0Tk+hNTVnTrirYInIDgC/BFAG8FtVfSK1f2XDOnz+ye8mx5yZGTaPe/PPzie3H/3puDlG2/HN9HzDa8Pz6SEcY7Ra9g/ChnUz5j5nptLPu1ptmWMMV9L7vPPobnMMoIuXQhEpA3gKwD0AtgJ4UES2djoerSzd/I51G4Cjqvquqs4DeB7Afb2ZFg26boK1CcCxRR8fzx8jiv+rUEQeFpFDInIoOz8XfThaJroJ1gkAmxd9fGP+2GVUdZeqblfV7eU1o10cjgZJN8H6O4AtInKTiFQBPABgX2+mRYOu43KDqrZE5BEAL2Gh3LBbVd/q2cxooHVVx1LV/QD2e/dvN0s4PzmR3GfzZ8+Y4xy7/4bk9ma9YU8ms+tL5VG77nNhOl13q46m61wA0DYKmwAwdcGu783PVJPbsxH7BWpuppbc3sp8L3L8Lx0KwWBRCAaLQjBYFILBohAMFoVgsChEXxf6leYF4++lD/nre54xx/nmn3+U3F45ma7nAEBznb1wrn3Grh1hbTO5udWya1RZ3d5n7tQac5+SUZorjdk1NXlvLL19nnUsKhCDRSEYLArBYFEIBotCMFgUgsGiEAwWhehrgVQFyNLryPClqr0uvl2xtjuuP66mr6YGgPKUXbisbEovKmzUjckCqI7bhcvsnD1ONp4u+o4Pp4u5ADAzZlzZ7TwV8YxFIRgsCsFgUQgGi0IwWBSCwaIQDBaFYLAoRF8LpIBdYPt3y75znaXUtK9yzhy3pMzG7CKqNtKFy5qjKDlvjAEAJXsYaCP9xZ0+P2IPYhWXHTdCBHjGoiAMFoVgsCgEg0UhGCwKwWBRCAaLQjBYFKK/K0jLQMtYofh643pznKyWHqPteVZGDxwA0JpdIG230hXDhtrFz4qjFUljvX1LgNKF9HmiNmJXWesNY9Wsox8P0H0vnfcBTAPIALRUdXs349HK0Ysz1p2qerYH49AKwt+xKES3wVIAL4vIayLy8MftsLjlSXtmtsvD0aDo9qXwdlU9ISIbABwQkXdU9dXFO6jqLgC7AKC2eXOP2k/SctfVGUtVT+RvTwPYi4VWc0RdNcIcE5GJi+8DuBvA4V5NjAZbNy+FGwHsFZGL4zyrqi8mP6OkaE2k6zEvfHireeByPV07mr/Orguhaf9MyYhjnOl0nUrX2LUjTx/sylq7jUuzkp5Ls2lf2W22gnH+MtNNk6Z3AdzS6efTysZyA4VgsCgEg0UhGCwKwWBRCAaLQjBYFKLvV0JbV9IemtxsDlGbSm/XbRfMMRoX7AV46iiiomIsBnT0ns4chctKzVGsNRYdQu25yLCxoJC3iqQiMVgUgsGiEAwWhWCwKASDRSEYLArBYFGI/hZIBeYVyOoo4q19L70q86Mv283GK46rgnXI/rkrD6ULip7nU6vZc5k+OWHPZSI9jueK6/qs0WDduYKUZywKwWBRCAaLQjBYFILBohAMFoVgsChE/xf6GXWsT13zX3OI9stHkttHvvMFc4xy2b5b39ys0cAawCfG04sKz88ZdSEA8/P2t0Hmnb1GEoaMmhsArNs4ndx+xlrYmOMZi0IwWBSCwaIQDBaFYLAoBINFIRgsCsFgUYj+FkhLivJIukg31bALiuPN+eT2luPK4qajKOlZGCdGCxDPQj9PsbbkKJBaiw5bLcetIo0vi/ZqoZ+I7BaR0yJyeNFj60XkgIgcyd9e4zscrRael8KnAey44rHHABxU1S0ADuYfE11iBitvCHDuiofvA7Anf38PgPt7PC8acJ3+8r5RVSfz909i4dbcRJd0/VehqioS124s7qWTTbOXzmrRabBOicgNAJC/Pb3Ujqq6S1W3q+r28sRYh4ejQdNpsPYB2Jm/vxPAC72ZDq0UnnLDcwD+CuBmETkuIg8BeALA10XkCICv5R8TXWJWCVX1wSU23XW1BxMBSqV0MXBqbsQcZ/xqD/wxasOO/sh1+3aSluGqo5eOo4ha99y1spIukFrFXACoX0hfRe7p+wPwv3QoCINFIRgsCsFgUQgGi0IwWBSCwaIQDBaF6P8l9gargAoAQ5s+mdzebts/L55Cn6cUODWbLuh6Lmv3rOzUiqO4aRR0q44VsSOj6abmJeMWCZf2c+1FdJUYLArBYFEIBotCMFgUgsGiEAwWhehrHUtEMTySvop5Zspe6Ff/4mhye1Z3tBkxbvEIAPU5u3VKtZau63hqallm7yNWv2cAatTmPLW79aPpr8sHjjojwDMWBWGwKASDRSEYLArBYFEIBotCMFgUgsGiEH1f6GcVDK/fbxclz9ySHmP9hrPmGPV5+yrntWvnzH1axvNpNBx9chwrCsdvmjL3sZbgeW5beXYmfeMW6/lexDMWhWCwKASDRSEYLArBYFEIBotCMFgUgsGiEH0tkLazEmY/Sq8QbdxqF/FKrXQpcPrYOnsyNXslZHMsvdoVAJqz6YKuOJpzl8qOq5zNPeweQp6rmLNW+lzjWRELdN5L5ycickJE3sj/3es6Gq0anfbSAYBfqOq2/N/+3k6LBl2nvXSIkrr55f0REflH/lK5ZFs5tjxZnToN1q8AfA7ANgCTAH6+1I5sebI6dRQsVT2lqpmqtgH8BsBtvZ0WDbqOgnWxQVPuGwAOL7UvrU5mHSvvpXMHgGtF5DiAxwHcISLbsLC27H0A3wucIw2gTnvp/K6jo7UE5Q/TKzd33vuKOczv992Z3D5x1K77Tt9s97gpOwqX2XT6WO1R+1aRmadw6VjxKsYK0cpG+48nnTQ6FTXZS4cKxGBRCAaLQjBYFILBohAMFoVgsChEf6+EVkCMdW8l83peYGguXUtRz4+L46rgprFwDgDaVeMJORb6eeZSnnO0cRnxtSNJjmElwlfG4hmLYjBYFILBohAMFoVgsCgEg0UhGCwKwWBRiP4WSEtAu5ou4pWtCipg3hOxba+JAxyHKZftnTKjV7PnNpDqecqO75QaCwY9t4rsFZ6xKASDRSEYLArBYFEIBotCMFgUgsGiEAwWhehvgbQNlBrpIl3mWP7ZrqW3DznuliSOxttZy15Bah/IsarTuD0jAJQdDdTbw+nt6piKWitiPc8HPGNREAaLQjBYFILBohAMFoVgsCgEg0Uh+lrHEthXQjfVrh2VGsb2lnMyhtqw3fJkrp6erzYdP7uOOwe6ru42vraeupy0jC+Mc7Ggp+XJZhF5RUT+KSJvicij+ePrReSAiBzJ3y55r3dafTw/By0AP1TVrQC+AuD7IrIVwGMADqrqFgAH84+JAPhankyq6uv5+9MA3gawCcB9APbku+0BcH/UJGnwXNUv7yLyGQC3AvgbgI2qOplvOglgY09nRgPNHSwRGQfwRwA/UNXzi7epqmKJa2cu66Uzy146q4UrWCJSwUKonlHVP+UPn7rYoSJ/e/rjPveyXjpj7KWzWnj+KhQsNAx4W1WfXLRpH4Cd+fs7AbzQ++nRoPLUsb4K4NsA3hSRN/LHfgzgCQB/EJGHAHwA4FsxU6RB5Gl58hcsXU6866qPaJwjz7eM1WqwC6CZsRAQsK8aBny3ioSxYFCqdssTTxG1bVxxvXAwexeTNRXeKpKKxGBRCAaLQjBYFILBohAMFoVgsCgEg0Uh+nsltMOxC/Z6QatAOrPJce/FIbvgODxs941unksXdBWOVZuOfjvtdfZcrGJtzfF86nMjxjHsaQA8Y1EQBotCMFgUgsGiEAwWhWCwKASDRSEYLAqx7AqkrbYj60ZtU+xFm0DmaPDtuS2idXm8Y4ySY5VpVre/VeXh9Dj1uqPJkPNWkBaesSgEg0UhGCwKwWBRCAaLQjBYFILBohB9rWNpCWiNpOskk7NrzHFKxh0cXVcNW609ADQzxyI94ypmdfS4bjuuhC7X7FpX27jNY8nxdTH7afNKaCoSg0UhGCwKwWBRCAaLQjBYFILBohAMFoUQ9TQK7tXBRM5g4X6lF10L4GzfJtC9QZpv1Fw/rarXWTv1NVj/d3CRQ6q6vbAJXKVBmm/Rc+VLIYVgsChE0cHaVfDxr9YgzbfQuRb6OxatXEWfsWiFKixYIrJDRP4lIkdFZFk30RSR90XkTRF5Q0QOFT2fK4nIbhE5LSKHFz1WaAfcQoIlImUATwG4B8BWAA/mXVuXsztVddsyLTc8DWDHFY8V2gG3qDPWbQCOquq7qjoP4HksdGylDqjqqwDOXfFwoR1wiwrWJgDHFn18PH9suVIAL4vIayLycNGTcSq0A+6yu3fDMnW7qp4QkQ0ADojIO/lZYiCoqor06KYMTkWdsU4A2Lzo4xvzx5YlVT2Rvz0NYC8WXsqXO1cH3ChFBevvALaIyE0iUgXwABY6ti47IjImIhMX3wdwN4DD6c9aFgrtgFvIS6GqtkTkEQAvASgD2K2qbxUxF4eNAPYudDDGEIBnVfXFYqd0ORF5DsAdAK4VkeMAHkfBHXBZeacQrLxTCAaLQjBYFILBohAMFoVgsCgEg0UhGCwK8T85zL2RahONtAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_spoken[0].reshape(fragment_length,13))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 771,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(45000, 325)\n",
      "(45000, 784)\n"
     ]
    }
   ],
   "source": [
    "print(train_spoken.shape)\n",
    "print(train_written.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 772,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = df_train_y.reshape(45000,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenate to oversample and shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 773,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = np.concatenate((train_y , train_written , train_spoken),axis=1)\n",
    "data_assignment = np.concatenate((assignment_written , assignment_spoken),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 774,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices_negative = np.where(data_train[:,0]==False)[0]\n",
    "indices_positive = np.where(data_train[:,0]==True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 775,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81292, 1110)"
      ]
     },
     "execution_count": 775,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices_positive = np.random.choice(indices_positive,size=len(indices_negative),replace=True)\n",
    "balanced_data = np.concatenate((data_train[indices_negative],data_train[indices_positive]),axis=0)\n",
    "balanced_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 776,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "data_train = shuffle(balanced_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 777,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test = \\\n",
    "            train_test_split(data_train, test_size=0.2, random_state=4099)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 778,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = X_train[:,0]\n",
    "X_train_written = X_train[:,1:785]\n",
    "X_train_spoken  = X_train[:,785:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 779,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = X_test[:,0]\n",
    "X_test_written = X_test[:,1:785]\n",
    "X_test_spoken  = X_test[:,785:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 780,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_assignment_written = data_assignment[:,:784]\n",
    "X_assignment_spoken = data_assignment[:,784:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 781,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train_spoken = train_spoken\n",
    "# X_train_written = train_written"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform back to correct format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 782,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(65033, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "X_train_written = np.array([x.reshape(28,28,1) for x in X_train_written])\n",
    "X_assignment_written = np.array([x.reshape(28,28,1) for x in X_assignment_written])\n",
    "X_test_written = np.array([x.reshape(28,28,1) for x in X_test_written])\n",
    "print(X_train_written.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 783,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(65033, 25, 13, 1)\n"
     ]
    }
   ],
   "source": [
    "X_train_spoken = np.array([x.reshape(fragment_length,13,1) for x in X_train_spoken])\n",
    "X_test_spoken = np.array([x.reshape(fragment_length,13,1) for x in X_test_spoken])\n",
    "X_assignment_spoken = np.array([x.reshape(fragment_length,13,1) for x in X_assignment_spoken])\n",
    "print(X_train_spoken.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 784,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_written /= 255\n",
    "X_test_written /= 255\n",
    "X_assignment_written /= 255\n",
    "\n",
    "X_train_spoken /= 8\n",
    "X_test_spoken /= 8\n",
    "X_assignment_spoken /= 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 785,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65033 train samples\n",
      "16259 test samples\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Concatenate, Dense, LSTM, Input, concatenate\n",
    "from keras.optimizers import Adagrad\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 2\n",
    "\n",
    "print(X_train_written.shape[0], 'train samples')\n",
    "print(X_test_written.shape[0], 'test samples')\n",
    "\n",
    "x = keras.layers.concatenate([spoken_model.output, written_model.output])\n",
    "output = Dense(1, kernel_initializer='zeros', activation='sigmoid', name='output')(x)\n",
    "\n",
    "final_model = Model(inputs=[spoken_model.input,written_model.input], outputs=[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 786,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model.compile(loss=keras.losses.binary_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 787,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_spoken = np.concatenate((X_train_spoken,X_test_spoken),axis=0)\n",
    "X_written = np.concatenate((X_train_written,X_test_written),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 788,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 1. 1. 0. 1. 0. 1. 1. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "print(y_train[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 724,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Input arrays should have the same number of samples as target arrays. Found 16259 input samples and 2200 target samples.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-724-91792e30a1f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfinal_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_train_spoken\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_train_written\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[0;34m,\u001b[0m                 \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m,\u001b[0m                 \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_test_spoken\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mX_test_written\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1648\u001b[0m                 \u001b[0mval_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1649\u001b[0m                 \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_sample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1650\u001b[0;31m                 batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1651\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muses_learning_phase\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearning_phase\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1652\u001b[0m                 \u001b[0mval_ins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval_x\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mval_y\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mval_sample_weights\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m   1488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1489\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcheck_array_lengths\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1490\u001b[0;31m             \u001b[0m_check_array_lengths\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1491\u001b[0m         _check_loss_and_target_compatibility(y,\n\u001b[1;32m   1492\u001b[0m                                              \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed_loss_fns\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/tensorflow/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_check_array_lengths\u001b[0;34m(inputs, targets, weights)\u001b[0m\n\u001b[1;32m    218\u001b[0m                          \u001b[0;34m'the same number of samples as target arrays. '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m                          \u001b[0;34m'Found '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' input samples '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m                          'and ' + str(list(set_y)[0]) + ' target samples.')\n\u001b[0m\u001b[1;32m    221\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset_w\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m         raise ValueError('All sample_weight arrays should have '\n",
      "\u001b[0;31mValueError\u001b[0m: Input arrays should have the same number of samples as target arrays. Found 16259 input samples and 2200 target samples."
     ]
    }
   ],
   "source": [
    "final_model.fit([X_train_spoken,X_train_written], y_train, \\\n",
    "                epochs=10, batch_size=300, \\\n",
    "                validation_data=([X_test_spoken,X_test_written], y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_spoken = np.concatenate((X_train_spoken,X_test_spoken),axis=0)\n",
    "X_written = np.concatenate((X_train_written,X_test_written),axis=0)\n",
    "Y = np.concatenate((y_train, y_test),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "81292/81292 [==============================] - 77s 947us/step - loss: 0.2400 - acc: 0.9082\n",
      "Epoch 2/10\n",
      "81292/81292 [==============================] - 78s 958us/step - loss: 0.2179 - acc: 0.9187\n",
      "Epoch 3/10\n",
      "81292/81292 [==============================] - 76s 937us/step - loss: 0.2027 - acc: 0.9241\n",
      "Epoch 4/10\n",
      "81292/81292 [==============================] - 77s 942us/step - loss: 0.1972 - acc: 0.9274\n",
      "Epoch 5/10\n",
      "81292/81292 [==============================] - 78s 960us/step - loss: 0.1862 - acc: 0.9307\n",
      "Epoch 6/10\n",
      "81292/81292 [==============================] - 76s 940us/step - loss: 0.1825 - acc: 0.9331\n",
      "Epoch 7/10\n",
      "81292/81292 [==============================] - 77s 949us/step - loss: 0.1742 - acc: 0.9358\n",
      "Epoch 8/10\n",
      "81292/81292 [==============================] - 77s 941us/step - loss: 0.1704 - acc: 0.9375\n",
      "Epoch 9/10\n",
      "81292/81292 [==============================] - 77s 953us/step - loss: 0.1653 - acc: 0.9397\n",
      "Epoch 10/10\n",
      "81292/81292 [==============================] - 77s 948us/step - loss: 0.1639 - acc: 0.9396\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fd34a1fa198>"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_model.fit([X_spoken,X_written], Y, \\\n",
    "                epochs=10, batch_size=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = final_model.predict([X_train_spoken,X_train_written],verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = final_model.evaluate([X_test_spoken, X_test_written], y_test,verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model.save('model4.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.19060262903874486\n",
      "Test accuracy: 0.9455071037579187\n"
     ]
    }
   ],
   "source": [
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate F1 Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For own test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = final_model.predict([X_test_spoken,X_test_written],verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.array([int(y[0]>0.5)for y in y_pred])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9477409460894184"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_test.astype(int),y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For given testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = final_model.predict([X_assignment_spoken,X_assignment_written],verbose=0)\n",
    "y_pred = np.array([y[0]<0.5 for y in y_pred])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15000,)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('result.npy', result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check results of written on emperically checked trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_written_pred =  written_model.predict(X_train_written).argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 9, 0, 2, 1, 8, 7, 7])"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_written_pred[:10] # ALL CORRECT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd3f3b22e48>"
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAECCAYAAAAYUakXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF8JJREFUeJzt3X+QXWV5B/Dvc8/dzWaTTUiyIdn8IJuEBEixCc4OioKNMjhIa4GxQ0tbJx2oQSsWrFop01Fmih2mKsp0OmgUamyBllGUTIsIRWdAK8gmxJCQ32ST7Gazu/m5Ccn+uOc+/WNPXpe493nu7tncc1O/nxkmm/u955w35959OPe+73lfUVUQEQFALusGEFH1YEEgooAFgYgCFgQiClgQiChgQSCiIJOCICLXi8h2EdklIvdk0QaLiLSJyOsislFEWqugPY+KSLeIbB722HQReV5EdiZ/Tquy9t0nIh3JOdwoIjdk2L75IvJTEXlDRLaIyF3J41VxDo32VfwcSqXHIYhIBGAHgOsAtAN4FcCtqvpGRRtiEJE2AC2qeijrtgCAiLwPwEkA31XVy5PH/gnAEVV9ICmq01T181XUvvsAnFTVr2TRpuFEpAlAk6puEJEGAOsB3ATgL1AF59Bo3y2o8DnM4grhSgC7VPVNVR0A8B8AbsygHecNVX0RwJGzHr4RwNrk57UYegNlokT7qoaqdqrqhuTnEwC2ApiLKjmHRvsqLouCMBfA/mF/b0dG/3iDAnhORNaLyOqsG1PCLFXtTH4+CGBWlo0p4U4R2ZR8pMjsI81wItIM4AoAr6AKz+FZ7QMqfA75peLIrlbVdwL4EIBPJpfEVUuHPvdV2xj0hwEsBrACQCeAr2bbHEBEJgP4PoC7VbV3eFYN53CE9lX8HGZREDoAzB/293nJY1VDVTuSP7sB/ABDH3OqTVfy2fPMZ9DujNvzNqrapaqxqhYBfAsZn0MRqcHQL9tjqvpU8nDVnMOR2pfFOcyiILwKYImILBSRWgB/AmBdBu0YkYhMSr7YgYhMAvBBAJvtrTKxDsCq5OdVAJ7OsC2/4cwvWuJmZHgORUQAPAJgq6o+OCyqinNYqn1ZnMOK9zIAQNJ98nUAEYBHVfVLFW9ECSKyCENXBQCQB/B41u0TkScArATQCKALwBcB/BDAkwAuArAXwC2qmskXeyXatxJDl7oKoA3AHcM+r1e6fVcDeAnA6wCKycP3Yuhzeubn0GjfrajwOcykIBBRdeKXikQUsCAQUcCCQEQBCwIRBSwIRBRkWhCqeFgwALYvrWpuXzW3DciufVlfIVT1iwK2L61qbl81tw3IqH1ZFwQiqiKpBiaJyPUAHsLQiMNvq+oD1vNrcxN1Yr4h/H2geBq1uYm/foLXlJyYsQ4MOjsYnUH0owYTyn6+1NaYedr2Se7t9XtA+1Ardb/ef7F49ibjSvKR/YSzXr8BPY1aGfb6ivP6FQpjbFl5hp+/s88dAHi/C+K1P+35H7b7Qe1HjZz93rOPL3Wl36unB45hoHDK3gGGhuaOSTLRyb9g2EQnIrLOmuhkYr4B75n5xyX36Z1Q6x8MAIW9+83c5bzgnvxs+y7uwv72VPvP1U8y8+Jbb6Xavye6YLr9hDi285pae/OenlG26CzO65ebPNnMdWDA3n2t3f7iiRNm7pG88+so9gW9LF1UMnt5xyNltSHNRwZOdEL0/0yagnA+THRCRKMw5o8M5Uq6T1YDQF1kX7IRUbbSXCGUNdGJqq5R1RZVbXnbF4hEVHXSFISqnuiEiEZvzB8ZVLUgIncC+DF+PdHJFnObwQIKnQdL5tGUKeYxC132DFf5hQvs7ffsNXOknBvC60WIli018/iNHWbu9SJESxebObqdWeWL9r8/Pnxu5w7xvmV3uyWd18/rBYh+5xIzj7dst4+fc7pli3YvTG6q/f4vnrRf/+KmbSUz1T5z2zNSfYegqs8AeCbNPoioenCkIhEFLAhEFLAgEFHAgkBEAQsCEQUsCEQUnPOhy6MR9/b6TzJ44wyimTPNXE+dMnNvHEC+abaZF5xxBuLcDRg12ncbFnbstrd3xnmkPf9ppb392Tt/6tyN6Y4z8O6GdcYZeLxxHjKh/Fvxx4pXCEQUsCAQUcCCQEQBCwIRBSwIRBSwIBBRwIJAREFVjUM417xZffPNF5m5Xtps5zvtWZ+jy5aYOWJ71ml3nMEFU+3dHztubz/DHueQdj4Ed5yGMVcG4I8jcV/f+fPMHDX2r0O8z57vwhvnEDnzHXjT9BedcTLjgVcIRBSwIBBRwIJARAELAhEFLAhEFLAgEFHAgkBEQUXHIUguZ65gLBfNMbePt+5M2QD7fvZC2z4zz8d2P3bBm0/Ayb1xAB7t60+1fbH3pJn3fOIqMz/9AXv759/1sJlf8+ynzbz5KTNG7bP2OIS0q2+n5Y0Dyc+eZeZSb698Fh86POo2nY1XCEQUsCAQUcCCQEQBCwIRBSwIRBSwIBBRwIJAREFFxyFosWivbeCMM8jPdcYpdHXbDYgiOy+qGRc6Ou3tHW77nX7kXEODmRdPnEh1/ONXzTfzdX/3ZTOfl59s5ne0X2fmO37/G2bedf1pM7/23z5n5osftNddSDvfgytnv/9U7fef9/4w123od9aUSKQqCCLSBuAEgBhAQVVb0uyPiLI1HlcI71fVQ+OwHyLKGL9DIKIgbUFQAM+JyHoRWT0eDSKi7KT9yHC1qnaIyIUAnheRbar64vAnJIViNQDUoT7l4YjoXEp1haCqHcmf3QB+AODKEZ6zRlVbVLWlBud+9VoiGrsxFwQRmSQiDWd+BvBBAJvHq2FEVHlpPjLMAvADGZpjIA/gcVV91tpAcjnkJpfuS/f60QsdB8w8mnWhmcc9Tj/u8kvNPLfLni8hbfvTjjPw7PvTZjPfcPc/29sX7P3fvu9qM//VI+8w8+sOXG7mPbfZ6xJ85Iafm/lr/2if37S8cR7e668njTE6Zcg1lx5HIm21Ze1jzAVBVd8EsHys2xNR9WG3IxEFLAhEFLAgEFHAgkBEAQsCEQUsCEQUVH4+hJP23P2W3KTSazoA/nwI0ZQp9vavbTFzccYJ5Bc1m7n22uMItH/AzD3xyneaeevdD5n5L/vt+/X/fvefmXn9bbGZz9j/CzP31F9orwvx5OXvMfPFb71s5uZ8AgByDfZ8DygW7dwT2+fP3Xz7rpKZanlrdvAKgYgCFgQiClgQiChgQSCigAWBiAIWBCIKWBCIKKjoOASZMAFR86KSebxjt7m9uaYDgGjpYjP39u/x5iNIO1+BN29/fuECMz+y0O5Hz8Pef43Y/eDt6+37/RelHGfgjQOY+V+l+9kBYOY6u/1eL7/22331sZN78rNnmXnBW1fE278xH4N01ZS1D14hEFHAgkBEAQsCEQUsCEQUsCAQUcCCQEQBCwIRBZWdD6G/3xwLkPp+9G57EepcXZ29fWT307vjILz5Fnp77eMX7Z7ywt52Mz+5YK6ZR2LX/6Kqmdd3ipmn5Y4D6Okx86hxhp0741QGm+zXr2ZTm5m333aZmZ9c3mfmjT8pPUYHAKattedz0BPGXCPOe+sMXiEQUcCCQEQBCwIRBSwIRBSwIBBRwIJARAELAhEFFR2H4NErLrGfsH2/vf1gwcyLfXY/sMdbd6HQZrfPI3n75dCC/e+rXXE01fH/9dDVZj59m71uhDcOwBMfOmzmfX9wpZl3t9jn76Xbv2zmR5xlFZbW2OuCAD91ctuW950285uW/Y2ZL/rb0vNRqJa3ZoR7hSAij4pIt4hsHvbYdBF5XkR2Jn9OK+toRFTVyvnI8B0A15/12D0AXlDVJQBeSP5OROc5tyCo6osAjpz18I0A1iY/rwVw0zi3i4gyMNYvFWepamfy80EA9mRxRHReSN3LoKoKoORdMSKyWkRaRaR1EOkmqSSic2usBaFLRJoAIPmz5HSxqrpGVVtUtaUG9t2MRJStsRaEdQBWJT+vAvD0+DSHiLLkjkMQkScArATQKCLtAL4I4AEAT4rI7QD2ArilrKOJQGpqS8b68iZ78/nzzDzeb/fD55svMvNC2z47f7PNzD1pxxl49EWn99fuxsc359nrKrxz4Qozr/2xPY4gumyJmU96yv7/0x82/sjMP1C/3cyn5Saa+e/98qNmfvqAPR9H7RF7Po0F/22v2yFFez6KpV3OOJwZ00vv+5jdtjPcgqCqt5aIri3rCER03uDQZSIKWBCIKGBBIKKABYGIAhYEIgpYEIgoqOx8CKrQwdL31OebZtubnzpl5tGypWZeeGOHmXvyC+bb+99r9xPn6utTHV9je279mb9y1jVw7okvlh6BDgA4dpW9/5pTV5n5s1/6qplPi+zz47X/4mfuMvNLvmnPN7BgzwEz19P2fBq6zF5XQVs327mZAt4oleiSi0uHJ8v7VecVAhEFLAhEFLAgEFHAgkBEAQsCEQUsCEQUsCAQUVBV6zIUOg+aeTRzpr0DZ12G1PMhOOMMXLU1ZiyT7Xn/Y6d9UZ89TiESu/4fjd8y8+9c86iZv+86M8b6fvvtds+OD5t57Z8PmvnSzlfNPLd0sZnHR4+bOYr2+YUzzsDjjaMp7t5r5rq3vXTYb6+pcQavEIgoYEEgooAFgYgCFgQiClgQiChgQSCigAWBiIKKjkOQXA65+tJ97cW37H5wdfJiT8+Y2nVGZMxrDwDI2XPbF3t7zTw+ZK9bACfPLb/M3n79NjNe+KO/NPPPX2Wve/DxCzrM/LlT9jiLL9z3cTOf+u8vm3m6VSuAeMfuVNvnFy4w88Iee5yAxx1n0O8shWis+6HubAtDeIVARAELAhEFLAhEFLAgEFHAgkBEAQsCEQUsCEQUVHQcghaL5liDyLtfPWU/sic+fOSc7t+TX9Rs5vGWnWY+sHK5me/50LdH26RR+cKOG8182vc2mrm96kJ6UlNr5taaIUD6cQYyYYKd1zrtc8YhaMEYqVHeMAT/CkFEHhWRbhHZPOyx+0SkQ0Q2Jv/dUN7hiKialfOR4TsArh/h8a+p6orkv2fGt1lElAW3IKjqiwCyvZYmoopI86XinSKyKflIMW3cWkREmRlrQXgYwGIAKwB0Aii5iqeIrBaRVhFpHYRzcwYRZWpMBUFVu1Q1VtUigG8BuNJ47hpVbVHVlhrY37ISUbbGVBBEpGnYX28GkG7+aSKqCu44BBF5AsBKAI0i0g7giwBWisgKDPVutgG4Yzwak3acQdp+Zk/UOMM5vj0fgLfuRM81TWZ+fImdb7/tYTPf6PRjf+94i5nff+HrZv71S//TzP9hzkfMXI4780mkHCfivf7euh+xM99GNGWKfXxrnACA4okTZu7Nx5Ez1v2QPrH3nXALgqreOsLDj5S1dyI6r3DoMhEFLAhEFLAgEFHAgkBEAQsCEQUsCEQUVHQ+hHPN62cWY956wO8n9tZVyDU0mHm0bKmZ3/SZn5j5vY3bzfxQbK9b8cSx95r5hruvMPP+xzeYeX0uNvPtfzXbzOf9xB4HUPecPU7Be/28dRWgzqQBzrIfsbMuR66uLlWuTvv0stLziei28kYJ8wqBiAIWBCIKWBCIKGBBIKKABYGIAhYEIgpYEIgoqOg4BKnJI984q2RePHbc3D7nzEdQaO8w86jJ7gcv7G83c493P3tUuNDMb2jYZOa7BwfN/BuHrzHzjXetMPPcS6+Z+Tu++9dmfv9HHjfzmz/wiplv+qzdzx7Nn2fm3uuXdl0Fbz4CFO1xGN44AhTtPLdgrpnHr20xDt5nH/vMMcp6FhH9VmBBIKKABYGIAhYEIgpYEIgoYEEgooAFgYiCys6HoHZfbLHP7istOuMMZIJ9z7fW2es2uMSe215q7f13X2OPQzhcrDfzOXn7fvunn323mS986Rdm7lFnav9bJtvjSA4MHjXzTbjAzL1xBu7r76xL4Ylm2uNg4q5uZ/tGMy90HDDz4l77/Z+7/NKSmez6ublt2EdZzyKi3wosCEQUsCAQUcCCQEQBCwIRBSwIRBSwIBBRUNFxCBoXUDx8ZMzbe+seePMR6IGuMR8bACB2/fTWfVj+sdfN/NqJ9v30i/7nU2Z+6QPG/fAAZJ59P72ePGnm9ZceM3PPhydvNvMffujTZj7hR6+aeW7KFDOPe5yFFRzeOAOPN1+Hx1t3RDdvK52N13wIIjJfRH4qIm+IyBYRuSt5fLqIPC8iO5M/p5V1RCKqWuV8ZCgA+IyqLgPwbgCfFJFlAO4B8IKqLgHwQvJ3IjqPuQVBVTtVdUPy8wkAWwHMBXAjgLXJ09YCuOlcNZKIKmNUXyqKSDOAKwC8AmCWqnYm0UEApSdLJKLzQtkFQUQmA/g+gLtV9W132ejQHUsj3rUkIqtFpFVEWgc13c0lRHRulVUQRKQGQ8XgMVV9Knm4S0SakrwJwIhfwarqGlVtUdWWGilvBVoiykY5vQwC4BEAW1X1wWHROgCrkp9XAXh6/JtHRJVUzjiE9wL4KIDXRWRj8ti9AB4A8KSI3A5gL4Bb3D0poIVCyTiaZvdcxkft++mjixeauXc/eVqH/+h3zfz+WQ85e7DnU6jbXmfmg8sXm/mxi+3tT3/Ynm9h85WPmbnnU2/ab5EJR9N9pIydMS75BfPNvLB3f6rj55svSrd/b90GR66u9Osrfc5kFgm3IKjqzwCU2tu1ZR2FiM4LHLpMRAELAhEFLAhEFLAgEFHAgkBEAQsCEQXirlk/jqbIdH2XlO6pzNXb6xLkptjzIRQOppzvIKXokovNfM/9E838pXd/w8wbo0mjbtNwg2rPt/DLfruvek50ysw/+PM7zXzp5+z5BNz5BqLIjL11F6IZ0+3je3N1OOtypB1HcC69oi+gV4+4gxF4hUBEAQsCEQUsCEQUsCAQUcCCQEQBCwIRBSwIRBRUdF0Gqa1Ffm7pe8YLbfvM7Yun7H5wT37uHDMvdBywd5Cz+8FRLJpx1GqPo3j6cns+g9unHrSP79hXOG3mn92+ysynf2LQzJfUOutiOOMEZII9o5YO2Mf3ZD7OwH3/2ONEvPlCZGrp95d02HNtnMErBCIKWBCIKGBBIKKABYGIAhYEIgpYEIgoYEEgoqCi8yFMzc/Uq6beXLoxUyab23vz2lvz0gNAsa/PzKXG7quN5tjLV7rtW7HMzLGjzYzTjsPw/n06OJBq+9xE5/w77c8tsdfViLfutLf3Xn9nHETacQbeOAFEzv9/Y3sci7WmCQDkpk4pmf3vwSdwfKCL8yEQUflYEIgoYEEgooAFgYgCFgQiClgQiChgQSCiwJ0PQUTmA/gugFkAFMAaVX1IRO4D8DEAPclT71XVZ6x9aTFG8UTpe+b16NEymz0yt5/ZIZcsMvPC5m1m7s77v/GNUbdpNKIppfuhAUAH7HEG6kw34I1T0GX2fA7RoeP29h3p1tXINdnjRHR/h507/fye2Hn/uuM4Fsy1D9B1yIwL7aX/feq9uIlyJkgpAPiMqm4QkQYA60Xk+ST7mqp+pawjEVHVcwuCqnYC6Ex+PiEiWwE4pYyIzkej+g5BRJoBXAHgleShO0Vkk4g8KiLOuE0iqnZlFwQRmQzg+wDuVtVeAA8DWAxgBYauIL5aYrvVItIqIq2Dmu4zPhGdW2UVBBGpwVAxeExVnwIAVe1S1VhViwC+BeDKkbZV1TWq2qKqLTViT6JJRNlyC4KICIBHAGxV1QeHPd407Gk3A9g8/s0jokoqp5fhvQA+CuB1EdmYPHYvgFtFZAWGuiLbANxxTlpIRBVTTi/DzwCMdB+1OeZgJBLlkTPuGY97ekpmZXHuZ4+W2v3k2Oesy+Bw5/13uP3U0y+wj99tn7+owV4XQpzz562rUNxkj9MoppxvID/P7twq7Nlr78BZdyGadaGZx13dZp6rrzfz4ml7XYx41x4z91jjYOSYsyZEgiMViShgQSCigAWBiAIWBCIKWBCIKGBBIKKABYGIgnIGJo0fASRfXn/oSKLGGfbuG+x1HYqddj+yNVdDJXj3w8e72+wdOP38cW+vmUeXLbG3d9ZFyDv9+IWD6eY7cNc18HjjIAbsOQPyTbPt7SP7vS2T5ph5vH2XvX+HNQ5GNS5rH7xCIKKABYGIAhYEIgpYEIgoYEEgooAFgYgCFgQiCkRT3qM+qoOJ9AAYftN6IwB7svlssX3pVHP7qrltwPi3b4GqzvSeVNGC8BsHF2lV1ZbMGuBg+9Kp5vZVc9uA7NrHjwxEFLAgEFGQdUFYk/HxPWxfOtXcvmpuG5BR+zL9DoGIqkvWVwhEVEVYEIgoYEEgooAFgYgCFgQiCv4PkfSBm7ibfXkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(X_train_written[7].reshape(28,28))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check results of spoken trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_transformer(mat):\n",
    "    new_mat = np.zeros(mat.shape) \n",
    "    new_mat[:,1] = -mat[:,0]\n",
    "    new_mat[:,2] = -mat[:,1]\n",
    "    new_mat[:,9] = -mat[:,2]\n",
    "    new_mat[:,3] = -mat[:,3]\n",
    "    new_mat[:,12] = -mat[:,4]\n",
    "    new_mat[:,0] = -mat[:,5]\n",
    "    new_mat[:,10] = -mat[:,6]\n",
    "    new_mat[:,6] = -mat[:,7]\n",
    "    new_mat[:,11] = -mat[:,8]  \n",
    "    new_mat[:,4] = -mat[:,9]\n",
    "    new_mat[:,8] = -mat[:,10]  \n",
    "    new_mat[:,7] = -mat[:,11] \n",
    "    new_mat[:,5] = -mat[:,12]\n",
    "    return new_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 713,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_spoken_pred = spoken_model.predict(X_train_spoken).argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 763,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9967875172097292"
      ]
     },
     "execution_count": 763,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_written_pred == y_spoken_pred ,train_y.flatten() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 735,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9993777777777778"
      ]
     },
     "execution_count": 735,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((y_written_pred == y_spoken_pred) == train_y.flatten()).sum()/45000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict for real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 759,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_spoken  = spoken_model.predict(X_assignment_spoken).argmax(axis=1) \n",
    "pred_written = written_model.predict(X_assignment_written).argmax(axis=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 762,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 762,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(pred_spoken == pred_written)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}