single_CNN_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install keras\n",
    "# !pip install matplotlib\n",
    "# !pip install sklearn\n",
    "# !pip install imblearn\n",
    "# !pip install python-resize-image\n",
    "# !pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Flatten, Input\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras.layers import Concatenate, Dense, LSTM, Input, concatenate\n",
    "from keras.optimizers import Adagrad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train_spoken_raw = np.load('spoken_train.npy')\n",
    "train_written_raw = np.load('written_train.npy')\n",
    "train_y_raw = np.load('match_train.npy')\n",
    "assignment_spoken_raw = np.load('spoken_test.npy')\n",
    "assignment_written_raw = np.load('written_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "fragment_length = 28"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Put data in correct datatype and format\n",
    "### Written"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_written = np.array([x.astype('float32') for x in train_written_raw])\n",
    "assignment_written = np.array([x.astype('float32') for x in assignment_written_raw])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_spoken = np.zeros((len(train_spoken_raw),13*fragment_length))\n",
    "for idx,im in enumerate(train_spoken_raw):\n",
    "    im = cv2.cvtColor(im.astype('float32'), cv2.COLOR_GRAY2BGR)\n",
    "    new_im = cv2.resize(im,(13,fragment_length), interpolation = cv2.INTER_CUBIC)\n",
    "    train_spoken[idx,:] = new_im[:,:,0].reshape(13*fragment_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "assignment_spoken = np.zeros((len(assignment_spoken_raw),13*fragment_length))\n",
    "for idx,im in enumerate(assignment_spoken_raw):\n",
    "    im = cv2.cvtColor(im.astype('float32'), cv2.COLOR_GRAY2BGR)\n",
    "    new_im = cv2.resize(im,(13,fragment_length), interpolation = cv2.INTER_CUBIC)\n",
    "    assignment_spoken[idx,:] = new_im[:,:,0].reshape(13*fragment_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = train_y_raw.reshape(45000,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenate to create train test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = np.concatenate((train_y , train_written , train_spoken),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_set, test_set = \\\n",
    "            train_test_split(train_data, test_size=0.2, random_state=123)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oversample / undersample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "indices_negative = np.where(train_set[:,0]==False)[0]\n",
    "indices_positive = np.where(train_set[:,0]==True)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Oversample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64932, 1149)"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices_positive = np.random.choice(indices_positive,size=len(indices_negative),replace=True)\n",
    "train_balanced = np.concatenate((train_set[indices_negative],train_set[indices_positive]),axis=0)\n",
    "balanced_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Undersample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "# indices_negative = np.random.choice(indices_negative,size=len(indices_positive),replace=False)\n",
    "# train_set_balanced = np.concatenate((train_set[indices_negative],train_set[indices_positive]),axis=0)\n",
    "# train_set_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data_balanced = train_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "train_balanced = shuffle(train_balanced)\n",
    "train_balanced = shuffle(train_balanced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = train_balanced[:,0]\n",
    "train_written = train_balanced[:,1:785]\n",
    "train_spoken  = train_balanced[:,785:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_y = test_set[:,0]\n",
    "test_written = test_set[:,1:785]\n",
    "test_spoken  = test_set[:,785:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform to proper 2D- format "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64932, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "train_written = np.array([x.reshape(28,28,1) for x in train_written])\n",
    "test_written = np.array([x.reshape(28,28,1) for x in test_written])\n",
    "assignment_written = np.array([x.reshape(28,28,1) for x in assignment_written])\n",
    "print(train_written.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64932, 28, 13, 1)\n"
     ]
    }
   ],
   "source": [
    "train_spoken = np.array([x.reshape(fragment_length,13,1) for x in train_spoken])\n",
    "test_spoken = np.array([x.reshape(fragment_length,13,1) for x in test_spoken])\n",
    "assignment_spoken = np.array([x.reshape(fragment_length,13,1) for x in assignment_spoken])\n",
    "print(train_spoken.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = train_y.reshape(len(train_y),1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_written /= 255\n",
    "test_written /= 255\n",
    "assignment_written /= 255\n",
    "\n",
    "train_spoken /= 8\n",
    "test_spoken /= 8\n",
    "assignment_spoken /= 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge written and spoken "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64932, 28, 28, 1)"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_written.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64932, 28, 13, 1)"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_spoken.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64932, 28, 3, 1)"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zeros_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "zeros_train = np.zeros(train_written.shape)[:,:,:3,:]\n",
    "train_x = np.concatenate((train_written,zeros_train,train_spoken),axis=2)\n",
    "zeros_test = np.zeros(test_written.shape)[:,:,:3,:]\n",
    "test_x = np.concatenate((test_written,zeros_test,test_spoken),axis=2)\n",
    "zeros_assignment = np.zeros(assignment_written.shape)[:,:,:3,:]\n",
    "assignment_x = np.concatenate((assignment_written,zeros_assignment,assignment_spoken),axis=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import Callback\n",
    "from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score\n",
    "class Metrics(Callback):\n",
    "\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.val_f1s = []\n",
    "        self.val_recalls = []\n",
    "        self.val_precisions = []\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        val_predict = (np.asarray(self.model.predict(self.validation_data[0]))).round()\n",
    "        val_targ = self.validation_data[1]\n",
    "        _val_f1 = f1_score(val_targ, val_predict)\n",
    "        self.val_f1s.append(_val_f1)\n",
    "        print(\"— val_f1: %2f\"%(_val_f1))\n",
    "        return 0\n",
    "    \n",
    "metrics = Metrics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model for combined images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "written_input = Input(shape=(28,28+13+3,1), name='written_input')\n",
    "x = Conv2D(50, kernel_size=(3, 3), activation='tanh')(written_input)\n",
    "x = Conv2D(100, (3, 3), activation='tanh')(x)\n",
    "x = MaxPooling2D(pool_size=(2,2))(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(100, activation='relu')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "output = Dense(1, activation='sigmoid', name='output')(x)\n",
    "# output = Dense(10, activation='softmax')(x)\n",
    "\n",
    "# written_model = Model(inputs=[written_input], outputs=[output1])\n",
    "final_model = Model(inputs=[written_input], outputs=[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model.compile(loss=keras.losses.binary_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_train_data_x = np.concatenate((train_x,test_x),axis=0)\n",
    "# all_train_data_y = np.concatenate((train_y,test_y.reshape(len(test_y),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 73932 samples, validate on 9000 samples\n",
      "Epoch 1/50\n",
      "23100/73932 [========>.....................] - ETA: 2:43 - loss: 0.0285 - acc: 0.9885"
     ]
    }
   ],
   "source": [
    "final_model.fit(train_x,test_x,\n",
    "          batch_size=100,\n",
    "          epochs=50,\n",
    "          verbose=1, \n",
    "          callbacks=[metrics],\n",
    "          validation_data=(test_x, test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "written_input (InputLayer)   (None, 28, 44, 1)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_37 (Conv2D)           (None, 26, 42, 50)        500       \n",
      "_________________________________________________________________\n",
      "conv2d_38 (Conv2D)           (None, 24, 40, 100)       45100     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_19 (MaxPooling (None, 12, 20, 100)       0         \n",
      "_________________________________________________________________\n",
      "dropout_37 (Dropout)         (None, 12, 20, 100)       0         \n",
      "_________________________________________________________________\n",
      "flatten_19 (Flatten)         (None, 24000)             0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 100)               2400100   \n",
      "_________________________________________________________________\n",
      "dropout_38 (Dropout)         (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 2,445,801\n",
      "Trainable params: 2,445,801\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "final_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "pre_test_y = final_model.predict(test_x).round()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9957343083485679"
      ]
     },
     "execution_count": 315,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(pre_test_y,test_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BELOW NOT UNNUSED "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submodel Spoken digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# spoken_input = Input(shape=(fragment_length,13,1), name='spoken_input')\n",
    "# x = Conv2D(32, kernel_size=(3, 3), activation='tanh')(spoken_input)\n",
    "# x = Conv2D(64, (3, 3), activation='relu')(x)\n",
    "# x = MaxPooling2D(pool_size=(2,2))(x)\n",
    "# x = Dropout(0.25)(x)\n",
    "# x = Flatten()(x)\n",
    "# x = Dense(125, activation='relu',kernel_initializer='uniform')(x)\n",
    "# output2 = Dropout(0.5)(x)\n",
    "# # output = Dense(10, activation='softmax')(x)\n",
    "\n",
    "# spoken_model = Model(inputs=[spoken_input], outputs=[output2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Last layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = keras.layers.concatenate([spoken_model.output, written_model.output])\n",
    "output = Dense(1, activation='sigmoid', name='output')(x)\n",
    "\n",
    "final_model = Model(inputs=[spoken_model.input,written_model.input], outputs=[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model.compile(loss=keras.losses.binary_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model.fit([train_spoken,train_written], train_y,\n",
    "          batch_size=200,\n",
    "          epochs=100,\n",
    "          verbose=1, \n",
    "          validation_data=([test_spoken,test_written], test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}