Real.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Neural Network (MLP) Algorithm for Yeast Data\n",
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from math import floor  #, ceil\n",
    "#from sklearn.metrics import classification_report\n",
    "#from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Preparation of the Data\n",
    "### Define methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def the_train_test_split(X, test_ratio=0.2):\n",
    "    if (test_ratio >= 1 or test_ratio < 0):\n",
    "        test_ratio = 0.2\n",
    "    row, _ = X.shape\n",
    "    train_count = floor(row * (1 - test_ratio))\n",
    "    train = X[:train_count]\n",
    "    test = X[train_count:]\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "       DBName     1     2     3     4    5    6     7     8 OutName\n0  ADT1_YEAST  0.58  0.61  0.47  0.13  0.5  0.0  0.48  0.22     MIT\n1  ADT2_YEAST  0.43  0.67  0.48  0.27  0.5  0.0  0.53  0.22     MIT\n2  ADT3_YEAST  0.64  0.62  0.49  0.15  0.5  0.0  0.53  0.22     MIT\n3  AAR2_YEAST  0.58  0.44  0.57  0.13  0.5  0.0  0.54  0.22     NUC\n4  AATM_YEAST  0.42  0.44  0.48  0.54  0.5  0.0  0.48  0.22     MIT",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>DBName</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>OutName</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ADT1_YEAST</td>\n      <td>0.58</td>\n      <td>0.61</td>\n      <td>0.47</td>\n      <td>0.13</td>\n      <td>0.5</td>\n      <td>0.0</td>\n      <td>0.48</td>\n      <td>0.22</td>\n      <td>MIT</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>ADT2_YEAST</td>\n      <td>0.43</td>\n      <td>0.67</td>\n      <td>0.48</td>\n      <td>0.27</td>\n      <td>0.5</td>\n      <td>0.0</td>\n      <td>0.53</td>\n      <td>0.22</td>\n      <td>MIT</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>ADT3_YEAST</td>\n      <td>0.64</td>\n      <td>0.62</td>\n      <td>0.49</td>\n      <td>0.15</td>\n      <td>0.5</td>\n      <td>0.0</td>\n      <td>0.53</td>\n      <td>0.22</td>\n      <td>MIT</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AAR2_YEAST</td>\n      <td>0.58</td>\n      <td>0.44</td>\n      <td>0.57</td>\n      <td>0.13</td>\n      <td>0.5</td>\n      <td>0.0</td>\n      <td>0.54</td>\n      <td>0.22</td>\n      <td>NUC</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>AATM_YEAST</td>\n      <td>0.42</td>\n      <td>0.44</td>\n      <td>0.48</td>\n      <td>0.54</td>\n      <td>0.5</td>\n      <td>0.0</td>\n      <td>0.48</td>\n      <td>0.22</td>\n      <td>MIT</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_name = \"yeast.csv\"\n",
    "md = pd.read_csv(file_name)\n",
    "\n",
    "# md.dropna(inplace = True)\n",
    "# md.replace('unknown', 0, inplace = True)\n",
    "md.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Prepare the data\n",
    "* Shuffle the data\n",
    "* Separate the input and output variables\n",
    "* Seperate the data into training and test sets\n",
    "* Normalize the data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "gl = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "# Shuffle the data to get more fair representative\n",
    "md.reindex(np.random.permutation(md.index))\n",
    "\n",
    "test_ratio = 0.2\n",
    "X = md.values[:, 1:9]\n",
    "Y = md.values[:, 9:]\n",
    "cat = pd.unique(Y[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for i in range(X.shape[1]):\n",
    "    X[:, i] = (X[:, i] - X[:, i].mean()) / X[:, i].std()\n",
    "\n",
    "#\n",
    "y = np.zeros((len(Y), 10))\n",
    "for i in range(len(Y)):\n",
    "    for j in range(10):\n",
    "        if cat[j] == Y[i]:\n",
    "            y[i][j] = 1\n",
    "        else:\n",
    "            y[i][j] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test = the_train_test_split(X.astype(\"float64\"), test_ratio=test_ratio)\n",
    "Y_train, Y_test = the_train_test_split(y.astype(\"float64\"), test_ratio=test_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "class Neural_Network():\n",
    "    def __init__(self, dimension=None, epochs=100, learning_rate=0.001, act_func=\"sigmoid\"):\n",
    "        if dimension is None:\n",
    "            dimension = [100, 100]\n",
    "        self.dim = dimension\n",
    "        self.ep = epochs\n",
    "        self.lr = learning_rate\n",
    "        # weight 0: input to 1st hidden\n",
    "        # weight 1:  1st hidden to  2nd hidden\n",
    "        # weight len(dim): last hidden to output\n",
    "        # in total 1+len(dim) weight matrices\n",
    "        self.weights = {}  # they will be defined when the input and output are given\n",
    "\n",
    "        # actv 0: fired from input layer\n",
    "        # actv 1: fired from 1st layer\n",
    "        # actv 1+len(dim): fired from output layer\n",
    "        self.actv = {}  # activation outputs\n",
    "        # z 0: input to 1st hidden\n",
    "        # z 1:  1st hidden to  2nd hidden\n",
    "        # z len(dim): last hidden to output\n",
    "        # in total 1+len(dim) weight matrices\n",
    "        self.z = {}  # middle values: inputs to next layers\n",
    "\n",
    "    def forward(self, x):\n",
    "        self.actv[0] = x\n",
    "        for i in range(len(self.dim) + 1):\n",
    "            #print(\"forward actv[\",i,\"] : \",self.actv[i])\n",
    "            # bias term is NOT added to each row with broadcasting\n",
    "            self.z[i] = np.dot(self.weights[i], self.actv[i])\n",
    "            #print(\"forward z[\",i,\"] : \",self.z[i])\n",
    "\n",
    "            if i == len(self.dim):\n",
    "                self.actv[i + 1] = self.actFunc(self.z[i], \"soft\")\n",
    "            else:\n",
    "                self.actv[i + 1] = self.actFunc(self.z[i], \"sigm\")\n",
    "            #print(\"forward actv[\",i+1,\"] : \",self.actv[i+1], \"#\")\n",
    "        return self.actv[1 + len(self.dim)]\n",
    "\n",
    "    def actFunc(self, t, type=\"sigm\"):\n",
    "        if type == \"sigm\":\n",
    "            return self.sigm(t)\n",
    "        elif type == \"sigm-d\":\n",
    "            return self.sigm(t, True)\n",
    "        elif type == \"soft\":\n",
    "            return self.softmax(t)\n",
    "        else:\n",
    "            return self.sigm(t)\n",
    "\n",
    "    def sigm(self, x, derivative=False):\n",
    "        # todo improve efficiency: e_x = np.exp(-x)\n",
    "        if derivative:\n",
    "            return (np.exp(-x)) / ((np.exp(-x) + 1) ** 2)\n",
    "        return 1 / (1 + np.exp(-x))\n",
    "\n",
    "    def softmax(self, x):\n",
    "        exps = np.exp(x - x.max())\n",
    "        return exps / np.sum(exps, axis=0)\n",
    "\n",
    "    def back_prop(self, y_exp, y_pred):\n",
    "        err = y_pred - y_exp\n",
    "        w_chng = {}\n",
    "        dimlen = len(self.dim)\n",
    "        # calculate the update for the last weights\n",
    "        w_chng[dimlen] = np.multiply(err, self.actv[1 + dimlen])\n",
    "        # calculate changes backwardly\n",
    "        for i in range(dimlen):\n",
    "            # e.g. the 1st weights (i.e. w[0])\n",
    "            # will be updated when i := dimlen-1\n",
    "            # so that err depends on w[1], previous err and z[0]\n",
    "            # and change depends on a[1]\n",
    "            #print( \"err: \", err.shape, \" ; sw.shape: \",self.weights[dimlen-i].T.shape,\" ; act.shape: \",self.actFunc(self.z[dimlen-i-1], \"sigm-d\").shape, \"; actv: \", self.actv[dimlen-i].shape )\n",
    "            err = np.multiply(np.dot(self.weights[dimlen - i].T, err), self.actFunc(self.z[dimlen - i - 1], \"sigm-d\"))\n",
    "            w_chng[dimlen - i - 1] = np.multiply(err, self.actv[dimlen - i])\n",
    "        #print(\"w_chng: \", type(w_chng), \" : \", w_chng)\n",
    "        return w_chng\n",
    "\n",
    "    def update_weights(self, w_changes):\n",
    "        # why calling items() https://stackoverflow.com/a/62173039/13555389\n",
    "        global gl\n",
    "        gl[\"w_changes\"] = w_changes\n",
    "        gl['wc'] = {}\n",
    "        for i, chng in w_changes.items():\n",
    "            #print(\"uw i=\",i, \" chng \", chng, \" shp: \", self.weights[i].T.shape)\n",
    "            #print(\"BEF: \", self.weights[i].T)\n",
    "            for k in range(len(self.weights[i].T)):\n",
    "                self.weights[i].T[k] = self.weights[i].T[k] - self.lr * chng\n",
    "            #print(\"AFT: \", self.weights[i].T)\n",
    "\n",
    "            #w_mtrx = w_mtrx - self.lr*chng\n",
    "\n",
    "    # THE FOLLOWING FUNCTION IS COPIED IN VERBATIM\n",
    "    def get_accuracy(self, x_val, y_val):\n",
    "        self.pred_indices = np.empty([y_val.shape[0], 2], \"int\")\n",
    "        #print(\"p_i shape:\",self.pred_indices.shape)\n",
    "        global gl\n",
    "        predictions = []\n",
    "        i = 0\n",
    "        for x, y in zip(x_val, y_val):\n",
    "            output = self.forward(x)\n",
    "            pred = np.argmax(output)\n",
    "            exp = np.argmax(y)\n",
    "            self.pred_indices[i][0], self.pred_indices[i][1] = pred, exp\n",
    "            #print(output, \" : \", pred, \" :: \", y, \" : \", exp)\n",
    "            predictions.append(pred == exp)\n",
    "            i = i + 1\n",
    "\n",
    "        #summed = sum(pred for pred in predictions) / 100.0\n",
    "\n",
    "        gl['p'] = predictions\n",
    "        gl['p_i'] = self.pred_indices\n",
    "        return sum(pred for pred in gl['p']) / len(gl['p'])  #np.average(summed)\n",
    "\n",
    "    def train(self, x_train, y_train, x_test, y_test):\n",
    "        # initialize weights! etc.!\n",
    "        # we seperate 1st and last because they depend on the size of x_train and y_train, respectively!\n",
    "        # weights from input to 1st hidden\n",
    "        self.weights[0] = np.random.randn(self.dim[0], x_train.shape[1]) * np.sqrt(1. / self.dim[0])\n",
    "        # set default weights to middle layer weights if there any\n",
    "        dimlen = len(self.dim)\n",
    "        for i in range(dimlen - 1):\n",
    "            self.weights[i + 1] = np.random.randn(self.dim[i + 1], self.dim[i]) * np.sqrt(1. / self.dim[i + 1])\n",
    "            #print(\"train: \" ,i+1, type( self.weights[i+1]))\n",
    "        # last weights\n",
    "        self.weights[dimlen] = np.random.randn(y_train.shape[1], self.dim[dimlen - 1]) * np.sqrt(1. / y_train.shape[1])\n",
    "        #print(\"train: \" ,dimlen, type( self.weights[dimlen]))\n",
    "\n",
    "        # train them all\n",
    "        start_time = time.time()\n",
    "        print(\"Started training!\")\n",
    "        for iteration in range(self.ep):\n",
    "            for x, y in zip(x_train, y_train):\n",
    "                output = self.forward(x)\n",
    "                changes_to_w = self.back_prop(y, output)\n",
    "                #print(\"x: \", x, \"y: \", y, \"o: \", np.argmax(output), \" \", output, \" /\\=\", changes_to_w)\n",
    "                self.update_weights(changes_to_w)\n",
    "\n",
    "            accuracy = self.get_accuracy(x_test, y_test)\n",
    "            #if iteration % 10 == 9:\n",
    "            print('{0}th epoch, {1:.2f} seconds wasted so far, for merely {2}  accuracy'.format(\n",
    "                iteration + 1, time.time() - start_time, accuracy))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "xor_x = np.array([[0, 0],\n",
    "                  [1, 0],\n",
    "                  [0, 1],\n",
    "                  [1, 1]\n",
    "                  ])\n",
    "#xor_y =np.array( [ [0], [1], [1],[0]])\n",
    "xor_y = np.array([[1, 0], [0, 1], [0, 1], [1, 0]])\n",
    "\n",
    "dnn = Neural_Network(dimension=[3], epochs=10000, learning_rate=1)\n",
    "dnn.train(xor_x, xor_y, xor_x, xor_y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "array([1., 1.])"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "4\n",
      "0.5\n"
     ]
    }
   ],
   "source": [
    "print(sum(pred for pred in gl['p']))\n",
    "print(len(gl['p']))\n",
    "print(sum(pred for pred in gl['p']) / len(gl['p']))\n",
    "summed = sum(pred for pred in gl['p']) / 100.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "399th epoch, 193.65 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "400th epoch, 194.17 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "401th epoch, 194.67 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "402th epoch, 195.09 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "403th epoch, 195.53 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "404th epoch, 195.92 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "405th epoch, 196.35 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "406th epoch, 196.81 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "407th epoch, 197.26 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "408th epoch, 197.67 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "409th epoch, 198.09 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "410th epoch, 198.51 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "411th epoch, 198.92 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "412th epoch, 199.35 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "413th epoch, 199.78 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "414th epoch, 200.20 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "415th epoch, 200.64 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "416th epoch, 201.10 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "417th epoch, 201.56 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "418th epoch, 202.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "419th epoch, 202.43 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "420th epoch, 202.84 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "421th epoch, 203.31 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "422th epoch, 203.72 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "423th epoch, 204.15 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "424th epoch, 204.65 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "425th epoch, 205.05 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "426th epoch, 205.50 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "427th epoch, 205.92 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "428th epoch, 206.33 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "429th epoch, 206.75 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "430th epoch, 207.17 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "431th epoch, 207.58 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "432th epoch, 208.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "433th epoch, 208.42 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "434th epoch, 208.89 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "435th epoch, 209.36 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "436th epoch, 209.77 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "437th epoch, 210.17 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "438th epoch, 210.58 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "439th epoch, 210.99 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "440th epoch, 211.41 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "441th epoch, 211.84 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "442th epoch, 212.31 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "443th epoch, 212.78 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "444th epoch, 213.17 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "445th epoch, 213.63 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "446th epoch, 214.09 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "447th epoch, 214.53 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "448th epoch, 214.95 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "449th epoch, 215.39 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "450th epoch, 215.86 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "451th epoch, 216.35 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "452th epoch, 216.85 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "453th epoch, 217.25 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "454th epoch, 217.69 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "455th epoch, 218.13 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "456th epoch, 218.55 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "457th epoch, 219.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "458th epoch, 219.42 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "459th epoch, 219.86 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "460th epoch, 220.36 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "461th epoch, 220.85 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "462th epoch, 221.30 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "463th epoch, 221.71 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "464th epoch, 222.11 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "465th epoch, 222.54 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "466th epoch, 223.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "467th epoch, 223.44 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "468th epoch, 223.91 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "469th epoch, 224.33 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "470th epoch, 224.75 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "471th epoch, 225.19 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "472th epoch, 225.60 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "473th epoch, 226.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "474th epoch, 226.40 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "475th epoch, 226.83 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "476th epoch, 227.27 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "477th epoch, 227.71 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "478th epoch, 228.10 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "479th epoch, 228.54 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "480th epoch, 228.97 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "481th epoch, 229.41 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "482th epoch, 229.83 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "483th epoch, 230.29 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "484th epoch, 230.76 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "485th epoch, 231.16 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "486th epoch, 231.58 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "487th epoch, 232.01 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "488th epoch, 232.43 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "489th epoch, 232.87 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "490th epoch, 233.27 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "491th epoch, 233.68 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "492th epoch, 234.10 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "493th epoch, 234.55 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "494th epoch, 234.94 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "495th epoch, 235.35 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "496th epoch, 235.74 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "497th epoch, 236.19 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "498th epoch, 236.65 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "499th epoch, 237.05 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "500th epoch, 237.54 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "501th epoch, 237.94 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "502th epoch, 238.38 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "503th epoch, 238.80 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "504th epoch, 239.22 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "505th epoch, 239.71 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "506th epoch, 240.11 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "507th epoch, 240.55 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "508th epoch, 240.99 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "509th epoch, 241.43 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "510th epoch, 241.88 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "511th epoch, 242.30 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "512th epoch, 242.77 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "513th epoch, 243.19 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "514th epoch, 243.68 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "515th epoch, 244.13 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "516th epoch, 244.55 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "517th epoch, 244.97 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "518th epoch, 245.44 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "519th epoch, 245.91 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "520th epoch, 246.34 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "521th epoch, 246.74 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "522th epoch, 247.18 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "523th epoch, 247.62 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "524th epoch, 248.01 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "525th epoch, 248.43 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "526th epoch, 248.85 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "527th epoch, 249.29 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "528th epoch, 249.75 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "529th epoch, 250.17 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "530th epoch, 250.58 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "531th epoch, 251.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "532th epoch, 251.45 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "533th epoch, 251.86 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "534th epoch, 252.28 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "535th epoch, 252.69 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "536th epoch, 253.11 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "537th epoch, 253.55 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "538th epoch, 253.95 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "539th epoch, 254.44 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "540th epoch, 254.84 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "541th epoch, 255.25 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "542th epoch, 255.70 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "543th epoch, 256.16 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "544th epoch, 256.64 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "545th epoch, 257.09 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "546th epoch, 257.56 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "547th epoch, 258.00 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "548th epoch, 258.44 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "549th epoch, 258.86 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "550th epoch, 259.30 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "551th epoch, 259.77 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "552th epoch, 260.19 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "553th epoch, 260.64 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "554th epoch, 261.08 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "555th epoch, 261.58 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "556th epoch, 262.07 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "557th epoch, 262.47 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "558th epoch, 262.88 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "559th epoch, 263.32 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "560th epoch, 263.71 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "561th epoch, 264.11 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "562th epoch, 264.52 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "563th epoch, 264.94 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "564th epoch, 265.36 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "565th epoch, 265.82 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "566th epoch, 266.24 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "567th epoch, 266.72 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "568th epoch, 267.17 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "569th epoch, 267.63 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "570th epoch, 268.08 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "571th epoch, 268.54 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "572th epoch, 268.93 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "573th epoch, 269.36 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "574th epoch, 269.77 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "575th epoch, 270.19 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "576th epoch, 270.60 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "577th epoch, 271.02 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "578th epoch, 271.44 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "579th epoch, 271.85 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "580th epoch, 272.27 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "581th epoch, 272.75 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "582th epoch, 273.24 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "583th epoch, 273.69 seconds wasted so far, for merely 0.07407407407407407  accuracy\n",
      "584th epoch, 274.11 seconds wasted so far, for merely 0.07407407407407407  accuracy\n"
     ]
    }
   ],
   "source": [
    "\n",
    "dnn = Neural_Network(dimension=[15, 20], epochs=1000, learning_rate=0.1)\n",
    "dnn.train(X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "#print(gl)\n",
    "for i, j in gl[\"w_changes\"].items():\n",
    "    print(\"i: \", i, \", j: \", j)\n",
    "\n",
    "gl['wc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# test accuracy\n",
    "print(dnn.get_accuracy(X_test, Y_test))\n",
    "pred_class = cat[gl['p_i'][:, 0]]\n",
    "exp_class = cat[gl['p_i'][:, 1]]\n",
    "print(exp_class)\n",
    "print(pred_class)\n",
    "ConfusionMatrixDisplay.from_predictions(exp_class, pred_class)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# training accuracy\n",
    "print(dnn.get_accuracy(X_train, Y_train))\n",
    "print(Y_test.shape[0])\n",
    "pred_class = cat[gl['p_i'][:, 0]]\n",
    "exp_class = cat[gl['p_i'][:, 1]]\n",
    "print(exp_class)\n",
    "print(pred_class)\n",
    "ConfusionMatrixDisplay.from_predictions(exp_class, pred_class)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "for i in range(3):\n",
    "    print(\"type z[\", i, \"] : \", type(dnn.z[i]))\n",
    "    print(\"shape z[\", i, \"] : \", dnn.z[i].shape)\n",
    "    #print(\"forward z[\",i,\"] : \",dnn.z[i])\n",
    "    print(\"\")\n",
    "    print(\"type actv[\", i, \"] : \", type(dnn.actv[i]))\n",
    "    print(\"shape actv[\", i, \"] : \", dnn.actv[i].shape)\n",
    "    #print(\"forward actv[\",i,\"] : \",dnn.actv[i])\n",
    "    print(\"\")\n",
    "\n",
    "    print(\"type weights[\", i, \"] : \", type(dnn.weights[i]))\n",
    "    print(\"shape weights[\", i, \"] : \", dnn.weights[i].shape)\n",
    "    #print(\"forward weights[\",i,\"] : \",dnn.weights[i])\n",
    "    print(\"-----------------\\n\")\n",
    "\n",
    "i = 3\n",
    "print(\"type actv[\", i, \"] : \", type(dnn.actv[i]))\n",
    "print(\"shape actv[\", i, \"] : \", dnn.actv[i].shape)\n",
    "print(\"forward actv[\", i, \"] : \", dnn.actv[i])\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "print(type(dnn.weights[0]))  #, \", shape: \"dnn.weights[0].shape)\n",
    "print(type(dnn.weights[1]))  #, \", shape: \"dnn.weights[1].shape)\n",
    "print(type(dnn.weights[2]))  #, \", shape: \"dnn.weights[2].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "dnn.weights[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "dnn.weights[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "dnn.weights[2]"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "0b647197a1fcc38d756609367937b3918f8af3cc96749ff11478d444b1db53c3"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 64-bit (windows store)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}