PokeV2_ModelPerfCheck.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ee6b0502-e38c-4fac-a077-48d029568286",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/bhawks/RN0X_Pokemon\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from resnet_rn06_eembc import * \n",
    "import qkeras\n",
    "!pwd\n",
    "model = qkeras.utils.load_qmodel('old_training/trained_model/deploy/resnet_v1_eembc_RN06_bilinear/small_model_best.h5')\n",
    "#model = qkeras.utils.load_qmodel('resnet_v1_eembc_RN06_PokeV2/small_model_best.h5')\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score, top_k_accuracy_score\n",
    "batch_size=32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "34838e0e-01b1-47a4-a996-57adb8a932d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1147 images belonging to 10 classes.\n",
      "Found 377 images belonging to 10 classes.\n",
      "Found 574 images belonging to 10 classes.\n",
      "Found 574 images belonging to 10 classes.\n",
      "Found 574 images belonging to 10 classes.\n"
     ]
    }
   ],
   "source": [
    "datagen = ImageDataGenerator(\n",
    "    rotation_range=60,\n",
    "    width_shift_range=0.15,\n",
    "    height_shift_range=0.15,\n",
    "    brightness_range=(0.8,1.5),\n",
    "    zoom_range = [1.0, 1.3], \n",
    "    horizontal_flip=True,\n",
    "    validation_split=0.25,\n",
    "    rescale=1./255 # normalize values to between 0-1\n",
    "    # preprocessing_function=random_crop,\n",
    "    #brightness_range=(0.9, 1.2),\n",
    "    #contrast_range=(0.9, 1.2)\n",
    ")\n",
    "tdatagen = ImageDataGenerator(rescale=1./255) # No augmentation\n",
    "tdatagen2 = ImageDataGenerator( # all augments, but minimum zoon to reduce interference of top bar?\n",
    "    rotation_range=60,\n",
    "    width_shift_range=0.15,\n",
    "    height_shift_range=0.15,\n",
    "    brightness_range=(0.8,1.5),\n",
    "    zoom_range = [1.10, 1.3],\n",
    "    horizontal_flip=True,\n",
    "    validation_split=0.25,\n",
    "    rescale=1./255\n",
    "    # normalize values to between 0-1\n",
    "    # preprocessing_function=random_crop,\n",
    "    #brightness_range=(0.9, 1.2),\n",
    "    #contrast_range=(0.9, 1.2)\n",
    ")\n",
    "tdatagen3 = ImageDataGenerator( # no augments, but minimum zoon to reduce interference of top bar?\n",
    "    #zoom_range = [1.10, 1.3],\n",
    "    rescale=1./255 # normalize values to between 0-1,\n",
    "    # preprocessing_function=random_crop,\n",
    "    #brightness_range=(0.9, 1.2),\n",
    "    #contrast_range=(0.9, 1.2)\n",
    ")\n",
    "train_generator = datagen.flow_from_directory('./data/SmallPokemonData/',\n",
    "                            target_size=(32,32),\n",
    "                            batch_size=batch_size,\n",
    "                            color_mode='rgb',\n",
    "                            class_mode='categorical',\n",
    "                            interpolation='bilinear',\n",
    "                            subset='training',\n",
    "                            keep_aspect_ratio=True,\n",
    "                            save_to_dir=\"/home/bhawks/RN0X_Pokemon/data/debug/datagen_Train/\"                                             \n",
    "                            )\n",
    "\n",
    "validation_generator = datagen.flow_from_directory('./data/SmallPokemonData/',\n",
    "                            target_size=(32,32),\n",
    "                            batch_size=batch_size,\n",
    "                            color_mode='rgb',\n",
    "                            class_mode='categorical',\n",
    "                            interpolation='bilinear',\n",
    "                            subset='validation',\n",
    "                            keep_aspect_ratio=True,\n",
    "                            shuffle=False,\n",
    "                            save_to_dir=\"/home/bhawks/RN0X_Pokemon/data/debug/datagen_Val/\"\n",
    "                            )    \n",
    "test_generator = tdatagen.flow_from_directory('./data/PokeCard/',\n",
    "                        target_size=(32,32),\n",
    "                        batch_size=batch_size,\n",
    "                        color_mode='rgb',\n",
    "                        class_mode='categorical',\n",
    "                        interpolation='bilinear',\n",
    "                        shuffle=False,\n",
    "                        save_to_dir=\"/home/bhawks/RN0X_Pokemon/data/debug/datagen_test_NoAug/\",\n",
    "                        keep_aspect_ratio=True,)#,\n",
    "                        #keep_aspect_ratio=True)\n",
    "\n",
    "test2_generator = tdatagen2.flow_from_directory('./data/PokeCard/',\n",
    "                        target_size=(32,32),\n",
    "                        batch_size=batch_size,\n",
    "                        color_mode='rgb',\n",
    "                        class_mode='categorical',\n",
    "                        interpolation='bilinear',\n",
    "                        shuffle=False,\n",
    "                        save_to_dir=\"/home/bhawks/RN0X_Pokemon/data/debug/datagen_test_fullAug/\",\n",
    "                            keep_aspect_ratio=True,)#,\n",
    "                        #keep_aspect_ratio=True)\n",
    "\n",
    "test3_generator = tdatagen3.flow_from_directory('./data/PokeCard/',\n",
    "                        target_size=(32,32),\n",
    "                        batch_size=batch_size,\n",
    "                        color_mode='rgb',\n",
    "                        class_mode='categorical',\n",
    "                        interpolation='bilinear',\n",
    "                        shuffle=False,\n",
    "                        save_to_dir=\"/home/bhawks/RN0X_Pokemon/data/debug/datagen_test_ZoomOnly/\",\n",
    "                            keep_aspect_ratio=True,)#,\n",
    "                        #keep_aspect_ratio=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4185ba27-17d9-4e1f-95af-059e8738c466",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 32, 32, 3)]       0         \n",
      "                                                                 \n",
      " q_conv2d_batchnorm (QConv2  (None, 8, 8, 32)          1025      \n",
      " DBatchnorm)                                                     \n",
      "                                                                 \n",
      " q_activation (QActivation)  (None, 8, 8, 32)          0         \n",
      "                                                                 \n",
      " q_conv2d_batchnorm_1 (QCon  (None, 8, 8, 32)          9377      \n",
      " v2DBatchnorm)                                                   \n",
      "                                                                 \n",
      " q_activation_1 (QActivatio  (None, 8, 8, 32)          0         \n",
      " n)                                                              \n",
      "                                                                 \n",
      " q_conv2d_batchnorm_2 (QCon  (None, 8, 8, 32)          9377      \n",
      " v2DBatchnorm)                                                   \n",
      "                                                                 \n",
      " q_activation_2 (QActivatio  (None, 8, 8, 32)          0         \n",
      " n)                                                              \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 2048)              0         \n",
      "                                                                 \n",
      " q_dense (QDense)            (None, 10)                20490     \n",
      "                                                                 \n",
      " softmax (Activation)        (None, 10)                0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 40269 (157.31 KB)\n",
      "Trainable params: 40074 (156.54 KB)\n",
      "Non-trainable params: 195 (792.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ff58039-31bd-46b6-b004-2daca316521c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n"
     ]
    }
   ],
   "source": [
    "data_list = []\n",
    "batch_index = 0\n",
    "while batch_index <= test_generator.batch_index:\n",
    "    data = test_generator.next()\n",
    "    data_list.append(data[0])\n",
    "    batch_index = batch_index + 1\n",
    "print(len(data_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2ca16a3c-bf95-4acf-9653-9a356653d75d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/18 [==============================] - 5s 242ms/step - loss: 1.4131 - accuracy: 0.6533\n",
      "[1.4130908250808716, 0.6533101201057434]\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(test_generator)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ebec07cd-2372-4bd7-aca6-b9a2cd831382",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/18 [==============================] - 5s 243ms/step\n",
      "18/18 [==============================] - 5s 250ms/step - loss: 1.4131 - accuracy: 0.6533\n",
      "[1.4130908250808716, 0.6533101201057434]\n",
      "[9.99998808e-01 1.06155070e-12 1.03272134e-10 6.12947071e-09\n",
      " 3.39609812e-10 1.80155162e-08 1.44215903e-07 5.73674476e-11\n",
      " 1.29318130e-08 1.08398717e-06]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5\n",
      " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
      " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9\n",
      " 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9]\n",
      "0.8205574912891986\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(test_generator)\n",
    "# evaluate with test dataset and share same prediction results\n",
    "evaluation = model.evaluate(test_generator)\n",
    "print(evaluation) \n",
    "y_test = test_generator.classes\n",
    "print(y_pred[0])\n",
    "print(y_test)\n",
    "score = top_k_accuracy_score(y_test, y_pred, k=2)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9725df63-ce08-45b2-8a63-3cd611d863e7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "574\n",
      "574\n",
      "Confusion Matrix - Test Generator\n",
      "[[ 18   0   0   0   0   0   3   1   0   3]\n",
      " [  0  34   1   0   0   0   1   5   1   0]\n",
      " [  0  17  45   0   0   1   8   1   4   0]\n",
      " [  0   0   1  35   0   4   2   0   1   0]\n",
      " [  0   1   0   0  30   0   2   0   0   1]\n",
      " [  1   5   3  10   5  33  17   0   2   3]\n",
      " [  0   1   0   1   1   1  18   2   3   2]\n",
      " [  0  15  18   1   2   0  10 110  10   4]\n",
      " [  0   1   1   2   1   0   5   1  36   2]\n",
      " [  1   2   2   0   0   0   0   2   4  16]]\n",
      "Classification Report - Test Generator\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   Bulbasaur       0.90      0.72      0.80        25\n",
      "  Charmander       0.45      0.81      0.58        42\n",
      "       Eevee       0.63      0.59      0.61        76\n",
      "      Gengar       0.71      0.81      0.76        43\n",
      "  Jigglypuff       0.77      0.88      0.82        34\n",
      "      Mewtwo       0.85      0.42      0.56        79\n",
      "        Onix       0.27      0.62      0.38        29\n",
      "     Pikachu       0.90      0.65      0.75       170\n",
      "     Snorlax       0.59      0.73      0.65        49\n",
      "    Squirtle       0.52      0.59      0.55        27\n",
      "\n",
      "    accuracy                           0.65       574\n",
      "   macro avg       0.66      0.68      0.65       574\n",
      "weighted avg       0.73      0.65      0.66       574\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "print(len(y_pred))\n",
    "print(len(y_test))\n",
    "print('Confusion Matrix - Test Generator')\n",
    "print(confusion_matrix(y_test, np.argmax(y_pred, axis=1)))\n",
    "print('Classification Report - Test Generator')\n",
    "target_names = test_generator.class_indices.keys() #['bulbasaur', 'charmander', 'eevee', 'gengar', 'jigglypuff', 'mewtwo', 'onix', 'pikachu', 'snorlax', 'squirtle']\n",
    "\n",
    "print(classification_report(y_test, np.argmax(y_pred, axis=1), target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4fb26214-ad29-4612-9bdf-cd524577dbaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/18 [==============================] - 6s 291ms/step - loss: 2.2142 - accuracy: 0.6150\n",
      "[2.2141733169555664, 0.6149826049804688]\n"
     ]
    }
   ],
   "source": [
    "score2 = model.evaluate(test2_generator)\n",
    "print(score2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "036ba3b9-b87b-435c-b316-767d2d5403f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/18 [==============================] - 6s 299ms/step\n",
      "18/18 [==============================] - 5s 288ms/step - loss: 1.9488 - accuracy: 0.6272\n",
      "[1.9487733840942383, 0.6271777153015137]\n",
      "[9.9892861e-01 2.2417657e-05 4.1883386e-08 2.6883960e-08 3.1207095e-10\n",
      " 5.2935520e-06 1.3366597e-05 4.2807464e-09 1.9790218e-06 1.0281664e-03]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5\n",
      " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
      " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9\n",
      " 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9]\n",
      "0.8031358885017421\n"
     ]
    }
   ],
   "source": [
    "    y_pred = model.predict(test2_generator)\n",
    "    # evaluate with test dataset and share same prediction results\n",
    "    evaluation = model.evaluate(test2_generator)\n",
    "    print(evaluation) \n",
    "    y_test = test2_generator.classes\n",
    "    print(y_pred[0])\n",
    "    print(y_test)\n",
    "    score = top_k_accuracy_score(y_test, y_pred, k=2)\n",
    "    print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7b29965c-d141-4abc-99cf-818a7e032b53",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "574\n",
      "574\n",
      "Confusion Matrix - Test2 Generator\n",
      "[[ 16   0   0   0   0   0   1   3   0   5]\n",
      " [  0  30   2   0   0   0   1   8   0   1]\n",
      " [  2  15  39   0   1   2   2  12   2   1]\n",
      " [  0   2   0  22   1  15   1   0   0   2]\n",
      " [  0   1   1   1  17   5   1   0   4   4]\n",
      " [  0   6   1   1   2  54   5   3   0   7]\n",
      " [  1   0   0   1   1   3  16   5   1   1]\n",
      " [  2  10   2   0   0   1   4 137   2  12]\n",
      " [  2   0   1   0   0   2   0   2  28  14]\n",
      " [  1   2   0   1   0   0   0   5   0  18]]\n",
      "Classification Report - Test2 Generator\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   Bulbasaur       0.67      0.64      0.65        25\n",
      "  Charmander       0.45      0.71      0.56        42\n",
      "       Eevee       0.85      0.51      0.64        76\n",
      "      Gengar       0.85      0.51      0.64        43\n",
      "  Jigglypuff       0.77      0.50      0.61        34\n",
      "      Mewtwo       0.66      0.68      0.67        79\n",
      "        Onix       0.52      0.55      0.53        29\n",
      "     Pikachu       0.78      0.81      0.79       170\n",
      "     Snorlax       0.76      0.57      0.65        49\n",
      "    Squirtle       0.28      0.67      0.39        27\n",
      "\n",
      "    accuracy                           0.66       574\n",
      "   macro avg       0.66      0.62      0.61       574\n",
      "weighted avg       0.71      0.66      0.67       574\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "print(len(y_pred))\n",
    "print(len(y_test))\n",
    "print('Confusion Matrix - Test2 Generator')\n",
    "print(confusion_matrix(y_test, np.argmax(y_pred, axis=1)))\n",
    "print('Classification Report - Test2 Generator')\n",
    "target_names = test2_generator.class_indices.keys() #['bulbasaur', 'charmander', 'eevee', 'gengar', 'jigglypuff', 'mewtwo', 'onix', 'pikachu', 'snorlax', 'squirtle']\n",
    "\n",
    "print(classification_report(y_test, np.argmax(y_pred, axis=1), target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "47b18b4d-256e-4b12-80db-c22733a71f5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/18 [==============================] - 5s 251ms/step\n",
      "18/18 [==============================] - 4s 236ms/step - loss: 1.4131 - accuracy: 0.6533\n",
      "[1.4130908250808716, 0.6533101201057434]\n",
      "[9.99998808e-01 1.06155070e-12 1.03272134e-10 6.12947071e-09\n",
      " 3.39609812e-10 1.80155162e-08 1.44215903e-07 5.73674476e-11\n",
      " 1.29318130e-08 1.08398717e-06]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5\n",
      " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
      " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9\n",
      " 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9]\n",
      "0.8205574912891986\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(test3_generator)\n",
    "# evaluate with test dataset and share same prediction results\n",
    "evaluation = model.evaluate(test3_generator)\n",
    "print(evaluation) \n",
    "y_test = test3_generator.classes\n",
    "print(y_pred[0])\n",
    "print(y_test)\n",
    "score = top_k_accuracy_score(y_test, y_pred, k=2)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fc5c7493-4b9c-44e8-8c76-86be548275cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "574\n",
      "574\n",
      "Confusion Matrix - Test3 Generator\n",
      "[[ 18   0   0   0   0   0   3   1   0   3]\n",
      " [  0  34   1   0   0   0   1   5   1   0]\n",
      " [  0  17  45   0   0   1   8   1   4   0]\n",
      " [  0   0   1  35   0   4   2   0   1   0]\n",
      " [  0   1   0   0  30   0   2   0   0   1]\n",
      " [  1   5   3  10   5  33  17   0   2   3]\n",
      " [  0   1   0   1   1   1  18   2   3   2]\n",
      " [  0  15  18   1   2   0  10 110  10   4]\n",
      " [  0   1   1   2   1   0   5   1  36   2]\n",
      " [  1   2   2   0   0   0   0   2   4  16]]\n",
      "Classification Report- Test3 Generator\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   Bulbasaur       0.90      0.72      0.80        25\n",
      "  Charmander       0.45      0.81      0.58        42\n",
      "       Eevee       0.63      0.59      0.61        76\n",
      "      Gengar       0.71      0.81      0.76        43\n",
      "  Jigglypuff       0.77      0.88      0.82        34\n",
      "      Mewtwo       0.85      0.42      0.56        79\n",
      "        Onix       0.27      0.62      0.38        29\n",
      "     Pikachu       0.90      0.65      0.75       170\n",
      "     Snorlax       0.59      0.73      0.65        49\n",
      "    Squirtle       0.52      0.59      0.55        27\n",
      "\n",
      "    accuracy                           0.65       574\n",
      "   macro avg       0.66      0.68      0.65       574\n",
      "weighted avg       0.73      0.65      0.66       574\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "print(len(y_pred))\n",
    "print(len(y_test))\n",
    "print('Confusion Matrix - Test3 Generator')\n",
    "print(confusion_matrix(y_test, np.argmax(y_pred, axis=1)))\n",
    "print('Classification Report- Test3 Generator')\n",
    "target_names = test3_generator.class_indices.keys() #['bulbasaur', 'charmander', 'eevee', 'gengar', 'jigglypuff', 'mewtwo', 'onix', 'pikachu', 'snorlax', 'squirtle']\n",
    "\n",
    "print(classification_report(y_test, np.argmax(y_pred, axis=1), target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cc93b5b4-4c30-42cb-8888-7c1e0323d49e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 2/12 [====>.........................] - ETA: 9s "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bhawks/.conda/envs/poke/lib/python3.8/site-packages/PIL/Image.py:981: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12/12 [==============================] - 8s 654ms/step\n",
      "12/12 [==============================] - 6s 421ms/step - loss: 0.4492 - accuracy: 0.8727\n",
      "[0.4491543173789978, 0.8726790547370911]\n",
      "[9.99999642e-01 3.71272367e-18 1.01263738e-14 1.03969305e-14\n",
      " 4.64664277e-13 8.16828549e-12 2.38120592e-07 2.20599125e-12\n",
      " 4.29453001e-11 8.88055567e-08]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
      " 4 4 4 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 5 5 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
      " 8 8 8 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9\n",
      " 9 9 9 9 9 9 9]\n",
      "0.9602122015915119\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(validation_generator)\n",
    "# evaluate with test dataset and share same prediction results\n",
    "evaluation = model.evaluate(validation_generator)\n",
    "print(evaluation) \n",
    "y_test = validation_generator.classes\n",
    "print(y_pred[0])\n",
    "print(y_test)\n",
    "score = top_k_accuracy_score(y_test, y_pred, k=2)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8d73e08b-9f96-4cce-9825-602ecee9d8b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "377\n",
      "377\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 2 1 1 4 1 1 2 2 1 1 8 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 5 4 7 1 2\n",
      " 2 1 2 2 2 1 7 2 2 7 2 2 2 1 2 2 8 2 2 2 7 2 7 2 6 2 2 2 2 2 2 1 2 3 9 6 3\n",
      " 3 3 3 4 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
      " 4 4 4 4 6 6 5 5 6 6 5 3 5 6 6 5 5 3 5 6 5 5 6 3 6 5 5 5 5 5 5 5 5 5 5 5 6\n",
      " 5 5 5 5 5 5 5 4 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 5 3 6 6 4 6 6 6 6 6 6 6 7\n",
      " 7 7 7 2 7 7 1 7 7 7 7 7 7 7 7 7 7 7 7 7 7 2 7 7 7 7 2 7 7 7 7 7 2 7 7 7 7\n",
      " 7 7 7 7 7 7 2 7 7 7 7 7 4 8 8 8 8 0 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
      " 8 2 8 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 8 9\n",
      " 9 9 9 9 9 9 9]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3\n",
      " 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3\n",
      " 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
      " 4 4 4 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
      " 5 5 5 5 5 5 5 5 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7\n",
      " 7 7 7 7 7 7 7 7 7 7 7 7 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
      " 8 8 8 8 8 8 8 8 8 8 8 8 8 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9\n",
      " 9 9 9 9 9 9 9]\n",
      "Confusion Matrix - Validation Data\n",
      "[[25  0  0  0  0  0  0  0  0  0]\n",
      " [ 0 38  3  0  1  0  0  0  1  0]\n",
      " [ 0  6 24  0  1  1  1  5  1  0]\n",
      " [ 0  0  1 44  1  0  1  0  0  1]\n",
      " [ 0  0  0  0 34  0  0  0  0  0]\n",
      " [ 0  0  0  3  1 27 10  0  0  0]\n",
      " [ 0  0  0  1  1  1 25  0  0  0]\n",
      " [ 0  1  5  0  0  0  0 44  0  0]\n",
      " [ 1  0  1  0  1  0  0  0 35  0]\n",
      " [ 0  0  0  0  0  0  0  0  1 30]]\n",
      "Classification Report - Validation Data\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   Bulbasaur       0.96      1.00      0.98        25\n",
      "  Charmander       0.84      0.88      0.86        43\n",
      "       Eevee       0.71      0.62      0.66        39\n",
      "      Gengar       0.92      0.92      0.92        48\n",
      "  Jigglypuff       0.85      1.00      0.92        34\n",
      "      Mewtwo       0.93      0.66      0.77        41\n",
      "        Onix       0.68      0.89      0.77        28\n",
      "     Pikachu       0.90      0.88      0.89        50\n",
      "     Snorlax       0.92      0.92      0.92        38\n",
      "    Squirtle       0.97      0.97      0.97        31\n",
      "\n",
      "    accuracy                           0.86       377\n",
      "   macro avg       0.87      0.87      0.87       377\n",
      "weighted avg       0.87      0.86      0.86       377\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(len(y_pred))\n",
    "print(len(y_test))\n",
    "print( np.argmax(y_pred, axis=1))\n",
    "print( y_test)\n",
    "print('Confusion Matrix - Validation Data')\n",
    "print(confusion_matrix(y_test, np.argmax(y_pred, axis=1)))\n",
    "print('Classification Report - Validation Data')\n",
    "target_names = validation_generator.class_indices.keys() #[str(i) for i in range(10)] # #['bulbasaur', 'charmander', 'eevee', 'gengar', 'jigglypuff', 'mewtwo', 'onix', 'pikachu', 'snorlax', 'squirtle']\n",
    "\n",
    "print(classification_report(y_test, np.argmax(y_pred, axis=1), target_names=target_names))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-poke]",
   "language": "python",
   "name": "conda-env-.conda-poke-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}