cat_dog.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cat_dog.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3.8 (XPython)",
      "language": "python",
      "name": "xpython"
    },
    "language_info": {
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "version": "3.8.8"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/forMwish/MyDeepLearn/blob/master/cat_dog.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cQQl-06bNHyn"
      },
      "source": [
        "# 1. 准备"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nGA_OO0wPDBc"
      },
      "source": [
        "## 1.1 挂载 google drive\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WRPImfTbMcii"
      },
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "\n",
        "gdrive_path = '/gdrive'\n",
        "drive.mount(gdrive_path, force_remount=True)\n",
        "\n",
        "os.chdir(\"%s/MyDrive\"%gdrive_path)\n",
        "os.listdir(\"%s/MyDrive\"%gdrive_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5SXkXoJX_FOw"
      },
      "source": [
        "## 1.2 配置 kaggle 并下载数据"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vLl4g7DS6Gyh"
      },
      "source": [
        "!pip install kaggle\n",
        "!mkdir ~/.kaggle\n",
        "!cp /gdrive/MyDrive/kaggle.json ~/.kaggle\n",
        "!chmod 600 ~/.kaggle/kaggle.json\n",
        "\n",
        "!kaggle datasets download -d tongpython/cat-and-dog\n",
        "!cp /gdrive/MyDrive/cat-and-dog.zip ~/\n",
        "!mkdir ~/cat-and-dog\n",
        "!unzip ~/cat-and-dog.zip -d ~/cat-and-dog"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9HPY9f0kHpV2"
      },
      "source": [
        "## 1.2 数据集处理"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKB2VyTjEIlt"
      },
      "source": [
        "import os, shutil\n",
        "import numpy as np\n",
        "import keras\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "\n",
        "\n",
        "original_dataset_dir = \"/root/cat-and-dog\"\n",
        "base_dir = \"/root/cat-and-dog-small\"\n",
        "try:\n",
        "  shutil.rmtree(base_dir)\n",
        "except:\n",
        "  pass\n",
        "os.mkdir(base_dir, mode=777)\n",
        "\n",
        "train_dir = os.path.join(base_dir, 'train')\n",
        "os.mkdir(train_dir)\n",
        "\n",
        "validation_dir = os.path.join(base_dir, 'validation')\n",
        "os.mkdir(validation_dir)\n",
        "\n",
        "test_dir = os.path.join(base_dir, 'test')\n",
        "os.mkdir(test_dir)\n",
        "\n",
        "train_cats_dir = os.path.join(train_dir, \"cats\")\n",
        "os.mkdir(train_cats_dir)\n",
        "\n",
        "train_dogs_dir = os.path.join(train_dir, \"dogs\")\n",
        "os.mkdir(train_dogs_dir)\n",
        "\n",
        "validation_cats_dir = os.path.join(validation_dir, \"cats\")\n",
        "os.mkdir(validation_cats_dir)\n",
        "\n",
        "validation_dogs_dir = os.path.join(validation_dir, \"dogs\")\n",
        "os.mkdir(validation_dogs_dir)\n",
        "\n",
        "test_cats_dir = os.path.join(test_dir, \"cats\")\n",
        "os.mkdir(test_cats_dir)\n",
        "\n",
        "test_dogs_dir = os.path.join(test_dir, \"dogs\")\n",
        "os.mkdir(test_dogs_dir)\n",
        "\n",
        "for file_name in [\"cat.%d.jpg\"%i for i in range(1, 1001)]:\n",
        "  src = os.path.join(original_dataset_dir, \"training_set/training_set/cats\", file_name)\n",
        "  dst = os.path.join(train_cats_dir, file_name)\n",
        "  shutil.copyfile(src, dst)\n",
        "\n",
        "for file_name in [\"cat.%d.jpg\"%i for i in range(1002, 1502)]:\n",
        "  src = os.path.join(original_dataset_dir, \"training_set/training_set/cats\", file_name)\n",
        "  dst = os.path.join(validation_cats_dir, file_name)\n",
        "  shutil.copyfile(src, dst)\n",
        "\n",
        "for file_name in [\"cat.%d.jpg\"%i for i in range(1503, 2003)]:\n",
        "  src = os.path.join(original_dataset_dir, \"training_set/training_set/cats\", file_name)\n",
        "  dst = os.path.join(test_cats_dir, file_name)\n",
        "  shutil.copyfile(src, dst)\n",
        "\n",
        "for file_name in [\"dog.%d.jpg\"%i for i in range(1, 1001)]:\n",
        "  src = os.path.join(original_dataset_dir, \"training_set/training_set/dogs\", file_name)\n",
        "  dst = os.path.join(train_dogs_dir, file_name)\n",
        "  shutil.copyfile(src, dst)\n",
        "\n",
        "for file_name in [\"dog.%d.jpg\"%i for i in range(1002, 1502)]:\n",
        "  src = os.path.join(original_dataset_dir, \"training_set/training_set/dogs\", file_name)\n",
        "  dst = os.path.join(validation_dogs_dir, file_name)\n",
        "  shutil.copyfile(src, dst)\n",
        "\n",
        "for file_name in [\"dog.%d.jpg\"%i for i in range(1503, 2003)]:\n",
        "  src = os.path.join(original_dataset_dir, \"training_set/training_set/dogs\", file_name)\n",
        "  dst = os.path.join(test_dogs_dir, file_name)\n",
        "  shutil.copyfile(src, dst)\n",
        "\n",
        "print('total training cat images:', len(os.listdir(train_cats_dir)))\n",
        "print('total validation cat images:', len(os.listdir(validation_cats_dir)))\n",
        "print('total test cat images:', len(os.listdir(test_cats_dir)))\n",
        "\n",
        "print('total training dog images:', len(os.listdir(train_dogs_dir)))\n",
        "print('total validation dog images:', len(os.listdir(validation_dogs_dir)))\n",
        "print('total test dog images:', len(os.listdir(test_dogs_dir)))\n",
        "\n",
        "# (train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
        "\n",
        "# train_images = train_images.reshape(60000, 28, 28, 1).astype(np.float32) / 255\n",
        "# test_images = test_images.reshape(10000, 28, 28, 1).astype(np.float32) / 255\n",
        "\n",
        "# train_labels = to_categorical(train_labels)\n",
        "# test_labels = to_categorical(test_labels)\n",
        "\n",
        "# validation_image = train_images[:10000]\n",
        "# validation_labels = train_labels[:10000]\n",
        "# partial_train_images = train_images[10000:]\n",
        "# partial_train_labels = train_labels[10000:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pH4X601Gy9TO"
      },
      "source": [
        "## 1.4 数据增强测试"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hr8gRQmAus-_"
      },
      "source": [
        "from keras.preprocessing import image\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "datagen = ImageDataGenerator(\n",
        " rotation_range=40,\n",
        " width_shift_range=0.2,\n",
        " height_shift_range=0.2,\n",
        " shear_range=0.5,\n",
        " zoom_range=0.2,\n",
        " horizontal_flip=True,\n",
        " fill_mode='nearest')\n",
        "\n",
        "fname = [os.path.join(train_cats_dir, fname) for fname in os.listdir(train_cats_dir)]\n",
        "img_path = fname[0]\n",
        "img = image.load_img(img_path, target_size=(150, 150))\n",
        "x = image.img_to_array(img)\n",
        "x = x.reshape((1,) + x.shape)\n",
        "\n",
        "i = 0\n",
        "for batch in datagen.flow(x, batch_size=1):\n",
        "  plt.figure(i)\n",
        "  imgplot = plt.imshow(image.array_to_img(batch[0]))\n",
        "  i += 1\n",
        "  if i % 4 == 0:\n",
        "    break\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6y7WoVtS8aWD"
      },
      "source": [
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "train_datagen = ImageDataGenerator(\n",
        "    rescale=1./255,\n",
        "    rotation_range=40,\n",
        "    width_shift_range=0.2,\n",
        "    height_shift_range=0.2,\n",
        "    shear_range=0.5,\n",
        "    zoom_range=0.2,\n",
        "    horizontal_flip=True,\n",
        "    fill_mode='nearest')\n",
        "\n",
        "test_datagen = ImageDataGenerator(rescale=1./255)\n",
        "\n",
        "train_generator = train_datagen.flow_from_directory(\n",
        "    train_dir,\n",
        "    target_size=(150, 150),\n",
        "    # classes=['dogs', 'cats'],\n",
        "    batch_size=20,\n",
        "    class_mode='binary'\n",
        ")\n",
        "\n",
        "validation_generator = test_datagen.flow_from_directory(\n",
        "    validation_dir,\n",
        "\n",
        "    target_size=(150, 150),\n",
        "    batch_size=20,\n",
        "    class_mode='binary'\n",
        ")\n",
        "\n",
        "print(train_generator.class_indices)\n",
        "print(validation_generator.class_indices)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMwzXDaqHxeW"
      },
      "source": [
        "# 2. 模型"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VTfes05BrsDx"
      },
      "source": [
        "## 2.1 构建"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lD3ShFylFPUt"
      },
      "source": [
        "from keras import models\n",
        "from keras import layers\n",
        "\n",
        "from keras import optimizers\n",
        "from keras import losses\n",
        "from keras import metrics\n",
        "\n",
        "\n",
        "network = models.Sequential()\n",
        "network.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(150, 150, 3)))\n",
        "network.add(layers.MaxPooling2D((2, 2)))\n",
        "network.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
        "network.add(layers.MaxPooling2D((2, 2)))\n",
        "network.add(layers.Conv2D(128, (3, 3), activation='relu'))\n",
        "network.add(layers.MaxPooling2D((2, 2)))\n",
        "network.add(layers.Conv2D(128, (3, 3), activation='relu'))\n",
        "network.add(layers.MaxPooling2D((2, 2)))\n",
        "network.add(layers.Flatten())\n",
        "network.add(layers.Dropout(0.5))\n",
        "network.add(layers.Dense(512, activation='relu'))\n",
        "network.add(layers.Dense(1, activation=\"sigmoid\"))\n",
        "network.compile(optimizer=optimizers.RMSprop(lr=0.001),\n",
        "        loss=losses.binary_crossentropy,\n",
        "        metrics=['acc'])\n",
        "        # metrics=metrics.binary_accuracy)\n",
        "network.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wjZ1_Hj-H1wl"
      },
      "source": [
        "## 2.2 训练"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RfFdr9zUat8"
      },
      "source": [
        "network.fit_generator(\n",
        "      train_generator, \n",
        "      steps_per_epoch=100,\n",
        "      epochs=100, \n",
        "      validation_data=validation_generator, \n",
        "      validation_steps=50)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LeSWDidHrsDy"
      },
      "source": [
        "# 3 model 数据存储"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8Cvi1T1PwD9"
      },
      "source": [
        "import pickle\n",
        "import os\n",
        "\n",
        "\n",
        "history_path = './history/cat_dog_v1.0.pickle'\n",
        "model_path = './model/cat_dog_v1.0'\n",
        "\n",
        "history_dir = os.path.dirname(history_path)\n",
        "model_dir = os.path.dirname(model_path)\n",
        "\n",
        "if not os.path.isdir(history_dir):\n",
        "  os.mkdir(history_dir)\n",
        "if not os.path.isdir(model_dir):\n",
        "  os.mkdir(model_dir)\n",
        "\n",
        "fp = open(history_path, 'wb')\n",
        "pickle.dump(network.history.history, fp)\n",
        "fp.close()\n",
        "\n",
        "network.save(model_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5oSCdtvXr3Qq"
      },
      "source": [
        "# 4. model 数据恢复"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ooBYRNVDsX_K"
      },
      "source": [
        "import pickle\n",
        "\n",
        "history_path = './history/cat_dog_v1.0.pickle'\n",
        "fp = open(history_path, 'rb')\n",
        "\n",
        "history = pickle.load(fp)\n",
        "fp.close()\n",
        "model_path = './model/cat_dog_v1.0'\n",
        "network = models.load_model(model_path)\n",
        "print(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "celPRAmTrsDz"
      },
      "source": [
        "# 5. history 分析\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6wIMvRLBrsDz"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "loss = history['loss']\n",
        "accuracy = history['acc']\n",
        "val_loss = history['val_loss']\n",
        "val_accuracy = history['val_acc']\n",
        "\n",
        "epoch = range(1, len(loss)+1)\n",
        "\n",
        "plt.figure(figsize=(20,10))\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.title('loss')\n",
        "plt.plot(epoch, loss, 'bo', label='loss')\n",
        "plt.plot(epoch, val_loss, 'r.', label='val_loss')\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.title('accuracy')\n",
        "plt.plot(epoch, accuracy, 'bo', label='accuracy')\n",
        "plt.plot(epoch, val_accuracy, 'r.', label='val_accuracy')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lswE5lrnH5Oi"
      },
      "source": [
        "## 5.1 测试集验证"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z_74DNDDHD4v"
      },
      "source": [
        "test_loss, test_acc = network.evaluate(test_images, test_labels)\n",
        "print(test_acc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IRa-83j_IBEj"
      },
      "source": [
        "## 其他"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H281-oE3IDIg"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "digit = train_images[4].reshape(28,28)\n",
        "print(digit.shape)\n",
        "plt.imshow(digit, cmap=plt.cm.binary)\n",
        "plt.show()\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kF_b80N5G__i"
      },
      "source": [
        "for j in range(0, 6, 2):\n",
        "  print(j)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xLICmwcWHCf-"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}