5.2.1 卷积神经网络-猫狗分类-制作数据集.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 制作数据集\n",
    "下载原始数据集：https://www.kaggle.com/c/dogs-vs-cats/data\n",
    "\n",
    "这个数据集包含25000张猫狗图像（每个类别都有12  500张），大小为543MB（压缩后）。\n",
    "\n",
    "本次训练并不使用全部图像，只是使用其中的4000幅图像（每个类别各2000张）。\n",
    "下载数据并解压之后，创建一个新数据集，其中包含三个子集：\n",
    "- train ： 2000张图像（1000张猫图像，1000张狗图像）\n",
    "- validation：1000张图像（500张猫图像，500张狗图像）\n",
    "- test：1000张图像（500张猫图像，500张狗图像）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 防止报错 OMP: Error #15: Initializing libiomp5.dylib, but found libiomp5.dylib already initialized.\n",
    "import os\n",
    "os.environ[\"KMP_DUPLICATE_LIB_OK\"]=\"TRUE\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 创建目录\n",
    "\n",
    "import os, shutil \n",
    "\n",
    "# 提取部分图片组成一个较小的数据集\n",
    "base_dir = './data/cats_and_dogs_small'  \n",
    "if not os.path.exists(base_dir):\n",
    "    os.mkdir(base_dir) \n",
    "\n",
    "# /cats_and_dogs_small/train\n",
    "train_dir = os.path.join(base_dir, 'train') \n",
    "if not os.path.exists(train_dir):\n",
    "    os.mkdir(train_dir) \n",
    "\n",
    "# /cats_and_dogs_small/validation\n",
    "validation_dir = os.path.join(base_dir, 'validation') \n",
    "if not os.path.exists(validation_dir):\n",
    "    os.mkdir(validation_dir) \n",
    "\n",
    "# /cats_and_dogs_small/test\n",
    "test_dir = os.path.join(base_dir, 'test') \n",
    "if not os.path.exists(test_dir):\n",
    "    os.mkdir(test_dir) \n",
    "\n",
    "# /cats_and_dogs_small/train/cats\n",
    "train_cats_dir = os.path.join(train_dir, 'cats')  \n",
    "if not os.path.exists(train_cats_dir):\n",
    "    os.mkdir(train_cats_dir) \n",
    "\n",
    "# /cats_and_dogs_small/train/dogs\n",
    "train_dogs_dir = os.path.join(train_dir, 'dogs')   \n",
    "if not os.path.exists(train_dogs_dir):\n",
    "    os.mkdir(train_dogs_dir) \n",
    "\n",
    "# /cats_and_dogs_small/validation/cats\n",
    "validation_cats_dir = os.path.join(validation_dir, 'cats') \n",
    "if not os.path.exists(validation_cats_dir):\n",
    "    os.mkdir(validation_cats_dir) \n",
    "\n",
    "# /cats_and_dogs_small/validation/dogs\n",
    "validation_dogs_dir = os.path.join(validation_dir, 'dogs')  \n",
    "if not os.path.exists(validation_dogs_dir):\n",
    "    os.mkdir(validation_dogs_dir) \n",
    "\n",
    "# /cats_and_dogs_small/test/cats\n",
    "test_cats_dir = os.path.join(test_dir, 'cats') \n",
    "if not os.path.exists(test_cats_dir):\n",
    "    os.mkdir(test_cats_dir) \n",
    "\n",
    "# /cats_and_dogs_small/test/dogs\n",
    "test_dogs_dir = os.path.join(test_dir, 'dogs')  \n",
    "if not os.path.exists(test_dogs_dir):\n",
    "    os.mkdir(test_dogs_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 复制图像到相应的目录\n",
    "\n",
    "#原始数据集的目录\n",
    "original_dataset_dir = '/mnt/资料/人工智能/python深度学习代码和数据集/python深度学习代码和数据集/data/kaggle_original_data'  \n",
    "\n",
    "# 制作数据集\n",
    "# 将前1000张猫的图像复制到train_cats_dir\n",
    "if not os.listdir(train_cats_dir):\n",
    "    fnames = ['cat.{}.jpg'.format(i) for i in range(1000)]  \n",
    "    for fname in fnames: \n",
    "        src = os.path.join(original_dataset_dir, fname) \n",
    "        dst = os.path.join(train_cats_dir, fname) \n",
    "        shutil.copyfile(src, dst) \n",
    "\n",
    "# 将接下来的500张猫的图像复制到validation_cats_dir\n",
    "if not os.listdir(validation_cats_dir):\n",
    "    fnames = ['cat.{}.jpg'.format(i) for i in range(1000, 1500)]  \n",
    "    for fname in fnames: \n",
    "        src = os.path.join(original_dataset_dir, fname) \n",
    "        dst = os.path.join(validation_cats_dir, fname) \n",
    "        shutil.copyfile(src, dst) \n",
    "\n",
    "# 将接下来的500张猫的图像复制到test_cats_dir\n",
    "if not os.listdir(test_cats_dir):\n",
    "    fnames = ['cat.{}.jpg'.format(i) for i in range(1500, 2000)]  \n",
    "    for fname in fnames: \n",
    "        src = os.path.join(original_dataset_dir, fname) \n",
    "        dst = os.path.join(test_cats_dir, fname) \n",
    "        shutil.copyfile(src, dst) \n",
    "\n",
    "# 将1000张狗的图像复制到train_dogs_dir  \n",
    "if not os.listdir(train_dogs_dir):\n",
    "    fnames = ['dog.{}.jpg'.format(i) for i in range(1000)]  \n",
    "    for fname in fnames: \n",
    "        src = os.path.join(original_dataset_dir, fname) \n",
    "        dst = os.path.join(train_dogs_dir, fname) \n",
    "        shutil.copyfile(src, dst) \n",
    "\n",
    "# 将接下来500张狗的图像复制到validation_dogs_dir    \n",
    "if not os.listdir(validation_dogs_dir):\n",
    "    fnames = ['dog.{}.jpg'.format(i) for i in range(1000, 1500)]  \n",
    "    for fname in fnames: \n",
    "        src = os.path.join(original_dataset_dir, fname) \n",
    "        dst = os.path.join(validation_dogs_dir, fname) \n",
    "        shutil.copyfile(src, dst) \n",
    "\n",
    "# 将接下来500张狗的图像复制到test_dogs_dir \n",
    "if not os.listdir(test_dogs_dir):\n",
    "    fnames = ['dog.{}.jpg'.format(i) for i in range(1500, 2000)]  \n",
    "    for fname in fnames: \n",
    "        src = os.path.join(original_dataset_dir, fname) \n",
    "        dst = os.path.join(test_dogs_dir, fname) \n",
    "        shutil.copyfile(src, dst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total training cat images: 1000\n",
      "total training dog images: 1000\n",
      "total validation cat images: 500\n",
      "total validation dog images: 500\n",
      "total test cat images: 500\n",
      "total test dog images: 500\n"
     ]
    }
   ],
   "source": [
    "# 查看生成的数据集\n",
    "\n",
    "print('total training cat images:', len(os.listdir(train_cats_dir))) \n",
    "\n",
    "print('total training dog images:', len(os.listdir(train_dogs_dir)))\n",
    "\n",
    "print('total validation cat images:', len(os.listdir(validation_cats_dir))) \n",
    "\n",
    "print('total validation dog images:', len(os.listdir(validation_dogs_dir))) \n",
    "\n",
    "print('total test cat images:', len(os.listdir(test_cats_dir))) \n",
    "\n",
    "print('total test dog images:', len(os.listdir(test_dogs_dir))) \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}