preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle as pc\n",
    "import numpy as np\n",
    "import subprocess\n",
    "import h5py\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "list_of_files = subprocess.getoutput(\"ls iccv09Data/labels/\").split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "715"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list_of_files[0:740])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def open_file(path):\n",
    "    \"\"\"\n",
    "        Takes a a text labels path and returns an text file object\n",
    "    \"\"\"\n",
    "    txt = open(path,'r')\n",
    "    y = txt.read()\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def numpify(txt):\n",
    "    \"\"\"\n",
    "        Takes a text object lable and returns an numpy array of it\n",
    "    \"\"\"\n",
    "    img = []\n",
    "    l = txt.split('\\n')\n",
    "    for i in l:\n",
    "        k = i.split(' ')\n",
    "        temp = []\n",
    "        for j in k:\n",
    "            temp.append(j)\n",
    "        if temp != ['']:\n",
    "            img.append(temp)\n",
    "#     print(len(img))\n",
    "#     print(len(img[0]))\n",
    "    return np.array(img,dtype='uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def one_hot_encodify(labels_mat):\n",
    "    \"\"\"\n",
    "        Takes an 240*320 numpy array of labels and returns an one hot encoding array of dimension 8*240*320\n",
    "    \"\"\"\n",
    "    cl = np.zeros((8,240,320))\n",
    "    classes = [i for i in range(8)]\n",
    "    for i in classes:\n",
    "        h = np.reshape(labels_mat,(240*320))\n",
    "        p = np.zeros((240*320))\n",
    "        for j in range(len(h)):\n",
    "            if h[j] == i:\n",
    "                  p[j] = 1\n",
    "        cl[i] = np.reshape(p,(240,320))\n",
    "    return cl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "241"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y.split('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y=open_file(\"./iccv09Data/labels/0000047.regions.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 48 ms, sys: 0 ns, total: 48 ms\n",
      "Wall time: 45.6 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 5, 5, 5],\n",
       "       [0, 0, 0, ..., 5, 5, 5],\n",
       "       [0, 0, 0, ..., 5, 5, 5],\n",
       "       ..., \n",
       "       [2, 2, 2, ..., 2, 2, 2],\n",
       "       [2, 2, 2, ..., 2, 2, 2],\n",
       "       [2, 2, 2, ..., 2, 2, 2]], dtype=uint8)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time numpify(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.84 s, sys: 0 ns, total: 5.84 s\n",
      "Wall time: 5.83 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(8, 240, 320)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time one_hot_encodify(numpify(y)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "one_hot_encodings = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "not_processed=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./h5py/h5py_0000072.regions.txt'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'./h5py/h5py_'+list_of_files[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with h5py.File('./h5py/h5py_'+list_of_files[3],'w') as hf:\n",
    "    hf.create_dataset('OHE', data=one_hot_encodify(numpify(open_file(\"./iccv09Data/labels/\" + list_of_files[3]))),compression='gzip',compression_opts=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def oheify(list_of_files):\n",
    "    for file in list_of_files:\n",
    "#         print(one_hot_encodify(numpify(open_file(\"./iccv09Data/labels/\" + file))))\n",
    "        try:\n",
    "            print(\"Processing file {}  ...\".format(file))\n",
    "            with h5py.File('./h5py/h5py_'+file,'w') as hf:\n",
    "                hf.create_dataset('OHE', data=one_hot_encodify(numpify(open_file(\"./iccv09Data/labels/\" + file))),compression='gzip',compression_opts=9)\n",
    "        except ValueError:\n",
    "            print(file)\n",
    "            not_processed.append(file)\n",
    "        except Exception as e:\n",
    "            print('Exception : ',file, ' ' , e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pool = Pool(processes=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(20):\n",
    "    pool.apply_async(oheify,[list_of_files[40*i:40*(i+1)]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(not_processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_files = [i for i in list_of_files if 'h5py_'+i not in not_retain]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "560"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(processed_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "h55ied = subprocess.getoutput('ls h5py/').split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'36864'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "not_retain = []\n",
    "for i in h55ied:\n",
    "    if int(subprocess.getoutput('stat --printf=\"%s\" h5py/'+i)) < 1000:\n",
    "        not_retain.append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "with open('processed_files.pkl','wb') as f:\n",
    "    pc.dump(processed_files,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle as pc\n",
    "pro = []\n",
    "with open('processed_files.pkl','rb') as f:\n",
    "    pro = pc.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "not_ther = [i for i in list_of_files if i not in pro]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in not_ther:\n",
    "    subprocess.getoutput('rm iccv09Data/images/'+i.split('.')[0]+'.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"  File: 'iccv09Data/images/9005273.jpg'\\n  Size: 11460     \\tBlocks: 24         IO Block: 4096   regular file\\nDevice: 805h/2053d\\tInode: 3181262     Links: 1\\nAccess: (0644/-rw-r--r--)  Uid: ( 1000/   dibya)   Gid: ( 1000/   dibya)\\nAccess: 2018-04-30 13:54:16.903570721 +0530\\nModify: 2009-07-08 05:20:32.000000000 +0530\\nChange: 2018-01-15 22:40:01.435712532 +0530\\n Birth: -\""
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0102435.jpg'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pro[45].split('.')[0]+'.jpg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in not_retain:\n",
    "    subprocess.getoutput('rm h5py/'+i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "with open('1_hot_encodings_processed.pkl','wb') as f:\n",
    "    pc.dump(one_hot_encodings,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "u = np.array(one_hot_encodings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "np.save(open(\"test\",'wb'),u)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}