TrainingData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Load the Pre Processed Data into sparse matrix for memory efficiency **\n",
    "\n",
    "Total Rows = 20104\n",
    "Total Cols = 13626\n",
    "Amount of Data for double = 8 * 20104 * 13626 = 2.09G\n",
    "\n",
    "If we have a limited RAM of 8G we are running out of RAM if we run some classification algorithm on this. So we need to optimize the memory. So storing the data in linked list sparse matrix and then for efficient arithmetic operations converting it to Compressed sparse row matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "\n",
    "class TrainingData:\n",
    "\n",
    "    def __init__(self, filename='training/data_with_comma.csv'):\n",
    "        (self.app_data, self.app_names_with_labels) = self.load_from_file(filename)\n",
    "        self.compressed_app_data = self.app_data.tocsr()\n",
    "        self.app_labels = np.array([ val[1] for val in self.app_names_with_labels ])\n",
    "        self.app_names = np.array([val[0] for val in self.app_names_with_labels])\n",
    "        self.total_sorted_app_labels = sorted(set(self.app_labels))\n",
    "    \n",
    "    def load_from_file(self, filename):\n",
    "        total_rows = 20104\n",
    "        total_cols = 13626\n",
    "        app_data = lil_matrix((total_rows, total_cols), dtype='double')\n",
    "        app_names_with_labels = []\n",
    "        app_name_to_label = dict({})\n",
    "        with open(filename,'r') as dest_f:\n",
    "            data_iter = csv.reader(dest_f, \n",
    "                                   delimiter = \",\")\n",
    "\n",
    "            for rowidx, row in enumerate(data_iter):\n",
    "                app_names_with_labels.append((row[0],row[1]))\n",
    "                for colidx, val in zip(*[iter(row[2:])]*2):\n",
    "                    app_data[rowidx,colidx] = float(val)\n",
    "        return (app_data, app_names_with_labels)\n",
    "    \n",
    "    def get_data_and_labels(self, idx):\n",
    "        return (self.compressed_app_data[idx], self.app_labels[idx])\n",
    "\n",
    "    def get_app_names(self, idx):\n",
    "        return self.app_names[idx]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}