LCD_e_vs_pi-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LCD e VS pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "gpu = 'gpu4'\n",
    "#file('/home/%s/.theanorc'%os.getenv('USER'),'w').write('[nvcc]\\nfastmath=True\\nflags =  -arch=sm_30\\n[global]\\n#mode=FAST_RUN\\ndevice=%s\\nfloatX=float32'%gpu)\n",
    "#import theano\n",
    "import keras\n",
    "#print(theano.config.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nn_packages import *\n",
    "#from io_functions import *\n",
    "import numpy as np\n",
    "import root_numpy as rnp\n",
    "import os\n",
    "import sys\n",
    "import re\n",
    "import glob\n",
    "import h5py\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "from keras.models import Sequential,Model\n",
    "from keras.layers import Dense, Activation,Input, Dense, Dropout, merge\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!which python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Shuffle the \"e vs pi\" files, for classification and regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Classification_Generator:\n",
    "    '''\n",
    "    Data generator class for directory of h5 files\n",
    "    '''\n",
    "\n",
    "    def __init__( self, batch_size, train_split=0.7, validation_split=0.2, test_split=0.1):\n",
    "        self.batch_size = batch_size\n",
    "        self.filelist=[]\n",
    "        for i in xrange(1,161):\n",
    "            self.filelist.append('/bigdata/shared/LCD/kaustuv1993/NewLCD/ChPiEle_Shuffled/ChPiEle_%d.h5'%i)\n",
    "        self.train_split = train_split \n",
    "        self.validation_split = validation_split\n",
    "        self.test_split = test_split\n",
    "        self.fileindex = 0\n",
    "        self.filesize = 0\n",
    "        self.position = 0\n",
    "        \n",
    "    #function to call when generating data for training\n",
    "    def train(self, modeltype=3):\n",
    "        '''\n",
    "        Generate data for training only\n",
    "        '''\n",
    "        length = len(self.filelist)\n",
    "        #deleting the validation and test set filenames from the filelist\n",
    "        del self.filelist[np.floor((1-(self.train_split))*length).astype(int):]\n",
    "        return self.batches(modeltype)\n",
    "    \n",
    "    \n",
    "    #function to call when generating data for testing\n",
    "    def test(self, modeltype=3):\n",
    "        '''\n",
    "        Generate data for testing only\n",
    "        '''\n",
    "        length = len(self.filelist)\n",
    "        #deleting the train and validation set filenames from the filelist\n",
    "        del self.filelist[:np.floor((1-self.test_split)*length).astype(int)+1]\n",
    "        return self.batches(modeltype)\n",
    "    \n",
    "    \n",
    "    #function to call when generating data for validating\n",
    "    def validation(self, modeltype=3):\n",
    "        '''\n",
    "        Generate data for validation only\n",
    "        '''\n",
    "        length = len(self.filelist)\n",
    "        #modifying the filename list to only include files for validation set\n",
    "        self.filelist = self.filelist[np.floor(self.train_split*length+1).astype(int):np.floor((self.train_split+self.validation_split)*length+1).astype(int)]\n",
    "        return self.batches(modeltype)\n",
    "\n",
    "        \n",
    "    #This function reads files to gather data until batch size is satisfied\n",
    "    def batch_helper(self, fileindex, position, batch_size):\n",
    "        '''\n",
    "        Reads files to gather data until batch size is satisfied, then yeilds\n",
    "        '''\n",
    "        f = h5py.File(self.filelist[fileindex],'r')\n",
    "        self.filesize = np.array(f['ECAL']).shape[0]\n",
    "\n",
    "\n",
    "        if (position + batch_size < self.filesize):\n",
    "            data_ECAL = np.array(f['ECAL'][position : position + batch_size])\n",
    "            #data_HCAL = np.array(f['HCAL'][position : position + batch_size])\n",
    "            target = np.array(f['target'][position : position + batch_size][:,0])\n",
    "            position += batch_size\n",
    "            f.close()\n",
    "            return data_ECAL, target, fileindex, position\n",
    "        \n",
    "        else:\n",
    "            data_ECAL = np.array(f['ECAL'][position : ])\n",
    "            #data_HCAL = np.array(f['HCAL'][position : ])\n",
    "            target = np.array(f['target'][position:][:,0])\n",
    "            #target = np.delete(target,0,1)\n",
    "            f.close()\n",
    "            \n",
    "\n",
    "            if (fileindex+1 < len(self.filelist)):\n",
    "                if(self.batch_size-data_ECAL.shape[0]>0):\n",
    "                    while(self.batch_size-data_ECAL.shape[0]>0):\n",
    "                        if(int(np.floor((self.batch_size-data_ECAL.shape[0]) / self.filesize))==0):\n",
    "                            number_of_files=1\n",
    "                        else:\n",
    "                            number_of_files=int(np.ceil((self.batch_size-data_ECAL.shape[0]) / self.filesize))\n",
    "                        for i in xrange(0,number_of_files):\n",
    "\n",
    "                            if fileindex + i + 1 > len(self.filelist):\n",
    "                                fileindex = -1 - i\n",
    "\n",
    "                            f = h5py.File(self.filelist[fileindex+i+1],'r')\n",
    "\n",
    "                            if (self.batch_size-data_ECAL.shape[0] < self.filesize):\n",
    "                                position = self.batch_size-data_ECAL.shape[0]\n",
    "                                data_temp_ECAL = np.array(f['ECAL'][ : position])\n",
    "                                #data_temp_HCAL = np.array(f['HCAL'][: position])\n",
    "                                target_temp = np.array(f['target'][:position][:,0])\n",
    "#target_temp = np.array(f['target'][:position][:,1]) for regression\n",
    "                            else:\n",
    "                                data_temp_ECAL = np.array(f['ECAL'])\n",
    "                                #data_temp_HCAL = np.array(f['HCAL'])\n",
    "                                target_temp = np.array(f['target'][:,0])\n",
    "\n",
    "                            f.close()\n",
    "                            data_ECAL = np.concatenate((data_ECAL, data_temp_ECAL), axis=0)\n",
    "                            #data_HCAL = np.concatenate((data_HCAL, data_temp_HCAL), axis=0)\n",
    "                            target = np.concatenate((target, target_temp), axis=0)\n",
    "\n",
    "                    if (fileindex +i+1 < len(self.filelist)):\n",
    "                        fileindex = fileindex +i+1\n",
    "                    else:\n",
    "                        fileindex = 0\n",
    "                else:\n",
    "                    position = 0\n",
    "                    fileindex=fileindex+1\n",
    "            else:\n",
    "                fileindex = 0\n",
    "                position = 0\n",
    "            \n",
    "            return data_ECAL, target, fileindex, position\n",
    "        \n",
    "        \n",
    "    #This function loops indefinitely and continues to return data of the specified batch size\n",
    "    def batches(self, modeltype):\n",
    "        '''\n",
    "        Loops indefinitely and continues to return data of specified batch size\n",
    "        '''\n",
    "        while (self.fileindex < len(self.filelist)):\n",
    "            data_ECAL, target, self.fileindex, self.position = self.batch_helper(self.fileindex, self.position, self.batch_size)\n",
    "            if data_ECAL.shape[0]!=self.batch_size:\n",
    "                continue\n",
    "\n",
    "            if modeltype == 3:\n",
    "                data_ECAL = data_ECAL.reshape((data_ECAL.shape[0],) + (1, 25, 25, 25))\n",
    "                #data_HCAL = data_HCAL.reshape((data_HCAL.shape[0],)+(1, 5, 5, 60))\n",
    "\n",
    "            elif modeltype == 2:\n",
    "                data_ECAL = data_ECAL.reshape((data_ECAL.shape[0],) + (25, 25, 25))\n",
    "                data_ECAL = np.swapaxes(data_ECAL, 1, 3)\n",
    "                #data_HCAL = data_HCAL.reshape((data_HCAL.shape[0],)+(4, 4, 60))\n",
    "                #data_HCAL = np.swapaxes(data_HCAL, 1, 3)\n",
    "\n",
    "            elif modeltype == 1:\n",
    "                data_ECAL= np.reshape(data_ECAL,(self.batch_size,-1))\n",
    "                #data_HCAL= np.reshape(data_HCAL,(self.batch_size,-1))\n",
    "\n",
    "            yield (data_ECAL, target)\n",
    "        self.fileindex = 0[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Functions to load and save regression models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadmodel(name, weights = False):\n",
    "    json_file = open('%s.json'%name, 'r')\n",
    "    loaded_model_json = json_file.read()\n",
    "    json_file.close()\n",
    "    model = model_from_json(loaded_model_json)\n",
    "    #load weights into new model\n",
    "    if weights == True:\n",
    "        model.load_weights('%s.h5'%name)\n",
    "    #print (model.summary())\n",
    "    print(\"Loaded model %s from disk\"%name)\n",
    "    return model\n",
    "\n",
    "def savemodel(model,name=\"neural network\"):\n",
    "    model_name = name\n",
    "    #model.summary()\n",
    "    model.save_weights('/home/vitoriabp/Models/Regression/%s_w.h5'%model_name, overwrite=True)\n",
    "    model_json = model.to_json()\n",
    "    with open(\"/home/vitoriabp/Models/Regression/%s_m.json\"%model_name, \"w\") as json_file:\n",
    "        json_file.write(model_json)\n",
    "        \n",
    "def savelosses(hist, name=\"neural network\"):    \n",
    "    loss = np.array(hist.history['loss'])\n",
    "    valoss = np.array(hist.history['val_loss'])\n",
    "    f = h5py.File(\"/home/vitoriabp/Models/Regression/%s_h.h5\"%name,\"w\")\n",
    "    f.create_dataset('loss',data=loss)\n",
    "    f.create_dataset('val_loss',data=valoss)\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "convolution3d_9 (Convolution3D)  (None, 21, 21, 1, 10) 6260        convolution3d_input_8[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling3d_6 (MaxPooling3D)    (None, 10, 10, 1, 10) 0           convolution3d_9[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)              (None, 1000)          0           maxpooling3d_6[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dense_16 (Dense)                 (None, 100)           100100      flatten_6[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)              (None, 100)           0           dense_16[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_17 (Dense)                 (None, 10)            1010        dropout_6[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_18 (Dense)                 (None, 1)             11          dense_17[0][0]                   \n",
      "====================================================================================================\n",
      "Total params: 107,381\n",
      "Trainable params: 107,381\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "Epoch 1/1000\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Error when checking model input: expected convolution3d_input_8 to have shape (None, 25, 25, 25, 1) but got array with shape (400, 1, 25, 25, 25)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mValueError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-01076f7068f3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0mcheck\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModelCheckpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"/home/vitoriabp/Models/Classification/%s_check.h5\"\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0mearly\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEarlyStopping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_loss'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'auto'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodeltype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamples_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnb_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodeltype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnb_val_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheck\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mearly\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0msavelosses\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/keras/models.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m    933\u001b[0m                                         \u001b[0mnb_worker\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnb_worker\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    934\u001b[0m                                         \u001b[0mpickle_safe\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpickle_safe\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 935\u001b[0;31m                                         initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m    936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    937\u001b[0m     def evaluate_generator(self, generator, val_samples,\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch)\u001b[0m\n\u001b[1;32m   1555\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   1556\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1557\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   1558\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1559\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1312\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1313\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1314\u001b[0;31m             check_batch_axis=True)\n\u001b[0m\u001b[1;32m   1315\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muses_learning_phase\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearning_phase\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_batch_axis, batch_size)\u001b[0m\n\u001b[1;32m   1027\u001b[0m                                    \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minternal_input_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m                                    \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1029\u001b[0;31m                                    exception_prefix='model input')\n\u001b[0m\u001b[1;32m   1030\u001b[0m         y = standardize_input_data(y, self.output_names,\n\u001b[1;32m   1031\u001b[0m                                    \u001b[0moutput_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    122\u001b[0m                             \u001b[0;34m' to have shape '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m                             \u001b[0;34m' but got array with shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m                             str(array.shape))\n\u001b[0m\u001b[1;32m    125\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking model input: expected convolution3d_input_8 to have shape (None, 25, 25, 25, 1) but got array with shape (400, 1, 25, 25, 25)"
     ]
    }
   ],
   "source": [
    "#scan over various models (with full z,y,x depths respectively) with different fractions of training set\n",
    "\n",
    "ts = [0.01,0.1,0.25,0.5,0.7]\n",
    "tsn = [\"0pt01\", \"0pt1\", \"0pt25\", \"0pt5\", \"0pt7\"]\n",
    "\n",
    "for j in xrange(1,6):\n",
    "    dim1 = [5, 5, 25]\n",
    "    dim2 = [5, 25, 5]\n",
    "    dim3 = [25, 5, 5]\n",
    "    scan = [\"xyscan\", \"xzscan\",\"yzscan\"]\n",
    "    pool = [(2,2,1), (2,1,2), (1,2,2)]\n",
    "    \n",
    "    for i in xrange(0,2):\n",
    "        model = Sequential()\n",
    "        model.add(Convolution3D(25, dim1[i], dim2[i], dim3[i], input_shape = (25, 25, 25, 1), activation='relu'))\n",
    "        model.add(MaxPooling3D(pool_size=pool[i]))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(100, activation='sigmoid'))\n",
    "        model.add(Dropout(0.25))\n",
    "        model.add(Dense(10, activation='sigmoid'))\n",
    "        model.add(Dense(1, activation='sigmoid'))\n",
    "        model.compile(loss='binary_crossentropy', optimizer='sgd')\n",
    "        model.summary()\n",
    "        \n",
    "        f=open(\"/home/vitoriabp/Models/Classification/Classification_Training_Log.txt\",\"w\")\n",
    "        \n",
    "        ds = Classification_Generator(400, train_split=ts[j])\n",
    "        vs = Classification_Generator(400)\n",
    "        \n",
    "        modelname=\"cnn3D_%s_cls_ChPiEle_ts%s\"%(scan[i],tsn[j])\n",
    "        check = ModelCheckpoint(filepath=\"/home/vitoriabp/Models/Classification/%s_check.h5\"%modelname, verbose=1)\n",
    "        early = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='auto')\n",
    "        hist = model.fit_generator(ds.train(modeltype=3), samples_per_epoch=50000, nb_epoch=1000, validation_data= vs.validation(modeltype=3), nb_val_samples=50000, verbose=1, callbacks=[check,early])\n",
    "        savelosses(hist,name=modelname)\n",
    "        \n",
    "        f.write(\"%s has trained, and early-stopped\"%modelname)\n",
    "        savemodel(model,name=modelname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}