CaloDNN-AnalyzePerformance.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Single Model Energy-dependent Classification Performance Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU 3\n",
      "Found 12 CPUs and 4 GPUs. Using 3 threads. max_threads = 12\n",
      "HyperParameter Scan:  240 possible combiniations.\n",
      "______________________________________\n",
      "ScanConfiguration\n",
      "______________________________________\n",
      "Picked combination:  0\n",
      "Combo[0]={'Width': 32, 'Depth': 1, 'lr': 0.01, 'optimizer': \"'RMSprop'\", 'decay': 0.01}\n",
      "Model Filename:  CaloDNN_32_1_0.01_RMSprop_0.01\n",
      "______________________________________\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "WARNING (theano.sandbox.cuda): The cuda backend is deprecated and will be removed in the next release (v0.10).  Please switch to the gpuarray backend. You can get more information about how to switch at this URL:\n",
      " https://github.com/Theano/Theano/wiki/Converting-to-the-new-gpu-back-end%28gpuarray%29\n",
      "\n",
      "Using gpu device 3: GeForce GTX 1080 (CNMeM is disabled, cuDNN 5105)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Searching in : /data/LCD/*/*.h5\n",
      "Found 639 files.\n",
      "Train Class Index Map: {'Pi0': 0, 'ChPi': 1, 'Gamma': 2, 'Ele': 3}\n",
      "Caching data on disk for faster processing after first epoch. Hope you have enough disk space.\n",
      "Loading Model From: TrainedModels.Test/CaloDNN_32_1_0.01_RMSprop_0.01_Merged.9\n",
      "Output Directory: TrainedModels/CaloDNN_32_1_0.01_RMSprop_0.01_Merged.9\n",
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 25, 25, 25)    0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_2 (InputLayer)             (None, 5, 5, 60)      0                                            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)              (None, 15625)         0                                            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)              (None, 1500)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 15625)         0                                            \n",
      "____________________________________________________________________________________________________\n",
      "activation_3 (Activation)        (None, 1500)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 15625)         62500                                        \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNorm (None, 1500)          6000                                         \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 32)            500032                                       \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 32)            48032                                        \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 32)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "activation_4 (Activation)        (None, 32)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 32)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 32)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 64)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_3 (Dense)                  (None, 4)             260                                          \n",
      "====================================================================================================\n",
      "Total params: 616,824.0\n",
      "Trainable params: 582,574.0\n",
      "Non-trainable params: 34,250.0\n",
      "____________________________________________________________________________________________________\n",
      "Compiling Model.\n",
      "Warning: optimizer configuration parameter epsilon was not set in configuration file. Will use default.\n",
      "Warning: optimizer configuration parameter rho was not set in configuration file. Will use default.\n",
      "Skipping Training.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "# Simply Run the usual experiment, using -L option to specify the model to load\n",
    "# Sets up the model and the data\n",
    "\n",
    "%run -im CaloDNN.ClassificationExperiment -- --NoAnalysis --NoTrain --cpu -L TrainedModels.Test/CaloDNN_32_1_0.01_RMSprop_0.01_Merged.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Searching in : /data/LCD/*/*.h5\n",
      "Found 639 files.\n",
      ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\n"
     ]
    }
   ],
   "source": [
    "TrainSampleList,TestSampleList,Norms,shapes=SetupData(FileSearch,\n",
    "                                                      ECAL,HCAL,True,NClasses,\n",
    "                                                      [float(NSamples)/MaxEvents,\n",
    "                                                       float(NTestSamples)/MaxEvents],\n",
    "                                                      Particles,\n",
    "                                                      BatchSize,\n",
    "                                                      multiplier,\n",
    "                                                      ECALShape,\n",
    "                                                      HCALShape,\n",
    "                                                      ECALNorm,\n",
    "                                                      HCALNorm)\n",
    "\n",
    "Test_genC = MakeGenerator(ECAL, HCAL, TestSampleList, NTestSamples, LCDNormalization(Norms),\n",
    "                          Merge=False,\n",
    "                          batchsize=BatchSize,\n",
    "                          shapes=shapes,\n",
    "                          n_threads=n_threads,\n",
    "                          multiplier=multiplier,\n",
    "                          cachefile=\"/tmp/CaloDNN-Analysis.h5\")\n",
    "\n",
    "print \"Loading Data into Memory:\"\n",
    "Test_genC.PreloadData(n_threads_cache)\n",
    "Test_X_ECAL, Test_X_HCAL, target, Test_Y = tuple(Test_genC.D)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/afarbin/.virtualenvs/keras2/local/lib/python2.7/site-packages/matplotlib/__init__.py:1401: UserWarning:  This call to matplotlib.use() has no effect\n",
      "because the backend has already been chosen;\n",
      "matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "  warnings.warn(_use_error_msg)\n"
     ]
    }
   ],
   "source": [
    "# Make the ROC Curves\n",
    "# Also performs inference on the test data, returning the results\n",
    "from DLAnalysis.Classification import *\n",
    "result,NewMetaData=MultiClassificationAnalysis(MyModel,[Test_X_ECAL,Test_X_HCAL],Test_Y,BatchSize,\n",
    "                                                       IndexMap={0:'Pi0', 2:'ChPi', 3:'Gamma', 1:'Ele'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Bin the data\n",
    "Energy=target[:,:,2].flatten()\n",
    "\n",
    "def AUCvsEnergy(E_min=10.,E_max=510.,E_bins=100.):\n",
    "    BD,E_binning=BinDataIndex(Energy, E_min, E_max, E_bins)\n",
    "    # Run the Classification Analysis in Bins\n",
    "    return BinMultiClassificationAnalysis(MyModel,Test_Y=Test_Y,Y_binning=E_binning,\n",
    "                                          bin_indecies=BD, result=result,\n",
    "                                          IndexMap={0:'Pi0', 2:'ChPi', 3:'Gamma', 1:'Ele'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Full Energy Range\n",
    "Res=AUCvsEnergy(10.,510.,50.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 10 to 100 GeV\n",
    "Res=AUCvsEnergy(10.,110.,20.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 250 to 350 GeV\n",
    "Res=AUCvsEnergy(250.,350.,50.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tmp=plt.hist(Energy,bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}