Relu_newtest-Copy0-Copy0.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:8fa671ccb19b2b814dc6de9677a27085147c5a91f63afb515ad9fcc85ed3eb9a"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import math, string, random, itertools, copy\n",
      "import numpy as np\n",
      "from numpy import float32\n",
      "from math import sqrt\n",
      "import matplotlib.pyplot as plt\n",
      "from scipy.stats import poisson\n",
      "from copy        import deepcopy\n",
      "import time\n",
      "import theanets"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import h5py\n",
      "f=h5py.File(\"data/data.hdf5\")\n",
      "f.keys()\n",
      "f['data'].keys()\n",
      "\n",
      "global input_matrix\n",
      "global target_matrix\n",
      "global decorator_matrix\n",
      "\n",
      "# Load Data \n",
      "input_matrix     = f['data']['input'].value\n",
      "target_matrix    = f['data']['target'].value\n",
      "decorator_matrix = f['data']['deco'].value\n",
      "\n",
      "f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "nInputs  = input_matrix.shape[1]\n",
      "nOutputs = target_matrix.shape[1]\n",
      "\n",
      "print \" Inputs=\", nInputs, \"       Outputs=\",nOutputs"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " Inputs= 82        Outputs= 1\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def allincluded(truncate, nHiddenLayers, nHiddenNeurons):\n",
      "    \n",
      "    IN_matrix     =input_matrix[0:truncate]\n",
      "    TAR_matrix    =target_matrix[0:truncate]  \n",
      "    \n",
      "    cut   =int(IN_matrix.shape[0]*0.8)\n",
      "    train =IN_matrix[:cut], TAR_matrix[:cut]    \n",
      "    valid =IN_matrix[cut:], TAR_matrix[cut:]\n",
      "\n",
      "    print \"Samples in Training Set=\", len(train[0])\n",
      "    print \"Samples in Validation Set=\", len(valid[0])\n",
      "    \n",
      "    # Build the NN\n",
      "    # -----------------------------------------------------------------------------------------------------------------------------------\n",
      "    # Topology: Feedforward\n",
      "    # Inputs = 82 (Jets 4-momenta (20 Jets maximum), METx, METy)\n",
      "    # Number of Hidden Layers = nHiddenLayers\n",
      "    # Activation function for Hidden Layers = RELU\n",
      "    # Outputs=1 (NN Trigger Bits= 0 or 1)\n",
      "    # Activation function for the Output Layer= RELU\n",
      "\n",
      "    exp=theanets.Experiment(theanets.feedforward.Regressor(layers=(nInputs,)+tuple([nHiddenNeurons for l in range(nHiddenLayers)])+ ( (nOutputs),)))  \n",
      "    # ------------------------------------------------------------------------------------------------------------------------------------\n",
      "    \n",
      "    fom          = 1.0\n",
      "    LearningRate = 0.0001\n",
      "    Epochs       = 0\n",
      "    batch_size   = 1000\n",
      "    \n",
      "    timing_start = time.mktime(time.gmtime())\n",
      "\n",
      "    for TrainError, ValError in exp.itertrain(train,valid, \n",
      "                                              learning_rate= LearningRate, \n",
      "                                              algo='rprop',\n",
      "                                              hidden_dropout=0.9, \n",
      "                                              iteration_size = 200,\n",
      "                                              batch_size=batch_size,\n",
      "                                              validate_every = 1,\n",
      "                                              patience=100, \n",
      "                                              min_improvement = 0.00001, \n",
      "                                              rms_halflife=0.99, \n",
      "                                              rms_regularizer=1e-09):\n",
      "        #print 'Epoch=', Epochs+1\n",
      "        if Epochs%1 == 0:\n",
      "            print 'Epoch=',Epochs+1, '     Training   Error =',TrainError['err'] , '           Validation Error =',ValError['err']\n",
      "        Epochs+=1\n",
      "        if Epochs == 20 :\n",
      "            break\n",
      "    \n",
      "    timing_end=time.mktime(time.gmtime())\n",
      "    \n",
      "    prediction = exp.network.predict(train[0])\n",
      "    goodness = np.asarray([[cut,(list(train[1][np.where(prediction>cut)]==1).count(True)+list(train[1][np.where(prediction<cut)]==0).count(True))/ float(train[0].shape[0])]  for cut in np.arange(0,1,0.05)])   \n",
      "    fom_train = max( goodness[:,1])    \n",
      "    \n",
      "    prediction = exp.network.predict(valid[0])\n",
      "    goodness   = np.asarray([[cut,(list(valid[1][np.where(prediction>cut)]==1).count(True)+list(valid[1][np.where(prediction<cut)]==0).count(True))/ float(valid[0].shape[0])]  for cut in np.arange(0,1,0.05)])\n",
      "    fom_valid  = max( goodness[:,1])     \n",
      "\n",
      "    MaxCut= goodness[np.where(goodness[:,1]==goodness[:,1].max())][0][0]\n",
      "\n",
      "    timing=timing_end-timing_start\n",
      "    \n",
      "    exp.save('network-pickle.pkl.gz')\n",
      "    \n",
      "    return fom_valid, fom_train, MaxCut,timing"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mapAlgo=[]\n",
      "\n",
      "for iLayers in range(1,11):\n",
      "    for nHiddenNeurons in range(200, 1400, 200):\n",
      "        for truncate in range(100000, 1100000, 100000):\n",
      "            fom_valid, fom_train, MaxCut, timing = allincluded(truncate, iLayers, nHiddenNeurons)\n",
      "            print  \"Samples=\",truncate,\" Hidden Neurons=\", nHiddenNeurons, \" Hid. Layers=\",iLayers, \"  FOM train=\",fom_train, \"FOM valid=\", fom_valid, \"  Time=\", timing, ' MaxCut=',MaxCut \n",
      "            mapAlgo.append((truncate, nHiddenNeurons, iLayers, fom_train, fom_valid, timing, MaxCut))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Samples in Training Set= 80000\n",
        "Samples in Validation Set= 20000\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1      Training   Error = 4.27229329225            Validation Error = 16.0707268537\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 2      Training   Error = 0.193835088077            Validation Error = 0.226921824835\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 3      Training   Error = 0.126481261518            Validation Error = 0.146811722609\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 4      Training   Error = 0.111481736695            Validation Error = 0.117295383038\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 5      Training   Error = 0.105606888514            Validation Error = 0.10905494478\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 6      Training   Error = 0.102129390123            Validation Error = 0.105141730477\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 7      Training   Error = 0.100140072158            Validation Error = 0.103690287079\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 8      Training   Error = 0.0989228090986            Validation Error = 0.102109908649\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 9      Training   Error = 0.0976234925083            Validation Error = 0.101127999042\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 10      Training   Error = 0.0968831602388            Validation Error = 0.100475976085\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 11      Training   Error = 0.0958606623011            Validation Error = 0.100019543978\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 12      Training   Error = 0.0954321800742            Validation Error = 0.0999223004105\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 13      Training   Error = 0.0947059014921            Validation Error = 0.0993504745003\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 14      Training   Error = 0.0940492359744            Validation Error = 0.0991934501878\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 15      Training   Error = 0.0934880080185            Validation Error = 0.0989266936543\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 16      Training   Error = 0.093284156787            Validation Error = 0.0987425372446\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 17      Training   Error = 0.0927762712375            Validation Error = 0.0984875611459\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 18      Training   Error = 0.0924102848055            Validation Error = 0.0983809676568\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 19      Training   Error = 0.0920663202566            Validation Error = 0.0982196049418\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 20      Training   Error = 0.091698411788            Validation Error = 0.0981677506471\n",
        "Samples="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 100000  Hidden Neurons= 200  Hid. Layers= 1   FOM train= 0.895725 FOM valid= 0.88205   Time= 139.0  MaxCut= 0.5\n",
        "Samples in Training Set= 160000\n",
        "Samples in Validation Set= 40000\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1      Training   Error = 0.406247186564            Validation Error = 1.95251225696\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 2      Training   Error = 0.209168765054            Validation Error = 0.228432987117\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 3      Training   Error = 0.144821887161            Validation Error = 0.177514136977\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 4      Training   Error = 0.121993142481            Validation Error = 0.127799174498\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 5      Training   Error = 0.113095963165            Validation Error = 0.11578501416\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 6      Training   Error = 0.108365348665            Validation Error = 0.109666039556\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 7      Training   Error = 0.105449868499            Validation Error = 0.106608019084\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 8      Training   Error = 0.103939497276            Validation Error = 0.104670912877\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 9      Training   Error = 0.102285973788            Validation Error = 0.103220372724\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 10      Training   Error = 0.101431328524            Validation Error = 0.102150512705\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 11      Training   Error = 0.100512659537            Validation Error = 0.101173355305\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 12      Training   Error = 0.0995002073524            Validation Error = 0.100673164636\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 13      Training   Error = 0.0990091352411            Validation Error = 0.0999409346908\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 14      Training   Error = 0.098510391522            Validation Error = 0.0997321713993\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 15      Training   Error = 0.0979219860715            Validation Error = 0.0992220395821\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 16      Training   Error = 0.097705089406            Validation Error = 0.0988673550224\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 17      Training   Error = 0.0971365679039            Validation Error = 0.0986104365396\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 18      Training   Error = 0.0969557682004            Validation Error = 0.0983763220396\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 19      Training   Error = 0.0963995455613            Validation Error = 0.0981976957792\n",
        "Epoch="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 20      Training   Error = 0.096242079738            Validation Error = 0.0979568335479\n",
        "Samples="
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 200000  Hidden Neurons= 200  Hid. Layers= 1   FOM train= 0.886525 FOM valid= 0.88295   Time= 123.0  MaxCut= 0.5\n",
        "Samples in Training Set= 240000\n",
        "Samples in Validation Set= 60000\n",
        "Epoch="
       ]
      }
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print mapAlgo"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(100000, 2, 200, 770.0, 0.88149999999999995, 0.89511249999999998, 0.5)]\n"
       ]
      }
     ],
     "prompt_number": 10
    }
   ],
   "metadata": {}
  }
 ]
}