10C-4T-3L-2KN-1M-Batch.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:97cfd5fb2f684164b9221f86d7dc6c0114ff9b3ef732041c803582f22799539e"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "                                                # Assembling algorithms need comments\n",
      "\n",
      "# Section 1 : importing\n",
      "\n",
      "import theanets\n",
      "import matplotlib.pyplot as plt\n",
      "import time\n",
      "import numpy as np\n",
      "from theanets.trainer import SupervisedPretrainer\n",
      "import math\n",
      "from random import uniform\n",
      "from math import cos,tan,sqrt\n",
      "from numpy import float32,int32\n",
      "dt = np.dtype(float32)\n",
      "import pickle\n",
      "from sklearn.cluster import KMeans\n",
      "\n",
      "###################################################################################################################################\n",
      "# section 2: model topology and parameters that should be set for each code\n",
      "\n",
      "nlayers = 4\n",
      "module_cell = 10 #per edge\n",
      "min_ntracks = 4\n",
      "max_ntracks = 4\n",
      "nsamples = 1000000\n",
      "BATCH_SIZE = 1000\n",
      "Validation_Batch_Size = 2000\n",
      "LR = 1e-3\n",
      "train_ratio = 0.9\n",
      "cut = int(nsamples * train_ratio)\n",
      "Input_File = \"network/4Layers_10by10Cells-4Tracks-_Input.npy\"\n",
      "Output_File = \"network/4Layers_10by10Cells-4Tracks_Output.npy\"\n",
      "Network_File = \"network/4Layers_10by10Cells-4Tracks-3L-2000N-1M_Network\"\n",
      "ncells=module_cell*module_cell*nlayers\n",
      "nhidden=ncells\n",
      "trainer = theanets.Experiment(theanets.feedforward.Regressor(layers=(ncells,2000,2000,2000,(ncells,'logistic'))))\n",
      "\n",
      "####################################################################################################################################\n",
      "# Section 3: assembling algorithms\n",
      "\n",
      "def clusterer (in_array,revised_number = 0) :\n",
      "    n_lines = len(in_array)\n",
      "    n_gaps = n_lines - 1\n",
      "    gap_matrix = np.zeros((n_gaps,3),dt)\n",
      "    length_array = np.zeros(n_gaps,dt)\n",
      "    for i in range (n_gaps) :\n",
      "        gap_matrix[i][2] = i\n",
      "        gap_matrix[i][0] = in_array[i+1] - in_array[i]\n",
      "        length_array[i] = in_array[i+1] - in_array[i]\n",
      "        gap_matrix[i][1] = (in_array[i+1] + in_array[i])/2.0\n",
      "    length_array = sorted(length_array)\n",
      "    gap_matrix.view('f32,f32,f32').sort(order=['f0'], axis=0)\n",
      "    out_matrix = np.zeros((int(n_lines),2),dtype=int32)\n",
      "    if revised_number == 0 :\n",
      "        col = np.reshape(length_array,(len(length_array),1))\n",
      "        algo = KMeans(n_clusters=2)\n",
      "        gap_candidates = algo.fit_predict(col)\n",
      "        if gap_candidates[0] == 0:\n",
      "            accepted = list(gap_candidates).count(1)\n",
      "        else :\n",
      "            accepted = list(gap_candidates).count(0)\n",
      "    else :\n",
      "        accepted = revised_number-1\n",
      "    for i in range (n_gaps-accepted) :\n",
      "        gap_matrix[i][1] = in_array[n_gaps]\n",
      "    gap_matrix.view('f32,f32,f32').sort(order=['f1'], axis=0)\n",
      "    if accepted == 0 :\n",
      "        out_matrix [0] = [0,n_gaps]\n",
      "        return accepted+1,out_matrix\n",
      "    out_matrix [0] = [0,gap_matrix[0][2]]\n",
      "    out_matrix [accepted] = [gap_matrix[accepted-1][2]+1,n_gaps]\n",
      "    for bunch in range (1,accepted) :\n",
      "        out_matrix [bunch] = [int(gap_matrix[bunch-1][2]+1),int(gap_matrix[bunch][2])]\n",
      "    return accepted+1,out_matrix\n",
      "\n",
      "def clustering (in_array) :\n",
      "    central_tracks ,_ = clusterer (in_array)\n",
      "    col = np.reshape(in_array,(len(in_array),1))\n",
      "    min_n = max(1,int((0.75*central_tracks)))\n",
      "    max_n = int(1.25*central_tracks)\n",
      "    if max_n <= min_n + 1 :\n",
      "        return clusterer (in_array,max_n)\n",
      "    goc = np.zeros(1+max_n-min_n)\n",
      "    for nclu in range(min_n ,max_n +1):\n",
      "        algo = KMeans(n_clusters=nclu)\n",
      "        track_candidates = algo.fit_predict(col)\n",
      "        track_candidate_values = [np.array(in_array)[np.where(track_candidates==n)] for n in range(nclu)]\n",
      "        goc[nclu - min_n] = algo.score(col)\n",
      "    d_goc = []\n",
      "    for i in range (max_n-min_n):\n",
      "        d_goc.append(goc[i+1] - goc[i])\n",
      "    col = np.reshape(d_goc,(len(d_goc),1))\n",
      "    algo = KMeans(n_clusters=2)\n",
      "    gap_candidates = algo.fit_predict(col)\n",
      "    if gap_candidates[0] == 0:\n",
      "        accepted = list(gap_candidates).count(1)\n",
      "    else :\n",
      "        accepted = list(gap_candidates).count(0)\n",
      "    revised_number = max_n-accepted\n",
      "    return clusterer (in_array,revised_number)\n",
      "def similarity (exptrack,nntrack) :\n",
      "    sim_counter = 0\n",
      "    for exphit in range (len(exptrack)) :\n",
      "        for nnhit in range (len(nntrack)) :\n",
      "            if exptrack[exphit] == nntrack[nnhit] :\n",
      "                sim_counter += 1\n",
      "    return sim_counter\n",
      "def got (exp,nn,cut = 0.1,requested_efficiency = 1) : # Dear reader, get ready for awful variable names and no comment !\n",
      "    goodTrack = 0\n",
      "    # creating MidPut\n",
      "    for i in range (len(nn)) :\n",
      "        if nn[i] < cut :\n",
      "            nn[i] = 0\n",
      "    exp_Output = np.array([[i,exp[i]] for i in range (len(exp)) if exp[i] != 0],dt)\n",
      "    nn_Output = np.array([[i,nn[i]] for i in range (len(nn)) if nn[i] != 0],dt)\n",
      "    exp_Output.view('f32,f32').sort(order=['f1'], axis=0)\n",
      "    nn_Output.view('f32,f32').sort(order=['f1'], axis=0)\n",
      "    n_exp,exp_matrix = clustering (exp_Output[:,1])\n",
      "    n_nn,nn_matrix = clustering (nn_Output[:,1])\n",
      "    expM = np.array([[exp_Output[i][0] for i in range (exp_matrix[track][0],exp_matrix[track][1]+1)] for track in range (n_exp)])\n",
      "    nnM = np.array([[nn_Output[i][0] for i in range (nn_matrix[track][0],nn_matrix[track][1]+1)] for track in range (n_nn)])\n",
      "    similarities = np.zeros(n_exp,dt)\n",
      "    # doesn't work quite well for efficiencies less than 1,I know the reason,but don't have time to fix it !\n",
      "    good_hit = 0\n",
      "    total_hit = len(expM[1]) * n_exp\n",
      "    for i in range (-len(expM[1]),0):\n",
      "      remained_NNtracks = np.array([j for j in range (len(nnM))])\n",
      "      for exptrack in range (n_exp) :\n",
      "          for nntrack in remained_NNtracks :\n",
      "              if similarity(expM[exptrack],nnM[nntrack]) == -i :\n",
      "                  good_hit += -i\n",
      "                  if similarity(expM[exptrack],nnM[nntrack]) >= requested_efficiency * len(expM[1]) :\n",
      "                    goodTrack +=1\n",
      "                  remained_NNtracks = np.array([remained_NNtracks[j] for j in range (len(remained_NNtracks)) if j != nntrack]) #like !\n",
      "                  break\n",
      "    return float(goodTrack)/len(expM),good_hit,total_hit\n",
      "def max_finder (listt) :\n",
      "    m = 0\n",
      "    for i in range (len(listt)) :\n",
      "        if listt[i] > listt[m] :\n",
      "            m = i\n",
      "    return m\n",
      "\n",
      "# \"Sorting\" assembling algorithm\n",
      "def compare (out,learn,n = 4,m = 4,efficiency = 1) : # n is nlayers , m is ntracks\n",
      "    counter = 0\n",
      "    output = np.zeros(len(out) , dt)\n",
      "    learned = np.zeros(len(learn) , dt)\n",
      "    Otracks = np.zeros((m,n),dt)\n",
      "    Ltracks = np.zeros((m,n),dt)\n",
      "    for i in range (len(out)) :\n",
      "        output[i] = out[i]\n",
      "    for i in range (len(learn)) :\n",
      "        learned[i] = learn[i]\n",
      "    for track in range (m) :\n",
      "        for layer in range (n) :\n",
      "            start = int(float(layer) * len(output) / n)\n",
      "            end = int(float(layer+1) * len(output) / n)\n",
      "            subOutput = output[start:end]\n",
      "            subLearned = learned[start:end]\n",
      "            Otracks[track][layer] = max_finder(subOutput)+start\n",
      "            Ltracks[track][layer] = max_finder(subLearned)+start\n",
      "            output[max_finder(subOutput)+start] = 0\n",
      "            learned[max_finder(subLearned)+start] = 0\n",
      "    for track in range (m) :\n",
      "        if max([similarity(Otracks[track],Ltracks[i]) for i in range(m)]) >= efficiency * n :\n",
      "            counter += 1\n",
      "    return counter\n",
      "\n",
      "###################################################################################################################################\n",
      "# Section 4: Loading batches of data from two numpy files for inputs and outputs\n",
      "\n",
      "def train_batch(): # returns a random batch from training data\n",
      "    X = np.load(Input_File, mmap_mode='r')[:nsamples]\n",
      "    Y = np.load(Output_File, mmap_mode='r')[:nsamples]\n",
      "    if cut == BATCH_SIZE:\n",
      "        i = 0\n",
      "    else:\n",
      "        i = np.random.randint(cut-BATCH_SIZE)\n",
      "    return (X[i:i+BATCH_SIZE],Y[i:i+BATCH_SIZE])\n",
      "def valid_batch(batchSize = BATCH_SIZE): # returns a random batch from validation data \n",
      "    X = np.load(Input_File, mmap_mode='r')[:nsamples]\n",
      "    Y = np.load(Output_File, mmap_mode='r')[:nsamples]\n",
      "    if cut == len(X)-batchSize:\n",
      "        i = cut\n",
      "    else:\n",
      "        i = np.random.randint(cut,len(X)-batchSize)\n",
      "    return (X[i:i+batchSize],Y[i:i+batchSize])\n",
      "\n",
      "#####################################################################################################################################\n",
      "# Section 5: Training the neural net\n",
      "\n",
      "trainer.load(Network_File)\n",
      "\n",
      "start = time.mktime(time.gmtime())\n",
      "print \"training started ... \"\n",
      "epoch_counter = 0\n",
      "for trn, val in trainer.itertrain(train_batch,valid_batch(BATCH_SIZE), algo='rprop',learning_rate = LR,\n",
      "                                  batch_size=BATCH_SIZE,validate_every = 1,patience=100000,min_improvement = 0.000001) :\n",
      "    trainer.save(Network_File)\n",
      "    if epoch_counter%1 == 0 :\n",
      "        print \"epoch \",epoch_counter+1,\"\\ttraining_error =\",1000*trn['err'],\" \\tvalidation_error =\",1000*val['err']\n",
      "    epoch_counter += 1\n",
      "    #if epoch_counter == 400 :\n",
      "    #  LR = 1e-3\n",
      "    #if epoch_counter == 1000 :\n",
      "    #  LR = 1e-4\n",
      "    #if epoch_counter == 2000 :\n",
      "    #  break\n",
      "print \"\\t training ended\"\n",
      "stop = time.mktime (time.gmtime())\n",
      "spent.append (stop - start)\n",
      "print \"training time (seconds) \\t\" , stop - start\n",
      "print spent\n",
      "\n",
      "trainer.save(Network_File)\n",
      "\n",
      "####################################################################################################################################\n",
      "# Section 6: Validating results\n",
      "\n",
      "A_Validation_Batch = valid_batch(Validation_Batch_Size)\n",
      "Predict = trainer.network.predict(A_Validation_Batch[0])\n",
      "\n",
      "# Performance based on Clustering assembling method for track requested efficiency = 1 and 0.75\n",
      "gots = np.zeros(5 , dt)\n",
      "start = time.mktime(time.gmtime())\n",
      "goodHit = 0\n",
      "totalHit = 0\n",
      "for i in range (Validation_Batch_Size) :\n",
      "    ratio_goodTracks,good_hit,total_hit = got(A_Validation_Batch[1][i],Predict[i],0.1,1)\n",
      "    goodHit += good_hit\n",
      "    totalHit += total_hit\n",
      "    if ratio_goodTracks == 1 :\n",
      "        print i\n",
      "    gots[int(4*ratio_goodTracks)] += 1\n",
      "print \"goh = \",float(goodHit)/totalHit\n",
      "stop = time.mktime (time.gmtime())\n",
      "spent.append (stop - start)\n",
      "print \"\\t\" , stop - start\n",
      "print \"gots :\\t\",gots\n",
      "print sum([i*gots[i] for i in range (5)])/(Validation_Batch_Size*0.04)\n",
      "\n",
      "gots = np.zeros(5 , dt)\n",
      "start = time.mktime(time.gmtime())\n",
      "goodHit = 0\n",
      "totalHit = 0\n",
      "for i in range (Validation_Batch_Size) :\n",
      "    ratio_goodTracks,good_hit,total_hit = got(A_Validation_Batch[1][i],Predict[i],0.1,0.75)\n",
      "    goodHit += good_hit\n",
      "    totalHit += total_hit\n",
      "    gots[int(4*ratio_goodTracks)] += 1\n",
      "print \"goh = \",float(goodHit)/totalHit\n",
      "stop = time.mktime (time.gmtime())\n",
      "spent.append (stop - start)\n",
      "print \"\\t\" , stop - start\n",
      "print \"gots :\\t\",gots\n",
      "print sum([i*gots[i] for i in range (5)])/(Validation_Batch_Size*0.04)\n",
      "\n",
      "# Performance based on Sorting assembling method for track requested efficiency = 1 and 0.75\n",
      "counter = np.zeros(5 , dt)\n",
      "start = time.mktime(time.gmtime())\n",
      "for i in range (Validation_Batch_Size) :\n",
      "    n_good_tracks = compare(A_Validation_Batch[1][i],Predict[i],4,4,1)\n",
      "    counter[n_good_tracks] += 1\n",
      "stop = time.mktime (time.gmtime())\n",
      "spent.append (stop - start)\n",
      "print \"\\t\" , stop - start\n",
      "print \"A-error: \",counter\n",
      "print sum([i*counter[i] for i in range (5)])/(Validation_Batch_Size*0.04)\n",
      "\n",
      "counter = np.zeros(5 , dt)\n",
      "start = time.mktime(time.gmtime())\n",
      "for i in range (Validation_Batch_Size) :\n",
      "    n_good_tracks = compare(A_Validation_Batch[1][i],Predict[i],4,4,0.75)\n",
      "    counter[n_good_tracks] += 1\n",
      "stop = time.mktime (time.gmtime())\n",
      "spent.append (stop - start)\n",
      "print \"\\t\" , stop - start\n",
      "print \"A-error: \",counter\n",
      "print sum([i*counter[i] for i in range (5)])/(Validation_Batch_Size*0.04)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}