Specialist_layer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 2 3 4 5]\n",
      " [0 1 2 4 5 3]\n",
      " [0 1 2 5 4 3]\n",
      " [0 1 2 3 5 4]\n",
      " [0 1 2 4 3 5]\n",
      " [0 1 2 5 3 4]]\n"
     ]
    }
   ],
   "source": [
    "from keras.objectives import categorical_crossentropy\n",
    "import theano.tensor as T\n",
    "import theano\n",
    "import numpy as np\n",
    "#categorical_crossentropy??\n",
    "#Loss:\n",
    "\n",
    "\n",
    "perm =[[0,1,2],[1,2,0],[2,1,0],[0,2,1],[1,0,2],[2,0,1]]\n",
    "perm = [[-3,-2,-1]+iperm for iperm in perm]\n",
    "perm = np.array(perm,dtype=np.int)\n",
    "perm += 3\n",
    "#print perm\n",
    "\n",
    "test_true = [[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]]]\n",
    "\n",
    "eps =1e-7\n",
    "test_pred = [[[1-eps,+eps],[1-eps,eps],[1-eps,eps]],[[eps,1-eps],[eps,1-eps],[eps,1-eps]],\n",
    "             [[eps,1-eps],[eps,1-eps],[eps,1-eps]]]\n",
    "\n",
    "\n",
    "def perm_loss(y_true,y_pred):\n",
    "    def loss(m,  y_true, y_pred,perm):\n",
    "\n",
    "        #return  perm[T.cast(m,\"int32\")]\n",
    "        return T.mean( T.sum(y_true[::,::,perm[m]] * T.log(y_pred),axis=-1),axis=-1)\n",
    "\n",
    "    #perm = np.array([[0,1],[1,0]],dtype=np.int)\n",
    "    perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                     [0, 1, 2, 4, 5, 3, 6],\n",
    "                     [0, 1, 2, 5, 4, 3, 6],\n",
    "                     [0, 1, 2, 3, 5, 4, 6],\n",
    "                     [0, 1, 2, 4, 3, 5, 6],\n",
    "                     [0, 1, 2, 5, 3, 4, 6]],dtype=np.int)\n",
    "    \n",
    "    \"\"\"perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                     [0, 1, 2, 3, 4, 5, 6]],dtype=np.int)\"\"\"\n",
    "    seq = T.arange(len(perm))\n",
    "    result, _ = theano.scan(fn=loss, outputs_info=None, \n",
    "                             sequences=seq, non_sequences=[y_true, y_pred,perm])\n",
    "    return -T.mean(T.max(result,axis=0)) #T.max(result.dimshuffle(1,2,0),axis=-1)\n",
    "\n",
    "#r =perm_loss(test_true,test_pred).eval()\n",
    "#print r\n",
    "#print r.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "def reverse(X):\n",
    "    return X[::,::,::-1]\n",
    "\n",
    "def output_shape(input_shape):\n",
    "    # here input_shape includes the samples dimension\n",
    "    return input_shape  # shap\n",
    "\n",
    "\n",
    "def sub_mean(X):\n",
    "    xdms = X.shape\n",
    "    return X.reshape(xdms[0])\n",
    "\n",
    "def old_version(ndim=2):\n",
    "\n",
    "    #middle = 50\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(200,5))\n",
    "    #graph.add_input(name='input2', input_shape=(None,2))\n",
    "\n",
    "    #nbr_filter = 10\n",
    "    graph.add_node(Convolution1D(nb_filter=5,filter_length=4,input_shape=(None,5),\n",
    "                                 border_mode=\"same\"),input='input1',name=\"conv1\")\n",
    "\n",
    "\n",
    "    graph.add_node(MaxPooling1D(pool_length=2),\n",
    "                   input='conv1',name=\"max1\")\n",
    "\n",
    "    graph.add_node(UpSampling1D(length=2),\n",
    "                   input='max1',name=\"input1b\")\n",
    "\n",
    "\n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=20, activation='sigmoid',input_shape=(200,10),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"allmost\",inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\")\n",
    "    graph.add_node(Lambda(reverse, output_shape),inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\",\n",
    "                   name=\"reversed0\")\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=20, activation='sigmoid',\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"allmost1\",input=\"reversed0\")\n",
    "\n",
    "\n",
    "    graph.add_node(Lambda(reverse, output_shape),input=\"allmost1\",name=\"reversed\")\n",
    "\n",
    "\n",
    "\n",
    "    #Here get the subcategory\n",
    "    graph.add_node(TimeDistributedDense(7,activation=\"softmax\"),inputs=[\"allmost\",\"reversed\"],\n",
    "                   name=\"output0\",merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    ##########################################\n",
    "    #First ehd here\n",
    "    #graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.compile('adadelta', {'output':'categorical_crossentropy' })\n",
    "    ################################################\n",
    "\n",
    "    #Here get the number of category\n",
    "    graph.add_node(LSTM(output_dim=27, activation='softmax',\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,27)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category00\")\n",
    "\n",
    "    graph.compile('adadelta', {'output':'categorical_crossentropy',\n",
    "                              'category':'categorical_crossentropy' })\n",
    "    \n",
    "    return graph\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "\n",
    "def reverse(X):\n",
    "    return X[::,::,::-1]\n",
    "\n",
    "def output_shape(input_shape):\n",
    "    # here input_shape includes the samples dimension\n",
    "    return input_shape  # shap\n",
    "\n",
    "\n",
    "def sub_mean(X):\n",
    "    xdms = X.shape\n",
    "    return X.reshape(xdms[0])\n",
    "\n",
    "def old_but_ok(ndim=2):\n",
    "#middle = 50\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(200,5))\n",
    "    #graph.add_input(name='input2', input_shape=(None,2))\n",
    "\n",
    "    #nbr_filter = 10\n",
    "\n",
    "    graph.add_node(Convolution1D(nb_filter=10,filter_length=4,input_shape=(None,5),\n",
    "                                 border_mode=\"same\"),input='input1',name=\"conv1\")\n",
    "\n",
    "\n",
    "    graph.add_node(MaxPooling1D(pool_length=2),\n",
    "                   input='conv1',name=\"max1\")\n",
    "\n",
    "    graph.add_node(UpSampling1D(length=2),\n",
    "                   input='max1',name=\"input1b\")\n",
    "\n",
    "\n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "    #First with 20 of activation\n",
    "\n",
    "    inside=50\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=inside, activation='sigmoid',input_shape=(200,15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"1allmost\",inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\")\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(Lambda(reverse, output_shape),inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\",\n",
    "                   name=\"reversed0\")\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=inside, activation='sigmoid',\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"allmost1\",input=\"reversed0\")\n",
    "\n",
    "    graph.add_node(Lambda(reverse, output_shape),input=\"allmost1\",name=\"reversed\")\n",
    "\n",
    "    #END first\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=inside, activation='sigmoid',input_shape=(200,2*inside+15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"allmost_l2\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"1allmost\",\"reversed\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    graph.add_node(Lambda(reverse, output_shape),inputs=[\"input1\",\"input1b\",\"1allmost\",\"reversed\"],merge_mode=\"concat\",\n",
    "                   concat_axis=-1,\n",
    "                   name=\"reversed0_l2\")\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=inside, activation='sigmoid',input_shape=(200,2*inside+15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"allmost1_l2\",input=\"reversed0_l2\")\n",
    "\n",
    "    graph.add_node(Lambda(reverse, output_shape),input=\"allmost1_l2\",name=\"reversed_l2\")\n",
    "\n",
    "    #END second\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"1allmost\",\"reversed\",\"allmost_l2\",\"reversed_l2\"],\n",
    "                   merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(7,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ##########################################\n",
    "    #First ehd here\n",
    "    #graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.compile('adadelta', {'output':'categorical_crossentropy' })\n",
    "    ################################################\n",
    "\n",
    "    #Here get the number of category\n",
    "    graph.add_node(LSTM(output_dim=12,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0_r\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(LSTM(output_dim=12,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False,go_backwards=True),\n",
    "                       name=\"category0_l\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(12,activation=\"softmax\"),inputs=[\"category0_l\",\"category0_r\"],concat_axis=1,merge_mode=\"concat\",\n",
    "                   name=\"category0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,12)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "    #graph.load_weights(\"step_check\")\n",
    "    #############################################\n",
    "    #Original end there\n",
    "    #graph.load_weights(\"step_check\")\n",
    "\n",
    "    #graph.add_output(name=\"category\",input=\"category0\")\n",
    "    #graph.compile('adadelta', {'output':'categorical_crossentropy'})\n",
    "\n",
    "    #############################################\n",
    "\n",
    "    #graph.add_node(TimeDistributedDense(1,activation=\"linear\"),input='output0',name=\"output1\")\n",
    "\n",
    "\n",
    "    #graph.load_weights(\"step_check_bigger\")\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category00\")\n",
    "\n",
    "    graph.compile('adadelta', {'output':perm_loss,\n",
    "                              'category':'categorical_crossentropy'})\n",
    "\n",
    "\n",
    "    graph.load_weights(\"old_weights/specialist_4_diff_size_50\")\n",
    "    \n",
    "    return graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using gpu device 0: GeForce GTX 660 (CNMeM is disabled, CuDNN not available)\n",
      "/usr/local/lib/python2.7/dist-packages/theano/tensor/signal/downsample.py:5: UserWarning: downsample module has been moved to the pool module.\n",
      "  warnings.warn(\"downsample module has been moved to the pool module.\")\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "from Bilayer import BiLSTM\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "def return_two_layer():\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "\n",
    "    #middle = 50\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(200,5))\n",
    "    #graph.add_input(name='input2', input_shape=(None,2))\n",
    "\n",
    "    #nbr_filter = 10\n",
    "\n",
    "    graph.add_node(Convolution1D(nb_filter=10,filter_length=4,input_shape=(None,5),\n",
    "                                 border_mode=\"same\"),input='input1',name=\"conv1\")\n",
    "\n",
    "\n",
    "    graph.add_node(MaxPooling1D(pool_length=2),\n",
    "                   input='conv1',name=\"max1\")\n",
    "\n",
    "    graph.add_node(UpSampling1D(length=2),\n",
    "                   input='max1',name=\"input1b\")\n",
    "\n",
    "\n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "    #First with 20 of activation\n",
    "\n",
    "    inside=50\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"l1\",inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"l2\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l1\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l1\",\"l2\"],\n",
    "                   merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(7,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ##########################################\n",
    "    #First ehd here\n",
    "    #graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.compile('adadelta', {'output':'categorical_crossentropy' })\n",
    "    ################################################\n",
    "\n",
    "    #Here get the number of category\n",
    "    graph.add_node(BiLSTM(output_dim=12,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0bi\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(12,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,12)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "    #graph.load_weights(\"step_check\")\n",
    "    #############################################\n",
    "    #Original end there\n",
    "    #graph.load_weights(\"step_check\")\n",
    "\n",
    "    #graph.add_output(name=\"category\",input=\"category0\")\n",
    "    #graph.compile('adadelta', {'output':'categorical_crossentropy'})\n",
    "\n",
    "    #############################################\n",
    "\n",
    "    #graph.add_node(TimeDistributedDense(1,activation=\"linear\"),input='output0',name=\"output1\")\n",
    "\n",
    "\n",
    "    #graph.load_weights(\"step_check_bigger\")\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category00\")\n",
    "\n",
    "    graph.compile('adadelta', {'output':perm_loss,\n",
    "                              'category':'categorical_crossentropy'})\n",
    "\n",
    "    graph.load_weights(\"saved_weights/two_bilayer_without_sub\")\n",
    "    return graph\n",
    "    #graph.load_weights(\"training_general_scale10\")\n",
    "    #############################################\n",
    "    #Second end there\n",
    "\n",
    "\n",
    "#############################################\n",
    "\n",
    "\n",
    "\n",
    "#history = graph.fit({'input1':X_train[::,1], 'input2':X2_train[::0], 'output':y_train}, nb_epoch=10)\n",
    "#predictions = graph.predict({'input1':X_test, 'input2':X2_test}) # {'output':...}\n",
    "#graph.save_weights(\"step_check\",overwrite=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "from Bilayer import BiLSTM\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "def return_three_layer():\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "\n",
    "    #middle = 50\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(200,5))\n",
    "    #graph.add_input(name='input2', input_shape=(None,2))\n",
    "\n",
    "    #nbr_filter = 10\n",
    "\n",
    "    graph.add_node(Convolution1D(nb_filter=10,filter_length=4,input_shape=(None,5),\n",
    "                                 border_mode=\"same\"),input='input1',name=\"conv1\")\n",
    "\n",
    "\n",
    "    graph.add_node(MaxPooling1D(pool_length=2),\n",
    "                   input='conv1',name=\"max1\")\n",
    "\n",
    "    graph.add_node(UpSampling1D(length=2),\n",
    "                   input='max1',name=\"input1b\")\n",
    "\n",
    "\n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "    #First with 20 of activation\n",
    "\n",
    "    inside=50\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"l1\",inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"l2\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l1\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+15),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"l3\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l2\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l1\",\"l2\",\"l3\"],\n",
    "                   merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(10,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=27,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0bi\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(27,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,27)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category0\")\n",
    "\n",
    "    graph.compile('adadelta', {'output':perm_loss,\n",
    "                              'category':'categorical_crossentropy'})\n",
    "    \n",
    "    graph.load_weights(\"three_layer_specialist\")\n",
    "    return graph\n",
    "\n",
    "#graph.load_weights(\"training_general_scale10\")\n",
    "#############################################\n",
    "#Second end there\n",
    "\n",
    "\n",
    "#############################################\n",
    "\n",
    "\n",
    "\n",
    "#history = graph.fit({'input1':X_train[::,1], 'input2':X2_train[::0], 'output':y_train}, nb_epoch=10)\n",
    "#predictions = graph.predict({'input1':X_test, 'input2':X2_test}) # {'output':...}\n",
    "#graph.save_weights(\"step_check\",overwrite=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "#print theano.__version__ , theano.__file__\n",
    "import keras\n",
    "#print keras.__version__, keras.__file__\n",
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "from Bilayer import BiLSTM\n",
    "    \n",
    "import theano.tensor as T\n",
    "import theano\n",
    "from keras.backend.common import _EPSILON\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def return_three_bis(ndim=2,inside=50):\n",
    "\n",
    "    #categorical_crossentropy??\n",
    "    #Loss:\n",
    "\n",
    "\n",
    "    perm =[[0,1,2],[1,2,0],[2,1,0],[0,2,1],[1,0,2],[2,0,1]]\n",
    "    perm = [[-3,-2,-1]+iperm for iperm in perm]\n",
    "    perm = np.array(perm,dtype=np.int)\n",
    "    perm += 3\n",
    "\n",
    "    test_true = [[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]]]\n",
    "\n",
    "    eps =1e-7\n",
    "    test_pred = [[[1-eps,+eps],[1-eps,eps],[1-eps,eps]],[[eps,1-eps],[eps,1-eps],[eps,1-eps]],\n",
    "                 [[eps,1-eps],[eps,1-eps],[eps,1-eps]]]\n",
    "\n",
    "\n",
    "    def perm_loss(y_true,y_pred):\n",
    "        def loss(m,  y_true, y_pred,perm):\n",
    "\n",
    "            #return  perm[T.cast(m,\"int32\")]\n",
    "            y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON)\n",
    "            return T.mean( T.sum(y_true[::,::,perm[m]] * T.log(y_pred),axis=-1),axis=-1)\n",
    "\n",
    "        #perm = np.array([[0,1],[1,0]],dtype=np.int)\n",
    "        perm = np.array([[0, 1, 2, 3, 4, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 5, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 4, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 3, 5, 4, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 3, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 3, 4, 6] + range(7,10)],dtype=np.int)\n",
    "\n",
    "        \"\"\"perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                         [0, 1, 2, 3, 4, 5, 6]],dtype=np.int)\"\"\"\n",
    "        seq = T.arange(len(perm))\n",
    "        result, _ = theano.scan(fn=loss, outputs_info=None, \n",
    "        sequences=seq, non_sequences=[y_true, y_pred,perm])\n",
    "        return -T.mean(T.max(result,axis=0)) #T.max(result.dimshuffle(1,2,0),axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "    \n",
    "    #middle = 50\n",
    "    add = 0\n",
    "    if ndim == 3:\n",
    "        add = 1\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(200,5+add))\n",
    "    #graph.add_input(name='input2', input_shape=(None,2))\n",
    "\n",
    "    #nbr_filter = 10\n",
    "\n",
    "    graph.add_node(Convolution1D(nb_filter=10,filter_length=4,input_shape=(None,5+add),\n",
    "                                 border_mode=\"same\"),input='input1',name=\"conv1\")\n",
    "\n",
    "\n",
    "    graph.add_node(MaxPooling1D(pool_length=2),\n",
    "                   input='conv1',name=\"max1\")\n",
    "\n",
    "    graph.add_node(UpSampling1D(length=2),\n",
    "                   input='max1',name=\"input1b\")\n",
    "\n",
    "\n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "    #First with 20 of activation\n",
    "\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,15+add),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"l1\",inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+15+add),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True,),name=\"l2\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l1\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+15+add),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"l3\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l2\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l1\",\"l2\",\"l3\"],merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(10,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(4,activation=\"softmax\"),input=\"output0\",\n",
    "                   name=\"output0b\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=27,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0bi\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(27,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,27)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    graph.add_output(name=\"outputtype\",input=\"output0b\")\n",
    "\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category0\")\n",
    "\n",
    "    graph.compile('adadelta', {'output':perm_loss,\n",
    "                              'category':'categorical_crossentropy',\n",
    "                              'outputtype':'categorical_crossentropy'})\n",
    "\n",
    "    #graph.load_weights(\"training_general_scale10\")\n",
    "    #############################################\n",
    "    #Second end there\n",
    "    \n",
    "    if ndim == 2 and inside == 50:\n",
    "        graph.load_weights(\"saved_weights/three_bilayer_sub_bis\")\n",
    "        \n",
    "    if ndim == 3 and inside == 50:\n",
    "        graph.load_weights(\"saved_weights/three_bilayer_sub_bis_3D_isotrope\")\n",
    "\n",
    "    #############################################\n",
    "\n",
    "    return graph\n",
    "\n",
    "    #history = graph.fit({'input1':X_train[::,1], 'input2':X2_train[::0], 'output':y_train}, nb_epoch=10)\n",
    "    #predictions = graph.predict({'input1':X_test, 'input2':X2_test}) # {'output':...}\n",
    "    #graph.save_weights(\"step_check\",overwrite=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def return_three_bis_three_level(ndim=2,inside=50):\n",
    "\n",
    "    #categorical_crossentropy??\n",
    "    #Loss:\n",
    "\n",
    "\n",
    "    perm =[[0,1,2],[1,2,0],[2,1,0],[0,2,1],[1,0,2],[2,0,1]]\n",
    "    perm = [[-3,-2,-1]+iperm for iperm in perm]\n",
    "    perm = np.array(perm,dtype=np.int)\n",
    "    perm += 3\n",
    "\n",
    "    test_true = [[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]]]\n",
    "\n",
    "    eps =1e-7\n",
    "    test_pred = [[[1-eps,+eps],[1-eps,eps],[1-eps,eps]],[[eps,1-eps],[eps,1-eps],[eps,1-eps]],\n",
    "                 [[eps,1-eps],[eps,1-eps],[eps,1-eps]]]\n",
    "\n",
    "\n",
    "    def perm_loss(y_true,y_pred):\n",
    "        def loss(m,  y_true, y_pred,perm):\n",
    "\n",
    "            #return  perm[T.cast(m,\"int32\")]\n",
    "            y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON)\n",
    "            return T.mean( T.sum(y_true[::,::,perm[m]] * T.log(y_pred),axis=-1),axis=-1)\n",
    "\n",
    "        #perm = np.array([[0,1],[1,0]],dtype=np.int)\n",
    "        perm = np.array([[0, 1, 2, 3, 4, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 5, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 4, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 3, 5, 4, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 3, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 3, 4, 6] + range(7,10)],dtype=np.int)\n",
    "\n",
    "        \"\"\"perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                         [0, 1, 2, 3, 4, 5, 6]],dtype=np.int)\"\"\"\n",
    "        seq = T.arange(len(perm))\n",
    "        result, _ = theano.scan(fn=loss, outputs_info=None, \n",
    "        sequences=seq, non_sequences=[y_true, y_pred,perm])\n",
    "        return -T.mean(T.max(result,axis=0)) #T.max(result.dimshuffle(1,2,0),axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "    \n",
    "    #middle = 50\n",
    "    add = 0\n",
    "    if ndim == 3:\n",
    "        add = 1\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(200,3*(5+add)))\n",
    "    #graph.add_input(name='input2', input_shape=(None,2))\n",
    "\n",
    "    #nbr_filter = 10\n",
    "\n",
    "    graph.add_node(Convolution1D(nb_filter=10,filter_length=4,input_shape=(None,3*(5+add)),\n",
    "                                 border_mode=\"same\"),input='input1',name=\"conv1\")\n",
    "\n",
    "\n",
    "    graph.add_node(MaxPooling1D(pool_length=2),\n",
    "                   input='conv1',name=\"max1\")\n",
    "\n",
    "    graph.add_node(UpSampling1D(length=2),\n",
    "                   input='max1',name=\"input1b\")\n",
    "\n",
    "\n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "    #First with 20 of activation\n",
    "\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,10+3*(5+add)),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"l1\",inputs=[\"input1\",\"input1b\"],concat_axis=-1,merge_mode=\"concat\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+10+3*(5+add)),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True,),name=\"l2\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l1\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',input_shape=(200,inside+10+3*(5+add)),\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"l3\",\n",
    "                       inputs=[\"input1\",\"input1b\",\"l2\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l1\",\"l2\",\"l3\"],merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(10,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(4,activation=\"softmax\"),input=\"output0\",\n",
    "                   name=\"output0b\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=27,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0bi\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(27,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,27)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    graph.add_output(name=\"outputtype\",input=\"output0b\")\n",
    "\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category0\")\n",
    "\n",
    "    graph.compile('adadelta', {'output':perm_loss,\n",
    "                              'category':'categorical_crossentropy',\n",
    "                              'outputtype':'categorical_crossentropy'})\n",
    "\n",
    "    #graph.load_weights(\"training_general_scale10\")\n",
    "    #############################################\n",
    "    #Second end there\n",
    "    \n",
    "   \n",
    "    #############################################\n",
    "\n",
    "    return graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "#print theano.__version__ , theano.__file__\n",
    "import keras\n",
    "#print keras.__version__, keras.__file__\n",
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "\n",
    "if  int(keras.__version__.split(\".\")[0]) >= 1.0 :\n",
    "    from Bilayer import BiLSTMv1 as BiLSTM\n",
    "else:\n",
    "    from Bilayer import BiLSTM, BiSimpleRNN    \n",
    "    \n",
    "import theano.tensor as T\n",
    "import theano\n",
    "from keras.backend.common import _EPSILON\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def return_three_bis_simpler(ndim=2,permute=True,extend=0):\n",
    "\n",
    "    #categorical_crossentropy??\n",
    "    #Loss:\n",
    "\n",
    "\n",
    "    perm =[[0,1,2],[1,2,0],[2,1,0],[0,2,1],[1,0,2],[2,0,1]]\n",
    "    perm = [[-3,-2,-1]+iperm for iperm in perm]\n",
    "    perm = np.array(perm,dtype=np.int)\n",
    "    perm += 3\n",
    "\n",
    "    test_true = [[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]]]\n",
    "\n",
    "    eps =1e-7\n",
    "    test_pred = [[[1-eps,+eps],[1-eps,eps],[1-eps,eps]],[[eps,1-eps],[eps,1-eps],[eps,1-eps]],\n",
    "                 [[eps,1-eps],[eps,1-eps],[eps,1-eps]]]\n",
    "\n",
    "\n",
    "    def perm_loss(y_true,y_pred):\n",
    "        def loss(m,  y_true, y_pred,perm):\n",
    "\n",
    "            #return  perm[T.cast(m,\"int32\")]\n",
    "            y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON)\n",
    "            return T.mean( T.sum(y_true[::,::,perm[m]] * T.log(y_pred),axis=-1),axis=-1)\n",
    "\n",
    "        #perm = np.array([[0,1],[1,0]],dtype=np.int)\n",
    "        perm = np.array([[0, 1, 2, 3, 4, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 5, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 4, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 3, 5, 4, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 3, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 3, 4, 6] + range(7,10)],dtype=np.int)\n",
    "\n",
    "        \"\"\"perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                         [0, 1, 2, 3, 4, 5, 6]],dtype=np.int)\"\"\"\n",
    "        seq = T.arange(len(perm))\n",
    "        result, _ = theano.scan(fn=loss, outputs_info=None, \n",
    "        sequences=seq, non_sequences=[y_true, y_pred,perm])\n",
    "        return -T.mean(T.max(result,axis=0)) #T.max(result.dimshuffle(1,2,0),axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "    \n",
    "    #middle = 50\n",
    "    add = 0\n",
    "    if ndim == 3:\n",
    "        add = 1\n",
    "   \n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(None,5+add))\n",
    " \n",
    "\n",
    "    #graph.add_node(Convolution1D(nb_filter=4,filter_length=3,input_shape=(None,2),\n",
    "    #                             border_mode=\"same\"),input='input1',name=\"output0\")\n",
    "\n",
    "    #66,4\n",
    "\n",
    "\n",
    "    #First with 20 of activation\n",
    "\n",
    "    inside=50\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),\n",
    "                       name=\"l1\",input=\"input1\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True,),name=\"l2\",\n",
    "                       inputs=[\"input1\",\"l1\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=inside, activation='tanh',\n",
    "                        inner_activation='hard_sigmoid',return_sequences=True),name=\"l3\",\n",
    "                       inputs=[\"input1\",\"l2\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l1\",\"l2\",\"l3\"],merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(10 + extend,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "    if permute:\n",
    "        graph.add_node(TimeDistributedDense(4,activation=\"softmax\"),input=\"output0\",\n",
    "                       name=\"output0b\")\n",
    "\n",
    "    graph.add_node(BiLSTM(output_dim=27,\n",
    "                        inner_activation='hard_sigmoid',return_sequences=False),\n",
    "                       name=\"category0bi\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(27,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "\n",
    "    graph.add_node(Reshape((1,27)),input=\"category0\",name = \"category00\")\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "    if permute:\n",
    "        graph.add_output(name=\"outputtype\",input=\"output0b\")\n",
    "\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category0\")\n",
    "\n",
    "    if permute:\n",
    "        graph.compile('adadelta', {'output':perm_loss,\n",
    "                              'category':'categorical_crossentropy',\n",
    "                              'outputtype':'categorical_crossentropy'})\n",
    "    else:\n",
    "        graph.compile('adadelta', {'output':\"categorical_crossentropy\",\n",
    "                              'category':'categorical_crossentropy'})\n",
    "\n",
    "    #graph.load_weights(\"training_general_scale10\")\n",
    "    #############################################\n",
    "    #Second end there\n",
    "   \n",
    "\n",
    "    #############################################\n",
    "\n",
    "    return graph\n",
    "\n",
    "    #history = graph.fit({'input1':X_train[::,1], 'input2':X2_train[::0], 'output':y_train}, nb_epoch=10)\n",
    "    #predictions = graph.predict({'input1':X_test, 'input2':X2_test}) # {'output':...}\n",
    "    #graph.save_weights(\"step_check\",overwrite=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "#print theano.__version__ , theano.__file__\n",
    "import keras\n",
    "#print keras.__version__, keras.__file__\n",
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "if  int(keras.__version__.split(\".\")[0]) >= 1.0 :\n",
    "    from Bilayer import BiLSTMv1 as BiLSTM\n",
    "else:\n",
    "    from Bilayer import BiLSTM, BiSimpleRNN    \n",
    "import theano.tensor as T\n",
    "import theano\n",
    "from keras.backend.common import _EPSILON\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def return_three_paper(ndim=2,inside=50,permutation=True,inputsize=5,simple=False):\n",
    "\n",
    "    #categorical_crossentropy??\n",
    "    #Loss:\n",
    "\n",
    "\n",
    "    perm =[[0,1,2],[1,2,0],[2,1,0],[0,2,1],[1,0,2],[2,0,1]]\n",
    "    perm = [[-3,-2,-1]+iperm for iperm in perm]\n",
    "    perm = np.array(perm,dtype=np.int)\n",
    "    perm += 3\n",
    "\n",
    "    test_true = [[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]]]\n",
    "\n",
    "    eps =1e-7\n",
    "    test_pred = [[[1-eps,+eps],[1-eps,eps],[1-eps,eps]],[[eps,1-eps],[eps,1-eps],[eps,1-eps]],\n",
    "                 [[eps,1-eps],[eps,1-eps],[eps,1-eps]]]\n",
    "\n",
    "\n",
    "    def perm_loss(y_true,y_pred):\n",
    "        def loss(m,  y_true, y_pred,perm):\n",
    "\n",
    "            #return  perm[T.cast(m,\"int32\")]\n",
    "            y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON)\n",
    "            return T.mean( T.sum(y_true[::,::,perm[m]] * T.log(y_pred),axis=-1),axis=-1)\n",
    "\n",
    "        #perm = np.array([[0,1],[1,0]],dtype=np.int)\n",
    "        perm = np.array([[0, 1, 2, 3, 4, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 5, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 4, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 3, 5, 4, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 3, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 3, 4, 6] + range(7,10)],dtype=np.int)\n",
    "\n",
    "        \"\"\"perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                         [0, 1, 2, 3, 4, 5, 6]],dtype=np.int)\"\"\"\n",
    "        seq = T.arange(len(perm))\n",
    "        result, _ = theano.scan(fn=loss, outputs_info=None, \n",
    "        sequences=seq, non_sequences=[y_true, y_pred,perm])\n",
    "        return -T.mean(T.max(result,axis=0)) #T.max(result.dimshuffle(1,2,0),axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "    \n",
    "    #middle = 50\n",
    "    add = 0\n",
    "    \n",
    "    if ndim == 3:\n",
    "        add = 1\n",
    "        \n",
    "    if simple:\n",
    "        Bi = BiSimpleRNN\n",
    "        \n",
    "    else:\n",
    "        Bi = BiLSTM\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(None,inputsize))\n",
    " \n",
    "\n",
    "    graph.add_node(Bi(output_dim=inside,activation='tanh',return_sequences=True,close=True,input_shape=(200,inputsize),),\n",
    "                       name=\"l1\",input=\"input1\")\n",
    "\n",
    "    graph.add_node(Bi(output_dim=inside,input_shape=(200,inputsize),\n",
    "                       return_sequences=True,close=True,activation='tanh'),name=\"l2\",\n",
    "                       inputs=[\"input1\",\"l1\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "    graph.add_node(Bi(output_dim=inside,activation='tanh', input_shape=(200,inputsize),\n",
    "                        return_sequences=True,close=True),name=\"l3\",\n",
    "                       inputs=[\"input1\",\"l2\"],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l1\",\"l2\",\"l3\"],merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(10,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "\n",
    "    graph.add_node(Bi(output_dim=27,activation='tanh',return_sequences=False,close=True),\n",
    "                       name=\"category0bi\",input=\"output0\")\n",
    "\n",
    "    graph.add_node(Dense(27,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "\n",
    "\n",
    "    graph.add_output(name=\"output\",input=\"output0\")\n",
    "\n",
    "    #graph.add_output(name=\"rOutput\",input=\"output1\")\n",
    "    graph.add_output(name=\"category\",input=\"category0\")\n",
    "\n",
    "    if permutation:\n",
    "        graph.compile('adadelta', {'output':perm_loss,\n",
    "                                  'category':'categorical_crossentropy'})\n",
    "    else:\n",
    "        graph.compile('adadelta', {'output':'categorical_crossentropy',\n",
    "                                  'category':'categorical_crossentropy'})\n",
    "        \n",
    "\n",
    "    #graph.load_weights(\"training_general_scale10\")\n",
    "    #############################################\n",
    "    #Second end there\n",
    "\n",
    "    #############################################\n",
    "\n",
    "    return graph\n",
    "\n",
    "    #history = graph.fit({'input1':X_train[::,1], 'input2':X2_train[::0], 'output':y_train}, nb_epoch=10)\n",
    "    #predictions = graph.predict({'input1':X_test, 'input2':X2_test}) # {'output':...}\n",
    "    #graph.save_weights(\"step_check\",overwrite=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "#print theano.__version__ , theano.__file__\n",
    "import keras\n",
    "#print keras.__version__, keras.__file__\n",
    "from keras.models import Graph\n",
    "from keras.layers.core import Dense, Dropout, Activation,TimeDistributedDense,Merge,Reshape\n",
    "from keras.layers.core import Lambda\n",
    "from keras.layers.convolutional import Convolution1D,MaxPooling1D,UpSampling1D\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM,GRU\n",
    "if  int(keras.__version__.split(\".\")[0]) >= 1.0 :\n",
    "    from Bilayer import BiLSTMv1 as BiLSTM\n",
    "    from Bilayer import BiSimpleRNNv1 as BiSimpleRNN\n",
    "\n",
    "else:\n",
    "    from Bilayer import BiLSTM, BiSimpleRNN\n",
    "    \n",
    "import theano.tensor as T\n",
    "import theano\n",
    "from keras.backend.common import _EPSILON\n",
    "#from keras.objectives import categorical_crossentropy\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def return_layer_paper(ndim=2,inside=50,permutation=True,inputsize=5,simple=False,\n",
    "                       n_layers=4,category=True,output=True):\n",
    "\n",
    "    #categorical_crossentropy??\n",
    "    #Loss:\n",
    "\n",
    "\n",
    "    perm =[[0,1,2],[1,2,0],[2,1,0],[0,2,1],[1,0,2],[2,0,1]]\n",
    "    perm = [[-3,-2,-1]+iperm for iperm in perm]\n",
    "    perm = np.array(perm,dtype=np.int)\n",
    "    perm += 3\n",
    "\n",
    "    test_true = [[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]],[[0,1],[0,1],[0,1]]]\n",
    "\n",
    "    eps =1e-7\n",
    "    test_pred = [[[1-eps,+eps],[1-eps,eps],[1-eps,eps]],[[eps,1-eps],[eps,1-eps],[eps,1-eps]],\n",
    "                 [[eps,1-eps],[eps,1-eps],[eps,1-eps]]]\n",
    "\n",
    "\n",
    "    def perm_loss(y_true,y_pred):\n",
    "        def loss(m,  y_true, y_pred,perm):\n",
    "\n",
    "            #return  perm[T.cast(m,\"int32\")]\n",
    "            y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON)\n",
    "            return T.mean( T.sum(y_true[::,::,perm[m]] * T.log(y_pred),axis=-1),axis=-1)\n",
    "\n",
    "        #perm = np.array([[0,1],[1,0]],dtype=np.int)\n",
    "        perm = np.array([[0, 1, 2, 3, 4, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 5, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 4, 3, 6] + range(7,10),\n",
    "                         [0, 1, 2, 3, 5, 4, 6] + range(7,10),\n",
    "                         [0, 1, 2, 4, 3, 5, 6] + range(7,10),\n",
    "                         [0, 1, 2, 5, 3, 4, 6] + range(7,10)],dtype=np.int)\n",
    "\n",
    "        \"\"\"perm = np.array([[0, 1, 2, 3, 4, 5, 6],\n",
    "                         [0, 1, 2, 3, 4, 5, 6]],dtype=np.int)\"\"\"\n",
    "        seq = T.arange(len(perm))\n",
    "        result, _ = theano.scan(fn=loss, outputs_info=None, \n",
    "        sequences=seq, non_sequences=[y_true, y_pred,perm])\n",
    "        return -T.mean(T.max(result,axis=0)) #T.max(result.dimshuffle(1,2,0),axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "    def reverse(X):\n",
    "        return X[::,::,::-1]\n",
    "\n",
    "    def output_shape(input_shape):\n",
    "        # here input_shape includes the samples dimension\n",
    "        return input_shape  # shap\n",
    "\n",
    "    def identity(X):\n",
    "        return X\n",
    "\n",
    "    def sub_mean(X):\n",
    "        xdms = X.shape\n",
    "        return X.reshape(xdms[0])\n",
    "    \n",
    "    #middle = 50\n",
    "    add = 0\n",
    "    \n",
    "    if ndim == 3:\n",
    "        add = 1\n",
    "        \n",
    "    if simple:\n",
    "        Bi = BiSimpleRNN\n",
    "        \n",
    "    else:\n",
    "        Bi = BiLSTM\n",
    "    graph = Graph()\n",
    "    graph.add_input(name='input1', input_shape=(None,inputsize))\n",
    " \n",
    "\n",
    "    graph.add_node(Bi(output_dim=inside,activation='tanh',return_sequences=True,close=True,input_shape=(200,inputsize),),\n",
    "                       name=\"l1\",input=\"input1\")\n",
    "    \n",
    "    \n",
    "    for layer in range(2,n_layers+1):\n",
    "\n",
    "        graph.add_node(Bi(output_dim=inside,input_shape=(200,inputsize),\n",
    "                           return_sequences=True,close=True,activation='tanh'),name=\"l%i\"%layer,\n",
    "                           inputs=[\"input1\",\"l%i\"%(layer-1)],merge_mode=\"concat\",concat_axis=-1)\n",
    "\n",
    "\n",
    "    graph.add_node(Dropout(0.4),inputs=[\"l%i\"%layer for layer in range(1,n_layers+1)],\n",
    "                   merge_mode=\"concat\",concat_axis=-1,name=\"output0_drop\")\n",
    "    #Here get the subcategory\n",
    "\n",
    "    graph.add_node(TimeDistributedDense(10,activation=\"softmax\"),input=\"output0_drop\",\n",
    "                   name=\"output0\")\n",
    "\n",
    "\n",
    "    \n",
    "    res = {}\n",
    "\n",
    "    if category:\n",
    "        graph.add_node(Bi(output_dim=27,activation='tanh',return_sequences=False,close=True),\n",
    "                           name=\"category0bi\",input=\"output0\")\n",
    "        graph.add_node(Dense(27,activation=\"softmax\"),input=\"category0bi\",name=\"category0\")\n",
    "        graph.add_output(name=\"category\",input=\"category0\")\n",
    "        res['category'] = 'categorical_crossentropy'\n",
    "    \n",
    "    if output:\n",
    "        graph.add_output(name=\"output\",input=\"output0\")\n",
    "\n",
    "        if permutation:\n",
    "            res['output'] = perm_loss\n",
    "        else:\n",
    "            res['output'] = 'categorical_crossentropy'\n",
    "        \n",
    "\n",
    "    graph.compile('adadelta', res)\n",
    "        \n",
    "\n",
    "    return graph\n",
    "\n",
    "    #history = graph.fit({'input1':X_train[::,1], 'input2':X2_train[::0], 'output':y_train}, nb_epoch=10)\n",
    "    #predictions = graph.predict({'input1':X_test, 'input2':X2_test}) # {'output':...}\n",
    "    #graph.save_weights(\"step_check\",overwrite=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}