Feature Extraction Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read in the data\n",
      "(7352, 1)\n",
      "(7352, 128, 9)\n",
      "WARNING:tensorflow:From c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From <ipython-input-1-69c1d93e2dab>:204: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "start training\n",
      "Training iter #1500:   Batch Loss = 2.863983, Accuracy = 0.14533333480358124\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 2.720302104949951, Accuracy = 0.1425178200006485\n",
      "Training iter #300000:   Batch Loss = 0.430425, Accuracy = 0.972000002861023\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.5676230192184448, Accuracy = 0.9267051219940186\n",
      "Training iter #600000:   Batch Loss = 0.341473, Accuracy = 0.9753333330154419\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.5598466396331787, Accuracy = 0.9334917068481445\n",
      "Training iter #900000:   Batch Loss = 0.258737, Accuracy = 0.9893333315849304\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.4117996394634247, Accuracy = 0.9501187801361084\n",
      "Training iter #1200000:   Batch Loss = 0.207006, Accuracy = 0.9933333396911621\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.4304578900337219, Accuracy = 0.9419748783111572\n",
      "Optimization Finished!\n",
      "Training time is: 1372.369987487793 seconds\n",
      "FINAL RESULT: Batch Loss = 0.3828376531600952, Accuracy = 0.9511367678642273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\sklearn\\utils\\validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVR)-------------\n",
      "Training time is: 31.166205167770386 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.8571428571428571\n",
      "\n",
      "Precision: 85.67684044882586%\n",
      "Recall: 85.71428571428571%\n",
      "f1_score: 85.61955651967943%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[443  21  32   0   0   0]\n",
      " [ 59 389  23   0   0   0]\n",
      " [ 31  15 374   0   0   0]\n",
      " [  0  17   0 347 121   6]\n",
      " [  0   2   0  94 436   0]\n",
      " [  0   0   0   0   0 537]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[15.032236    0.7125891   1.08585     0.          0.          0.        ]\n",
      " [ 2.002036   13.199864    0.7804547   0.          0.          0.        ]\n",
      " [ 1.0519172   0.5089922  12.690872    0.          0.          0.        ]\n",
      " [ 0.          0.5768578   0.         11.774686    4.1058702   0.20359688]\n",
      " [ 0.          0.06786563  0.          3.1896844  14.794705    0.        ]\n",
      " [ 0.          0.          0.          0.          0.         18.22192   ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVO)-------------\n",
      "Training time is: 31.257577896118164 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.8571428571428571\n",
      "\n",
      "Precision: 85.67684044882586%\n",
      "Recall: 85.71428571428571%\n",
      "f1_score: 85.61955651967943%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[443  21  32   0   0   0]\n",
      " [ 59 389  23   0   0   0]\n",
      " [ 31  15 374   0   0   0]\n",
      " [  0  17   0 347 121   6]\n",
      " [  0   2   0  94 436   0]\n",
      " [  0   0   0   0   0 537]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[15.032236    0.7125891   1.08585     0.          0.          0.        ]\n",
      " [ 2.002036   13.199864    0.7804547   0.          0.          0.        ]\n",
      " [ 1.0519172   0.5089922  12.690872    0.          0.          0.        ]\n",
      " [ 0.          0.5768578   0.         11.774686    4.1058702   0.20359688]\n",
      " [ 0.          0.06786563  0.          3.1896844  14.794705    0.        ]\n",
      " [ 0.          0.          0.          0.          0.         18.22192   ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------lstm-----------------\n",
      "(?, 32)\n",
      "Training iter #1500:   Batch Loss = 2.274322, Accuracy = 0.34200000762939453\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 1.7587647438049316, Accuracy = 0.30166271328926086\n",
      "Training iter #300000:   Batch Loss = 0.024958, Accuracy = 0.996666669845581\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.18356430530548096, Accuracy = 0.9524940848350525\n",
      "Training iter #600000:   Batch Loss = 0.060137, Accuracy = 0.9766666889190674\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.2611507177352905, Accuracy = 0.9501187801361084\n",
      "Optimization Finished!\n",
      "Training time is: 512.5083673000336 seconds\n",
      "FINAL LSTM RESULT: Batch Loss = 0.24866531789302826, Accuracy = 0.9575839638710022\n",
      "Testing Accuracy: 95.75839638710022%\n",
      "\n",
      "Precision: 95.97518910756236%\n",
      "Recall: 95.75839837122497%\n",
      "f1_score: 95.77410410709045%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[469   0  26   0   0   1]\n",
      " [  0 439  32   0   0   0]\n",
      " [  7   7 406   0   0   0]\n",
      " [  0  17   0 444  26   4]\n",
      " [  0   1   0   0 531   0]\n",
      " [  0   0   0   0   4 533]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[15.91449     0.          0.88225317  0.          0.          0.03393281]\n",
      " [ 0.         14.896504    1.08585     0.          0.          0.        ]\n",
      " [ 0.2375297   0.2375297  13.776723    0.          0.          0.        ]\n",
      " [ 0.          0.5768578   0.         15.066169    0.88225317  0.13573125]\n",
      " [ 0.          0.03393281  0.          0.         18.018324    0.        ]\n",
      " [ 0.          0.          0.          0.          0.13573125 18.08619   ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Experiment 2 (CNN LSTM as feature extractor) of feature extractor training with the CNNLSTM model model 3.\n",
    "# CNN LSTM model is used as a feature extrator, SVM (ovo or ovr) and a single layer of LSTM is used as personlized classifier\n",
    "# The personlized classifier is trained with 100% dataset\n",
    "# The structure based on Guillaume Chevalier https://github.com/guillaume-chevalier/LSTM-Human-Activity-Recognition\n",
    "# The structure include read in of the input, formatting the input and the conversion of the one-hot encoding.\n",
    "# The SVM is model is from sklearn library\n",
    "# The CNN LSTM model and the feature extration method is written by myself\n",
    "# The credit of this code goes to Guillaume Chevalier under MIT License.\n",
    "# Python version 3.6\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf  # Version 1.0\n",
    "from sklearn import metrics\n",
    "import time\n",
    "import os\n",
    "\n",
    "# Those are separate normalised input features for the neural network\n",
    "INPUT_SIGNAL_TYPES = [\n",
    "    \"body_acc_x_\",\n",
    "    \"body_acc_y_\",\n",
    "    \"body_acc_z_\",\n",
    "    \"body_gyro_x_\",\n",
    "    \"body_gyro_y_\",\n",
    "    \"body_gyro_z_\",\n",
    "    \"total_acc_x_\",\n",
    "    \"total_acc_y_\",\n",
    "    \"total_acc_z_\"\n",
    "]\n",
    "\n",
    "# Output classes to learn how to classify\n",
    "LABELS = [\n",
    "    \"WALKING\", \n",
    "    \"WALKING_UPSTAIRS\", \n",
    "    \"WALKING_DOWNSTAIRS\", \n",
    "    \"SITTING\", \n",
    "    \"STANDING\", \n",
    "    \"LAYING\"\n",
    "] \n",
    "\n",
    "# Pathes to dataset\n",
    "DATASET_PATH = \"UCI HAR Dataset/\"\n",
    "TRAIN = \"train/\"\n",
    "TEST = \"test/\"\n",
    "\n",
    "\n",
    "def load_X(X_signals_paths):\n",
    "    X_signals = []\n",
    "    \n",
    "    for signal_type_path in X_signals_paths:\n",
    "        file = open(signal_type_path, 'r')\n",
    "        # Read dataset from disk, dealing with text files' syntax\n",
    "        X_signals.append(\n",
    "            [np.array(serie, dtype=np.float32) for serie in [\n",
    "                row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "            ]]\n",
    "        )\n",
    "        file.close()\n",
    "    \n",
    "    return np.transpose(np.array(X_signals), (1, 2, 0))\n",
    "\n",
    "def load_y(y_path):\n",
    "    file = open(y_path, 'r')\n",
    "    # Read dataset from disk, dealing with text file's syntax\n",
    "    y_ = np.array(\n",
    "        [elem for elem in [\n",
    "            row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "        ]], \n",
    "        dtype=np.int32\n",
    "    )\n",
    "    file.close()\n",
    "    \n",
    "    # Substract 1 to each output class for friendly 0-based indexing \n",
    "    return y_ - 1\n",
    "\n",
    "X_train_signals_paths = [\n",
    "    DATASET_PATH + TRAIN + \"Inertial Signals/\" + signal + \"train.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "X_test_signals_paths = [\n",
    "    DATASET_PATH + TEST + \"Inertial Signals/\" + signal + \"test.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "y_train_path = DATASET_PATH + TRAIN + \"y_train.txt\"\n",
    "y_test_path = DATASET_PATH + TEST + \"y_test.txt\"\n",
    "\n",
    "X_for_train = load_X(X_train_signals_paths)\n",
    "X_test = load_X(X_test_signals_paths)\n",
    "y_for_train = load_y(y_train_path)\n",
    "y_test = load_y(y_test_path)\n",
    "\n",
    "print(\"read in the data\")\n",
    "\n",
    "# randomly shuffle the array\n",
    "def rand_shuffle_data(train_X, train_Y):\n",
    "    item_size = np.array(train_X).shape\n",
    "    sample_size = item_size[0]\n",
    "    np.random.seed(42)\n",
    "    value_list = np.arange(sample_size)\n",
    "    np.random.shuffle(value_list)\n",
    "    output_X = np.zeros(np.array(train_X).shape)\n",
    "    output_Y = np.zeros(np.array(train_Y).shape)\n",
    "    print(output_Y.shape)\n",
    "    cnt = 0\n",
    "    for idx in value_list:\n",
    "        output_X[cnt,:,:] = train_X[idx,:,:]\n",
    "        output_Y[cnt,:] = train_Y[idx,:]\n",
    "        cnt += 1\n",
    "    return value_list, output_X, output_Y\n",
    "\n",
    "# The pre-trained model\n",
    "def cnn_LSTM_net(_X, _weights, _biases):\n",
    "    _X = tf.reshape(_X, [-1, n_steps, n_input]) \n",
    "    # convlution layers\n",
    "    conv1 = tf.layers.separable_conv1d(_X, 32, 5, activation=tf.nn.relu, name='conv1_layer')\n",
    "    conv1 = tf.layers.max_pooling1d(conv1, 20, 2, name='max1_layer')\n",
    "    conv2 = tf.layers.separable_conv1d(conv1, 64, 5, activation=tf.nn.relu, name='conv2_layer')\n",
    "    conv2 = tf.transpose(conv2, [1, 0, 2])\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)\n",
    "    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell], state_is_tuple=True)\n",
    "    # Get LSTM cell output\n",
    "    outputs, states = tf.nn.dynamic_rnn(lstm_cells, conv2, dtype=tf.float32)\n",
    "    lstm_last_output = outputs[-1]\n",
    "    # Linear activation\n",
    "    return tf.matmul(lstm_last_output, _weights['out']) + _biases['out']\n",
    "\n",
    "# The personlized model\n",
    "def lstm_net(_X, _weights, _biases):\n",
    "    conv2 = tf.transpose(_X, [1, 0, 2])\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)\n",
    "    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell], state_is_tuple=True)\n",
    "    # Get LSTM cell output\n",
    "    outputs, states = tf.nn.dynamic_rnn(lstm_cells, conv2, dtype=tf.float32)\n",
    "    lstm_last_output = outputs[-1]\n",
    "    # Linear activation\n",
    "    print(lstm_last_output.shape)\n",
    "    return tf.matmul(lstm_last_output, _weights['out']) + _biases['out']\n",
    "\n",
    "def extract_batch_size(_train, step, batch_size):\n",
    "    # Function to fetch a \"batch_size\" amount of data from \"(X|y)_train\" data. \n",
    "    \n",
    "    shape = list(_train.shape)\n",
    "    shape[0] = batch_size\n",
    "    batch_s = np.empty(shape)\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        # Loop index\n",
    "        index = ((step-1)*batch_size + i) % len(_train)\n",
    "        batch_s[i] = _train[index] \n",
    "\n",
    "    return batch_s\n",
    "\n",
    "\n",
    "def one_hot(y_):\n",
    "    # Function to encode output labels from number indexes \n",
    "    # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]\n",
    "    \n",
    "    y_ = y_.reshape(len(y_))\n",
    "    n_values = int(np.max(y_)) + 1\n",
    "    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS\n",
    "\n",
    "\n",
    "percentage_list = [1]\n",
    "\n",
    "for percent_train in percentage_list:\n",
    "    \n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    rand_shuffle_idx, train_X, train_y = rand_shuffle_data(X_for_train, y_for_train)\n",
    "    pick_samples = int(round((np.array(X_for_train).shape[0])*percent_train))\n",
    "    X_train = X_for_train[0:pick_samples, :, :]\n",
    "    y_train = y_for_train[0:pick_samples,:]\n",
    "    \n",
    "    print(np.array(X_train).shape)\n",
    "    \n",
    "    # Input data and LSTM internal structure\n",
    "    training_data_count = len(X_train)  # 7352 training series (with 50% overlap between each serie)\n",
    "    test_data_count = len(X_test)  # 2947 testing series\n",
    "    n_height = 1\n",
    "    n_steps = len(X_train[0])  # 128 timesteps per series\n",
    "    n_input = len(X_train[0][0])\n",
    "    n_hidden = 32\n",
    "    n_classes = 6 # Total classes (should go up, or should go down)\n",
    "\n",
    "    # Training parameters\n",
    "    learning_rate = 0.0025\n",
    "    lambda_loss_amount = 0.0015\n",
    "    training_iters = training_data_count * 200  # Loop 200 times on the dataset\n",
    "    batch_size = 1500\n",
    "    display_iter = 300000  # To show test set accuracy during training\n",
    "    \n",
    "    # Graph input/output\n",
    "    x = tf.placeholder(tf.float32, [None, n_steps, n_input])\n",
    "    y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "\n",
    "    weights = {\n",
    "        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0))\n",
    "    }\n",
    "    biases = {\n",
    "        'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "    }\n",
    "\n",
    "    pred = cnn_LSTM_net(x,weights,biases)\n",
    "\n",
    "    # Loss, optimizer and evaluation\n",
    "    l2 = lambda_loss_amount * sum(\n",
    "        tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()\n",
    "    ) # L2 loss prevents this overkill neural network to overfit the data\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=pred)) + l2# Softmax loss\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer\n",
    "\n",
    "    correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "    # To keep track of training's performance\n",
    "    test_losses = []\n",
    "    test_accuracies = []\n",
    "    train_losses = []\n",
    "    train_accuracies = []\n",
    "\n",
    "    # Launch the graph\n",
    "    print(\"start training\")\n",
    "    start_time = time.time()\n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    # Perform Training steps with \"batch_size\" amount of example data at each loop\n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters:\n",
    "        batch_xs =         extract_batch_size(X_train, step, batch_size)\n",
    "        batch_ys = one_hot(extract_batch_size(y_train, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer, cost, accuracy],\n",
    "            feed_dict={\n",
    "                x: batch_xs, \n",
    "                y: batch_ys\n",
    "            }\n",
    "        )\n",
    "        train_losses.append(loss)\n",
    "        train_accuracies.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost, accuracy], \n",
    "            feed_dict={\n",
    "                x: X_test,\n",
    "                y: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses.append(loss2)\n",
    "        test_accuracies.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time) + \" seconds\")\n",
    "\n",
    "    one_hot_predictions, accuracy, final_loss = sess.run(\n",
    "        [pred, accuracy, cost],\n",
    "        feed_dict={\n",
    "            x: X_test,\n",
    "            y: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss) + \\\n",
    "          \", Accuracy = {}\".format(accuracy))\n",
    "\n",
    "    \n",
    "    # feature extration\n",
    "    feature_tensor = sess.graph.get_tensor_by_name('conv2_layer/Relu:0')\n",
    "    features = sess.run(feature_tensor, feed_dict={x:X_train})\n",
    "    feature_shape = features.shape\n",
    "    test_features = sess.run(feature_tensor,feed_dict={x:X_test})\n",
    "    test_feature_shape = test_features.shape\n",
    "    reformatted_features = np.zeros((feature_shape[0], feature_shape[1]*feature_shape[2]))\n",
    "    test_reformatted_features = np.zeros((test_feature_shape[0], test_feature_shape[1]*test_feature_shape[2]))\n",
    "    c = 0\n",
    "    for row in features:\n",
    "        reformatted_features[c,:] = row.flatten()\n",
    "        c += 1\n",
    "    d = 0\n",
    "    for row in test_features:\n",
    "        test_reformatted_features[d,:] = row.flatten()\n",
    "        d += 1\n",
    "    \n",
    "    # classify with svm\n",
    "    from sklearn.svm import SVC\n",
    "    svm_starttime = time.time()\n",
    "    clf = SVC()\n",
    "    clf.fit(reformatted_features, y_train)\n",
    "    print(\"-------------SVM(OVR)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_1 = clf.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_1)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnnlstm_svm_ovr_conf.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnnlstm_svm_ovr_conf.png\", bbox_inches='tight')\n",
    "    \n",
    "    \n",
    "    svm2_starttime = time.time()\n",
    "    clf2 = SVC(decision_function_shape='ovo')\n",
    "    clf2.fit(reformatted_features, y_train)\n",
    "    print(\"-------------SVM(OVO)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm2_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf2.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_2 = clf2.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnnlstm_svm_ovo_conf.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnnlstm_svm_ovo_conf.png\", bbox_inches='tight')\n",
    "    \n",
    "    start_time_lstm = time.time()\n",
    "    print(\"-------------lstm-----------------\")\n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    learning_rate_lstm = 0.0025\n",
    "    \n",
    "    weights2 = {\n",
    "        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0))\n",
    "    }\n",
    "    biases2 = {\n",
    "        'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "    }\n",
    "    \n",
    "    x_feat = tf.placeholder(tf.float32, [None, feature_shape[1], feature_shape[2]])\n",
    "    y2 = tf.placeholder(tf.float32, [None, n_classes])\n",
    "    pred_lstm = lstm_net(x_feat, weights2, biases2)\n",
    "    cost2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y2, logits=pred_lstm))# Softmax loss\n",
    "    optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate_lstm).minimize(cost2)\n",
    "    correct_pred2 = tf.equal(tf.argmax(pred_lstm,1), tf.argmax(y2,1))\n",
    "    accuracy2 = tf.reduce_mean(tf.cast(correct_pred2, tf.float32))\n",
    "    \n",
    "    test_losses2 = []\n",
    "    test_accuracies2 = []\n",
    "    train_losses2 = []\n",
    "    train_accuracies2 = []\n",
    "    training_iters2 = training_data_count * 100  # Loop 100 times on the dataset\n",
    "    \n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    \n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters2:\n",
    "        batch_xxs =         extract_batch_size(features, step, batch_size)\n",
    "        batch_yys = one_hot(extract_batch_size(y_train, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer2, cost2, accuracy2],\n",
    "            feed_dict={\n",
    "                x_feat: batch_xxs, \n",
    "                y2: batch_yys\n",
    "            }\n",
    "        )\n",
    "        train_losses2.append(loss)\n",
    "        train_accuracies2.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost2, accuracy2], \n",
    "            feed_dict={\n",
    "                x_feat: test_features,\n",
    "                y2: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses2.append(loss2)\n",
    "        test_accuracies2.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters2):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time_lstm) + \" seconds\")\n",
    "\n",
    "    one_hot_predictions2, accuracy2, final_loss2 = sess.run(\n",
    "        [pred_lstm, accuracy2, cost2],\n",
    "        feed_dict={\n",
    "            x_feat: test_features,\n",
    "            y2: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL LSTM RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss2) + \\\n",
    "          \", Accuracy = {}\".format(accuracy2))\n",
    "    \n",
    "    predictions2 = one_hot_predictions2.argmax(1)\n",
    "    \n",
    "    print(\"Testing Accuracy: {}%\".format(100*accuracy2))\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, predictions2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, predictions2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnnlstm_feat_lstm_train_conf.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnnlstm_feat_lstm_train_conf.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read in the data\n",
      "(7352, 1)\n",
      "(7352, 128, 9)\n",
      "WARNING:tensorflow:From c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From <ipython-input-1-4b47ce760439>:207: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "start training\n",
      "Training iter #1500:   Batch Loss = 2.586469, Accuracy = 0.14533333480358124\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 2.5635905265808105, Accuracy = 0.1425178200006485\n",
      "Training iter #300000:   Batch Loss = 0.392232, Accuracy = 0.9779999852180481\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.5255632996559143, Accuracy = 0.9311164021492004\n",
      "Training iter #600000:   Batch Loss = 0.314711, Accuracy = 0.9806666374206543\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.42468589544296265, Accuracy = 0.9423142075538635\n",
      "Training iter #900000:   Batch Loss = 0.222358, Accuracy = 0.996666669845581\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.3646925687789917, Accuracy = 0.9470648169517517\n",
      "Training iter #1200000:   Batch Loss = 0.211350, Accuracy = 0.9866666793823242\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.3892090916633606, Accuracy = 0.9355276823043823\n",
      "Optimization Finished!\n",
      "Training time is: 1335.0050539970398 seconds\n",
      "FINAL RESULT: Batch Loss = 0.2926364541053772, Accuracy = 0.9501187801361084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\sklearn\\utils\\validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVR)-------------\n",
      "Training time is: 9.784814596176147 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.8445877163216831\n",
      "\n",
      "Precision: 84.94110903225469%\n",
      "Recall: 84.45877163216831%\n",
      "f1_score: 84.5084431983994%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[473   4  19   0   0   0]\n",
      " [ 56 397  18   0   0   0]\n",
      " [ 41  44 335   0   0   0]\n",
      " [  0   0   0 351 140   0]\n",
      " [  0   0   0 108 424   0]\n",
      " [  0   1   0   0  27 509]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.05022     0.13573125  0.6447235   0.          0.          0.        ]\n",
      " [ 1.9002376  13.471326    0.6107906   0.          0.          0.        ]\n",
      " [ 1.3912454   1.4930438  11.367493    0.          0.          0.        ]\n",
      " [ 0.          0.          0.         11.910418    4.7505937   0.        ]\n",
      " [ 0.          0.          0.          3.664744   14.387512    0.        ]\n",
      " [ 0.          0.03393281  0.          0.          0.916186   17.271801  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVO)-------------\n",
      "Training time is: 9.791866779327393 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.8445877163216831\n",
      "\n",
      "Precision: 84.94110903225469%\n",
      "Recall: 84.45877163216831%\n",
      "f1_score: 84.5084431983994%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[473   4  19   0   0   0]\n",
      " [ 56 397  18   0   0   0]\n",
      " [ 41  44 335   0   0   0]\n",
      " [  0   0   0 351 140   0]\n",
      " [  0   0   0 108 424   0]\n",
      " [  0   1   0   0  27 509]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.05022     0.13573125  0.6447235   0.          0.          0.        ]\n",
      " [ 1.9002376  13.471326    0.6107906   0.          0.          0.        ]\n",
      " [ 1.3912454   1.4930438  11.367493    0.          0.          0.        ]\n",
      " [ 0.          0.          0.         11.910418    4.7505937   0.        ]\n",
      " [ 0.          0.          0.          3.664744   14.387512    0.        ]\n",
      " [ 0.          0.03393281  0.          0.          0.916186   17.271801  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------lstm-----------------\n",
      "(?, 32)\n",
      "Training iter #1500:   Batch Loss = 2.907452, Accuracy = 0.30266666412353516\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 2.131521701812744, Accuracy = 0.34747201204299927\n",
      "Training iter #300000:   Batch Loss = 0.011924, Accuracy = 0.9993333220481873\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.23089906573295593, Accuracy = 0.9446895122528076\n",
      "Training iter #600000:   Batch Loss = 0.019851, Accuracy = 0.9933333396911621\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.27710995078086853, Accuracy = 0.9429928660392761\n",
      "Optimization Finished!\n",
      "Training time is: 500.97373962402344 seconds\n",
      "FINAL LSTM RESULT: Batch Loss = 0.2707633972167969, Accuracy = 0.9487614631652832\n",
      "Testing Accuracy: 94.87614631652832%\n",
      "\n",
      "Precision: 95.19054266963273%\n",
      "Recall: 94.87614523243977%\n",
      "f1_score: 94.91503334948601%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[466   0  30   0   0   0]\n",
      " [  0 450  15   0   6   0]\n",
      " [  5  10 405   0   0   0]\n",
      " [  0   0   0 455  36   0]\n",
      " [  0   1   0   5 526   0]\n",
      " [ 15   0   0   1  27 494]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[15.812691    0.          1.0179844   0.          0.          0.        ]\n",
      " [ 0.         15.269765    0.5089922   0.          0.20359688  0.        ]\n",
      " [ 0.16966406  0.3393281  13.742789    0.          0.          0.        ]\n",
      " [ 0.          0.          0.         15.43943     1.2215812   0.        ]\n",
      " [ 0.          0.03393281  0.          0.16966406 17.84866     0.        ]\n",
      " [ 0.5089922   0.          0.          0.03393281  0.916186   16.76281   ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Experiment 2 (CNN LSTM as feature extractor) of feature extractor training with the CNNLSTM model model 3.\n",
    "# CNN LSTM model is used as a feature extrator, SVM (ovo or ovr) and a single layer of LSTM is used as personlized classifier\n",
    "# The personlized classifier works on the 50% dataset\n",
    "# The structure based on Guillaume Chevalier https://github.com/guillaume-chevalier/LSTM-Human-Activity-Recognition\n",
    "# The structure include read in of the input, formatting the input and the conversion of the one-hot encoding.\n",
    "# The SVM is model is from sklearn library\n",
    "# The CNN LSTM model and the feature extration method is written by myself\n",
    "# The credit of this code goes to Guillaume Chevalier under MIT License.\n",
    "# Python version 3.6\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf  # Version 1.0\n",
    "from sklearn import metrics\n",
    "import time\n",
    "import os\n",
    "\n",
    "# Those are separate normalised input features for the neural network\n",
    "INPUT_SIGNAL_TYPES = [\n",
    "    \"body_acc_x_\",\n",
    "    \"body_acc_y_\",\n",
    "    \"body_acc_z_\",\n",
    "    \"body_gyro_x_\",\n",
    "    \"body_gyro_y_\",\n",
    "    \"body_gyro_z_\",\n",
    "    \"total_acc_x_\",\n",
    "    \"total_acc_y_\",\n",
    "    \"total_acc_z_\"\n",
    "]\n",
    "\n",
    "# Output classes to learn how to classify\n",
    "LABELS = [\n",
    "    \"WALKING\", \n",
    "    \"WALKING_UPSTAIRS\", \n",
    "    \"WALKING_DOWNSTAIRS\", \n",
    "    \"SITTING\", \n",
    "    \"STANDING\", \n",
    "    \"LAYING\"\n",
    "] \n",
    "\n",
    "# Pathes to dataset\n",
    "DATASET_PATH = \"UCI HAR Dataset/\"\n",
    "TRAIN = \"train/\"\n",
    "TEST = \"test/\"\n",
    "\n",
    "\n",
    "def load_X(X_signals_paths):\n",
    "    X_signals = []\n",
    "    \n",
    "    for signal_type_path in X_signals_paths:\n",
    "        file = open(signal_type_path, 'r')\n",
    "        # Read dataset from disk, dealing with text files' syntax\n",
    "        X_signals.append(\n",
    "            [np.array(serie, dtype=np.float32) for serie in [\n",
    "                row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "            ]]\n",
    "        )\n",
    "        file.close()\n",
    "    \n",
    "    return np.transpose(np.array(X_signals), (1, 2, 0))\n",
    "\n",
    "def load_y(y_path):\n",
    "    file = open(y_path, 'r')\n",
    "    # Read dataset from disk, dealing with text file's syntax\n",
    "    y_ = np.array(\n",
    "        [elem for elem in [\n",
    "            row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "        ]], \n",
    "        dtype=np.int32\n",
    "    )\n",
    "    file.close()\n",
    "    \n",
    "    # Substract 1 to each output class for friendly 0-based indexing \n",
    "    return y_ - 1\n",
    "\n",
    "X_train_signals_paths = [\n",
    "    DATASET_PATH + TRAIN + \"Inertial Signals/\" + signal + \"train.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "X_test_signals_paths = [\n",
    "    DATASET_PATH + TEST + \"Inertial Signals/\" + signal + \"test.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "y_train_path = DATASET_PATH + TRAIN + \"y_train.txt\"\n",
    "y_test_path = DATASET_PATH + TEST + \"y_test.txt\"\n",
    "\n",
    "X_for_train = load_X(X_train_signals_paths)\n",
    "X_test = load_X(X_test_signals_paths)\n",
    "y_for_train = load_y(y_train_path)\n",
    "y_test = load_y(y_test_path)\n",
    "\n",
    "print(\"read in the data\")\n",
    "\n",
    "# randomly shuffle the array\n",
    "def rand_shuffle_data(train_X, train_Y):\n",
    "    item_size = np.array(train_X).shape\n",
    "    sample_size = item_size[0]\n",
    "    np.random.seed(42)\n",
    "    value_list = np.arange(sample_size)\n",
    "    np.random.shuffle(value_list)\n",
    "    output_X = np.zeros(np.array(train_X).shape)\n",
    "    output_Y = np.zeros(np.array(train_Y).shape)\n",
    "    print(output_Y.shape)\n",
    "    cnt = 0\n",
    "    for idx in value_list:\n",
    "        output_X[cnt,:,:] = train_X[idx,:,:]\n",
    "        output_Y[cnt,:] = train_Y[idx,:]\n",
    "        cnt += 1\n",
    "    return value_list, output_X, output_Y\n",
    "\n",
    "# The pre-trained model\n",
    "def cnn_LSTM_net(_X, _weights, _biases):\n",
    "    _X = tf.reshape(_X, [-1, n_steps, n_input]) \n",
    "    # convlution layers\n",
    "    conv1 = tf.layers.separable_conv1d(_X, 32, 5, activation=tf.nn.relu, name='conv1_layer')\n",
    "    conv1 = tf.layers.max_pooling1d(conv1, 20, 2, name='max1_layer')\n",
    "    conv2 = tf.layers.separable_conv1d(conv1, 64, 5, activation=tf.nn.relu, name='conv2_layer')\n",
    "    conv2 = tf.transpose(conv2, [1, 0, 2])\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)\n",
    "    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell], state_is_tuple=True)\n",
    "    # Get LSTM cell output\n",
    "    outputs, states = tf.nn.dynamic_rnn(lstm_cells, conv2, dtype=tf.float32)\n",
    "    lstm_last_output = outputs[-1]\n",
    "    # Linear activation\n",
    "    return tf.matmul(lstm_last_output, _weights['out']) + _biases['out']\n",
    "\n",
    "# The personlized model\n",
    "def lstm_net(_X, _weights, _biases):\n",
    "    conv2 = tf.transpose(_X, [1, 0, 2])\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)\n",
    "    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell], state_is_tuple=True)\n",
    "    # Get LSTM cell output\n",
    "    outputs, states = tf.nn.dynamic_rnn(lstm_cells, conv2, dtype=tf.float32)\n",
    "    lstm_last_output = outputs[-1]\n",
    "    # Linear activation\n",
    "    print(lstm_last_output.shape)\n",
    "    return tf.matmul(lstm_last_output, _weights['out']) + _biases['out']\n",
    "\n",
    "def extract_batch_size(_train, step, batch_size):\n",
    "    # Function to fetch a \"batch_size\" amount of data from \"(X|y)_train\" data. \n",
    "    \n",
    "    shape = list(_train.shape)\n",
    "    shape[0] = batch_size\n",
    "    batch_s = np.empty(shape)\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        # Loop index\n",
    "        index = ((step-1)*batch_size + i) % len(_train)\n",
    "        batch_s[i] = _train[index] \n",
    "\n",
    "    return batch_s\n",
    "\n",
    "\n",
    "def one_hot(y_):\n",
    "    # Function to encode output labels from number indexes \n",
    "    # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]\n",
    "    \n",
    "    y_ = y_.reshape(len(y_))\n",
    "    n_values = int(np.max(y_)) + 1\n",
    "    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS\n",
    "\n",
    "\n",
    "percentage_list = [1]\n",
    "\n",
    "for percent_train in percentage_list:\n",
    "    \n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    rand_shuffle_idx, train_X, train_y = rand_shuffle_data(X_for_train, y_for_train)\n",
    "    pick_samples = int(round((np.array(X_for_train).shape[0])*percent_train))\n",
    "    X_train = X_for_train[0:pick_samples, :, :]\n",
    "    y_train = y_for_train[0:pick_samples,:]\n",
    "    \n",
    "    print(np.array(X_train).shape)\n",
    "    \n",
    "    # Input data and LSTM internal structure\n",
    "    training_data_count = len(X_train)  # 7352 training series (with 50% overlap between each serie)\n",
    "    test_data_count = len(X_test)  # 2947 testing series\n",
    "    n_height = 1\n",
    "    n_steps = len(X_train[0])  # 128 timesteps per series\n",
    "    n_input = len(X_train[0][0])\n",
    "    n_hidden = 32\n",
    "    n_classes = 6 # Total classes (should go up, or should go down)\n",
    "\n",
    "    # Training parameters\n",
    "    learning_rate = 0.0025\n",
    "    lambda_loss_amount = 0.0015\n",
    "    training_iters = training_data_count * 200  # Loop 200 times on the dataset\n",
    "    batch_size = 1500\n",
    "    display_iter = 300000  # To show test set accuracy during training\n",
    "    \n",
    "    # Graph input/output\n",
    "    x = tf.placeholder(tf.float32, [None, n_steps, n_input])\n",
    "    y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "\n",
    "    weights = {\n",
    "        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0))\n",
    "    }\n",
    "    biases = {\n",
    "        'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "    }\n",
    "\n",
    "    pred = cnn_LSTM_net(x,weights,biases)\n",
    "\n",
    "    # Loss, optimizer and evaluation\n",
    "    l2 = lambda_loss_amount * sum(\n",
    "        tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()\n",
    "    ) # L2 loss prevents this overkill neural network to overfit the data\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=pred)) + l2# Softmax loss\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer\n",
    "\n",
    "    correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "    # To keep track of training's performance\n",
    "    test_losses = []\n",
    "    test_accuracies = []\n",
    "    train_losses = []\n",
    "    train_accuracies = []\n",
    "\n",
    "    # Launch the graph\n",
    "    print(\"start training\")\n",
    "    start_time = time.time()\n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    # Perform Training steps with \"batch_size\" amount of example data at each loop\n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters:\n",
    "        batch_xs =         extract_batch_size(X_train, step, batch_size)\n",
    "        batch_ys = one_hot(extract_batch_size(y_train, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer, cost, accuracy],\n",
    "            feed_dict={\n",
    "                x: batch_xs, \n",
    "                y: batch_ys\n",
    "            }\n",
    "        )\n",
    "        train_losses.append(loss)\n",
    "        train_accuracies.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost, accuracy], \n",
    "            feed_dict={\n",
    "                x: X_test,\n",
    "                y: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses.append(loss2)\n",
    "        test_accuracies.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time) + \" seconds\")\n",
    "\n",
    "    one_hot_predictions, accuracy, final_loss = sess.run(\n",
    "        [pred, accuracy, cost],\n",
    "        feed_dict={\n",
    "            x: X_test,\n",
    "            y: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss) + \\\n",
    "          \", Accuracy = {}\".format(accuracy))\n",
    "\n",
    "    \n",
    "    pick_samples_1 = int(round((np.array(X_for_train).shape[0])*0.5))\n",
    "    X_train_1 = X_for_train[0:pick_samples_1, :, :]\n",
    "    y_train_1 = y_for_train[0:pick_samples_1,:]\n",
    "    \n",
    "    # feature extration\n",
    "    feature_tensor = sess.graph.get_tensor_by_name('conv2_layer/Relu:0')\n",
    "    features = sess.run(feature_tensor, feed_dict={x:X_train_1})\n",
    "    feature_shape = features.shape\n",
    "    test_features = sess.run(feature_tensor,feed_dict={x:X_test})\n",
    "    test_feature_shape = test_features.shape\n",
    "    reformatted_features = np.zeros((feature_shape[0], feature_shape[1]*feature_shape[2]))\n",
    "    test_reformatted_features = np.zeros((test_feature_shape[0], test_feature_shape[1]*test_feature_shape[2]))\n",
    "    c = 0\n",
    "    for row in features:\n",
    "        reformatted_features[c,:] = row.flatten()\n",
    "        c += 1\n",
    "    d = 0\n",
    "    for row in test_features:\n",
    "        test_reformatted_features[d,:] = row.flatten()\n",
    "        d += 1\n",
    "    \n",
    "    # classify with svm\n",
    "    from sklearn.svm import SVC\n",
    "    svm_starttime = time.time()\n",
    "    clf = SVC()\n",
    "    clf.fit(reformatted_features, y_train_1)\n",
    "    print(\"-------------SVM(OVR)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_1 = clf.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_1)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnnlstm_svm_ovr_conf1.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnnlstm_svm_ovr_conf1.png\", bbox_inches='tight')\n",
    "    \n",
    "    \n",
    "    svm2_starttime = time.time()\n",
    "    clf2 = SVC(decision_function_shape='ovo')\n",
    "    clf2.fit(reformatted_features, y_train_1)\n",
    "    print(\"-------------SVM(OVO)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm2_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf2.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_2 = clf2.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnnlstm_svm_ovo_conf1.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnnlstm_svm_ovo_conf1.png\", bbox_inches='tight')\n",
    "    \n",
    "    start_time_lstm = time.time()\n",
    "    print(\"-------------lstm-----------------\")\n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    learning_rate_lstm = 0.0025\n",
    "    \n",
    "    weights2 = {\n",
    "        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0))\n",
    "    }\n",
    "    biases2 = {\n",
    "        'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "    }\n",
    "    \n",
    "    x_feat = tf.placeholder(tf.float32, [None, feature_shape[1], feature_shape[2]])\n",
    "    y2 = tf.placeholder(tf.float32, [None, n_classes])\n",
    "    pred_lstm = lstm_net(x_feat, weights2, biases2)\n",
    "    cost2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y2, logits=pred_lstm))# Softmax loss\n",
    "    optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate_lstm).minimize(cost2)\n",
    "    correct_pred2 = tf.equal(tf.argmax(pred_lstm,1), tf.argmax(y2,1))\n",
    "    accuracy2 = tf.reduce_mean(tf.cast(correct_pred2, tf.float32))\n",
    "    \n",
    "    test_losses2 = []\n",
    "    test_accuracies2 = []\n",
    "    train_losses2 = []\n",
    "    train_accuracies2 = []\n",
    "    training_iters2 = training_data_count * 100  # Loop 100 times on the dataset\n",
    "    \n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    \n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters2:\n",
    "        batch_xxs =         extract_batch_size(features, step, batch_size)\n",
    "        batch_yys = one_hot(extract_batch_size(y_train_1, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer2, cost2, accuracy2],\n",
    "            feed_dict={\n",
    "                x_feat: batch_xxs, \n",
    "                y2: batch_yys\n",
    "            }\n",
    "        )\n",
    "        train_losses2.append(loss)\n",
    "        train_accuracies2.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost2, accuracy2], \n",
    "            feed_dict={\n",
    "                x_feat: test_features,\n",
    "                y2: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses2.append(loss2)\n",
    "        test_accuracies2.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters2):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time_lstm) + \" seconds\")\n",
    "\n",
    "    one_hot_predictions2, accuracy2, final_loss2 = sess.run(\n",
    "        [pred_lstm, accuracy2, cost2],\n",
    "        feed_dict={\n",
    "            x_feat: test_features,\n",
    "            y2: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL LSTM RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss2) + \\\n",
    "          \", Accuracy = {}\".format(accuracy2))\n",
    "    \n",
    "    predictions2 = one_hot_predictions2.argmax(1)\n",
    "    \n",
    "    print(\"Testing Accuracy: {}%\".format(100*accuracy2))\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, predictions2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, predictions2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnnlstm_feat_lstm_train_conf1.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnnlstm_feat_lstm_train_conf1.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read in the data\n",
      "(7352, 1)\n",
      "(7352, 128, 9)\n",
      "start training\n",
      "Training iter #1500:   Batch Loss = 3.004152, Accuracy = 0.23066666722297668\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 2.982813835144043, Accuracy = 0.21750932931900024\n",
      "Training iter #300000:   Batch Loss = 0.878896, Accuracy = 0.9293333292007446\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 1.0750219821929932, Accuracy = 0.8062436580657959\n",
      "Training iter #600000:   Batch Loss = 0.557821, Accuracy = 0.8573333621025085\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.661785900592804, Accuracy = 0.8581608533859253\n",
      "Training iter #900000:   Batch Loss = 0.330677, Accuracy = 0.9526666402816772\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.5054173469543457, Accuracy = 0.8808958530426025\n",
      "Training iter #1200000:   Batch Loss = 0.278224, Accuracy = 0.9539999961853027\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.4370726943016052, Accuracy = 0.8893790245056152\n",
      "Training iter #1500000:   Batch Loss = 0.310223, Accuracy = 0.9393333196640015\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.4038749039173126, Accuracy = 0.8910756707191467\n",
      "Training iter #1800000:   Batch Loss = 0.205200, Accuracy = 0.9853333234786987\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.38511598110198975, Accuracy = 0.8937903046607971\n",
      "Optimization Finished!\n",
      "Training time is: 902.053948879242 seconds\n",
      "FINAL CNN RESULT: Batch Loss = 0.3840663433074951, Accuracy = 0.8941296339035034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\sklearn\\utils\\validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVR)-------------\n",
      "Training time is: 189.09908866882324 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.5873769935527655\n",
      "\n",
      "Precision: 65.33044800918265%\n",
      "Recall: 58.737699355276554%\n",
      "f1_score: 49.75906649475647%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[493   0   3   0   0   0]\n",
      " [471   0   0   0   0   0]\n",
      " [247   0 173   0   0   0]\n",
      " [  0   0   0  23 468   0]\n",
      " [  0   0   0   0 532   0]\n",
      " [  3  24   0   0   0 510]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.728876    0.          0.10179844  0.          0.          0.        ]\n",
      " [15.982355    0.          0.          0.          0.          0.        ]\n",
      " [ 8.381405    0.          5.8703766   0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.7804547  15.880556    0.        ]\n",
      " [ 0.          0.          0.          0.         18.052256    0.        ]\n",
      " [ 0.10179844  0.8143875   0.          0.          0.         17.305735  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz8AAAM+CAYAAAA0Cb/mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4JVV97//3pxscEARlUAYFNAQBEcQWFRPFaAwanBK9gEPEaFBzNRE1TvBTNCqJETWKxouKiAOgieSniEGSXIIiKN3KqCBzhGZqmlEQoc/3/lG14+ZwhurmnD57U+/X8+yHvatW1Vq19vFxf/v7rVWpKiRJkiTp/m7RQg9AkiRJktYGgx9JkiRJvWDwI0mSJKkXDH4kSZIk9YLBjyRJkqReMPiRJEmS1AsGP5IkSZJ6weBHkiRJUi8Y/EiSJEnqBYMfSZIkSb2wzkIPQJIkSdKa2yupFQs9iGksg5Oqaq+FHseAwY8kSZI0xlYASxd6ENMIbLLQYxhm2ZskSZKkXjDzI0mSJI27xSOa01g1sdAjuIcRnSVJkiRJmlsGP5IkSZJ6wbI3SZIkaZwFWJyFHsXUVi30AO7JzI8kSZKkXjD4kSRJktQLlr1JkiRJYy2ju9rbiNW9jeosSZIkSdKcMviRJEmS1AuWvUmSJEnjLMA6I7ra24gx8yNJkiSpFwx+JEmSJPWCZW+SJEnSOAsjvNrbaHGWJEmSJPWCwY8kSZKkXrDsTZIkSRp3i13trQuDH0mSJEkLIsmRwN7AdVX1+HbbccD2bZONgJuqatcpjr0cuBVYBdxdVUtm68/gR5IkSdJCOQo4HDh6sKGq9hm8T3IYcPMMxz+rqlZ07czgR5IkSRpnydiu9lZVpybZZqp9SQL8L+AP5qq/8ZwlSZIkSfd3vw9cW1UXTbO/gO8lWZbkgC4nNPMjSZIkab5skmTp0OcjquqIjsfuBxwzw/6nV9XyJJsBJye5oKpOnemEBj+SJEnSOBvth5yu6LIQwWRJ1gH+BHjSdG2qann73+uSHA/sDswY/IzsLEmSJEnqrecAF1TVlVPtTPKQJBsM3gPPBc6b7aQGP5IkSZIWRJJjgNOB7ZNcmeS17a59mVTylmSLJCe2Hx8B/CDJ2cCPge9U1b/N2l9Vzd3oJUmSJK1VSx6wuJY+Yv2FHsaUcuUty9ak7G2+mPmRJEmS1AsGP5IkSZJ6wdXeJEmSpHE2xg85XducJUmSJEm9YPAjSZIkqRcse5MkSZLG3eIs9AjGgpkfSZIkSb1g8CNJkiSpFyx7kyRJksZZgHXMaXThLEmSJEnqBYMfSZIkSb1g2ZskSZI0zhJXe+vIzI8kSZKkXjD4kSRJktQLlr1JkiRJ426xOY0unCVJkiRJvWDwI0mSJKkXLHuTJEmSxlmw7K0jZ0mSJElSLxj8SJIkSeoFy94kSZKkceZDTjsz8yNJkiSpFwx+JEmSJPWCZW+SJEnSuHO1t06cJUmSJEm9YPAjSZIkqRcse5MkSZLGWXC1t47M/EiSJEnqBYMfSZIkSb1g2ZskSZI01uJqbx05S5IkSZJ6weBHkiRJUi9Y9iZJkiSNM1d768zMjyRJkqReMPiRJEmS1AuWvUmSJEnjLLjaW0fOkiRJkqReMPiRJEmS1AuWvUmSJEnjztXeOjHzI0mSJKkXDH4kSXMmyYOTfDvJzUm+cR/O84ok35vLsS2UJL+f5MKFHockybI3SeqlJC8H3go8DrgVOAv4UFX94D6e+qXAI4CNq+ruNT1JVX0V+Op9HMu8S1LAdlV18XRtqur7wPZrb1SSeidxtbeOnCVJ6pkkbwU+AXyYJlB5NPAZ4EVzcPqtgV/cl8Dn/iSJ/8goSSPE4EeSeiTJhsAHgP9dVd+sql9V1V1V9e2q+pu2zQOTfCLJ8vb1iSQPbPftmeTKJG9Lcl2Sq5O8pt33fuC9wD5Jbkvy2iSHJPnKUP/bJKlBUJBk/ySXJrk1yWVJXjG0/QdDx+2R5My2nO7MJHsM7Tslyd8mOa09z/eSbDLN9Q/G/46h8b84yfOT/CLJyiTvGWq/e5LTk9zUtj08yQPafae2zc5ur3efofO/M8k1wBcH29pjHtv2sVv7eYskK5LseZ++WElSJwY/ktQvTwMeBBw/Q5uDgKcCuwK7ALsDBw/tfySwIbAl8Frg00keVlXvo8kmHVdV61fVF2YaSJKHAJ8EnldVGwB70JTfTW73cOA7bduNgY8B30my8VCzlwOvATYDHgC8fYauH0kzB1vSBGufA14JPAn4feC9SR7Ttl0FHAhsQjN3zwb+EqCqntG22aW93uOGzv9wmizYAcMdV9UlwDuBryZZD/gicFRVnTLDeCVpdosXjeZrxIzeiCRJ82ljYMUsZWmvAD5QVddV1fXA+4FXDe2/q91/V1WdCNzGmt/TMgE8PsmDq+rqqjp/ijZ/DFxUVV+uqrur6hjgAuAFQ22+WFW/qKo7gK/TBG7TuYvm/qa7gGNpApt/rKpb2/7PB54AUFXLquqMtt/Lgf8DPLPDNb2vqu5sx3MPVfU54CLgR8DmNMGmJGktMPiRpH65AdhklntRtgCuGPp8Rbvtf84xKXi6HVh/dQdSVb8C9gHeAFyd5DtJHtdhPIMxbTn0+ZrVGM8NVbWqfT8ITq4d2n/H4Pgkv5vkhCTXJLmFJrM1ZUndkOur6teztPkc8HjgU1V15yxtJUlzxOBHkvrldODXwItnaLOcpmRr4NHttjXxK2C9oc+PHN5ZVSdV1R/SZEAuoAkKZhvPYExXreGYVsc/0Yxru6p6KPAeYLYnCdZMO5OsT7PgxBeAQ9qyPklac6F5yOkovkaMwY8k9UhV3Uxzn8un2xv910uybpLnJflI2+wY4OAkm7YLB7wX+Mp055zFWcAzkjy6XWzh3YMdSR6R5IXtvT930pTPrZriHCcCv5vk5UnWSbIPsCNwwhqOaXVsANwC3NZmpd44af+1wGPuddTM/hFYVlWvo7mX6bP3eZSSpE4MfiSpZ6rqYzTP+DkYuB74JfAm4F/bJh8ElgLnAOcCP2m3rUlfJwPHtedaxj0DlkXA22gyOytp7qX5yynOcQOwd9v2BuAdwN5VtWJNxrSa3k6zmMKtNFmp4ybtPwT4Ursa3P+a7WRJXgTsRVPqB833sNtglTtJ0vxK1YzZeUmSJEkjbMnDHlxL99x2oYcxpfzrz5dV1ZKFHseAmR9JkiRJvWDwI0mSJKkXZlrqVJIkSdI4GMGV1UaRmR9JkiRJvWDwI0mzSHJokrcs9Dhmk2SbJDV4gGmS7yZ59Rz3sX+SH8zlOdeGdlntU5PcmuSwBeh/ZOctySlJXjdP5/5YkjfM3lKS1g6DH0maQZJNgT8D/s9Cj2V1VdXzqupLa6u/ycHXGhz/qCRnJFk5OUBJ8m9J7stqQQcAK4CHVtXbpuj7qCSdl/Ne3faznOs+zdt8jWua81+e5Dmrccg/AAclecB8jUkS7UNOF43ma8SM3ogkabTsD5xYVXfM9Ynn4sfu/cy7gS8B2wIvHgQ77UNNL62qpffh3FsDPyuf77BWVdXVwAXACxd6LJIEBj+SNJvnAf81+JBkzyRXJnlbkuuSXJ3kNUP7N0xydJLrk1yR5OAki9p9+yc5LcnHk6wEDpm07aYklybZo93+y7aPVw+d/4+T/DTJLe3+Q6Yb+HA5U5LfSfJfSW5OsiLJcUPtHpfk5DbjcuHwwzqTbJzkW21/PwYeO8Ncndr+96YktyV5WpJF7Rxc0V7L0Uk2nOb4bYH/rKqbgTOBxyR5KPAu4D0z9DsY6x5Jzmyv8cwke7TbjwJeDbyjHddzJh13APCKof3fbrfv0M7hTUnOT/LCWdq/K8klbWndz5K8ZLYxTzdv7fn+PMnPk9yY5KQkW7fb0/69XNde6zlJHj/duKaYpz9MckF77OE0/2Y82PfYJP+Z5Ib27+SrSTZq930ZeDTw7fb872i3fyPJNe35Tk2y06QuTwH+uONcSNK8MviRpJntDFw4adsjgQ2BLYHXAp9O8rB236fafY8BnklTMveaoWOfAlwKbAZ8aGjbOcDGwNeAY4EnA78DvBI4PMn6bdtftefciOYH5RuTvLjDdfwt8D3gYcBW7ThJ8hDg5LbfzYD9gM8M/YD9NPBrYHPgz9vXdJ7R/nejqlq/qk6nyZztDzyrnZP1gcOnOf484A/bH9tLgJ+14/5EVd0008UleTjwHeCTNPP4MeA7STauqv2BrwIfacf178PHVtURk/a/IMm6wLdp5mwz4M3AV5NsP1X79lSXAL9P8/2/H/hKks1nGnfrXvPWfqfvAf4E2BT4PnBM2+657TG/S/N3sA9wwwzjGp6nTYB/AQ4GNmnH/PThJsChwBbADsCjgEPaeXoV8N/AC9rzf6Q95rvAdu08/aQdw7CfA7t0mAdJayppVnsbxdeIMfiRpJltBNw6adtdwAeq6q6qOhG4Ddg+yWKaH6Lvrqpbq+py4DDgVUPHLq+qT1XV3UOldJdV1RerahVwHM0Pzg9U1Z1V9T3gNzSBEFV1SlWdW1UTVXUOzQ/iZ3a4jrtoSr+2qKpfV9Xg5vu9gcvb/u+uqp/Q/Dh+aXs9fwq8t6p+VVXn0ZSlrY5XAB+rqkur6jaa0rZ9M3XJ36E0wcN/0QRd6wJPoMk0fK3NKrxpmn7+GLioqr7cXscxNOVW9woAOnoqTaD2d1X1m6r6T+AEmuBwSlX1japa3n43xwEXAbuvYf+vBw6tqp9X1d3Ah4Fd2+zPXcAGwOOAtG2u7nje59OU//1zVd0FfAK4ZugaLq6qk9u/vetpgsgZ/76q6sj27/1OmkBpl0nZvVtp/nckSQvO4EeSZnYjzQ/NYTe0P0gHbqf5obwJ8ADgiqF9V9BkiAZ+OUUf1w69vwOgqiZvWx8gyVOS/N80ZXU3A29o+53NO2j+Vf/HbQnXIIOzNfCUtrTrpiQ30QQsj6TJOKwzaczD19bFFtx7PtYBHjG5YVWtrKp9qmoX4B9pslNvpil7Ow94DvCGJDt26GfQ15ZTtO067l9W1UTX8yX5syRnDc3j4+n23Uxla+Afh861kub727INxA6nCRCvTXJEWx7YxRYMfZ/tPVD/8znJZkmOTXJVkluAr8x0DUkWJ/m7ttzvFuDydtfwMRsAM2buJGltMfiRpJmdQ1Ne1MUKfpthGXg0cNXQ5/t6w/3XgG8Bj6qqDYHPMnTPxnSq6pqq+ouq2oImq/CZJL9D88P3v6pqo6HX+lX1RuB64G6aTNTw9UzbzRTblnPv+bibewZ8UzkAOKPNNu0MLK2q3wDn0gQVs/Uz6OuqKdpOZfLYlwOPSnu/1hTnu0f7NiPzOeBNwMZVtRFNwNal5mOqefsl8PpJ38uDq+qHAFX1yap6ErATzd/n38xwrmFXM/R9Jgn3/H4Pbc/xhKp6KE3Z5fA1TD7/y4EX0QSmGwLbDE491GYH4OxZxiXpvlroVd1c7U2S7hdOpFtZGW3Z2teBDyXZoP1B/Faafz2fKxsAK6vq10l2p/nxOaskL0uyVfvxRpofsatoSrl+N8mrkqzbvp6cZIf2er5JszDDem3GZabnBl0PTNDc2zNwDHBgkm3b+5Y+DBw3KXM2eaybAf+b9l4T4DLgWe3xS2jumZrsxPY6Xp5knTQrxO3YXl8X104a949o7q96Rzsne9KU0B07TfuH0Mzp9e01vIapg7SpTDVvnwXePbj3Ks1CGi9r3z+5zQCu247x1zTf5VTjmuw7wE5J/qQtPfwrmizfwAY0ZZw3JdmS3wZVA5PPvwFwJ3ADsB7N9zvZM2nuC5KkBWfwI0kzOxp4fpIHd2z/ZpofpJcCP6DJ1Bw5h+P5S+ADSW4F3ksTbHXxZOBHSW6jyRz9dVVdVlW30txAvy9NtuMa4O+BB7bHvYmm5O4a4Cjgi9N1UFW30yzicFpbrvVUmmv/Ms2KZpfR/FB/8yxj/SjNPU+3tZ8PBf6AJhvyramWvK6qG2juX3obzQ/xdwB7V9WKWfoa+AKwYzvuf22zTC+kWe1vBfAZ4M+q6oJp2v+M5v6u02kChJ2B07p0PNW8VdXxNN/DsW052XntWAAeSpNlupGmFO8Gmjm717im6GsF8DLg79rjtps0zvcDuwE30wRK35x0ikOBg9vzv53mfx9X0GTEfgacMdy4XfBhR+BeY5GkhRAfeSBJM0vyYeC6qvrEQo9FGidpHlZ7SVV9ZqHHIt2fLdlkvVq69/YLPYwp5UtnLauq+/KQ6jnlA/YkaRZVNeszZiTdW1W9baHHIEnDLHuTJEmS1AtmfiRJkqSxlpFcWW0UOUuSJEmSesHgR5IkSVIvWPamObXJuotqmwf6Z7UQlv/qCQs9BEmSeuEmLuf2WtHlIcZrR4DFozOcUeavVM2pbR64Dkt3fsRCD6OXDjnjXo8+0Vo0sdjHBiyURav8P3xJa9cRjMzKzVpNlr1JkiRJ6gUzP5IkSdI4C6721pGzJEmSJKkXDH4kSZIk9YJlb5IkSdJY8yGnXTlLkiRJknrB4EeSJElSL1j2JkmSJI2zAIt85lkXZn4kSZIk9YLBjyRJkqResOxNkiRJGneu9taJsyRJkiSpFwx+JEmSJPWCZW+SJEnSOAuw2NXeujDzI0mSJKkXDH4kSZIk9YJlb5IkSdJYi6u9deQsSZIkSeoFgx9JkiRJvWDZmyRJkjTOXO2tMzM/kiRJknrB4EeSJElSL1j2JkmSJI27ReY0unCWJEmSJPWCwY8kSZKkXjD4kSRJksZZ0qz2NoqvWYeeI5Ncl+S8oW2HJLkqyVnt6/nTHLtXkguTXJzkXV2myuBHkiRJ0kI5Cthriu0fr6pd29eJk3cmWQx8GngesCOwX5IdZ+vM4EeSJEnSgqiqU4GVa3Do7sDFVXVpVf0GOBZ40WwHudqbJEmSNM4CLL7f5TTelOTPgKXA26rqxkn7twR+OfT5SuAps530fjdLkiRJkkbGJkmWDr0O6HDMPwGPBXYFrgYOm6LNVDcU1WwnNvMjSZIkab6sqKolq3NAVV07eJ/kc8AJUzS7EnjU0OetgOWzndvgR5IkSRp3HVZWGxdJNq+qq9uPLwHOm6LZmcB2SbYFrgL2BV4+27kNfiRJkiQtiCTHAHvSlMddCbwP2DPJrjRlbJcDr2/bbgF8vqqeX1V3J3kTcBKwGDiyqs6frT+DH0mSJEkLoqr2m2LzF6Zpuxx4/tDnE4F7LYM9E4MfSZIkaZwlsMh1zLpwliRJkiT1gsGPJEmSpF6w7E2SJEkad/ej1d7mk5kfSZIkSb1g8CNJkiSpFyx7kyRJksZZgMXmNLpwlhZQko8necvQ55OSfH7o82FJ3tq+PzDJr5NsOLR/zyQnTHHeU5Isad9vk+SiJH803D7J/kkmkjxh6LjzkmzTvl8/yT8luSTJT5MsS/IXcz8LkiRJ0tph8LOwfgjsAZBkEbAJsNPQ/j2A09r3+wFnAi/pevIkW9E89fZtVXXSFE2uBA6a5vDPAzcC21XVE4G9gId37VuSJEkaNQY/C+s02uCHJug5D7g1ycOSPBDYAfhpkscC6wMH0wRBXTwS+B5wcFV9a5o2JwA7Jdl+eGPb3+7tsRMAVXV9Vf1990uTJEnSWrMoo/kaMQY/C6iqlgN3J3k0TRB0OvAj4GnAEuCcqvoNTcBzDPB9YPskm3U4/dHA4VX1jRnaTAAfAd4zaftOwNmDwEeSJEm6PzD4WXiD7M8g+Dl96PMP2zb7Ase2wcg3gZd1OO+/A69Kst4s7b4GPDXJttM1SHJQkrOSLJ9m/wFJliZZev1dxkuSJEkaTQY/C29w38/ONGVvZ9BkfvYATmsXJNgOODnJ5TSBUJfSt4/QZJG+kWTaVf2q6m7gMOCdQ5t/BuzS3odEVX2oqnYFHjrNOY6oqiVVtWTTdf2TkiRJWquSZrW3UXyNmNEbUf+cBuwNrKyqVVW1EtiIJgA6nSbQOaSqtmlfWwBbJtm6w7kPBG4BvpBkpqLLo4DnAJsCVNXFwFLgg0kWAyR5EM1CipIkSdJYMvhZeOfSrPJ2xqRtN1fVCppMz/GTjjm+3Q7w7CRXDr2eNmhUVQW8GticJhM0pfa+ok8Cw/cSvQ7YGLg4yTKaMrp3TnG4JEmSNBZ8yOkCq6pVTConq6r9h97f616cqnrr0McHT3HaPYfa/gZ47tC+U9rtR9FkfAbtPkkTAA0+3wK8vsMlSJIkaaGN4Mpqo8jMjyRJkqReMPiRJEmS1AsGP5IkSZJ6wXt+JEmSpHEWRnJZ6VHkLEmSJEnqBYMfSZIkSb1g2ZskSZI01uJS1x2Z+ZEkSZLUCwY/kiRJknrBsjdJkiRpnLnaW2fOkiRJkqReMPiRJEmS1AuWvUmSJEnjztXeOjHzI0mSJKkXDH4kSZIk9YJlb5IkSdI4S1ztrSNnSZIkSVIvGPxIkiRJ6gXL3iRJkqRx52pvnZj5kSRJktQLBj+SJEmSesGyN0mSJGmcBVd768hZkiRJktQLBj+SJEmSesGyN0mSJGmsxdXeOjLzI0mSJKkXDH4kSZIk9YJlb5IkSdI4C7DInEYXzpIkSZKkXjD4kSRJktQLlr1JkiRJ426xq711YeZHkiRJUi8Y/EiSJEnqBcveJEmSpHGWuNpbR86SJEmSpF4w+JEkSZLUC5a9SZIkSeNukau9dWHmR5IkSVIvGPxIkiRJ6gXL3iRJkqRxFnzIaUdmfiRJkiT1gsGPJEmSpF6w7E1z64HrwLYPW+hR9NMZCz2Aflu0ynIDSdIC8iGnnThLkiRJknrB4EeSJElSL1j2JkmSJI2zhAkfctqJmR9JkiRJvWDwI0mSJKkXLHuTJEmSxlgBE6721omzJEmSJKkXDH4kSZIk9YJlb5IkSdKYc7W3bsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTGKmHVYnMaXThLkiRJknrB4EeSJElSL1j2JkmSJI05V3vrxsyPJEmSpF4w+JEkSZLUC5a9SZIkSeMsUIvMaXThLEmSJEnqBYMfSZIkSb1g2ZskSZI0xgpXe+vKzI8kSZKkXjD4kSRJktQLlr1JkiRJ4yyx7K0jMz+SJEmSesHgR5IkSVIvWPYmSZIkjbFmtTdzGl04S5IkSZJ6weBHkiRJUi9Y9iZJkiSNOVd768bMjyRJkqReMPiRJEmS1AuWvUmSJEljrBJWxZxGF86SJEmSpF4w+JEkSZLUC5a9SZIkSWPO1d66MfMjSZIkqRcMfiRJkiT1gmVvkiRJ0piz7K0bMz+SJEmSFkSSI5Ncl+S8oW3/kOSCJOckOT7JRtMce3mSc5OclWRpl/4MfiRJkiQtlKOAvSZtOxl4fFU9AfgF8O4Zjn9WVe1aVUu6dGbZmyRJkjTGKlCLxjOnUVWnJtlm0rbvDX08A3jpXPU3nrMkSZIkaRxskmTp0OuA1Tz+z4HvTrOvgO8lWdb1vGZ+JEmSJM2XFV1L0iZLchBwN/DVaZo8vaqWJ9kMODnJBVV16kznNPiRJEmSxlrud6u9JXk1sDfw7KqqqdpU1fL2v9clOR7YHZgx+BmLsrckH0/ylqHPJyX5/NDnw5K8tX1/YJJfJ9lwaP+eSU6Y4rynJFnSvt8myUVJ/mi4fZL9k0wkecLQcecNahOTrJ/kn5JckuSnbdrtL2a4lnuNJclRSV46NKYLk5yd5LQk27fb927Pf3aSnyV5fZKD2tUtzkqyauj9Xw2d++wkx3Ts78wkuw61+/N2BY1z2mt+0XTXJUmSJM2FJHsB7wReWFW3T9PmIUk2GLwHngucN1XbYWMR/AA/BPYASLII2ATYaWj/HsBp7fv9gDOBl3Q9eZKtgJOAt1XVSVM0uRI4aJrDPw/cCGxXVU+kWa3i4V37nsYrqmoX4EvAPyRZFzgCeEG7/YnAKVX1oXZ1i12BOwbvq+qT7XXtQPMdP6P9o5itv88A/9Aeu1V7zb/XrrTxVOCc+3hdkiRJ0v9o/5H+dGD7JFcmeS1wOLABTSnbWUk+27bdIsmJ7aGPAH6Q5Gzgx8B3qurfZutvXMreTgM+3r7fiSaq2zzJw4DbgR2AnyZ5LLA+8DfAe2iWzpvNI4GjgYOr6lvTtDmBJoDYvqouHGxs+9sdeHlVTQBU1fXA36/e5U3rVOAtNF/+OsANbR93AhfOcNzAy4Ev08zPC4FjZm7O6TRzB7AZcCtwW9vnbYP3kiRJGiGBifFd7W2/KTZ/YZq2y4Hnt+8vBXZZ3f7GYpbaC707yaNpsjynAz8CngYsAc6pqt/QZH2OAb5PEz1u1uH0RwOHV9U3ZmgzAXyEJqAathNw9iDwmQcvAM6tqpXAt4ArkhyT5BVtBmw2+wDH0czJVH9Yk+0F/Gv7/mzgWuCyJF9M8oLpDkpywGAFj+t/fXeHbiRJkqS1byyCn9ZpNIHPIPg5fejzD9s2+wLHtsHIN4GXdTjvvwOvSrLeLO2+Bjw1ybbTNRi6B2f5DOeZ8oatSdu/muQs4OnA2wGq6nXAs2nSem8HjpxpsEmeDFxfVVcA/wHs1mbKpvLVJFfS1FZ+qu1vFU0w9FKah0t9PMkhUw686oiqWlJVSzZ90LgkEyVJktQ34xT8DO772Zmm7O0MmszPHsBp7YIE29HUBl5OEwh1yXZ8hCaL9I0k0/5yr6q7gcNoAoSBnwG7DLIwg3twgIfO0N8NwOQg5OHAiqHPr2jv3XlxVf1yaAznVtXHgT8E/nSW69oPeFw7F5e0Y5rumFcA29IEeJ8e6q+q6sdVdSjNfM7WpyRJktayAiaSkXyNmnEKfk6jWe5uZVWtakvBNqIJgE6n+bF/SFVt0762ALZMsnWHcx8I3AJ8IZnxWzoKeA6wKUBVXQwsBT6YZDFAkgcBM53jImCLdjEC2vHtApw13QHtinJ7Dm3aFbhihvaLaLJeTxjMB/AiZggGq+ou4GCa7NYO7Q1lu3XtU5IkSRp14xT8nEuzytsZk7bdXFUraDITx0865vh2O8Cz2xUkBq+nDRq1a4e/GticJhM0pfa+ok/SLAYw8DpgY+DiJMtoyujeOcXhg3PcCbwS+GJb2vbPwOuq6uZpr7wJpt7RLkl9FvB+YP8Z2j8DuKqqrhradiqwY5LNZxjbHTTZrbcD6wIfTXJB2+c+wF8362jpAAAgAElEQVTP0KckSZI00jLNM4OkNbJk4/Vq6R/9zkIPo5cOOcaVyCVJWhuOYAnLa+nI1HTt/PjN6/h/ee1CD2NK2z3uQ8uqaslCj2NgnDI/kiRJkrTGXJprniTZmeYZO8PurKqnLMR4JEmSpL4z+JknVXUuzSIBkiRJ0rypZGwfcrq2OUuSJEmSesHgR5IkSVIvWPYmSZIkjblVI/hA0VFk5keSJElSLxj8SJIkSeoFy94kSZKkMVbgam8dOUuSJEmSesHgR5IkSVIvWPYmSZIkjbVQrvbWiZkfSZIkSb1g8CNJkiSpFyx7kyRJksZZYGKRZW9dmPmRJEmS1AsGP5IkSZJ6wbI3SZIkaYwVMBFzGl04S5IkSZJ6weBHkiRJUi9Y9iZJkiSNOVd768bMjyRJkqReMPiRJEmS1AuWvUmSJEnjLGEilr11YeZHkiRJUi8Y/EiSJEnqBcveJEmSpDFWwKpF5jS6cJYkSZIk9YLBjyRJkqResOxNkiRJGnOu9taNmR9JkiRJvWDwI0mSJKkXLHuTJEmSxlhh2VtXZn4kSZIk9YLBjyRJkqResOxNkiRJGmcJ5UNOO3GWJEmSJPWCwY8kSZKkXrDsTZIkSRpzrvbWjZkfSZIkSb1g8CNJkiSpFyx705z6720fxZu/8o8LPYxe2viYhR5Bvx1z4WULPYTe2m/7bRd6CJK0oHzIaXdmfiRJkiT1gsGPJEmSpF6w7E2SJEkac5a9dWPmR5IkSVIvGPxIkiRJ6gXL3iRJkqQxVgkTMafRhbMkSZIkqRcMfiRJkiT1gmVvkiRJ0phztbduzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMYKWLXIsrcuzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNM58yGlnzpIkSZKkXjD4kSRJktQLlr1JkiRJY658yGknZn4kSZIk9YLBjyRJkqResOxNkiRJGmMFTGDZWxdmfiRJkiT1gsGPJEmSpF6w7E2SJEkacxOu9taJmR9JkiRJvWDwI0mSJKkXLHuTJEmSxlqYiDmNLpwlSZIkSb1g8CNJkiSpFyx7kyRJksZY4WpvXZn5kSRJktQLBj+SJEmSesGyN0mSJGmcBVZZ9taJmR9JkiRJvWDwI0mSJKkXLHuTJEmSxpirvXVn5keSJElSLxj8SJIkSeoFy94kSZKksRYmzGl04ixJkiRJ6gWDH0mSJEm9YNmbJEmSNObK1d46MfMjSZIkqRcMfiRJkiT1wrwFP0k+nuQtQ59PSvL5oc+HJXlr+/7AJL9OsuHQ/j2TnDDFeU9JsqR9v02Si5L80XD7JPsnmUjyhKHjzkuyTft+/ST/lOSSJD9NsizJX8xwLdskuaNt+/MkP07y6kltXpzknCQXJDk3yYvb7bskOWuo3X5Jbk+ybvt55yTnDF3b0qG2S5Kc0r5fL8lX23Ofl+QHSbZOclb7uibJVUOfH9Ae95IkleRxk67nvKF5vrm9tguSfHSo3SOSnJDk7CQ/S3LidHMkSZKkhTF4yOkovkbNfGZ+fgjsAZBkEbAJsNPQ/j2A09r3+wFnAi/pevIkWwEnAW+rqpOmaHIlcNA0h38euBHYrqqeCOwFPHyWLi+pqidW1Q7AvsCBSV7TjmUX4KPAi6rqccALgY+2wde5wNZJNmjPswdwAfDEoc+nDfWzWZLnTdH/XwPXVtXOVfV44LXANVW1a1XtCnwW+Pjgc1X9pj1uP+AH7Zin8/12Hp4I7J3k6e32DwAnV9UuVbUj8K5Z5kiSJEkaWfMZ/JxGG/zQBD3nAbcmeViSBwI7AD9N8lhgfeBgmh/qXTwS+B5wcFV9a5o2JwA7Jdl+eGPb3+7tsRMAVXV9Vf191wurqkuBtwJ/1W56O/Dhqrqs3X8ZcCjwN20fZwJPads+Cfg0v52bPWgCxYF/oJmLyTYHrhoaw4VVdedM40yyPvB0mkBppuBncM47gLOALYf6vHJo/zmznUOSJEkaVfMW/FTVcuDuJI+m+YF/OvAj4GnAEuCcNjuxH3AM8H1g+ySbdTj90cDhVfWNGdpMAB8B3jNp+07A2YPA5z74CTAoJdsJWDZp/1J+m+n6IbBHkoe04zqFewY/w5mf04E7kzxr0vmOBN6Z5PQkH0yyXYcxvhj4t6r6BbAyyW4zNU7yMGA74NR206eBLyT5v0kOSrLFNMcdkGRpkqV3XH9zh2FJkiRpLk2QkXyNmvle8GCQ/RkEP6cPfR5kO/YFjm2DkW8CL+tw3n8HXpVkvVnafQ14apJtp2vQ/qg/K8nyDv3e49BJ72uK/YNtg3nYHTizqi4BfifJpsD6bSZp2AeZlP2pqrOAx9Bkhh4OnJlkh1nGuB9wbPv+WKbPrP1+e9/RNcAJVXVN2+dJbZ+fown0ftqO+R6q6oiqWlJVSx686YaTd0uSJEkjYb6Dn8F9PzvTlL2dQZP52QM4rb0nZjvg5CSX0wRCXUrfPkKTRfpGkmmfVVRVdwOHAe8c2vwzYJf2PiSq6kPtPTMPXb1L44nAz9v359Nks4bt1vYFzXU/Gfg9mgAQmnKyfblnydtg3P8JPAh46qTtt1XVN6vqL4GvAM+fbnBJNgb+APh8O7d/A+yTTHnn2fer6gk039Mbk+w61OfKqvpaVb2KpnzvGdP1KUmSJK2OJEcmuW6wGFe77eFJTm4XNju5rU6a6thXt20umrwY2XTWRuZnb2BlVa2qqpXARjQB0Ok0gc4hVbVN+9oC2DLJ1h3OfSBwC01Z1kw5taOA5wCbAlTVxTQlaR9MshggyYOge16uXTXuo8Cn2k0fBd49tJrcNjTldoe1fd4K/BLYn98GP6cDb2GK4Kf1IeAdQ30+ffDFtyu57QhcMcMwXwocXVVbt3P7KOAymgBsSm153KG0wWKSPxhk19oFGx4L/PcMfUqSJGktK8JEFo3kq4OjaBYfG/Yu4D+qajvgP5hi0a0kDwfeR3Nf/e7A+6YLkobNd/BzLs0qb2dM2nZzVa2gyXwcP+mY4/ntzfnPTnLl0Otpg0ZVVcCraW7K/8h0A2jvK/okMHwv0euAjYGLkyyjKaN75xSHD3vsYKlr4OvAp6rqi20fZ7XHfzvJBcC3gXe02wdOAx5YVb9sP59OU1I2ZfBTVScC1w/3D/xXknOBn9IEcP8yw3j3495z+y/Ay2e5zs8Cz2hLBZ8ELG1L4k4HPl9VZ85yvCRJktRJVZ0KrJy0+UXAl9r3X6K5j32yP6JZlXhlVd0InMy9g6h7SRNDSHNjsyXb1z4//uxCD6OXNl48eY0MrU3HXHjZQg+ht/bbftrbOiVpXhzBEpbX0pG5m3+bJz2m3vujv13oYUzpteu+cllVTb495B7aqqkT2se5kOSmqtpoaP+NVfWwSce8HXhQVX2w/fz/AXdU1UeZwbT3y0iSJEkaD6O4slprkyRLhz4fUVVHzMF5p7rgWbM6Bj9DkuwMfHnS5jur6ilTtZckSZI0oxWzZX6mcG2Szavq6iSbA9dN0eZKYM+hz1vRPE5mRgY/Q6rqXGDXWRtKkiRJmi/form3/+/a//7/U7Q5Cfjw0CIHzwXePduJDX4kSZKkMVaBiRkXPx5dSY6hyeBskuRKmhXc/g74epLX0qw0/LK27RLgDVX1uqpameRvaR7FAvCBdmXpGRn8SJIkSVoQVTXdMz6fPUXbpTSrNg8+HwkcuTr9zfdS15IkSZI0Esz8SJIkSWNu1eiu9jZSzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMaKjO1qb2ubmR9JkiRJvWDwI0mSJKkXLHuTJEmSxly52lsnZn4kSZIk9YLBjyRJkqResOxNkiRJGnMTMafRhbMkSZIkqRcMfiRJkiT1wrRlb0keOtOBVXXL3A9HkiRJ0uooYMLV3jqZ6Z6f82nmcngmB58LePQ8jkuSJEmS5tS0wU9VPWptDkSSJEmS5lOn1d6S7As8pqo+nGQr4BFVtWx+hyZJkiRpdrHsraNZFzxIcjjwLOBV7abbgc/O56AkSZIkaa51yfzsUVW7JfkpQFWtTPKAeR6XJEmSJM2pLsHPXUkW0SxyQJKNgYl5HZUkSZKkzix766bLc34+DfwLsGmS9wM/AP5+XkclSZIkSXNs1sxPVR2dZBnwnHbTy6rqvPkdliRJkiTNrU6rvQGLgbtoSt+6ZIskSZIkrQUFrIplb110We3tIOAYYAtgK+BrSd493wOTJEmSpLnUJfPzSuBJVXU7QJIPAcuAQ+dzYJIkSZI0l7oEP1dMarcOcOn8DEeSJEnS6nK1t26mDX6SfJymhPB24PwkJ7Wfn0uz4pskSZIkjY2ZMj+DFd3OB74ztP2M+RuOJEmSJM2PaYOfqvrC2hyIJEmSpNVXhAkXZO5k1nt+kjwW+BCwI/Cgwfaq+t15HJckSZIkzakuIeJRwBeBAM8Dvg4cO49jkiRJkqQ51yX4Wa+qTgKoqkuq6mDgWfM7LEmSJEldFRnJ16jpstT1nUkCXJLkDcBVwGbzOyxJkiRJmltdgp8DgfWBv6K592dD4M/nc1CSJEmSNNdmDX6q6kft21uBV83vcCRJkiStLh9y2s1MDzk9nuahplOqqj+ZlxFJkiRJ0jyYKfNz+Fobhe431lm2ARsvdj0M9c9+22+70EPoreseM+2/02kt+Mweuy70EHrrkK+cvdBDkMbOTA85/Y+1ORBJkiRJq6+w7K0rHwUrSZIkqRcMfiRJkiT1QpelrgFI8sCqunM+ByNJkiRp9Vn21s2smZ8kuyc5F7io/bxLkk/N+8gkSZIkaQ51KXv7JLA3cANAVZ0NuJyXJEmSpLHSpextUVVdkdwjlbZqnsYjSZIkaTUUYZVlb510CX5+mWR3oJIsBt4M/GJ+hyVJkiRJc6tL2dsbgbcCjwauBZ7abpMkSZKksTFr5qeqrgP2XQtjkSRJkrQGyrK3TmYNfpJ8jubBsfdQVQfMy4gkSZIkaR50uefn34fePwh4CfDL+RmOJEmSJM2PLmVvxw1/TvJl4OR5G5EkSZKk1eJDTrvpsuDBZNsCW8/1QCRJkiRpPnW55+dGfnvPzyJgJfCu+RyUJEmSJM21GYOfNE823QW4qt00UVX3WvxAkiRJ0sIoYFVZ9tbFjGVvbaBzfFWtal8GPpIkSZLGUpd7fn6cZLd5H4kkSZIkzaNpy96SrFNVdwO/B/xFkkuAXwGhSQoZEEmSJEkjwNXeupnpnp8fA7sBL15LY5EkSZKkeTNT8BOAqrpkLY1FkiRJkubNTMHPpkneOt3OqvrYPIxHkiRJ0mooQln21slMwc9iYH1wJiVJkiSNv5mCn6ur6gNrbSSSJEmSNI9mvedHkiRJ0mib6PQEG800S89ea6OQJEmSpHk2bfBTVSvX5kAkSZIkaT7NVPYmSZIkaQxMlHesdGFxoCRJkqReMPiRJEmS1AuWvUmSJEljrIBVLtTciZkfSZIkSb1g8CNJkiSpFyx7kyRJksZaKFd768TMjyRJkqReMPiRJEmS1AuWvUmSJEljrIAJV3vrxMyPJEmSpF4w+JEkSZLUC5a9SZIkSeOsYJWrvXVi5keSJElSLxj8SJIkSeoFy94kSZKkMeZqb92Z+ZEkSZLUCwY/IyzJQUnOT3JOkrOSPCXJKUmWJPlRu+2/k1zfvj8rybXTbN8myeVJNmnPXUkOG+rr7UkOGfr8yrbf85OcneTzSTZagGmQJEmS5oRlbyMqydOAvYHdqurONmh5wGB/VT2lbbc/sKSq3jTp+HttT+6RDr0T+JMkh1bViknH7gUcCDyvqq5Kshh4NfAI4KY5u0hJkiTNiXK1t07M/IyuzYEVVXUnQFWtqKrlc3j+u4EjaIKcyQ4C3l5VV7V9r6qqI6vqwjnsX5IkSVqrDH5G1/eARyX5RZLPJHnmPPTxaeAVSTactH0n4CddT5LkgCRLkyy9nevndICSJEnSXDH4GVFVdRvwJOAA4HrguLaUbS77uAU4Gvir6dok2bm9Z+iSJPtMc54jqmpJVS1Zj03ncoiSJEmaVZgY0deoMfgZYW252SlV9T7gTcCfzkM3nwBeCzxkaNv5wG7tGM6tql2B7wIPnof+JUmSpLXC4GdEJdk+yXZDm3YFrpjrfqpqJfB1mgBo4FDgo0m2Gtpm4CNJkqSx5mpvo2t94FPt8tJ3AxfTlMD98zz0dRhNZgmAqjoxyabAd9uV3m4CzgNOmoe+JUmSdB8UMOFqb50Y/IyoqloG7DHFrj0ntTsKOGqK4++1vaq2GXq//tD7a4H1JrX9EvCl1Ru1JEmSNLose5MkSZLUC2Z+JEmSpDG3yrK3Tsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTmagQfKDqKzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMZ8yGl3Zn4kSZIkLYgk2yc5a+h1S5K3TGqzZ5Kbh9q8d037M/MjSZIkaUFU1YXArgBJFgNXAcdP0fT7VbX3fe3P4EeSJEkaZ5X7y0NOnw1cUlVXzFcHlr1JkiRJGgX7AsdMs+9pSc5O8t0kO61pB2Z+JEmSJM2XTZIsHfp8RFUdMblRkgcALwTePcU5fgJsXVW3JXk+8K/AdmsyGIMfSZIkaYw1q70t9CimtaKqlnRo9zzgJ1V17eQdVXXL0PsTk3wmySZVtWJ1B2PZmyRJkqSFth/TlLwleWSStO93p4lhbliTTsz8SJIkSVowSdYD/hB4/dC2NwBU1WeBlwJvTHI3cAewb1WtUa7L4EeSJEkaczXGq71V1e3AxpO2fXbo/eHA4XPRl2VvkiRJknrB4EeSJElSL1j2JkmSJI2xZrW38S17W5vM/EiSJEnqBYMfSZIkSb1g2ZskSZI05iaw7K0LMz+SJEmSesHgR5IkSVIvWPYmSZIkjbECVrnaWydmfiRJkiT1gsGPJEmSpF6w7E2SJEkaZxXKsrdOzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNOYmJix768LMjyRJkqReMPiRJEmS1AuWvUmSJEljzIecdmfmR5IkSVIvGPxIkiRJ6gXL3iRJkqRxVjBh2VsnZn4kSZIk9YLBjyRJkqResOxNkiRJGnNl2VsnZn4kSZIk9YKZH0nSWNvsUv+1c0Fdes5Cj0CSOjPzI0mSJKkXzPxIkiRJY6yIS113ZOZHkiT9v/buO9yysrz7+Pc3owgqJYogUkQQG0VBmvAGC8ZoghrbC2NiRFGjiQUNqLESIjYQY8UM6ouaxC4GNSqogCJFB0IVBVRQigFsFBvM3O8fax/ZHs8Mm2H2Xmef5/u5rn3NXuXs9TuLYWbu89zreSSpCRY/kiRJkppg25skSZI05VZU3wmmgyM/kiRJkppg8SNJkiSpCba9SZIkSVOsCpavcLa3UTjyI0mSJKkJFj+SJEmSmmDbmyRJkjTlykVOR+LIjyRJkqQmWPxIkiRJaoJtb5IkSdKUW2Hb20gc+ZEkSZLUBIsfSZIkSU2w7U2SJEmaYoWLnI7KkR9JkiRJTbD4kSRJktQE294kSZKkaVZxtrcROfIjSZIkqQkWP5IkSZKaYNubJEmSNMUKqBV9p5gOjvxIkiRJaoLFjyRJkqQm2PYmSZIkTTlnexuNIz+SJEmSmmDxI0mSJKkJtr1JkiRJ06xgxQrb3kbhyI8kSZKkJlj8SJIkSWqCbW+SJEnSFCtgubO9jcSRH0mSJElNsPiRJEmS1ATb3iRJkqQpV872NhJHfiRJkiQ1weJHkiRJUhNse5MkSZKmWAErqu8U08GRH0mSJElNsPiRJEmS1ATb3iRJkqRpVmG5s72NxJEfSZIkSU2w+JEkSZLUBIufCUny6iQXJDk3ydlJThz8ekmSXw7en51kj8H590hyU5K/m/U5lyb59ND2U5McM3i/f5JrkvxPkouTfHnm8wbHj0ny1MH7k5IsGzq2c5KThrZ3HZxzcZKzknwhyfbjuj+SJElaPQWsWJF5+ZpvfOZnApI8DNgH2KmqfptkQ2CtqroyySOAg6pqn1lf9jTgdGAJ8G+zju2cZNuqumCOy328ql44uO4jgc8keWRVXTjHuRsleVxVfXFW3o2BTwBPr6pTB/v+D7A1cN5t+NYlSZKkecORn8nYBLi2qn4LUFXXVtWVt/I1S4B/BDZLsumsY0cAr7q1i1bVicBS4HkrOeVw4DVz7H8h8KGZwmfwWadU1Wdv7ZqSJEnSfGXxMxnHA5snuSjJe5M8fFUnJ9kcuGdVfYtuBGbfWad8AtgpyX1HuPZZwANWcuw04LeDEaJh2w6+TpIkSVOgKvPyNd9Y/ExAVd0APJRuBOYa4ONJ9l/Fl+xHV+AAfIxuFGjYcrpRm38a4fK39rvuDcw9+nPLByRnJLkwyTtWcvx5SZYlWfYrrhkhkiRJkjR5Fj8TUlXLq+qkqno9XVvZU1Zx+hJg/ySXAscBD06yzaxzPgLsBWxxK5feEZjreZ+ZXF8D1gZ2H9p9AbDT0Dm7Aa8F1l/JZyytqp2rauc7c49biSNJkiT1w+JnApLcf1bx8hDgspWdC9ylqjatqi2rakvgTXSjQb9XVTcBbwcOXMV1H0432nT0rUQ8DHj50PZ76IqvPYb23flWPkOSJEl9KFixYn6+5htne5uMuwLvSrIBcDNwCSufhGAJcOysfZ+ma3/7l1n7P8Aft6ztO5iZ7c7AD4GnrGSmt9+rqv9Ocs3Q9k+S7Au8ZTDZwtXAtcChq/ocSZIkaT6z+JmAqjoT2GMlx04CThraPmSOc84FHjR4v+XQ/t8C9xraPgY4ZhU59h96/4hZxx46a/t0YJUTM0iSJEnTxOJHkiRJmmIzi5zq1vnMjyRJkqQmWPxIkiRJaoJtb5IkSdI0K1hu29tIHPmRJEmS1ASLH0mSJElNsO1NkiRJmmJFnO1tRI78SJIkSWqCxY8kSZKkJtj2JkmSJE25WtF3gungyI8kSZKkJlj8SJIkSWqCbW+SJEnSNCtYXs72NgpHfiRJkiQ1weJHkiRJUhNse5MkSZKmWIGLnI7IkR9JkiRJTbD4kSRJktQE294kSZKkKbfCRU5H4siPJEmSpCZY/EiSJEnqTZJLk5yX5Owky+Y4niTvTHJJknOT7LS617LtTZIkSZpmBTX9s709sqquXcmxxwHbDF67AUcNfr3NHPmRJEmSNJ89EfhwdU4HNkiyyep8kMWPJEmSpD4VcHySM5M8b47jmwI/Htq+fLDvNrPtTZIkSZpi83yR0w1nPceztKqWzjpnz6q6MslGwAlJvltVXx86Ptc3V6sTxuJHkiRJ0rhcW1U7r+qEqrpy8OvVSY4FdgWGi5/Lgc2HtjcDrlydMLa9SZIkSepFkrskWXfmPfAY4PxZpx0H/O1g1rfdgV9W1VWrcz1HfiRJkqRpVrB8ehc53Rg4Ngl0tcl/VtWXkjwfoKreB/w38BfAJcCvgGet7sUsfiRJkiT1oqp+ADx4jv3vG3pfwD+sievZ9iZJkiSpCY78SJIkSVOsyHye7W1eceRHkiRJUhMsfiRJkiQ1wbY3SZIkaZoV1HLb3kbhyI8kSZKkJlj8SJIkSWqCbW+SJEnSFCumepHTiXLkR5IkSVITLH4kSZIkNcG2N0mSJGnKucjpaBz5kSRJktQEix9JkiRJTbDtTWvcisXVd4QmXXvvvhO0baMf2G6gNh2Cf+b35ZB7b9B3hGZ9/qob+o7whwpWONvbSBz5kSRJktQEix9JkiRJTbDtTZIkSZpymaezvc23xlhHfiRJkiQ1weJHkiRJUhNse5MkSZKmWcHi5fOz7e3mvgPM4siPJEmSpCZY/EiSJElqgm1vkiRJ0hQLsMhFTkfiyI8kSZKkJlj8SJIkSWqCbW+SJEnSNKuwaJ4ucjrfOPIjSZIkqQkWP5IkSZKaYNubJEmSNOWyvO8E08GRH0mSJElNsPiRJEmS1ATb3iRJkqQploLFzvY2Ekd+JEmSJDXB4keSJElSE2x7kyRJkqbcohV9J5gOjvxIkiRJaoLFjyRJkqQm2PYmSZIkTbEULFrubG+jcORHkiRJUhMsfiRJkiQ1wbY3SZIkacrFRU5H4siPJEmSpCZY/EiSJElqgm1vkiRJ0hRLweLlfaeYDo78SJIkSWqCxY8kSZKkJtj2JkmSJE21sMjZ3kbiyI8kSZKkJlj8SJIkSWqCbW+SJEnSNCtY5GxvI3HkR5IkSVITLH4kSZIkNcG2N0mSJGmKBYizvY3EkR9JkiRJTbD4kSRJktQE294kSZKkaVaw2NneRuLIjyRJkqQmWPxMiSQ3rOLYOUk+OrT9vCQfH9peL8n3k9wnyTFJnjrYf1KSZUPn7ZzkpKHtXQfnXJzkrCRfSLL9Gv/mJEmSpAmw+JlySR5I999xryR3Gew+GtgsyaMH24cCH6yqH87xERsledwcn7sx8AngVVW1TVXtBLwJ2HqNfxOSJElabQEWrZifr/nG4mf6PR34CHA88ASAqirgBcC/JtkZ2Bs4fCVffzjwmjn2vxD4UFWdOrOjqk6pqs+uweySJEnSxFj8TL99gY8DHwWWzOysqnOBLwNfBV5cVb9bydefBvw2ySNn7d8WOGvNx5UkSZL6YfEzxZLsAlxTVZfRFTk7JfmToVPeA1xRVSfeyke9gblHf4avdUaSC5O8Y45jz0uyLMmyX3HNbfwuJEmSdLsULFqeefmabyx+ptsS4AFJLgW+D6wHPGXo+IrBa5Wq6mvA2sDuQ7svAHYaOmc34LXA+nN8/dKq2rmqdr4z91iNb0OSJEkaP4ufKZVkEfA0YIeq2rKqtgSeyFDr2210GPDyoe33APsn2WNo351X87MlSZKk3rnI6fS4c5LLh7aPpGtpu2Jo39eBByXZpKquui0fXlX/neSaoe2fJNkXeEuSTYGrgWvpZo6TJEnSPJJ5OLPafGTxMyWqaq5RuiNnnbMc2GRo+1Jgu1nn7D/0/hGzjj101vbpwMNXM7IkSZI0r9j2JkmSJKkJjvxIkiRJUywFi+fhzGrzkWLlKcsAACAASURBVCM/kiRJkppg8SNJkiSpCba9SZIkSVNu0fK+E0wHR34kSZIkNcHiR5IkSVITbHuTJEmSplgKFq1wtrdROPIjSZIkqQkWP5IkSZKaYNubJEmSNOXibG8jceRHkiRJUhMsfiRJkiQ1wbY3SZIkaZpVWLzc2d5G4ciPJEmSpCZY/EiSJElqgm1vkiRJ0hRLwSJnexuJIz+SJEmSmmDxI0mSJKkJtr1JkiRJU27Rir4TTAdHfiRJkiQ1weJHkiRJUhNse5MkSZKmWUFc5HQkjvxIkiRJaoLFjyRJkqQm2PYmSZIkTbEAi13kdCSO/EiSJElqgsWPJEmSpCbY9iZJkiRNs4JFtr2NxJEfSZIkSU2w+JEkSZLUBNveJEmSpCkWYNGULnKaZHPgw8A9gRXA0qp6x6xzHgH8F/DDwa7PVNWhq3M9ix9JkiRJfbkZ+MeqOivJusCZSU6oqu/MOu8bVbXP7b2YbW+SJEmSelFVV1XVWYP31wMXApuO63qO/EiSJEnTrCAr+g5x+yXZEtgROGOOww9Lcg5wJXBQVV2wOtew+JEkSZI0LhsmWTa0vbSqls4+KcldgU8DB1bVdbMOnwXcu6puSPIXwGeBbVYnjMWPJEmSpHG5tqp2XtUJSe5IV/j8R1V9Zvbx4WKoqv47yXuTbFhV197WMBY/kiRJ0hQLsHhKFzlNEuADwIVVdeRKzrkn8L9VVUl2pZu34Kercz2LH0mSJEl92RN4BnBekrMH+14FbAFQVe8Dngq8IMnNwK+B/aqqVudiFj+SJEmSelFVp9ANXq3qnHcD714T17P4kSRJkqZZTe8ip5PmOj+SJEmSmmDxI0mSJKkJtr1pjbqKM6/9l+WLLus7x+2wIXCbp02cF37Qd4DbbXrv/cLg/e+P974/U33v/3ma/7btTPP9v3ffAf5AwaIpne1t0ix+tEZV1T36znB7JFl2a3PRazy89/3y/vfHe98f732/vP/qg21vkiRJkprgyI8kSZI0xYJtb6Ny5Ef6Q0v7DtAw732/vP/98d73x3vfL++/Ji6ruTiqJEmSpHlgvQ0fWrvsc3rfMeb0tQ+tdeZ8erbLtjdJkiRpmrnI6chse5MkSZLUBIsfSZIkSU2w7U3SxCV5PHBuVV022H4d8BTgMuAlVfXDPvO1JMndgb2AH1XVmX3nWeiSrAdsXFUXD7afBqwzOPzlqvrf3sJJmlrO9jY6R37UpCQHJDl4aPuKJNcluT7JC/rM1ojDgGsAkuwD/A3wbOA44H095lrwknw+yXaD95sA59Pd+48kObDXcG04AthzaPtNwC50Beg/95KoEUm2TfKEoe23J/ng4LVTn9la4P3XfGHxo1Y9H/jg0PbVVbUecA9gST+RmlJV9avB+ycDH6iqM6vq/XT/DTQ+96mq8wfvnwWcUFWPB3ajK4I0XrsAHxravr6qXlRVzwG26ylTK94MXDu0/efAF4ATgdf1kqgt3n/NC7a9qVWLquqnQ9ufBKiq3yRZZyVfozUnSe4K/ArYG3jv0LG1+4nUjJuG3u8NHA1QVdcnWdFPpKbcof5wjYlnDL3fYNJhGrNJVZ06tH1dVX0aIMnf9ZSpJd7/cSrb3kZl8aNWrT+8UVVvBEiyCLh7L4na8q/A2cB1wIVVtQwgyY7AVX0Ga8CPk7wIuBzYCfgSwKDov2OfwRqxIsk9q+onADOjcEk2BSw+x2vd4Y2q2n1oc6MJZ2mR91/zgm1vatXxSd4wx/5DgeMnHaY1VfVB4OHAAcBfDB36CbB/H5kacgCwLd193reqfjHYvzvw//oK1ZDDgc8l2SvJuoPXw4HPDo5pfK5MstvsnUl2B67sIU9rvP+aFxz5UasOBt6f5BLgnMG+BwPLgOf0lqohVXUFcMWs3esBBwHPnXyiNlTV1XTPvM3ef2KSH/QQqSlV9e9JrgXeQFeEQjfpxOuq6ov9JWvCK4CPJzkGOGuw76HAM4F9+wrVEO//ONn2NjKLHzWpqm4EliTZilv+AfKdqvp+j7GakWQHulmv7kX3E+930T33sxvwth6jNSHJw4BNga9X1dWD/x6vBP4U2LzXcA2oqi8xaDfU5FTVtwajDP/ALSPMFwC7O8X4+Hn/NV9Y/KhJSbYYvL2ZW0Z+fr+/qn7UR66GHA0cBZwGPJbup4D/Cfx1Vf2mz2ALXZLDgX3onrl6RZLPA38PvBFnexu7wZpWK1NV9S8TC9OgwT+ynVmsJ95/zQcWP2rVF4CiWxdsRtFNs7wRsLiPUA25U1UdM3j/vSQHAa+sKgftx+8vgR0HMxv+CV2v/Q4zi25q7G6cY99d6J7Fujtg8TMmSU6k+3N+LlVVe08yT2u8/+MVwqLlufUTZfGjNlXV9sPbSbak60d+NN1PwDVeaw9mdpv5k/oGYIckAaiqs1b6lbq9fj0zulZVP0/yPQufyamq37d1JlkXeAndeksfw5bPcTtojn27Ay8Hrp5wlhZ5/zUvWPyoaUm2AV7NLc+avLiqblr1V2kN+Alw5Eq2C3jUxBO1Y+skxw1tbzm8XVVPmONrtAYluRvwMuCv6RY83amqft5vqoWvqs6ceT+YYe+1wJ2A5zvZxPh5/zVfWPyoSUm2oyt6tgXeChxgy9XkVNUj+s7QsCfO2na0YYIGz1w9GVgKbF9VN/QcqSlJ/pzuH92/AQ6rqhN7jtQU7/8YOdvbyPKHC01LbUiyHPgx3bM/f/THRVW9eOKhGpLkyas6XlWfmVQWaZKSrAB+SzfZyvBfwKF77mG9XoI1IMm36Z7rPJxuspU/YLvteHn/x2uDDXauh+91Rt8x5nTc5+5wZlXt3HeOGY78qFUHsPIHLzV+j1/FsQIsfsYkyXnM/Xt/5h/fO0w4UlOqysXF+3Mj3fOFTx28htluO37ef80LFj9q0tBMY+pBVT1rZceSbDzJLA3ap+8ALRs877NSVfWzSWVpje22/fL+j1dsexuZxY+alORzrGLkx4e+JyvJ+sBTgKcDD6RbgFNjUFWXzbU/yZ509/8fJpuoOWfyx9Pszyhgq8nGaYfttv3y/mu+sPhRq47oO0DrkqwDPIHuH9w7AesCfwV8vc9cLUnyELr7/3+BH2K74SQ8YmUFqMbOdtt+ef81L1j8qFVrVdUJcx1I8hbg5AnnaUqS/wD2Ao4H3g18Dbikqk7qM1cLktwP2A9YAvwU+Djd5DeP7DVYO46lK/Y1Yatqt9VEHGLhP162vY3GBy/Vqvck+cvhHUkWJTkGeHA/kZqyHfBz4ELgu4Npxp2AYjK+C+wNPL6q/k9VvYs5ZjzU2LgEe4+S3D/J25J8YfA6YvADAY3fV5O8Mok/eFev/A2oVj0G+FKSO1XVZwYtWJ8ErmPVQ/NaA6rqwUkeQNdy9ZUkVwPrJrlnVf2k53gL3VPoRn5OTPIl4GP4D/JJ2jTJO1d20Gn2xyfJw+haq5YOXgF2BE5K8uSqOr3PfA3YETgUODPJi6rKFmf1wuJHTaqqS5M8Gvhyko2AZwBnVNXLeo7WjKr6LvA64HVJdqZrw/pWksurao9+0y1cVXUscGySu9A9Y/VSYOMkRwHHVtXxvQZc+H5NN+mBJu91wJJZ7bWfTfI14PXA43pJ1Yiquh54aZKH0o0CXQ6swGn214hutjd/jjUKFzlVk5LM9NxvAnwYOAF468xxF1sbryQvrKp3z7E/wF5V5TNXY5LkDlV186x9dwOeBuxbVa61MUZJzqoqn/npQZKLqmrOFrck36uq+086U2uSPAp4B/Bl4D10xQ+w8pkoNZq7rbdz7b3bt/qOMadPfWWxi5xK88Dbht6fC2w8tM/F1sbv2XQTHfyB6n4aY+EzXt9i1gP3g7Vl/m3w0nht0neAhl2/imM3TixFo5J8jG4Zg6dX1Xl951G7LH7UpFXNbJVk90lmkSbMvoh++UxbfzZfyfNWwbXFJuGrVXX0XAeSbFxV/zvpQAuNs72NxuJH+mOfALboO8QCt0OS6+bYP9P7vd6kAzXkHklW+mxbVR05yTANste8Pwev4tiyiaVo1OzCx8Wt1ReLH+mP+ZPx8TuvqnbsO0SjFgN3xd/nfdnM2d76UVUf6jtD61zcWvOBxY/0x/zJrBayq6rq0L5DNMzZ3nqS5P+x8j/fq6oOmGSe1ri49ZiVbW+jsvhRk5J8jrn/Egxw9wnHadEn+w7QMEd8+vVTRyB68/k59m0BHEg3Iqrx+qPFrZP4w0ZNnMWPWnXEah7TmnFNkm2q6uLB9NYfpOv9vhTY36nGx+qJSe5YVTdBt+I98BfAZVX1mX6jNeF3fQdoVVV9euZ9kq2AV9GNRLwZ+EBfuVrh4taaLyx+1KSVrSOTZHNgP5xuedxeAhwzeL8E2AG4D90K4O8A/rSfWE34d+AA4OIk9wVOA/4D2CfJLlX1T72mW/j+YWidsT9i4T9eSR4IvJruz5rDgefPXvdK4+Pi1uMT295GZvGj5iXZkG6BxyV0s80c22+iJtw8M/IA7AN8uKp+SvfTwLeu4ut0+/1JVV08eP9M4KNV9aIka9E9i2LxM15H0LXczrQfzm77cY2xMUnySWBnuv8GLwWWA+t1g8+/X+9KE1JVy4BlSQ6m+4GYNBEWP2pSknWBJ9ENv9+PruDZqqo26zVYO1Yk2YSu/3tv4LChY+v0E6kZw//YfhTdT7+pqt8lWTH3l2gNegXw46q6CiDJM7ml5fOQ/mI1YRe63/8HAf842DdchG7VR6jWVdWKJC8F3t53FrXB4ketuppupfvXAKdUVSV5Us+ZWvI6unU1FgPHVdUFAEkeDvygz2ANODfJEcAVwH3pZl4iyQa9pmrH+4BHAyTZC3gT8CLgIcBS4Kn9RVvYqmrLvjNopZyIZQ2w7W00i/oOIPXkVcDawFHAPyXZuuc8TamqzwP3Bh5YVc8dOrQM2LefVM14LnAtsCXwmKr61WD/g3Cyj0lYPNRetS+wtKo+XVWvpStGNUFJtk7y6iTn952lcc76pomx+FGTqurtVbUb3WJrAT4L3CvJK5Lcr990C1+SbYBPAd9I8tEkmwJU1Y1VdUO/6Ra2qvp1Vb25ql5SVecM7T+1qj7SZ7ZGLE4y03WxN91aJzPsxpiAJJskOTDJt4AL6O77kp5jLXhJrk9y3Ryv64F79Z1P7fAPWjUpyYHAKcDZVXUYcFiS7en+Avwi4EjQeH0Q+DDdqt5PAN4FPLnXRI1IciKrXuhx70nmadBHgZOTXEu34Ok3AAYz7/2yz2ALXZLn0v0ZvxnwCeA5wH9V1T/3GqwRVbVu3xkWsm62N7sHR2Hxo1ZtBrwTeECSc4FTgW8CR1TVq3pN1oZ1q+rowfvDkzi97+QcNMe+3YGX0z0LpzGqqsOSfBXYBDi+qmYK0UV0z/5ofN5DN7X70wczjeEim1J7LH7UpKo6CGAwve/OwB7As4Gjk/yiqh7UZ74GrJ1kR255yHWd4W3XOhmfqjpz5v1ggonXAneiW+/ki70Fa0hVnT7Hvov6yNKYe9Eta3Bkko3pRn/u2G8kSZNm8aPWrQOsB6w/eF0JnNdrojb8BDhyJduFa52MVZI/pyt6fgMcVlUn9hxJGruqupZukpujkmxGt6D11UkuBI511F/TztneRmPxoyYlWQpsC1wPnEHX9nZkVf2812CNqKpH9J2hVUm+DdyDbn2f0wb7dpo57qibFqoku8+MulXV5XSzGx6R5P50hZCkBlj8qFVb0LX6XEy33snlwC96TdSQJLMnNyi66ZfPrqrre4jUkhuBG+jWk5m9poyjblrI3gvsNHtnVX0PcNIDqREWP2pSVT02SehGf/agW+17uyQ/A06rqtf3GnDhe/wc++4G7JDkgKr62hzHtQY46iZJC1DZ9jYqix81azDL0vlJfkE3xewvgX2AXQGLnzGqqmfNtT/JvekeQt5tsonakeQcumneTwW+WVWX9ptImpitkhy3soNV9YRJhpHUD4sfNSnJi+lGfPYEbqKb5vo0uvVnnPCgJ1V1WRJnXxqvv6b7vf9nwOuT3IWuEDoVOLWqzugznDRG1wBv6zuEpH5Z/KhVWwKfAl5aVVf1nEUDgwePf9t3joWsqs4HzgeWAiTZkO5h7wPpHgBf3F86aaxuqKqT+w4hjUNsexuZxY+aVFUv6ztDy5J8ju7h+mF3o1v48W8mn6gdSRYDO3LLyOfWdJN+vJ/B7G/SAvXzJPesqp8AJPlb4CnAZcAhVfWzXtNJmgiLH0l9OGLWdgE/BS6uqt/1kKcl1wEX0q12/8qq+mHPeaRJ2QD4HUCSvYA3Ay8CHkI3Ejp79kNJC5DFj6SJG7X1JMlpVfWwcedpzHOAhw1+fdZg3Z/T6GY5vKLXZNJ4LRoa3dkXWFpVnwY+neTsHnNJa4Rtb6Ox+JE0n63dd4CFpqo+CnwUIMmd6WY33BN4U5K1qurefeaTxugOSe5QVTcDewPPGz7WUyZJE+b/7JLms9nPBWkNGMzwthu3PPezC/BjulkPpYXqo8DJSa4Ffg18AyDJfemWOpDUAIsfSWpIkv8BtgCW0U1v/Tbg9Kq6oddg0phV1WFJvko3scrxg7XeABbRPfsjTS1nexudxY+k+Sx9B1iAngmcN/QPP6kZVXX6HPsu6iOLpH4s6juAJK3CM/oOsNBU1bnAtkk+lGRZkm8P3u/QdzZJksbN4kfSxCU5IMnBQ9tXJLkuyfVJXjCzf7Agp9agJE8EjgVOBp5NN+vbyXQzXj2xz2ySpNVUsOjm+fmab2x7k9SH5wOPHdq+uqo2TbI2cDxwVD+xmnAo8GdVdenQvnOSfA34r8FLkqQFyZEfSX1YVFU/Hdr+JEBV/QZYp59IzbjjrMIHgMG+O048jSRJE+TIj6Q+rD+8UVVvBEiyCLh7L4nacVOSLarqR8M7k9wbmIcNCpKkUSxa7hxBo3DkR1Ifjk/yhjn2H0rX9qbxeT3wlST7J9k+yXZJnkV331/XczZJksbKkR9JfTgYeH+SS4BzBvseTLf2zHN6S9WAqvpskh8C/0i3tkmAC4D/W1XnrPKLJUmachY/kiauqm4EliTZCth2sPs7VfX9HmM1Y1Dk/G3fOSRJa4aLnI7OtjdJE5dkiyRb0D1jcs7gddPQfo1RkmcmOTPJjYPXsiQWQ5KkBc+RH0l9+AJQdC1XMwq4B7ARsLiPUC0YFDkHAi8DzqL7b7ATcHgSqurDfeaTJGmcLH4kTVxVbT+8nWRL4BXAo4E39hCpJX8PPGnWdNdfS/IU4GOAxY8kTSHb3kZj25uk3iTZJskxwBeBM4EHVdW7+k214K23inV+1pt4GkmSJsiRH0kTl2Q74NV0kx28FTigqvyZ1WT8ejWPSZI09Sx+JPXhHODHdM/+7Arsmtzy+E9VvbinXC14YJJz59gfYKtJh5Ek3X7O9jY6ix9JfTiAboIDTd4D+w4gSVJfLH4kTVxVHdN3hlZV1WWjnJfktKp62LjzSJI0SRY/kiYuyedYxchPVT1hgnE0t7X7DiBJGpFtbyOz+JHUhyP6DqBbZVuiJGnBsfiR1Ie1quqEuQ4keQtw8oTzSJKkBlj8SOrDe5K8tKq+MLMjySLgg8A9+4ulIbn1UyRJ84Vtb6Ox+JHUh8cAX0pyp6r6TJJ1gE8C1wGP7zeaBp7RdwBJkta0RX0HkNSeqroUeDTwL0meD3wFuKiqnl5VN/UaboFLckCSg4e2r0hyXZLrk7xgZn9Vnd9PQkmSxseRH0kTl2SnwduXAx8GTgD+fWZ/VZ3VV7YGPB947ND21VW1aZK1geOBo/qJJUlaXS5yOjqLH0l9eNvQ+3OBjYf2FfCoiSdqx6Kq+unQ9icBquo3g/ZDSZIWLIsfSRNXVY9c2bEku08yS4PWH96oqjfC7yecuHsviSRJmhCLH0nzzSeALfoOsYAdn+QNVfWaWfsPpWt7kyRNm4JFN/cdYjpY/Eiab5xiebwOBt6f5BLgnMG+BwPLgOf0lkqSpAmw+JE031TfARayqroRWJJkK2Dbwe7vVNX3e4wlSdJEWPxImrgkn2PuIif43MlYJZlpKbyZW0Z+fr+/qn7URy5J0uoLzvY2KosfSX04YjWP6fb7Al3hOdxeWMA9gI2AxX2EkiRpEix+JE1cVZ081/4kmwP7AXMe1+1XVdsPbyfZEngF3aKzb+whkiRJE2PxI6lXSTYEngYsATYFju03URuSbAO8GtiNbo2lF1fVTf2mkiStFhc5HZnFj6SJS7Iu8CTg6cD96Aqerapqs16DNSDJdnRFz7bAW4EDqsq/MiVJTbD4kdSHq4FvAa8BTqmqSvKknjO14hzgx3TP/uwK7Jrc8vhPVb24p1ySpAYleSzwDrpnTt9fVW+edfxOwIeBhwI/BfatqktX93oWP5L68Cq6Z3uOAv4zycd7ztOSA3A6cUlacKax7S3JYuA9wJ8BlwPfTnJcVX1n6LQDgJ9X1X2T7Ae8Bdh3da9p8SNp4qrq7cDbB2vNLAE+C9wrySuAY6vqol4DLmBVdUzfGSRJGtgVuKSqfgCQ5GPAE4Hh4ueJwCGD958C3p0kVbVaP8iz+JE0cUkOBE4Bzq6qw4DDkmxPVwh9Edi6z3wL2SrWWAKgqp4wwTiSpLZtSteKPeNyuol45jynqm5O8ku6NQGvXZ0LWvxI6sNmwDuBByQ5FzgV+CZwRFW9qtdkC5/rKEnSAnMVZ375ELJh3zlWYu0ky4a2l1bV0sH7zHH+7B/QjXLOyCx+JE1cVR0EkGQtYGdgD+DZwNFJflFVD+oz3wK3VlWdMNeBJG/BNZYkaepU1WP7zrCaLgc2H9reDLhyJedcnuQOwPrAz1b3gotW9wslaQ1YB1iP7g+y9en+wDuj10QL33uS/OXwjiSLkhwDPLifSJKkRn0b2CbJfQY/EN0POG7WOccBzxy8fyrwtdV93gcc+ZHUgyRL6daZuZ6u2DkVOLKqft5rsDY8BvhSkjtV1WeSrAN8ErgOeHy/0SRJLRk8w/NC4Mt0U11/sKouSHIosKyqjgM+AHwkySV0Iz773Z5r5nYUTpK0WpJ8CdgQOJ+u8DkNOP/2/CRHo0uyGd1fNO8CngGcUVUv6zeVJEnjZ/EjqRfpVtbclu55nz2A7eh+onNaVb2+z2wLWZKdBm83oVs07gTgrTPHq+qsPnJJkjQJFj+SejUYhdiTrgDaB7h7VW3Qb6qFK8mJqzhcVfWoiYWRJGnCLH4kTVySF9MVO3sCN9FNc33a4NfzqmpFj/GalWT3qjq97xySJI2LxY+kiUtyJIO1farqqr7zqJPkR1W1Rd85JEkaF4sfSRIASX5cVZvf+pmSJE0n1/mRJM3wp2GSpAXNdX4kqSFJPsfcRU6Au084jiRJE2XbmyQ1JMnDV3W8qk6eVBZJkibN4keSRJLNgf2q6vC+s0iSNC4+8yNJjUqyYZIXJPk6cBKwcc+RJEkaK5/5kaSGJFkXeBLwdOB+wLHAVlW1Wa/BJEmaANveJKkhSX4NfAt4DXBKVVWSH1TVVj1HkyRp7Gx7k6S2vApYGzgK+KckW/ecR5KkiXHkR5IalGQrYAmwH7AN8Hrg2Kq6qNdgkiSNkcWPJDUkyYHAKcDZVXXzYN/2dIXQvlXlSJAkacGy+JGkhiQ5AtgDeABwLnAq8E3gtKr6WZ/ZJEkaN4sfSWpQkrWAnekKoYcNXr+oqgf1GkySpDFyqmtJatM6wHrA+oPXlcB5vSaSJGnMHPmRpIYkWQpsC1wPnAGcDpxeVT/vNZgkSRPgVNeS1JYtgDsBPwGuAC4HftFrIkmSJsSRH0lqTJLQjf7sMXhtB/yMbtKD1/eZTZKkcbL4kaRGJdkM2JOuANoHuHtVbdBvKkmSxsfiR5IakuTFdMXOnsBNDKa5Hvx6XlWt6DGeJElj5WxvktSWLYFPAS+tqqt6ziJJ0kQ58iNJkiSpCc72JkmSJKkJFj+SJEmSmmDxI0m6TZIsT3J2kvOTfDLJnW/HZz0iyecH75+Q5JWrOHeDJH+/Gtc4JMlBo+6fdc4xSZ56G661ZZLzb2tGSdJkWPxIkm6rX1fVQ6pqO+B3wPOHD6Zzm/9+qarjqurNqzhlA+A2Fz+SJM2w+JEk3R7fAO47GPG4MMl7gbOAzZM8JslpSc4ajBDdFSDJY5N8N8kpwJNnPijJ/knePXi/cZJjk5wzeO0BvBnYejDqdPjgvIOTfDvJuUn+eeizXp3ke0m+Atz/1r6JJM8dfM45ST49azTr0Um+keSiJPsMzl+c5PCha//d7b2RkqTxs/iRJK2WJHcAHgecN9h1f+DDVbUjcCPwGuDRVbUTsAx4WZK1gaOBxwN/CtxzJR//TuDkqnowsBNwAfBK4PuDUaeDkzwG2AbYFXgI8NAkeyV5KLAfsCNdcbXLCN/OZ6pql8H1LgQOGDq2JfBw4C+B9w2+hwOAX1bVLoPPf26S+4xwHUlSj1znR5J0W62T5OzB+28AHwDuBVxWVacP9u8OPAj4ZhKAtegWU30A8MOquhggyb8Dz5vjGo8C/hagqpYDv0zyJ7POeczg9T+D7bvSFUPrAsdW1a8G1zhuhO9puyRvoGutuyvw5aFjnxgs/npxkh8MvofHADsMPQ+0/uDaF41wLUlSTyx+JEm31a+r6iHDOwYFzo3Du4ATqmrJrPMeAqypBeYCvKmq/m3WNQ5cjWscA/xVVZ2TZH/gEUPHZn9WDa79oqoaLpJIsuVtvK4kaYJse5MkjcPpwJ5J7guQ5M5J7gd8F7hPkq0H5y1Zydd/FXjB4GsXJ1kPuJ5uVGfGl4FnDz1LtGmSjYCvA09Ksk6Sdela7G7NusBVSe4I/PWsY09LsmiQeSvge4Nrv2BwPknul+QuI1xHktQjR34kSWtcVV0zGEH5aJI7DXa/pqouSvI84AtJrgVOAbab4yNeAixNcgCwHHhBVZ2W5JuDqaS/OHju54HAaYORpxuAv6mqs5J8HDgbuIyuNe/WvBY4Y3D+efxhSa1M8gAAAG1JREFUkfU94GRgY+D5VfWbJO+nexborHQXvwb4q9HujiSpL6laU90HkiRJkjR/2fYmSZIkqQkWP5IkSZKaYPEjSZIkqQkWP5IkSZKaYPEjSZIkqQkWP5IkSZKaYPEjSZIkqQkWP5IkSZKa8P8BmhhfZqpVwbYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x864 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVO)-------------\n",
      "Training time is: 189.22538781166077 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.5873769935527655\n",
      "\n",
      "Precision: 65.33044800918265%\n",
      "Recall: 58.737699355276554%\n",
      "f1_score: 49.75906649475647%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[493   0   3   0   0   0]\n",
      " [471   0   0   0   0   0]\n",
      " [247   0 173   0   0   0]\n",
      " [  0   0   0  23 468   0]\n",
      " [  0   0   0   0 532   0]\n",
      " [  3  24   0   0   0 510]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.728876    0.          0.10179844  0.          0.          0.        ]\n",
      " [15.982355    0.          0.          0.          0.          0.        ]\n",
      " [ 8.381405    0.          5.8703766   0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.7804547  15.880556    0.        ]\n",
      " [ 0.          0.          0.          0.         18.052256    0.        ]\n",
      " [ 0.10179844  0.8143875   0.          0.          0.         17.305735  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz8AAAM+CAYAAAA0Cb/mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4JVV97//3pxscEARlUAYFNAQBEcQWFRPFaAwanBK9gEPEaFBzNRE1TvBTNCqJETWKxouKiAOgieSniEGSXIIiKN3KqCBzhGZqmlEQoc/3/lG14+ZwhurmnD57U+/X8+yHvatW1Vq19vFxf/v7rVWpKiRJkiTp/m7RQg9AkiRJktYGgx9JkiRJvWDwI0mSJKkXDH4kSZIk9YLBjyRJkqReMPiRJEmS1AsGP5IkSZJ6weBHkiRJUi8Y/EiSJEnqBYMfSZIkSb2wzkIPQJIkSdKa2yupFQs9iGksg5Oqaq+FHseAwY8kSZI0xlYASxd6ENMIbLLQYxhm2ZskSZKkXjDzI0mSJI27xSOa01g1sdAjuIcRnSVJkiRJmlsGP5IkSZJ6wbI3SZIkaZwFWJyFHsXUVi30AO7JzI8kSZKkXjD4kSRJktQLlr1JkiRJYy2ju9rbiNW9jeosSZIkSdKcMviRJEmS1AuWvUmSJEnjLMA6I7ra24gx8yNJkiSpFwx+JEmSJPWCZW+SJEnSOAsjvNrbaHGWJEmSJPWCwY8kSZKkXrDsTZIkSRp3i13trQuDH0mSJEkLIsmRwN7AdVX1+HbbccD2bZONgJuqatcpjr0cuBVYBdxdVUtm68/gR5IkSdJCOQo4HDh6sKGq9hm8T3IYcPMMxz+rqlZ07czgR5IkSRpnydiu9lZVpybZZqp9SQL8L+AP5qq/8ZwlSZIkSfd3vw9cW1UXTbO/gO8lWZbkgC4nNPMjSZIkab5skmTp0OcjquqIjsfuBxwzw/6nV9XyJJsBJye5oKpOnemEBj+SJEnSOBvth5yu6LIQwWRJ1gH+BHjSdG2qann73+uSHA/sDswY/IzsLEmSJEnqrecAF1TVlVPtTPKQJBsM3gPPBc6b7aQGP5IkSZIWRJJjgNOB7ZNcmeS17a59mVTylmSLJCe2Hx8B/CDJ2cCPge9U1b/N2l9Vzd3oJUmSJK1VSx6wuJY+Yv2FHsaUcuUty9ak7G2+mPmRJEmS1AsGP5IkSZJ6wdXeJEmSpHE2xg85XducJUmSJEm9YPAjSZIkqRcse5MkSZLG3eIs9AjGgpkfSZIkSb1g8CNJkiSpFyx7kyRJksZZgHXMaXThLEmSJEnqBYMfSZIkSb1g2ZskSZI0zhJXe+vIzI8kSZKkXjD4kSRJktQLlr1JkiRJ426xOY0unCVJkiRJvWDwI0mSJKkXLHuTJEmSxlmw7K0jZ0mSJElSLxj8SJIkSeoFy94kSZKkceZDTjsz8yNJkiSpFwx+JEmSJPWCZW+SJEnSuHO1t06cJUmSJEm9YPAjSZIkqRcse5MkSZLGWXC1t47M/EiSJEnqBYMfSZIkSb1g2ZskSZI01uJqbx05S5IkSZJ6weBHkiRJUi9Y9iZJkiSNM1d768zMjyRJkqReMPiRJEmS1AuWvUmSJEnjLLjaW0fOkiRJkqReMPiRJEmS1AuWvUmSJEnjztXeOjHzI0mSJKkXDH4kSXMmyYOTfDvJzUm+cR/O84ok35vLsS2UJL+f5MKFHockybI3SeqlJC8H3go8DrgVOAv4UFX94D6e+qXAI4CNq+ruNT1JVX0V+Op9HMu8S1LAdlV18XRtqur7wPZrb1SSeidxtbeOnCVJ6pkkbwU+AXyYJlB5NPAZ4EVzcPqtgV/cl8Dn/iSJ/8goSSPE4EeSeiTJhsAHgP9dVd+sql9V1V1V9e2q+pu2zQOTfCLJ8vb1iSQPbPftmeTKJG9Lcl2Sq5O8pt33fuC9wD5Jbkvy2iSHJPnKUP/bJKlBUJBk/ySXJrk1yWVJXjG0/QdDx+2R5My2nO7MJHsM7Tslyd8mOa09z/eSbDLN9Q/G/46h8b84yfOT/CLJyiTvGWq/e5LTk9zUtj08yQPafae2zc5ur3efofO/M8k1wBcH29pjHtv2sVv7eYskK5LseZ++WElSJwY/ktQvTwMeBBw/Q5uDgKcCuwK7ALsDBw/tfySwIbAl8Frg00keVlXvo8kmHVdV61fVF2YaSJKHAJ8EnldVGwB70JTfTW73cOA7bduNgY8B30my8VCzlwOvATYDHgC8fYauH0kzB1vSBGufA14JPAn4feC9SR7Ttl0FHAhsQjN3zwb+EqCqntG22aW93uOGzv9wmizYAcMdV9UlwDuBryZZD/gicFRVnTLDeCVpdosXjeZrxIzeiCRJ82ljYMUsZWmvAD5QVddV1fXA+4FXDe2/q91/V1WdCNzGmt/TMgE8PsmDq+rqqjp/ijZ/DFxUVV+uqrur6hjgAuAFQ22+WFW/qKo7gK/TBG7TuYvm/qa7gGNpApt/rKpb2/7PB54AUFXLquqMtt/Lgf8DPLPDNb2vqu5sx3MPVfU54CLgR8DmNMGmJGktMPiRpH65AdhklntRtgCuGPp8Rbvtf84xKXi6HVh/dQdSVb8C9gHeAFyd5DtJHtdhPIMxbTn0+ZrVGM8NVbWqfT8ITq4d2n/H4Pgkv5vkhCTXJLmFJrM1ZUndkOur6teztPkc8HjgU1V15yxtJUlzxOBHkvrldODXwItnaLOcpmRr4NHttjXxK2C9oc+PHN5ZVSdV1R/SZEAuoAkKZhvPYExXreGYVsc/0Yxru6p6KPAeYLYnCdZMO5OsT7PgxBeAQ9qyPklac6F5yOkovkaMwY8k9UhV3Uxzn8un2xv910uybpLnJflI2+wY4OAkm7YLB7wX+Mp055zFWcAzkjy6XWzh3YMdSR6R5IXtvT930pTPrZriHCcCv5vk5UnWSbIPsCNwwhqOaXVsANwC3NZmpd44af+1wGPuddTM/hFYVlWvo7mX6bP3eZSSpE4MfiSpZ6rqYzTP+DkYuB74JfAm4F/bJh8ElgLnAOcCP2m3rUlfJwPHtedaxj0DlkXA22gyOytp7qX5yynOcQOwd9v2BuAdwN5VtWJNxrSa3k6zmMKtNFmp4ybtPwT4Ursa3P+a7WRJXgTsRVPqB833sNtglTtJ0vxK1YzZeUmSJEkjbMnDHlxL99x2oYcxpfzrz5dV1ZKFHseAmR9JkiRJvWDwI0mSJKkXZlrqVJIkSdI4GMGV1UaRmR9JkiRJvWDwI0mzSHJokrcs9Dhmk2SbJDV4gGmS7yZ59Rz3sX+SH8zlOdeGdlntU5PcmuSwBeh/ZOctySlJXjdP5/5YkjfM3lKS1g6DH0maQZJNgT8D/s9Cj2V1VdXzqupLa6u/ycHXGhz/qCRnJFk5OUBJ8m9J7stqQQcAK4CHVtXbpuj7qCSdl/Ne3faznOs+zdt8jWua81+e5Dmrccg/AAclecB8jUkS7UNOF43ma8SM3ogkabTsD5xYVXfM9Ynn4sfu/cy7gS8B2wIvHgQ77UNNL62qpffh3FsDPyuf77BWVdXVwAXACxd6LJIEBj+SNJvnAf81+JBkzyRXJnlbkuuSXJ3kNUP7N0xydJLrk1yR5OAki9p9+yc5LcnHk6wEDpm07aYklybZo93+y7aPVw+d/4+T/DTJLe3+Q6Yb+HA5U5LfSfJfSW5OsiLJcUPtHpfk5DbjcuHwwzqTbJzkW21/PwYeO8Ncndr+96YktyV5WpJF7Rxc0V7L0Uk2nOb4bYH/rKqbgTOBxyR5KPAu4D0z9DsY6x5Jzmyv8cwke7TbjwJeDbyjHddzJh13APCKof3fbrfv0M7hTUnOT/LCWdq/K8klbWndz5K8ZLYxTzdv7fn+PMnPk9yY5KQkW7fb0/69XNde6zlJHj/duKaYpz9MckF77OE0/2Y82PfYJP+Z5Ib27+SrSTZq930ZeDTw7fb872i3fyPJNe35Tk2y06QuTwH+uONcSNK8MviRpJntDFw4adsjgQ2BLYHXAp9O8rB236fafY8BnklTMveaoWOfAlwKbAZ8aGjbOcDGwNeAY4EnA78DvBI4PMn6bdtftefciOYH5RuTvLjDdfwt8D3gYcBW7ThJ8hDg5LbfzYD9gM8M/YD9NPBrYHPgz9vXdJ7R/nejqlq/qk6nyZztDzyrnZP1gcOnOf484A/bH9tLgJ+14/5EVd0008UleTjwHeCTNPP4MeA7STauqv2BrwIfacf178PHVtURk/a/IMm6wLdp5mwz4M3AV5NsP1X79lSXAL9P8/2/H/hKks1nGnfrXvPWfqfvAf4E2BT4PnBM2+657TG/S/N3sA9wwwzjGp6nTYB/AQ4GNmnH/PThJsChwBbADsCjgEPaeXoV8N/AC9rzf6Q95rvAdu08/aQdw7CfA7t0mAdJayppVnsbxdeIMfiRpJltBNw6adtdwAeq6q6qOhG4Ddg+yWKaH6Lvrqpbq+py4DDgVUPHLq+qT1XV3UOldJdV1RerahVwHM0Pzg9U1Z1V9T3gNzSBEFV1SlWdW1UTVXUOzQ/iZ3a4jrtoSr+2qKpfV9Xg5vu9gcvb/u+uqp/Q/Dh+aXs9fwq8t6p+VVXn0ZSlrY5XAB+rqkur6jaa0rZ9M3XJ36E0wcN/0QRd6wJPoMk0fK3NKrxpmn7+GLioqr7cXscxNOVW9woAOnoqTaD2d1X1m6r6T+AEmuBwSlX1japa3n43xwEXAbuvYf+vBw6tqp9X1d3Ah4Fd2+zPXcAGwOOAtG2u7nje59OU//1zVd0FfAK4ZugaLq6qk9u/vetpgsgZ/76q6sj27/1OmkBpl0nZvVtp/nckSQvO4EeSZnYjzQ/NYTe0P0gHbqf5obwJ8ADgiqF9V9BkiAZ+OUUf1w69vwOgqiZvWx8gyVOS/N80ZXU3A29o+53NO2j+Vf/HbQnXIIOzNfCUtrTrpiQ30QQsj6TJOKwzaczD19bFFtx7PtYBHjG5YVWtrKp9qmoX4B9pslNvpil7Ow94DvCGJDt26GfQ15ZTtO067l9W1UTX8yX5syRnDc3j4+n23Uxla+Afh861kub727INxA6nCRCvTXJEWx7YxRYMfZ/tPVD/8znJZkmOTXJVkluAr8x0DUkWJ/m7ttzvFuDydtfwMRsAM2buJGltMfiRpJmdQ1Ne1MUKfpthGXg0cNXQ5/t6w/3XgG8Bj6qqDYHPMnTPxnSq6pqq+ouq2oImq/CZJL9D88P3v6pqo6HX+lX1RuB64G6aTNTw9UzbzRTblnPv+bibewZ8UzkAOKPNNu0MLK2q3wDn0gQVs/Uz6OuqKdpOZfLYlwOPSnu/1hTnu0f7NiPzOeBNwMZVtRFNwNal5mOqefsl8PpJ38uDq+qHAFX1yap6ErATzd/n38xwrmFXM/R9Jgn3/H4Pbc/xhKp6KE3Z5fA1TD7/y4EX0QSmGwLbDE491GYH4OxZxiXpvlroVd1c7U2S7hdOpFtZGW3Z2teBDyXZoP1B/Faafz2fKxsAK6vq10l2p/nxOaskL0uyVfvxRpofsatoSrl+N8mrkqzbvp6cZIf2er5JszDDem3GZabnBl0PTNDc2zNwDHBgkm3b+5Y+DBw3KXM2eaybAf+b9l4T4DLgWe3xS2jumZrsxPY6Xp5knTQrxO3YXl8X104a949o7q96Rzsne9KU0B07TfuH0Mzp9e01vIapg7SpTDVvnwXePbj3Ks1CGi9r3z+5zQCu247x1zTf5VTjmuw7wE5J/qQtPfwrmizfwAY0ZZw3JdmS3wZVA5PPvwFwJ3ADsB7N9zvZM2nuC5KkBWfwI0kzOxp4fpIHd2z/ZpofpJcCP6DJ1Bw5h+P5S+ADSW4F3ksTbHXxZOBHSW6jyRz9dVVdVlW30txAvy9NtuMa4O+BB7bHvYmm5O4a4Cjgi9N1UFW30yzicFpbrvVUmmv/Ms2KZpfR/FB/8yxj/SjNPU+3tZ8PBf6AJhvyramWvK6qG2juX3obzQ/xdwB7V9WKWfoa+AKwYzvuf22zTC+kWe1vBfAZ4M+q6oJp2v+M5v6u02kChJ2B07p0PNW8VdXxNN/DsW052XntWAAeSpNlupGmFO8Gmjm717im6GsF8DLg79rjtps0zvcDuwE30wRK35x0ikOBg9vzv53mfx9X0GTEfgacMdy4XfBhR+BeY5GkhRAfeSBJM0vyYeC6qvrEQo9FGidpHlZ7SVV9ZqHHIt2fLdlkvVq69/YLPYwp5UtnLauq+/KQ6jnlA/YkaRZVNeszZiTdW1W9baHHIEnDLHuTJEmS1AtmfiRJkqSxlpFcWW0UOUuSJEmSesHgR5IkSVIvWPamObXJuotqmwf6Z7UQlv/qCQs9BEmSeuEmLuf2WtHlIcZrR4DFozOcUeavVM2pbR64Dkt3fsRCD6OXDjnjXo8+0Vo0sdjHBiyURav8P3xJa9cRjMzKzVpNlr1JkiRJ6gUzP5IkSdI4C6721pGzJEmSJKkXDH4kSZIk9YJlb5IkSdJY8yGnXTlLkiRJknrB4EeSJElSL1j2JkmSJI2zAIt85lkXZn4kSZIk9YLBjyRJkqResOxNkiRJGneu9taJsyRJkiSpFwx+JEmSJPWCZW+SJEnSOAuw2NXeujDzI0mSJKkXDH4kSZIk9YJlb5IkSdJYi6u9deQsSZIkSeoFgx9JkiRJvWDZmyRJkjTOXO2tMzM/kiRJknrB4EeSJElSL1j2JkmSJI27ReY0unCWJEmSJPWCwY8kSZKkXjD4kSRJksZZ0qz2NoqvWYeeI5Ncl+S8oW2HJLkqyVnt6/nTHLtXkguTXJzkXV2myuBHkiRJ0kI5Cthriu0fr6pd29eJk3cmWQx8GngesCOwX5IdZ+vM4EeSJEnSgqiqU4GVa3Do7sDFVXVpVf0GOBZ40WwHudqbJEmSNM4CLL7f5TTelOTPgKXA26rqxkn7twR+OfT5SuAps530fjdLkiRJkkbGJkmWDr0O6HDMPwGPBXYFrgYOm6LNVDcU1WwnNvMjSZIkab6sqKolq3NAVV07eJ/kc8AJUzS7EnjU0OetgOWzndvgR5IkSRp3HVZWGxdJNq+qq9uPLwHOm6LZmcB2SbYFrgL2BV4+27kNfiRJkiQtiCTHAHvSlMddCbwP2DPJrjRlbJcDr2/bbgF8vqqeX1V3J3kTcBKwGDiyqs6frT+DH0mSJEkLoqr2m2LzF6Zpuxx4/tDnE4F7LYM9E4MfSZIkaZwlsMh1zLpwliRJkiT1gsGPJEmSpF6w7E2SJEkad/ej1d7mk5kfSZIkSb1g8CNJkiSpFyx7kyRJksZZgMXmNLpwlhZQko8necvQ55OSfH7o82FJ3tq+PzDJr5NsOLR/zyQnTHHeU5Isad9vk+SiJH803D7J/kkmkjxh6LjzkmzTvl8/yT8luSTJT5MsS/IXcz8LkiRJ0tph8LOwfgjsAZBkEbAJsNPQ/j2A09r3+wFnAi/pevIkW9E89fZtVXXSFE2uBA6a5vDPAzcC21XVE4G9gId37VuSJEkaNQY/C+s02uCHJug5D7g1ycOSPBDYAfhpkscC6wMH0wRBXTwS+B5wcFV9a5o2JwA7Jdl+eGPb3+7tsRMAVXV9Vf1990uTJEnSWrMoo/kaMQY/C6iqlgN3J3k0TRB0OvAj4GnAEuCcqvoNTcBzDPB9YPskm3U4/dHA4VX1jRnaTAAfAd4zaftOwNmDwEeSJEm6PzD4WXiD7M8g+Dl96PMP2zb7Ase2wcg3gZd1OO+/A69Kst4s7b4GPDXJttM1SHJQkrOSLJ9m/wFJliZZev1dxkuSJEkaTQY/C29w38/ONGVvZ9BkfvYATmsXJNgOODnJ5TSBUJfSt4/QZJG+kWTaVf2q6m7gMOCdQ5t/BuzS3odEVX2oqnYFHjrNOY6oqiVVtWTTdf2TkiRJWquSZrW3UXyNmNEbUf+cBuwNrKyqVVW1EtiIJgA6nSbQOaSqtmlfWwBbJtm6w7kPBG4BvpBkpqLLo4DnAJsCVNXFwFLgg0kWAyR5EM1CipIkSdJYMvhZeOfSrPJ2xqRtN1fVCppMz/GTjjm+3Q7w7CRXDr2eNmhUVQW8GticJhM0pfa+ok8Cw/cSvQ7YGLg4yTKaMrp3TnG4JEmSNBZ8yOkCq6pVTConq6r9h97f616cqnrr0McHT3HaPYfa/gZ47tC+U9rtR9FkfAbtPkkTAA0+3wK8vsMlSJIkaaGN4Mpqo8jMjyRJkqReMPiRJEmS1AsGP5IkSZJ6wXt+JEmSpHEWRnJZ6VHkLEmSJEnqBYMfSZIkSb1g2ZskSZI01uJS1x2Z+ZEkSZLUCwY/kiRJknrBsjdJkiRpnLnaW2fOkiRJkqReMPiRJEmS1AuWvUmSJEnjztXeOjHzI0mSJKkXDH4kSZIk9YJlb5IkSdI4S1ztrSNnSZIkSVIvGPxIkiRJ6gXL3iRJkqRx52pvnZj5kSRJktQLBj+SJEmSesGyN0mSJGmcBVd768hZkiRJktQLBj+SJEmSesGyN0mSJGmsxdXeOjLzI0mSJKkXDH4kSZIk9YJlb5IkSdI4C7DInEYXzpIkSZKkXjD4kSRJktQLlr1JkiRJ426xq711YeZHkiRJUi8Y/EiSJEnqBcveJEmSpHGWuNpbR86SJEmSpF4w+JEkSZLUC5a9SZIkSeNukau9dWHmR5IkSVIvGPxIkiRJ6gXL3iRJkqRxFnzIaUdmfiRJkiT1gsGPJEmSpF6w7E1z64HrwLYPW+hR9NMZCz2Aflu0ynIDSdIC8iGnnThLkiRJknrB4EeSJElSL1j2JkmSJI2zhAkfctqJmR9JkiRJvWDwI0mSJKkXLHuTJEmSxlgBE6721omzJEmSJKkXDH4kSZIk9YJlb5IkSdKYc7W3bsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTGKmHVYnMaXThLkiRJknrB4EeSJElSL1j2JkmSJI05V3vrxsyPJEmSpF4w+JEkSZLUC5a9SZIkSeMsUIvMaXThLEmSJEnqBYMfSZIkSb1g2ZskSZI0xgpXe+vKzI8kSZKkXjD4kSRJktQLlr1JkiRJ4yyx7K0jMz+SJEmSesHgR5IkSVIvWPYmSZIkjbFmtTdzGl04S5IkSZJ6weBHkiRJUi9Y9iZJkiSNOVd768bMjyRJkqReMPiRJEmS1AuWvUmSJEljrBJWxZxGF86SJEmSpF4w+JEkSZLUC5a9SZIkSWPO1d66MfMjSZIkqRcMfiRJkiT1gmVvkiRJ0piz7K0bMz+SJEmSFkSSI5Ncl+S8oW3/kOSCJOckOT7JRtMce3mSc5OclWRpl/4MfiRJkiQtlKOAvSZtOxl4fFU9AfgF8O4Zjn9WVe1aVUu6dGbZmyRJkjTGKlCLxjOnUVWnJtlm0rbvDX08A3jpXPU3nrMkSZIkaRxskmTp0OuA1Tz+z4HvTrOvgO8lWdb1vGZ+JEmSJM2XFV1L0iZLchBwN/DVaZo8vaqWJ9kMODnJBVV16kznNPiRJEmSxlrud6u9JXk1sDfw7KqqqdpU1fL2v9clOR7YHZgx+BmLsrckH0/ylqHPJyX5/NDnw5K8tX1/YJJfJ9lwaP+eSU6Y4rynJFnSvt8myUVJ/mi4fZL9k0wkecLQcecNahOTrJ/kn5JckuSnbdrtL2a4lnuNJclRSV46NKYLk5yd5LQk27fb927Pf3aSnyV5fZKD2tUtzkqyauj9Xw2d++wkx3Ts78wkuw61+/N2BY1z2mt+0XTXJUmSJM2FJHsB7wReWFW3T9PmIUk2GLwHngucN1XbYWMR/AA/BPYASLII2ATYaWj/HsBp7fv9gDOBl3Q9eZKtgJOAt1XVSVM0uRI4aJrDPw/cCGxXVU+kWa3i4V37nsYrqmoX4EvAPyRZFzgCeEG7/YnAKVX1oXZ1i12BOwbvq+qT7XXtQPMdP6P9o5itv88A/9Aeu1V7zb/XrrTxVOCc+3hdkiRJ0v9o/5H+dGD7JFcmeS1wOLABTSnbWUk+27bdIsmJ7aGPAH6Q5Gzgx8B3qurfZutvXMreTgM+3r7fiSaq2zzJw4DbgR2AnyZ5LLA+8DfAe2iWzpvNI4GjgYOr6lvTtDmBJoDYvqouHGxs+9sdeHlVTQBU1fXA36/e5U3rVOAtNF/+OsANbR93AhfOcNzAy4Ev08zPC4FjZm7O6TRzB7AZcCtwW9vnbYP3kiRJGiGBifFd7W2/KTZ/YZq2y4Hnt+8vBXZZ3f7GYpbaC707yaNpsjynAz8CngYsAc6pqt/QZH2OAb5PEz1u1uH0RwOHV9U3ZmgzAXyEJqAathNw9iDwmQcvAM6tqpXAt4ArkhyT5BVtBmw2+wDH0czJVH9Yk+0F/Gv7/mzgWuCyJF9M8oLpDkpywGAFj+t/fXeHbiRJkqS1byyCn9ZpNIHPIPg5fejzD9s2+wLHtsHIN4GXdTjvvwOvSrLeLO2+Bjw1ybbTNRi6B2f5DOeZ8oatSdu/muQs4OnA2wGq6nXAs2nSem8HjpxpsEmeDFxfVVcA/wHs1mbKpvLVJFfS1FZ+qu1vFU0w9FKah0t9PMkhUw686oiqWlJVSzZ90LgkEyVJktQ34xT8DO772Zmm7O0MmszPHsBp7YIE29HUBl5OEwh1yXZ8hCaL9I0k0/5yr6q7gcNoAoSBnwG7DLIwg3twgIfO0N8NwOQg5OHAiqHPr2jv3XlxVf1yaAznVtXHgT8E/nSW69oPeFw7F5e0Y5rumFcA29IEeJ8e6q+q6sdVdSjNfM7WpyRJktayAiaSkXyNmnEKfk6jWe5uZVWtakvBNqIJgE6n+bF/SFVt0762ALZMsnWHcx8I3AJ8IZnxWzoKeA6wKUBVXQwsBT6YZDFAkgcBM53jImCLdjEC2vHtApw13QHtinJ7Dm3aFbhihvaLaLJeTxjMB/AiZggGq+ou4GCa7NYO7Q1lu3XtU5IkSRp14xT8nEuzytsZk7bdXFUraDITx0865vh2O8Cz2xUkBq+nDRq1a4e/GticJhM0pfa+ok/SLAYw8DpgY+DiJMtoyujeOcXhg3PcCbwS+GJb2vbPwOuq6uZpr7wJpt7RLkl9FvB+YP8Z2j8DuKqqrhradiqwY5LNZxjbHTTZrbcD6wIfTXJB2+c+wF8362jpAAAgAElEQVTP0KckSZI00jLNM4OkNbJk4/Vq6R/9zkIPo5cOOcaVyCVJWhuOYAnLa+nI1HTt/PjN6/h/ee1CD2NK2z3uQ8uqaslCj2NgnDI/kiRJkrTGXJprniTZmeYZO8PurKqnLMR4JEmSpL4z+JknVXUuzSIBkiRJ0rypZGwfcrq2OUuSJEmSesHgR5IkSVIvWPYmSZIkjblVI/hA0VFk5keSJElSLxj8SJIkSeoFy94kSZKkMVbgam8dOUuSJEmSesHgR5IkSVIvWPYmSZIkjbVQrvbWiZkfSZIkSb1g8CNJkiSpFyx7kyRJksZZYGKRZW9dmPmRJEmS1AsGP5IkSZJ6wbI3SZIkaYwVMBFzGl04S5IkSZJ6weBHkiRJUi9Y9iZJkiSNOVd768bMjyRJkqReMPiRJEmS1AuWvUmSJEnjLGEilr11YeZHkiRJUi8Y/EiSJEnqBcveJEmSpDFWwKpF5jS6cJYkSZIk9YLBjyRJkqResOxNkiRJGnOu9taNmR9JkiRJvWDwI0mSJKkXLHuTJEmSxlhh2VtXZn4kSZIk9YLBjyRJkqResOxNkiRJGmcJ5UNOO3GWJEmSJPWCwY8kSZKkXrDsTZIkSRpzrvbWjZkfSZIkSb1g8CNJkiSpFyx705z6720fxZu/8o8LPYxe2viYhR5Bvx1z4WULPYTe2m/7bRd6CJK0oHzIaXdmfiRJkiT1gsGPJEmSpF6w7E2SJEkac5a9dWPmR5IkSVIvGPxIkiRJ6gXL3iRJkqQxVgkTMafRhbMkSZIkqRcMfiRJkiT1gmVvkiRJ0phztbduzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMYKWLXIsrcuzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNM58yGlnzpIkSZKkXjD4kSRJktQLlr1JkiRJY658yGknZn4kSZIk9YLBjyRJkqResOxNkiRJGmMFTGDZWxdmfiRJkiT1gsGPJEmSpF6w7E2SJEkacxOu9taJmR9JkiRJvWDwI0mSJKkXLHuTJEmSxlqYiDmNLpwlSZIkSb1g8CNJkiSpFyx7kyRJksZY4WpvXZn5kSRJktQLBj+SJEmSesGyN0mSJGmcBVZZ9taJmR9JkiRJvWDwI0mSJKkXLHuTJEmSxpirvXVn5keSJElSLxj8SJIkSeoFy94kSZKksRYmzGl04ixJkiRJ6gWDH0mSJEm9YNmbJEmSNObK1d46MfMjSZIkqRcMfiRJkiT1wrwFP0k+nuQtQ59PSvL5oc+HJXlr+/7AJL9OsuHQ/j2TnDDFeU9JsqR9v02Si5L80XD7JPsnmUjyhKHjzkuyTft+/ST/lOSSJD9NsizJX8xwLdskuaNt+/MkP07y6kltXpzknCQXJDk3yYvb7bskOWuo3X5Jbk+ybvt55yTnDF3b0qG2S5Kc0r5fL8lX23Ofl+QHSbZOclb7uibJVUOfH9Ae95IkleRxk67nvKF5vrm9tguSfHSo3SOSnJDk7CQ/S3LidHMkSZKkhTF4yOkovkbNfGZ+fgjsAZBkEbAJsNPQ/j2A09r3+wFnAi/pevIkWwEnAW+rqpOmaHIlcNA0h38euBHYrqqeCOwFPHyWLi+pqidW1Q7AvsCBSV7TjmUX4KPAi6rqccALgY+2wde5wNZJNmjPswdwAfDEoc+nDfWzWZLnTdH/XwPXVtXOVfV44LXANVW1a1XtCnwW+Pjgc1X9pj1uP+AH7Zin8/12Hp4I7J3k6e32DwAnV9UuVbUj8K5Z5kiSJEkaWfMZ/JxGG/zQBD3nAbcmeViSBwI7AD9N8lhgfeBgmh/qXTwS+B5wcFV9a5o2JwA7Jdl+eGPb3+7tsRMAVXV9Vf191wurqkuBtwJ/1W56O/Dhqrqs3X8ZcCjwN20fZwJPads+Cfg0v52bPWgCxYF/oJmLyTYHrhoaw4VVdedM40yyPvB0mkBppuBncM47gLOALYf6vHJo/zmznUOSJEkaVfMW/FTVcuDuJI+m+YF/OvAj4GnAEuCcNjuxH3AM8H1g+ySbdTj90cDhVfWNGdpMAB8B3jNp+07A2YPA5z74CTAoJdsJWDZp/1J+m+n6IbBHkoe04zqFewY/w5mf04E7kzxr0vmOBN6Z5PQkH0yyXYcxvhj4t6r6BbAyyW4zNU7yMGA74NR206eBLyT5v0kOSrLFNMcdkGRpkqV3XH9zh2FJkiRpLk2QkXyNmvle8GCQ/RkEP6cPfR5kO/YFjm2DkW8CL+tw3n8HXpVkvVnafQ14apJtp2vQ/qg/K8nyDv3e49BJ72uK/YNtg3nYHTizqi4BfifJpsD6bSZp2AeZlP2pqrOAx9Bkhh4OnJlkh1nGuB9wbPv+WKbPrP1+e9/RNcAJVXVN2+dJbZ+fown0ftqO+R6q6oiqWlJVSx686YaTd0uSJEkjYb6Dn8F9PzvTlL2dQZP52QM4rb0nZjvg5CSX0wRCXUrfPkKTRfpGkmmfVVRVdwOHAe8c2vwzYJf2PiSq6kPtPTMPXb1L44nAz9v359Nks4bt1vYFzXU/Gfg9mgAQmnKyfblnydtg3P8JPAh46qTtt1XVN6vqL4GvAM+fbnBJNgb+APh8O7d/A+yTTHnn2fer6gk039Mbk+w61OfKqvpaVb2KpnzvGdP1KUmSJK2OJEcmuW6wGFe77eFJTm4XNju5rU6a6thXt20umrwY2XTWRuZnb2BlVa2qqpXARjQB0Ok0gc4hVbVN+9oC2DLJ1h3OfSBwC01Z1kw5taOA5wCbAlTVxTQlaR9MshggyYOge16uXTXuo8Cn2k0fBd49tJrcNjTldoe1fd4K/BLYn98GP6cDb2GK4Kf1IeAdQ30+ffDFtyu57QhcMcMwXwocXVVbt3P7KOAymgBsSm153KG0wWKSPxhk19oFGx4L/PcMfUqSJGktK8JEFo3kq4OjaBYfG/Yu4D+qajvgP5hi0a0kDwfeR3Nf/e7A+6YLkobNd/BzLs0qb2dM2nZzVa2gyXwcP+mY4/ntzfnPTnLl0Otpg0ZVVcCraW7K/8h0A2jvK/okMHwv0euAjYGLkyyjKaN75xSHD3vsYKlr4OvAp6rqi20fZ7XHfzvJBcC3gXe02wdOAx5YVb9sP59OU1I2ZfBTVScC1w/3D/xXknOBn9IEcP8yw3j3495z+y/Ay2e5zs8Cz2hLBZ8ELG1L4k4HPl9VZ85yvCRJktRJVZ0KrJy0+UXAl9r3X6K5j32yP6JZlXhlVd0InMy9g6h7SRNDSHNjsyXb1z4//uxCD6OXNl48eY0MrU3HXHjZQg+ht/bbftrbOiVpXhzBEpbX0pG5m3+bJz2m3vujv13oYUzpteu+cllVTb495B7aqqkT2se5kOSmqtpoaP+NVfWwSce8HXhQVX2w/fz/AXdU1UeZwbT3y0iSJEkaD6O4slprkyRLhz4fUVVHzMF5p7rgWbM6Bj9DkuwMfHnS5jur6ilTtZckSZI0oxWzZX6mcG2Szavq6iSbA9dN0eZKYM+hz1vRPE5mRgY/Q6rqXGDXWRtKkiRJmi/form3/+/a//7/U7Q5Cfjw0CIHzwXePduJDX4kSZKkMVaBiRkXPx5dSY6hyeBskuRKmhXc/g74epLX0qw0/LK27RLgDVX1uqpameRvaR7FAvCBdmXpGRn8SJIkSVoQVTXdMz6fPUXbpTSrNg8+HwkcuTr9zfdS15IkSZI0Esz8SJIkSWNu1eiu9jZSzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMaKjO1qb2ubmR9JkiRJvWDwI0mSJKkXLHuTJEmSxly52lsnZn4kSZIk9YLBjyRJkqResOxNkiRJGnMTMafRhbMkSZIkqRcMfiRJkiT1wrRlb0keOtOBVXXL3A9HkiRJ0uooYMLV3jqZ6Z6f82nmcngmB58LePQ8jkuSJEmS5tS0wU9VPWptDkSSJEmS5lOn1d6S7As8pqo+nGQr4BFVtWx+hyZJkiRpdrHsraNZFzxIcjjwLOBV7abbgc/O56AkSZIkaa51yfzsUVW7JfkpQFWtTPKAeR6XJEmSJM2pLsHPXUkW0SxyQJKNgYl5HZUkSZKkzix766bLc34+DfwLsGmS9wM/AP5+XkclSZIkSXNs1sxPVR2dZBnwnHbTy6rqvPkdliRJkiTNrU6rvQGLgbtoSt+6ZIskSZIkrQUFrIplb110We3tIOAYYAtgK+BrSd493wOTJEmSpLnUJfPzSuBJVXU7QJIPAcuAQ+dzYJIkSZI0l7oEP1dMarcOcOn8DEeSJEnS6nK1t26mDX6SfJymhPB24PwkJ7Wfn0uz4pskSZIkjY2ZMj+DFd3OB74ztP2M+RuOJEmSJM2PaYOfqvrC2hyIJEmSpNVXhAkXZO5k1nt+kjwW+BCwI/Cgwfaq+t15HJckSZIkzakuIeJRwBeBAM8Dvg4cO49jkiRJkqQ51yX4Wa+qTgKoqkuq6mDgWfM7LEmSJEldFRnJ16jpstT1nUkCXJLkDcBVwGbzOyxJkiRJmltdgp8DgfWBv6K592dD4M/nc1CSJEmSNNdmDX6q6kft21uBV83vcCRJkiStLh9y2s1MDzk9nuahplOqqj+ZlxFJkiRJ0jyYKfNz+Fobhe431lm2ARsvdj0M9c9+22+70EPoreseM+2/02kt+Mweuy70EHrrkK+cvdBDkMbOTA85/Y+1ORBJkiRJq6+w7K0rHwUrSZIkqRcMfiRJkiT1QpelrgFI8sCqunM+ByNJkiRp9Vn21s2smZ8kuyc5F7io/bxLkk/N+8gkSZIkaQ51KXv7JLA3cANAVZ0NuJyXJEmSpLHSpextUVVdkdwjlbZqnsYjSZIkaTUUYZVlb510CX5+mWR3oJIsBt4M/GJ+hyVJkiRJc6tL2dsbgbcCjwauBZ7abpMkSZKksTFr5qeqrgP2XQtjkSRJkrQGyrK3TmYNfpJ8jubBsfdQVQfMy4gkSZIkaR50uefn34fePwh4CfDL+RmOJEmSJM2PLmVvxw1/TvJl4OR5G5EkSZKk1eJDTrvpsuDBZNsCW8/1QCRJkiRpPnW55+dGfnvPzyJgJfCu+RyUJEmSJM21GYOfNE823QW4qt00UVX3WvxAkiRJ0sIoYFVZ9tbFjGVvbaBzfFWtal8GPpIkSZLGUpd7fn6cZLd5H4kkSZIkzaNpy96SrFNVdwO/B/xFkkuAXwGhSQoZEEmSJEkjwNXeupnpnp8fA7sBL15LY5EkSZKkeTNT8BOAqrpkLY1FkiRJkubNTMHPpkneOt3OqvrYPIxHkiRJ0mooQln21slMwc9iYH1wJiVJkiSNv5mCn6ur6gNrbSSSJEmSNI9mvedHkiRJ0mib6PQEG800S89ea6OQJEmSpHk2bfBTVSvX5kAkSZIkaT7NVPYmSZIkaQxMlHesdGFxoCRJkqReMPiRJEmS1AuWvUmSJEljrIBVLtTciZkfSZIkSb1g8CNJkiSpFyx7kyRJksZaKFd768TMjyRJkqReMPiRJEmS1AuWvUmSJEljrIAJV3vrxMyPJEmSpF4w+JEkSZLUC5a9SZIkSeOsYJWrvXVi5keSJElSLxj8SJIkSeoFy94kSZKkMeZqb92Z+ZEkSZLUCwY/IyzJQUnOT3JOkrOSPCXJKUmWJPlRu+2/k1zfvj8rybXTbN8myeVJNmnPXUkOG+rr7UkOGfr8yrbf85OcneTzSTZagGmQJEmS5oRlbyMqydOAvYHdqurONmh5wGB/VT2lbbc/sKSq3jTp+HttT+6RDr0T+JMkh1bViknH7gUcCDyvqq5Kshh4NfAI4KY5u0hJkiTNiXK1t07M/IyuzYEVVXUnQFWtqKrlc3j+u4EjaIKcyQ4C3l5VV7V9r6qqI6vqwjnsX5IkSVqrDH5G1/eARyX5RZLPJHnmPPTxaeAVSTactH0n4CddT5LkgCRLkyy9nevndICSJEnSXDH4GVFVdRvwJOAA4HrguLaUbS77uAU4Gvir6dok2bm9Z+iSJPtMc54jqmpJVS1Zj03ncoiSJEmaVZgY0deoMfgZYW252SlV9T7gTcCfzkM3nwBeCzxkaNv5wG7tGM6tql2B7wIPnof+JUmSpLXC4GdEJdk+yXZDm3YFrpjrfqpqJfB1mgBo4FDgo0m2Gtpm4CNJkqSx5mpvo2t94FPt8tJ3AxfTlMD98zz0dRhNZgmAqjoxyabAd9uV3m4CzgNOmoe+JUmSdB8UMOFqb50Y/IyoqloG7DHFrj0ntTsKOGqK4++1vaq2GXq//tD7a4H1JrX9EvCl1Ru1JEmSNLose5MkSZLUC2Z+JEmSpDG3yrK3Tsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTmagQfKDqKzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMZ8yGl3Zn4kSZIkLYgk2yc5a+h1S5K3TGqzZ5Kbh9q8d037M/MjSZIkaUFU1YXArgBJFgNXAcdP0fT7VbX3fe3P4EeSJEkaZ5X7y0NOnw1cUlVXzFcHlr1JkiRJGgX7AsdMs+9pSc5O8t0kO61pB2Z+JEmSJM2XTZIsHfp8RFUdMblRkgcALwTePcU5fgJsXVW3JXk+8K/AdmsyGIMfSZIkaYw1q70t9CimtaKqlnRo9zzgJ1V17eQdVXXL0PsTk3wmySZVtWJ1B2PZmyRJkqSFth/TlLwleWSStO93p4lhbliTTsz8SJIkSVowSdYD/hB4/dC2NwBU1WeBlwJvTHI3cAewb1WtUa7L4EeSJEkaczXGq71V1e3AxpO2fXbo/eHA4XPRl2VvkiRJknrB4EeSJElSL1j2JkmSJI2xZrW38S17W5vM/EiSJEnqBYMfSZIkSb1g2ZskSZI05iaw7K0LMz+SJEmSesHgR5IkSVIvWPYmSZIkjbECVrnaWydmfiRJkiT1gsGPJEmSpF6w7E2SJEkaZxXKsrdOzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNOYmJix768LMjyRJkqReMPiRJEmS1AuWvUmSJEljzIecdmfmR5IkSVIvGPxIkiRJ6gXL3iRJkqRxVjBh2VsnZn4kSZIk9YLBjyRJkqResOxNkiRJGnNl2VsnZn4kSZIk9YKZH0nSWNvsUv+1c0Fdes5Cj0CSOjPzI0mSJKkXzPxIkiRJY6yIS113ZOZHkiT9v/buO9yysrz7+Pc3owgqJYogUkQQG0VBmvAGC8ZoghrbC2NiRFGjiQUNqLESIjYQY8UM6ouaxC4GNSqogCJFB0IVBVRQigFsFBvM3O8fax/ZHs8Mm2H2Xmef5/u5rn3NXuXs9TuLYWbu89zreSSpCRY/kiRJkppg25skSZI05VZU3wmmgyM/kiRJkppg8SNJkiSpCba9SZIkSVOsCpavcLa3UTjyI0mSJKkJFj+SJEmSmmDbmyRJkjTlykVOR+LIjyRJkqQmWPxIkiRJaoJtb5IkSdKUW2Hb20gc+ZEkSZLUBIsfSZIkSU2w7U2SJEmaYoWLnI7KkR9JkiRJTbD4kSRJktQE294kSZKkaVZxtrcROfIjSZIkqQkWP5IkSZKaYNubJEmSNMUKqBV9p5gOjvxIkiRJaoLFjyRJkqQm2PYmSZIkTTlnexuNIz+SJEmSmmDxI0mSJKkJtr1JkiRJ06xgxQrb3kbhyI8kSZKkJlj8SJIkSWqCbW+SJEnSFCtgubO9jcSRH0mSJElNsPiRJEmS1ATb3iRJkqQpV872NhJHfiRJkiQ1weJHkiRJUhNse5MkSZKmWAErqu8U08GRH0mSJElNsPiRJEmS1ATb3iRJkqRpVmG5s72NxJEfSZIkSU2w+JEkSZLUBIufCUny6iQXJDk3ydlJThz8ekmSXw7en51kj8H590hyU5K/m/U5lyb59ND2U5McM3i/f5JrkvxPkouTfHnm8wbHj0ny1MH7k5IsGzq2c5KThrZ3HZxzcZKzknwhyfbjuj+SJElaPQWsWJF5+ZpvfOZnApI8DNgH2KmqfptkQ2CtqroyySOAg6pqn1lf9jTgdGAJ8G+zju2cZNuqumCOy328ql44uO4jgc8keWRVXTjHuRsleVxVfXFW3o2BTwBPr6pTB/v+D7A1cN5t+NYlSZKkecORn8nYBLi2qn4LUFXXVtWVt/I1S4B/BDZLsumsY0cAr7q1i1bVicBS4HkrOeVw4DVz7H8h8KGZwmfwWadU1Wdv7ZqSJEnSfGXxMxnHA5snuSjJe5M8fFUnJ9kcuGdVfYtuBGbfWad8AtgpyX1HuPZZwANWcuw04LeDEaJh2w6+TpIkSVOgKvPyNd9Y/ExAVd0APJRuBOYa4ONJ9l/Fl+xHV+AAfIxuFGjYcrpRm38a4fK39rvuDcw9+nPLByRnJLkwyTtWcvx5SZYlWfYrrhkhkiRJkjR5Fj8TUlXLq+qkqno9XVvZU1Zx+hJg/ySXAscBD06yzaxzPgLsBWxxK5feEZjreZ+ZXF8D1gZ2H9p9AbDT0Dm7Aa8F1l/JZyytqp2rauc7c49biSNJkiT1w+JnApLcf1bx8hDgspWdC9ylqjatqi2rakvgTXSjQb9XVTcBbwcOXMV1H0432nT0rUQ8DHj50PZ76IqvPYb23flWPkOSJEl9KFixYn6+5htne5uMuwLvSrIBcDNwCSufhGAJcOysfZ+ma3/7l1n7P8Aft6ztO5iZ7c7AD4GnrGSmt9+rqv9Ocs3Q9k+S7Au8ZTDZwtXAtcChq/ocSZIkaT6z+JmAqjoT2GMlx04CThraPmSOc84FHjR4v+XQ/t8C9xraPgY4ZhU59h96/4hZxx46a/t0YJUTM0iSJEnTxOJHkiRJmmIzi5zq1vnMjyRJkqQmWPxIkiRJaoJtb5IkSdI0K1hu29tIHPmRJEmS1ASLH0mSJElNsO1NkiRJmmJFnO1tRI78SJIkSWqCxY8kSZKkJtj2JkmSJE25WtF3gungyI8kSZKkJlj8SJIkSWqCbW+SJEnSNCtYXs72NgpHfiRJkiQ1weJHkiRJUhNse5MkSZKmWIGLnI7IkR9JkiRJTbD4kSRJktQE294kSZKkKbfCRU5H4siPJEmSpCZY/EiSJEnqTZJLk5yX5Owky+Y4niTvTHJJknOT7LS617LtTZIkSZpmBTX9s709sqquXcmxxwHbDF67AUcNfr3NHPmRJEmSNJ89EfhwdU4HNkiyyep8kMWPJEmSpD4VcHySM5M8b47jmwI/Htq+fLDvNrPtTZIkSZpi83yR0w1nPceztKqWzjpnz6q6MslGwAlJvltVXx86Ptc3V6sTxuJHkiRJ0rhcW1U7r+qEqrpy8OvVSY4FdgWGi5/Lgc2HtjcDrlydMLa9SZIkSepFkrskWXfmPfAY4PxZpx0H/O1g1rfdgV9W1VWrcz1HfiRJkqRpVrB8ehc53Rg4Ngl0tcl/VtWXkjwfoKreB/w38BfAJcCvgGet7sUsfiRJkiT1oqp+ADx4jv3vG3pfwD+sievZ9iZJkiSpCY78SJIkSVOsyHye7W1eceRHkiRJUhMsfiRJkiQ1wbY3SZIkaZoV1HLb3kbhyI8kSZKkJlj8SJIkSWqCbW+SJEnSFCumepHTiXLkR5IkSVITLH4kSZIkNcG2N0mSJGnKucjpaBz5kSRJktQEix9JkiRJTbDtTWvcisXVd4QmXXvvvhO0baMf2G6gNh2Cf+b35ZB7b9B3hGZ9/qob+o7whwpWONvbSBz5kSRJktQEix9JkiRJTbDtTZIkSZpymaezvc23xlhHfiRJkiQ1weJHkiRJUhNse5MkSZKmWcHi5fOz7e3mvgPM4siPJEmSpCZY/EiSJElqgm1vkiRJ0hQLsMhFTkfiyI8kSZKkJlj8SJIkSWqCbW+SJEnSNKuwaJ4ucjrfOPIjSZIkqQkWP5IkSZKaYNubJEmSNOWyvO8E08GRH0mSJElNsPiRJEmS1ATb3iRJkqQploLFzvY2Ekd+JEmSJDXB4keSJElSE2x7kyRJkqbcohV9J5gOjvxIkiRJaoLFjyRJkqQm2PYmSZIkTbEULFrubG+jcORHkiRJUhMsfiRJkiQ1wbY3SZIkacrFRU5H4siPJEmSpCZY/EiSJElqgm1vkiRJ0hRLweLlfaeYDo78SJIkSWqCxY8kSZKkJtj2JkmSJE21sMjZ3kbiyI8kSZKkJlj8SJIkSWqCbW+SJEnSNCtY5GxvI3HkR5IkSVITLH4kSZIkNcG2N0mSJGmKBYizvY3EkR9JkiRJTbD4kSRJktQE294kSZKkaVaw2NneRuLIjyRJkqQmWPxMiSQ3rOLYOUk+OrT9vCQfH9peL8n3k9wnyTFJnjrYf1KSZUPn7ZzkpKHtXQfnXJzkrCRfSLL9Gv/mJEmSpAmw+JlySR5I999xryR3Gew+GtgsyaMH24cCH6yqH87xERsledwcn7sx8AngVVW1TVXtBLwJ2HqNfxOSJElabQEWrZifr/nG4mf6PR34CHA88ASAqirgBcC/JtkZ2Bs4fCVffzjwmjn2vxD4UFWdOrOjqk6pqs+uweySJEnSxFj8TL99gY8DHwWWzOysqnOBLwNfBV5cVb9bydefBvw2ySNn7d8WOGvNx5UkSZL6YfEzxZLsAlxTVZfRFTk7JfmToVPeA1xRVSfeyke9gblHf4avdUaSC5O8Y45jz0uyLMmyX3HNbfwuJEmSdLsULFqeefmabyx+ptsS4AFJLgW+D6wHPGXo+IrBa5Wq6mvA2sDuQ7svAHYaOmc34LXA+nN8/dKq2rmqdr4z91iNb0OSJEkaP4ufKZVkEfA0YIeq2rKqtgSeyFDr2210GPDyoe33APsn2WNo351X87MlSZKk3rnI6fS4c5LLh7aPpGtpu2Jo39eBByXZpKquui0fXlX/neSaoe2fJNkXeEuSTYGrgWvpZo6TJEnSPJJ5OLPafGTxMyWqaq5RuiNnnbMc2GRo+1Jgu1nn7D/0/hGzjj101vbpwMNXM7IkSZI0r9j2JkmSJKkJjvxIkiRJUywFi+fhzGrzkWLlKcsAACAASURBVCM/kiRJkppg8SNJkiSpCba9SZIkSVNu0fK+E0wHR34kSZIkNcHiR5IkSVITbHuTJEmSplgKFq1wtrdROPIjSZIkqQkWP5IkSZKaYNubJEmSNOXibG8jceRHkiRJUhMsfiRJkiQ1wbY3SZIkaZpVWLzc2d5G4ciPJEmSpCZY/EiSJElqgm1vkiRJ0hRLwSJnexuJIz+SJEmSmmDxI0mSJKkJtr1JkiRJU27Rir4TTAdHfiRJkiQ1weJHkiRJUhNse5MkSZKmWUFc5HQkjvxIkiRJaoLFjyRJkqQm2PYmSZIkTbEAi13kdCSO/EiSJElqgsWPJEmSpCbY9iZJkiRNs4JFtr2NxJEfSZIkSU2w+JEkSZLUBNveJEmSpCkWYNGULnKaZHPgw8A9gRXA0qp6x6xzHgH8F/DDwa7PVNWhq3M9ix9JkiRJfbkZ+MeqOivJusCZSU6oqu/MOu8bVbXP7b2YbW+SJEmSelFVV1XVWYP31wMXApuO63qO/EiSJEnTrCAr+g5x+yXZEtgROGOOww9Lcg5wJXBQVV2wOtew+JEkSZI0LhsmWTa0vbSqls4+KcldgU8DB1bVdbMOnwXcu6puSPIXwGeBbVYnjMWPJEmSpHG5tqp2XtUJSe5IV/j8R1V9Zvbx4WKoqv47yXuTbFhV197WMBY/kiRJ0hQLsHhKFzlNEuADwIVVdeRKzrkn8L9VVUl2pZu34Kercz2LH0mSJEl92RN4BnBekrMH+14FbAFQVe8Dngq8IMnNwK+B/aqqVudiFj+SJEmSelFVp9ANXq3qnHcD714T17P4kSRJkqZZTe8ip5PmOj+SJEmSmmDxI0mSJKkJtr1pjbqKM6/9l+WLLus7x+2wIXCbp02cF37Qd4DbbXrv/cLg/e+P974/U33v/3ma/7btTPP9v3ffAf5AwaIpne1t0ix+tEZV1T36znB7JFl2a3PRazy89/3y/vfHe98f732/vP/qg21vkiRJkprgyI8kSZI0xYJtb6Ny5Ef6Q0v7DtAw732/vP/98d73x3vfL++/Ji6ruTiqJEmSpHlgvQ0fWrvsc3rfMeb0tQ+tdeZ8erbLtjdJkiRpmrnI6chse5MkSZLUBIsfSZIkSU2w7U3SxCV5PHBuVV022H4d8BTgMuAlVfXDPvO1JMndgb2AH1XVmX3nWeiSrAdsXFUXD7afBqwzOPzlqvrf3sJJmlrO9jY6R37UpCQHJDl4aPuKJNcluT7JC/rM1ojDgGsAkuwD/A3wbOA44H095lrwknw+yXaD95sA59Pd+48kObDXcG04AthzaPtNwC50Beg/95KoEUm2TfKEoe23J/ng4LVTn9la4P3XfGHxo1Y9H/jg0PbVVbUecA9gST+RmlJV9avB+ycDH6iqM6vq/XT/DTQ+96mq8wfvnwWcUFWPB3ajK4I0XrsAHxravr6qXlRVzwG26ylTK94MXDu0/efAF4ATgdf1kqgt3n/NC7a9qVWLquqnQ9ufBKiq3yRZZyVfozUnSe4K/ArYG3jv0LG1+4nUjJuG3u8NHA1QVdcnWdFPpKbcof5wjYlnDL3fYNJhGrNJVZ06tH1dVX0aIMnf9ZSpJd7/cSrb3kZl8aNWrT+8UVVvBEiyCLh7L4na8q/A2cB1wIVVtQwgyY7AVX0Ga8CPk7wIuBzYCfgSwKDov2OfwRqxIsk9q+onADOjcEk2BSw+x2vd4Y2q2n1oc6MJZ2mR91/zgm1vatXxSd4wx/5DgeMnHaY1VfVB4OHAAcBfDB36CbB/H5kacgCwLd193reqfjHYvzvw//oK1ZDDgc8l2SvJuoPXw4HPDo5pfK5MstvsnUl2B67sIU9rvP+aFxz5UasOBt6f5BLgnMG+BwPLgOf0lqohVXUFcMWs3esBBwHPnXyiNlTV1XTPvM3ef2KSH/QQqSlV9e9JrgXeQFeEQjfpxOuq6ov9JWvCK4CPJzkGOGuw76HAM4F9+wrVEO//ONn2NjKLHzWpqm4EliTZilv+AfKdqvp+j7GakWQHulmv7kX3E+930T33sxvwth6jNSHJw4BNga9X1dWD/x6vBP4U2LzXcA2oqi8xaDfU5FTVtwajDP/ALSPMFwC7O8X4+Hn/NV9Y/KhJSbYYvL2ZW0Z+fr+/qn7UR66GHA0cBZwGPJbup4D/Cfx1Vf2mz2ALXZLDgX3onrl6RZLPA38PvBFnexu7wZpWK1NV9S8TC9OgwT+ynVmsJ95/zQcWP2rVF4CiWxdsRtFNs7wRsLiPUA25U1UdM3j/vSQHAa+sKgftx+8vgR0HMxv+CV2v/Q4zi25q7G6cY99d6J7Fujtg8TMmSU6k+3N+LlVVe08yT2u8/+MVwqLlufUTZfGjNlXV9sPbSbak60d+NN1PwDVeaw9mdpv5k/oGYIckAaiqs1b6lbq9fj0zulZVP0/yPQufyamq37d1JlkXeAndeksfw5bPcTtojn27Ay8Hrp5wlhZ5/zUvWPyoaUm2AV7NLc+avLiqblr1V2kN+Alw5Eq2C3jUxBO1Y+skxw1tbzm8XVVPmONrtAYluRvwMuCv6RY83amqft5vqoWvqs6ceT+YYe+1wJ2A5zvZxPh5/zVfWPyoSUm2oyt6tgXeChxgy9XkVNUj+s7QsCfO2na0YYIGz1w9GVgKbF9VN/QcqSlJ/pzuH92/AQ6rqhN7jtQU7/8YOdvbyPKHC01LbUiyHPgx3bM/f/THRVW9eOKhGpLkyas6XlWfmVQWaZKSrAB+SzfZyvBfwKF77mG9XoI1IMm36Z7rPJxuspU/YLvteHn/x2uDDXauh+91Rt8x5nTc5+5wZlXt3HeOGY78qFUHsPIHLzV+j1/FsQIsfsYkyXnM/Xt/5h/fO0w4UlOqysXF+3Mj3fOFTx28htluO37ef80LFj9q0tBMY+pBVT1rZceSbDzJLA3ap+8ALRs877NSVfWzSWVpje22/fL+j1dsexuZxY+alORzrGLkx4e+JyvJ+sBTgKcDD6RbgFNjUFWXzbU/yZ509/8fJpuoOWfyx9Pszyhgq8nGaYfttv3y/mu+sPhRq47oO0DrkqwDPIHuH9w7AesCfwV8vc9cLUnyELr7/3+BH2K74SQ8YmUFqMbOdtt+ef81L1j8qFVrVdUJcx1I8hbg5AnnaUqS/wD2Ao4H3g18Dbikqk7qM1cLktwP2A9YAvwU+Djd5DeP7DVYO46lK/Y1Yatqt9VEHGLhP162vY3GBy/Vqvck+cvhHUkWJTkGeHA/kZqyHfBz4ELgu4Npxp2AYjK+C+wNPL6q/k9VvYs5ZjzU2LgEe4+S3D/J25J8YfA6YvADAY3fV5O8Mok/eFev/A2oVj0G+FKSO1XVZwYtWJ8ErmPVQ/NaA6rqwUkeQNdy9ZUkVwPrJrlnVf2k53gL3VPoRn5OTPIl4GP4D/JJ2jTJO1d20Gn2xyfJw+haq5YOXgF2BE5K8uSqOr3PfA3YETgUODPJi6rKFmf1wuJHTaqqS5M8Gvhyko2AZwBnVNXLeo7WjKr6LvA64HVJdqZrw/pWksurao9+0y1cVXUscGySu9A9Y/VSYOMkRwHHVtXxvQZc+H5NN+mBJu91wJJZ7bWfTfI14PXA43pJ1Yiquh54aZKH0o0CXQ6swGn214hutjd/jjUKFzlVk5LM9NxvAnwYOAF468xxF1sbryQvrKp3z7E/wF5V5TNXY5LkDlV186x9dwOeBuxbVa61MUZJzqoqn/npQZKLqmrOFrck36uq+086U2uSPAp4B/Bl4D10xQ+w8pkoNZq7rbdz7b3bt/qOMadPfWWxi5xK88Dbht6fC2w8tM/F1sbv2XQTHfyB6n4aY+EzXt9i1gP3g7Vl/m3w0nht0neAhl2/imM3TixFo5J8jG4Zg6dX1Xl951G7LH7UpFXNbJVk90lmkSbMvoh++UxbfzZfyfNWwbXFJuGrVXX0XAeSbFxV/zvpQAuNs72NxuJH+mOfALboO8QCt0OS6+bYP9P7vd6kAzXkHklW+mxbVR05yTANste8Pwev4tiyiaVo1OzCx8Wt1ReLH+mP+ZPx8TuvqnbsO0SjFgN3xd/nfdnM2d76UVUf6jtD61zcWvOBxY/0x/zJrBayq6rq0L5DNMzZ3nqS5P+x8j/fq6oOmGSe1ri49ZiVbW+jsvhRk5J8jrn/Egxw9wnHadEn+w7QMEd8+vVTRyB68/k59m0BHEg3Iqrx+qPFrZP4w0ZNnMWPWnXEah7TmnFNkm2q6uLB9NYfpOv9vhTY36nGx+qJSe5YVTdBt+I98BfAZVX1mX6jNeF3fQdoVVV9euZ9kq2AV9GNRLwZ+EBfuVrh4taaLyx+1KSVrSOTZHNgP5xuedxeAhwzeL8E2AG4D90K4O8A/rSfWE34d+AA4OIk9wVOA/4D2CfJLlX1T72mW/j+YWidsT9i4T9eSR4IvJruz5rDgefPXvdK4+Pi1uMT295GZvGj5iXZkG6BxyV0s80c22+iJtw8M/IA7AN8uKp+SvfTwLeu4ut0+/1JVV08eP9M4KNV9aIka9E9i2LxM15H0LXczrQfzm77cY2xMUnySWBnuv8GLwWWA+t1g8+/X+9KE1JVy4BlSQ6m+4GYNBEWP2pSknWBJ9ENv9+PruDZqqo26zVYO1Yk2YSu/3tv4LChY+v0E6kZw//YfhTdT7+pqt8lWTH3l2gNegXw46q6CiDJM7ml5fOQ/mI1YRe63/8HAf842DdchG7VR6jWVdWKJC8F3t53FrXB4ketuppupfvXAKdUVSV5Us+ZWvI6unU1FgPHVdUFAEkeDvygz2ANODfJEcAVwH3pZl4iyQa9pmrH+4BHAyTZC3gT8CLgIcBS4Kn9RVvYqmrLvjNopZyIZQ2w7W00i/oOIPXkVcDawFHAPyXZuuc8TamqzwP3Bh5YVc8dOrQM2LefVM14LnAtsCXwmKr61WD/g3Cyj0lYPNRetS+wtKo+XVWvpStGNUFJtk7y6iTn952lcc76pomx+FGTqurtVbUb3WJrAT4L3CvJK5Lcr990C1+SbYBPAd9I8tEkmwJU1Y1VdUO/6Ra2qvp1Vb25ql5SVecM7T+1qj7SZ7ZGLE4y03WxN91aJzPsxpiAJJskOTDJt4AL6O77kp5jLXhJrk9y3Ryv64F79Z1P7fAPWjUpyYHAKcDZVXUYcFiS7en+Avwi4EjQeH0Q+DDdqt5PAN4FPLnXRI1IciKrXuhx70nmadBHgZOTXEu34Ok3AAYz7/2yz2ALXZLn0v0ZvxnwCeA5wH9V1T/3GqwRVbVu3xkWsm62N7sHR2Hxo1ZtBrwTeECSc4FTgW8CR1TVq3pN1oZ1q+rowfvDkzi97+QcNMe+3YGX0z0LpzGqqsOSfBXYBDi+qmYK0UV0z/5ofN5DN7X70wczjeEim1J7LH7UpKo6CGAwve/OwB7As4Gjk/yiqh7UZ74GrJ1kR255yHWd4W3XOhmfqjpz5v1ggonXAneiW+/ki70Fa0hVnT7Hvov6yNKYe9Eta3Bkko3pRn/u2G8kSZNm8aPWrQOsB6w/eF0JnNdrojb8BDhyJduFa52MVZI/pyt6fgMcVlUn9hxJGruqupZukpujkmxGt6D11UkuBI511F/TztneRmPxoyYlWQpsC1wPnEHX9nZkVf2812CNqKpH9J2hVUm+DdyDbn2f0wb7dpo57qibFqoku8+MulXV5XSzGx6R5P50hZCkBlj8qFVb0LX6XEy33snlwC96TdSQJLMnNyi66ZfPrqrre4jUkhuBG+jWk5m9poyjblrI3gvsNHtnVX0PcNIDqREWP2pSVT02SehGf/agW+17uyQ/A06rqtf3GnDhe/wc++4G7JDkgKr62hzHtQY46iZJC1DZ9jYqix81azDL0vlJfkE3xewvgX2AXQGLnzGqqmfNtT/JvekeQt5tsonakeQcumneTwW+WVWX9ptImpitkhy3soNV9YRJhpHUD4sfNSnJi+lGfPYEbqKb5vo0uvVnnPCgJ1V1WRJnXxqvv6b7vf9nwOuT3IWuEDoVOLWqzugznDRG1wBv6zuEpH5Z/KhVWwKfAl5aVVf1nEUDgwePf9t3joWsqs4HzgeWAiTZkO5h7wPpHgBf3F86aaxuqKqT+w4hjUNsexuZxY+aVFUv6ztDy5J8ju7h+mF3o1v48W8mn6gdSRYDO3LLyOfWdJN+vJ/B7G/SAvXzJPesqp8AJPlb4CnAZcAhVfWzXtNJmgiLH0l9OGLWdgE/BS6uqt/1kKcl1wEX0q12/8qq+mHPeaRJ2QD4HUCSvYA3Ay8CHkI3Ejp79kNJC5DFj6SJG7X1JMlpVfWwcedpzHOAhw1+fdZg3Z/T6GY5vKLXZNJ4LRoa3dkXWFpVnwY+neTsHnNJa4Rtb6Ox+JE0n63dd4CFpqo+CnwUIMmd6WY33BN4U5K1qurefeaTxugOSe5QVTcDewPPGz7WUyZJE+b/7JLms9nPBWkNGMzwthu3PPezC/BjulkPpYXqo8DJSa4Ffg18AyDJfemWOpDUAIsfSWpIkv8BtgCW0U1v/Tbg9Kq6oddg0phV1WFJvko3scrxg7XeABbRPfsjTS1nexudxY+k+Sx9B1iAngmcN/QPP6kZVXX6HPsu6iOLpH4s6juAJK3CM/oOsNBU1bnAtkk+lGRZkm8P3u/QdzZJksbN4kfSxCU5IMnBQ9tXJLkuyfVJXjCzf7Agp9agJE8EjgVOBp5NN+vbyXQzXj2xz2ySpNVUsOjm+fmab2x7k9SH5wOPHdq+uqo2TbI2cDxwVD+xmnAo8GdVdenQvnOSfA34r8FLkqQFyZEfSX1YVFU/Hdr+JEBV/QZYp59IzbjjrMIHgMG+O048jSRJE+TIj6Q+rD+8UVVvBEiyCLh7L4nacVOSLarqR8M7k9wbmIcNCpKkUSxa7hxBo3DkR1Ifjk/yhjn2H0rX9qbxeT3wlST7J9k+yXZJnkV331/XczZJksbKkR9JfTgYeH+SS4BzBvseTLf2zHN6S9WAqvpskh8C/0i3tkmAC4D/W1XnrPKLJUmachY/kiauqm4EliTZCth2sPs7VfX9HmM1Y1Dk/G3fOSRJa4aLnI7OtjdJE5dkiyRb0D1jcs7gddPQfo1RkmcmOTPJjYPXsiQWQ5KkBc+RH0l9+AJQdC1XMwq4B7ARsLiPUC0YFDkHAi8DzqL7b7ATcHgSqurDfeaTJGmcLH4kTVxVbT+8nWRL4BXAo4E39hCpJX8PPGnWdNdfS/IU4GOAxY8kTSHb3kZj25uk3iTZJskxwBeBM4EHVdW7+k214K23inV+1pt4GkmSJsiRH0kTl2Q74NV0kx28FTigqvyZ1WT8ejWPSZI09Sx+JPXhHODHdM/+7Arsmtzy+E9VvbinXC14YJJz59gfYKtJh5Ek3X7O9jY6ix9JfTiAboIDTd4D+w4gSVJfLH4kTVxVHdN3hlZV1WWjnJfktKp62LjzSJI0SRY/kiYuyedYxchPVT1hgnE0t7X7DiBJGpFtbyOz+JHUhyP6DqBbZVuiJGnBsfiR1Ie1quqEuQ4keQtw8oTzSJKkBlj8SOrDe5K8tKq+MLMjySLgg8A9+4ulIbn1UyRJ84Vtb6Ox+JHUh8cAX0pyp6r6TJJ1gE8C1wGP7zeaBp7RdwBJkta0RX0HkNSeqroUeDTwL0meD3wFuKiqnl5VN/UaboFLckCSg4e2r0hyXZLrk7xgZn9Vnd9PQkmSxseRH0kTl2SnwduXAx8GTgD+fWZ/VZ3VV7YGPB947ND21VW1aZK1geOBo/qJJUlaXS5yOjqLH0l9eNvQ+3OBjYf2FfCoiSdqx6Kq+unQ9icBquo3g/ZDSZIWLIsfSRNXVY9c2bEku08yS4PWH96oqjfC7yecuHsviSRJmhCLH0nzzSeALfoOsYAdn+QNVfWaWfsPpWt7kyRNm4JFN/cdYjpY/Eiab5xiebwOBt6f5BLgnMG+BwPLgOf0lkqSpAmw+JE031TfARayqroRWJJkK2Dbwe7vVNX3e4wlSdJEWPxImrgkn2PuIif43MlYJZlpKbyZW0Z+fr+/qn7URy5J0uoLzvY2KosfSX04YjWP6fb7Al3hOdxeWMA9gI2AxX2EkiRpEix+JE1cVZ081/4kmwP7AXMe1+1XVdsPbyfZEngF3aKzb+whkiRJE2PxI6lXSTYEngYsATYFju03URuSbAO8GtiNbo2lF1fVTf2mkiStFhc5HZnFj6SJS7Iu8CTg6cD96Aqerapqs16DNSDJdnRFz7bAW4EDqsq/MiVJTbD4kdSHq4FvAa8BTqmqSvKknjO14hzgx3TP/uwK7Jrc8vhPVb24p1ySpAYleSzwDrpnTt9fVW+edfxOwIeBhwI/BfatqktX93oWP5L68Cq6Z3uOAv4zycd7ztOSA3A6cUlacKax7S3JYuA9wJ8BlwPfTnJcVX1n6LQDgJ9X1X2T7Ae8Bdh3da9p8SNp4qrq7cDbB2vNLAE+C9wrySuAY6vqol4DLmBVdUzfGSRJGtgVuKSqfgCQ5GPAE4Hh4ueJwCGD958C3p0kVbVaP8iz+JE0cUkOBE4Bzq6qw4DDkmxPVwh9Edi6z3wL2SrWWAKgqp4wwTiSpLZtSteKPeNyuol45jynqm5O8ku6NQGvXZ0LWvxI6sNmwDuBByQ5FzgV+CZwRFW9qtdkC5/rKEnSAnMVZ375ELJh3zlWYu0ky4a2l1bV0sH7zHH+7B/QjXLOyCx+JE1cVR0EkGQtYGdgD+DZwNFJflFVD+oz3wK3VlWdMNeBJG/BNZYkaepU1WP7zrCaLgc2H9reDLhyJedcnuQOwPrAz1b3gotW9wslaQ1YB1iP7g+y9en+wDuj10QL33uS/OXwjiSLkhwDPLifSJKkRn0b2CbJfQY/EN0POG7WOccBzxy8fyrwtdV93gcc+ZHUgyRL6daZuZ6u2DkVOLKqft5rsDY8BvhSkjtV1WeSrAN8ErgOeHy/0SRJLRk8w/NC4Mt0U11/sKouSHIosKyqjgM+AHwkySV0Iz773Z5r5nYUTpK0WpJ8CdgQOJ+u8DkNOP/2/CRHo0uyGd1fNO8CngGcUVUv6zeVJEnjZ/EjqRfpVtbclu55nz2A7eh+onNaVb2+z2wLWZKdBm83oVs07gTgrTPHq+qsPnJJkjQJFj+SejUYhdiTrgDaB7h7VW3Qb6qFK8mJqzhcVfWoiYWRJGnCLH4kTVySF9MVO3sCN9FNc33a4NfzqmpFj/GalWT3qjq97xySJI2LxY+kiUtyJIO1farqqr7zqJPkR1W1Rd85JEkaF4sfSRIASX5cVZvf+pmSJE0n1/mRJM3wp2GSpAXNdX4kqSFJPsfcRU6Au084jiRJE2XbmyQ1JMnDV3W8qk6eVBZJkibN4keSRJLNgf2q6vC+s0iSNC4+8yNJjUqyYZIXJPk6cBKwcc+RJEkaK5/5kaSGJFkXeBLwdOB+wLHAVlW1Wa/BJEmaANveJKkhSX4NfAt4DXBKVVWSH1TVVj1HkyRp7Gx7k6S2vApYGzgK+KckW/ecR5KkiXHkR5IalGQrYAmwH7AN8Hrg2Kq6qNdgkiSNkcWPJDUkyYHAKcDZVXXzYN/2dIXQvlXlSJAkacGy+JGkhiQ5AtgDeABwLnAq8E3gtKr6WZ/ZJEkaN4sfSWpQkrWAnekKoYcNXr+oqgf1GkySpDFyqmtJatM6wHrA+oPXlcB5vSaSJGnMHPmRpIYkWQpsC1wPnAGcDpxeVT/vNZgkSRPgVNeS1JYtgDsBPwGuAC4HftFrIkmSJsSRH0lqTJLQjf7sMXhtB/yMbtKD1/eZTZKkcbL4kaRGJdkM2JOuANoHuHtVbdBvKkmSxsfiR5IakuTFdMXOnsBNDKa5Hvx6XlWt6DGeJElj5WxvktSWLYFPAS+tqqt6ziJJ0kQ58iNJkiSpCc72JkmSJKkJFj+SJEmSmmDxI0m6TZIsT3J2kvOTfDLJnW/HZz0iyecH75+Q5JWrOHeDJH+/Gtc4JMlBo+6fdc4xSZ56G661ZZLzb2tGSdJkWPxIkm6rX1fVQ6pqO+B3wPOHD6Zzm/9+qarjqurNqzhlA+A2Fz+SJM2w+JEk3R7fAO47GPG4MMl7gbOAzZM8JslpSc4ajBDdFSDJY5N8N8kpwJNnPijJ/knePXi/cZJjk5wzeO0BvBnYejDqdPjgvIOTfDvJuUn+eeizXp3ke0m+Atz/1r6JJM8dfM45ST49azTr0Um+keSiJPsMzl+c5PCha//d7b2RkqTxs/iRJK2WJHcAHgecN9h1f+DDVbUjcCPwGuDRVbUTsAx4WZK1gaOBxwN/CtxzJR//TuDkqnowsBNwAfBK4PuDUaeDkzwG2AbYFXgI8NAkeyV5KLAfsCNdcbXLCN/OZ6pql8H1LgQOGDq2JfBw4C+B9w2+hwOAX1bVLoPPf26S+4xwHUlSj1znR5J0W62T5OzB+28AHwDuBVxWVacP9u8OPAj4ZhKAtegWU30A8MOquhggyb8Dz5vjGo8C/hagqpYDv0zyJ7POeczg9T+D7bvSFUPrAsdW1a8G1zhuhO9puyRvoGutuyvw5aFjnxgs/npxkh8MvofHADsMPQ+0/uDaF41wLUlSTyx+JEm31a+r6iHDOwYFzo3Du4ATqmrJrPMeAqypBeYCvKmq/m3WNQ5cjWscA/xVVZ2TZH/gEUPHZn9WDa79oqoaLpJIsuVtvK4kaYJse5MkjcPpwJ5J7guQ5M5J7gd8F7hPkq0H5y1Zydd/FXjB4GsXJ1kPuJ5uVGfGl4FnDz1LtGmSjYCvA09Ksk6Sdela7G7NusBVSe4I/PWsY09LsmiQeSvge4Nrv2BwPknul+QuI1xHktQjR34kSWtcVV0zGEH5aJI7DXa/pqouSvI84AtJrgVOAbab4yNeAixNcgCwHHhBVZ2W5JuDqaS/OHju54HAaYORpxuAv6mqs5J8HDgbuIyuNe/WvBY4Y3D+efxhSa1M8gAAAG1JREFUkfU94GRgY+D5VfWbJO+nexborHQXvwb4q9HujiSpL6laU90HkiRJkjR/2fYmSZIkqQkWP5IkSZKaYPEjSZIkqQkWP5IkSZKaYPEjSZIkqQkWP5IkSZKaYPEjSZIkqQkWP5IkSZKa8P8BmhhfZqpVwbYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x864 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------lstm-----------------\n",
      "(?, 32)\n",
      "Training iter #1500:   Batch Loss = 1.979642, Accuracy = 0.1653333306312561\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 1.8602725267410278, Accuracy = 0.18221920728683472\n",
      "Training iter #300000:   Batch Loss = 0.159613, Accuracy = 0.9559999704360962\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.1847238838672638, Accuracy = 0.935188353061676\n",
      "Training iter #600000:   Batch Loss = 0.090398, Accuracy = 0.9739999771118164\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.1605163961648941, Accuracy = 0.9467254877090454\n",
      "Training iter #900000:   Batch Loss = 0.046877, Accuracy = 0.9860000014305115\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.14072047173976898, Accuracy = 0.9531727433204651\n",
      "Optimization Finished!\n",
      "Training time is: 763.8484580516815 seconds\n",
      "FINAL LSTM RESULT: Batch Loss = 0.1496160626411438, Accuracy = 0.9538514018058777\n",
      "Testing Accuracy: 95.38514018058777%\n",
      "\n",
      "Precision: 95.5310242638082%\n",
      "Recall: 95.38513742789277%\n",
      "f1_score: 95.38797023936841%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[480   0  16   0   0   0]\n",
      " [  0 452  17   0   2   0]\n",
      " [  9  33 378   0   0   0]\n",
      " [  0   0   0 444  47   0]\n",
      " [  0   8   0   0 524   0]\n",
      " [  2   0   0   2   0 533]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.28775     0.          0.542925    0.          0.          0.        ]\n",
      " [ 0.         15.337631    0.5768578   0.          0.06786563  0.        ]\n",
      " [ 0.3053953   1.1197829  12.826604    0.          0.          0.        ]\n",
      " [ 0.          0.          0.         15.066169    1.5948422   0.        ]\n",
      " [ 0.          0.2714625   0.          0.         17.780794    0.        ]\n",
      " [ 0.06786563  0.          0.          0.06786563  0.         18.08619   ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz8AAAM+CAYAAAA0Cb/mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4JVV57/Hvr1tQsRGQQRmEVkIQEEFsQTExGIegIQ6JXsAh4BDUXJMIGCd4FI1KNCJGwRhERI0CMRGvIgYwuQRFUBplFgUUAjRT0zKJIvR57x9V57o5nKG6OafP3tT38zz76V1Vq2qtWqfR8/b71qpUFZIkSZL0ULdgvgcgSZIkSWuCwY8kSZKkXjD4kSRJktQLBj+SJEmSesHgR5IkSVIvGPxIkiRJ6gWDH0mSJEm9YPAjSZIkqRcMfiRJkiT1gsGPJEmSpF542HwPQJIkSdLq2zOp5fM9iCmcD6dV1Z7zPY5xBj+SJEnSCFsOLJ3vQUwhsNF8j2GQZW+SJEmSesHMjyRJkjTqFg5pTmPl2HyP4H6GdJYkSZIkaXYZ/EiSJEnqBcveJEmSpFEWYGHmexSTWznfA7g/Mz+SJEmSesHgR5IkSVIvWPYmSZIkjbQM72pvQ1b3NqyzJEmSJEmzyuBHkiRJUi9Y9iZJkiSNsgAPG9LV3oaMmR9JkiRJvWDwI0mSJKkXLHuTJEmSRlkY4tXehouzJEmSJKkXDH4kSZIk9YJlb5IkSdKoW+hqb12Y+ZEkSZLUCwY/kiRJknrBsjdJkiRplCUju9pbkuOAvYCbq+rJ7b6TgG3bJusDt1XVzpOcezVwJ7ASuK+qlszUn8GPJEmSpPlyPHAU8IXxHVW19/j3JEcAt09z/nOqannXzgx+JEmSJM2LqjoryeLJjiUJ8L+AP5yt/gx+JEmSpFH20H3J6e8DN1XVFVMcL+D0JAX8c1UdM9MFDX4kSZIkzZWNkiwd2D6mS5DS2hc4YZrjz6qqZUk2Ac5IcnlVnTXdBQ1+JEmSJM2V5V0WIpgoycOAPwWeNlWbqlrW/nlzkpOBXQGDH0mSJOkh7aH3ktPnAZdX1XWTHUzyKGBBVd3Zfn8B8P6ZLvqQLA6UJEmSNPySnACcA2yb5Lokr28P7cOEkrckmyU5td18LPDdJBcCPwC+WVX/MWN/VTV7o5ckSZK0Ri1Ze2Etfeyi+R7GpHLdHeevTtnbXLHsTZIkSRplI/yS0zXNWZIkSZLUCwY/kiRJknrBsjdJkiRp1D30VnubE2Z+JEmSJPWCwY8kSZKkXrDsTZIkSRplAR5mTqMLZ0mSJElSLxj8SJIkSeoFy94kSZKkUZa42ltHZn4kSZIk9YLBjyRJkqResOxNkiRJGnULzWl04SxJkiRJ6gWDH0mSJEm9YNmbJEmSNMqCZW8dOUuSJEmSesHgR5IkSVIvWPYmSZIkjTJfctqZmR9JkiRJvWDwI0mSJKkXLHuTJEmSRp2rvXXiLEmSJEnqBYMfSZIkSb1g2ZskSZI0yoKrvXVk5keSJElSLxj8SJIkSeoFy94kSZKkkRZXe+vIWZIkSZLUCwY/kiRJknrBsjdJkiRplLnaW2dmfiRJkiT1gsGPJEmSpF6w7E2SJEkaZcHV3jpyliRJkiT1gsGPJEmSpF6w7E2SJEkada721omZH0mSJEm9YPAjSZo1SR6Z5BtJbk/ylQdxnVclOX02xzZfkvx+kp/M9zgkSZa9SVIvJXklcBDwJOBO4ALgg1X13Qd56ZcDjwU2rKr7VvciVfUl4EsPcixzLkkB21TVlVO1qarvANuuuVFJ6p3E1d46cpYkqWeSHAR8HPgQTaCyJfAp4CWzcPmtgJ8+mMDnoSSJ/8goSUPE4EeSeiTJesD7gf9dVV+tql9W1b1V9Y2q+tu2zcOTfDzJsvbz8SQPb4/tkeS6JAcnuTnJDUle2x57H/AeYO8kdyV5fZLDkvzLQP+Lk9R4UJBk/yQ/S3Jnkp8nedXA/u8OnLd7kvPacrrzkuw+cOzMJH+X5Oz2Oqcn2WiK+x8f/9sHxv/SJC9K8tMkK5K8e6D9rknOSXJb2/aoJGu3x85qm13Y3u/eA9d/R5Ibgc+N72vP2brtY5d2e7Mky5Ps8aB+sJKkTgx+JKlfngk8Ajh5mjaHAM8AdgZ2AnYFDh04/jhgPWBz4PXA0Uk2qKr30mSTTqqqRVX12ekGkuRRwCeAF1bVusDuNOV3E9s9Bvhm23ZD4GPAN5NsONDslcBrgU2AtYG3TdP142jmYHOaYO0zwKuBpwG/D7wnyRPbtiuBA4GNaObuucBfAlTVs9s2O7X3e9LA9R9DkwU7YLDjqroKeAfwpSTrAJ8Djq+qM6cZryTNbOGC4fwMmeEbkSRpLm0ILJ+hLO1VwPur6uaqugV4H/CageP3tsfvrapTgbtY/WdaxoAnJ3lkVd1QVZdO0uaPgSuq6otVdV9VnQBcDvzJQJvPVdVPq+pXwL/SBG5TuZfm+aZ7gRNpApt/rKo72/4vBZ4CUFXnV9W5bb9XA/8M/EGHe3pvVd3Tjud+quozwBXA94FNaYJNSdIaYPAjSf1yK7DRDM+ibAZcM7B9Tbvv/19jQvB0N7BoVQdSVb8E9gbeBNyQ5JtJntRhPONj2nxg+8ZVGM+tVbWy/T4enNw0cPxX4+cn+d0kpyS5MckdNJmtSUvqBtxSVb+eoc1ngCcDn6yqe2ZoK0maJQY/ktQv5wC/Bl46TZtlNCVb47Zs962OXwLrDGw/bvBgVZ1WVc+nyYBcThMUzDSe8TFdv5pjWhX/RDOubarq0cC7gZneJFjTHUyyiGbBic8Ch7VlfZK0+kLzktNh/AwZgx9J6pGqup3mOZej2wf910myVpIXJvlI2+wE4NAkG7cLB7wH+JeprjmDC4BnJ9myXWzhXeMHkjw2yYvbZ3/uoSmfWznJNU4FfjfJK5M8LMnewPbAKas5plWxLnAHcFeblXrzhOM3AU98wFnT+0fg/Kp6A82zTJ9+0KOUJHVi8CNJPVNVH6N5x8+hwC3AtcBbgK+1TT4ALAUuAi4GftjuW52+zgBOaq91PvcPWBYAB9NkdlbQPEvzl5Nc41Zgr7btrcDbgb2qavnqjGkVvY1mMYU7abJSJ004fhjw+XY1uP8108WSvATYk6bUD5qfwy7jq9xJkuZWqqbNzkuSJEkaYks2eGQt3eMJ8z2MSeVrPz6/qpbM9zjGmfmRJEmS1AsGP5IkSZJ6YbqlTiVJkiSNgiFcWW0YmfmRJEmS1AsGP5I0gySHJ3nrfI9jJkkWJ6nxF5gm+VaS/Wa5j/2TfHc2r7kmtMtqn5XkziRHzEP/QztvSc5M8oY5uvbHkrxp5paStGYY/EjSNJJsDPw58M/zPZZVVVUvrKrPr6n+JgZfq3H+45Ocm2TFxAAlyX8keTCrBR0ALAceXVUHT9L38Uk6L+e9qu1nuNaDmre5GtcU1786yfNW4ZR/AA5JsvZcjUkS7UtOFwznZ8gM34gkabjsD5xaVb+a7QvPxi+7DzHvAj4PPAF46Xiw077U9GdVtfRBXHsr4LLy/Q5rVFXdAFwOvHi+xyJJYPAjSTN5IfDf4xtJ9khyXZKDk9yc5IYkrx04vl6SLyS5Jck1SQ5NsqA9tn+Ss5McmWQFcNiEfbcl+VmS3dv917Z97Ddw/T9O8qMkd7THD5tq4IPlTEl+J8l/J7k9yfIkJw20e1KSM9qMy08GX9aZZMMkX2/7+wGw9TRzdVb7521J7kryzCQL2jm4pr2XLyRZb4rznwD8V1XdDpwHPDHJo4F3Au+ept/xse6e5Lz2Hs9Lsnu7/3hgP+Dt7bieN+G8A4BXDRz/Rrt/u3YOb0tyaZIXz9D+nUmuakvrLkvyspnGPNW8tdd7XZIfJ/lFktOSbNXuT/v35eb2Xi9K8uSpxjXJPD0/yeXtuUfR/Jvx+LGtk/xXklvbvydfSrJ+e+yLwJbAN9rrv73d/5UkN7bXOyvJDhO6PBP4445zIUlzyuBHkqa3I/CTCfseB6wHbA68Hjg6yQbtsU+2x54I/AFNydxrB87dDfgZsAnwwYF9FwEbAl8GTgSeDvwO8GrgqCSL2ra/bK+5Ps0vlG9O8tIO9/F3wOnABsAW7ThJ8ijgjLbfTYB9gU8N/AJ7NPBrYFPgde1nKs9u/1y/qhZV1Tk0mbP9gee0c7IIOGqK8y8Bnt/+sr0EuKwd98er6rbpbi7JY4BvAp+gmcePAd9MsmFV7Q98CfhIO65vD55bVcdMOP4nSdYCvkEzZ5sAfwV8Kcm2k7VvL3UV8Ps0P//3Af+SZNPpxt16wLy1P9N3A38KbAx8BzihbfeC9pzfpfl7sDdw6zTjGpynjYB/Bw4FNmrH/KzBJsDhwGbAdsDjgcPaeXoN8D/An7TX/0h7zreAbdp5+mE7hkE/BnbqMA+SVlfSrPY2jJ8hY/AjSdNbH7hzwr57gfdX1b1VdSpwF7BtkoU0v4i+q6rurKqrgSOA1wycu6yqPllV9w2U0v28qj5XVSuBk2h+4Xx/Vd1TVacDv6EJhKiqM6vq4qoaq6qLaH4h/oMO93EvTenXZlX166oaf/h+L+Dqtv/7quqHNL8cv7y9nz8D3lNVv6yqS2jK0lbFq4CPVdXPquoumtK2fTJ5yd/hNMHDf9MEXWsBT6HJNHy5zSq8ZYp+/hi4oqq+2N7HCTTlVg8IADp6Bk2g9vdV9Zuq+i/gFJrgcFJV9ZWqWtb+bE4CrgB2Xc3+3wgcXlU/rqr7gA8BO7fZn3uBdYEnAWnb3NDxui+iKf/7t6q6F/g4cOPAPVxZVWe0f/duoQkip/37VVXHtX/f76EJlHaakN27k+a/I0madwY/kjS9X9D8ojno1vYX0nF30/yivBGwNnDNwLFraDJE466dpI+bBr7/CqCqJu5bBJBktyT/N01Z3e3Am9p+Z/J2mn/V/0FbwjWewdkK2K0t7botyW00AcvjaDIOD5sw5sF762IzHjgfDwMeO7FhVa2oqr2raifgH2myU39FU/Z2CfA84E1Jtu/Qz3hfm0/Stuu4r62qsa7XS/LnSS4YmMcn0+1nM5mtgH8cuNYKmp/f5m0gdhRNgHhTkmPa8sAuNmPg59k+A/X/t5NskuTEJNcnuQP4l+nuIcnCJH/flvvdAVzdHho8Z11g2sydJK0pBj+SNL2LaMqLuljObzMs47YErh/YfrAP3H8Z+Drw+KpaD/g0A89sTKWqbqyqv6iqzWiyCp9K8js0v/j+d1WtP/BZVFVvBm4B7qPJRA3ez5TdTLJvGQ+cj/u4f8A3mQOAc9ts047A0qr6DXAxTVAxUz/jfV0/SdvJTBz7MuDxaZ/XmuR692vfZmQ+A7wF2LCq1qcJ2LrUfEw2b9cCb5zwc3lkVX0PoKo+UVVPA3ag+fv5t9Nca9ANDPw8k4T7/3wPb6/xlKp6NE3Z5eA9TLz+K4GX0ASm6wGLxy890GY74MIZxiXpwZrvVd1c7U2SHhJOpVtZGW3Z2r8CH0yybvsL8UE0/3o+W9YFVlTVr5PsSvPL54ySvCLJFu3mL2h+iV1JU8r1u0lek2St9vP0JNu19/NVmoUZ1mkzLtO9N+gWYIzm2Z5xJwAHJnlC+9zSh4CTJmTOJo51E+B/0z5rAvwceE57/hKaZ6YmOrW9j1cmeViaFeK2b++vi5smjPv7NM9Xvb2dkz1oSuhOnKL9o2jm9Jb2Hl7L5EHaZCabt08D7xp/9irNQhqvaL8/vc0ArtWO8dc0P8vJxjXRN4EdkvxpW3r41zRZvnHr0pRx3pZkc34bVI2beP11gXuAW4F1aH6+E/0BzXNBkjTvDH4kaXpfAF6U5JEd2/8VzS+kPwO+S5OpOW4Wx/OXwPuT3Am8hybY6uLpwPeT3EWTOfqbqvp5Vd1J8wD9PjTZjhuBDwMPb897C03J3Y3A8cDnpuqgqu6mWcTh7LZc6xk09/5FmhXNfk7zi/pfzTDWj9I883RXu3048Ic02ZCvT7bkdVXdSvP80sE0v4i/HdirqpbP0Ne4zwLbt+P+WptlejHNan/LgU8Bf15Vl0/R/jKa57vOoQkQdgTO7tLxZPNWVSfT/BxObMvJLmnHAvBomizTL2hK8W6lmbMHjGuSvpYDrwD+vj1vmwnjfB+wC3A7TaD01QmXOBw4tL3+22j++7iGJiN2GXDuYON2wYftgQeMRZLmQ3zlgSRNL8mHgJur6uPzPRZplKR5We1VVfWp+R6L9FC2ZKN1aule2873MCaVz19wflU9mJdUzypfsCdJM6iqGd8xI+mBqurg+R6DJA2y7E2SJElSL5j5kSRJkkZahnJltWHkLEmSJEnqBYMfSZIkSb1g2Ztm1UZrL6zF66w138PopWW3d32liObCSv/XdN4snPKNQZI0N27jau6u5V1eYrxmBFg4PMMZZv7ftWbV4nXWYumzF8/3MHrpsG884NUnWoPu2MDXBsyXR9/i/+FLWrOOYWhWbtYqsuxNkiRJUi+Y+ZEkSZJGWXC1t46cJUmSJEm9YPAjSZIkqRcse5MkSZJGmi857cpZkiRJktQLBj+SJEmSesGyN0mSJGmUBVjgO8+6MPMjSZIkqRcMfiRJkiT1gmVvkiRJ0qhztbdOnCVJkiRJvWDwI0mSJKkXLHuTJEmSRlmAha721oWZH0mSJEm9YPAjSZIkqRcse5MkSZJGWlztrSNnSZIkSVIvGPxIkiRJ6gXL3iRJkqRR5mpvnZn5kSRJktQLBj+SJEmSesGyN0mSJGnULTCn0YWzJEmSJKkXDH4kSZIk9YJlb5IkSdIoS1ztrSMzP5IkSZLmRZLjktyc5JKBfYcluT7JBe3nRVOcu2eSnyS5Msk7u/Rn8CNJkiRpvhwP7DnJ/iOrauf2c+rEg0kWAkcDLwS2B/ZNsv1MnVn2JkmSJI2yAAtHM6dRVWclWbwap+4KXFlVPwNIciLwEuCy6U4azVmSJEmS9FD2liQXtWVxG0xyfHPg2oHt69p90zL4kSRJkjRXNkqydOBzQIdz/gnYGtgZuAE4YpI2k63wUDNd2LI3SZIkadQN72pvy6tqyaqcUFU3jX9P8hnglEmaXQc8fmB7C2DZTNc28yNJkiRpaCTZdGDzZcAlkzQ7D9gmyROSrA3sA3x9pmub+ZEkSZI0L5KcAOxBUx53HfBeYI8kO9OUsV0NvLFtuxlwbFW9qKruS/IW4DRgIXBcVV06U38GP5IkSdIoS2DBaBZ0VdW+k+z+7BRtlwEvGtg+FXjAMtjTGc1ZkiRJkqRVZPAjSZIkqRcse5MkSZJG3fCu9jZUzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMoCLDSn0YWzNI+SHJnkrQPbpyU5dmD7iCQHtd8PTPLrJOsNHN8jyQPeeJvkzCRL2u+Lk1yR5I8G2yfZP8lYkqcMnHdJksXt90VJ/inJVUl+lOT8JH8x+7MgSZIkrRkGP/Pre8DuAEkWABsBOwwc3x04u/2+L82bbF/W9eJJtqB58dPBVXXaJE2uAw6Z4vRjgV8A21TVU4E9gcd07VuSJEkaNgY/8+ts2uCHJui5BLgzyQZJHg5sB/woydbAIuBQmiCoi8cBpwOHVtXXp2hzCrBDkm0Hd7b97dqeOwZQVbdU1Ye735okSZLWmAUZzs+QMfiZR+1bau9LsiVNEHQO8H3gmcAS4KKq+g1NwHMC8B1g2ySbdLj8F4Cjquor07QZAz4CvHvC/h2AC8cDH0mSJOmhwOBn/o1nf8aDn3MGtr/XttkHOLENRr4KvKLDdb8NvCbJOjO0+zLwjCRPmKpBkkOSXJBk2RTHD0iyNMnSW36zssPQJEmSpDXP4Gf+jT/3syNN2du5NJmf3YGz2wUJtgHOSHI1TSDUpfTtIzRZpK8kmXJVv6q6DzgCeMfA7suAndrnkKiqD1bVzsCjp7jGMVW1pKqWbLz2wg5DkyRJ0qxJmtXehvEzZIZvRP1zNrAXsKKqVlbVCmB9mgDoHJpA57CqWtx+NgM2T7JVh2sfCNwBfDbJdEWXxwPPAzYGqKorgaXAB5IsBEjyCJqFFCVJkqSRZPAz/y6mWeXt3An7bq+q5TSZnpMnnHNyux/guUmuG/g8c7xRVRWwH7ApTSZoUu1zRZ8ABp8legOwIXBlkvNpyujeMcnpkiRJ0kjwJafzrKpWMqGcrKr2H/j+gGdxquqggc1HTnLZPQba/gZ4wcCxM9v9x9NkfMbbfYImABrfvgN4Y4dbkCRJ0nwbwpXVhpGZH0mSJEm9YPAjSZIkqRcMfiRJkiT1gs/8SJIkSaMsDOWy0sPIWZIkSZLUCwY/kiRJknrBsjdJkiRppMWlrjsy8yNJkiSpFwx+JEmSJPWCZW+SJEnSKHO1t86cJUmSJEm9YPAjSZIkqRcse5MkSZJGnau9dWLmR5IkSVIvGPxIkiRJ6gXL3iRJkqRRlrjaW0fOkiRJkqReMPiRJEmS1AuWvUmSJEmjztXeOjHzI0mSJKkXDH4kSZIk9YJlb5IkSdIoC6721pGzJEmSJKkXDH4kSZIk9YJlb5IkSdJIi6u9dWTmR5IkSVIvGPxIkiRJ6gXL3iRJkqRRFmCBOY0unCVJkiRJvWDwI0mSJKkXLHuTJEmSRt1CV3vrwsyPJEmSpF4w+JEkSZLUC5a9SZIkSaMscbW3jpwlSZIkSb1g8CNJkiSpFyx7kyRJkkbdAld768LMjyRJkqReMPiRJEmS1AuWvUmSJEmjLPiS047M/EiSJEnqBYMfSZIkSb1g2Ztm1bLbn8xh31g638PopcMO/r35HkKvvfXL35nvIUiS+syXnHbiLEmSJEnqBYMfSZIkSb1g2ZskSZI0yhLGfMlpJ2Z+JEmSJPWCwY8kSZKkXrDsTZIkSRphBYy52lsnzpIkSZKkXjD4kSRJktQLlr1JkiRJI87V3rox8yNJkiSpFwx+JEmSJPWCZW+SJEnSCKuElQvNaXThLEmSJEnqBYMfSZIkSb1g2ZskSZI04lztrRszP5IkSZJ6weBHkiRJUi9Y9iZJkiSNskAtMKfRhbMkSZIkqRcMfiRJkiT1gmVvkiRJ0ggrXO2tKzM/kiRJknrB4EeSJElSL1j2JkmSJI2yxLK3jsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTCmtXezGl04SxJkiRJ6gWDH0mSJEm9YNmbJEmSNOJc7a0bMz+SJEmSesHgR5IkSVIvWPYmSZIkjbBKWBlzGl04S5IkSZJ6weBHkiRJUi9Y9iZJkiSNOFd768bMjyRJkqReMPiRJEmS1AuWvUmSJEkjzrK3bsz8SJIkSeoFgx9JkiRJ8yLJcUluTnLJwL5/SHJ5kouSnJxk/SnOvTrJxUkuSLK0S38GP5IkSdIIq0AtWDCUnw6OB/acsO8M4MlV9RTgp8C7pjn/OVW1c1Ut6dKZwY8kSZKkeVFVZwErJuw7varuazfPBbaYrf4MfiRJkiTNlY2SLB34HLCK578O+NYUxwo4Pcn5Xa/ram+SJEnSSMswr/a2vGtJ2kRJDgHuA740RZNnVdWyJJsAZyS5vM0kTWkkMj9Jjkzy1oHt05IcO7B9RJKD2u8HJvl1kvUGju+R5JRJrntmkiXt98VJrkjyR4Ptk+yfZCzJUwbOuyTJ4vb7oiT/lOSqJD9qI8+/mOZeHjCWJMcnefnAmH6S5MIkZyfZtt2/V3v9C5NcluSNSQ5pH/C6IMnKge9/PXDtC5Oc0LG/85LsPNDude1DZBe19/ySqe5LkiRJmi1J9gP2Al5VVTVZm6pa1v55M3AysOtM1x2J4Af4HrA7QJIFwEbADgPHdwfObr/vC5wHvKzrxZNsAZwGHFxVp03S5DrgkClOPxb4BbBNVT2V5oGtx3TtewqvqqqdgM8D/5BkLeAY4E/a/U8FzqyqD7YPeO0M/Gr8e1V9or2v7Wh+xs9O8qgO/X0K+If23C3ae/699mGzZwAXPcj7kiRJkqaVZE/gHcCLq+ruKdo8Ksm649+BFwCXTNZ20KgEP2fTBj80Qc8lwJ1JNkjycGA74EdJtgYWAYfSBEFdPA44HTi0qr4+RZtTgB3GszDj2v52bc8dA6iqW6rqw91vbVpnAb8DrEtTonhr28c9VfWTDue/Evgizf29uEP7c4DN2++bAHcCd7V93lVVP1+l0UuSJGnuBcYWLBjKz4xDbyqUzgG2TXJdktcDR9H8/ntGW9X06bbtZklObU99LPDdJBcCPwC+WVX/MVN/I/HMT1vLd1+SLWmCoPFf0p8J3A5cVFW/SbIvcALwHZoJ3KRNg03nCzTBy1emaTMGfAR4N7DfwP4dgAvHA5858CfAxVW1IsnXgWuS/CdNMHZCh373Bp4PbAu8hWZuprMn8LX2+4XATcDP2z6/WlXfmOyk9gGzAwDWY8uZ70qSJEkCqmqyhMVnp2i7DHhR+/1nwE6r2t+oZH7gt9mf8eDnnIHt77Vt9gFObIOCrwKv6HDdbwOvSbLODO2+DDwjyROmajDwDM6yaa4zac3ihP1fSnIB8CzgbQBV9QbguTSR7duA46YbbJKnA7dU1TXAfwK7JNlgiuZfSnIdTXrxk21/K2mCoZfTrK9+ZJLDJh141TFVtaSqlqzDxtMNS5IkSZo3oxT8jD/3syNN2du5NJmf3YGz2wUJtqFJj11NEwh1KX37CPB94CtJpsyEtWuNH0ETIIy7DNipfQ6J8WdwgEdP09+twMQg5DHA8oHtV7XP7ry0qq4dGMPFVXUkTTbnz2a4r32BJ7VzcVU7pqnOeRXwBJoA7+iB/qqqflBVh9PM50x9SpIkaQ0rYCwZys+wGaXg52yaFR9WVNXKqloBrE8TAJ1D88v+YVW1uP1sBmyeZKsO1z4QuAP4bDLtT+l44HnQpDeq6kpgKfCBJAsBkjwCmO4aVwCbtYsR0I5vJ+CCqU5oV5TbY2DXzsA107RfQJP1esr4fAAvYZpgsKrupXlW6hlJtmtrKnfp2qckSZI07EYp+LmYZpW3cyfsu72qltNkJk6ecM7J7X6A57YPUY1/njneqF0+bz9gU5pM0KSq6jfAJ2gWAxj3BmBD4Mok59OU0b1jktPHr3EP8Grgc21p278Bb6iq26e88yaYenu7JPUFwPuA/adp/2zg+qq6fmDfWcC44PP7AAAgAElEQVT2STadZmy/osluvQ1YC/hoksvbPvcG/maaPiVJkqShlimWzZZWy2ZZUgewdL6H0UuHHfx78z2EXnvrl78z30PorfVvGL6yCkkPbcewhGW1dGj+x2fHJ29aJ//76+d7GJPa5kkfPH91X3I6F0Yp8yNJkiRJq20klroeRUl2pHnHzqB7qmq3+RiPJEmS1HcGP3Okqi6mWSRAkiRJmjOVdHqhqCx7kyRJktQTBj+SJEmSesGyN0mSJGnErRzCF4oOIzM/kiRJknrB4EeSJElSL1j2JkmSJI2wAld768hZkiRJktQLBj+SJEmSesGyN0mSJGmkhXK1t07M/EiSJEnqBYMfSZIkSb1g2ZskSZI0ygJjCyx768LMjyRJkqReMPiRJEmS1AuWvUmSJEkjrICxmNPowlmSJEmS1AsGP5IkSZJ6wbI3SZIkacS52ls3Zn4kSZIk9YLBjyRJkqResOxNkiRJGmUJY7HsrQszP5IkSZJ6weBHkiRJUi9Y9iZJkiSNsAJWLjCn0YWzJEmSJKkXDH4kSZIk9YJlb5IkSdKIc7W3bsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTCCsveujLzI0mSJKkXDH4kSZIk9YJlb5IkSdIoSyhfctqJsyRJkiSpFwx+JEmSJPWCZW+SJEnSiHO1t27M/EiSJEnqBYMfSZIkSb1g2ZtmVS2AX69T8z2MXnrdGWfN9xB67cNfO2i+h9BbR+925HwPQZLmlS857c7MjyRJkqReMPiRJEmS1AuWvUmSJEkjzrK3bsz8SJIkSeoFgx9JkiRJvWDZmyRJkjTCKmEs5jS6cJYkSZIk9YLBjyRJkqResOxNkiRJGnGu9taNmR9JkiRJvWDwI0mSJKkXLHuTJEmSRlgBKxdY9taFmR9JkiRJvWDwI0mSJKkXLHuTJEmSRpkvOe3MWZIkSZLUCwY/kiRJknrBsjdJkiRpxJUvOe3EzI8kSZKkXjD4kSRJktQLlr1JkiRJI6yAMSx768LMjyRJkqReMPiRJEmS1AuWvUmSJEkjbszV3jox8yNJkiSpFwx+JEmSJPWCZW+SJEnSSAtjMafRhbMkSZIkqRcMfiRJkiT1gmVvkiRJ0ggrXO2tKzM/kiRJknrB4EeSJElSL1j2JkmSJI2ywErL3jox8yNJkiSpFwx+JEmSJPWCZW+SJEnSCHO1t+7M/EiSJEnqBYMfSZIkSb1g2ZskSZI00sKYOY1OnCVJkiRJvWDwI0mSJKkXLHuTJEmSRly52lsnZn4kSZIk9YLBjyRJkqRemLPgJ8mRSd46sH1akmMHto9IclD7/cAkv06y3sDxPZKcMsl1z0yypP2+OMkVSf5osH2S/ZOMJXnKwHmXJFncfl+U5J+SXJXkR0nOT/IX09zL4iS/atv+OMkPkuw3oc1Lk1yU5PIkFyd5abt/pyQXDLTbN8ndSdZqt3dMctHAvS0daLskyZnt93WSfKm99iVJvptkqyQXtJ8bk1w/sL12e97LklSSJ024n0sG5vn29t4uT/LRgXaPTXJKkguTXJbk1KnmSJIkSfNj/CWnw/gZNnOZ+fkesDtAkgXARsAOA8d3B85uv+8LnAe8rOvFk2wBnAYcXFWnTdLkOuCQKU4/FvgFsE1VPRXYE3jMDF1eVVVPrartgH2AA5O8th3LTsBHgZdU1ZOAFwMfbYOvi4GtkqzbXmd34HLgqQPbZw/0s0mSF07S/98AN1XVjlX1ZOD1wI1VtXNV7Qx8GjhyfLuqftOety/w3XbMU/lOOw9PBfZK8qx2//uBM6pqp6raHnjnDHMkSZIkDa25DH7Opg1+aIKeS4A7k2yQ5OHAdsCPkmwNLAIOpflFvYvHAacDh1bV16docwqwQ5JtB3e2/e3anjsGUFW3VNWHu95YVf0MOAj463bX24APVdXP2+M/Bw4H/rbt4zxgt7bt04Cj+e3c7E4TKI77B5q5mGhT4PqBMfykqu6ZbpxJFgHPogmUpgt+xq/5K+ACYPOBPq8bOH7RTNeQJEmShtWcBT9VtQy4L8mWNL/gnwN8H3gmsAS4qM1O7AucAHwH2DbJJh0u/wXgqKr6yjRtxoCPAO+esH8H4MLxwOdB+CEwXkq2A3D+hONL+W2m63vA7kke1Y7rTO4f/Axmfs4B7knynAnXOw54R5JzknwgyTYdxvhS4D+q6qfAiiS7TNc4yQbANsBZ7a6jgc8m+b9JDkmy2RTnHZBkaZKld9ctHYYlSZKk2TRGhvIzbOZ6wYPx7M948HPOwPZ4tmMf4MQ2GPkq8IoO1/028Jok68zQ7svAM5I8YaoG7S/1FyRZ1qHf+5064XtNcnx83/g87AqcV1VXAb+TZGNgUZtJGvQBJmR/quoC4Ik0maHHAOcl2W6GMe4LnNh+P5GpM2u/3z53dCNwSlXd2PZ5WtvnZ2gCvR+1Y76fqjqmqpZU1ZJ1HnhYkiRJGgpzHfyMP/ezI03Z27k0mZ/dgbPbZ2K2Ac5IcjVNINSl9O0jNFmkrySZ8l1FVXUfcATwjoHdlwE7tc8hUVUfbJ+ZefSq3RpPBX7cfr+UJps1aJe2L2ju++nA79EEgNCUk+3D/Uvexsf9X8AjgGdM2H9XVX21qv4S+BfgRVMNLsmGwB8Cx7Zz+7fA3smkT559p6qeQvNzenOSnQf6XFFVX66q19CU7z17qj4lSZKkYbYmMj97ASuqamVVrQDWpwmAzqEJdA6rqsXtZzNg8yRbdbj2gcAdNGVZ0+XUjgeeB2wMUFVX0pSkfSDJQoAkj4Duebl21biPAp9sd30UeNfAanKLacrtjmj7vBO4Ftif3wY/5wBvZZLgp/VB4O0DfT6rLUujXclte+CaaYb5cuALVbVVO7ePB35OE4BNqi2PO5w2WEzyh+PZtXbBhq2B/5mmT0mSJK1hRRjLgqH8zCTJcUluHl+JuN33mCRnpFnV+Yzx34EnOXe/ts0VE1dinspcBz8X06zydu6EfbdX1XKazMfJE845md8+nP/cJNcNfJ453qiqCtiP5qH8j0w1gPa5ok8Ag88SvQHYELgyyfk0ZXTvmOT0QVuPL3UN/Cvwyar6XNvHBe3530hyOfAN4O3t/nFnAw+vqmvb7XNoSsomDX6q6lRg8AGarYH/TnIx8COaAO7fpxnvvjxwbv8deOUM9/lp4NltqeDTgKVtSdw5wLFVdd4M50uSJEldHU+z8vKgdwL/WVXbAP/JJCsOJ3kM8F6aRcV2Bd47VZB0v/OaGEKaHZsuXFL7rWN8NB9ufqL/Lc+nD3/m4PkeQm8dvduR8z0EST1zDEtYVkuH5mn+xU97Yr3n+38338OY1OvXevX5VTXx8ZD7aaumTmlf50KSnwB7VNUNSTYFzqyqiSs479u2eWO7/c9tuxOm62vK52UkSZIkjYZhXFntQXhsVd0A0AZAk60GvTnNYyXjruO3r2uZksHPgCQ7Al+csPueqtptsvaSJEmSprVRkqUD28dU1TGzcN3Jor0Zy2AMfgZU1cXAzjM2lCRJktTF8pnK3iZxU5JNB8rebp6kzXXAHgPbW9C8S3Nac73ggSRJkqQ5VIGxZCg/q+nrNAub0f75fyZpcxrwgiQbtAsdvKDdNy2DH0mSJEnzIskJNKsKb9uu7vx64O+B5ye5Anh+u02SJUmOheZdlMDf0byH8jzg/e2+aVn2JkmSJGleVNW+Uxx67iRtl9K8smZ8+zjguFXpz+BHkiRJGnErH1qrvc0Zy94kSZIk9YLBjyRJkqResOxNkiRJGmHFg1pZrVfM/EiSJEnqBYMfSZIkSb1g2ZskSZI04srV3jox8yNJkiSpFwx+JEmSJPWCZW+SJEnSiBuLOY0unCVJkiRJvWDwI0mSJKkXpix7S/Lo6U6sqjtmfziSJEmSVkUBY6721sl0z/xcSjOXgzM5vl3AlnM4LkmSJEmaVVMGP1X1+DU5EEmSJEmaS51We0uyD/DEqvpQki2Ax1bV+XM7NEmSJEkzi2VvHc244EGSo4DnAK9pd90NfHouByVJkiRJs61L5mf3qtolyY8AqmpFkrXneFySJEmSNKu6BD/3JllAs8gBSTYExuZ0VJIkSZI6s+ytmy7v+Tka+Hdg4yTvA74LfHhORyVJkiRJs2zGzE9VfSHJ+cDz2l2vqKpL5nZYkiRJkjS7Oq32BiwE7qUpfeuSLZIkSZK0BhSwMpa9ddFltbdDgBOAzYAtgC8neddcD0ySJEmSZlOXzM+rgadV1d0AST4InA8cPpcDkyRJkqTZ1CX4uWZCu4cBP5ub4UiSJElaVa721s2UwU+SI2lKCO8GLk1yWrv9ApoV3yRJkiRpZEyX+Rlf0e1S4JsD+8+du+FIkiRJ0tyYMvipqs+uyYFIkiRJWnVFGHNB5k5mfOYnydbAB4HtgUeM76+q353DcUmSJEnSrOoSIh4PfA4I8ELgX4ET53BMkiRJkjTrugQ/61TVaQBVdVVVHQo8Z26HJUmSJKmrIkP5GTZdlrq+J0mAq5K8Cbge2GRuhyVJkiRJs6tL8HMgsAj4a5pnf9YDXjeXg5IkSZKk2TZj8FNV32+/3gm8Zm6HI0mSJGlV+ZLTbqZ7yenJNC81nVRV/emcjEiSJEmS5sB0mZ+j1tgo9JCRMXjEXf7Lw3zY8iLnfT4dvduR8z2E3jrsPX8430PotZd/74z5HkJvPfnbC+d7CNLIme4lp/+5JgciSZIkadUVlr115atgJUmSJPWCwY8kSZKkXuiy1DUASR5eVffM5WAkSZIkrTrL3rqZMfOTZNckFwNXtNs7JfnknI9MkiRJkmZRl7K3TwB7AbcCVNWFwHPmclCSJEmSNNu6lL0tqKprkvul0lbO0XgkSZIkrYIirLTsrZMuwc+1SXYFKslC4K+An87tsCRJkiRpdnUpe3szcBCwJXAT8Ix2nyRJkiSNjBkzP1V1M7DPGhiLJEmSpNVQlr11MmPwk+QzNC+OvZ+qOmBORiRJkiRJc6DLMz/fHvj+COBlwLVzMxxJkiRJmhtdyt5OGtxO8kXgjDkbkSRJkqRV4ktOu+my4MFETwC2mu2BSJIkSdJc6vLMzy/47TM/C4AVwDvnclCSJEmSNNumDX7SvNl0J+D6dtdYVT1g8QNJkiRJ86OAlWXZWxfTlr21gc7JVbWy/Rj4SJIkSRpJXZ75+UGSXeZ8JJIkSZI0h6Yse0vysKq6D/g94C+SXAX8EghNUsiASJIkSRoCrvbWzXTP/PwA2AV46RoaiyRJkiTNmemCnwBU1VVraCySJEmSNGemC342TnLQVAer6mNzMB5JkiRJq6AIZdlbJ9MFPwuBReBMSpIkSRp90wU/N1TV+9fYSCRJkiRpDs34zI8kSZKk4TbW6Q02mm6WnrvGRiFJkiRJc2zK4KeqVqzJgUiSJEnSXJqu7E2SJEnSCBgrn1jpwuJASZIkSb1g8CNJkiSpFyx7kyRJkkZYAStdqLkTMz+SJEmSesHgR5IkSVIvWPYmSZIkjbRQrvbWiZkfSZIkSb1g8CNJkiSpFyx7kyRJkkZYAWOu9taJmR9JkiRJvWDwI0mSJKkXLHuTJEmSRlnBSld768TMjyRJkqReMPiRJEmS1AuWvUmSJEkjzNXeujPzI0mSJKkXDH6GWJJDklya5KIkFyTZLcmZSZYk+X6773+S3NJ+vyDJTVPsX5zk6iQbtdeuJEcM9PW2JIcNbL+67ffSJBcmOTbJ+vMwDZIkSdKssOxtSCV5JrAXsEtV3dMGLWuPH6+q3dp2+wNLquotE85/wP7kfunQe4A/TXJ4VS2fcO6ewIHAC6vq+iQLgf2AxwK3zdpNSpIkaVaUq711YuZneG0KLK+qewCqanlVLZvF698HHEMT5Ex0CPC2qrq+7XtlVR1XVT+Zxf4lSZKkNcrgZ3idDjw+yU+TfCrJH8xBH0cDr0qy3oT9OwA/7HqRJAckWZpk6d3cMqsDlCRJkmaLwc+Qqqq7gKcBBwC3ACe1pWyz2ccdwBeAv56qTZId22eGrkqy9xTXOaaqllTVknXYeDaHKEmSpBmFsSH9DBuDnyHWlpudWVXvBd4C/NkcdPNx4PXAowb2XQrs0o7h4qraGfgW8Mg56F+SJElaIwx+hlSSbZNsM7BrZ+Ca2e6nqlYA/0oTAI07HPhoki0G9hn4SJIkaaS52tvwWgR8sl1e+j7gSpoSuH+bg76OoMksAVBVpybZGPhWu9LbbcAlwGlz0LckSZIehALGXO2tE4OfIVVV5wO7T3JojwntjgeOn+T8B+yvqsUD3xcNfL8JWGdC288Dn1+1UUuSJEnDy7I3SZIkSb1g5keSJEkacSste+vEzI8kSZKkXjD4kSRJktQLlr1JkiRJI66G8IWiw8jMjyRJkqReMPiRJEmS1AuWvUmSJEkjzJecdmfmR5IkSVIvGPxIkiRJmhdJtk1ywcDnjiRvndBmjyS3D7R5z+r2Z9mbJEmSNMoqI/uS06r6CbAzQJKFwPXAyZM0/U5V7fVg+zPzI0mSJGkYPBe4qqqumasODH4kSZIkzZWNkiwd+BwwTdt9gBOmOPbMJBcm+VaSHVZ3MJa9SZIkSSOsWe1tvkcxpeVVtWSmRknWBl4MvGuSwz8Etqqqu5K8CPgasM3qDMbMjyRJkqT59kLgh1V108QDVXVHVd3Vfj8VWCvJRqvTicGPJEmSpPm2L1OUvCV5XJK033eliWFuXZ1OLHuTJEmSRlyN6GpvAEnWAZ4PvHFg35sAqurTwMuBNye5D/gVsE9VrVahn8GPJEmSpHlTVXcDG07Y9+mB70cBR81GX5a9SZIkSeoFMz+SJEnSCGtWexvdsrc1ycyPJEmSpF4w+JEkSZLUC5a9SZIkSSNuDMveujDzI0mSJKkXDH4kSZIk9YJlb5IkSdIIK2Clq711YuZHkiRJUi8Y/EiSJEnqBcveJEmSpFFWoSx768TMjyRJkqReMPiRJEmS1AuWvUmSJEkjbmzMsrcuzPxIkiRJ6gWDH0mSJEm9YNmbJEmSNMJ8yWl3Zn4kSZIk9YLBjyRJkqResOxNkiRJGmUFY5a9dWLmR5IkSVIvGPxIkiRJ6gXL3iRJkqQRV5a9dWLmR5IkSVIvmPmRHiJ+88ia7yH02tq/8l/c5sth7/+v+R5Cr/3boofP9xB66zDume8hSCPHzI8kSZKkXjDzI0mSJI2wIi513ZGZH0mSJEm9YPAjSZIkqRcse9P/a+/Owywrq3uPf3/diqAyRFFEBhHEiUFBJuEGVIzRBDVOF9rEiKJGEwc0osZZI04gxhED6kVN4ixe1KigAooM2hBGUUAFZfACTgyiQve6f+xTciyrm0PR5+w69X4/z3Oe3vvdu85etWmqa5219vtKkiRpyq103qORWPmRJEmS1ASTH0mSJElNsO1NkiRJmmJVsGKls72NwsqPJEmSpCaY/EiSJElqgm1vkiRJ0pQrFzkdiZUfSZIkSU0w+ZEkSZLUBNveJEmSpCm30ra3kVj5kSRJktQEkx9JkiRJTbDtTZIkSZpihYucjsrKjyRJkqQmmPxIkiRJaoJtb5IkSdI0qzjb24is/EiSJElqgsmPJEmSpCbY9iZJkiRNsQJqZd9RTAcrP5IkSZKaYPIjSZIkqQm2vUmSJElTztneRmPlR5IkSVITTH4kSZIkNcG2N0mSJGmaFaxcadvbKKz8SJIkSWqCyY8kSZKkJtj2JkmSJE2xAlY429tIrPxIkiRJaoLJjyRJkqQm2PYmSZIkTblytreRWPmRJEmS1ASTH0mSJElNsO1NkiRJmmIFrKy+o5gOVn4kSZIkNcHkR5IkSVITbHuTJEmSplmFFc72NhIrP5IkSZKaYPIjSZIkqQkmPxOS5FVJzktydpIzkxw/+POiJL8ebJ+ZZPfB+XdLcmOSf5j1Phcn+ezQ/pOTHDXY3j/JVUn+J8mFSb46836D40clefJg+4Qky4eO7ZTkhKH9XQbnXJjkjCRfSrLduO6PJEmS5qeAlSuzIF8Ljc/8TECShwL7ADtW1e+SbAisVVWXJ3kY8NKq2mfWlz0FOBVYBvz7rGM7Jdmmqs6b43KfrKrnD677cOBzSR5eVefPce7dkzymqr48K96NgE8BT62qkwdj/wvYCjjnVnzrkiRJ0oJh5WcyNgaurqrfAVTV1VV1+S18zTLgn4FNk2wy69ihwCtv6aJVdTxwBPCcVZxyCPDqOcafD3xkJvEZvNdJVfX5W7qmJEmStFCZ/EzGscBmSS5I8v4ke63u5CSbAfeoqu/QVWD2nXXKp4Adk9xnhGufAdx/FcdOAX43qBAN22bwdZIkSZoCVVmQr4XG5GcCquo64CF0FZirgE8m2X81X7IfXYID8Am6KtCwFXRVm38Z4fK39LfuTcxd/bn5DZLTkpyf5F2rOP6cJMuTLP8NV40QkiRJkjR5Jj8TUlUrquqEqnodXVvZk1Zz+jJg/yQXA8cAD0qy9axzPgbsCWx+C5feAZjreZ+ZuL4BrA3sNjR8HrDj0Dm7Aq8B1l/FexxRVTtV1U535G63EI4kSZLUD5OfCUhyv1nJy4OBS1Z1LnCnqtqkqraoqi2At9BVg/6gqm4E3gkcuJrr7kVXbTryFkI8GHjZ0P776JKv3YfG7ngL7yFJkqQ+FKxcuTBfC42zvU3GnYH3JNkAuAm4iFVPQrAMOHrW2Gfp2t/+ddb4h/jTlrV9BzOz3RH4MfCkVcz09gdV9d9Jrhra/1mSfYG3DSZbuBK4Gnjj6t5HkiRJWshMfiagqk4Hdl/FsROAE4b2Xz/HOWcDDxxsbzE0/jvgnkP7RwFHrSaO/Ye2Hzbr2ENm7Z8KrHZiBkmSJGmamPxIkiRJU2xmkVPdMp/5kSRJktQEkx9JkiRJTbDtTZIkSZpmBStsexuJlR9JkiRJTTD5kSRJktQE294kSZKkKVbE2d5GZOVHkiRJUhNMfiRJkiQ1wbY3SZIkacrVyr4jmA5WfiRJkiQ1weRHkiRJUhNse5MkSZKmWcGKcra3UVj5kSRJktQEkx9JkiRJTbDtTZIkSZpiBS5yOiIrP5IkSZKaYPIjSZIkqQm2vUmSJElTbqWLnI7Eyo8kSZKkJpj8SJIkSepNkouTnJPkzCTL5zieJO9OclGSs5PsON9r2fYmSZIkTbOCmv7Z3h5eVVev4thjgK0Hr12Bwwd/3mpWfiRJkiQtZI8HPlqdU4ENkmw8nzcy+ZEkSZLUpwKOTXJ6kufMcXwT4KdD+5cOxm41294kSZKkKbbAFzndcNZzPEdU1RGzztmjqi5PcnfguCTfr6pvDh2f65ur+QRj8iNJkiRpXK6uqp1Wd0JVXT7488okRwO7AMPJz6XAZkP7mwKXzycY294kSZIk9SLJnZKsO7MNPAo4d9ZpxwB/P5j1bTfg11V1xXyuZ+VHkiRJmmYFK6Z3kdONgKOTQJeb/FdVfSXJcwGq6gPAfwN/BVwE/AZ4xnwvZvIjSZIkqRdV9SPgQXOMf2Bou4B/WhPXs+1NkiRJUhOs/EiSJElTrMhCnu1tQbHyI0mSJKkJJj+SJEmSmmDbmyRJkjTNCmqFbW+jsPIjSZIkqQkmP5IkSZKaYNubJEmSNMWKqV7kdKKs/EiSJElqgsmPJEmSpCbY9iZJkiRNORc5HY2VH0mSJElNMPmRJEmS1ATb3qRFYq0bLHdLmrzXX/e7vkNo1uvx535fvth3ALMVrHS2t5FY+ZEkSZLUBJMfSZIkSU2w7U2SJEmaclmgs71V3wHMYuVHkiRJUhNMfiRJkiQ1wbY3SZIkaZoVLF2xMNvebuo7gFms/EiSJElqgsmPJEmSpCbY9iZJkiRNsQBLXOR0JFZ+JEmSJDXB5EeSJElSE2x7kyRJkqZZhSULdJHThcbKjyRJkqQmmPxIkiRJaoJtb5IkSdKUy4q+I5gOVn4kSZIkNcHkR5IkSVITbHuTJEmSplgKljrb20is/EiSJElqgsmPJEmSpCbY9iZJkiRNuSUr+45gOlj5kSRJktQEkx9JkiRJTbDtTZIkSZpiKViywtneRmHlR5IkSVITTH4kSZIkNcG2N0mSJGnKxUVOR2LlR5IkSVITTH4kSZIkNcG2N0mSJGmKpWDpir6jmA5WfiRJkiQ1weRHkiRJUhNse5MkSZKmWljibG8jsfIjSZIkqQkmP5IkSZKaYNubJEmSNM0Kljjb20is/EiSJElqgsmPJEmSpCbY9iZJkiRNsQBxtreRWPmRJEmS1ASTH0mSJElNsO1NkiRJmmYFS53tbSRWfiRJkiQ1weRnSiS5bjXHzkry8aH95yT55ND+ekl+mOTeSY5K8uTB+AlJlg+dt1OSE4b2dxmcc2GSM5J8Kcl2a/ybkyRJkibA5GfKJXkA3X/HPZPcaTB8JLBpkkcO9t8IfLiqfjzHW9w9yWPmeN+NgE8Br6yqratqR+AtwFZr/JuQJEnSvAVYsnJhvhYak5/p91TgY8CxwOMAqqqA5wH/lmQnYG/gkFV8/SHAq+cYfz7wkao6eWagqk6qqs+vwdglSZKkiTH5mX77Ap8EPg4smxmsqrOBrwJfB15YVb9fxdefAvwuycNnjW8DnLHmw5UkSZL6YfIzxZLsDFxVVZfQJTk7JvmzoVPeB1xWVcffwlu9ibmrP8PXOi3J+UneNcex5yRZnmT5b7jqVn4XkiRJuk0KlqzIgnwtNCY/020ZcP8kFwM/BNYDnjR0fOXgtVpV9Q1gbWC3oeHzgB2HztkVeA2w/hxff0RV7VRVO92Ru83j25AkSZLGz+RnSiVZAjwF2L6qtqiqLYDHM9T6disdDLxsaP99wP5Jdh8au+M831uSJEnqnYucTo87Jrl0aP8wupa2y4bGvgk8MMnGVXXFrXnzqvrvJFcN7f8syb7A25JsAlwJXE03c5wkSZIWkCzAmdUWIpOfKVFVc1XpDpt1zgpg46H9i4FtZ52z/9D2w2Yde8is/VOBveYZsiRJkrSg2PYmSQQGafIAACAASURBVJIkqQlWfiRJkqQploKlC3BmtYXIyo8kSZKkJpj8SJIkSWqCbW+SJEnSlFuyou8IpoOVH0mSJElNMPmRJEmS1ATb3iRJkqQploIlK53tbRRWfiRJkiQ1weRHkiRJUhNse5MkSZKmXJztbSRWfiRJkiQ1weRHkiRJUhNse5MkSZKmWYWlK5ztbRRWfiRJkiQ1weRHkiRJUhNse5MkSZKmWAqWONvbSKz8SJIkSWqCyY8kSZKkJtj2JkmSJE25JSv7jmA6WPmRJEmS1ASTH0mSJElNsO1NkiRJmmYFcZHTkVj5kSRJktQEkx9JkiRJTbDtTZIkSZpiAZa6yOlIrPxIkiRJaoLJjyRJkqQm2PYmSZIkTbOCJba9jcTKjyRJkqQmmPxIkiRJaoJtb5IkSdIUC7DERU5HYuVHkiRJUi+SbJbk+CTnJzkvyYvmOOdhSX6d5MzB67XzvZ6VH0mSJEl9uQn456o6I8m6wOlJjquq780671tVtc9tvZjJjyRJkjTNCrKy7yDmp6quAK4YbF+b5HxgE2B28rNG2PYmSZIkaVw2TLJ86PWcVZ2YZAtgB+C0OQ4/NMlZSb6cZJv5BmPlR5IkSdK4XF1VO93SSUnuDHwWOLCqrpl1+AzgXlV1XZK/Aj4PbD2fYEx+JEmSpCkWYOkUL3Ka5PZ0ic9/VtXnZh8fToaq6r+TvD/JhlV19a29lm1vkiRJknqRJMCHgPOr6rBVnHOPwXkk2YUuh/n5fK5n5UeSJElSX/YAngack+TMwdgrgc0BquoDwJOB5yW5CbgB2K+qaj4XM/mRJEmSpllN7yKnVXUSXefe6s55L/DeNXE9294kSZIkNcHkR5IkSVITbHvTGnUFp1/9BnJJ33HcBhsCt3rmEK0R3vt+ef/7473vz1Tf+zf0HcBtN833/159B/BHCpZM8Wxvk2TyozWqqu7Wdwy3RZLlo8xFrzXPe98v739/vPf98d73y/uvPtj2JkmSJKkJVn4kSZKkKRZsexuVlR/pjx3RdwAN8973y/vfH+99f7z3/fL+a+Iyz/WBJEmSJC0A6234kNp5n1P7DmNO3/jIWqcvpGe7bHuTJEmSptkUL3I6aba9SZIkSWqCyY8kSZKkJtj2JmnikjwWOLuqLhnsvxZ4EnAJ8KKq+nGf8bUkyV2BPYGfVNXpfcez2CVZD9ioqi4c7D8FWGdw+KtV9f96C07S1HK2t9FZ+VGTkhyQ5KCh/cuSXJPk2iTP6zO2RhwMXAWQZB/g74BnAscAH+gxrkUvyReTbDvY3hg4l+7efyzJgb0G14ZDgT2G9t8C7EyXgL6hl4gakWSbJI8b2n9nkg8PXjv2GVsLvP9aKEx+1KrnAh8e2r+yqtYD7gYs6yekplRV/Waw/UTgQ1V1elV9kO6/gcbn3lV17mD7GcBxVfVYYFe6JEjjtTPwkaH9a6vqBVX1LGDbnmJqxVuBq4f2/xL4EnA88NpeImqL918Lgm1vatWSqvr50P6nAarqt0nWWcXXaM1JkjsDvwH2Bt4/dGztfkJqxo1D23sDRwJU1bVJVvYTUlNuV3+8xsTThrY3mHQwjdm4qk4e2r+mqj4LkOQfeoqpJd7/cSrb3kZl8qNWrT+8U1VvBkiyBLhrLxG15d+AM4FrgPOrajlAkh2AK/oMrAE/TfIC4FJgR+ArAIOk//Z9BtaIlUnuUVU/A5ipwiXZBDD5HK91h3eqareh3btPOJYWef+1INj2plYdm+RNc4y/ETh20sG0pqo+DOwFHAD81dChnwH79xFTQw4AtqG7z/tW1a8G47sB/6evoBpyCPCFJHsmWXfw2gv4/OCYxufyJLvOHkyyG3B5D/G0xvuvBcHKj1p1EPDBJBcBZw3GHgQsB57VW1QNqarLgMtmDa8HvBR49uQjakNVXUn3zNvs8eOT/KiHkJpSVf+R5GrgTXRJKHSTTry2qr7cX2RNeDnwySRHAWcMxh4CPB3Yt6+gGuL9Hyfb3kZm8qMmVdX1wLIkW3LzLyDfq6of9hhWM5JsTzfr1T3pPvF+D91zP7sC7+gxtCYkeSiwCfDNqrpy8N/jFcCfA5v1GlwDquorDNoNNTlV9Z1BleGfuLnCfB6wm1OMj5/3XwuFyY+alGTzweZN3Fz5+cN4Vf2kj7gaciRwOHAK8Gi6TwH/C/jbqvptn4EtdkkOAfahe+bq5Um+CPwj8Gac7W3sBmtarUpV1b9OLJgGDX7Jdmaxnnj/tRCY/KhVXwKKbl2wGUU3zfLdgaV9BNWQO1TVUYPtHyR5KfCKqrJoP35/DewwmNnwz+h67befWXRTY3f9HGN3onsW666Ayc+YJDme7uf8XKqq9p5kPK3x/o9XCEtW5JZPlMmP2lRV2w3vJ9mCrh/5kXSfgGu81h7M7Dbzk/o6YPskAaiqM1b5lbqtbpiprlXVL5P8wMRncqrqD22dSdYFXkS33tInsOVz3F46x9huwMuAKyccS4u8/1oQTH7UtCRbA6/i5mdNXlhVN67+q7QG/Aw4bBX7BTxi4hG1Y6skxwztbzG8X1WPm+NrtAYluQvwEuBv6RY83bGqftlvVItfVZ0+sz2YYe81wB2A5zrZxPh5/7VQmPyoSUm2pUt6tgHeDhxgy9XkVNXD+o6hYY+ftW+1YYIGz1w9ETgC2K6qrus5pKYk+Uu6X7p/CxxcVcf3HFJTvP9j5GxvI8sfLzQttSHJCuCndM/+/MmPi6p64cSDakiSJ67ueFV9blKxSJOUZCXwO7rJVob/AQ7dcw/r9RJYA5J8l+65zkPoJlv5I7bbjpf3f7w22GCn2mvP0/oOY07HfOF2p1fVTn3HMcPKj1p1AKt+8FLj99jVHCvA5GdMkpzD3H/3Z3753n7CITWlqlxcvD/X0z1f+OTBa5jttuPn/deCYPKjJg3NNKYeVNUzVnUsyUaTjKVB+/QdQMsGz/usUlX9YlKxtMZ22355/8crtr2NzORHTUryBVZT+fGh78lKsj7wJOCpwAPoFuDUGFTVJXONJ9mD7v7/02Qjas7p/Ok0+zMK2HKy4bTDdtt+ef+1UJj8qFWH9h1A65KsAzyO7hfuHYF1gb8BvtlnXC1J8mC6+/+/gR9ju+EkPGxVCajGznbbfnn/tSCY/KhVa1XVcXMdSPI24MQJx9OUJP8J7AkcC7wX+AZwUVWd0GdcLUhyX2A/YBnwc+CTdJPfPLzXwNpxNF2yrwlbXbutJuL1Jv7jZdvbaHzwUq16X5K/Hh5IsiTJUcCD+gmpKdsCvwTOB74/mGbcCSgm4/vA3sBjq+p/VdV7mGPGQ42NS7D3KMn9krwjyZcGr0MHHwho/L6e5BVJ/OBdvfIvoFr1KOArSe5QVZ8btGB9GriG1ZfmtQZU1YOS3J+u5eprSa4E1k1yj6r6Wc/hLXZPoqv8HJ/kK8An8BfySdokybtXddBp9scnyUPpWquOGLwC7ACckOSJVXVqn/E1YAfgjcDpSV5QVbY4qxcmP2pSVV2c5JHAV5PcHXgacFpVvaTn0JpRVd8HXgu8NslOdG1Y30lyaVXt3m90i1dVHQ0cneROdM9YvRjYKMnhwNFVdWyvAS5+N9BNeqDJey2wbFZ77eeTfAN4HfCYXqJqRFVdC7w4yUPoqkCXAitxmv01opvtzc+xRuEip2pSkpme+42BjwLHAW+fOe5ia+OV5PlV9d45xgPsWVU+czUmSW5XVTfNGrsL8BRg36pyrY0xSnJGVfnMTw+SXFBVc7a4JflBVd1v0jG1JskjgHcBXwXeR5f8AKueiVKjuct6O9Xeu36n7zDm9JmvLXWRU2kBeMfQ9tnARkNjLrY2fs+km+jgj1T3aYyJz3h9h1kP3A/Wlvn3wUvjtXHfATTs2tUcu35iUTQqySfoljF4alWd03c8apfJj5q0upmtkuw2yVikCbMvol8+09afzVbxvFVwbbFJ+HpVHTnXgSQbVdX/m3RAi42zvY3G5Ef6U58CNu87iEVu+yTXzDE+0/u93qQDasjdkqzy2baqOmySwTTIXvP+HLSaY8snFkWjZic+Lm6tvpj8SH/KT8bH75yq2qHvIBq1FLgz/j3vy6bO9taPqvpI3zG0zsWttRCY/Eh/yk9mtZhdUVVv7DuIhjnbW0+S/B9W/fO9quqAScbTGhe3HrOy7W1UJj9qUpIvMPc/ggHuOuFwWvTpvgNomBWffv3cCkRvvjjH2ObAgXQVUY3XnyxuncQPGzVxJj9q1aHzPKY146okW1fVhYPprT9M1/t9MbC/U42P1eOT3L6qboRuxXvgr4BLqupz/YbWhN/3HUCrquqzM9tJtgReSVeJeCvwob7iaoWLW2uhMPlRk1a1jkySzYD9cLrlcXsRcNRgexmwPXBvuhXA3wX8eT9hNeE/gAOAC5PcBzgF+E9gnyQ7V9W/9Brd4vdPQ+uM/QkT//FK8gDgVXQ/aw4Bnjt73SuNj4tbj09sexuZyY+al2RDugUel9HNNnN0vxE14aaZygOwD/DRqvo53aeBb1/N1+m2+7OqunCw/XTg41X1giRr0T2LYvIzXofStdzOtB/ObvtxjbExSfJpYCe6/wYvBlYA63XF5z+sd6UJqarlwPIkB9F9ICZNhMmPmpRkXeAJdOX3+9IlPFtW1aa9BtaOlUk2puv/3hs4eOjYOv2E1IzhX7YfQffpN1X1+yQr5/4SrUEvB35aVVcAJHk6N7d8vr6/sJqwM93f/5cC/zwYG05Ct+wjqNZV1cokLwbe2XcsaoPJj1p1Jd1K968GTqqqSvKEnmNqyWvp1tVYChxTVecBJNkL+FGfgTXg7CSHApcB96GbeYkkG/QaVTs+ADwSIMmewFuAFwAPBo4AntxfaItbVW3RdwxaJSdiWQNsexvNkr4DkHrySmBt4HDgX5Js1XM8TamqLwL3Ah5QVc8eOrQc2LefqJrxbOBqYAvgUVX1m8H4A3Gyj0lYOtRetS9wRFV9tqpeQ5eMaoKSbJXkVUnO7TuWxjnrmybG5EdNqqp3VtWudIutBfg8cM8kL09y336jW/ySbA18BvhWko8n2QSgqq6vquv6jW5xq6obquqtVfWiqjpraPzkqvpYn7E1YmmSma6LvenWOplhN8YEJNk4yYFJvgOcR3ffl/Uc1qKX5Nok18zxuha4Z9/xqR3+oFWTkhwInAScWVUHAwcn2Y7uH8AvA1aCxuvDwEfpVvV+HPAe4Im9RtSIJMez+oUe955kPA36OHBikqvpFjz9FsBg5r1f9xnYYpfk2XQ/4zcFPgU8C/i/VfWGXgNrRFWt23cMi1k325vdg6Mw+VGrNgXeDdw/ydnAycC3gUOr6pW9RtaGdavqyMH2IUmc3ndyXjrH2G7Ay+iehdMYVdXBSb4ObAwcW1UziegSumd/ND7vo5va/amDmcZwkU2pPSY/alJVvRRgML3vTsDuwDOBI5P8qqoe2Gd8DVg7yQ7c/JDrOsP7rnUyPlV1+sz2YIKJ1wB3oFvv5Mu9BdaQqjp1jrEL+oilMfekW9bgsCQb0VV/bt9vSJImzeRHrVsHWA9Yf/C6HDin14ja8DPgsFXsF651MlZJ/pIu6fktcHBVHd9zSNLYVdXVdJPcHJ5kU7oFra9Mcj5wtFV/TTtnexuNyY+alOQIYBvgWuA0ura3w6rql70G1oiqeljfMbQqyXeBu9Gt73PKYGzHmeNW3bRYJdltpupWVZfSzW54aJL70SVCkhpg8qNWbU7X6nMh3XonlwK/6jWihiSZPblB0U2/fGZVXdtDSC25HriObj2Z2WvKWHXTYvZ+YMfZg1X1A8BJD6RGmPyoSVX16CShq/7sTrfa97ZJfgGcUlWv6zXAxe+xc4zdBdg+yQFV9Y05jmsNsOomSYtQ2fY2KpMfNWswy9K5SX5FN8Xsr4F9gF0Ak58xqqpnzDWe5F50DyHvOtmI2pHkLLpp3k8Gvl1VF/cbkTQxWyY5ZlUHq+pxkwxGUj9MftSkJC+kq/jsAdxIN831KXTrzzjhQU+q6pIkzr40Xn9L93f/L4DXJbkTXSJ0MnByVZ3WZ3DSGF0FvKPvICT1y+RHrdoC+Azw4qq6oudYNDB48Ph3fcexmFXVucC5wBEASTake9j7QLoHwJf2F500VtdV1Yl9ByGNQ2x7G5nJj5pUVS/pO4aWJfkC3cP1w+5Ct/Dj300+onYkWQrswM2Vz63oJv34IIPZ36RF6pdJ7lFVPwNI8vfAk4BLgNdX1S96jU7SRJj8SOrDobP2C/g5cGFV/b6HeFpyDXA+3Wr3r6iqH/ccjzQpGwC/B0iyJ/BW4AXAg+kqobNnP5S0CJn8SJq4UVtPkpxSVQ8ddzyNeRbw0MGfzxis+3MK3SyHl/UamTReS4aqO/sCR1TVZ4HPJjmzx7ikNcK2t9GY/EhayNbuO4DFpqo+DnwcIMkd6WY33AN4S5K1qupefcYnjdHtktyuqm4C9gaeM3ysp5gkTZj/s0tayGY/F6Q1YDDD267c/NzPzsBP6WY9lBarjwMnJrkauAH4FkCS+9AtdSCpASY/ktSQJP8DbA4sp5ve+h3AqVV1Xa+BSWNWVQcn+TrdxCrHDtZ6A1hC9+yPNLWc7W10Jj+SFrL0HcAi9HTgnKFf/KRmVNWpc4xd0EcskvqxpO8AJGk1ntZ3AItNVZ0NbJPkI0mWJ/nuYHv7vmOTJGncTH4kTVySA5IcNLR/WZJrklyb5Hkz44MFObUGJXk8cDRwIvBMulnfTqSb8erxfcYmSZqngiU3LczXQmPbm6Q+PBd49ND+lVW1SZK1gWOBw/sJqwlvBP6iqi4eGjsryTeA/zt4SZK0KFn5kdSHJVX186H9TwNU1W+BdfoJqRm3n5X4ADAYu/3Eo5EkaYKs/Ejqw/rDO1X1ZoAkS4C79hJRO25MsnlV/WR4MMm9gAXYoCBJGsWSFc4RNAorP5L6cGySN80x/ka6tjeNz+uAryXZP8l2SbZN8gy6+/7anmOTJGmsrPxI6sNBwAeTXAScNRh7EN3aM8/qLaoGVNXnk/wY+Ge6tU0CnAf876o6a7VfLEnSlDP5kTRxVXU9sCzJlsA2g+HvVdUPewyrGYMk5+/7jkOStGa4yOnobHuTNHFJNk+yOd0zJmcNXjcOjWuMkjw9yelJrh+8licxGZIkLXpWfiT14UtA0bVczSjgbsDdgaV9BNWCQZJzIPAS4Ay6/wY7Aockoao+2md8kiSNk8mPpImrqu2G95NsAbwceCTw5h5Cask/Ak+YNd31N5I8CfgEYPIjSVPItrfR2PYmqTdJtk5yFPBl4HTggVX1nn6jWvTWW806P+tNPBpJkibIyo+kiUuyLfAquskO3g4cUFV+ZjUZN8zzmCRJU8/kR1IfzgJ+Svfszy7ALsnNj/9U1Qt7iqsFD0hy9hzjAbacdDCSpNvO2d5GZ/IjqQ8H0E1woMl7QN8BSJLUF5MfSRNXVUf1HUOrquqSUc5LckpVPXTc8UiSNEkmP5ImLskXWE3lp6oeN8FwNLe1+w5AkjQi295GZvIjqQ+H9h2AbpFtiZKkRcfkR1If1qqq4+Y6kORtwIkTjkeSJDXA5EdSH96X5MVV9aWZgSRLgA8D9+gvLA3JLZ8iSVoobHsbjcmPpD48CvhKkjtU1eeSrAN8GrgGeGy/oWngaX0HIEnSmrak7wAktaeqLgYeCfxrkucCXwMuqKqnVtWNvQa3yCU5IMlBQ/uXJbkmybVJnjczXlXn9hOhJEnjY+VH0sQl2XGw+TLgo8BxwH/MjFfVGX3F1oDnAo8e2r+yqjZJsjZwLHB4P2FJkubLRU5HZ/IjqQ/vGNo+G9hoaKyAR0w8onYsqaqfD+1/GqCqfjtoP5QkadEy+ZE0cVX18FUdS7LbJGNp0PrDO1X1ZvjDhBN37SUiSZImxORH0kLzKWDzvoNYxI5N8qaqevWs8TfStb1JkqZNwZKb+g5iOpj8SFponGJ5vA4CPpjkIuCswdiDgOXAs3qLSpKkCTD5kbTQVN8BLGZVdT2wLMmWwDaD4e9V1Q97DEuSpIkw+ZE0cUm+wNxJTvC5k7FKMtNSeBM3V37+MF5VP+kjLknS/AVnexuVyY+kPhw6z2O67b5El3gOtxcWcDfg7sDSPoKSJGkSTH4kTVxVnTjXeJLNgP2AOY/rtquq7Yb3k2wBvJxu0dk39xCSJEkTY/IjqVdJNgSeAiwDNgGO7jeiNiTZGngVsCvdGksvrKob+41KkjQvLnI6MpMfSROXZF3gCcBTgfvSJTxbVtWmvQbWgCTb0iU92wBvBw6oKv/JlCQ1weRHUh+uBL4DvBo4qaoqyRN6jqkVZwE/pXv2Zxdgl+Tmx3+q6oU9xSVJalCSRwPvonvm9INV9dZZx+8AfBR4CPBzYN+quni+1zP5kdSHV9I923M48F9JPtlzPC05AKcTl6RFZxrb3pIsBd4H/AVwKfDdJMdU1feGTjsA+GVV3SfJfsDbgH3ne02TH0kTV1XvBN45WGtmGfB54J5JXg4cXVUX9BrgIlZVR/UdgyRJA7sAF1XVjwCSfAJ4PDCc/DweeP1g+zPAe5Okqub1QZ7Jj6SJS3IgcBJwZlUdDBycZDu6ROjLwFZ9xreYrWaNJQCq6nETDEeS1LZN6FqxZ1xKNxHPnOdU1U1Jfk23JuDV87mgyY+kPmwKvBu4f5KzgZOBbwOHVtUre41s8XMdJUlaZK7g9K++nmzYdxyrsHaS5UP7R1TVEYPtzHH+7A/oRjlnZCY/kiauql4KkGQtYCdgd+CZwJFJflVVD+wzvkVurao6bq4DSd6GayxJ0tSpqkf3HcM8XQpsNrS/KXD5Ks65NMntgPWBX8z3gkvm+4WStAasA6xH94NsfbofeKf1GtHi974kfz08kGRJkqOAB/UTkiSpUd8Ftk5y78EHovsBx8w65xjg6YPtJwPfmO/zPmDlR1IPkhxBt87MtXTJzsnAYVX1y14Da8OjgK8kuUNVfS7JOsCngWuAx/YbmiSpJYNneJ4PfJVuqusPV9V5Sd4ILK+qY4APAR9LchFdxWe/23LN3IbESZLmJclXgA2Bc+kSn1OAc2/LJzkaXZJN6f6heQ/wNOC0qnpJv1FJkjR+Jj+SepFuZc1t6J732R3Ylu4TnVOq6nV9xraYJdlxsLkx3aJxxwFvnzleVWf0EZckSZNg8iOpV4MqxB50CdA+wF2raoN+o1q8khy/msNVVY+YWDCSJE2YyY+kiUvyQrpkZw/gRrpprk8Z/HlOVa3sMbxmJdmtqk7tOw5JksbF5EfSxCU5jMHaPlV1Rd/xqJPkJ1W1ed9xSJI0LiY/kiQAkvy0qja75TMlSZpOrvMjSZrhp2GSpEXNdX4kqSFJvsDcSU6Au044HEmSJsq2N0lqSJK9Vne8qk6cVCySJE2ayY8kiSSbAftV1SF9xyJJ0rj4zI8kNSrJhkmel+SbwAnARj2HJEnSWPnMjyQ1JMm6wBOApwL3BY4GtqyqTXsNTJKkCbDtTZIakuQG4DvAq4GTqqqS/Kiqtuw5NEmSxs62N0lqyyuBtYHDgX9JslXP8UiSNDFWfiSpQUm2BJYB+wFbA68Djq6qC3oNTJKkMTL5kaSGJDkQOAk4s6puGoxtR5cI7VtVVoIkSYuWyY8kNSTJocDuwP2Bs4GTgW8Dp1TVL/qMTZKkcTP5kaQGJVkL2IkuEXro4PWrqnpgr4FJkjRGTnUtSW1aB1gPWH/wuhw4p9eIJEkaMys/ktSQJEcA2wDXAqcBpwKnVtUvew1MkqQJcKprSWrL5sAdgJ8BlwGXAr/qNSJJkibEyo8kNSZJ6Ko/uw9e2wK/oJv04HV9xiZJ0jiZ/EhSo5JsCuxBlwDtA9y1qjboNypJksbH5EeSGpLkhXTJzh7AjQymuR78eU5VrewxPEmSxsrZ3iSpLVsAnwFeXFVX9ByLJEkTZeVHkiRJUhOc7U2SJElSE0x+JEmSJDXB5EeSdKskWZHkzCTnJvl0kjvehvd6WJIvDrYfl+QVqzl3gyT/OI9rvD7JS0cdn3XOUUmefCuutUWSc29tjJKkyTD5kSTdWjdU1YOralvg98Bzhw+mc6v/famqY6rqras5ZQPgVic/kiTNMPmRJN0W3wLuM6h4nJ/k/cAZwGZJHpXklCRnDCpEdwZI8ugk309yEvDEmTdKsn+S9w62N0pydJKzBq/dgbcCWw2qTocMzjsoyXeTnJ3kDUPv9aokP0jyNeB+t/RNJHn24H3OSvLZWdWsRyb5VpILkuwzOH9pkkOGrv0Pt/VGSpLGz+RHkjQvSW4HPAY4ZzB0P+CjVbUDcD3wauCRVbUjsBx4SZK1gSOBxwJ/DtxjFW//buDEqnoQsCNwHvAK4IeDqtNBSR4FbA3sAjwYeEiSPZM8BNgP2IEuudp5hG/nc1W18+B65wMHDB3bAtgL+GvgA4Pv4QDg11W18+D9n53k3iNcR5LUI9f5kSTdWuskOXOw/S3gQ8A9gUuq6tTB+G7AA4FvJwFYi24x1fsDP66qCwGS/AfwnDmu8Qjg7wGqagXw6yR/NuucRw1e/zPYvzNdMrQucHRV/WZwjWNG+J62TfImuta6OwNfHTr2qcHirxcm+dHge3gUsP3Q80DrD659wQjXkiT1xORHknRr3VBVDx4eGCQ41w8PAcdV1bJZ5z0YWFMLzAV4S1X9+6xrHDiPaxwF/E1VnZVkf+BhQ8dmv1cNrv2CqhpOkkiyxa28riRpgmx7kySNw6nAHknuA5DkjknuC3wfuHeSrQbnLVvF138deN7ga5cmWQ+4lq6qM+OrwDOHniXaJMndgW8CT0iyTpJ16Vrsbsm6wBVJbg/87axjT0myZBDzlsAPBtd+3uB8ktw3yZ1GuI4kqUdWfiRJa1xVXTWooHw8yR0Gw6+uqguSPAf4UpKrgZOAbed4ixcBRyQ5AFgBPK+qTkny7cFU0l8ePPfzAOCUQeXpOuDvquqMJJ8EzgQuoWvN0ISV0wAAAHdJREFUuyWvAU4bnH8Of5xk/QA4EdgIeG5V/TbJB+meBToj3cWvAv5mtLsjSepLqtZU94EkSZIkLVy2vUmSJElqgsmPJEmSpCaY/EiSJElqgsmPJEmSpCaY/EiSJElqgsmPJEmSpCaY/EiSJElqgsmPJEmSpCb8f6xvxsi6I+0lAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x864 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Experiment 1 (CNN as feature extractor) of feature extractor training with the CNN model model 2.\n",
    "# CNN model is used as a feature extrator, SVM (ovo or ovr) and a single layer of LSTM is used as personlized classifier\n",
    "# The personlized classifier is trained with 100% dataset\n",
    "# The structure based on Guillaume Chevalier https://github.com/guillaume-chevalier/LSTM-Human-Activity-Recognition\n",
    "# The structure include read in of the input, formatting the input and the conversion of the one-hot encoding.\n",
    "# The SVM is model is from sklearn library\n",
    "# The CNN model and the feature extration method is written by myself\n",
    "# The credit of this code goes to Guillaume Chevalier under MIT License.\n",
    "# Python version 3.6\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf  # Version 1.0.0 (some previous versions are used in past commits)\n",
    "from sklearn import metrics\n",
    "import time\n",
    "import os\n",
    "\n",
    "# Those are separate normalised input features for the neural network\n",
    "INPUT_SIGNAL_TYPES = [\n",
    "    \"body_acc_x_\",\n",
    "    \"body_acc_y_\",\n",
    "    \"body_acc_z_\",\n",
    "    \"body_gyro_x_\",\n",
    "    \"body_gyro_y_\",\n",
    "    \"body_gyro_z_\",\n",
    "    \"total_acc_x_\",\n",
    "    \"total_acc_y_\",\n",
    "    \"total_acc_z_\"\n",
    "]\n",
    "\n",
    "# Output classes to learn how to classify\n",
    "LABELS = [\n",
    "    \"WALKING\", \n",
    "    \"WALKING_UPSTAIRS\", \n",
    "    \"WALKING_DOWNSTAIRS\", \n",
    "    \"SITTING\", \n",
    "    \"STANDING\", \n",
    "    \"LAYING\"\n",
    "] \n",
    "\n",
    "# Pathes to dataset\n",
    "DATASET_PATH = \"UCI HAR Dataset/\"\n",
    "TRAIN = \"train/\"\n",
    "TEST = \"test/\"\n",
    "\n",
    "\n",
    "def load_X(X_signals_paths):\n",
    "    X_signals = []\n",
    "    \n",
    "    for signal_type_path in X_signals_paths:\n",
    "        file = open(signal_type_path, 'r')\n",
    "        # Read dataset from disk, dealing with text files' syntax\n",
    "        X_signals.append(\n",
    "            [np.array(serie, dtype=np.float32) for serie in [\n",
    "                row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "            ]]\n",
    "        )\n",
    "        file.close()\n",
    "    \n",
    "    return np.transpose(np.array(X_signals), (1, 2, 0))\n",
    "\n",
    "def load_y(y_path):\n",
    "    file = open(y_path, 'r')\n",
    "    # Read dataset from disk, dealing with text file's syntax\n",
    "    y_ = np.array(\n",
    "        [elem for elem in [\n",
    "            row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "        ]], \n",
    "        dtype=np.int32\n",
    "    )\n",
    "    file.close()\n",
    "    \n",
    "    # Substract 1 to each output class for friendly 0-based indexing \n",
    "    return y_ - 1\n",
    "\n",
    "X_train_signals_paths = [\n",
    "    DATASET_PATH + TRAIN + \"Inertial Signals/\" + signal + \"train.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "X_test_signals_paths = [\n",
    "    DATASET_PATH + TEST + \"Inertial Signals/\" + signal + \"test.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "y_train_path = DATASET_PATH + TRAIN + \"y_train.txt\"\n",
    "y_test_path = DATASET_PATH + TEST + \"y_test.txt\"\n",
    "\n",
    "X_for_train = load_X(X_train_signals_paths)\n",
    "X_test = load_X(X_test_signals_paths)\n",
    "y_for_train = load_y(y_train_path)\n",
    "y_test = load_y(y_test_path)\n",
    "\n",
    "print(\"read in the data\")\n",
    "\n",
    "# randomly shuffle the array\n",
    "def rand_shuffle_data(train_X, train_Y):\n",
    "    item_size = np.array(train_X).shape\n",
    "    sample_size = item_size[0]\n",
    "    np.random.seed(42)\n",
    "    value_list = np.arange(sample_size)\n",
    "    np.random.shuffle(value_list)\n",
    "    output_X = np.zeros(np.array(train_X).shape)\n",
    "    output_Y = np.zeros(np.array(train_Y).shape)\n",
    "    print(output_Y.shape)\n",
    "    cnt = 0\n",
    "    for idx in value_list:\n",
    "        output_X[cnt,:,:] = train_X[idx,:,:]\n",
    "        output_Y[cnt,:] = train_Y[idx,:]\n",
    "        cnt += 1\n",
    "    return value_list, output_X, output_Y\n",
    "\n",
    "# The pre-trained model\n",
    "def cnn_net(_X):\n",
    "    _X = tf.reshape(_X, [-1, n_steps, n_input]) \n",
    "    # convlution layers\n",
    "    conv1 = tf.layers.separable_conv1d(_X, 32, 5, activation=tf.nn.relu, name='conv1_layer') \n",
    "    conv1 = tf.layers.max_pooling1d(conv1, 20, 2, name='max1_layer')\n",
    "    conv2 = tf.layers.separable_conv1d(conv1, 64, 5, activation=tf.nn.relu, name='conv2_layer')\n",
    "    fc1 = tf.contrib.layers.flatten(conv2)\n",
    "    fc1 = tf.layers.dense(fc1, 1024)\n",
    "    out = tf.layers.dense(fc1, n_classes)\n",
    "    return out\n",
    "\n",
    "# The personalized classifier\n",
    "def lstm_net(_X, _weights, _biases):\n",
    "    conv2 = tf.transpose(_X, [1, 0, 2])\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)\n",
    "    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell], state_is_tuple=True)\n",
    "    # Get LSTM cell output\n",
    "    outputs, states = tf.nn.dynamic_rnn(lstm_cells, conv2, dtype=tf.float32)\n",
    "    lstm_last_output = outputs[-1]\n",
    "    # Linear activation\n",
    "    print(lstm_last_output.shape)\n",
    "    return tf.matmul(lstm_last_output, _weights['out']) + _biases['out']\n",
    "\n",
    "def extract_batch_size(_train, step, batch_size):\n",
    "    # Function to fetch a \"batch_size\" amount of data from \"(X|y)_train\" data. \n",
    "    \n",
    "    shape = list(_train.shape)\n",
    "    shape[0] = batch_size\n",
    "    batch_s = np.empty(shape)\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        # Loop index\n",
    "        index = ((step-1)*batch_size + i) % len(_train)\n",
    "        batch_s[i] = _train[index] \n",
    "\n",
    "    return batch_s\n",
    "\n",
    "\n",
    "def one_hot(y_):\n",
    "    # Function to encode output labels from number indexes \n",
    "    # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]\n",
    "    \n",
    "    y_ = y_.reshape(len(y_))\n",
    "    n_values = int(np.max(y_)) + 1\n",
    "    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS\n",
    "\n",
    "\n",
    "percentage_list = [1]\n",
    "\n",
    "for percent_train in percentage_list:\n",
    "    \n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    rand_shuffle_idx, train_X, train_y = rand_shuffle_data(X_for_train, y_for_train)\n",
    "    pick_samples = int(round((np.array(X_for_train).shape[0])*percent_train))\n",
    "    X_train = X_for_train[0:pick_samples, :, :]\n",
    "    y_train = y_for_train[0:pick_samples,:]\n",
    "    \n",
    "    print(np.array(X_train).shape)\n",
    "    \n",
    "    # Input data and LSTM internal structure\n",
    "    training_data_count = len(X_train)  # 7352 training series (with 50% overlap between each serie)\n",
    "    test_data_count = len(X_test)  # 2947 testing series\n",
    "    n_height = 1\n",
    "    n_steps = len(X_train[0])  # 128 timesteps per series\n",
    "    n_input = len(X_train[0][0])\n",
    "    n_hidden = 32\n",
    "    n_classes = 6 # Total classes (should go up, or should go down)\n",
    "\n",
    "    # Training parameters\n",
    "    learning_rate = 0.0001\n",
    "    lambda_loss_amount = 0.0015\n",
    "    training_iters = training_data_count * 250  # Loop 300 times on the dataset\n",
    "    batch_size = 1500\n",
    "    display_iter = 300000  # To show test set accuracy during training\n",
    "    \n",
    "    # Graph input/output\n",
    "    x = tf.placeholder(tf.float32, [None, n_steps, n_input])\n",
    "    y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "    \n",
    "    pred = cnn_net(x)\n",
    "\n",
    "    # Loss, optimizer and evaluation\n",
    "    l2 = lambda_loss_amount * sum(\n",
    "        tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()\n",
    "    ) # L2 loss prevents this overkill neural network to overfit the data\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=pred)) + l2# Softmax loss\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer\n",
    "\n",
    "    correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "    # To keep track of training's performance\n",
    "    test_losses = []\n",
    "    test_accuracies = []\n",
    "    train_losses = []\n",
    "    train_accuracies = []\n",
    "\n",
    "    # Launch the graph\n",
    "    print(\"start training\")\n",
    "    start_time = time.time()\n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    # Perform Training steps with \"batch_size\" amount of example data at each loop\n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters:\n",
    "        batch_xs =         extract_batch_size(X_train, step, batch_size)\n",
    "        batch_ys = one_hot(extract_batch_size(y_train, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer, cost, accuracy],\n",
    "            feed_dict={\n",
    "                x: batch_xs, \n",
    "                y: batch_ys\n",
    "            }\n",
    "        )\n",
    "        train_losses.append(loss)\n",
    "        train_accuracies.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost, accuracy], \n",
    "            feed_dict={\n",
    "                x: X_test,\n",
    "                y: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses.append(loss2)\n",
    "        test_accuracies.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time) + \" seconds\")\n",
    "    # Accuracy for test data\n",
    "\n",
    "    one_hot_predictions, accuracy, final_loss = sess.run(\n",
    "        [pred, accuracy, cost],\n",
    "        feed_dict={\n",
    "            x: X_test,\n",
    "            y: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL CNN RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss) + \\\n",
    "          \", Accuracy = {}\".format(accuracy))\n",
    "\n",
    "    \n",
    "    \n",
    "    # extract the features\n",
    "    feature_tensor = sess.graph.get_tensor_by_name('conv2_layer/Relu:0')\n",
    "    features = sess.run(feature_tensor, feed_dict={x:X_train})\n",
    "    feature_shape = features.shape\n",
    "    test_features = sess.run(feature_tensor,feed_dict={x:X_test})\n",
    "    test_feature_shape = test_features.shape\n",
    "    reformatted_features = np.zeros((feature_shape[0], feature_shape[1]*feature_shape[2]))\n",
    "    test_reformatted_features = np.zeros((test_feature_shape[0], test_feature_shape[1]*test_feature_shape[2]))\n",
    "    c = 0\n",
    "    for row in features:\n",
    "        reformatted_features[c,:] = row.flatten()\n",
    "        c += 1\n",
    "    d = 0\n",
    "    for row in test_features:\n",
    "        test_reformatted_features[d,:] = row.flatten()\n",
    "        d += 1\n",
    "        \n",
    "    # classify with svm\n",
    "    from sklearn.svm import SVC\n",
    "    svm_starttime = time.time()\n",
    "    clf = SVC()\n",
    "    clf.fit(reformatted_features, y_train)\n",
    "    print(\"-------------SVM(OVR)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_1 = clf.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_1)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnn_svm_ovr_conf.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnn_svm_ovr_conf.png\", bbox_inches='tight')\n",
    "    \n",
    "    \n",
    "    svm2_starttime = time.time()\n",
    "    clf2 = SVC(decision_function_shape='ovo')\n",
    "    clf2.fit(reformatted_features, y_train)\n",
    "    print(\"-------------SVM(OVO)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm2_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf2.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_2 = clf2.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "     # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnn_svm_ovo_conf.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnn_svm_ovo_conf.png\", bbox_inches='tight')\n",
    "    \n",
    "    start_time_lstm = time.time()\n",
    "    print(\"-------------lstm-----------------\")\n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    weights = {\n",
    "        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0))\n",
    "    }\n",
    "    biases = {\n",
    "        'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "    }\n",
    "    \n",
    "    learning_rate_lstm = 0.0025\n",
    "    \n",
    "    x_feat = tf.placeholder(tf.float32, [None, feature_shape[1], feature_shape[2]])\n",
    "    y2 = tf.placeholder(tf.float32, [None, n_classes])\n",
    "    pred_lstm = lstm_net(x_feat, weights, biases)\n",
    "    cost2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y2, logits=pred_lstm))# Softmax loss\n",
    "    optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate_lstm).minimize(cost2)\n",
    "    correct_pred2 = tf.equal(tf.argmax(pred_lstm,1), tf.argmax(y2,1))\n",
    "    accuracy2 = tf.reduce_mean(tf.cast(correct_pred2, tf.float32))\n",
    "    \n",
    "    test_losses2 = []\n",
    "    test_accuracies2 = []\n",
    "    train_losses2 = []\n",
    "    train_accuracies2 = []\n",
    "    training_iters2 = training_data_count * 150  # Loop 300 times on the dataset\n",
    "    \n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    \n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters2:\n",
    "        batch_xxs =         extract_batch_size(features, step, batch_size)\n",
    "        batch_yys = one_hot(extract_batch_size(y_train, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer2, cost2, accuracy2],\n",
    "            feed_dict={\n",
    "                x_feat: batch_xxs, \n",
    "                y2: batch_yys\n",
    "            }\n",
    "        )\n",
    "        train_losses2.append(loss)\n",
    "        train_accuracies2.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost2, accuracy2], \n",
    "            feed_dict={\n",
    "                x_feat: test_features,\n",
    "                y2: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses2.append(loss2)\n",
    "        test_accuracies2.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters2):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time_lstm) + \" seconds\")\n",
    "\n",
    "    one_hot_predictions2, accuracy2, final_loss2 = sess.run(\n",
    "        [pred_lstm, accuracy2, cost2],\n",
    "        feed_dict={\n",
    "            x_feat: test_features,\n",
    "            y2: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL LSTM RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss2) + \\\n",
    "          \", Accuracy = {}\".format(accuracy2))\n",
    "    \n",
    "    predictions2 = one_hot_predictions2.argmax(1)\n",
    "    \n",
    "    print(\"Testing Accuracy: {}%\".format(100*accuracy2))\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, predictions2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, predictions2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnn_feat_lstm_train_conf.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnn_feat_lstm_train_conf.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "read in the data\n",
      "(7352, 1)\n",
      "(7352, 128, 9)\n",
      "WARNING:tensorflow:From c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From <ipython-input-1-ec5bdf120e78>:197: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "start training\n",
      "Training iter #1500:   Batch Loss = 3.012986, Accuracy = 0.4339999854564667\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 2.9847655296325684, Accuracy = 0.3885307013988495\n",
      "Training iter #300000:   Batch Loss = 0.860838, Accuracy = 0.9453333616256714\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 1.038323163986206, Accuracy = 0.8439090847969055\n",
      "Training iter #600000:   Batch Loss = 0.569806, Accuracy = 0.8679999709129333\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.6699758172035217, Accuracy = 0.8717339634895325\n",
      "Training iter #900000:   Batch Loss = 0.345212, Accuracy = 0.9546666741371155\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.5313519239425659, Accuracy = 0.8832710981369019\n",
      "Training iter #1200000:   Batch Loss = 0.278755, Accuracy = 0.9506666660308838\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.4601738452911377, Accuracy = 0.8907363414764404\n",
      "Training iter #1500000:   Batch Loss = 0.309608, Accuracy = 0.9279999732971191\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.4211867153644562, Accuracy = 0.8995589017868042\n",
      "Training iter #1800000:   Batch Loss = 0.201193, Accuracy = 0.9860000014305115\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.3989870846271515, Accuracy = 0.9029521346092224\n",
      "Optimization Finished!\n",
      "Training time is: 877.1098103523254 seconds\n",
      "FINAL CNN RESULT: Batch Loss = 0.3966541290283203, Accuracy = 0.903630793094635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\edwar\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\sklearn\\utils\\validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVR)-------------\n",
      "Training time is: 56.123528718948364 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.5232439769256871\n",
      "\n",
      "Precision: 80.21921607477537%\n",
      "Recall: 52.324397692568716%\n",
      "f1_score: 39.2259574098903%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[496   0   0   0   0   0]\n",
      " [455   4   0   0  12   0]\n",
      " [417   0   3   0   0   0]\n",
      " [  0   0   0   1 490   0]\n",
      " [  4   0   0   0 528   0]\n",
      " [  0   0   0   0  27 510]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.830675    0.          0.          0.          0.          0.        ]\n",
      " [15.43943     0.13573125  0.          0.          0.40719375  0.        ]\n",
      " [14.149983    0.          0.10179844  0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.03393281 16.627077    0.        ]\n",
      " [ 0.13573125  0.          0.          0.         17.916527    0.        ]\n",
      " [ 0.          0.          0.          0.          0.916186   17.305735  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------SVM(OVO)-------------\n",
      "Training time is: 55.77107119560242 seconds\n",
      "FINAL SCORE FOR SVM OVR:\n",
      "0.5232439769256871\n",
      "\n",
      "Precision: 80.21921607477537%\n",
      "Recall: 52.324397692568716%\n",
      "f1_score: 39.2259574098903%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[496   0   0   0   0   0]\n",
      " [455   4   0   0  12   0]\n",
      " [417   0   3   0   0   0]\n",
      " [  0   0   0   1 490   0]\n",
      " [  4   0   0   0 528   0]\n",
      " [  0   0   0   0  27 510]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.830675    0.          0.          0.          0.          0.        ]\n",
      " [15.43943     0.13573125  0.          0.          0.40719375  0.        ]\n",
      " [14.149983    0.          0.10179844  0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.03393281 16.627077    0.        ]\n",
      " [ 0.13573125  0.          0.          0.         17.916527    0.        ]\n",
      " [ 0.          0.          0.          0.          0.916186   17.305735  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------lstm-----------------\n",
      "(?, 32)\n",
      "Training iter #1500:   Batch Loss = 2.305381, Accuracy = 0.15733332931995392\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 2.1666078567504883, Accuracy = 0.15982355177402496\n",
      "Training iter #300000:   Batch Loss = 0.113284, Accuracy = 0.9639999866485596\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.20133183896541595, Accuracy = 0.9297590851783752\n",
      "Training iter #600000:   Batch Loss = 0.120129, Accuracy = 0.9626666903495789\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.19851039350032806, Accuracy = 0.9287410974502563\n",
      "Training iter #900000:   Batch Loss = 0.048739, Accuracy = 0.9879999756813049\n",
      "PERFORMANCE ON TEST SET: Batch Loss = 0.15200914442539215, Accuracy = 0.9487614631652832\n",
      "Optimization Finished!\n",
      "Training time is: 755.5035235881805 seconds\n",
      "FINAL LSTM RESULT: Batch Loss = 0.14458534121513367, Accuracy = 0.9487614631652832\n",
      "Testing Accuracy: 94.87614631652832%\n",
      "\n",
      "Precision: 94.93527471466814%\n",
      "Recall: 94.87614523243977%\n",
      "f1_score: 94.89346112601368%\n",
      "\n",
      "Confusion Matrix:\n",
      "[[479   0  16   0   0   1]\n",
      " [  0 439  28   0   4   0]\n",
      " [  3  27 390   0   0   0]\n",
      " [  0   0   0 451  40   0]\n",
      " [  0   3   0  26 503   0]\n",
      " [  0   0   0   3   0 534]]\n",
      "\n",
      "Confusion matrix (normalised to % of total test data):\n",
      "[[16.253817    0.          0.542925    0.          0.          0.03393281]\n",
      " [ 0.         14.896504    0.9501188   0.          0.13573125  0.        ]\n",
      " [ 0.10179844  0.916186   13.233797    0.          0.          0.        ]\n",
      " [ 0.          0.          0.         15.303699    1.3573124   0.        ]\n",
      " [ 0.          0.10179844  0.          0.88225317 17.068205    0.        ]\n",
      " [ 0.          0.          0.          0.10179844  0.         18.120121  ]]\n",
      "Note: training and testing data is not equally distributed amongst classes, \n",
      "so it is normal that more than a 6th of the data is correctly classifier in the last category.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x1200 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Experiment 1 (CNN as feature extractor) of feature extractor training with the CNN model model 2.\n",
    "# CNN model is used as a feature extrator, SVM (ovo or ovr) and a single layer of LSTM is used as personlized classifier\n",
    "# The personlized classifier is trained with 50% dataset\n",
    "# The structure based on Guillaume Chevalier https://github.com/guillaume-chevalier/LSTM-Human-Activity-Recognition\n",
    "# The structure include read in of the input, formatting the input and the conversion of the one-hot encoding.\n",
    "# The SVM is model is from sklearn library\n",
    "# The CNN model and the feature extration method is written by myself\n",
    "# The credit of this code goes to Guillaume Chevalier under MIT License.\n",
    "# Python version 3.6\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf  # Version 1.0.0 (some previous versions are used in past commits)\n",
    "from sklearn import metrics\n",
    "import time\n",
    "import os\n",
    "\n",
    "# Those are separate normalised input features for the neural network\n",
    "INPUT_SIGNAL_TYPES = [\n",
    "    \"body_acc_x_\",\n",
    "    \"body_acc_y_\",\n",
    "    \"body_acc_z_\",\n",
    "    \"body_gyro_x_\",\n",
    "    \"body_gyro_y_\",\n",
    "    \"body_gyro_z_\",\n",
    "    \"total_acc_x_\",\n",
    "    \"total_acc_y_\",\n",
    "    \"total_acc_z_\"\n",
    "]\n",
    "\n",
    "# Output classes to learn how to classify\n",
    "LABELS = [\n",
    "    \"WALKING\", \n",
    "    \"WALKING_UPSTAIRS\", \n",
    "    \"WALKING_DOWNSTAIRS\", \n",
    "    \"SITTING\", \n",
    "    \"STANDING\", \n",
    "    \"LAYING\"\n",
    "] \n",
    "\n",
    "# Pathes to dataset\n",
    "DATASET_PATH = \"UCI HAR Dataset/\"\n",
    "TRAIN = \"train/\"\n",
    "TEST = \"test/\"\n",
    "\n",
    "\n",
    "def load_X(X_signals_paths):\n",
    "    X_signals = []\n",
    "    \n",
    "    for signal_type_path in X_signals_paths:\n",
    "        file = open(signal_type_path, 'r')\n",
    "        # Read dataset from disk, dealing with text files' syntax\n",
    "        X_signals.append(\n",
    "            [np.array(serie, dtype=np.float32) for serie in [\n",
    "                row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "            ]]\n",
    "        )\n",
    "        file.close()\n",
    "    \n",
    "    return np.transpose(np.array(X_signals), (1, 2, 0))\n",
    "\n",
    "def load_y(y_path):\n",
    "    file = open(y_path, 'r')\n",
    "    # Read dataset from disk, dealing with text file's syntax\n",
    "    y_ = np.array(\n",
    "        [elem for elem in [\n",
    "            row.replace('  ', ' ').strip().split(' ') for row in file\n",
    "        ]], \n",
    "        dtype=np.int32\n",
    "    )\n",
    "    file.close()\n",
    "    \n",
    "    # Substract 1 to each output class for friendly 0-based indexing \n",
    "    return y_ - 1\n",
    "\n",
    "X_train_signals_paths = [\n",
    "    DATASET_PATH + TRAIN + \"Inertial Signals/\" + signal + \"train.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "X_test_signals_paths = [\n",
    "    DATASET_PATH + TEST + \"Inertial Signals/\" + signal + \"test.txt\" for signal in INPUT_SIGNAL_TYPES\n",
    "]\n",
    "y_train_path = DATASET_PATH + TRAIN + \"y_train.txt\"\n",
    "y_test_path = DATASET_PATH + TEST + \"y_test.txt\"\n",
    "\n",
    "X_for_train = load_X(X_train_signals_paths)\n",
    "X_test = load_X(X_test_signals_paths)\n",
    "y_for_train = load_y(y_train_path)\n",
    "y_test = load_y(y_test_path)\n",
    "\n",
    "print(\"read in the data\")\n",
    "\n",
    "# randomly shuffle the array\n",
    "def rand_shuffle_data(train_X, train_Y):\n",
    "    item_size = np.array(train_X).shape\n",
    "    sample_size = item_size[0]\n",
    "    np.random.seed(42)\n",
    "    value_list = np.arange(sample_size)\n",
    "    np.random.shuffle(value_list)\n",
    "    output_X = np.zeros(np.array(train_X).shape)\n",
    "    output_Y = np.zeros(np.array(train_Y).shape)\n",
    "    print(output_Y.shape)\n",
    "    cnt = 0\n",
    "    for idx in value_list:\n",
    "        output_X[cnt,:,:] = train_X[idx,:,:]\n",
    "        output_Y[cnt,:] = train_Y[idx,:]\n",
    "        cnt += 1\n",
    "    return value_list, output_X, output_Y\n",
    "\n",
    "# The pre-trained model\n",
    "def cnn_net(_X):\n",
    "    _X = tf.reshape(_X, [-1, n_steps, n_input]) \n",
    "    # convlution layers\n",
    "    conv1 = tf.layers.separable_conv1d(_X, 32, 5, activation=tf.nn.relu, name='conv1_layer') \n",
    "    conv1 = tf.layers.max_pooling1d(conv1, 20, 2, name='max1_layer')\n",
    "    conv2 = tf.layers.separable_conv1d(conv1, 64, 5, activation=tf.nn.relu, name='conv2_layer')\n",
    "    fc1 = tf.contrib.layers.flatten(conv2)\n",
    "    fc1 = tf.layers.dense(fc1, 1024)\n",
    "    out = tf.layers.dense(fc1, n_classes)\n",
    "    return out\n",
    "\n",
    "# The personalized classifier\n",
    "def lstm_net(_X, _weights, _biases):\n",
    "    conv2 = tf.transpose(_X, [1, 0, 2])\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True)\n",
    "    lstm_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell], state_is_tuple=True)\n",
    "    # Get LSTM cell output\n",
    "    outputs, states = tf.nn.dynamic_rnn(lstm_cells, conv2, dtype=tf.float32)\n",
    "    lstm_last_output = outputs[-1]\n",
    "    # Linear activation\n",
    "    print(lstm_last_output.shape)\n",
    "    return tf.matmul(lstm_last_output, _weights['out']) + _biases['out']\n",
    "\n",
    "def extract_batch_size(_train, step, batch_size):\n",
    "    # Function to fetch a \"batch_size\" amount of data from \"(X|y)_train\" data. \n",
    "    \n",
    "    shape = list(_train.shape)\n",
    "    shape[0] = batch_size\n",
    "    batch_s = np.empty(shape)\n",
    "\n",
    "    for i in range(batch_size):\n",
    "        # Loop index\n",
    "        index = ((step-1)*batch_size + i) % len(_train)\n",
    "        batch_s[i] = _train[index] \n",
    "\n",
    "    return batch_s\n",
    "\n",
    "\n",
    "def one_hot(y_):\n",
    "    # Function to encode output labels from number indexes \n",
    "    # e.g.: [[5], [0], [3]] --> [[0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0]]\n",
    "    \n",
    "    y_ = y_.reshape(len(y_))\n",
    "    n_values = int(np.max(y_)) + 1\n",
    "    return np.eye(n_values)[np.array(y_, dtype=np.int32)]  # Returns FLOATS\n",
    "\n",
    "\n",
    "percentage_list = [1]\n",
    "\n",
    "for percent_train in percentage_list:\n",
    "    \n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    rand_shuffle_idx, train_X, train_y = rand_shuffle_data(X_for_train, y_for_train)\n",
    "    pick_samples = int(round((np.array(X_for_train).shape[0])*percent_train))\n",
    "    X_train = X_for_train[0:pick_samples, :, :]\n",
    "    y_train = y_for_train[0:pick_samples,:]\n",
    "    \n",
    "    print(np.array(X_train).shape)\n",
    "    \n",
    "    # Input data and LSTM internal structure\n",
    "    training_data_count = len(X_train)  # 7352 training series (with 50% overlap between each serie)\n",
    "    test_data_count = len(X_test)  # 2947 testing series\n",
    "    n_height = 1\n",
    "    n_steps = len(X_train[0])  # 128 timesteps per series\n",
    "    n_input = len(X_train[0][0])\n",
    "    n_hidden = 32\n",
    "    n_classes = 6 # Total classes (should go up, or should go down)\n",
    "\n",
    "    # Training parameters\n",
    "    learning_rate = 0.0001\n",
    "    lambda_loss_amount = 0.0015\n",
    "    training_iters = training_data_count * 250  # Loop 300 times on the dataset\n",
    "    batch_size = 1500\n",
    "    display_iter = 300000  # To show test set accuracy during training\n",
    "    \n",
    "    # Graph input/output\n",
    "    x = tf.placeholder(tf.float32, [None, n_steps, n_input])\n",
    "    y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "    \n",
    "    pred = cnn_net(x)\n",
    "\n",
    "    # Loss, optimizer and evaluation\n",
    "    l2 = lambda_loss_amount * sum(\n",
    "        tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables()\n",
    "    ) # L2 loss prevents this overkill neural network to overfit the data\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=pred)) + l2# Softmax loss\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Adam Optimizer\n",
    "\n",
    "    correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "    # To keep track of training's performance\n",
    "    test_losses = []\n",
    "    test_accuracies = []\n",
    "    train_losses = []\n",
    "    train_accuracies = []\n",
    "\n",
    "    # Launch the graph\n",
    "    print(\"start training\")\n",
    "    start_time = time.time()\n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    # Perform Training steps with \"batch_size\" amount of example data at each loop\n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters:\n",
    "        batch_xs =         extract_batch_size(X_train, step, batch_size)\n",
    "        batch_ys = one_hot(extract_batch_size(y_train, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer, cost, accuracy],\n",
    "            feed_dict={\n",
    "                x: batch_xs, \n",
    "                y: batch_ys\n",
    "            }\n",
    "        )\n",
    "        train_losses.append(loss)\n",
    "        train_accuracies.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost, accuracy], \n",
    "            feed_dict={\n",
    "                x: X_test,\n",
    "                y: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses.append(loss2)\n",
    "        test_accuracies.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time) + \" seconds\")\n",
    "    # Accuracy for test data\n",
    "\n",
    "    one_hot_predictions, accuracy, final_loss = sess.run(\n",
    "        [pred, accuracy, cost],\n",
    "        feed_dict={\n",
    "            x: X_test,\n",
    "            y: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL CNN RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss) + \\\n",
    "          \", Accuracy = {}\".format(accuracy))\n",
    "\n",
    "    pick_samples_1 = int(round((np.array(X_for_train).shape[0])*0.5))\n",
    "    X_train_1 = X_for_train[0:pick_samples_1, :, :]\n",
    "    y_train_1 = y_for_train[0:pick_samples_1,:]\n",
    "    \n",
    "    # extract the features\n",
    "    feature_tensor = sess.graph.get_tensor_by_name('conv2_layer/Relu:0')\n",
    "    features = sess.run(feature_tensor, feed_dict={x:X_train_1})\n",
    "    feature_shape = features.shape\n",
    "    test_features = sess.run(feature_tensor,feed_dict={x:X_test})\n",
    "    test_feature_shape = test_features.shape\n",
    "    reformatted_features = np.zeros((feature_shape[0], feature_shape[1]*feature_shape[2]))\n",
    "    test_reformatted_features = np.zeros((test_feature_shape[0], test_feature_shape[1]*test_feature_shape[2]))\n",
    "    c = 0\n",
    "    for row in features:\n",
    "        reformatted_features[c,:] = row.flatten()\n",
    "        c += 1\n",
    "    d = 0\n",
    "    for row in test_features:\n",
    "        test_reformatted_features[d,:] = row.flatten()\n",
    "        d += 1\n",
    "        \n",
    "    # classify with svm\n",
    "    from sklearn.svm import SVC\n",
    "    svm_starttime = time.time()\n",
    "    clf = SVC()\n",
    "    clf.fit(reformatted_features, y_train_1)\n",
    "    print(\"-------------SVM(OVR)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_1 = clf.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_1, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_1)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnn_svm_ovr_conf1.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnn_svm_ovr_conf1.png\", bbox_inches='tight')\n",
    "    \n",
    "    \n",
    "    svm2_starttime = time.time()\n",
    "    clf2 = SVC(decision_function_shape='ovo')\n",
    "    clf2.fit(reformatted_features, y_train_1)\n",
    "    print(\"-------------SVM(OVO)-------------\")\n",
    "    print(\"Training time is: \" + str(time.time() - svm2_starttime) + \" seconds\")\n",
    "    print(\"FINAL SCORE FOR SVM OVR:\")\n",
    "    print(clf2.score(test_reformatted_features, y_test))\n",
    "    \n",
    "    svm_pred_2 = clf2.predict(test_reformatted_features)\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, svm_pred_2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, svm_pred_2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "     # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnn_svm_ovo_conf1.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnn_svm_ovo_conf1.png\", bbox_inches='tight')\n",
    "    \n",
    "    start_time_lstm = time.time()\n",
    "    print(\"-------------lstm-----------------\")\n",
    "    tf.reset_default_graph() # reset my graph\n",
    "    \n",
    "    weights = {\n",
    "        'out': tf.Variable(tf.random_normal([n_hidden, n_classes], mean=1.0))\n",
    "    }\n",
    "    biases = {\n",
    "        'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "    }\n",
    "    \n",
    "    learning_rate_lstm = 0.0025\n",
    "    \n",
    "    x_feat = tf.placeholder(tf.float32, [None, feature_shape[1], feature_shape[2]])\n",
    "    y2 = tf.placeholder(tf.float32, [None, n_classes])\n",
    "    pred_lstm = lstm_net(x_feat, weights, biases)\n",
    "    cost2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y2, logits=pred_lstm))# Softmax loss\n",
    "    optimizer2 = tf.train.AdamOptimizer(learning_rate=learning_rate_lstm).minimize(cost2)\n",
    "    correct_pred2 = tf.equal(tf.argmax(pred_lstm,1), tf.argmax(y2,1))\n",
    "    accuracy2 = tf.reduce_mean(tf.cast(correct_pred2, tf.float32))\n",
    "    \n",
    "    test_losses2 = []\n",
    "    test_accuracies2 = []\n",
    "    train_losses2 = []\n",
    "    train_accuracies2 = []\n",
    "    training_iters2 = training_data_count * 150  # Loop 300 times on the dataset\n",
    "    \n",
    "    sess = tf.InteractiveSession(config=tf.ConfigProto(log_device_placement=True))\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    \n",
    "    step = 1\n",
    "    while step * batch_size <= training_iters2:\n",
    "        batch_xxs =         extract_batch_size(features, step, batch_size)\n",
    "        batch_yys = one_hot(extract_batch_size(y_train_1, step, batch_size))\n",
    "\n",
    "        # Fit training using batch data\n",
    "        _, loss, acc = sess.run(\n",
    "            [optimizer2, cost2, accuracy2],\n",
    "            feed_dict={\n",
    "                x_feat: batch_xxs, \n",
    "                y2: batch_yys\n",
    "            }\n",
    "        )\n",
    "        train_losses2.append(loss)\n",
    "        train_accuracies2.append(acc)\n",
    "        \n",
    "        loss2, acc2 = sess.run(\n",
    "            [cost2, accuracy2], \n",
    "            feed_dict={\n",
    "                x_feat: test_features,\n",
    "                y2: one_hot(y_test)\n",
    "            }\n",
    "        )\n",
    "        test_losses2.append(loss2)\n",
    "        test_accuracies2.append(acc2)\n",
    "\n",
    "        # Evaluate network only at some steps for faster training: \n",
    "        if (step*batch_size % display_iter == 0) or (step == 1) or (step * batch_size > training_iters2):\n",
    "\n",
    "            # To not spam console, show training accuracy/loss in this \"if\"\n",
    "            print(\"Training iter #\" + str(step*batch_size) + \\\n",
    "                  \":   Batch Loss = \" + \"{:.6f}\".format(loss) + \\\n",
    "                  \", Accuracy = {}\".format(acc))\n",
    "\n",
    "            # Evaluation on the test set (no learning made here - just evaluation for diagnosis)\n",
    "            print(\"PERFORMANCE ON TEST SET: \" + \\\n",
    "                  \"Batch Loss = {}\".format(loss2) + \\\n",
    "                  \", Accuracy = {}\".format(acc2))\n",
    "\n",
    "        step += 1\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Training time is: \" + str(time.time() - start_time_lstm) + \" seconds\")\n",
    "\n",
    "    one_hot_predictions2, accuracy2, final_loss2 = sess.run(\n",
    "        [pred_lstm, accuracy2, cost2],\n",
    "        feed_dict={\n",
    "            x_feat: test_features,\n",
    "            y2: one_hot(y_test)\n",
    "        }\n",
    "    )\n",
    "\n",
    "    print(\"FINAL LSTM RESULT: \" + \\\n",
    "          \"Batch Loss = {}\".format(final_loss2) + \\\n",
    "          \", Accuracy = {}\".format(accuracy2))\n",
    "    \n",
    "    predictions2 = one_hot_predictions2.argmax(1)\n",
    "    \n",
    "    print(\"Testing Accuracy: {}%\".format(100*accuracy2))\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Precision: {}%\".format(100*metrics.precision_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"Recall: {}%\".format(100*metrics.recall_score(y_test, predictions2, average=\"weighted\")))\n",
    "    print(\"f1_score: {}%\".format(100*metrics.f1_score(y_test, predictions2, average=\"weighted\")))\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Confusion Matrix:\")\n",
    "    confusion_matrix = metrics.confusion_matrix(y_test, predictions2)\n",
    "    print(confusion_matrix)\n",
    "    normalised_confusion_matrix = np.array(confusion_matrix, dtype=np.float32)/np.sum(confusion_matrix)*100\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Confusion matrix (normalised to % of total test data):\")\n",
    "    print(normalised_confusion_matrix)\n",
    "    print(\"Note: training and testing data is not equally distributed amongst classes, \")\n",
    "    print(\"so it is normal that more than a 6th of the data is correctly classifier in the last category.\")\n",
    "    \n",
    "    # Plot Results: \n",
    "    width = 12\n",
    "    height = 12\n",
    "    f = plt.figure(figsize=(width, height))\n",
    "    plt.imshow(\n",
    "        normalised_confusion_matrix, \n",
    "        interpolation='nearest', \n",
    "        cmap=plt.cm.rainbow\n",
    "    )\n",
    "    plt.title(\"Confusion matrix \\n(normalised to % of total test data)\")\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(tick_marks, LABELS, rotation=90)\n",
    "    plt.yticks(tick_marks, LABELS)\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.show()\n",
    "    f.savefig(\"cnn_feat_lstm_train_conf1.pdf\", bbox_inches='tight')\n",
    "    f.savefig(\"cnn_feat_lstm_train_conf1.png\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}