mlp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration:  0  Error:  29.1014462511\n",
      "Iteration:  100  Error:  20.9086265635\n",
      "Iteration:  200  Error:  20.4527019487\n",
      "Iteration:  300  Error:  20.1177932175\n",
      "Iteration:  400  Error:  19.7190499632\n",
      "Iteration:  500  Error:  19.1962051803\n",
      "Iteration:  600  Error:  18.5578113616\n",
      "Iteration:  700  Error:  17.8318816406\n",
      "Iteration:  800  Error:  17.0163389317\n",
      "Iteration:  900  Error:  16.0272666973\n",
      "Iteration:  1000  Error:  14.6349030539\n",
      "Iteration:  1100  Error:  12.494824663\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD9CAYAAAC1DKAUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXl8lNXV+L8nISFhDQIKCSJoLaIQ\n2UQ0IAoKCAoREXCp+qqltVaordFg+2qqtlDTtwp1K7VWWq2askQwWlCWnxKXyiIRVAQRJQurBgQS\nyHJ/fzwzYZI8k8xk9pnz/XzymZn73Oe5J08m9zz3nHPPEWMMiqIoSuwRF2oBFEVRlNCgCkBRFCVG\nUQWgKIoSo6gCUBRFiVFUASiKosQoqgAURVFiFL8oABF5XkT2icgWN8cvFZFDIvKx4+dBf4yrKIqi\ntJxWfrrOC8CTwD+a6POuMeYqP42nKIqi+IhfVgDGmHeAb/1xLUVRFCU4BNMHcJGIbBaRN0XkvCCO\nqyiKotjgLxNQc2wEzjDGHBGR8UA+cLZdRxGZAcwAaNu27eBzzjknSCIqiqJEPhs2bDhgjOnqSV/x\nVy4gEekFvG6M6edB313AEGPMgab6DRkyxKxfv94v8imKosQCIrLBGDPEk75BMQGJSDcREcf7oY5x\nDwZjbEVRFMUev5iARORl4FKgi4gUAw8BCQDGmGeBKcCdIlINVADTjaYhVRRFCSl+UQDGmOubOf4k\nVpiooiiKEiboTmBFUZQYRRWAoihKjBKsMNCwI39TCbkrtlFaXkFqSjJZY/uQOTCtxf0URVEijZhU\nAPmbSpi95BMqqmoAKCmvYPaSTwDqTe6e9lMimKI8WPUwHCqGjj1g9IOQPjXUUilKUIhJE1Duim11\nk7qTiqoacldsa1E/JUIpyoPlM+HQbsBYr8tnWu2KEgPEpAIoLa/wqN3TfkqEsuphqGrwt6yqsNoV\nJQaISQWQmpLsUbun/ZQI5VCxd+2KEmXEpALIGtuH5IT4em3JCfFkje3Ton5KhNKxh3ftihJlxKQC\nyByYxpzJ/UlLSUaAtJRk5kzu38ix62k/JUIZ/SAkNFjNJSRb7XYU5cHj/SAnxXpVX4ES4fgtGVwg\n0GRwSsDxNArI6TB29RkkJMPV8zVqSAkrvEkGF5NhoO7QmP8YJH2qZxN4Uw5jVQBKhKIKwIHG/CtN\nog5jJQqJLR9AEzZcjflXmkQdxkoUEjsKoJlNPw1j+yfGrWNd4kzerbhGHX6K9w5jRYkAYscE1IwN\nNzUlmRKHEpgYt465Cc/RRk5Y/ZzKAtTeG8U06QNy/t01bYQSRcSOAmjKhpufT1bb48w+3I6KWriv\nVd7Jyd+JOvwiC5fonmPJ3XisahoLjwytm9gxhtyCrZQerSY1sZbL2lWx+FBrKhxWQKcPaP3X37Lm\n8/0OpdCFrLEr1CekBI4g56aKHQXQsYfD/NOA8hq45hoyAfqOJHfkLaSe6qZUsTr8IoMGIZttKsq4\nzzzNt3EnWFY+nKxXNkBtLVXxCQCUnIjjpYMJGKl/mYqqGl764BucgdIaGKAElIahxkGwPMSOD+CS\n2TiqVLrQCi78JWzcCNu2kfnEAxT+sJy444n211CHX2RgY+5rIye4r5Xlx6mS+LrJ34kR+3+Fhrtk\nNDBACRghyE0VGyuAzz+Hmx6FVofgqk7Q+oT98uqHP4QxY6AoDV67G2oqXS6SAKP+N+iiKy3AzUot\nVQ765fKaDFAJCCEINY5+BfDaa/CjH0FSEvxjOYwb1/w5DR1+xxPh9e/gvwvhxSvglFMCK7PSYvI3\nlTCMLnRjf6NjpaZzk+cK9Z/4G352oskAlYDgzkwdQMuDX0xAIvK8iOwTkS1ujouIzBeRHSJSJCKD\n/DFuszz1FGRmkn/JFDJmvkjvtTVkzF1N/qaS5s9Nnwr3bIGccvj9Xrjrz/D22zB4MGzeHHjZFa9x\nbub7/YnrOGbqm/GOmUQeq7YUe0KckBBf3+CfnBDPjcN61sv7dOOwno2TAcbDZed0JWPuanpnF3j+\nfVKU5ghBqLG/VgAvAE8C/3Bz/ErgbMfPhcAzjtfA8fe/w89/Tv5N9zC71xVUHKkCWujIE4Gf/hQG\nDYJrr4WRI+HNN+GiiwIlvdICnJv5ljEcqqxorlQ5yB7pzIKEm1h+fChpziggR//m0n4MOeOUk/2O\nfstlO/7LYsY1ihYCdQwrPhKCUGO/JYMTkV7A68aYfjbH/gKsNca87Pi8DbjUGFPW1DVbnAwuLw+u\nvx4uv5yMS++j5FBloy5pKckUZo/y/trffAOXXw6lpbBsGYxqwTWUgNA7u8DWZCPAV3Mn+D5AWRkZ\nc96mpE1jE2CLv0+K0pBdu6C4GIYPb9Hp3iSDC1YUUBrgatwqdrQ1QkRmiMh6EVm/f39jO26zHDwI\nd9wBGRmwdCmlNpM/+ODI69kT3nkHeveG8ePh//2/ll1H8Tup8VX27f6y2XfvTqnN5A/qGFb8xPHj\ncN11kJkJR44EfLiwCwM1xiwwxgwxxgzp2rWr1+fnf1NJxq9epffF95Mx/wNS2jQM/bTwaVLo1g3W\nrIEzz4SJE9UnEA48/zxZS58guba+EvB3AR+tEqcElF/8Atavh+eeg3btAj5csBRACXC6y+cejja/\n4nQCllTUYrDss0cqq20dfj5PCl26wIoV0KGDFVn01Ve+XU9pOW+9BTNmkNkziTnXDQxoAR+tEqcE\njBdfhGefhawsawUQBILlA5gA/BwYj+X8nW+MGdrcNb31AWTMXV2Xz8eVlOQE2rZuFZg8/59+atnq\nunWDDz6wFIISPD791HLGn3EGFBZC+/YBH7IuZ9B3x0g9vJ+s0WeROeWSgI+rRDHbt8OAATBkCKxa\nBa1aHp8T9IIwIvIycCnQRUSKgYdwbLs1xjwLvIE1+e8AjgH/449xG+LODnuoooqPHxoTiCGhegtk\nnQKVxTD3BzDtz3D+tMCMpdTnwAGYMAHatIHXXw/K5A9WtE/mwDQoL4fzz4e3E2HcpqAs2ZUopLbW\n8lsmJsLLL/s0+XuLX0YyxlzfzHED3OWPsZrCNaNnw/aA4Jq7QwQSj8OSn1nvNWlcYKmpgRtugLIy\nePddyzkfbFJS4B//gMsug9mz4c9/Dr4MSuTzl79YgSXPPw+pqUEdOuycwL4QdPusXe4OqYbXZwdm\nPOUkv/2tZft/6im44ILQyTFyJNx1Fzz9tJVTSlG84Ztv4L77rNDyW28N+vBRpQAyB6YxZ3L/gDoB\n6+EuR8fxA7DbZku34h/eeAMeeQRuuw1uvz3U0liydOliKYLa2lBLo0QSd99tfWcWLLAsB0Em6nIB\n1dlnA4jTCfhqbWd6xNmkjj5srI1oa9cG1Z4XrdQr1NI+gazXnibz/PPhySdDLZpFSgrk5sItt8AL\nL1iKSVGaY9UqazPpnDnWvqIQEFUrgGBQF2paXsFj1VMb5ZwhIRn63GZFpDz0UGiEjCJc77cBSr6v\nYvaI/yH/93+F5DCKvf/Rj6xosPvvt5zDitIUNTXwq19Z0Wu/+EXIxNDHUy9xLR6/rNYl50zcQfbR\nhTlHr2N9/BVk3ZlC5pyHye8/mtxdBCYENQZwvd9OKhKSyC36nszxIRLKDhGYP9/KF5WbC7/7Xagl\nUsKQk6vZY6ReNIus9A5kJiU1XY40gPhtH0AgaHEuoADiLt9MQ5IT4rj207Us7n0hFa1au7THB9Yv\nEQEU7Cxg3sZ57Dm6h25tuzFr0CwmnNk4V0/BzgLuWzUXSSjHVKVwfP9Yqg8PBKz8Po9PGxCSf5om\nuf56a1n/5ZfW3hBFceBczbo+0CQnxHHt4B4s3lDSoL3l80Q45gKKGjwNKa2oquXlH15SN/lPjFvH\nusSZbI2bxrDXRlohpFFEwc4CxiwaQ/rCdMYsGkPBzgK3/XLey6HsaBkGQ9nRMnLey2nU39kvLrEc\nEYhLLCep+xJaddgEQMfkhPqmIUdWzpCnZn7kEThxAh59NLRyKOFFUR7DXhvJ1rhprEucycS4dYBj\nnvhwd+NVbpAqz6kC8BK7UFN31DiWChPj1jE34Tl6xB0gTrCKlSyfGRFKwJOJ3dNJHWDexnlU1tRP\n0FdZU8m8jfOa7SdxVbTuuoLkhHhEqPunadVhE23Pmkv8WVk8uPF6t8onKPzgB3DXaEhYCDkp8Hi/\niPg7KwHEsV+oG/uJE+gRd4C5Cc/VKYEaN1aYYCQYVAXgJXahpp3cJJyLd4R13dcqjzZyov7BANf6\n9AeeTuyeTuoAe47usR2rYbu7fnEJ5cyZ3J/yY1bSt1YdNpHUfUndSsG0+s6t8gkk+ZtKyJi7mlkP\nzKYiZQN0jAPMycLeqgRil2ZqVMe7Cf8MRoJBVQAtIHNgGoXZo/hq7gQKs0fx0NXn2W5Au/7C00lO\niCdVbEJFIaC1PpvDkyd7Tyd2Tyd1gG5t7e3iDdvd9everjuZA9Pq/jlad12BxNXPAOpO+QQK10il\nrFZ5JBN5yl4JIE3UqHadJ1wJVoJBVQB+wN0GtEcz+zNncn/2iX1a6+Lazn4vKehPk42nE7unkzrA\nrEGzSIpPqteWFJ/ErEGzGverjWvcr8uF8Hg/1lVOprD1TOIS7EMu7WT31E/hLa6RSuGo7JUQ46am\n7z7pUm+eCNoGVhc0DNRPuNuAljkwDeJ/D8tmQvXJZaCzRq0/Swo6J3bnU7tzYgfqRdk09WTv2q9b\n226UHW1ctK3hxD5r0Kx644L9pO4qh7sooJPhcNCl1Vg69vx/HG5dYfXrciETCv8KVRXWP4ocoFt1\nGntsfDINZfT03rQEV1ttqelCDzslEMDC3kp44vwuDzl8NXMT/0Yyx08eTEim29W/JzPd+p8PxgZW\nO3QFEAzSp8LE+ZB8GrUGimu7kF11h7WPAM88/qEw2Xj6tD7hzAnkXJxD97bdEYTubbuTc3GO24l1\nwpkTWDllJUW3FLFyysp6k79rZM/+6pEc/CaHh9PftPptWtrIlvqL774jqYETzU5Gb/wU3uJqq7Xd\nHGhaBbSwtxJ+uH6XX6sdzv0nbqektjMGgY6nw9XzwyJhpK4AgkX6VEifylnZr1tfAiwHZuuuK5CE\ncsqrUijYWeE2Ht6Tp1dvTDaePNk397TesK+vT9K2m76qa8ldsc16OrIxo0w4egwQ5vXu16SM3vgp\nvCVrbJ+6+G7n5sD7E6yC9HKiNbxdCb8c6/M4SuTQ8Lu8rHY4y04MJy0licJ7RodQsvqoAggATW10\nSu2YTMmhyrroFacDUxLL3ZokQm2y8XVi94iiPF49NpvU1gcoNV14rHpq3QqpzsTSsYcVVdNQxlan\nMGHKyiYv7+m9aQnOpbtzU9qGDlfw0difW+0bN8KcwfC3v8Evf+nzWEpk4C6Es7TcvkZ5qFATkBf4\nw8GaNe4ckuO9i14Jtckm4DjipJ37JBrGSdeZWEY/aOVaciUh2SPziqf3pqU0jAyrs+cOGmTlCHry\nSSv/ixITRErtaF0B4FlqAn85WJ0Tw/9u9jx6JVxNNn6jiTjpt8zIk+FwTpvpqoctc1DHHtbk74Et\n1Zt743dmzYLrrrOqlk2aFPjxlJDjahZ0Eo61o6M2F5A3+WbszCENn4jHLBpjOwl3b9udlS7mh/SF\n6RibbEGCUHRL0cnr5V1OWcXeZq/njYwRS04K2NyzWoRlk7aGJDrC0+9PU9RL8HX0IFl7PiDz308H\nSGIl3MjfVELuS4WUxieT2jGZrCv7BifBW6znAgpEagJ/x8TPGnIPSTTYQVybwFdfXNJob0DYmWxa\ngHOnbO/sgsZ7H9yESMZ17BGyyd/T7487GqWxbtuZ2T1Hk7/s/cAJroQVmT2TKHzyZr76/k0KH7g8\n9IkKbfCLAhCRcSKyTUR2iEi2zfFbRWS/iHzs+LnDH+O6IxCpCTye2L2xw494hO4VCWDAVKVQUTaZ\nqsMDbRObuQudjAQaTYYNf7+ES+FEgxWAh7b9QOCPkFG3aazf1Q1hMcPf/w7Hj8Odd4ZaErf4rABE\nJB54CrgSOBe4XkTOten6qjFmgOPnOV/HbYpApCYIhIN1wpkTWHntCjp8dC9HdmTXpTqG4GUDDAa2\nk2FVDTnLtpLxu7fovXscOVW3cSy5O4RBnLQ/QkbdRoHEJcGhQy2SS4kgamvhmWdgxAjo3z/U0rjF\nH07gocAOY8xOABF5BZgEfOqHa7cIb0L+PA2JDJiDtWtXStt3tj0UjGyAwcDd71FeUUU5gAgvxF/B\nq0fGhUWtBH+EjKamJFNi83unHj4A//pXWD8VKn5gxQrYuTPsCwP5wwSUBrgGZxc72hpyrYgUicgi\nETndD+O6xZuQP6+f2ANghomUkLGW4nkNhfBY9fgjZNQubXhyQhxZu9bCX/4CYRx8oXhPIx/Xiyvh\n1FNh8uRQi9YkwQoDXQ68bIw5LiI/ARYCo+w6isgMYAZAz549WzSYtyF/oQ6JzBp7DrMXb6ai+uSk\nEI4hYy3FLiTOHeGw6vFHyGjDzWF1FctSd5H/56/Jffg/lFbUhk8lM6XFNKz0VVJewexuw+G2nmQm\nJjZzdmjxOQxURC4CcowxYx2fZwMYY+a46R8PfGuM6djctcOxJGSgyN9UQu6i9ZRWx5GaYMi6dnBU\nTQoNa54e+7ac7+JaN+qXlpJMYbbts0FUkP/edmYvLqIi4eQKQ8uERjYZc1fbmvvS2rWi8DfBTwHi\nTRioP1YAHwFni0hvoASYDtzQQKDuxhinUXUi8Jkfxo0qMgemkXl+d7j0Uigqgp+FzIUSEOplO3zr\nLfJnzWf21b+kQk6aSaJp1eOO3Hd215v84aTpSxVAZOLW4X+kOsiSeI/PPgBjTDXwc2AF1sSeZ4zZ\nKiIPi8hER7eZIrJVRDYDM4FbfR03KomLs3LGHD8OP/1pdNqJDx+G228n0+xlzrWhyYEeStzniAm9\n6UtpGZHsw/OLD8AY8wbwRoO2B13ezwZm+2OsqOfss62C4vfeCy++CD/6Uagl8i/33gslJfDee2QO\n7U3m0N6hliiouI0OioDJQrHHPu1DXESsZqNyJ3BEU5QHcc/Dgx1g089h7bOhlsh/rFwJf/2rpQQu\nvDDU0oQE++ig6Dd9RTN1FQE7tEZMLWm1x5gzOT0iVrNRmwsoInFkxXRNjFZhEsmuuoP1Ha6I7GiR\nb7+F9HTo0MFKkZyU1Pw5EUBLcgblbyoh942tlB4+TqqcIGvasMj9uyonefZZa3/HRx/BEI98sAHB\nGyewKoBw4vF+tvnui2u7MPzE/MbRIkV5LcqMGRKuvx4WLYIPPoDBg0MtjV/wOUnf+PHwySewaxf5\nRXsah4yqUogsLr4Yvv/eCuIQCZkYwY4CUvyFm8LhqXIQaBAt0nC1cGi39RkCrwSaUTwNQz6z2h8k\n85VXLN9GlEz+4HmhHrdkngM/fBfzSGcuMJ0ZXDWVEob7tU60EiR27ID334c//CGkk7+3qA8gnHCT\nFbPUnEwVUVJeQcbc1RQvmt0ohz5VFdbEHEiciufQbsCcVDxFeYCbxG8748mfdAfcf39gZQsyPuUM\nKsqDff+ClDgEQ5rUL4ITLruiFQ958UVr4r/hhub7hhGqAMIJm4pXx0wij1WffLoWrEk1VQ7YX8PN\nKsJv2BRvcVU89lkwW5M7ZAq0iq4Fp6eJBG1Z9TBU2xfBcaKhoRGCMZYCGDUKetg/xIUrqgDCifSp\nVhbMjqdjEEpMF7Kr7qirjSumtq5sSqnpYn8NN6sIv+FOwTjaG05aE+PWsS5xJu9WTbV8HEV5dmdH\nJD7lDGrG3AcaGhoxvP8+fPllRIZsqwIIN9Knwj1bkJxyPsp8hw0drrA2SnVsjeGkbfGx6qkcMw3y\njAQjh747BeNod520JsatY27Cc3W1fhuaiyIdnwr1NGPu09DQCOKf/4Tk5LBP/GaHRgFFEBmPrqDE\nZXv5xLh13Ncqj9S4g8QFKwrIJlSVhOS6/P35m0rqEtutS7QKvTei4+lwz5bAyhnu2IX8kkj2iSgI\n+Y0lNv4LXrwT2gMpp4dFJJ5GAUUpWRP6Mfvfm6iotVYCy2qH85YZyZxJQUyh0Exh9sw+nWDDq+Se\neRmpXULkp4gEGt7H8hqSe0xi3r22ORSVcKQoD5bPgg6Oz8GMxPMTugKIMPI3lZD7ygeUkkhqqxqy\nplwQPk+KxsBNN8HLL8Prr8O2+2z3NegKwIb0dGjXDt57L9SSKJ7iZt9OqL/fMV8UPprJHJhG4e8z\n+erTBRTOvZbMfWE0kc6ZY1W7euQRa5OTTVRTKGv9hjU33mg5E3fuDLUkiqc0ExARCagCiETi462J\ntl8/mDqV/OUf1K9G5FJMPmgsXAi//rU1kT3wgNXmEtUUDrV+w5rrr7deX345tHIontOqk317oCPx\n/IiagCKZb74hf/pMZl98MxWtThZXCVSBkUY7fJ2OyhUr4KqrrFoGBQUQ5lWQwpaMDDhyBDZvDrUk\niif8z0WQ+ikkuLS5BESECjUBxQo9e5J75U/rTf4QmF2ktjt8l3xC/ksrrfC3886DxYt18veFadOs\nPDKffx5qSZQmyN9UQsbv36b3qQ+QY+7gWHJ3InWFqwogwik9al9n19+7SG13+FbVkFtYCr17W6uA\nDh3cnK14xJQpVjqBV18NtSSKG+oehA4fx0gcL9SOYvCRJ8iftNVy/EbQ5A8aBhrxeFNgxK0Jx4aG\nfe3GACht3wVWr4ZTT/XtF1EgNRVGjIC8PHjooVBLo9jg9kEoQkt66gogwrEtMFJ1nKxe9X07bk04\nNg5ju77u8humdmitk78HFOwsYMyiMaQvTGfMojEU7Cyw7zhtGnz6KWwJo+gupY5oK+mpCiDCqatG\n5Kyt264Vcz7+N5k3jYXHHoPaWqDpJ5eG2PU10EgJJCfEcdl53UIfgRTmOOsGlB0tw2AoO1pGzns5\n9krg2mut2tB50ZEuI9qI5Pq/dqgCiAIyB6ZRmD2Kr+ZOoPA3Y8lc+hfLMXv//XD11fD11149ubjr\na4wh7dA+K31xShLXDu7B4g0lHq0qYpmm6gY04rTTrGiqV1+1NtYpYUXW2D4k11bVa4vkvE1+UQAi\nMk5EtonIDhHJtjneWkRedRz/UER6+WNcxQ0dOpCf/ScyHlhK734/JeeJXAoT72Zn6xtYlzizLuc8\n2D+5uHuaSTv2HYXTz+SruVdRmD2aNZ/v93hVEct4XTdg6lT44gs1A4UB+ZtK6q1wOXqUOW/+mTRT\naa24U5IDEnIdLHx2AotIPPAUcAVQDHwkIsuMMZ+6dLsd+M4Y8wMRmQ78AZjm69iKPfmbSpi9dAsV\nNQlMjF/HfR1epY2cAKCHo/AIVfBW/EiyxvZp5PC97IddWLx+NxW1J6+ZbGrIuvFiGHZWXVu02UMD\nRbe23Sg7WmbbbktmJvzsZ1YJzf79Ayyd4g6nL8z5kFNSXsHsgi+YU1ND4ZSecMEFIZbQd/yxAhgK\n7DDG7DTGnABeASY16DMJWOh4vwgYLRJBddMiDFcb/n2t8uomfydt5AQPxL/MnD3vwJIlzH51Yz0z\nzuLCHVz70XLSDu9DjCGtfQJzpg8m02Xyh+izhwYKr+sGnHYaXHKJpQCUkGHrNzNC7ujbQlr03Z/4\nIww0DXDNiFQMXOiujzGmWkQOAZ2BRukiRWQGMAOgZ8+efhAv9nB9AndXOaxb3HdkFrxAxqTfUdGx\n/nNARUJr1lw0gcL7LrMSlLkha2yfek9IENn20EDhrA8wb+M89hzdQ7e23Zg1aFbTdQOuvRbuvhs+\n+wz69g2SpIorble4bU6JqLq/TRF2TmBjzAJjzBBjzJCuXbuGWpyIxPUJ3G3lsJTTobSU0o72IZyl\nJ+KanPzBJgIpwu2hgWTCmRNYOWUlRbcUsXLKyuaLxjiLiyxeHHjhFFvcrnDbxNu2RyL+UAAlwOku\nn3s42mz7iEgroCNwECUguO4NaK5yWMMKXusSZ7Kz9Q28nzTLo8pd9SKQskfp5O8vUlOt3EBqBgoZ\ntntsqo+TdVW/EEnkf/yhAD4CzhaR3iKSCEwHljXoswy4xfF+CrDahHMWugjH9cl8ee1wHkv4mdt8\nJc4vecPyjd3YH1XlGyOSa6+1EsPt2BFqSWKSRivcw/uYU/M5mYNPb/bcSMEv2UBFZDzwBBAPPG+M\n+Z2IPAysN8YsE5Ek4J/AQOBbYLoxptnE55oNNDjkbyph2GsjrUm/IVq8JXR88w2ccYZVZyG7UXS1\nEkz+/W8rPHftWhg5MtTSNIk32UA1HbRikZOCtd+3IQI55cGWRnEydKjlcPzww1BLEttMn27lvCor\ns+pxhDGaDlrxHndFLCKouEVUMnky/Pe/UBw5VaaijspKq85FZmbYT/7eogpAsdDyjeHJNddYr/n5\noZUjlnnrLatQjzMyK4pQBaBYaPnG8KRPHzj3XFiyJNSSxC5LlkDHjjBqVKgl8TtaD0A5SfpUnfDD\nkcmT4fe/hwMHoIubfR1KYKiuhmXLrJKnUVjtTlcAihKm1CUiqxpGxoznyH9xZahFij3efRe+/TYq\nzT+gCkBRwpJGRXk6nsrs0jaaajvYLF0KSUkwdmyoJQkIqgAUJQyxTUQWl0Dum5+FSKIYxBjL+T5m\nDLRtG2ppAoIqAEUJI5ylIw91m0Xbs+bSqsOmesdLD1e6OVPxOxs2wO7dJyOxohB1AitKmOAsHVlZ\nU4kISGI5Sd2XUAlUHx4IQGr1sdAKGUssXWrF/V99daglCRi6AlCUMMGudKTEVdG66woAkk01WWtf\ngBMnbM5W/EZRHjzeD1o9Cfd2gpJVoZYoYKgCUJQwwV2JSEkot1Jt94knc8ObVj4aJTAU5VlJEA/t\nBgGSTkR1UkRVAIoSJrgrEZnarruVavv60ZYzUncFB45VD0NVg0IwVRVWu59x+nvSF6YzZtEYCnYW\n+H2M5lAFoChhQrOlI5OTYdw4SwHU1tpcQfGZQ25yLrlrbyFOf0/Z0TIMhrKjZeS8lxN0JaAKQFHC\nhAlnTiDn4hy6t+2OIHRv252ci3OYcOSoZZPOSYEBG6Dzfvjoo1CLG50EKSminb+nsqaSeRvn+XWc\n5tAoIEUJIyacOaF+uUinTdpplqj+Fq5Ohtf+ABdqfiC/M/pBWPozMFUn2wKQFNGdv8dde6DQFYCi\nhDN2NulEobjVZjLmrtadwf6Veb9LAAAgAElEQVQmfSps7wVH4ghkUkR3/h537YFCFYCihDNubM+p\ncpCS8gpmL/lElYCfyN9UQsbv36Z32kNkxP+V/ElbrWp4AUiQ2Ky/J0ioAlCUcMaN7bnUdAagoqqG\n3BXbgilRVFKXe+nwcYzEUSLJAVWubv09rua/IKA+AEUJZ0Y/WN8HABwziTxWffKptLS8wu5MxQts\ncy85lGvmwDSvrlWws4B5G+ex5+geurXtxqxBs2wn9kb+nhDgkwIQkVOAV4FewC5gqjHmO5t+NcAn\njo/fGGMm+jKuosQMTvPDqoepPVRMaW1nHqueyrLa4XVdUlOS3ZyseIo7JeqtcnVN5wHUhXcCIZ/s\n7fDVBJQNrDLGnA2scny2o8IYM8Dxo5O/onhD+lS4ZwvLJm3lCvNUvck/WWrJGtsnhMJFB+6UqLfK\nNVzCOz3FVwUwCVjoeL8QyPTxeoqiuCFzYBpzJvcnLSUZAdKOfsucr9/22kShNCZrbB+STXW9tuSE\neK+Va7iEd3qKrwrgNGNMmeP9HuA0N/2SRGS9iHwgIqokFKWFZA5MozB7FF/NnUBh681kLn4Wvv8+\n1GJFPJnnd2dO4QukVR2xlGtKMnMm9/dauYZLeKenNKsARORtEdli8zPJtZ8xxgDGzWXOMMYMAW4A\nnhCRs5oYb4ZDWazfv3+/N7+LosQWkyZZmUHffDPUkkQ+GzaQWZhP4fnHLeWaPapFK6twCe/0lGad\nwMaYy90dE5G9ItLdGFMmIt2BfW6uUeJ43Skia4GBwJdu+i4AFgAMGTLEnUJRFOXii60i8a+9BlP9\nH6seU+TnW7n/J/jmqHU6ej2JAgoHfA0DXQbcAsx1vL7WsIOIdAKOGWOOi0gXIAN4zMdxFUWJj4eJ\nE2HxYmslkJgYaokil9deg0sugc6dfb5UOIR3eoqvPoC5wBUish243PEZERkiIs85+vQF1ovIZmAN\nMNcY86mP4ypKzFOws4Axl35B+hM9GJN3eUjSCUcF27fD1q2WSa0JwiF9s7/xaQVgjDkIjLZpXw/c\n4Xj/HtDfl3EURalPXbx5bSWIUFbzXVjHm4c1rzkMF00ogEiL7/cUTQWhKBFIpMWbhzX5+TBgAPTq\n5bZLtN5vVQCKEoFEWrx52LJ3L7z3HmQ2HZ0erfdbFYCiRCBu480TOpwsHvN4v6itZes3li8HY5pV\nAJEW3+8pqgAUJQKxjTevNszas9sqaI6xXqO4oLlfyM+3TD/p6U12i7T4fk9RBaAoEUijdMI17cgp\nPsCEw+X1OwaooHlU8P338PbbcM01INJk13BJ3+xvNB20okQo9eLNv/4anu9vP5H5uaB51LBiBRw/\n3qz5x0kkxfd7iq4AFCUaOOMMqEiwP+bnguZRQ34+BVf0YMzeR6Mqtt8bVAEoSrTQdiycaJA9JQAF\nzaOCEycoKFtNzvRTKDtWhsHUxfbHkhJQBaAo0cKUX8PyCohLIZAFzaOCtWuZN749lfG19ZqjIbbf\nG9QHoCjRwrnnQmUveO8s+M9/Qi1NeLNkCXuG2ZvMIj223xt0BaAo0YKIFdGyejUcOhRqacKPoryT\neyTavkK34/bTX6TH9nuDKgBFiXDyN5WQMXc1vbMLyGh7Gfk/uBgKYseO7RFFedaeCOceifaGWd9/\nR5LUXwVEQ2y/N6gCUJQIJn9TCbOXfEJJeQUGKKmoZfb4meSv3BRq0cKLVQ9beyJcmHDkMDnfn4i6\n2H5vEKuQV3gyZMgQs379+lCLoShhS8bc1ZSUVzRqTzl2mLbdu1J66DipKclkje0TU7WDC3YW1C/K\n8tUWJhw9atNTIKfcpj1yEZENjgqMzaIrAEWJYEptJn+A8uT2lBw6jgH21r7HbzZMp3+MxLo7UzeX\nHXUJ7+x6CgVt2zTqW1zbmYy5q8nfVBICSUOPKgBFiWBSU5LtDzh2BLfqsImk7kuQhHKIkVh329TN\nIsw7pVO9tmMmkceqp1JSXsHsJZ/EpBJQBaAoEUzW2D4kJ8S7Pd666wokrqpeW7THurtN3dwqHo7G\nU2uguLYL2VV3sKx2OAAVVTXkrtgWTDHDAt0HoCgRjNOun7tiG6XlFaSmJHPsRDXfHbMmfevJvzHR\nHOverW03yo6WNW5P7AJ/3MJZ9y/H0DhnkjtzWjSjKwBFiXAyB6ZRmD2Kr+ZOoDB7FA9dfR7JCda/\ntqlKsT0nmmPd3aZuLvsBAKntE23Pc2tOi2J8UgAicp2IbBWRWhFx63UWkXEisk1EdohIti9jKorS\nNJkD05gzOZ206iOc2DcGamMr1t1t6uYXN8BFF5E1/rxGZrPkhHiyxvYJkcShw1cT0BZgMvAXdx1E\nJB54CrgCKAY+EpFlxphPfRxbURQ3ZA5MI/PLeLjuQQqW/YF5x98+GRI5aFbExro3Cu9087s0St28\ncyds2gR//KOt2SzWwmSd+KQAjDGfAUjTxRSGAjuMMTsdfV8BJgGqABQlkIwfD23aMOHNXUx4emWo\npfEZZ3inM8LHGdEENK/QFi+2XidPBhwKMgYn/IYEwweQBux2+VzsaFMUJZC0aQMTJsCSJVBTE2pp\nfMY2vNPTiKbFi2HwYOjdO0DSRSbNKgAReVtEttj8TAqEQCIyQ0TWi8j6/fv3B2IIRYkdpkyBvXvh\n3XdDLYnPuA3vbC6iafdu+PBDuPbaAEgV2TRrAjLGXO7jGCXA6S6fezja3I23AFgAVioIH8dWlNhm\nwgRIToZFi+DSS5vs6ql9PVS4De9sLqLp3/+2Xq+7LgBSRTbBMAF9BJwtIr1FJBGYDiwLwriKorRt\na/kCFi9u0gxkmz4hiDuGC3YWMGbRmCZLM7oN72wuoikvDwYNgh/8wJ8iRwW+hoFeIyLFwEVAgYis\ncLSnisgbAMaYauDnwArgMyDPGLPVN7EVRfGYKVNgzx4oLHTbxSf7uo94qnzchnc2tUrZtcsy/0zV\nqmh2aDZQRYl2jhyBrl3h9tvhySdtu6QvTMfQeC4QhKJbigIq3phFY2xNO93bdmflFB+jl3Jz4b77\nrDDQGHEAazZQRVFO0q4dXHWVZQuvrrbt4s6OHowdwy127npCXh5ccEHMTP7eogpAUWKB6dNh3z5Y\nu9b2sDf2dU/s9d70C5jy+fJLWL+e/Il3nKyYFsOpn+3QZHCKEguMHw/t28Mrr8DljQP7nHb05qKA\nPN2M5c2mrVmDZtXrCy1PV5G/qeTkDl85wWWX/4TFlT2oqLYSvTlTPwO6EYwI9AFUVVVRXFxMZWWl\nm7Oih6SkJHr06EFCQkLznRWlOW6+GZYvt/YFJNonRGsOT+313tr1/RGC6iyPWVF1MtpJjMHYZCpI\nS0mmMHuUV9ePFLzxAUTcCqC4uJj27dvTq1ev5lJQRDTGGA4ePEhxcTG91X6p+IPp0+Gf/4SVKy2f\nQAvw1F7vrV2/Ue6eFpC7Ylu9yR+wnfwhNlM/2xFxPoDKyko6d+4c1ZM/WPmVOnfuHBMrHSVIXH45\nnHKKZQZqIZ7a60PhVPZmUo/F1M92RJwCgGaTz0UNsfJ7KkEiMdFKh5CfD8eOtegSnjqLW7xpywfc\nTeoN/4tiNfWzHRGpACKB8ePHU15uX41JUULGDTfA0aPw2mstOt3TzVgt2rTlI3blMZMT4rlxWE/S\nUpIRLNv/nMn91QHsIOKcwJ999hl9+/YNkUTNY4zBGENcnH90a7j/vkqEUVtrxcSfdx688UaopfE7\n+ZtKyH35fUqlNakdk8m6sm/MTfa6ESzA/OlPf6Jfv37069ePJ554gl27dtGnTx9uvvlm+vXrx+7d\nu+nVqxcHDhwA4JFHHqFPnz4MHz6c66+/nj/+8Y8h/g2UmCUuDm680XIE790bamn8Tma/Uyn8+518\ntetFCh+4POYmf2+JuCigevziF/Dxx/695oAB8MQTbg9v2LCBv//973z44YcYY7jwwgsZOXIk27dv\nZ+HChQwbNqxe/48++ojFixezefNmqqqqGDRoEIMHD/avzIriAXUx8rUZpP74r2Q9/zqZs28PtVj+\nZeVKOHDAMnUpzRLZCiAErFu3jmuuuYa2bdsCMHnyZN59913OOOOMRpM/QGFhIZMmTSIpKYmkpCSu\nvvrqYIusKI1i5Es6nsrsb0/AppLoekp+4QXo0gWuvDLUkkQEka0AmnhSDzZOhaAo4YhdjHxFfCK5\nr2+JHgXw7bewbBnceWeLN7rFGuoD8JIRI0aQn5/PsWPHOHr0KEuXLmXEiBFu+2dkZLB8+XIqKys5\ncuQIr7/+ehClVRSLhjHyE+PWsS5xJu9WT4XH+0FRXogk8yOvvgonTsAtt4RakoghslcAIWDQoEHc\neuutDB06FIA77riDTp06ue1/wQUXMHHiRNLT0znttNPo378/HTt2DJa4igJYMfIlDiUwMW4dcxOe\no42csA4e2g3LZ1rv0yMwb35RHqx6GMp3Q1YniN8ODAy1VBGBhoEGgSNHjtCuXTuOHTvGJZdcwoIF\nCxg0aJBH50bi76uEH64+gHWJM+kRd6Bxp46nwz1bgi9cC3A6tIccfou5iX8jmeMnDyYkw9XzI1OZ\n+YGozgUUicyYMYNPP/2UyspKbrnlFo8nf0XxF047f+6KbaRW2Ez+AIeKgyhRy3FVZq8m5tWf/AGq\nKqwVQYwqAG9QBRAE/vWvf4VaBEUhc2CapQgeP90y+zSkY4/gC9UCXB3aqRLZyizUqBNYUWKN0Q9a\nZpJ6JFjtEYCrQ7vUdLHvFCHKLNSoAlCUWCN9qmUj73g6IFDRCj5oB/2vC7VkHuGa9O2x6qkcMw1C\nPhOSI0aZhRqfFICIXCciW0WkVkTcOh1EZJeIfCIiH4uIVnlXlFCTPtVy+OaUw9l/gBVfw3vvhVoq\nj3BN+rasdjjZVXdQUtsZg1hKLYYdwN7i6wpgCzAZeMeDvpcZYwZ46p1WFCVIXH89dOwITz8dakk8\nInNgGnMm93dk+DRs+O6HfDQyH8kpt5SaTv4e45MCMMZ8ZozZ5i9hIoHy8nKebsE/iqaHVsKWtm3h\n1lvh3/+OmARxmQPTKPzVCL5a+GMKS5aSOTo91CJFJMHyARhgpYhsEJEZQRoTsELGMuaupnd2ARlz\nV5O/qcSn67lTANXV1U2e98Ybb5CSkuLT2IoSMO68E6qq4LnnQi2J5+Tnw549MCOoU0pU0awCEJG3\nRWSLzc8kL8YZbowZBFwJ3CUilzQx3gwRWS8i6/fv3+/FEI1xxguXlFdggJLyCmYv+cQnJZCdnc2X\nX37JgAEDuOCCCxgxYgQTJ07k3HPPBSAzM5PBgwdz3nnnsWDBgrrznOmhd+3aRd++ffnxj3/Meeed\nx5gxY6io0PqkSojp08cqGfnss9DMw0zYMG8enHmmJn7zgWYVgDHmcmNMP5sfj0sKGWNKHK/7gKXA\n0Cb6LjDGDDHGDOnataunQ9himwCrqobcFS23Ws2dO5ezzjqLjz/+mNzcXDZu3Mi8efP44osvAHj+\n+efZsGED69evZ/78+Rw8eLDRNbZv385dd93F1q1bSUlJYfHixS2WR1H8xl13QXExLF8eakmaZ/16\nKCyEu++G+Pjm+yu2BNwEJCJtRaS98z0wBst5HHDcFYn2pnh0cwwdOpTevXvXfZ4/fz7nn38+w4YN\nY/fu3Wzfvr3ROb1792bAgAEADB48mF27dvlNHkVpMVddBaefDvPnh1qS5pk3D9q3h9tuC7UkEY2v\nYaDXiEgxcBFQICIrHO2pIuKsN3casE5ENgP/BQqMMf/xZVxPcVck2l17S3BNA7127Vrefvtt3n//\nfTZv3szAgQOprKxsdE7r1q3r3sfHxzfrP1CUQFHPR/bHd8if8RtYuxY2bAi1aO4pK7Myf952G3To\nEGppIhpfo4CWGmN6GGNaG2NOM8aMdbSXGmPGO97vNMac7/g5zxjzO38I7gnuikRnje3T4mu2b9+e\n77//3vbYoUOH6NSpE23atOHzzz/ngw8+aPE4ihJobH1klaeTP2gc5OaGWjz3PPOM5ae4++5QSxLx\nRPVO4PrxwpCWksycyf19KoDRuXNnMjIy6NevH1lZWfWOjRs3jurqavr27Ut2drZthTBFCRdsfWTV\nteRefocVErpzZ4gka4KjR639CldfDWedFWppIp6oTwZXlwDLj7hL7ta6dWvefPNN22NOO3+XLl3Y\nsuWkC+Tee+/1q2yK4ilufWSSZDlWH38c/vznIEvVDH/5Cxw8CNnZoZYkKojqFYCiKO5p0kd2443w\nt79ZBdbDhcpKyzQ1ejRcdFGopYkKVAEoSozSpI8sK8uacP/v/0IknQ3PP29t/Pr1r0MtSdSgCkBR\nYpQmfWTnngvTp1shoSFOD5G/qYSMOavo/XVPMma9RH7Hs0MqTzQR9T4ARVHc06SP7KGHrHDLP/wB\n/vSn4ArmwLX6FyKUJHVk9tItIOJ3314soisARVHs6dMHbr7Zirop8S2HVksJxG5+5SSqABRFcc+D\nD0JNDfwuaNt36hGM3fyxjCqAANOuXTsASktLmTJlim2fSy+9lPXrtU6OEob07m1l21ywALYEJYNL\nPVLbJ9q3+3E3fywT/QqgKA8e7wc5KdZrUV5IxEhNTWXRokUhGVtRfOLhh62UCzNngjFBHTpr74ck\nV9VPp+Lrbn7lJNGtAIryYPlMOLQbMNbr8pk+KYHs7Gyeeuqpus85OTk8+uijjB49mkGDBtG/f39e\ne61xotRdu3bRr18/ACoqKpg+fTp9+/blmmuu0XTQSnjTubNlAlqzBvz4ENNsrY6NG8l8Ooc5NZ/7\ndTe/cpLojgJa9TBUNZhcqyqs9haWjZs2bRq/+MUvuOuuuwDIy8tjxYoVzJw5kw4dOnDgwAGGDRvG\nxIkTERHbazzzzDO0adOGzz77jKKiIgYNGtQiWRQlaDjNQL/8JYwfb1UR84F60T2crNWx/utvWfP5\nfkrLK0it+I6soVeRmXMXmVpMKSBEtwI4VOxduwcMHDiQffv2UVpayv79++nUqRPdunXjnnvu4Z13\n3iEuLo6SkhL27t1Lt27dbK/xzjvvMHPmTADS09NJT9dydkp4kb+phNwV26yJOCWZrLF9yHzySRg+\nHB54wErH7APuonte+uAbnEamkuROzL70DvjqKJkDVQEEguhWAB17OMw/Nu0+cN1117Fo0SL27NnD\ntGnTeOmll9i/fz8bNmwgISGBXr162aaBVpRIwPXpfGLcOu47lkdq/kG+SzyVedm/ZGHtpaQ+VEBW\n5oAWm2LcRfE09DBU1Aq5K7apySdARLcPYPSDkNAgWiAh2Wr3gWnTpvHKK6+waNEirrvuOg4dOsSp\np55KQkICa9as4euvv27y/EsuuaQuodyWLVsoKirySR5F8SfOp/OJceuYm/AcPeIOECeGTlV7uS/x\nH1wd/x4lx2H24qKTdnsvgy28ieLRkM/AEd0KIH0qXD0fOp4OiPV69fwW2/+dnHfeeXz//fekpaXR\nvXt3brzxRtavX0///v35xz/+wTnnnNPk+XfeeSdHjhyhb9++PPjggwwePNgneRTFnzgn3Pta5dFG\nTtQ71kZOcF8ra3KvqK4ld8XnLQq2sMtDZO8x05DPQBLdJiCwJnsfJ3w7Pvnkk7r3Xbp04f3337ft\nd+TIEcAqCu9MA52cnMwrr7zid5kUxR+kpiRTUl5BqthnAk2Vk3WuS8srWhRs4TTpuPoZLqvZz+Jv\nW1GRkFTXT0M+A0t0rwAURfEa59N5qelie7zUdK57n3pov72fDZoNtsgcmEZh9ii+mjuBwgEnePTh\nm5mzew1pKUka8hkkon8FoCiKVzgn3OcKbuK+qqfrmYGOmUQeq7ae6pMT4sg6XAStayHF5lnSXbBF\nUZ61OjhUbPXpPBF+/H9w7rlkPvswmaec4vffSbHH16LwuSLyuYgUichSEbGN1RKRcSKyTUR2iIiW\n8lGUMCdzYBo5v/ktba59qs6Hdiy5O48l/IzltcMdT+fpZC54FIrPgBMN4nfcBVvY+Qs+fxLG9IRV\nq0An/6Aixoet3SIyBlhtjKkWkT8AGGPub9AnHvgCuAIoBj4CrjfGfNrc9YcMGWIa5sj57LPP6Nu3\nb4tljjRi7fdVIpDKSrjnSmi9HjrGcSypG4/VTGfhkaEn9xA4zTiP97M3GbVPg181mBIarhRGPxgQ\nf160ISIbjDFDPOnrkwnIGLPS5eMHgF22s6HADmPMTodwrwCTgGYVgKIo4U3dhrGO95IqJ7js/bdY\n3P9yKhJaAyd3+ILDtOTOL/B9af3PzpWC07nsjCwCVQJ+xJ9O4NsAu4roaYCryi92tCmKEsE4N4yV\nlFdggBKTyEuDJtRN/k4qqmrIffFdGDECDtXYX6yhv6CpyCLFbzSrAETkbRHZYvMzyaXPr4Fq4CVf\nBRKRGSKyXkTW79+/39fL+Z3y8nKefvrpFp37xBNPcOzYMT9LpCihwS6dgzuDcmmrdtCqFbS6FOLr\nKwhbf0EA0rgojWlWARhjLjfG9LP5eQ1ARG4FrgJuNPYOhRLgdJfPPRxt7sZbYIwZYowZ0rVrV69+\nGTsKdhYwZtEY0hemM2bRGAp2Fvh0PVUAimLhzQ7d1E5trGyiucth0pPNb850F0HkYxoXpT4++QBE\nZBxwHzDSGONuZvsIOFtEemNN/NOBG3wZ11MKdhaQ814OlTVWXp6yo2XkvJcDwIQzJ7TomtnZ2Xz5\n5ZcMGDCAK664glNPPZW8vDyOHz/ONddcw29/+1uOHj3K1KlTKS4upqamhv/93/9l7969lJaWctll\nl9GlSxfWrFnjr19TUUKCc8NYQ4T6K4FGm7k82Zw5+sH6PgDwSxoXpT6++gCeBNoDb4nIxyLyLICI\npIrIGwDGmGrg58AK4DMgzxiz1cdxPWLexnl1k7+TyppK5m1seSbDuXPnctZZZ/Hxxx9zxRVXsH37\ndv773//y8ccfs2HDBt555x3+85//kJqayubNm9myZQvjxo1j5syZpKamsmbNGp38lajALp1DckI8\nNw7r6Xv+/gClcVHq42sU0A/ctJcC410+vwG84ctYLWHP0T1etXvLypUrWblyJQMHDgSstA/bt29n\nxIgR/OpXv+L+++/nqquuYsSIEX4ZT1HCCbt0DvVCPn0lQGlclJNE9U7gbm27UXa0zLbdHxhjmD17\nNj/5yU8aHdu4cSNvvPEGv/nNbxg9ejQPPqhLVyX6yByYpqkaIpiozgU0a9AskuKT6rUlxScxa9Cs\nFl+zffv2fP/99wCMHTuW559/vi7hW0lJSV2xmDZt2nDTTTeRlZXFxo0bG52rKLFEs+UflZAQ1SsA\np6N33sZ57Dm6h25tuzFr0KwWO4ABOnfuTEZGBv369ePKK6/khhtu4KKLLgKgXbt2vPjii+zYsYOs\nrCzi4uJISEjgmWeeAWDGjBmMGzeuzhegKLGAu/KPgK4eQoxPqSACjaaCiL3fV4k+Muauto0WSktJ\npjB7VAgkim68SQUR1SYgRVFCj7v9AlrpK/SoAlAUJaC4q+illb5CjyoARVECirv9AlrpK/REpBPY\nGIOIuwqi0UM4+2cUxVMCvl9AaTERpwCSkpI4ePAgnTt3jmolYIzh4MGDJCUlNd9ZUcIc3S8QnkSc\nAujRowfFxcWEY6ZQf5OUlESPHpr8SlGUwBBxCiAhIYHevXuHWgxFUZSIR53AiqIoMYoqAEVRlBhF\nFYCiKEqMEtapIERkP/B1C0/vAhzwozj+QuXyDpXLO1Qu74hGuc4wxnhUTjGsFYAviMh6T/NhBBOV\nyztULu9Qubwj1uVSE5CiKEqMogpAURQlRolmBbAg1AK4QeXyDpXLO1Qu74hpuaLWB6AoiqI0TTSv\nABRFUZQmiGgFICLXichWEakVEbcecxEZJyLbRGSHiGS7tPcWkQ8d7a+KSKKf5DpFRN4Ske2O1042\nfS4TkY9dfipFJNNx7AUR+crl2IBgyeXoV+My9jKX9lDerwEi8r7j710kItNcjvn1frn7vrgcb+34\n/Xc47kcvl2OzHe3bRGSsL3K0QK5fisinjvuzSkTOcDlm+zcNkly3ish+l/HvcDl2i+Pvvl1Ebgmy\nXI+7yPSFiJS7HAvI/RKR50Vkn4hscXNcRGS+Q+YiERnkcsz/98oYE7E/QF+gD7AWGOKmTzzwJXAm\nkAhsBs51HMsDpjvePwvc6Se5HgOyHe+zgT800/8U4FugjePzC8CUANwvj+QCjrhpD9n9An4InO14\nnwqUASn+vl9NfV9c+vwMeNbxfjrwquP9uY7+rYHejuvEB1Guy1y+Q3c65WrqbxokuW4FnrQ59xRg\np+O1k+N9p2DJ1aD/3cDzQbhflwCDgC1ujo8H3gQEGAZ8GMh7FdErAGPMZ8aYbc10GwrsMMbsNMac\nAF4BJomIAKOARY5+C4FMP4k2yXE9T687BXjTGHPMT+O7w1u56gj1/TLGfGGM2e54XwrsAzza7OIl\ntt+XJuRdBIx23J9JwCvGmOPGmK+AHY7rBUUuY8wal+/QB0AwUsl6cr/cMRZ4yxjzrTHmO+AtYFyI\n5LoeeNlPY7vFGPMO1sOeOyYB/zAWHwApItKdAN2riFYAHpIG7Hb5XOxo6wyUG2OqG7T7g9OMMWWO\n93uA05rpP53GX77fOZaAj4tI6yDLlSQi60XkA6dZijC6XyIyFOup7kuXZn/dL3ffF9s+jvtxCOv+\neHJuIOVy5XasJ0kndn/TYMp1rePvs0hETvfy3EDKhcNU1htY7dIcqPvVHO7kDsi9Cvt00CLyNtDN\n5tCvjTGvBVseJ03J5frBGGNExG2olUO79wdWuDTPxpoIE7HCwe4HHg6iXGcYY0pE5ExgtYh8gjXJ\ntRg/369/ArcYY2odzS2+X9GIiNwEDAFGujQ3+psaY760v4LfWQ68bIw5LiI/wVo9jQrS2J4wHVhk\njKlxaQvl/QoaYa8AjDGX+3iJEuB0l889HG0HsZZXrRxPcc52n+USkb0i0t0YU+aYsPY1campwFJj\nTJXLtZ1Pw8dF5O/AvSxXLZkAAAIvSURBVMGUyxhT4njdKSJrgYHAYkJ8v0SkA1CApfw/cLl2i++X\nDe6+L3Z9ikWkFdAR6/vkybmBlAsRuRxLqY40xhx3trv5m/pjQmtWLmPMQZePz2H5fJznXtrg3LV+\nkMkjuVyYDtzl2hDA+9Uc7uQOyL2KBRPQR8DZYkWwJGL9sZcZy7OyBsv+DnAL4K8VxTLH9Ty5biPb\no2MSdNrdMwHbiIFAyCUinZwmFBHpAmQAn4b6fjn+dkux7KOLGhzz5/2y/b40Ie8UYLXj/iwDposV\nJdQbOBv4rw+yeCWXiAwE/gJMNMbsc2m3/ZsGUa7uLh8nAp853q8Axjjk6wSMof5KOKByOWQ7B8up\n+r5LWyDvV3MsA252RAMNAw45HnACc6/86eEO9g9wDZYt7DiwF1jhaE8F3nDpNx74AkuD/9ql/Uys\nf9AdwL+B1n6SqzOwCtgOvA2c4mgfAjzn0q8XlmaPa3D+auATrInsRaBdsOQCLnaMvdnxens43C/g\nJqAK+NjlZ0Ag7pfd9wXLpDTR8T7J8fvvcNyPM13O/bXjvG3AlX7+vjcn19uO/wPn/VnW3N80SHLN\nAbY6xl8DnONy7m2O+7gD+J9gyuX4nAPMbXBewO4X1sNemeO7XIzlq/kp8FPHcQGecsj8CS7RjYG4\nV7oTWFEUJUaJBROQoiiKYoMqAEVRlBhFFYCiKEqMogpAURQlRlEFoCiKEqOoAlAURYlRVAEoiqLE\nKKoAFEVRYpT/D2CH0JaArPSKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f851b9ce0f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# %load marsland/mlp.py\n",
    "# The sinewave regression example\n",
    "import numpy as np\n",
    "\n",
    "class mlp:\n",
    "    \"\"\" A Multi-Layer Perceptron\"\"\"\n",
    "    \n",
    "    def __init__(self,inputs,targets,nhidden,beta=1,momentum=0.9,outtype='logistic'):\n",
    "        \"\"\" Constructor \"\"\"\n",
    "        # Set up network size\n",
    "        self.nin = np.shape(inputs)[1]\n",
    "        self.nout = np.shape(targets)[1]\n",
    "        self.ndata = np.shape(inputs)[0]\n",
    "        self.nhidden = nhidden\n",
    "\n",
    "        self.beta = beta\n",
    "        self.momentum = momentum\n",
    "        self.outtype = outtype\n",
    "    \n",
    "        # Initialise network\n",
    "        self.weights1 = (np.random.rand(self.nin+1,self.nhidden)-0.5)*2/np.sqrt(self.nin)\n",
    "        self.weights2 = (np.random.rand(self.nhidden+1,self.nout)-0.5)*2/np.sqrt(self.nhidden)\n",
    "\n",
    "    def earlystopping(self,inputs,targets,valid,validtargets,eta,niterations=100):\n",
    "    \n",
    "        valid = np.concatenate((valid,-np.ones((np.shape(valid)[0],1))),axis=1)\n",
    "        \n",
    "        old_val_error1 = 100002\n",
    "        old_val_error2 = 100001\n",
    "        new_val_error = 100000\n",
    "        \n",
    "        count = 0\n",
    "        while (((old_val_error1 - new_val_error) > 0.001) or ((old_val_error2 - old_val_error1)>0.001)):\n",
    "            count+=1\n",
    "            print(count)\n",
    "            self.mlptrain(inputs,targets,eta,niterations)\n",
    "            old_val_error2 = old_val_error1\n",
    "            old_val_error1 = new_val_error\n",
    "            validout = self.mlpfwd(valid, internal=True)\n",
    "            new_val_error = 0.5*np.sum((validtargets-validout)**2)\n",
    "            \n",
    "        print(\"Stopped\", new_val_error,old_val_error1, old_val_error2)\n",
    "        return new_val_error\n",
    "\n",
    "    def mlptrain(self,inputs,targets,eta,niterations):\n",
    "        \"\"\" Train the thing \"\"\"    \n",
    "        # Add the inputs that match the bias node\n",
    "        inputs = np.concatenate((inputs,-np.ones((np.shape(inputs)[0],1))),axis=1)\n",
    "        change = range(self.ndata)\n",
    "    \n",
    "        updatew1 = np.zeros((np.shape(self.weights1)))\n",
    "        updatew2 = np.zeros((np.shape(self.weights2)))\n",
    "            \n",
    "        for n in range(niterations):\n",
    "    \n",
    "            self.outputs = self.mlpfwd(inputs, internal=True)\n",
    "\n",
    "            error = 0.5*np.sum((self.outputs-targets)**2)\n",
    "            if (np.mod(n,100)==0):\n",
    "                print(\"Iteration: \",n, \" Error: \",error)\n",
    "\n",
    "            # Different types of output neurons\n",
    "            if self.outtype == 'linear':\n",
    "                deltao = (self.outputs-targets)/self.ndata\n",
    "            elif self.outtype == 'logistic':\n",
    "                deltao = self.beta*(self.outputs-targets)*self.outputs*(1.0-self.outputs)\n",
    "            elif self.outtype == 'softmax':\n",
    "                deltao = (self.outputs-targets)*(self.outputs*(-self.outputs)+self.outputs)/self.ndata \n",
    "            else:\n",
    "                print(\"error\")\n",
    "            \n",
    "            deltah = self.hidden*self.beta*(1.0-self.hidden)*(np.dot(deltao,np.transpose(self.weights2)))\n",
    "                      \n",
    "            updatew1 = eta*(np.dot(np.transpose(inputs),deltah[:,:-1])) + self.momentum*updatew1\n",
    "            updatew2 = eta*(np.dot(np.transpose(self.hidden),deltao)) + self.momentum*updatew2\n",
    "            self.weights1 -= updatew1\n",
    "            self.weights2 -= updatew2\n",
    "                \n",
    "            # Randomise order of inputs (not necessary for matrix-based calculation)\n",
    "            #np.random.shuffle(change)\n",
    "            #inputs = inputs[change,:]\n",
    "            #targets = targets[change,:]\n",
    "            \n",
    "    def mlpfwd(self,inputs, internal=False):\n",
    "        \"\"\" Run the network forward \"\"\"\n",
    "\n",
    "        if not internal:\n",
    "            inputs = np.concatenate((inputs,-np.ones((np.shape(inputs)[0],1))),axis=1)\n",
    "\n",
    "        self.hidden = np.dot(inputs,self.weights1)\n",
    "        self.hidden = 1.0/(1.0+np.exp(-self.beta*self.hidden))\n",
    "        self.hidden = np.concatenate((self.hidden,-np.ones((np.shape(inputs)[0],1))),axis=1)\n",
    "\n",
    "        outputs = np.dot(self.hidden,self.weights2)\n",
    "\n",
    "        # Different types of output neurons\n",
    "        if self.outtype == 'linear':\n",
    "            return outputs\n",
    "        elif self.outtype == 'logistic':\n",
    "            return 1.0/(1.0+np.exp(-self.beta*outputs))\n",
    "        elif self.outtype == 'softmax':\n",
    "            normalisers = np.sum(np.exp(outputs),axis=1)*np.ones((1,np.shape(outputs)[0]))\n",
    "            return np.transpose(np.transpose(np.exp(outputs))/normalisers)\n",
    "        else:\n",
    "            print(\"error\")\n",
    "\n",
    "    def confmat(self,inputs,targets):\n",
    "        \"\"\"Confusion matrix\"\"\"\n",
    "\n",
    "        # Add the inputs that match the bias node\n",
    "        inputs = np.concatenate((inputs,-np.ones((np.shape(inputs)[0],1))),axis=1)\n",
    "        outputs = self.mlpfwd(inputs)\n",
    "        \n",
    "        nclasses = np.shape(targets)[1]\n",
    "\n",
    "        if nclasses==1:\n",
    "            nclasses = 2\n",
    "            outputs = np.where(outputs>0.5,1,0)\n",
    "        else:\n",
    "            # 1-of-N encoding\n",
    "            outputs = np.argmax(outputs,1)\n",
    "            targets = np.argmax(targets,1)\n",
    "\n",
    "        cm = np.zeros((nclasses,nclasses))\n",
    "        for i in range(nclasses):\n",
    "            for j in range(nclasses):\n",
    "                cm[i,j] = np.sum(np.where(outputs==i,1,0)*np.where(targets==j,1,0))\n",
    "\n",
    "        print(\"Confusion matrix is:\")\n",
    "        print(cm)\n",
    "        print(\"Percentage Correct: \",np.trace(cm)/np.sum(cm)*100)\n",
    "\n",
    "\n",
    "import pylab as pl\n",
    "\n",
    "# Set up the data\n",
    "ndata = 120\n",
    "x = np.linspace(0,1,ndata).reshape((ndata,1))\n",
    "w = np.sin(2*np.pi*x) + np.cos(4*np.pi*x)\n",
    "t = np.sin(2*np.pi*x) + np.cos(4*np.pi*x) + np.random.randn(ndata).reshape((ndata,1))*0.1\n",
    "x = (x-0.5)*2\n",
    "\n",
    "\n",
    "\n",
    "# Split into training, testing, and validation sets\n",
    "train = x[0::2,:]\n",
    "test = x[1::4,:]\n",
    "valid = x[3::4,:]\n",
    "traintarget = t[0::2,:]\n",
    "testtarget = t[1::4,:]\n",
    "validtarget = t[3::4,:]\n",
    "\n",
    "\n",
    "# Plot the data\n",
    "pl.plot(x, w, '-r', label='orig')\n",
    "pl.plot(train, traintarget ,'o', label='train')\n",
    "# pl.plot(test, testtarget ,'o', label='test')\n",
    "pl.plot(valid, validtarget ,'o', label='valid')\n",
    "\n",
    "\n",
    "\n",
    "# Perform basic training with a small MLP\n",
    "net = mlp(train,traintarget,3,outtype='linear')\n",
    "net.mlptrain(train,traintarget,0.05,ndata*10)\n",
    "\n",
    "# Use early stopping\n",
    "# net.earlystopping(train,traintarget,valid,validtarget,0.25)\n",
    "\n",
    "o = net.mlpfwd(test)\n",
    "pl.plot(test, o ,'o', label='test')\n",
    "\n",
    "pl.legend(loc='lower left')\n",
    "# pl.xlabel('x')\n",
    "# pl.ylabel('t')\n",
    "\n",
    "# Test out different sizes of network\n",
    "#count = 0\n",
    "#out = zeros((10,7))\n",
    "#for nnodes in [1,2,3,5,10,25,50]:\n",
    "#    for i in range(10):\n",
    "#        net = mlp.mlp(train,traintarget,nnodes,outtype='linear')\n",
    "#        out[i,count] = net.earlystopping(train,traintarget,valid,validtarget,0.25)\n",
    "#    count += 1\n",
    "#    \n",
    "#test = concatenate((test,-ones((shape(test)[0],1))),axis=1)\n",
    "#outputs = net.mlpfwd(test)\n",
    "#print 0.5*sum((outputs-testtarget)**2)\n",
    "#\n",
    "#print out\n",
    "#print out.mean(axis=0)\n",
    "#print out.var(axis=0)\n",
    "#print out.max(axis=0)\n",
    "#print out.min(axis=0)\n",
    "\n",
    "pl.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}