diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f7a9ad67..fa2cc64d 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -12,7 +12,9 @@ Because GitHub's [graph of contributors](http://github.com/secondmind-labs/GPflu [Felix Leibfried](https://github.com/fleibfried), [John A. McLeod](https://github.com/johnamcleod), [Hugh Salimbeni](https://github.com/hughsalimbeni), -[Marcin B. Tomczak](https://github.com/marctom) +[Marcin B. Tomczak](https://github.com/marctom), +[Sebastian Popescu](https://github.com/SebastianPopescu), +[Alessandro Vullo](https://github.com/avullo), Feel free to add yourself when you first contribute to GPflux's code, tests, or documentation! \ No newline at end of file diff --git a/docs/notebooks/deep_cde.ipynb b/docs/notebooks/deep_cde.ipynb index eb069dbb..ff48c7a2 100644 --- a/docs/notebooks/deep_cde.ipynb +++ b/docs/notebooks/deep_cde.ipynb @@ -2,18 +2,22 @@ "cells": [ { "cell_type": "markdown", + "id": "61537fc9", + "metadata": {}, "source": [ "# Deep Gaussian processes with Latent Variables\n", "\n", "In this notebook, we explore the use of Deep Gaussian processes and Latent Variables to model a dataset with heteroscedastic noise. The model can be seen as a deep GP version of or as doing variational inference in models from . We start by fitting a single layer GP model to show that it doesn't result in a satisfactory fit for the noise.\n", "\n", "This notebook is inspired by [prof. Neil Lawrence's Deep Gaussian process talk](https://inverseprobability.com/talks/notes/deep-gps.html), which we highly recommend watching." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, + "id": "1d1e7e03", + "metadata": {}, + "outputs": [], "source": [ "import tensorflow as tf\n", "import gpflow\n", @@ -24,76 +28,73 @@ "\n", "import tensorflow_probability as tfp\n", "from sklearn.neighbors import KernelDensity\n" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "5b0bccd7", + "metadata": { + "lines_to_next_cell": 2 + }, "source": [ "## Load data\n", "\n", "The data comes from a motorcycle accident simulation [1] and shows some interesting behaviour. In particular the heteroscedastic nature of the noise." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, + "id": "08bbc183", + "metadata": {}, + "outputs": [], "source": [ "def motorcycle_data():\n", - " \"\"\" Return inputs and outputs for the motorcycle dataset. We normalise the outputs. \"\"\"\n", + " \"\"\"Return inputs and outputs for the motorcycle dataset. We normalise the outputs.\"\"\"\n", " import pandas as pd\n", + "\n", " df = pd.read_csv(\"./data/motor.csv\", index_col=0)\n", " X, Y = df[\"times\"].values.reshape(-1, 1), df[\"accel\"].values.reshape(-1, 1)\n", " Y = (Y - Y.mean()) / Y.std()\n", " X /= X.max()\n", " return X, Y" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, + "id": "3d0e2bdf", + "metadata": {}, + "outputs": [], "source": [ "X, Y = motorcycle_data()\n", "num_data, d_xim = X.shape\n", "\n", "X_MARGIN, Y_MARGIN = 0.1, 0.5\n", "fig, ax = plt.subplots()\n", - "ax.scatter(X, Y, marker='x', color='k');\n", - "ax.set_ylim(Y.min() - Y_MARGIN, Y.max() + Y_MARGIN);\n", - "ax.set_xlim(X.min() - X_MARGIN, X.max() + X_MARGIN);" - ], - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAb90lEQVR4nO3df4wkaVkH8O+zc4IYcWbirjmzcAwkYLicN33MhL7FsHt3EPdyMXPooQEFJUE3YG67Zlr/gFwy06t/GXMzN6MkuokEMAHxB+j6KwjKQSS3s8zBAMcp5EA4wQUWuG4lRmW7Hv/ort7qmqrqqq6qt+rt+n6Sym7/mKq3uquffvt5f4mqgoiI7HWs7AIQEVE2DORERJZjICcishwDORGR5RjIiYgsd1MZBz1+/LguLS2VcWgiIms9/vjj31bVE8H7SwnkS0tLODg4KOPQRETWEpGvht3P1AoRkeUYyImILMdATkRkOQZyIiLLMZATEVmOgZyIyHIM5ERElmMgJyKyHAM5EZHlGMiJiCzHQE5EZDkGciIiyzGQExFZLnMgF5Hni8hHReRJEfm8iDh5FIyIiJLJYxrb6wB+Q1U/JSLPBfC4iHxYVZ/MYd9ERDRB5hq5ql5V1U8N//9fAP4FwMms+yUqm6rG3i5rX0RBuebIRWQJwB0A9kMeOyciByJycO3atTwPS5S7TqeDjY2NUcBVVWxsbKDT6ZS6L6IwuQVyEflhAH8BYF1V/zP4uKpeVNVVVV09ceLISkVElaGq6Ha72N3dHQXgjY0N7O7uotvtpqpN57kvokiqmnkD8AMAPgSgneT5KysrSlRlruuq4zgKYLQ5jqOu65a6L6o3AAcaElNFM9YIREQAvBvAd1V1PcnfrK6uKtfspKpTVRw7duNHq+u6GFzu5e6L6ktEHlfV1eD9eaRWfgrAGwHcIyKHw+2+HPZLVBodpkD8/HnuPPeVtTJFlLn7oar+MwBWLWhmeIF3d3cXjuNgZ2dndBsAdnZ2Etemg/uan5/HpUuXRvva3t5Gu93GwsICGz9pahzZSRQgIlhYWBgFcRHBzs4OHMfBwsJCqpSIf1/b29vo9Xo4PDxEo9HA/Pw82u02Gz4ps8w58mkwR042UNWxoB28Pc2+/DV0j/8LgyhOVI6cgZzIMDZ80rSKbOwkooTybEQl8jCQExkSbPh0XReO44wNFiKaRh6TZhFRAlGNqABSN6IS+TFHTmRYno2oVC/MkRNVRDBoM4hTVgzkRESWYyAnIrIcAzlRCsE2JfY0oSpgICdKiAtEUFUxkBMloFwggiqM/ciJEvD3+d7d3R3NlcJ5UqgK2I+cKAXOk0JlYj9yoow4TwpVFQM5UQKcJ4WqjDlyqo0sQ+Oj5klR1bF5Uooabs9h/RSHgZxqodPpoNvtjoKwV8NOs8Rap9MZC6AXLlwAAGxtbQG4UWufn58fPebdnyXo5lF2mm1MrdDMy7ProL/m3e12sbe3d2Sfly5dguu6o+dl6WvObo+UiKoa31ZWVpTIJNd11XEcBTDaHMdR13Vz3Wej0Rjbt/d4lmMVUXayE4ADDYmp7H5ItaEFdB0M7rPf748WVPbk0de8iLKTfdj9kGpNC+g6GLbPdruN7e3tsfvyCOLs9kixwqrpRW9MrZBJYSmOrCmPuH166RXkkAYpouxkL0SkVthrhWZeEUushe1ze3sbH/vYx3B4eDi632uYBKarmXN5OEqCOXKqDS2gL3ZwH1tbW+j1erl3FSyi7GSfqBw5AznRUF7BkkGXisLGTqIYec41nmRNzmAFqowKFc0OBnKqPTU86IYLVFDe2NhJtWdyrnH/lwaAsQZRx3GYhqGpMEdONKSGBt34a/weLlBBSTBHThTDC65+RQ268f8C8JgM4szPzx4Gcqo9fw3ZxFzjJr80gsLy8+vr62P5eQZ2++QSyEXknSLyLRF5Io/9EZkUNejGcZzcB92Y/tIIHjvYqHvq1Cns7e3hmWeeGY0SZMOrffJq7HwXgN8H8J6c9kdkVHCucS+Y553uKHOkZlSjrh8bXu2UW2OniCwB+BtVvW3Sc9nYSXVX5qChYKNuq9XC3t7e6DYbXqur9MZOETknIgcicnDt2jVThyWqpCSDhooQlp8PYhC3j7FArqoXVXVVVVdPnDhh6rBENBSWnw/WxgFOkWsj9lohqolgft6v2WwabXilfHFkJ1GNBBt1FxcX0Wq18Mgjj3CKXIvl0tgpIu8DcBeA4wC+CWBLVf8o6vls7CSqDs7WaI+oxs5cauSq+vo89kNE5pXV8Er5YY6ciMhyDOREOeEcJlQWBnKiHMzKHOP8MrITAzlRRqYXpijKrHwZ1RG7H9LMK7pXhsmFKYrCBS/sxoUlaKZ1Oh10u93cV7UPY2phiqKkWfBi0pejyS6NprtPltlds/S5VohMM5nyKHOO8bwkXfBiUgrGZIrGdDqosuknbw5ik9vKyooSmeC6rjqOowBGm+M46rpuIcfw9h28bYMkr9Wkc+33+8ZeC9OvexXeZwAHGhJTGchp5rmuOxac0nzggs+N+tutra2xD7P3Id/a2pq63CalCVKTAr6JL8+kZbH9eEFRgZw5cpppmiLvG5Q2v66WD3VPc746oT1g0uN5MnmsMo7nF5UjZ42cZlaWn8JV+BmdVNJfDXntizXy6tXIGchpJnkfLC/l0e/3R/cnTXmU/aFNwnRKhznyaubI2Y+cZo4/RdDpdOC6Ltrt9ihFkLRvt9eLw5+WqVK/cFXzfb8nrTl67NgxY2uSml7/tMz1VicKi+5Fb6yRU1HyrDXlUSPPM+1RVBmnPW6W2ybLYvvx/FDX1ErUi17mmzFJ2rJV+VzKkFcAzvqFYCrtkaVXTnA/WW4Xidf4QFQgn+kBQV7nfdd1AdzowXDmzJnMnfq9v426nfZ5wTInLVtlByiUKOnAlkn7CPsZ7ThOop/RqmYGI3n79fNf80nFXUeqOva4t5m6zniNJxAW3YveTNTI/TWoRqMx1ghz/PhxIzWttDWytLXAKjS+VFGe6YYsNcGi0x5x13ij0dDNzc3U+wleR81mU8+fP6+tVksBaKvV0larpc1m08h1xmt8HOqYWun3+9poNMY+SMvLy6MLPewDNqlLmuu6Yxd12O2o5wVb9/379Ad77+/iPvzebf+H1/98b//B58+6qn3w80p7RNnc3Bxdy9777r+d9HhhXzpe0A7+3/+4idezyC/EYKo1KvVaFVGBfOYHBLmui7m5ubH7Wq0WdnZ2xu73fopGDYDwekLMz8+j2+0CAPb29kaPnzx5Eg888AAWFhbQ6/Wwvb2NjY0N7O/vY39/f/S8RqOBtbW10XNe8YpXABisYr64uIitrS2sr6+P7dsNDDjwl6XX6+Hhhx/GTTfd6IB0+vRp3HHHHUYmiqoikxNlxfGOO81gpDS8XjlZj6N6dKALgCPn4H/cVE+NsLJlPXbwc7S9vY12uz26XcXPSy0HBIV9kwPQ8+fP6/nz54/c59U44oYke7Udr2bvbd79wX+Dz/M/dvvtt4/d/+CDDx4pV7A8YWXxUkXeliR1ZHvjUZUb5rzjme7j7L8G0u4/rtYb3HfYdVmkImrkYZ+j4L9VTN2gbqmVsDcqbAs+FvVzMepLIeyLIG6LK0uwHFG5yLiyBC/GsAvf9nlBbCm/yV4rWQJd3JdOq9U6Urnwp1lszpHHfY6qGMRVaxjIVcPzh8Eacr/fT1yTiaqZ+POIUY9P+iKI+qB4/w9++KPK4uXMNzc3Q8+rajnktGwrf9G/DPJ6PcK+dPwViWazmei6LEKRX4hRn6OqXUeeWgZy1UEw9/9EDAbbuJqrX1wg9mot3kUfFcjDGl/jtqgG2Ek1ievXr8fW0IruTVE028uft7wCXdh15t+3t3n7Nvl6F/GFyBq5RYFcVccuQC+w+oNqsOtWmhx5VH4t+Dx/bSYuRx6sXYedy6TcXjDHF5Ujt6EGEsX28uetyJp/2e0NRZi1HPlcGa2yFy9e7Jw7d87Y8UQEIoLLly+j0WjgkUcewbFjx3D16lX0ej2sra3hnnvuwdmzZ9HtdrGwsIC777577O+9v11eXkaj0cAHPvAB9Hq90e1ut4vXvOY1kY8vLi7iOc95zmgfy8vLWFpaGuvR8vTTT+N73/ve6Hav18PZs2fHWufTlMXrtRA8L1Ud9ajxdLvdI8eqKtvLX4Tgeec9p0lR+y5Lks9RMA5UwYULF652Op2LRx4Ii+5Fb2XOtRL8hk3T33pSn9NJ/3r/96d4Wq2WXr9+faw24E+NxKV6ov4NpmP8t23LMQfZXn6TbKlJl1nOJJ/ZKgFnPxwI1ib8fVPDHg97bNp//f9fXFwc6+u7trYGAFhbW8Pc3NzEWdWijnHhwoVRH2rgRu3V6xNb6RncErC9/KZUpS/9JGWXM8ln1gph0b3ojbMfDmT5dRC1v6S1VVtqa1FsL3+RbPnVUnQ5Z/EaQZ0bO+ukzj06bPjgmiqjLddBUeW0ZaxBWgzkNVLHHh3TfHBNB/4yVvOx4TrIu5y2/CKZBgN5TdhSE8vTNB/cqi2Rlvf7Y8t1UFQ5bTn/tBjIa2CWayKTpPnglvU6mQoutlwHJnLkNvwiSYOBvCbiapq2dbVKK+6DG9awXEaNzURwcd34EZlVUtQvI9bIp9gA3AvgCwCeAvC2Sc9nIC9WWNDyPjCbm5ujka3e7Sp+wNOK++BGBYuo+WjKKGNeqjKkPo282yps+UUyjcICOYA5AF8C8CIAzwLwGQC3xv0NA7k5W1tbodMD3HzzzZUfjpxU3Ac3bKa+4NBsEzU2E8FllgNYWuy1kj6QnwLwId/ttwN4e9zfMJCbERXQgtssfMgnpZSCNeEk89GYLGNeZjWlMA0buqOmFRXIM68QJCKvBXCvqv7q8PYbATRV9cGovzG5QlDdqR5dpSbINbjSS5FUdew8/LdVx1eY2dzcRK/XMz6iMK6MeR4j79V0qBqiVgg6FvbkggpwTkQOROTg2rVrpg5be2Erygf5VyivumA5/bejJnfygrSft7SXf0j2zs5O4cPCi56AKuxcq/T+xr1/lEFYNT3NBqZWKs11j87BDtgxZWfQtIN+6pI3rvq5zmre2iQUOGnWJwG8WEReCODrAF4H4Bdz2C9lpMPa2d7eHprNJprNJi5fvowrV67gla98JU6fPo2FhQWcOXOm8hNOqSq63e4oRbSzszNKGTmOE5miqNMkW1U+12nfP0ooLLqn3QDcB+CLGPReeWjS81kjNyesO5q3RJdt/cizNOTNYsNXlKqeKxtis0NRjZ3TYGOnWWqggc0UZUOe1fj+ZVN6YyeVZ1ZWeNGKN+RRPL5/xWEgJyt4QcDLqbquC8dxsLu7y2BgAb5/xardCkFkpyo35NFkfP+KxRw5WWWW8v11xPcvG+bIKZHgF3vVfvLOSr7fr+qveZ5m8f2rAgZyGul0OmP5Si+vWaXFemcNX3PKAwN5TUyq9alvwIYXWLzGqW63O9O1xLLwNafchHUuL3rjgCCzkg6N5oAN8/iaUxqIGBDEGvmM00Ctz3XdsVqf67qj54ZNsOX1MKBi8DWnPDCQzzgvUHh9dufm5kZ9ebe3t9Fut0f5WOWADeP4mlMuwqrpRW9MrZjX7/fHfr7716x0HOfI7arNnDeLqj5bIVUPCpz9kCpOVdFut8fum5ubA4CxARqzMGBDLeqnPCuvOZWPA4JmnAaGRm9vb4+COAD0+/2xSYxsCoRBnU4H3W7X+Ko/Wdn8mpNZHBBUU/5an5cT92u322P5WFsHbKjFXflsfc2pQsLyLUVvzJGblyQnbjt25aNZB85HTramHtJQ5XzXNLuiUits7KyRTqczln/1GtdmJdB5X0x+GxsbM3WORGGYI68Zm/KxwV+Lcb8evSDO+a6pjlgjp0pKmwZiVz6qMwZyqhzV6VZcn/XUEVEUBnKqHH9tend3dxTQ/bXtuL+Nu000i9hrhSqLPVCIxnFAEB2RpjHRtKgeKFUqI1FVMJDXVJVXpmEPFKJ0mCOvoWkbE01hDxSidJgjryl/rdeTpDHRpOAXStlfMERli8qRM5DXmC2NiQzoRANs7KQxtjQmTsrlV7nBlsgUBvIasqUx0Z/LD5uadmtrq7INtkQmsbGzhmxpTIwbGOTNrV7VBlsik5gjrzFbcs9RuXwbGmyJ8sQcOR1hw3D2uFy+v8buYRCnOmIgr7kqNxZOyuW7rmtFgy1R0RjIa6zKozuBo7l8YFDjdhwH8/PzVjTYEpmQqbFTRH4eQAfASwG8XFWZ+LZE1Ud3erypaS9cuDCan9wL6qdOnUKz2ax0gy2REWELeSbdMAjgPwHgUQCrSf+Oiy9Xgy2LFfvL6ZXPu91qtcbKW7WyE+UJRS6+LCKPAvhNTVgjZ6+V6lCLRneyhwrVXem9VkTknIgciMjBtWvXTB2WYnjB0a+q+WX2UCGKNjGQi8hHROSJkO3+NAdS1YuquqqqqydOnJi+xJQLfw3XhsZCm750iEyb2Nipqq82URAyy5bRncDRLx1/wyzAmjkRh+jXmC2LFdv0pUNUhkyNnSLyswB+D8AJAF0Ah6p6dtLfsbGTpuH/0gm7TTTroho7M9XIVfWDAD6YZR9ESdkwpQBRGTiyk4jIcgzklFowHceeI0TlYiCnVKo+PwtRHTGQU2I6YcUe1syJysHuh5RY3Io9Vey2SFQXXCGIUrNlfhaiWVP6XCs0GzhUnqh6GMgpMdvmZyGqC+bIKTEOlSeqJubIKbWih8pzKD5ROObIKTdFDpVnP3Wi9BjIqTLYT51oOsyR0xFlpTbYT51oOsyR05hOpzNarV5ERrXihYUFY+kN9lMnCsccOU1UhdQG+6kTTUFVjW8rKytK1eS6rjqOowBGm+M46rruVPuKux13bO+YwdtEdQbgQENiKlMrdITmkNqYNkVThdQOUVUVskIQzR4vcPptbGykamxUX4oGwNhiyY7jxDae2rKOKFGlhFXTi96YWqmmPFMbeaZoiGgATK1QEnmmNrTg3ifKEaBUM0ytUCJ5pTa8LwA/L0Xj7TdrOZlLJxpgIKcjsg7B94Lq7u4ums0mms0mgMEgH+8X4OLi4tQBVzPk4IlmEQM55c6bJbHVagEA9vb20Gq10Gq1sL+/j/39/UwBlyNAicYxR06F8a4tr7bsySvgFp2DJ6oajuykqQW/7JN++YvIWO3ZExfEkx4rKgdfRsWEqGwM5BQr67SyaQJu0mP5c/BcqYiIgZxi+BsVp5l7JU3ATXOsqJWKHMfhSkVUT2Gdy4veOCDIHlkH9mxtbY0939vf1tZW5mOlncuFyHbggCCaliZoVNSYwTlxj01zLKK6YmMnTUUT5Lgn5baT9ktPciwiOoqBnCJ5gTUux60Z8+hpjkVE4TggiCJFNSoCGGtUzGNwTtJjEdFRzJHTREly3HnlttPk04nqppAcuYj8roj8q4h8VkQ+KCILWfZH1RQXWFUVrutOzG0HKwz+fHrSYxNRuKw58g8DuE1VbwfwRQBvz14kqqqwRs0777wTJ0+eHOW2+/0+Go3GWG47qjH0rrvuyjTYiIgGMgVyVf0HVb0+vHkZwPOyF4mqKKxRc319HVeuXME3vvENNBoNbG9vo91u4/DwEI1GA/Pz8wAQ2Rja6/VKXeiZaGaEdS6fZgPw1wDeEPP4OQAHAA5uueWWafrCU8nCBuy0Wi1ttVpHBvH0+/3Yv/Oew1WEiJLDtAOCROQjAG4OeeghVf2r4XMeArAK4Od00g7Bxk6baUijpqpibm5u7L6kjaFh9wPMjROFmbqxU1Vfraq3hWxeEH8TgJ8B8EtJgjjZS4fpFD/HcbCysjJ2X1hDZ1hjqOu6OHXq1Nj96+vrWF9fZ56cKI2wanrSDcC9AJ4EcCLN33GuFfv40yNh6ZRGozGWKvFSJHELOi8vL0emaFqtFlMsRAEoYq4VEXkKwLMBfGd412VVfcukv2NqxU7+dTIBjKVE+v0+jh07Nqp9+9fOjFpf8/DwEMvLywAGqwh5ms0mHnvsMaZXiAKiUiscEESpeNdL3Ko/GjFgKKw/urc/TpRFNBknzaLcTJoTxR+go/gDNSfKIsqGgZxSSbKoA1f6ITKLk2ZRap1OZyxV4gVzrybuDQACBhNq+YN18O84URZRdsyRU+78NW1P3GyIUflzIhrHxk4ySnOaDZGIbmBjJxnj1cj9mPMmKg4DOeWKDZhE5rGxk3LFBkwi85gjp0KwAZMof8yRk1HBoM0gTlQcBnIiIssxkBMRWY6BnIjIcgzkRESWYyAnIrIcAzkRkeUYyImILMdATkRkOQZyIiLLMZATEVmOgZyIyHIM5ERElmMgJyKyHAM5EZHlGMiJiCxXysISInINwFcz7uY4gG/nUJyqq8N51uEcAZ7nrCnjPF+gqieCd5YSyPMgIgdhK2XMmjqcZx3OEeB5zpoqnSdTK0RElmMgJyKynM2B/GLZBTCkDudZh3MEeJ6zpjLnaW2OnIiIBmyukRMRERjIiYisV/lALiL3isgXROQpEXlbyOPPFpH3Dx/fF5El86XMJsE5tkXkSRH5rIj8o4i8oIxyZjXpPH3Pe0BEVEQq0bUrrSTnKSK/MHxPPy8i7zVdxjwkuG5vEZGPisinh9fufWWUMwsReaeIfEtEnoh4XERkb/gafFZEXma6jAAAVa3sBmAOwJcAvAjAswB8BsCtgef8OoA/GP7/dQDeX3a5CzjHuwH80PD/b7XtHJOe5/B5zwXwcQCXAayWXe6C3s8XA/g0gMXh7R8ru9wFnedFAG8d/v9WAF8pu9xTnOdpAC8D8ETE4/cB+HsAAuBOAPtllLPqNfKXA3hKVb+sqv8H4E8A3B94zv0A3j38/58DeJWIiMEyZjXxHFX1o6r638OblwE8z3AZ85DkvQSA3wbwOwD+x2ThcpTkPH8NwDtU9RkAUNVvGS5jHpKcpwL4keH/5wH8h8Hy5UJVPw7guzFPuR/Ae3TgMoAFEflxM6W7oeqB/CSAf/fd/trwvtDnqOp1AD0AP2qkdPlIco5+b8agBmCbiec5/Fn6fFX9W5MFy1mS9/MlAF4iIp8Qkcsicq+x0uUnyXl2ALxBRL4G4O8AnDdTNKPSfn4LcZPpA9L0ROQNAFYBnCm7LHkTkWMAtgG8qeSimHATBumVuzD4dfVxEflJVe2WWqr8vR7Au1T1YRE5BeCPReQ2VXXLLtisqXqN/OsAnu+7/bzhfaHPEZGbMPgJ9x0jpctHknOEiLwawEMA1lT1fw2VLU+TzvO5AG4D8KiIfAWDfOMlCxs8k7yfXwNwSVW/r6r/BuCLGAR2myQ5zzcD+FMAUNXHAPwgBhNNzZJEn9+iVT2QfxLAi0XkhSLyLAwaMy8FnnMJwK8M//9aAP+kw1YIS0w8RxG5A8AfYhDEbcynAhPOU1V7qnpcVZdUdQmDtoA1VT0op7hTS3LN/iUGtXGIyHEMUi1fNlnIHCQ5z6cBvAoAROSlGATya0ZLWbxLAH552HvlTgA9Vb1qvBRltwonaDW+D4May5cAPDS877cw+JADg4vjzwA8BeAKgBeVXeYCzvEjAL4J4HC4XSq7zEWcZ+C5j8LCXisJ30/BII30JIDPAXhd2WUu6DxvBfAJDHq0HAL46bLLPMU5vg/AVQDfx+CX1JsBvAXAW3zv5TuGr8HnyrpmOUSfiMhyVU+tEBHRBAzkRESWYyAnIrIcAzkRkeUYyImILMdATkRkOQZyIiLL/T9SNsQe3ygj1wAAAABJRU5ErkJggg==" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "metadata": {} + "ax.scatter(X, Y, marker=\"x\", color=\"k\")\n", + "ax.set_ylim(Y.min() - Y_MARGIN, Y.max() + Y_MARGIN)\n", + "ax.set_xlim(X.min() - X_MARGIN, X.max() + X_MARGIN)" + ] }, { "cell_type": "markdown", + "id": "7c40b3e1", + "metadata": {}, "source": [ "## Standard single layer Sparse Variational GP\n", "\n", "We first show that a single layer SVGP performs quite poorly on this dataset. In the following code block we define the kernel, inducing variable, GP layer and likelihood of the shallow GP:" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, + "id": "54e4e5a8", + "metadata": { + "lines_to_end_of_cell_marker": 2 + }, + "outputs": [], "source": [ "NUM_INDUCING = 20\n", "\n", @@ -101,25 +102,24 @@ "inducing_variable = gpflow.inducing_variables.InducingPoints(\n", " np.linspace(X.min(), X.max(), NUM_INDUCING).reshape(-1, 1)\n", ")\n", - "gp_layer = gpflux.layers.GPLayer(\n", - " kernel, inducing_variable, num_data=num_data, num_latent_gps=1\n", - ")\n", - "likelihood_layer = gpflux.layers.LikelihoodLayer(gpflow.likelihoods.Gaussian(0.1))\n", - "\n" - ], - "outputs": [], - "metadata": {} + "gp_layer = gpflux.layers.GPLayer(kernel, inducing_variable, num_data=num_data, num_latent_gps=1)\n", + "likelihood_layer = gpflux.layers.LikelihoodLayer(gpflow.likelihoods.Gaussian(0.1))" + ] }, { "cell_type": "markdown", + "id": "9914574b", + "metadata": {}, "source": [ "We can now encapsulate `gp_layer` in a GPflux DeepGP model:" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, + "id": "e2471c80", + "metadata": {}, + "outputs": [], "source": [ "\n", "single_layer_dgp = gpflux.models.DeepGP([gp_layer], likelihood_layer)\n", @@ -129,47 +129,16 @@ "history = model.fit({\"inputs\": X, \"targets\": Y}, epochs=int(1e3), verbose=0)\n", "fig, ax = plt.subplots()\n", "ax.plot(history.history[\"loss\"])\n", - "ax.set_xlabel('Epoch')\n", - "ax.set_ylabel('Loss')" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "WARNING:tensorflow:From /home/vincent/anaconda3/envs/gpflow2/lib/python3.7/site-packages/tensorflow/python/ops/linalg/linear_operator_diag.py:175: calling LinearOperator.__init__ (from tensorflow.python.ops.linalg.linear_operator) with graph_parents is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Do not pass `graph_parents`. They will no longer be used.\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "Text(0, 0.5, 'Loss')" - ] - }, - "metadata": {}, - "execution_count": 5 - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAe10lEQVR4nO3deZwcZ33n8c+vqo+5RzOakSxLNpJvywfgDD6w8cvG5gyBHBAwEBzjxJuEBXMsV3Zfyx7Jbk4IBJbFQLgPB+OA8SYGbAzGBBskI5+ykeVLt0bX3NPnb/+omtFoRrbH0vTUTNX3/Xr1S93VrX5+NSV95+mnnn7K3B0REcmOIOkCRERkfin4RUQyRsEvIpIxCn4RkYxR8IuIZEwu6QJmo6enx1evXp10GSIii8r69ev3uHvv9O2LIvhXr17NunXrki5DRGRRMbMnD7ddQz0iIhmj4BcRyRgFv4hIxij4RUQyRsEvIpIxCn4RkYxR8IuIZEyqg//Ge7by1bsOO41VRCSzGhb8ZvZPZrbbzB6Ysq3bzH5oZpviP7sa1T7A9+7dzvW/3NLIJkREFp1G9vi/CLxy2rYPAbe5+8nAbfHjhgmDgFpdF5oREZmqYcHv7ncA+6Ztfh3wpfj+l4DfblT7ALnAFPwiItPM9xj/cnffEd/fCSx/uhea2TVmts7M1vX39x9RY2FoVOv1I/q7IiJpldjJXY8u9vu03XF3v87d+9y9r7d3xuJys6Iev4jITPMd/LvMbAVA/OfuRjYWBkZVwS8icoj5Dv6bgCvj+1cC321kY+rxi4jM1MjpnN8Afg6camZbzexq4K+Al5nZJuDy+HHDhEGgHr+IyDQNuxCLu1/xNE9d1qg2p1OPX0RkplR/czcMjGpNs3pERKZKdfCrxy8iMlOqgz+ax6/gFxGZKtXBrx6/iMhMqQ7+iVk90XfFREQE0h78ZgCo0y8iclCqgz8XRsGv9XpERA5KdfCHQRT8GucXETko1cGfU/CLiMyQ6uBXj19EZKZUB/9Ej19z+UVEDkp18IdBtHvq8YuIHJTq4FePX0RkplQH/+QYf03BLyIyIdXBr3n8IiIzpTr4NatHRGSmVAe/xvhFRGZKdfBrVo+IyEypDn71+EVEZkp18B8c49fJXRGRCakO/skev6ZziohMSnXwB5rVIyIyQ6qDX2P8IiIzpTr4J8f4delFEZFJqQ7+3MR0To3xi4hMSnXwhxrqERGZIdXBP7FWj07uiogclOrgP9jj1zx+EZEJqQ5+XXNXRGSmVAe/xvhFRGZKdfDntEibiMgMqQ5+9fhFRGZKdfBPjvHXdHJXRGRCIsFvZu8xswfN7AEz+4aZNTWinTBUj19EZLp5D34zWwm8C+hz9zOBEHhTI9rSrB4RkZmSGurJAc1mlgNagO2NaERj/CIiM8178Lv7NuDvgKeAHcCAu/9g+uvM7BozW2dm6/r7+4+ordDU4xcRmS6JoZ4u4HXAGuBYoNXM3jr9de5+nbv3uXtfb2/vEbWlHr+IyExJDPVcDjzu7v3uXgFuBF7ciIbMjDAwXXpRRGSKJIL/KeB8M2sxMwMuAzY2qrEo+Bv17iIii08SY/x3AzcA9wD3xzVc16j2curxi4gcIpdEo+7+EeAj89FWGJjG+EVEpkj1N3dhosev4BcRmZD64A+DQD1+EZEpUh/8ucB0zV0RkSlSH/wa4xcROVTqgz8XalaPiMhUqQ9+9fhFRA6V+uDXrB4RkUOlPvg1q0dE5FCpD/5cYFS1ZoOIyKTUB7/G+EVEDpX64C+EARX1+EVEJqU++HOhUdUXuEREJqU++PNhQEVDPSIikzIQ/EalqqEeEZEJqQ/+XBBQ1Td3RUQmpT7487mAisb4RUQmpT/4A9OsHhGRKdIf/GGgWT0iIlOkPvhzoXr8IiJTpT748/oCl4jIITIQ/KaTuyIiU6Q++HOhpnOKiEyV+uCPhnocd/X6RUQgC8EfGIBW6BQRiaU++HNhtIua0ikiEkl98OfDqMdf1sweEREgE8E/0eNX8IuIQJaCX2P8IiJABoI/NzHUo6WZRUSADAT/xBi/evwiIpEMBH+0i1q2QUQkkvrgzwUKfhGRqVIf/IVcPNSjefwiIkBCwW9mS8zsBjN72Mw2mtkFjWpLPX4RkUPlEmr348At7v56MysALY1q6OAYv3r8IiKQQPCbWSdwMfCHAO5eBsqNam9iVo96/CIikSSGetYA/cAXzOxXZvY5M2ttVGOTa/VoaWYRESCZ4M8B5wCfdvcXAiPAh6a/yMyuMbN1Zrauv7//iBubXKunqqEeERFIJvi3Alvd/e748Q1EvwgO4e7XuXufu/f19vYecWPFXAhokTYRkQnzHvzuvhPYYmanxpsuAx5qVHvFXLSLpUqtUU2IiCwqszq5a2YnEvXSS2Z2CXA28GV3P3CE7b4T+Fo8o+cx4KojfJ9nVczHwa+1ekREgNn3+L8N1MzsJOA64Djg60faqLtviIdxznb333b3/Uf6Xs9mYqhHwS8iEplt8NfdvQr8DvCP7v5+YEXjypo7E0M94xrqEREBZh/8FTO7ArgSuDnelm9MSXNrcoxfPX4REWD2wX8VcAHwl+7+uJmtAb7SuLLmjplRyAWUqurxi4jALE/uuvtDwLsAzKwLaHf3v25kYXOpmAsoVdTjFxGBWfb4zezHZtZhZt3APcBnzeyjjS1t7hRzoYZ6RERisx3q6XT3QeB3iaZxngdc3riy5lZTXkM9IiITZhv8OTNbAfw+B0/uLhrFXKAev4hIbLbB/z+A7wOb3f2XZnYCsKlxZc2tYi7UGL+ISGy2J3e/BXxryuPHgN9rVFFzraihHhGRSbM9ubvKzP7FzHbHt2+b2apGFzdXNNQjInLQbId6vgDcBBwb374Xb1sUNKtHROSg2QZ/r7t/wd2r8e2LwJGvlTzPonn8GuoREYHZB/9eM3urmYXx7a3A3kYWNpeKefX4RUQmzDb43040lXMnsAN4PfE1cxcD9fhFRA6aVfC7+5Pu/lp373X3Ze7+2yymWT25gHH1+EVEgKO7Atd756yKBmsphIyWq0mXISKyIBxN8NucVdFgrcUc45U6VV13V0TkqILf56yKBmsrRt9TGylrnF9E5Bm/uWtmQxw+4A1obkhFDTAZ/KUqnc2L4voxIiIN84zB7+7t81VII7XGwT9c0ji/iMjRDPUsGm0KfhGRSZkI/tYpQz0iIlmXieBvU/CLiEzKVPAPjSv4RUQyEfytxRBQj19EBDIT/JrHLyIyIRPBX8wF5EPTUI+ICBkJfjOjs7nAwFg56VJERBKXieAHWNpaYN+Igl9EJDPB39WaZ/9IJekyREQSl5ng724tsG9UPX4RkcwEf1dLgf0a6hERyU7wd7cW2D9apl5fNKtJi4g0RGaCv6ulQN1hcFzj/CKSbYkFv5mFZvYrM7t5Ptpb2lYAYM9waT6aExFZsJLs8V8LbJyvxo7paAJg54CCX0SyLZHgN7NVwG8Cn5uvNld0RhcM2z4wNl9NiogsSEn1+P8B+ADwtFc/N7NrzGydma3r7+8/6gaXdxYB2DkwftTvJSKymM178JvZa4Dd7r7+mV7n7te5e5+79/X29h51u8VcSE9bgR3q8YtIxiXR478QeK2ZPQF8E3ipmX11Phpe0dnM9gPq8YtIts178Lv7h919lbuvBt4E/Mjd3zofbR/T2aShHhHJvMzM4wc4trNJJ3dFJPMSDX53/7G7v2a+2luxpJmh8aq+xCUimZapHv8py9sAeHjHUMKViIgkJ1PBf8axnQA8uH0g4UpERJKTqeBf1l6kp63Ag9sHky5FRCQxmQp+M2PtsZ0KfhHJtEwFP8DaFR08unuIcvVpvzQsIpJqmQv+M47toFJzfr1LJ3hFJJsyGfwAD2m4R0QyKnPBv3ppKy2FUDN7RCSzMhf8QWCcvqJDJ3hFJLMyF/wQDfds3DGo6++KSCZlNvhHyjWe3DeadCkiIvMuo8Gvb/CKSHZlMvhPXt5GPjTu26rgF5HsyWTwF3MhZ63sZN0T+5IuRURk3mUy+AFetKab+7cNMFKqJl2KiMi8ymzwv3ztcio15//dtyPpUkRE5lVmg/+c47tY2lrgrsf3Jl2KiMi8ymzwmxkXn9LLDx/axXillnQ5IiLzJrPBD/CGvlUMjVe55YGdSZciIjJvMh38569ZyvHdLVz/yy1JlyIiMm8yHfxBYLz2+cdy9+N7NbtHRDIj08EPcO6abuoOP3hIwz0ikg2ZD/6LTurhzJUd/M0tj+gkr4hkQuaDPwiMD7ziNHYMjHOz5vSLSAZkPvgBLjyph7UrOvjYD39NtaZr8YpIuin4gTAw3vOyU9h2YIwv//zJpMsREWkoBX/sstOWce7qbj5zx2YGRitJlyMi0jAK/lgQGB945ansGynzjq/fg7uuziUi6aTgn6JvdTcfftXp3PnoHu58dE/S5YiINISCf5o3n3c8K5c085++dS+D4xryEZH0UfBP05QP+cvfOZNdgyXee/29muUjIqmj4D+MS05dxn/5zdO5deMuPn/n40mXIyIypxT8T+Pqi9bwkpN7+NvvP8J9Ww8kXY6IyJyZ9+A3s+PM7HYze8jMHjSza+e7htkwMz55xTn0thf5oy+tY9uBsaRLEhGZE0n0+KvA+9x9LXA+8A4zW5tAHc+qsyXPZ9/Wx0ipyts+fzdP7R1NuiQRkaM278Hv7jvc/Z74/hCwEVg533XM1pkrO/nkm89hc/8IV33xFzy0fTDpkkREjkqiY/xmthp4IXD3YZ67xszWmdm6/v7++S7tEJeetowvv/1cBsaq/O6nf8ZPNyVbj4jI0Ugs+M2sDfg28G53n9GNdvfr3L3P3ft6e3vnv8BpLj6ll3+79iWsXtrK1V9cx60P7Uq6JBGRI5JI8JtZnij0v+buNyZRw5HobS/yzWvO5/QV7fzJV9fz8Vs3UatraYfDqdedD994n2ZEiSxASczqMeDzwEZ3/+h8t3+0lrQU+OofnccFJy7lY7f+mj/56np2DY4nXdaCs+3AGN/4xRau/tK6pEsRkWmS6PFfCPwB8FIz2xDfXp1AHUesvSnPl99+Lu9/xan85JF+zvtft/Hmz95FqaoreE3Ysj+aAaWrmoksPEnM6rnT3c3dz3b3F8S3f53vOo6WmfGOS0/i+++5mLNXdfLvm/fyH76ynmFdtB2AnQPRp6DALOFKRGQ6fXP3KK3paeWm/3gR//U1a/nxI/1c8re3c+M9WxnK+AJvQ+PRL8AwUPCLLDS5pAtIi7dftIZTj2nn73/wCO/953sBeM3ZK/jYG19APsze79eJX3yKfZGFJ3uJ1EAXntTDN6+5gA++8jQAbr5vB6/75M84MFpOuLL5N9HjL2t1U5EFR8E/xwq5gD+95EQ2/eWr+PNXn8aju4d57Sd/xt2P7U26tHk1GAf/0HiVz/xkc8LViMhUCv4GyYcB11x8Il//4/MAeON1d/Hef97A43tGEq5sfkw9x/G//+1hfd9BZAFR8DdY3+pubnn3S7jm4hP43r3becXH7uADN9zL+if3JV1aQw2NV2nKB7QUQgC27tcCdyILhYJ/HrQUcvz5q0/nJ++/lN96/rF8d8N2fu/TP+eDN9zHvpF0jv8Pl6qcc3wXX7n6XAA29w8nXJGITFDwz6NjlzTz97//fO74wKX85lkruH7dFi78qx/xP29+iB0D6Vrvf2i8QntTjhN62gB4rD8bQ1wii4GmcyZgeUcTn3rLOVy7a4j/++PNfPHfn+CffvY4L3peN3/0kjW8bO1ybJF/8WlovEp7U56u1gLdrQX1+EUWEPX4E3TK8nY++sYXcPv7LuG9l5/CY3tGuOYr63n1J+7kO7/atqiXO4iCP+pXnNjbymb1+EUWDAX/AnD80hbeednJ3PnBS/nIb61lvFLj3ddvoO8vbuX937qXB7YNJF3ic1KrO8OlqMcPcPLydjZuH9RaRiILhIZ6FpCmfMhVF67hLec9j7se28v167Zw073b+db6rZzY28qrzlzBK844htNWtC/obwNPrFfUEff4X752OV+/+yl+/Eg/rzjjmCRLExEU/AtSIRdw8Sm9XHxKLwNjFW7asI1/e2An/+fHj/LJ2x9laWuBy09fzgUnLuW8E7o5pqNpQZ0TmJjDPzHUc9FJPfS0Fbjxnq0KfpEFQMG/wHU25/mDC1bzBxesZu9wiR89vJvbNu7mlgd3cv26LQA050N624ucc/wSXveClZy0rI2VS5oJElogbaLHPzHUkwsDfvecVXzup49x12N7Of+EpYnUJSIRBf8isrStyBv6juMNfcdRrzsPbB9gw5YD/OqpA9y79QDf2bCd72zYDkSfGk7oaeWFxy9hrFxjZVczKzqbWdZe5KKTe2gpNO7QT6zT01Y82MafXXIit23cxZ997R4++7Y+fuN5XQ1rX0SemYJ/kQoC4+xVSzh71RLedkG0baRU5YFtA2zuH+Gx/mEe2TXEzffuYGjaNQIKYcCpx7RzXHczpyxvZ1VXC4VcwBnHdtDVUqCjKUfuKM4h9A+VAFjaVpjctqSlwOeufBF/+IVfcMV1d/HHF6/hmpecSGdL/ojbEZEjo+BPkdZijvNOWMp5U4ZS3J1a3Xli7yh1d/qHStzx637u3zbAjx7ezb/ev/Ow79XelAOH47pbOK67mY6mPLkwoKetwPKOJp7aN8ruwXEuOHEpv/G8LnrainQ25zGzyeUZVnW1HPKea3pa+e47LuS/3fQgn7p9M9fd8RgvPrGHc9d0c2JvG73tRZa2FuhuK5ALrKGfSkSyzNwX/uJZfX19vm6drt3aCGPlGjsHx9k5MM6uwXH2j5bZP1phYLTMwFiFnYPj7B+psH+0zO64J/908qFRCANGyjWO6Wjirj+/7Glfe//WAb6zYRt3/LqfTbtnfrkrDIyuljzFXEhHc54zju2gkAvoasnT1RJ9KayrtUB3S4GmfEh3a4H2phzFXLCgTnSLJMnM1rt73/Tt6lJlXHMhZE1PK2t6Wp/1te5OqVpn70iZwbHK5Hz9XYPj7Bkus3tonM27R2grhrzj0pOe8b3OWtXJWas6ARgYq7Bl3yj9wyX6h0r88vF9VOtOSyGkVK2za3Ccn27qp1JzBuJ2n05g0bTYnrYiLYUQM6O9KUdPW4FSpU4hFxAERmdznkIYkA+N5R1NbNo1zLKOIt2t0SeagbEKbcUcZtHJ86Z8SLlapzU+b1F3p6MpP3kiu5gLWNKSpykfMlquUanVcY9+gbUUQloKIQdGKyxpyVOu1SmEAbkwYNfgOKVKneFSlbNWdnJgrMxTe0dpLeZY1lEkFwTkQmNwrEJPW5FaPToGzfmQ8UqNYj6I99sIzBgcr9CcD8mHAYGBx7W6Qy4wxqt1irkAd6jWo/cZKdcYK9cIDLpbC/QPl8Cj4bkwMDbtHqK1kKO1mKOQCyiEAftHy5SrdY7pbGLPcImetiLu0Xc4mgvRz+rB7QM05UNOO6adUrWOWdTRaC6EVGrOln2jnLSsjXwYUKrWKMW1lap1dg+W6GrJs6SlwMBYhe7WaNiwXK2zdf8ox3VHnyZHSlW+dvdTXHrqMk5f0c7ekTJP7h2hrZhnxZImQjPy8XEGqNadnQPj9LQV2TU4zuqe1slPxSPlGi2FkO0Hxlje0USpUicIorW2DBiv1mjOh5Mdi0qtzvYDY+TCgO9u2MZTe0e59vKT6W0rkgsDqrU6dY+uSdGSjxYrrLlTdycfBJOTLybar7kzOFalu7XAaLk6+bNuREdGPX5ZVOp1Z2i8yr7RMvtGyuwfKTNSrnJgtMJIucpoqcZwqcqe4RKj5Rrlap1yrc7e4RKlahTGxXzA/pEyB8YqGLAYV4w2g0b81w3s0J9HPjQqtUMbCgOb/OV7uDoKuYBy9eAFeKa/5/T2cuGhrz+cQhhQd6c65Y2m1jEbucAO+fsTtQYG45Vnbn9iPwu5IPrlVKk/7UWGCmFAMR8wXKoS2Mwac4FRc6cQBtTqPqOmqT+vfGj867tewsnL22e9n4fWrR6/pEAQGJ0teTpb8rP6lPJMPA6SkVKVXBiQC6Ke9d6RMm3FHGPxkhnjlahHbGaMV2qMlqvU6lCuRc+3FfOUq/XodZXaZA+7KR9QrTv1ulOuOYVcwEipSmsxR7VWp1KrYxjNhZB9I2XCwHB3zIxl7UXGq3VqtTrVerRtIP5F1VIIGRir4EBoRksxZKRUJQwCOppyHBitkA8DzKIQmegxDo5XaC3kGC3XqNXrdLUWGB6vUq07y9qL5AJjz3CZSr1O/1CJVV0tlCo1ivmQ4kTgVeuMxJ9y2ppyjJSqdLcWGR6vMl6t0VoIGSpVyQcBD+8c4vjuFpoLAaEZThSegUXHsbe9yNb9Y9TdqdUci2td2lpgSUue4VKNfSMlth8YZ3lH02RAP7B9gPPXdDNWqdFcyLFt/xhnruxgx8A4gRnVWrRvY+UaHc05KjWf7AAYUMxFP799IyWWdzRRrTu5wCjmAjBj73C0vbWYo153RssTxzyqsVJzmvIh7k4uNMIgYO9wiUIuYElzgcHx6FNpR3OewbEKXS0Fxqu1ySVYdhwYZ3lHkaZ8SBjY5L+9kXKVJc0F9gyX6IinQo9Xa/S0FY/q3/nhKPgls8yMfGgsaTk4+6gpH7KsoynBqkQab+F+719ERBpCwS8ikjEKfhGRjFHwi4hkjIJfRCRjFPwiIhmj4BcRyRgFv4hIxiyKJRvMrB948gj/eg+wZw7LWQy0z9mgfc6Go9nn57l77/SNiyL4j4aZrTvcWhVppn3OBu1zNjRinzXUIyKSMQp+EZGMyULwX5d0AQnQPmeD9jkb5nyfUz/GLyIih8pCj19ERKZQ8IuIZEyqg9/MXmlmj5jZo2b2oaTrmQtmdpyZ3W5mD5nZg2Z2bby928x+aGab4j+74u1mZp+Ifwb3mdk5ye7BkTOz0Mx+ZWY3x4/XmNnd8b5db2aFeHsxfvxo/PzqJOs+Uma2xMxuMLOHzWyjmV2Q9uNsZu+J/10/YGbfMLOmtB1nM/snM9ttZg9M2facj6uZXRm/fpOZXflcakht8JtZCHwKeBWwFrjCzNYmW9WcqALvc/e1wPnAO+L9+hBwm7ufDNwWP4Zo/0+Ob9cAn57/kufMtcDGKY//GviYu58E7AeujrdfDeyPt38sft1i9HHgFnc/DXg+0b6n9jib2UrgXUCfu58JhMCbSN9x/iLwymnbntNxNbNu4CPAecC5wEcmflnMirun8gZcAHx/yuMPAx9Ouq4G7Od3gZcBjwAr4m0rgEfi+58Brpjy+snXLaYbsCr+D/FS4GbAiL7NmJt+vIHvAxfE93Px6yzpfXiO+9sJPD697jQfZ2AlsAXojo/bzcAr0nicgdXAA0d6XIErgM9M2X7I657tltoePwf/EU3YGm9Ljfij7QuBu4Hl7r4jfmonsDy+n5afwz8AHwDq8eOlwAF3r8aPp+7X5D7Hzw/Er19M1gD9wBfi4a3PmVkrKT7O7r4N+DvgKWAH0XFbT7qP84TnelyP6ninOfhTzczagG8D73b3wanPedQFSM08XTN7DbDb3dcnXcs8ygHnAJ929xcCIxz8+A+k8jh3Aa8j+qV3LNDKzCGR1JuP45rm4N8GHDfl8ap426JnZnmi0P+au98Yb95lZivi51cAu+Ptafg5XAi81syeAL5JNNzzcWCJmeXi10zdr8l9jp/vBPbOZ8FzYCuw1d3vjh/fQPSLIM3H+XLgcXfvd/cKcCPRsU/zcZ7wXI/rUR3vNAf/L4GT4xkBBaKTRDclXNNRMzMDPg9sdPePTnnqJmDizP6VRGP/E9vfFs8OOB8YmPKRclFw9w+7+yp3X010HH/k7m8BbgdeH79s+j5P/CxeH79+UfWM3X0nsMXMTo03XQY8RIqPM9EQz/lm1hL/O5/Y59Qe5yme63H9PvByM+uKPym9PN42O0mf5GjwCZRXA78GNgP/Oel65mifLiL6GHgfsCG+vZpobPM2YBNwK9Adv96IZjdtBu4nmjGR+H4cxf5fAtwc3z8B+AXwKPAtoBhvb4ofPxo/f0LSdR/hvr4AWBcf6+8AXWk/zsB/Bx4GHgC+AhTTdpyBbxCdw6gQfbK7+kiOK/D2eN8fBa56LjVoyQYRkYxJ81CPiIgchoJfRCRjFPwiIhmj4BcRyRgFv4hIxij4RQAzq5nZhim3OVvN1cxWT12JUSRpuWd/iUgmjLn7C5IuQmQ+qMcv8gzM7Akz+xszu9/MfmFmJ8XbV5vZj+I10m8zs+Pj7cvN7F/M7N749uL4rUIz+2y81vwPzKw5sZ2SzFPwi0Sapw31vHHKcwPufhbwSaJVQgH+EfiSu58NfA34RLz9E8BP3P35RGvrPBhvPxn4lLufARwAfq/B+yPytPTNXRHAzIbdve0w258AXuruj8WL4+1096Vmtodo/fRKvH2Hu/eYWT+wyt1LU95jNfBDjy6ygZl9EMi7+180fs9EZlKPX+TZ+dPcfy5KU+7X0Pk1SZCCX+TZvXHKnz+P7/870UqhAG8Bfhrfvw34U5i8RnDnfBUpMlvqdYhEms1sw5THt7j7xJTOLjO7j6jXfkW87Z1EV8d6P9GVsq6Kt18LXGdmVxP17P+UaCVGkQVDY/wizyAe4+9z9z1J1yIyVzTUIyKSMerxi4hkjHr8IiIZo+AXEckYBb+ISMYo+EVEMkbBLyKSMf8fzQ4l5vupbIsAAAAASUVORK5CYII=" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "metadata": {} + "ax.set_xlabel(\"Epoch\")\n", + "ax.set_ylabel(\"Loss\")" + ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, + "id": "7aa743e0", + "metadata": {}, + "outputs": [], "source": [ "fig, ax = plt.subplots()\n", "num_data_test = 200\n", @@ -190,66 +159,47 @@ "ax.set_xlim(X.min() - X_MARGIN, X.max() + X_MARGIN)\n", "ax.plot(X, Y, \"kx\", alpha=0.5)\n", "ax.plot(X_test, mu, \"C1\")\n", - "ax.set_xlabel('time')\n", - "ax.set_ylabel('acc')\n" - ], - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "Text(0, 0.5, 'acc')" - ] - }, - "metadata": {}, - "execution_count": 6 - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9eXBj93Xv+b0XOxdw30BwAbubve/NVktUi2rJslqWZTuOYSue95KXehk5Ve+l8mbevKpkpl7FVVMzk5pUZZKpl8nYWZxMnMQK7HiRLcvaWq0Wpe5m783eu3m5gisAEiR24P7mj3MvdpAgsZK8nyoWuwny4gK493fO7yzfwzHGoKCgoKCw/eBLfQIKCgoKCqVBMQAKCgoK2xTFACgoKChsUxQDoKCgoLBNUQyAgoKCwjZFXeoTWA+NjY2su7u71KehoKCgUHyCK0A4APDrXLaZiFv3R9zBCKtJfmhTGYDu7m5cuXKl1KehoKCgUFxW5oC7PwWMJoBbZ+AmsIz6p74xm+4hJQSkoKCgUM6IIjD+GaCrXv/ivwaKAVBQUFAoZ5wC7QD0KRGcnFEMgIKCgkK5Eg4AE58BFY0FObxiABQUFBTKlbn7QDgIaPQFObxiABQUFBTKkcAKYL8GVDYV7CkUA6CgoKBQjkzfoqSvqnDFmooBUFBQUCg3fC5g7k7BYv8yigFQUFBQKDemrgNqPcAXdolWDICCgoJCObEyDzifABUNBX8qxQAoKCgolAuMAVNXAG0lwHEFfzrFACgoFBtRBCJhutkVFOJZmQWWJgFDXVGeblNpASkobFrCAcA5Aiw8BrwLABMBTgVUtwB1FqCuu2C13gqbBMaAySGSfCgSigFQUCgkjFEr/+gngBgE9LVAVTOV94kiEFgGRgeB8YuA6QjQvA9Qa0t91gqlwG0HlmeAGnPRnrJkBoDjuA4A/x+AFgAMwHcZY39eqvNRUMg7okgL+9wwNfOokzx8nidvT1cNiGFg6howfx/oOUM7A4XtgyiS9683FvVpS5kDCAP4z4yxfQBOAfgPHMftK+H5KCjkD1EExj6hWm5je+rinwyvBoxtAK8C7v0UmL6t5Ai2E+4pwDMP6LaJAWCMTTPGrkn/XgZwD0B7qc5HQSGvTA4B8w9o8V+PhK+2CqhuIwGwsU8pWaywtRFFYOIyhQeLTFlUAXEc1w3gKIBLaR57g+O4KxzHXZmfny/2qSkorIvBwUEI1z4Gpm8A1SaA4yBMzmDw2p3sD8KrAWM7Bt9/C8L5fwIioehDgiBgcHCwAGeuUDKWxgGfE9BVFf2pS24AOI6rAvAjAP+JMeZOfpwx9l3G2AnG2ImmpsKJIiko5ANTQzVsf/ffICwC4HkIkzOwvXMBpuZ1NvVwPEyW3bD9+C0I5/4BCAchCAJsNhtMJlNBzl2hBIgRYGIIMNSX5OlLWgXEcZwGtPj/I2PsX0t5LgoKOcMYLPw0rC+dgu39S+g74MLQ8ENYz56Gxdy67sNZzK2wfuULsP30l+ibmsXQNGD9xuuwWCwFOHmFkrA4DviXgJrSRL9LtgPgOI4D8DcA7jHG/rRU56GgkDdco8DiGCy9e9F3oBfnh26j70DvhhZ/GYu5FX3Hj+H84EX0mVSwdHbk73wVSkskDExcAioK2PTFGOB1QKPi0tYWlzIE1A/g3wJ4geO4G9LXF0p4PgoKGycSAsY/BSoaIUzOYGj4IQb6DmJo+CGEyZkNHzZ6rP6nMTR0GcKFf6GkocLmxzlCfSCaisIcP+QDZm4Ck1eh4dNHe0oWAmKMfQKg8GIXCgrFYP4hEPJBcARhe+dCNOzTbW5J+P96kPMH1rOnYZ9z4NTJPth+9CNYVVpYnv11CKOjsNvt6O/vL9CLUigY4SBVilUWSO55eRaYlQoPVuktKHkSWEFh0xMOAParQEUT7HOOhMXeYm6NLuDrJf5YpuYGXLz1AKdO9sF+7xKEiz9XEsKbmYWHQNi3dn/IeomEgbm7gP0GoDUAhtUHyStSEAoKuTL/kG48tRb9x/anPGwxt24oDxB/LNmQ2N65gL79O2H7x+/B+pvfUhLCm5GQD5i8kv9Rj0EPTRELrQCV9Vn1nyg7AAWFXIiEgOnrQGXhtdst5lZKLl+5g77jx2FhY8DCk4I/r0KemRmm5Kwqj5pPXifJjkSCVFKaZfPh5jIAPhfw5COKbylt8grlgGuMbrp83swZSEgu331CvQYjH1ApocLmwL8EzNwCqvLo/bunKZ+gMay7mWxzGQAxArgngXs/A+69BSxNKYZAoXSIImC/XhTt9viE8JlThykc9P5FCC4RePQuLQIK5c/kEDkLvCr3YzEGOEcp7GOoBdS6dR9icxkAgMak1ZgpgfLgbeD+24BnodRnpbAdWZklj65QZXxxZEwuO1fIAD14m6SEFcqX5RnAIeRn1CNjgOMx6U1V1JF8yAbYvElgnZG+fIvAnR8DzfuB9mPKUA2F4jF/j7bdRSCr5PKDt4HeV0hVVKG8iIRp7oOhJvdRj4wBC49ozkRFfU6D4zffDiAZQy1gNFFZ1fAPKSaroFBogh66AQ3FV3BMi6YC0NeQEVByAuXH/H3A78p92pfs+btyX/yBrWAAAMp4V7fQTfDwXWDkPBDyl/qsFLYyzlHy5NYj9SyKFDJy24HFCWB5GvC789fZq6mgReHhr4C5+/k5pkLu+N1S01dz7sdyjQKOEar0yXHxBzZzCCgdGgOJKjkFurmUyUoKhYAxYPY2oM8y+SuKwLKdPLdwiPrfOdAcPADgNTQT2GjKfRykWg9UtQDCeTI27ccB1da6zTcVokjjQHkNoNLkdqylSeo5yYPnL7P1rgyOo0U/sELVQuaTQOvBvL1hCgrwLAABD1CzepclACDopZZ8n5Na8tOFACJhwPmItvVNe4Dq1tzixCoNYDQDM7dpypTluaKPGlSQWHhI075ynfO7MkfXUUVtXteyrbsq6qroRpq8DDx8h2K2Cgr5wDmSnVcdXAEmL2PwxgMIblVCr4Aw7cLgsJSvUqlpS6/RU0nfzDBpxeQCz9Nu2LcIDP+IqkUUEbni4nUCY4NAVY6hH/8SMH2TjPgGq30ysXUNAEBvVo2ZPLbhf6XYq4JCLkTCwMKDtWv/gx5q9+d4mNpaYTs/DGHaBYAWf9v5YZgakrxylZY6ij2z5Ljkw2mpbCDjInwM3P0p3QNK70zhCQeAxx/QiM9cmgSDXmDqGoW3C9BsuLUNgExVEyXI7v+C3kwxUuozUtiseOakzt9V4rmRMHny4ABtJSxtdbAOHIDt/DDOXR+B7fwwrAMHYGlLY0Q4joyLGKE5sf6UIXnrR60lR0gM0j1w9yeUJwsHcj+2QipihOL+wZXcQm+REHn+HFewcuOtlwPIhLaCOuXs18gLUuKiChvB8Xh1BUfGaIcQWkkY82dpq0Pf7nacvzWKgUPd6Rf/eHRVkmjYECVy81FuKvfOBFaAJx8AnAqo7QDqe+hcdUYlV5YrjNGQF+cIYMxhypcokqpn0FPQgTHbxwAA1H5tbAe8DoqLdp8GGnbk3pihsD2IhGPNN5nwzFO1RtLvCNMuDD2YwsChbgw9mEJ3a93aRkBjAMABU1eA9hP56znQVdGXKAIr84BzTKpMUkkhowY6f42BjJ1KqmDhNXQPcSqpBDbuvmFM+ooATEz8Auh1cHzsWFsRUaSw3+wwrTO5rCuuUeoczkfX8CpsLwMgU9FA298nH5Kl7no69wYNha2PZw4QQ5kTcZEQeW266oT+ADnmL4d9ulvrVg8DxSN3tk9dAcx91OyVL3iejIpsWESRJFZcAnU5A7SIMcSVrcbnDyQjwFjSz6XHEPd38X/PqWj3XdFApa+Gegp7bebdhzzece4OUN2+vv6QZDzz1OlbUV9w53R7GgCAwkE1ZtJzuf1Durma9ig10wqZcY0CqlUEt5wCIIZTnAm7w52w2Ms5AbvDvbYBACQjwIDJq0BHX+GcFZ4HtJX0lQ3xyeT1LFRMpCon9xTgkOSsVWqgcTdQbyGd/M20K/e7qe9iZVby/HNY/GVNf31xwnGb2OTmAY6jkWwVDaSlPfxD6SZWksQKxODgIARBuiYWHgP6WgiTMxi8difxF4MeYHEM0KV66P0HuhIWern8s/9AV/RnCWWh6dAYKHQydZUqQ7I592t3UuYRpz33jSKHgda7WHM8GTVDHekWGduoqW7hAXD3ZxSeXXhMO6pyJhIGZu7Q+frduS/+cvEAr8m9ITBLtrcBkFFpqGZapaHSrds2YP6RUiWhAJPJBJvNBuHOVUAMQZhegO2dCzA1J8VmnaMU2sjCazM1GFPKQv/kzQuIRBLDKClGQSupjtqvZXVtmpobYHvnQtQIyJLSKedeDqjU5PnXtAPgAOEj4OabwNy93Hsi8k3ITx25t/8FmPiMQjWr5YWygTHSCwp5AF2WO7A8wLFNVBN8Ym83u/KTvyz8E4V81LnJqYGGnUBDD12c5Zi8EiOU1PbMU3WT1wmEJA9RY6DzrrPQjbUBvXAFQBAE2P7q/0JfVwWGHs+lDngPLANjn0ox2+x8Kjkv0Le7HUMPpnBqbwcu3puIhoqS8wYJ+N1kDEzH1rwm5UW/70AvhoYfbmg4fckIBwDvAoXdzH1UsFGqezDko3vM8YRyJIzFmvfywdIk7SYqG/If/vI6Yf7mn01NusWUdmQl4J0OjQHQtNPiujhKW1OOpy1ebSeFjPQ1RdumpRAJU0LSMUJJbDEkbasrqGpDngokhqmF3DVK3qn5hJLn2ACW7m70WYw4f/0xBp46krqAukapSWcd2//kstDnDnejo7kmwShkTBLrjTQdb+Y20HZ41eEi0TGSQ7cx0Hdw8yz+ADksxnYyBKOfAParQHsf5QkKbQiCHnqP3TPA0gQ5hIxJTlVLfuPz/sWYzEORcx/KSrAavCpWhiWKdEEsTUoVDYzqpqtbSeXPUENdf9rKwnyIYoQ8EKdAFQJikDyj1YZBqLSAQUtVHpEQ5Tkcj4GdLypVT+tAuHcDQ7fuY+CpExgafohuc0tsIQ16qVxvnSWamcpCs+4VMNSRdzx7F2jZn3FBShgjmXzumwW1jnawYT8w+rHUG3GMdrb58MAZAwJu2j0vTdKXvIvm1eRQGU25P086wgHAfovWjTzLPGSDYgCyhefJ85KbxxgDIgHSXl94FDMKvBqoaKTu48pGyShUkeewrkoJRl6I10HSwbKnr9LQzb9eL17Oc3idwJ2fALtfofNTWBVBEGD75+/D+uJJWPYeRLe5JTqa0WJupUoWjl+X95+pLPTU3o719QoY6kllVKWmnV3S9RU/RtJibk09982GWh+3I/iUHJrGXgoNVTZlP2ZRjJC+jtdJI2YXJ+leBgPUBrpfizHnQRTJ82dhQJPH8t51oOQA8o0YoVrqkD+xioGLq7nWGcniqzQUmgGoNC4SBPzLtOh7F8jjkS9KnTF/oZvAMp3fvteKMs92MzM4OAiT5y4sbbW0MIAWVvucA/2HdgGjF2g3tY4Zr4PDYzA1GBMW949vjuIHH93Cf/n66bVzAPEwEfA4KU/VsDPBCAxeuwNTc0PCYh899zQTxjYdYoR25ZEg3Uc17UBVG6Cvpl0Dx0uOWpDuJa+DGt+8C7ESVo204JciLOp4QjvyQjtiq+QAFANQLEQRiPipoiESpPh8gii89G+Vli5etaGwF6VvkRaLvV+KVZcopBLyATf+CahuS93BLdmBueG8dGumMwrCtAt2hzuhXDQtokjebMOO7dvZLkZoxxzyJnUfQ1rsOXK4NAZpN17iAkjPPDB5La/a/hlRksBlAM8DfEVRBohnhaGWLsLRT4Cdn9vcXZiFxDNPC0jyosoYVYNoqvLyNOkWeUtbFnIRAH12FfXkTYKl7AS2BbwqMURbzgRXitrstRqKAdjOVDZR89LcXaD1QKnPpjxxjadPNAbcQMiLwZHl7Dz3SJDGNLqnKGzB8RTWqzEDjbtyV3vkedqJOEbIA27YVfLFRSEN4SBgv1HUZq/VUK6Q7U5VK2mY+BZLfSblhyhSiWea7l647QCvTtvUlaD1z0SSdb70HeDRryiZL8emXQL97NL/Czx+f80O38HhsejzyCQ0i/E8UFlPTWmzd6Qwo0LZICd9w4GiNnuthrID2O6o1JR3mLgI7Hp5+4UOVsO/KGn/J90mkTB58roaWNr4qNZ/Sv1+0APc/zlVitX3AJ2ngGpT7D1mjPo5pq6T7vvcPaDneaDlQNrPQTY26ZrFonA8NROtzFAxQuvBgmnJK6wDxihE55krq+o7xQAo0AW5OE7hoLruUp9N+bA8k94g+pzkzUkhlrT1+0EPcPMHFCrqPZt+Uec4GuC++yw16T1+j8aXLo4Du15KmQAVP1hm1WYxjqOcgH+ZSiXbDucuVaCQG0sTa0uJlwAlBKRAGBooVKEI4cVwjkRLPxNYmkzICyQ3dY1NzpBOTMANHPgaeeFr7awqG4FD3wC6nqGczM03046EjDc2fbvbV08S66upWXBiiLRryl1cbauyPEu7u4ryk7wur7NRKB3aCmqOcQqlPpPyIByg7XpyiaxcTy5Vc8WHYc4c7YH1uf0I3Pk5mMcB7P81mriVLRwPdPXT33kdwI1/pM8kjmRjk5wTSEGjI69zaYwGlC/PbGg4fMGVRbcqHgcwfYOkY9bRK1IsSmoAOI77W47j5jiOGy7leShIVDQCk5cUTxGgBZix1Hpxj4O+Sx59ita/OIJegwujhkMbD6c17KTdQDhAYSTJCKQYGykctKYR4HnqGlbpqPxw/FPqYVjH55ysLGp75wK+84O3E5RFFYOQhNdJyq06Y3kKSaL0O4C/A3C2xOegIKPRU+OTaxVd+u3Ckj29Not7KiGpmqD173UCwsdAw05YTryU2/Mb24CDVpIouPkm4HevOlgmK9RaSW2SpwY24TwwM0y9DmH/qn9qMbfCevY0bO9cwLmLN3F1+FFCWKuspaZLgWeBZjdoK0tf7rlKs29Jk8CMsY85jusu5TkoJGGoI6+lvqfs4pVFxSWQ5xZPyEeVQYY0iTzGKInLqymBm49qqupW4ODXgVv/Atz6AfoPvZ7S6JR1s1g8ah19iSLJIixPI9qFrq+NiRqq1HEzDnhYGivRt9uM859dxctP7UW3qRG2t95D3z4Lhu6MwPq5PljqNLRLig6L4Sn0wcszhbfBNeWeBmZu0fVTysU/5KP5xFNXM/5K2VcBcRz3BoA3AKCztbwy6FsSTQWwNEWe7nri11uJwAolcJMVIL0uROfgJjN/n6p3dn4ufeJ4o1S3AoesZARu24DDv5E/6Q6eT1SFFcOkE+VzxooB4l6qML2EocGHGNjViqHBc+h+thd9zSGc/+hDDBw0w8JNAvbJ1OeJVzzRSLpWFfX03NqqsoyNbwhRJMdh4TFJO5dA3RMA5Xns1+maFMNUaZaBsjcAjLHvAvguQFpAECNxlSrSVSWrMW6VC6nU6KvpAtquBsDrSP/zpPBPFDFMoZ+qZiq5zDfVbcCBXycDMPwjyg8UwrPk1YA2/ZIgTLtguzwJ6+dOkoJpVye+89ZlgOPw8om9pF7aya0hXMfovfIt0vxcQOpbaCRjq9+Aym25EPYDs/eocKAY+j7JMAY4HlEl3/I07bZa9gOmo5Lg5J+l/bPN9W6HvMCTDzM/znH0wjV6Sda1QorB6SkBptZvjy1oruiMtAvwOsuubrkoLI7TtRJP2C+Ff9IscDO3aMew6/OFExmrMZNw350fA3d/TAahiB5mcv4BAMBxON5rwpmjPVFJ61XVSzlJkE2lASB1wooiGYTlWXLgjCbAaCYnZDPAGA1dmpWS38Vu8hIj5OlPXCLHRV8L7HiRFn95AqDXmfHPN5cBYOLqCxJjAJN2CL5FSchLjNvGcoCmkoa36GpiyoBq/ebvgGWMFqlIMFbdoVKTIdzIKEiVFph/AHQ9nd/zLHcYIwOgSwrj+BYBpBGFkwft1JgL30TXsIPmODx4G7j3c2Dfl4qmapksVmd3uPGtL/alTUivKyfB8/Re66rovnXbaf6FoY4mfxlK4E1nS2AZWHhICd9ix/vFCE2Em7gMBJbI8Oz5ItC0e13XREkNAMdx/wzgeQCNHMdNAvgjxtjf5HBAmuPLq9MvekykNv6VOfJw5dgkJ8VC9Ub6IDWGzbFjCAfJK12Zo2Reclmf/Pp01UBNJ1Ddkn2YrKKeDIDpaP7mnm4GAm6SUEh2NNzTqbsCgG7CoIe882I4ES37ydA/+RB4+C7QWxr5jpzUSzPBq2KDWIJeYOoa7eAbdkoDX8rkXgwsk0aU207rTDG9fnl4/OgFKg+ubgN2vgDUb0wGvNRVQL9R1CfkeLLSyZZalIaxLM+Q95GctNJKHko5hJMiIUrSLU1JsWqpekNTAegyLO4hH5X9uUaA5n3ZhXV4FU0qWhwHmnrz+hLKGo8DYMlefhjwOaiZJx4mAlNXKGxRkyK1Xjjaj9NnOv4ZGeee54v33MVCW0FfIT/pJGn0pHBa1VyaXJ88fGZxnCILKg3dR8WaK8AYGZ3Rj8nhq2yiMGCdJScHYHOFgAoFzwO8HkCShycnreIrI1IqGqopnKSTjIPGkP/YrOzpL9tj+vRqPXlL2VyAcqgr5CdZgKbdNNx+LQOmr6XY5nYyAItjqVU2/iUplJj0fi08osd6ni/W2cXo6icjMDlEn23HU8U/h2Kg0dNXOEi7LbWWSpSrWgsfchFFIOimKWJLE+QIqPUku13MXZdnnnZ8i+O01uz+AjlyeTgHxQCsRkLSKgnZOPiXKAbI4trr1XopnFQrGQZpx5BtN2A4SAlv/xJZe78k1azS0aK8Ua9DI53D/H0KIzTtXv0i0lZSQ9R2SQaLIt3oyYlezxwVFyQzOUSfR8PO4pxfPBxHJadhP1UgqfWFqUAqF9RaQN1AO+D5B2R8je30pavK34IcDlIYcGWOFFUjkVieothlnWE/MDpIFXlqHbDjBfqM83geigHYKKsZh0gI8C3FvHUZngfUFXSzqrWxrWx0bmmA4smyjjvHSZ5+Xf4ucF5F3aCuUfp34661f99VfiqGBcG/SF5e/A0milShkrwrWJ6lcrsdL+YUBshpFCTHkTcYDgCP3qNrpWn3hs9lUyCHXqIJ43H6bIxmEltbT1+BKNIiG/LGikaCK/QYry5djwJj1MAlfEzn1nYE6H62ILLeigEoBMmlbjKiSIt7yENhJcTtGjieEtjaysJfdFzc9ChdNTUbZaKiDpi9SxfhVu+zSFcuF/QAYgjgk8oSp2/QItGyL6enzErjfzV4FVUD3bbR7AHGgOY9OZ3TesnJiG0UXkXVfAB57c7HgEOa/auvlqpyKgCNFtEyQCYv+D66/4Ke2M6d4ymPlk9nayOszAGP3iXnwmgiNdnqzI1cuaIYgGLC8wBf+jFwACSBsBryNHRVmbtXVVryMFdmUztjtxqLo6lelt+VuiCEg8D8PaBpT/rKoHWQtcb/aqi0tFAM/ytw/y3aTbYdyum81kPORixX4gs7mAiEQ3S9iuFUHRyOJ8Ot0pCRKJfKIjFC5cQTF+maymOcfzUUA7CdUWnIIE3fpiRipptBraPdwlY2AKJI+Y4KWngHr92BqbkBFhYr/4x6tfUuCvOtN+bORDKm4UAsqazWwdJakzpQZr2odcDBrwF3f0pjJiMBwNy3/uNsgLwYsXzB8SSBjQ30vpSK5RngwS+plLt5H8X6izTFrUzMn0LJ0FVS0mspjYaLjKGOtthbWSbav0geoxT/NzU3wPb2RxBGJwC1PnHW7+wwSWdXt2V3bMYAv5tCTCodVbE076PabZUOgjCGoeHHGDjYmZ3GfyZUGpol0NgLjHxElSNs/dr/GyGbQTVrzTRec+ZxHinmc2V6vtGpeUxe/hlw/fsUmtr/VWDPq0Ud4akYAAUqJ114kHkoOa+i5OjyTPrHtwKeBcTqeyX54+cPwfbJQ5y7IcRCHLWg+GzL/uy252KY+jUq6mnaV0cfdbjWtAP13RC4DtjuhmB99QWc2VMP69Pd2Wn8Z4JXAXtfA0zHSAVy+EdpJ4vlm2wG1cihIvmxBKOaxeP5pJjPle757CP3YHzwA5j9D2hi3Infpk7vIqOEgBTI6+VUgPMJXYzp0FbQUOutKhC3OE4J+DgsNUDfnqTQzOgn9GDz3rWPGZGEz1r2AjUdaQ2Gfc4B66tnYDG3AkEPLIZ7sIaDsM8vbTyEwvHAzhepWejx+8C1vwd6XyHDUwDiY/6WtrqMukBrhYqKGUoqdthKfr6ffHwT1k4n2oNPENZXAXutJZ3DrewAFAh9NZXV+ZfTP66rodLRcLCop1UURJGUPuMT4YxBePwQQ08cMa/W7qR5vbWdiTLKSQwOj0GYmqfF33QYqO2EMDWbdlpW/7H9tPgDZIBMx2DZdxT9PVWxcuCN0nYIOPpvKOw0/EPg/tsF2Q2sZ1DNWqGidc08zpFiPhcAWPRL+F3THbQHn2BKuwOap/59SRd/QDEACjJSQhKOR+kf53mKJ8syvlsJ/yJVYcSVuQqCANuFu7A+fyg6fnHws4vUnNe8f9XDmeqrYPvgKgSxBahuXd+0LJ6n3oy2Q2RAcjW4Vc3A8d8COk5R5dLlv6JdTB4NQcJUNAlLW13aEtC1QkXrnnmcA0V7rnAAePgr4LYNvpCI6xUD+KexJghzhQ/NrYUSAlKIoa2iZhj/InW4JqPWA86RrRcGkjWV4rBPTsDa35vg1dbv4iAu8uDXaJ6z1PGwvvIcbJ88Qt8Sh6Hhh7CePR3z9LPB2EZJXft1AFW5yR7wasByGmg9AAgXgPHPII5fgqe6B9VdR+jz5NUFr91fK1SUbSipGOeSNxxPgEfvggU9uLzchpYjL+NYexPqzK6Cvbb1oBgAhRgcR7sA5yhgOpL6uL6GuoIjz5TtkOsNsTgGaCpipZ/mVvTvMAJBPrYo7u9EjXcMaOhZXV475APUelgOnkKf/w7OD93GQN/B9S3+MpWNlMy1X0PORgCgaq59XwK8Dqw8+hRa1yNg+DHAa+DVN2N2lqF3xw7AEY4JH3JcrJaeRagSTAzHvstfkRAARpIZciOkrpquGamMdrVQkaWtbs3H80nBn0ueXTJ3D6hoxC39KeAwxQoAACAASURBVDTv6kZ3EV7belAMgEIi2ioK8wSWU+PcvIpu9pU5qmLZCogiKatW1FPp5zsXYP3807AEHRCcIoWBBg6QEF9wGWg8nflYTAQCHqDzJITpBQwNP8RA30EMDT9Et7llg0aggSS5p64CnJGavnKlogHGw69hdGoe1y5/ipOtEWiWpvFUtQ/cnB2Yy/0pElDrgIom9BvbAI0JCGqjCfd4CemCSExnoGDPJcs1P/mQSju7ngE6TuFwmi76Qr229aAYAIVE5Klqi+NU6piMSgc4ha1jAOL0fyzmVljPnqZB580hDI3GeYlPrlOl1GrCb343UNcBwREgQyKFfbrNLQn/X4v4nQgAoLIRAjPDfu0y+o/vy5sYWHd7E4Tug/jrW6MYOHQELYe7SAsn6CEPNuRP/ANeTUOGeHWsm5ZXS8Pe1aS4EJF2BOEA9Zf4lyRRw1kyYpNDdCyjieSdG3eln7K2GfEvUdWVc4TUSg9aKQdTxihJ4DRkahL5y59dKmrzyHpYb2PLqr+vkyqCwoHUP9TX0gUencu8yfE6E8ozLeZW9O1sxvnhqVhlCGOkPlnXnTn8I1fs1O+g0s64xV42LPa5DLOGk5B3IsIk9V0IkzOwXbgL0+6jgHcxb+99ShJ01k0hG6OJDF3rgcSv5j3087pumn9Q1YLBkRUIrgiVCWsqAL0RwhKHwfEg0LQbg0stEKpOUDVS/+8Dh78JV/0xrHi8gHAeGPpraoSavpX3CrOiNXsxEZi8Alz5Hs0T6TkDHP3vyn7xBxQDkBZTgxF/8uYFfHxzFEAsYVRfVYE/+ZcLG24eyfaCXPeFK4bRXRnAvcsfwjX8ISB8DNfwB7hz+Ry6qiKpeihYoxFGloRYTlPxo1KTOFqmwembjcXxhM5LYXIGQzfvYOBIT6wyxDNH3uxqyV+/mzp71brE0k4Ji7kV/cdWrx6K/13r2dOwvXMB5y7ejO0e9h4htU+vK+cOX2HahT958wJO7e2IVjnZzg/j45uj61ogM11HdscyhGlXwuPC7DJsN1z4q2Et5nu+Bpx8g2YpREIkX3Hx/6FqmTxdW0Vp9lqeAa7/IzByjpLpJ34bMJ8o3qCYHFF9+9vfLvU5ZM13/+//89tvfOlUwZ+nrtqAaoMOf/yDj8EBuHhvAqf2duDO2BzO9vXi3SuPEQxF8N7VJ+vK4kciLHoB1lUbohfkqb0dqKs2rO/3/G4akDF6AXj0PoxL97DL4ILBNw3RbUeFbxq9BheMS/coJslE8kikC7Ou2hC9QdK+Fl5F3nG6BqaQj2YLbHZtIFGkkkhDDcDx5Gm//RGsR+pwdN9OmBrp/dmnm4HBN01D39PF4CNhmp7WeiBviql1xioEg2GcH7qNZ47uw9F9UuhJX0PJ2OVpMlwbFAu7OzaH/V0tuHhvIqrkGYkw/OCjWzjb15twPa56nhmuI/ln+7ubsb+7Gd/5+RDev/oErhU/fvPzR+k6U+tJz7/tCDWpsQglTaeu0ghObQW93g2+xjWv8VwIeoDHHwKP36N7q/dloPt0eY5PDfnwpz+6uPw//uEf/WnyQ0oOIAPPHe7G5PwSvvera9jT0QSn24tvvXYSlrY63Hwyjb9/9zqO95qiF1OmEjpZKtfucMPUYIR14AC+8/MhtDdWY2phGZV6TcLvxVclyL93b2wer585hIX5GRhdw2gIjlOIBkBQW4d57Q609+zBhFeHn12bwrw7gOcPduL53bWk8TN3Fxg5B9/oZSy2DWDEVxV9LnOTEf/w/g187tgOvDP0EGf7pNJHlRbwuyE8fgD7ipjoveprKCRiOlZa6dxc8S9KUs90G9jnHLCeOQILxgGOi1ZqqEf+lXR/kjqFowTcQOPuvFZGCZMz6ZPIHEex87A0wrRyY3Ma5Ou0o7kmoRv2v3z99LoXyPiGqngxu/hOW9eyD3qtGi+f2Jl6fI4jZ8JoAizPAfabVPl020aVUO0nqPN6A7mPTOe2YSIhjF77EJ2Be4AYxnLtQdTsOwNh3gv7nfHovV4wGew8oxgAmUiQauBX5gHPPDxuJwbgxCuvhCGGHQDHwyiMIDTK4wv6IA6Yw+AiTjjuBhHS1uK963N4qf9YymFlD+TU3o7od9eyD7POFWg1KnzhZG/K4/KOw7XsQyXz4HcO8WieewdmHXXprvA1+GS+FYOzBoTU1fjWaychhIHv/OoywHF4+cROXH4wha62BlhMR6ik0ylA9eA9tEy+Da/hMGzDenQ11+H779/E54/vxPffv4nfeuloYl32gh+2i+/B+vWvJ74otY5m5/pcm3tQjNeJqFY8qCsXC4+BxZgXb2nQAo+cQH1/+mOIEdpVGbMUhssCuXEsYxKZ52kOgRikZjFDmp6NLMnHApmcS+hurYtWuPTtbsdPBu8BAL7Svzfh8bRoKoCup0kzae4+zVx++A4NR2k7TLsFXQbp8nWc27oRI8DMTWD8IrqDHjzx18LZeArn7i7iFJuO3rNFlcHOAxxLEx8uV07saGRX/uo/5edgTCQPyilQbfvyDORmIJHXYD6ggdZQhRl3BBq9Hg/GF1BlUMOg5lGvZ2ir5sCHvDCo4hJynIrkhA0NtDBW0PfRJQ7/cuEBzI1GfPftK2irr4axQoe6agO+9UWS7LWdH4a5sRrvf3oNv/NsG7jlafRUrKBFR4nYFb4W1xercH3JiH+9OoM3Xj2BW09mMLngxond7bjyYArmppro8a48pPh1wpY3HITn1k9RuTKKix4z/t2bs/jd104iGIokGJ6L9yYkj3AS1pMdsDz9RUCtT6xOcc8AXacgePSw2+3o78+wQJYRg4ODMJlMsFgkTZwn5yDcvwW7OxTb4YwOSl3RUqhnZhh4+Evg2G8CVWkGc/hclBTN41jIlCogkFGwzzkSd2KREIVLQj4aQboB5PDiRvVwkhuqkmcBfOety3Ct+FFXpce3XjsJAOtrgGKM8jRTV6j4gJM6pU3HKHy0yg50tXPL+jWGAxRqnbpKOz2jGbCchuCtlO5ZI77/wU38m88dxuS8u+SNXWnxOmH+5p9NTbpFc/JD288ArMxRSGTuXmz8W3Ub3cTVrUBlEwYfuRARKfYvf6Af3xzF//y372KnqQG/9fmjOHO0B+euj+Ddi8M4ZtbB2mei5JXXSd/9S4jvLg1wesx7GeY8DKJKD3NTDaoMWkzMLqK7QQfmd0MT8UDD09/4IjxQ3QZD225MRJrwz5+OwdxkxKV7k3hid2KHqR5f6d+LmyMz+IufXMIrT/Xif/j1ZwAg4QZM2Y4yBvvFH8IUGsU7C224uGzCwKFunDnaEw1jBUORqEd4ZnctSRfXtCd6ps1VEGYWYbvugtVqjS2qZYwgCLDZbHS+XV0QfvFnsH10C9YvDNBiGw6Qpxm/q7n7E4pHP/W7qYsNE8kDL9C4vqwI+6kCRQyvqk+UjnwskJmmgckOiKWtDid6qWR41esyG3wumsQ2c5s+q8omEi9s3J12V7DhSWWMkTb/zG36igTJ2HQ+TeuEdB2cuz6C87dGYazQwe0NRO+jsmPbGwAxTAu+/RoZAI4H6iwUV6zrTnvzJl88wrQrGpP3BcJJnnIaz0mUlCC9DrhmpzA+MQEdAhADK+io0yIcEVFloJJCd5DHvJ8H01bj/XtOqGrbMbGiwRuvPRU95sc3R/GDj27h2f1d0V0EAMw4l/Hl/r1498rjrLwQYdqFH56/ja+3jKGDn8H5yDEMzWlTFoHo63p2FywttdTQwnFRI9B3YBeGrlyF9ff+V1h6s1DGLBNkI9B3aA+GfvEPsH7lCzFP27NA0guyARDDwGf/DWjaS0m+ZPxuaREq8ZY/6CUjANB8hywp5CjHgo6JjITofp6+TvczANR00s6gtpN23uvNTTGRPn/HIxo676WwL5r2AO3HU8amyveJsgMoIus2AEEPMHWNYnchHw3xaDtMH2rykO9VSPaMPr45ij/+wcf4g9efw3OHu1f1nOIreOJDLNl+lz2m5GP85VuX8WjSgV3mBvzv//4lfHZ3HN975xp+++wxfPPF9JOq4s/lyv1RvNH+EKqwB7dqPo/zD5wJz5ngEZ40w3Ly5aiHee7iTZI42N+GM6//HlC3ORJeMufOncP5X/4YAxY9zrxwJvbA/ANKrsvhFNcoJSL3fzW9VrvHAXSczCkGnzeCHmqy4lTrurY3PV4HVbnN3Qd80kxnTQX1KVQ0UN+KoVaSteDpi0Wo0z2wQjv1ZTt97hGpD6HGTGtEY2/axP9a93TZGYFtZwDkm8F+g6o8GnaSFc+gyb4Wyd7M4PAYIhEGlYqLejPZVgHJC6v8f7nyJtPjsqRu/DEA4L9+733sbK+HPxhBR5MRHn8I5iYjPhkey1jJkXIutYB49e/h5uvg6n4N71x5FKsCkhCmXVKM/1mgoSduB9CLoes3Yf3Ka7A8/811v6elIroDaNdgaOgyrK99PrYDGP2EyjjlUs/HHwAzt4Cn/2NqhY881rGrv3wqoQLLtBPgVZkrlrYqjNFivjRBOQP3FO3QkMX6VtFIne1GE1DbvWaSea17uuyqgLaNAQh6pYX/Om3fm/dS3G4zV6okkctuJC3269S+3vtK5lBGJAyE/RDUu2B7dzBajSKMT8H21nuw/v7/BsuO/CVBC0U0B/C1r8GyOEhVTu99Rq+ntVaK/0uSzYwBQ39F/z/w66kH8zipEidbSQzGKIYdTpJXYCDjUlGfnx6CwDIlLMGtq1pmSyJGKHHrc5F3z0Tq/eB5QFtN74+2amsJG6ZjFQOwNcpAQ9LCP3WdPP7mvUDnM1tq4ZdJVjFUqTj8wevPQaUiL3TdKoNtR4C5ewg9+gD2cAO6zLFyxoRdTSAI++xEosRBZzusLxyFfeTepjAAdrudEsCtdYAjCEuXOSrRYKlTIb4kFD4neZTmk6kHYiL9amVTdk8c9FD3bn030HKAtG9UWrpWfYtU3TJ3jwxAZVNuOwpdNQ2Dt18nD3id1UEFjd3nkazOk1fRe71VtIYKwOY2ACGf5PFfo8RQ016qIa7IYvDGJiX5JsxZ1ZDjgF2fh/rq32H+1vsQVV9MW84Hjkf/zlqgKVXiwNKW4liUJdFSVcdINDJgMbeSQZu/n9ho5Byh7/VpqjqCHioJzUae2eMgY7H3i6m9ArwOqG6hr7ZDFL6ZfwBUNUUllDeEtpLkCKZvkiHT12VtVOS+lUxlneXCZjnPcmdzGoCQj24W+zXa2jXtoVBPZWOpz2xzUtkIrvUgjs8M47ufXEHvzh2plU26KkqUNeyKaQUB1BXseAy0Hy3NuW+EpQlAm1T5tTKXWA3mGgMM9ek96HAwOxkMj4MSsr0vrx2T11YCPQNUlfbkHKD255ZcVusB03EqeXbbsw4xFXtW7kYp1Hlulh1QvtgcikVRGCXqLn8XmLhIpZzHfxvY+5qy+OdK1zPgeB5fbnekn5HKqykXEEia86rRA4ElKeG2CWCMFBu1VNE0eO0OBGGUkrpSLFiYWkDENZ6+ukmMkCDeWmEF3yJ1TGez+MdT1wXs/wq93yvzsXOUlEFlhMmZtDOGE1CpqU6+eR/FwbMcA1nsWbkbpRDnWRQBuTJicxkA/xIw/hl5Scf/HU03Uhb+/KCrxmLNPrSEJnD2QEP6Gak8T3IZKfCxeuxyx78ERAK0OEKSXv7lRxBmlgDQDX/x0kWoEAFqUw3A4PX7EFb0ifODkxfjcICSvbte2lg1jqEW2PMqfV+eSy8Pne2MYY4D6jpph8xraFeyxrD5Ys7lzYVCnGf8zuLc9ZGyGNtYSDaXAeA1wLHfAvZ9OfsEnEJWCNMufP+eGoxX4VT1TPQmSLipdFVUXicmSRFrK2Ix83LH60B8stdiboW1fxdsnz6O3vCv7NLS79R2pvy5qa4Ctk+fZF6MmUjG0DKQWwhHW0Hqo5UNsNTw6eWh1zNhTG+knoWmPVT/7nWlNQTxsfR4mehyMwKFPM/NsgPKB5vLAGgrN8WQhc2I3eHGq6ePgW89CMwOw1KvjlYTReHVlGwPLif+sbaaDEMkVNyT3ghLEymd3xZjBH17u6I3fG14jhK2ycNfxDAs7Y2wvvpi5sXYsyANTsmDJIBGT7uIynpYann0HejF+aHb6DvQu7HxkryKdgPd1NOBgIcMYsATNeqrzcpdFSbmPKNgPWz4PLNgs+yA1kQMS5PdvBl/paRJYI7jzgL4cwAqAH/NGPvjUp7Pdiaa4PL1UfXI1FVYep6H3eFOaHKx1PIQHt2HPVQFU3NDTKBMbqXPoypm3hFFahLSx3l0IR+EyTkMPZrFwKFu3Ho4hufbZsB1nkpNCAZWIHgrYQ+7ootxwsD3cAAAR/LF+UKtA3a+BOGDv8PQtRsY6DuS24xh+Zj1PbTD8TqB5RkMDt2Aqa4C/T01ABjgW4Qw64bduYL+vSZYaqsl9VTQBiq5fUiuMorvK5J/T6UlY5anUZZA4Wb6JvfRdLfW5T8MJBtLUZTeL/k946TNKUfvJ8choTQ5doC4vxeps1mMxMqT5cOptFQWXFeHQARpxvuV0ABwHKcC8BcAXgIwCWCI47ifMcbuluqcFEBhi8ZdCE3dxJRuf4Kc9Xd+PoSOhkp4fH6cev5s1PsFQDf30mR5G4DAEiWyVbHLXhAE2AYfwvo5mvWwr3IR3AyDXWxIX2o4bMepE0fSa/V75gHL83mXYhAmp2G7tgDry6dhMdWte8ZwRng17airmmE60gjb2+dgbTHD0mKEMDEF25VHsJ45AtQ1U4JcJc3/lb/zatpVcKqYAZAXpUiQci0hnySS6JB2iBxJNWgyjNYsMavtLDYsIx3yJu6OOV6apyy9lxwXZ1ClhV0MS4u7ZCCiNoKjv+fVtMCrNGTQ1QZ6T1U6Kk9W6WPXeWAZEZGlnSNask5gjuOeBvBtxtjL0v//EAAYY/9Hpr/Jqxy0QmYWx4Fbb+JtVw/2nnwBAPCdnw9hbHYRM85l/Pcv7sWkWA/rF19M9H7DfuDQN8pHGiGZhcfU7RtnpAY/+CVMej8s3VK8/9G7EGfu4mLtl/HMQUtMHK+3FUN3x3Dq+bO4ePthrBtaVkh94Tgs5hbSDeLzG1mNyli31QP33gJUWgjznlR56BxJkPkYfpi7gYmHMSDkoV3E0hRVk/FqyivlcWdQFogiKQ2LYXptlU1UrKKppMVapSnuPRJYRv1T33jk9LHe5IdK+c63A5iI+/8kgKdKdC5bBzESGxrOqzYmL1DTAVQ04kXdEv78/G307TbDtexDc20l+na34+G0CwPP7klcHNS6mAx2OYijpWNxNCX+37+jGuDjztc1Br6uE88cIHnraELw6n0MPH0cKo06/cD3kXuwPP2lvC/+ABJnLfS+Atz7GSwtxvwtzhIWc2v60FY+4DiSXdBWUegpuAIszwGLY+QdawybX8RODJMUBxhQLWkL6YwFuSbyRVZnxnHcr3EcVxP3/1qO475SuNNKeO43OI67wnHclXm3f+0/2E4wUdpiO2Nf8QmfkDf284An7XD4tHAcYDoCXcCBF3dWRCc6PbXHjPevPUFvZwuGrt9OqU0Hx5VvOagoAkv2RM38sJ/eP1n8zbdIYyLj6v+jCcH97Rh6spAyqAUALE0VtEjXFKEjurKBegu8TmpIyyPJYyhTPt98oq2iRLTlOWoi1OilKXNLqVVm5Y4oUlVVYAWo3wF0P0c6UYbasl78geyrgP6IMbYk/4cxtgjgj3J87ikAHXH/N0s/S4Ax9l3G2AnG2IkmYxkOXC42TJS0ZZxSs5GetI86ngJ2nAF6nic5jK6n6d+WARoJqaskDz3ky+55mvdB5FQIT92EPxgGB+CWMIs/eP05TDq9ONVTB9svziUuEtpKwPkk7y85VwYHByHcuyFtyWlHJEzOYPDStUSjuDhG32tjCq+288Ownt6LM0e6otU/KQujf5kSv8Xa1htNwI4XgZXZNWv6syV+2M+ZU4ejZacFNQJATP/I3EfXrLENCCzS9S3FzQeHx1IqcYRpFwaHxwp7bmvBGDVAyk5D92kabp9cPVbGZGsA0v1eruGjIQC7OI6zcBynBfA6gJ/leMytCWOSN++gRV9bBbQeosXdfII8T70xfSxVraUbrP043WTyFKs1dgPCvBd3PXU4VuvGH37jGbx4fAfAGDqaa2AdOAiViqPk2Jwj9kfaKmB5WqqGKR9MJhNsb/4ThClqYovW7lchUQlycYxeg6QlFU0INuqAqhYSv5PE46IEveTpVRc5+d3QQ0N63NN58Zjtc470oa3411podNXkzHQ/R1r8YT/gccBUoym/7txwgO5HQx19Do27stOGKjOyXcSvcBz3p6CqHQD4DwCu5vLEjLEwx3H/EcCvQGWgf8sYW6O3vciIEfKYI8H0pW9A4s/lWbIqbe6JrehzS6WFhjqaa2Co37iHUVEPdJwifZiV2VUnJ9kdbvTsOgnN1NvoVs+je+AATvS2RzVRLPVaQB+BZc/B2B9xPC1GnvnihEOyxGKxUKPQz99D3xKLJTjDjwFe2lXKEhFxI/+ipYZeZ3QiVFQ8TsbnAna8UJqtfst+WiTt16T5uBs/h3TJ5JTXWizUOvKoa8yAzwmLQYC1zw/b+1fQt78HQw/tpevOFUXy+FU6oP0YzRIo16KHLMh2lfo9AP8VwJug5e49kBHICcbY2wDezvU4eSfkI8+OV1GZXHwGn1eBNkRMSriGyUCEA9Qg5V+Wyg3lsi9GZXK8hhYJXi3dqLLlYFLZV4j+hkXoMU5Fz1vVIk00ytO2Uq2l3cPsMLAyk1E5tf9AFy2KjkFg9jbQsi+xzlpjoLmpcqVD9Pg6WkjLyAAgEoalhqHv8IFYgrO1FhB8Mclwr4N2Wcndv3Jzk74GKYSDFIJL0zFcFDiOhqOHfKRmusaQ9E2HHB6qbIKlaS/6vB/h/KUbGDhggqWxBAnjwDLd5/U9pEOm2vzVS1m9AsaYB8AfFPhcSk/IT4kcQw3FzSvqV/fkeRUALQD5YmyJPRYOkncW9gFBH5XAhXz0M7nGFxwdQ60DtDW0BdZWUSxdbSicV8nz5D2yCHm3map2OI5+b+xTuvjjE6g8H4uBxs9d0BtJFqLjqfJJgHkdECZnMXTnUax2v14LS/xauTRO32s6Ev82HKDXl25oiNcJmI+XdiHgeQpBRIKAcxSoyUKldBMiLHgwNOnHwEtfwNC1G+ieXYSlTupD0FXnZ5hOJuR1obKBQqnx98EmJ6srl+O49wBYpeQvOI6rA/ADuYZ/0yOKgN9FDSrm4xsbKp2MWivFBMtURZBXAc37gclLtNvJVILXtJcMwPx9yiEkH8Mzn2gAVFq6YfyLZTOQR7h7FbYPr8H65VdgMbdSI9WPfgzriRZYuqVzXJygkr1kTz/kI28vGSYCECnpV2p4FVXTiGGS5Ch2PqLAxCeoLeZWdHe2w/bLC7C+cBQWY0TSpwpLHceV+XM8QgEqV9XmcV0oM7J9pxrlxR8AGGMuAFtDlCfkp6EZ9T1A5ykKu2yxDzkjai3QeoR2KWLaRkFaxKtaaOh2MtpKSvomJ5R5njToywT7wxuwvvJ8YoLzqQ7Y3VIFDWOkEZRpZnS6HZJ/iaqFysUbVGkoF1HdRp/JFiJtgvqV07AvBYCm3WT8TMcoPxZwS2XPy5mv6dUQRfpbzwIAEWg7TNMFt+i6kO3eVeQ4rpMxNg4AHMd1I6tpy2WO300fasfJ7Ts2Tl9NQ14WHtEWNx3Ne4GRjyjhGf8+RcXhVhIXQl01DYnJNGO4mIR86N/TkugVhwOwNOhg6ZB+5l0gT782KfwTCVOyT5NmdxT0Aj37CnfeG0GlofLQkXPUbVvduiUWrTUT1LyaFujKRvrM/IvUj7IyR7k1MEl2QZKv4FRxKbgIXcORADkCHA9UNtOsZ3351/HnSrYG4H8B8AnHcedBb91pAG8U7KwKjTygW18LtB3MbfzeVqC2k8oJQ970i13THjIAc/co3pwARzdcvAHQVNAOIOjZmB5+PvEs0Pf4hTCQpGa6KDWkJydzgytkFJIX0XCQkuBVJaiQWQu1lnYCwseA4wktZDlUB206VHHGoHkvXdPBFXL2AsuSLo+HFn8OpJmjqwb0HZS/0hm3RHI3W7JNAr/DcdwJ0KJ/HcBPAGTZUVRmMFEq62ujbr2tpkOyEXgVvRcTlyj5nLzg6aqpwmThYaoB0Ogx+OllmA5qYh4Zx0Gwz8PueBv9r1iL8xoysTQZ6/SV8bkSk4ZL4+nj/2KEyvyS8bkoNFBC7zCqD2SJ5SAEQYDdbqeu5J7n6bOcHabmqu14nXMcOSDaSgpjKqSQrRTE7wD4AMB/BvA/AfgHAN8u3GkVCFGkdvPaLqDlwPa8KTJhqKXh5MnesUxjLyV8fYuJP1frYapisL39UeKQlI9uw6QvcUMYY1SRlDzXd2WWFkf5dxYnU8M/TKQFRJcmiS+G04+LLCImkwk2mw2CIACQVE1tNphMUhUQr6LO2s6ngeWZ7DvAFbYV2bowvw+gD8AYY+wMgKMAFlf/kzJDlDz/hh2UONrisb0N0bCTygmlztKEFvzGXQAAh3AzsQWf42BprYH1+cOJQ1JefRGW6lBph8T4XNKs37gdQFiSKJa7Nj3zlASvSQr/RMs/k5yEoIeqQUpc4WSxWGC1WmGz2XDu3DnYbDZYrdaEHQE4jkKcu1+hMkY5HKagIJHtKuhnjPkBgOM4HWPsPoDdhTutPBNd/HtokdsCibGCICs1SruAhAHZ+hoEdA3w2e+mtuCrNLDUiIkTqzpNFEJJO0O4SKQTpkve4SzJ8f+kHUDInz5s4HfTkPUywGKxoK+vD+fPn0dfX1/i4h9PjRk48FWgqomkvsOKqKICka0BmOQ4rhYU+3+P47ifAiixElOWMJG6PJXFPztqO6hygonRYRh/8uYF/NP7N3F53oB27QosDZpEMS5NBYTHDzB0+0GikqRaB7hKeJk4R0hvPh6fMzH0tzgBqBoMrQAAIABJREFU6GpS4/+Mpf8ZGCVWywBBEDA0NISBgQEMDQ1Fw0Fp0VUBOz9PVUKBFUr6b4YRngoFJSsDwBj7NcbYImPs2yBJiL8BUBQ56JxgDPA4Sd9FWfyzQ1tFCfLACgDSw3/2QBe+96trcBs6wAFYGLmVIMYlzLphu3Af1heOJypJLvipEqUU8r7hAEldJFchrcyR9DAQq/9P9v7FCJUMapL+NuihXUGyUSkBcszfarXizJkz0XDQqkaA54HGncBBK4kI+lzUROV3r3+erzxv1u+m48iy475Fep/ypFKqUFjWnQVljJ0vxInkHcbogqw1UwJTWfyzp84SbSYSpl2YXHDjt18+hr94/waeO1uB5fFhWAesUV0gu8MN67O7YamnyyleSdJSy5PXXZmmmqaQeObJ8MSXQMr6/3L83jNPP0sJ/3gpXJKcJwosA21HCnveWWK32xNi/nJOwG63Zw4FyWj0QNshKu91T1GXt9sujSZksZGDQGzmrFw2CdDvyPNmK2okAUQNIDKqpw96yChEpHkFKi2VXG+j8srNwtb9RHwuqmpp2qskfNeLvhow1EMYt8P26ZOY8iIHvP3kAqw73OCbYpO1+g90Ucx8eSZaHRNt1FmepTBLsQ2AayxVQC+wnKT/T/o/QzM8GkVXTOguEoSwyGCfuhNrQpLDP2Uy8zhhSpiExWJZe/GPR60lKYt6C/U2BNzSl1dqjBJp0VbpAa2B+mU0BurzSKeNlIw8t2JpgnaC4QD9vaF2e/UmlDFb0wD4FqljtWW/svhvlPpu2K/eSRiIPjnvxuHug+C5y5h7chvNu+O0gdQ68vTDgcSF11ADLDwgcb1i7cJEkQbT6JO6uz2OxIVraQLQ16CxuS1p+PsSbPccsL56Jva7ZRT+KQhqLaBuzK+hlmvwaztIHHBllnYbTgEAD1TWp/ZoKBSVrWcA/Mt00bUe2j51/oxJ5Y2y5waQDDVH9e7aqvVvvw0N6D/YDWj0sclYAwdgaa1F+NNhuEZvwWPcGfOa5cXdv0QS2jJqvRQbdhWvdNLroARn8mtemYt1OjNGTWINu6LJbtv5YfTtasHQg0lYf+PfJmrhB5ap+UthY/AqmmRmNAHmFdoRzNyknUdFfcqs5i2BnFcp493O1lohAyt005uObMrpPOtCDNNiK0/fMtRSsttQD2gkDzzkozi3a5zisfqa7D1YnqdcgONRbDKWtNirm3uxSxzGxQVX4lAOlZYW2aoknUCOp8W2WAZgaZL0XuIJSsZRJyV2PXMJ8f/o8Pdr9zFw6liaQShidCiMQo7oqgDTYZJqcI0B9qvkJOiN5SOutxFCPronwWK5FCAWPuRVtCsto7Vp6xiAkBcAA0xHt662jxgh3Z2wH+C10mCKLhqaoVnlNYsRSvJNXKLFsbo1u91RdTPgeID+/Z2J4ZuGHeCnb+CZ9qSQjqZCEuASE0Nv+hpg/gHQerDwYSDGAMcjCj3FE1hGgn6hrP8j6f9Hh7/vM2Ho8QK6983EjEDISyHF5I5ihdxQa4GmXVSi7Z4Cpq7R9amp2Fx5gsAK3Zf6GqC9j/JEump6fYzR/ep3A+5JYPYe4PHT9ZRJgr2IbA0DEPLTVrLjZOnFx/KNGCGvIuQjD6JhJ3UzVzZnH9bhVeTpGttJG2bikrTtXuMC1BhICye4kvi+1nSQAXGOJOrh8yqqGAkuJ9bQawzAkr04YSD/Ii32xqTBKN6FxHjz4jhVpuiNsRDXc/thqQW6NXsS9Ofhd9ONrVAYeBU1INZ0kAMxcxtYHKVdXEVDdgnnUhAJ0vka6oHes3R/JeccOU5KnBuoKKX1MOWeJofI2FU2l3RHsPkNQDhIHpq5r3DbRzFC1REhL2KtE9FBwFL5nDThS60jb1yt35gHI3sMgWVqyOJUFIpp2EGeey43A89T+V9lI/Dgl/SztYxAbQcwdT3RAKg0dMM6nwDshUSvnuMBryu1iYpXkdddYAMw+O5bMMEBS5wBECamYb9xHf1HSM4CTKSbr4ma2WPD3ysAnoelswPWsxoqYzW30o5mi07aKis4jhbJ6hYyuo7H5LCEgxQ20hnLp5zbs0DrQvdpkknJdiKZSk1OU20nvb7xSwAY7eJL8No2twGISKVr7ScyjzXMhXCQEoocT6GWum66CDUGKYTCKBYv68sEVsjT9MxLMgSSkWCgC4RXx80EBgCR9MvFsDS8Qvp9Qy1N66ppp8U63x6Q0QTseRW4/wtpkP0q4SNDHV20YiTxIq/voR2Az5k4V1hjoB6C+u6k49TSMPrWg4WrzGIMJs0ybO8Pw1rVBIu5lYTpfvEhrPv1sbDXyhzlA6T4f3T4u28xuqOJlrFGgmTQt+u8iFKhN9LQ9dZDwLIdmLlDYSJOBVTUli7MGwlSaXNdJ9DVv3Gnk1eRA1LTQbuB+Qc0j2MthyzPbF4DEAmTl2A6mnmQyUZhLKZh03mKFruMMXZdUthJkkgSRZoDHPRIHZPL9O+wL9aCz6vpuNoquuC11XRBrRbPzxfVrcDOzwEPf0UGIZMHw6tpa+ueSvTq63vou3Mk0QCoddKAdV9iZYdaR2WYXgc1WRUCnwuWRj2sr56B7Z0L6DvQi6Hhh7AO7IdFE6dJtCjP/00zAF6f5Ej43eThlYvnud1Qqclbru2kz8I1RrsCj4OuKUNdYecBx+Nz0e7c8hw1l+bDkdFWAD0D5HgIH9M6UdVctOttcxoAeepP2+HUipOcjx2ihqbGXqCjb+M5BZ6nxbycqxrqukgSYOoqCYZloroltmjK6GsoP+AcSZ0VDFDeIrm0T6UGXKOFMwBLkxTCMbdGhekG+g7CYhSBYJxRXZqghSO+IkpuEEuWf44EUgfFK5QGvZHUTVv2k4O28IjCKCxS2HstEiZZkaoWUlYtxG6wthM48OvAxGVpN9BYlNLYTZJmjyN+8c93WV7YT80q3c+SVd5qCeV0tB2h99HrzPw7WiOFoSJJ+i71PbTohpN0/1V6MqLJGOqA+Xupx8kHokghJkMthMkZDA0/JGG62w9IH0cOGcjx/+TpX2E/GbX4xDoTQQ1LBTJYChuD58kpsTwLHPkNoOcMfb5LkxSekSUocoUxivV75oCOUxQ2LWQoUGOgdaf3ZYoWLM+uX6NpnWw+A1CoxT/ko/BE78s0HWu7bPl5ngxe2J95YeZ58oKDK4kzAup7ACZiduRO4owAjYFyIcmCYCoN5VUKMbTcuwAEViDMLEYreM6cOgzrC0dg++Q+hNkl+r3lGVogUvT//UBlkvxzYIXyMGVUt62QhFpHZaT7vgQc/BrtEPzLNBPZ69y4EKF/icKeVS3kmbcdLF6oqa6LXku9harngisFe6rNZQA4npq88u75B2jx3/2FVM9wO2CoA8wnAc9s5t+pagbESOKMgJp2RHgtZp/cSpwRwPPkufjdqcfRVpKnnm8cjwG1FvY5R6x8E4ClTgvrs3tgd0jnkkn/n7HU3oGgB6jfkf9zVSgMhjqg/TjtCna/QmFNzzwZA88COR+rEQ5QgYDbTnm5PV8Edr1UmAKTtZB3A3tfpXvJPVUQ+e7NlQPQGPI/2zMSpg9950uptePbieY9lFzLNBheWwVo9LA0a2KyCbvb0eapwj7jCtStSTcJpyKvPLnsU19DW/XAcv5ituEgMP8QqGxA/7GkcM3KNFX0yN3Ri+OUtI4P70XHPyZ3SbPC5SsUCgevop1bTTuNxPTMUQmya5SuSY6LyaUAIJlTRtd98z4quS7xxLcoRhOw/6uUF5i8AioZbcybzM3mMgDIc1iGMWBlmgadN6xDRXErotJQxdPj94CaNAaA48ijcgox2YRbo/j63h6o/VcodxK/M9NW0dCRhqQKGo4DwAOOEZIDyAfuKQo3Jd8U4QCFceS4rRghb7D1QOrv6WoS/z4cpAUh3Uxghc2DWkvXbY2ZZiSHfFI1np+uGU5FDYKycF05hn5VGrpm63to9zxzGwCjBrRkxdt1srlCQPlmZRZo3E1VBQpAbRd1JmYaDF/RBDAxJptwqBsfjjLyo5wjib+rUtMimi5+WVlPQmD52NIyBkzfTGk8G7x2B8KTx/Qf6aa2jz6i5rrkqp6wn153PIEluuHKcUFQ2DgaA3nQNWbq66ntoISyrqr8P2ttBVXtHX6dQrYhL4WrPAsbvpe2rwHwL5F31/l0+X/wxYLnKYaaLnYPALoqCPM+2D66CevAAZw52oNXTx/FdLAS/tmHaY7H0US2ZFRaMg6yHk8ueObpKyl8Y2pugO3tcxDmPQBI6+fx3RsAgEvTXCyRDQCMQXAFMHjtTuxnkdDqpbEKCqVCY6AdwaHXqTKpYSc5be5pySDM0//DASkJzjIeansagEiItoE7zigVHskY2ynpFfSkPsZxsAf0sJ6yRFVALW11MLTuhs4/T4qb8agrqIszHYZawH49cUDLRpi9m7Ze2mJqhrXPBNunIzh3fYRyFi0RoLIJzU1NsUQ2E2mk5YfXYWqWGtqYJKVd7CE2Cgrrgecp7Nr1NHDkm8CBrwI7XyQVAV0NrXOeeSCwgghD2nKoTZYDyBPLM4DldPkkesoJeRfw+IO0fRD9Tz9FzSpx1HXtB5zXANcI0BIXX9dI3b/JXcEAHXtpkuL3G/W0Zb2YdFO6giuwNFejb08Hzt8axfMHO2DwXgFaDyXq/+9sxNDjBVi/8XpM/TPokUS6couvKigUDY4jp0qWhY9HFOEO/E7aWtLttwPwOGjBadxd6jMpX2o7Kd6Y3OAFkFwFr06s8a9qoYSpM81Aco4jcbh06GuBiaGN12rP3KIEWTrRPa8Twpw7mquwjz6kc5bKfKOJ7Bsj6Du8L1H/P7iSqHKqoLCZWUWyYnsZgEiQkoDdzyqjIleDV5EIV7ruYHnbGR/u4ThKmLqE1M5FjSFzGEhXRWV5S+PpH18NnwuYu58xTCM8vAPbZ6PRXMUrvXowBoz5qfQ0msg+YMbQo1kIk3Gdy4zlv9xYQaEM2V6r4MocJX2VwR5rI1fAiJGUhwYfLUKwOxJ+Nsu10I7BnbTYq/WSiFaa3QRApWzjF9du0klm6jqFaNJ5/0Ev7LPzsJ45HM1V1EXmEdQ3YHIxEKf/vw9nDndGxeOEyZmY6qmi/qmwDSiJAeA4zspx3B2O40SO404U5Ul9LvJcG3uL8nSbHm0FvVe+1F2AqaMDtsFHEKbICAjTLvzz9SUwcDQjIB5OarLxZQgDaStoNzFzK/tzc43RTNmKDCqwPhf697XD0laHweExjE7NA247dI0W9B/owpWHU7C01cHSVAHoamDpNMN69jTscw6qnqhpL17bv4JCCSnVDmAYwFcBfFyUZxMjVDPb1a+EftZD0+609cWWTjOsn38KtnM3ohU2X37uCLiaDmrwSkatT90ZxFPVTBVBK3Nrn1PIR7K5lQ2Zy3dXZqKS2qYGIy5eukSKkbWdEKZdEKZdONHbnlD/bzG3ov/Yfjp+bffa56GgsAUoyWrIGLvHGHtQtCdcmaO5AUrVz/qoaKBeiZA35SHLrv3o29mA87dG0bebvG3UWyimn9xHoDGQ1lKmMA+vooTwo3cz9yAAZIyE85RnyCSVGw5S7kJS/7S01eHlXj1EBnw8FqbQjzzgnjGgIkn/R261V1DYBpS9O8xx3Bscx13hOO7K/GKGDtXVCHlpsWg5mP+T2+pwHDWc+JZSHhIcfgw9msXAoW4M/f/t3WtwnOd12PH/WdyI+2VxJ0BieZNIQqJFE7IUxqRpK5asOlLSFB1n4jpuPLHGnuZDL2mT5kNv05lm3OZDJunY6iRx7nGQiVU5dqRaFkMpqJmCoiUZkESR0pIUABKEcCGI+y726Yez2PsCILDYJXbPb4ZDYPfF4nkJ4j37nud5zrk0omvqveHCaYm7gsWDpoHWKDldVgUIXP5+6iCwEoCrfw/TI2v3gFhNNcXMDdQHbzFbVM/LPx6JBqvVyerSmA1kKwENHIntLI3JU9sWAETkJREZTPHn6bt5Hefcs865E865E011d1k8zDld9tl10jZ8bVbdXsDFre7xD9+k76X/R++Zj3Cme3dkTb1/WvTimRgAAIrLdd3/WioadLJ46K+1uNvSrKZkbo/A0HO6yijVmv9YM6Px6/eDS7iZUYZul8cHq+CS3nXE1v9ZuqPLRG1nuCkQ27YRzDn32Ha99obNT2htbevotHmlFRoE5sYjZXEjJZcrlmDycmRj1ejEDL6GfXBzMLk4W0l5tKXeWv1cK+r1mKuv6A528QAh3dlYvc7FP7isKaiY8r1j/rdowdF1/0O07d9HV2u9poE+1onvSML6g+BCcploY/JY/u4EDgV13X/nx+wd3VY13QfTVyMX1pPHw8XzFu/AhJZy8LXVa2plckUndKc/iN9MtfozmJvQVTZrKd6lJSnuVor0T2jST0hKaPPdFxln7+luRkeG8SXW/0fSrywyJg/lahnoz4rIMPAo8F0ReTHj32R2XCd+c9HMId9Ut2rZ3MQ9AaWV4CmJ3xVc26nv/FOlgUoqNrfpa6NmRpLKN7TJh3jq98Qt6/S11nLyyO74AnLBJT2fe7mHszEZlqtVQN92znU458qccy3Ouccz+g0C81qHpqV7/WPN+opKtK7/4nT846u9WWN3BReVaB598r3kQm8lu/SuIV256a0ILGhZ3NhmNgtTWvW1oSv+2HT5//q9mR+XMfewe34V0KbMT+iaf5v4zRzvfgik2M1b2ZTc+7dhn154U23+8hSlbhi/VXPjWn46Nt03Fa5NVJ9Q1ydV/f+V5c2lnYzZwfIvAMxPQk1HePWKyZiqZn13n9g4frVjVmwNoIZ9+vfEleTXKavSvrwpSkxsmnPa6rEkoaXj5FVdlZRY1iFV/1+w/L8pOPkVAFZ3/O55xCZ+M81TBI0HktNAxaV6gY2t9bOrVt9hxwSA/sFruvzSU6xBZGES//DN+CYsm7V0R1NAsXd8oRWdb0h8978aqOLy/4ua+09R/tqYfJZfAWBuHNo+Yjt+t0u9D1ZSpIGqW/UCHKvxgE7KhhvLtHtrok1YSsvxD12k74VXo01YtmJmJLl2z8yobuxKrI2eNv+fcJwxBSB/AkBgMdw82Xb8bpvKpuRVP5B6pZX3oP49ocXhYpuwnB28Sd8PLtB75qH4OvybEVzSDWalCat3pvy6HDRc/z96fKr8f2D9PQbG5KH8CQBz4zrxW7LGJiOzNUXF0LAfFhLSQCWVGnxjA0Nlk27emrgceSjShOXNq/Tc14GvKk2J6LsxcxOE5CJ/U1ehuj25q1fK+j9Y/t8UpPwIAPOT2uXLbuO3X0OKNJCIvoNObBLTeEBLN4eLwEWasDzYxcD7k/jffTs5dXQ3Qiv6Tr80ob/D8hzMjiUv/3QhHVds/Z/Aok5kl1ZgTKHZ+QEgFNTb+r2P2sRvNlQ1hzeFJXT+qmhMTg15D2oZ5il/tAlLuENX7yce0J4CQ69tfix3xjQYFSVsaF9tTVm/L/7x4JJOWMfm/5ct/28K184PALO3oP24dXDKlqIS3e27lFCxc1eNpmJil4PW7tYicBOXGZ2YiZZhJjwn8MmPMnrtSmSi+K6EgjDxburubhNXdE4gsWpocFFTU7FWgusXmDMmT+3sALA8q8v5Wm3Hb1Z590Mg4aJdVKLtHWOXg4pHj514n5NHOiIX/1W+3V5OHt2Tupn8eu7c0M1bRQmb/VYCmv/37k++I3ROVwDFP6jjNqYA7dwA4EI6Gdl1Si8+JntWG6YnlnqoSrEc1HtA0zS3P0j9Wruqtdxzqgb06QQWtFx0WXQyN7LPYPo6hALgPYD/xhT9g9f0gJDl/41JtHMDwOw4NB+22/dcKK3QIJCYuimv1RLOseq7NOf+4WVSEg+UVcLYYPIu41Scg/FLgMTl/lf3Gcx88BYUleBfqKbv3CDt3nCKKLgI5d741ULLVv/HFLadGQAC8/quf3d2+smbFLwHk4u6lVTqbtzYC3lRia4cmrgcPz8Q93Xlmjr68FLy5HKimRu6wich9+9rq6f31BE8U+9xS5rpe/XtuDkHgovJcwIrAahp38DJGpOfdl4AcCGtKe87ZWv+c6m6JS4F1H9xCP/IWHhXsN4ZRFIwTYf1bmE6TRoIdBJ/ejhawC2VuXEY+7FuPEux4stXMU9VUYBzo6XR1o+xEieMneX/TWHbeQFgdhxajljnplzbVaepoPCkb3uzl74XXsU/7SC0Eln22e6t0eJwRSUw/k761xPREh4fXoGxIX13vioU0jmEkYt6Efek7mM0c+1NAk5o2fdgtPUj6H4BKdI7lFWBRa1ZZPl/U8B2WEcwp+/6O3pyPRAjommgW0NQ1Yyvo5XeJz5O3/fO0eOdY+DqbXpPPxB9F+49AB++CwceS67bs8rjgcoGXeFzZwyqmvTY+Um9gyivS3vx949O0DR5meXavZw6fojOtqbIvgNfQylUNibn/5vuz/A/ijE7y866A/AUw/5PJm/vN7lR16ErbsJ8Ha30PHg/5y5N0LO/KT4F03RY8/BTV9d8yf6hD/BPo8t75ydh9hb+sRn6/XNpL/4As2N+qooCVHZoLajYPsWsLKXO/1v9H1PgdlYA2FWb/ItscqeiUS/K4Ylb//BNBgbf5fQjJxh453o0BQO6GqikXNM7a4hUDR2bgbIq/JNB+v7+3ehqnjQeqJrUsXiju399bfWc7A6v8ilL8fVW/8cUuJ2VApKdFa/yXlGxNt+ZG8c/sUjfC6/S+8TH8TVX01UyEU3BtNVrKqf5iDaMD8zHt26MEVs1tOe+3QxcGolfzZNKKKjzC40HU28MKy7T4LMqaOv/jYGddgdg7j0N+yAwz+itCb34d7RCaSW+zlZ6f/KQpmBWtXTrKq5bb6/5kvFVQ1Os5kk0cUUno1P1gA4s6J6F2FVD1v/XGMACgNmqqiZwjpPHj8bX9q9uxecti6ZgQNN3VS1wc3DNl4yrGhq7miedsSGt/ZNY+x/07iAm1dN/cQj/9dG4/L/f76e/v3/t72FMHrIAYLamrFpX5ySWgKhoTL3xq/UBmLulHbtSWF0+6murp6s1mg7y35iKL+2wamlWawm1HE1OEa7uU4jJ/7c3e+n7wWv4x3QTm9/vp6+vj/Z22xBmCo8FALN13gOwmFAdtKwmXDY6ofl781HN049cTPlSq1VDTxzaTd85vVPoPd3NhXdH4ks7rLrxBuBSFwQMhtf6x/QK9rXU0fuZT9D3v/+Gs2fP0tfXR29vLz6fL/nrjclzFgDM1tW0a93/WB6PpnwC8/GPF5fqXcCHl/Tde4KT3XvxtdXHTQZfvanv/pMmg0NBuPG6dilLVQ48sKA7k2Mt38H3wMP09PRw7tw5enp67OJvCpYFALN1Fd7wctCEIFDdEr+jd1X7cU0P3Xh9zZdddzJ4/JIGmN3H079IYmBYCeCfDDAwMMDp06cZGBjA799EOWpj8oAFALN1nqJwk5hocbj+i0P4xxf1k/BcQCSHX16naaPRH8X3D0iw5mSwczDyGlR46R8maaLYPzJO/ztj8eUfwo/3ffdlent7OXPmDL29vfT19VkQMAXJAoDJjAZfXJOY9mYvfS+dx39bILgYXxsIYM9PaI5+JHVLyKQWkjGTwYAu/Zwdg44e2htr457z35ii7+wbtHfujS//EFhg9M4KvT//C5G0j8/no7e3l9HR1JPSxuQzcYlNPe5hJ06ccBcuXMj1MEwqS7Pw5l9Aze7IQ/7hm/Q99z162mDg6kxyDn/o21oh9OEvJ1V27R+8Rru3Ju54/40pRidmtIvYxW9qyunEL4F4IgEjsnns4Q58xz+hNYBWzd7SSehOKyNuCouIvOacS/qPb3cAJjPKqrRCaMxyUF9HKz0PdXNu8AN67mtPzuHvPal1eq7/36SXW50MjhUp7TD+Dsx9qF8fXvoZP1/Qjq+tVlcAxQpZ/19jYlkAMJnTsD9uOah/+CYDb1/j9EP3MfD2taQ8ff/VBWZq7tcloXdu6tekWusfK7AI75+Fyua4ap5x8wVvXcM/LfGtQlfvdK3+jzERFgBM5tTujqwE8g/fjNQGOnP6JL2P+OJz+Gjhtz+4VEWwqBwuv4h/dCL1Wv9Y778My/Nw3xOR8g5J8wWP+ug7fx3/8M3o1wUW9OJvTYSMibAAYDKnwqsF4kIr8bWByhvwtdREyzOH+drqeerUR/jOeAfM3uLOG9+h99TR9LV/xob0T+fHoo3piW4eW/06X0sNvf/ok4zemoh+7dIM1Nt6f2Ni7axqoObetrocdHaMk8ePRh8vrYSyanxNLmVe/+reBzh3/Q6n62/A8lvgTia3fJx4Dy79rdb72fto3FNx9YYCS/q99nYRt7/Lrei+BGNMRE7uAETkayLyjoi8KSLfFpG6XIzDbIOE5aARdR3a1SvBau4+1PkoP55vhOs/hKG/1kle5zTd895ZXTFU3QJHfnbNxjAE5jQVFcuFAI/1/zUmQa7uAL4P/LpzLigivwn8OvDvcjQWk0mVTUkP9V8cor2uAl/MkuO+c4PcmpplbjEQSd+88rqHbw29Qq9cwzP5B6xQTBFB/YK2Y+A7jX98jhcGfsQTPYdSLxH1VSVP9C7Pa8qoOKFXgDEFLid3AM65/+OcC/9mcx7oyMU4zDYoq9J32jE1gNqbvfS9fAH/VAgCC/hvTPHapRF++NYHPHK4E19bPf4bU5x/Z5i27tO8VvsZOPhp5moPcvb2HkY6fwYOfhr/+Dx95wY5urcleePXuUHaa8u0Omlp/O5flmf1zsQYE+demAP4JeBb6Z4UkS8DXwbYsydFvXdz7/Ee0B2+4a5fkYbxz32XnjZh4OoMz/z0w4DeCayEQik7f9W0HaOreYo/PTdIz3RR3DGdzbXJXcNqSU7/gKaSrJWoMUm27Q5ARF4SkcEUf56OOeY3gCDwp+lexzn3rHPuhHPuRFNTcnrB3IOq25J6Afg6WimvruO581cim8J8bfXqnBGHAAAKoklEQVR0NNXwxy+9nrbzV7qCcKkfDyWnf0JBnTNIVS3UmAK3bXcAzrnH1npeRL4IfBb4lNtJ9SjM+ioawFMSvfii+wJGxm+zGCrhxfNv09XawAe3bvMnL73B5x87xsClEbpa65OCQGJBuNVj+s4N8tqlER7vOaiPN5aDp4TRO1fjVyAtzUJth65QMsbEyUkKSESeAP4tcNo5N7/e8WaH8RRpz93bw1DRENkU9sznnoT5Kb7xzT/n3//e97k5eYf/9Iuf4tSxrrjNXKtBIPGxrla98D9yuJPXLo2ACF2t+vg3vv0KVDbxzBeOxY8lsGD9f41JI1cbwX4HqAa+LyKvi8jXczQOs10afJG6QLGbwnwHDvH4if0Eg0FOH+vi1LEugEgDmNiNYkkbvMLHDF0b45mffphnPtujDWNuTALw0WPd8X2JQVNRsQXhjDERObkDcM4dyMX3NVlU2aybucIN41f5R8cZGF7gq5/pZsB/G/+NqbgLfGwKKG6DV5ivrZ6vPPWxyOc99+3m3MVLPP6TxznzmTPxBwcXw0XqEorCGWMAKwVhtkvJLp0MXo62fYzUB/rs45x5YDe9p44k1Qe6G5H5gSNtDFydia/9A1qYrsHeaxiTjgUAs30a9scFgEgqaJ8PKpvxeUvj0j79g8kVQ9NVB43MD5w8yJkTh+n97E/R98Kr8UEgFITa9u05N2PygAUAs31qWiEUXQ568vjRaI6+fi8El6M1/tHqoCk3eKWoDhqZH/CWgHcfvs42ep/4eLQAnAsBAhWW/zcmnXthI5jJV7tqYVeN1vBPLMO8q0537QYWoKQciE7yJm3wSrE/4GT3Xn2HvzwXmeT1dbRGA8zyLNS0W/kHY9ZgdwBme3kPwuLt5MdFoPFgUoG4dBu/Ulqcgfp9qYvDLc1Bw74tDt6Y/GYBwGyvus5Ik5gkFV69S4ipG5S48SvtBPFKMFx+OkXpBwAcVLemec4YAxYAzHYrb9D0z8py8nMi0HgIlubBueTOXuF0UMogsHRb7yBi2z6uCixAWY2mn4wxaVkAMNvL49GL/MJ06ufL67VR++JM2o1fsZvDAL3AF4eXmaayOKPf0xizJpsENtuvbg/cfDP1c6tzAXPjnDy8W1tKxkjcHIYLaX5/z8PpG8O40BqpIWPMKrsDMNuvslFTNSvB1M+XlEPzEViYils2mtL8lC4hTVfdcyWgQcS6fxmzLgsAZvt5ijQls7jGjt+aNvDuh/nJpFLSEQvTUOmFxjV29y5O6/fy2H9tY9ZjvyUmO+p9EAysfUzDfqjfA3OTEIyZNA6twPyE7htofXDtnsArAajvysiQjcl3NgdgsqOyCYrLdDVQUZrNWR4PNB/W9M34O9EyEuIB7yFdUrpWXf+VoKaaUvQlNsYkswBgssPjgab74dbQ+u0Zq1v0mMCctnMsKV/7Xf+qxSndeGbNX4zZEEsBmexp6NLyDRshAqVVmvbZyMUfNMVkzd+N2TALACZ7Kry6QSvcKCajVpY1xWTpH2M2zAKAyR4RaO3WlTqZNj8FLYct/WPMXbAAYLKrbo/m9dMt9dysUFBXGhljNswCgMmu0kqo7dRyDZmyPKf7Ayps85cxd8MCgMm+lqNJZaC3ZHEamrsz93rGFAgLACb7qtu0WXsmJoNXgiDFuoHMGHNXLACY7PN4oO0hrf2zVfMTekdRXLb11zKmwFgAMLlRvxekKH2BuI1wDkIBrSZqjLlrFgBMbpTsgrZjMP/h5l9jYVLr/pTXZWxYxhQSCwAmd5rvB0QLuN0tF9I5hPbjGR+WMYVCnHO5HsOGicg4cG2LL9MIbOFt546xI86zsoTyqlKpDITcXUWBYo8Uzyy58sUgw9s1tnvIjvhZZoCd5/bZ65xL2ia/owJAJojIBefciVyPY7sVwnkWwjmCnWe+uZfO01JAxhhToCwAGGNMgSrEAPBsrgeQJYVwnoVwjmDnmW/umfMsuDkAY4wxqhDvAIwxxmABwBhjClbeBgAReUJELonIFRH5tRTPl4nIt8LP/4OIdGV/lFuzgXP8VyLyloi8KSI/EJG9uRjnVq13njHH/ZyIOBG5J5bY3a2NnKeI/NPwz3RIRP4s22PMhA38v90jImdF5Efh/7tP5mKcWyEivy8it0RkMM3zIiK/Hf43eFNEcrOj0TmXd3+AIuA9YB9QCrwBHEk45qvA18Mffw74Vq7HvQ3neAaoCH/8lZ12jhs9z/Bx1cArwHngRK7HvU0/z4PAj4D68OfNuR73Np3ns8BXwh8fAa7metybOM9TwHFgMM3zTwJ/CwjwCPAPuRhnvt4BPAxccc6975xbBv4CeDrhmKeBPwx//FfAp0REsjjGrVr3HJ1zZ51z8+FPzwMdWR5jJmzkZwnwX4DfBBazObgM2sh5/jLwu865KQDn3K0sjzETNnKeDqgJf1wLjGZxfBnhnHsFmFzjkKeBP3LqPFAnIm3ZGV1UvgaA3cAHMZ8Phx9LeYxzLgjcBrxZGV1mbOQcY30Jfcex06x7nuHb507n3HezObAM28jP8xBwSET6ReS8iDyRtdFlzkbO8z8CnxeRYeB7wK9kZ2hZdbe/v9uiONvf0GSfiHweOAGczvVYMk1EPMBvAV/M8VCyoRhNA30CvZt7RUQecM5N53RUmffzwDedc/9DRB4F/lhEup3LdCNpk693ACNAZ8znHeHHUh4jIsXoreZEVkaXGRs5R0TkMeA3gKecc0tZGlsmrXee1UA38HcichXNpz6/AyeCN/LzHAaed84FnHN+4F00IOwkGznPLwF/CeCc+yGwCy2glk829Pu73fI1AAwAB0XEJyKl6CTv8wnHPA/8YvjjfwK87MKzMzvEuucoIg8B30Av/jsxXwzrnKdz7rZzrtE51+Wc60LnOp5yzl3IzXA3bSP/Z59D3/0jIo1oSuj9bA4yAzZynteBTwGIyGE0AIxndZTb73ngC+HVQI8At51zN7I9iLxMATnngiLyL4AX0VUHv++cGxKR/wxccM49D/weemt5BZ2s+VzuRnz3NniOXwOqgL7w/PZ159xTORv0JmzwPHe8DZ7ni8CnReQtYAX4VefcTrpr3eh5/mvgf4nIv0QnhL+4w96cISJ/jgbrxvBcxn8ASgCcc19H5zaeBK4A88A/z8k4d9i/qzHGmAzJ1xSQMcaYdVgAMMaYAmUBwBhjCpQFAGOMKVAWAIwxpkBZADAmDRGpE5Gvhj9uF5G/yvWYjMkkWwZqTBrhEuF/45zrzvFQjNkWebkRzJgM+W/AfhF5HbgMHHbOdYvIF4GfASrRUgz/HS1t/M+AJeBJ59ykiOwHfhdoQjf7/LJz7p3sn4YxqVkKyJj0fg14zzn3EeBXE57rBv4x0AP8V2DeOfcQ8EPgC+FjngV+xTn3UeDfAP8zK6M2ZoPsDsCYzTnrnLsD3BGR28B3wo//GHhQRKqAnyBahgOgLPvDNCY9CwDGbE5sZdVQzOch9PfKA0yH7x6MuSdZCsiY9O6g5abvmnNuBvCLSC9EesAey+TgjNkqCwDGpBGutNkfbuz9tU28xC8AXxKRN4AhUreyNCZnbBmoMcYUKLsDMMaYAmUBwBhjCpQFAGOMKVAWAIwxpkBZADDGmAJlAcAYYwqUBQBjjClQ/x+As7Ag/zT/GgAAAABJRU5ErkJggg==" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "metadata": {} + "ax.set_xlabel(\"time\")\n", + "ax.set_ylabel(\"acc\")\n" + ] }, { "cell_type": "markdown", + "id": "be5c1a4e", + "metadata": {}, "source": [ - "The errorbars of the single layer model are not good: we observe an overestimation of the error bars on the left and right. " - ], - "metadata": {} + "The errorbars of the single layer model are not good: we observe an overestimation of the error bars on the left and right." + ] }, { "cell_type": "markdown", + "id": "77cb254c", + "metadata": {}, "source": [ "## Deep Gaussian process with latent variables\n", "\n", - "To tackle the problem we suggest a Deep Gaussian process with a latent variable in the first layer. The latent variable will be able to capture the \n", - "heteroscedasticity, while the two-layered deep GP is able to model the sharp transitions. \n", + "To tackle the problem we suggest a Deep Gaussian process with a latent variable in the first layer. The latent variable will be able to capture the\n", + "heteroscedasticity, while the two-layered deep GP is able to model the sharp transitions.\n", "\n", - "Note that a GPflux Deep Gaussian process by itself (i.e. without the latent variable layer) is not able to capture the heteroscedasticity of this dataset. This is a consequence of the noise-less hidden layers and the doubly-stochastic variational inference training procedure, as forumated in . On the contrary, the original deep GP suggested by Damianou and Lawrence , using a different variational approximation for training, can model this dataset without a latent variable, as shown in [this blogpost](https://inverseprobability.com/talks/notes/deep-gps.html). " - ], - "metadata": {} + "Note that a GPflux Deep Gaussian process by itself (i.e. without the latent variable layer) is not able to capture the heteroscedasticity of this dataset. This is a consequence of the noise-less hidden layers and the doubly-stochastic variational inference training procedure, as forumated in . On the contrary, the original deep GP suggested by Damianou and Lawrence , using a different variational approximation for training, can model this dataset without a latent variable, as shown in [this blogpost](https://inverseprobability.com/talks/notes/deep-gps.html)." + ] }, { "cell_type": "markdown", + "id": "83ce6ea5", + "metadata": {}, "source": [ "### Latent Variable Layer\n", "\n", "This layer concatenates the inputs with a latent variable. See Dutordoir, Salimbeni et al. Conditional Density with Gaussian processes (2018) for full details. We choose a one-dimensional input and a full parameterisation for the latent variables. This means that we do not need to train a recognition network, which is useful for fitting but can only be done in the case of small datasets, as is the case here." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, + "id": "8b6cf7b2", + "metadata": {}, + "outputs": [], "source": [ "w_dim = 1\n", "prior_means = np.zeros(w_dim)\n", @@ -257,32 +207,34 @@ "encoder = gpflux.encoders.DirectlyParameterizedNormalDiag(num_data, w_dim)\n", "prior = tfp.distributions.MultivariateNormalDiag(prior_means, prior_std)\n", "lv = gpflux.layers.LatentVariableLayer(prior, encoder)" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "121b7798", + "metadata": {}, "source": [ "### First GP layer\n", "\n", "GP Layer with two dimensional input because it acts on the inputs and the one-dimensional latent variable. We use a Squared Exponential kernel, a zero mean function, and inducing points, whose pseudo input locations are carefully chosen." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, + "id": "3eb1860e", + "metadata": {}, + "outputs": [], "source": [ "\n", - "kernel = gpflow.kernels.SquaredExponential(lengthscales=[.05, .2], variance=1.)\n", + "kernel = gpflow.kernels.SquaredExponential(lengthscales=[0.05, 0.2], variance=1.0)\n", "inducing_variable = gpflow.inducing_variables.InducingPoints(\n", " np.concatenate(\n", " [\n", " np.linspace(X.min(), X.max(), NUM_INDUCING).reshape(-1, 1),\n", " np.random.randn(NUM_INDUCING, 1),\n", " ],\n", - " axis=1\n", + " axis=1,\n", " )\n", ")\n", "gp_layer = gpflux.layers.GPLayer(\n", @@ -292,22 +244,24 @@ " num_latent_gps=1,\n", " mean_function=gpflow.mean_functions.Zero(),\n", ")" - ], - "outputs": [], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "b93fe2e1", + "metadata": {}, "source": [ "### Second GP layer\n", "\n", "Final layer GP with Squared Exponential kernel" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, + "id": "226ef693", + "metadata": {}, + "outputs": [], "source": [ "\n", "kernel = gpflow.kernels.SquaredExponential()\n", @@ -321,151 +275,82 @@ " num_latent_gps=1,\n", " mean_function=gpflow.mean_functions.Identity(),\n", ")\n", - "gp_layer2.q_sqrt.assign(gp_layer.q_sqrt * 1e-5);" - ], - "outputs": [], - "metadata": {} + "gp_layer2.q_sqrt.assign(gp_layer.q_sqrt * 1e-5)" + ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, + "id": "1c383ad3", + "metadata": {}, + "outputs": [], "source": [ "\n", "likelihood_layer = gpflux.layers.LikelihoodLayer(gpflow.likelihoods.Gaussian(0.01))\n", "gpflow.set_trainable(likelihood_layer, False)\n", "dgp = gpflux.models.DeepGP([lv, gp_layer, gp_layer2], likelihood_layer)\n", "gpflow.utilities.print_summary(dgp, fmt=\"notebook\")" - ], - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "" - ], - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
name class transform prior trainable shape dtype value
DeepGP.f_layers[0]._layers[0].means\n", - "DeepGP.f_layers[0].encoder.means ParameterIdentity True (94, 1) float64[[2.01673752e-02...
DeepGP.f_layers[0]._layers[0].stds\n", - "DeepGP.f_layers[0].encoder.stds ParameterSoftplus True (94, 1) float64[[1.e-05...
DeepGP.f_layers[1].kernel.variance ParameterSoftplus True () float641.0
DeepGP.f_layers[1].kernel.lengthscales ParameterSoftplus True (2,) float64[0.05 0.2 ]
DeepGP.f_layers[1].inducing_variable.Z ParameterIdentity True (20, 2) float64[[0.04166667, 0.66191201...
DeepGP.f_layers[1].q_mu ParameterIdentity True (20, 1) float64[[0....
DeepGP.f_layers[1].q_sqrt ParameterFillTriangular True (1, 20, 20)float64[[[1., 0., 0....
DeepGP.f_layers[2].kernel.variance ParameterSoftplus True () float641.0
DeepGP.f_layers[2].kernel.lengthscales ParameterSoftplus True () float641.0
DeepGP.f_layers[2].inducing_variable.Z ParameterIdentity True (20, 1) float64[[2.48779388...
DeepGP.f_layers[2].q_mu ParameterIdentity True (20, 1) float64[[0....
DeepGP.f_layers[2].q_sqrt ParameterFillTriangular True (1, 20, 20)float64[[[1.e-05, 0.e+00, 0.e+00...
DeepGP.likelihood_layer.likelihood.varianceParameterSoftplus + Shift False () float640.009999999999999998
" - ] - }, - "metadata": {} - } - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "36132b7d", + "metadata": {}, "source": [ "### Fit\n", "\n", "We can now fit the model. Because of the `DirectlyParameterizedEncoder` it is important to set the batch size to the number of datapoints and turn off shuffle. This is so that we use the associated latent variable for each datapoint. If we would use an amortized encoder network this would not be necessary." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, + "id": "9f21ce5a", + "metadata": {}, + "outputs": [], "source": [ "model = dgp.as_training_model()\n", "model.compile(tf.optimizers.Adam(0.005))\n", - "history = model.fit({\"inputs\": X, \"targets\": Y}, epochs=int(20e3), verbose=0, batch_size=num_data, shuffle=False)" - ], - "outputs": [], - "metadata": {} + "history = model.fit(\n", + " {\"inputs\": X, \"targets\": Y}, epochs=int(20e3), verbose=0, batch_size=num_data, shuffle=False\n", + ")" + ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, + "id": "128f730a", + "metadata": {}, + "outputs": [], "source": [ "gpflow.utilities.print_summary(dgp, fmt=\"notebook\")" - ], - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "" - ], - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
name class transform prior trainable shape dtype value
DeepGP.f_layers[0]._metrics[0]._non_trainable_weights[0]\n", - "DeepGP.f_layers[0]._metrics[0].total ResourceVariable False () float641.0780249257681602
DeepGP.f_layers[0]._metrics[0]._non_trainable_weights[1]\n", - "DeepGP.f_layers[0]._metrics[0].count ResourceVariable False () float641.0
DeepGP.f_layers[0]._layers[0].means\n", - "DeepGP.f_layers[0].encoder.means Parameter Identity True (94, 1) float64[[0.03641934...
DeepGP.f_layers[0]._layers[0].stds\n", - "DeepGP.f_layers[0].encoder.stds Parameter Softplus True (94, 1) float64[[0.8204074...
DeepGP.f_layers[1]._metrics[0]._non_trainable_weights[0]\n", - "DeepGP.f_layers[1]._metrics[0].total ResourceVariable False () float640.4804222860958632
DeepGP.f_layers[1]._metrics[0]._non_trainable_weights[1]\n", - "DeepGP.f_layers[1]._metrics[0].count ResourceVariable False () float641.0
DeepGP.f_layers[1].kernel.variance Parameter Softplus True () float640.8957803239319818
DeepGP.f_layers[1].kernel.lengthscales Parameter Softplus True (2,) float64[0.13181667 4.24491404]
DeepGP.f_layers[1].inducing_variable.Z Parameter Identity True (20, 2) float64[[0.10892382, -0.15883231...
DeepGP.f_layers[1].q_mu Parameter Identity True (20, 1) float64[[3.44379760e-02...
DeepGP.f_layers[1].q_sqrt Parameter FillTriangular True (1, 20, 20)float64[[[8.69748551e-02, 0.00000000e+00, 0.00000000e+00...
DeepGP.f_layers[2]._metrics[0]._non_trainable_weights[0]\n", - "DeepGP.f_layers[2]._metrics[0].total ResourceVariable False () float640.14355680909786334
DeepGP.f_layers[2]._metrics[0]._non_trainable_weights[1]\n", - "DeepGP.f_layers[2]._metrics[0].count ResourceVariable False () float641.0
DeepGP.f_layers[2].kernel.variance Parameter Softplus True () float640.10763915659754926
DeepGP.f_layers[2].kernel.lengthscales Parameter Softplus True () float640.7436233151733747
DeepGP.f_layers[2].inducing_variable.Z Parameter Identity True (20, 1) float64[[3.28383233e+00...
DeepGP.f_layers[2].q_mu Parameter Identity True (20, 1) float64[[-0.25372763...
DeepGP.f_layers[2].q_sqrt Parameter FillTriangular True (1, 20, 20)float64[[[9.55995241e-01, 0.00000000e+00, 0.00000000e+00...
DeepGP.likelihood_layer.likelihood.varianceParameter Softplus + Shift False () float640.009999999999999998
" - ] - }, - "metadata": {} - } - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "19f6a9f1", + "metadata": {}, "source": [ "### Prediction and plotting code" - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, + "id": "d6359dee", + "metadata": {}, + "outputs": [], "source": [ "Xs = np.linspace(X.min() - X_MARGIN, X.max() + X_MARGIN, num_data_test).reshape(-1, 1)\n", "\n", + "\n", "def predict_y_samples(prediction_model, Xs, num_samples=25):\n", " samples = []\n", " for i in tqdm(range(num_samples)):\n", " out = prediction_model(Xs)\n", - " s = out.y_mean + out.y_var ** .5 * tf.random.normal(tf.shape(out.y_mean), dtype=out.y_mean.dtype)\n", + " s = out.y_mean + out.y_var ** 0.5 * tf.random.normal(\n", + " tf.shape(out.y_mean), dtype=out.y_mean.dtype\n", + " )\n", " samples.append(s)\n", " return tf.concat(samples, axis=1)\n", "\n", @@ -473,10 +358,10 @@ "def plot_samples(ax, N_samples=25):\n", " samples = predict_y_samples(dgp.as_prediction_model(), Xs, N_samples).numpy().T\n", " Xs_tiled = np.tile(Xs, [N_samples, 1])\n", - " ax.scatter(Xs_tiled.flatten(), samples.flatten(), marker='.', alpha=0.2, color='C0')\n", + " ax.scatter(Xs_tiled.flatten(), samples.flatten(), marker=\".\", alpha=0.2, color=\"C0\")\n", " ax.set_ylim(-2.5, 2.5)\n", " ax.set_xlim(min(Xs), max(Xs))\n", - " ax.scatter(X, Y, marker='.', color='C1')\n", + " ax.scatter(X, Y, marker=\".\", color=\"C1\")\n", "\n", "\n", "def plot_latent_variables(ax):\n", @@ -484,58 +369,46 @@ " if isinstance(l, gpflux.layers.LatentVariableLayer):\n", " m = l.encoder.means.numpy()\n", " s = l.encoder.stds.numpy()\n", - " ax.errorbar(X.flatten(), m.flatten(), yerr=s.flatten(), fmt='o')\n", - " return\n" - ], - "outputs": [], - "metadata": {} + " ax.errorbar(X.flatten(), m.flatten(), yerr=s.flatten(), fmt=\"o\")\n", + " return" + ] }, { "cell_type": "markdown", - "source": [], - "metadata": {} + "id": "27137aaf", + "metadata": {}, + "source": [] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, + "id": "ac5c63a6", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], "source": [ "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4))\n", "plot_samples(ax1)\n", "plot_latent_variables(ax2)" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "100%|██████████| 25/25 [00:01<00:00, 24.88it/s]\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAAD4CAYAAAAnxwjSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9eYyk6X3f93neu86u7q4+pnt6pnt3rh3uyR0uyV1RR0SFlh3LMmOEsQ07QQRIQIBAgR0aYRDDTgBDRGQ4UGwEkRIpkmxBoaBQBA1LZESRFCnuLpfLnd2d3dnd2Tm7p++uu977ePJHdffO9HTP2TPVPfN8gMVOv1NV/VTVO+/zfX/H9yeklCgUCoVCoVAo7hyt3wtQKBQKhUKh2K8oIaVQKBQKhUJxlyghpVAoFAqFQnGXKCGlUCgUCoVCcZcoIaVQKBQKhUJxlxj9+KXValVOT0/341crFIo+8eMf/3hNSjnS73XcK+r6pVA8etzs+tUXITU9Pc3rr7/ej1+tUCj6hBDiSr/XsBuo65dC8ehxs+uXSu0pFAqFQqFQ3CVKSCkUCoVCoVDcJUpIKRQKhUKhUNwlSkgpFAqFQqFQ3CVKSCkUCoVCoVDcJUpIKfYNXpSw0gnwoqTfS1EoFAqFAuiT/YFCsRNelNANE4q2Qd4yrjv++uU6mYQ4STk2XmakZF/3GIVCoVAoHjRqF1LsGa4VS5qAU9NDAHTDBD9KySQULINX51u4UUrBMjg2XlKCSqFQKBR9Q+0+ij1DN0zIJAwXbGpuyGon5ErNXY9CZYBk3otBSoaLFm/ONXGjlKGCyanpoRvE1E7RLYVCodiJL/zmKwB85Vc+3eeVKPYLandR7BmKtoEmoOaGaKJ37FphNVMtAFCwdOpuDAgmKzncdcG0UypwI7qlxJRCoVAodhu1syj2DHnL4NT00GYUCeBKzd0UVhspvIKtM1vz0JG4UUKcpPhRihd9JKa2Rre2Ci2FQqFQKHYDtbMo9hR56/o03LXCKm8ZeFHC2YU2mQTbNBgvO8zVPS6tuVypuZuRp63RrQ1hplAoFArFbqJ2F8WeZquw2hppypCYhnZD5GlrdEtFoxQKxW6iaqkUGygfKcUD53b9oLZ73NZI02jJ3jHylLcMRkuOElGKmyKEmBJCfEcIcVYI8a4Q4lf7vSZFf/ja6XlOzzb54aU6L33523zt9Hy/l6TYB6gdRvFAud0icC9K+MGHa7hRQsEyeOloFehFpE5OlDefn0k2f1aRJ8VdkgD/WEr5hhCiBPxYCPHnUsqz/V6Y4sHxtdPzfOmrZ4jSDID5ps+XvnoGgF98brKfS1PscdSuo3igbFcEvnH8WiG02gl5f6lNyTG5tNql4Oi4QYJp6GiiJ542aqVUV57iXpBSLgKL63/uCCHeAyYBJaT6yINOnf36Nz/Aj9Prjvlxyq9/8wMlpBQ3Re08igfK1tScJrguQrURXWq4EU0vxotiFpohxZxJx495aqKCnyS8v9TBj1ImKvm76spTHlOK7RBCTAPPAT/ccvyXgV8GOHTo0ANfl+L+s9D07+i4QrHBPe8gQogp4PeBMUACvyWl/I17fV3Fw8nWIvBrI1QLLY+Xz9ewDY235ppcbXh0wwTH1CiYOu/MNekECZ0g5oWZYVY7IQA5S9+sjdoQSBtpv43jWzv/lMeUYitCiCLw/wL/rZSyfe3fSSl/C/gtgFOnTsk+LE9xn5mo5JjfRjRNVHJ9WI1iP7Ebu4eqL1DcEVs78eIk5cJqlzhNKTsmOdPATzIcU0cCi02fH882CNMMUxeYukYlbzGQN5kayjNaslnthPiRx1zdI5OS8ytdjowW0YQgjBMSKShYGi8dHVEeU4obEEKY9ETUH0gpv9rv9SgePF/83HG+9NUz16X3cqbOFz93vI+rUuwH7nn3UPUFintHABLH7NU/+XGCsf7/tU4ESISUrHZClpoBUSYp2AafnBlmtGRz+kqTM/NNmn5MwdR47vAwqZTkTIPFts+lNZexksNcI9kcdqw8phQbCCEE8NvAe1LKf9Xv9Sj6w0Yd1D/547eJ0ozJSo4vfu74tvVRG919UZrx0pe/vePjFI8Gu7qD7FRfsP53qsZAcQO96JBksGARxCnHxsvkLJ2ZaoH/63sXWG4FSClYaofkLZ2JSo4klQwVLEZKNm/NNTk922C+GWBoUOtkTAwGpKlkqe2TpBmWLhACepnnG9OLKhr1yPMS8A+AM0KIN9eP/Q9Syj/t45oeafolVH7xuUn+8LVZYOcid9Xdp9jKru0gN6svAFVjoNiGudconPsOnc4RPig9RZpKxkoOYGGbOk9PVUhlL1qlaWDpGl6cMeO/y4uLH/DO6lN8z5/h3HIHTZOMlfIcGy/w+EiBSt4kzSQ5S2fAMUmByUqOkZIN3JheVDy6SCn/il5YVLEH2OtCRXX3KbayKzuJqi9Q3DFzr8Hv/QL5NOJvayav/9T/zXejab7xzhIFW2e6WqCSMxktW7hhyjNTA7wwM4x34RWOfuMfQxpxUph8OPxrXNIfI04luhCYus5Q0SJK5Q3DjhUKxd5nrwsV1d2n2Mo9O5ur+gLF3eCe+w4yjRAyRaQx2aW/wosShMhY7YSstgMODuUZKTocGytSdkyGizZj9dchjdDI0GTM4fYbhElGJW/xxESJ4+MlvCglTrLNGqiCrXOl5vL+YptvnV1mrRv0++0rFIod2OtCZacuPtXd9+iyGxEpVV+guC02rAmabsQPazP858JAk5JMGLwuT/LBYoc1N8bQ4b3FNm6Y0gkTRkoOmYRzSx1eWZnil4SJLmMyYdI+8GmetSpousZ0tcBKO6TsmGgCZqoFRko23TCh4UXM1X3C9Tvdz54cU6k9hWIP0m8bglsZgKruPsVWdqNrT9UXKG7JhneTH6W8fGGNevw48vi/oVp7jdc5yYJ+gobvognJCzPDXK37XFhzKdg6a4st8pbGpTWXbzYPca7wP/Ns+i5Xis/ilZ7E6wScmh4mSFKmhnIMFezNC/GGb9T7ix2WWj5FxwSksjxQKPYoe12o3El3n+LRQO0kigfChndTzjTImTq2qXE6O4I5cpyaG5LGKYM5iyBJWetEaALylsZwwaaphRwYyNHxOzS8iJXsCD/MHucXrQVeWP0DvhsdZbX9EpYlcAyd8yurREmKIWCkZOOGKZMVB4AwSQmTTFkeKBR7lP0gVG6nu0/x6KB2E8UDYWM0jB8llByTx8aKRFHGCzNDnJlv8tZci4KtMzWYo5K3aHoJc3WPC6tdDpQdFlo+bpRgGRo5TeNYfJZfnf9nGMT8NAb/yv51Fgae5umDA8zVPYqOwbuLbQ4O5VnthKy0Q2brLmXHWLdCUCgUe5XdFioPem6f4tFiTwgpNffs4eda76bnpwc3x7fkLYOcpZOzdBxT5+mDFTIJp680COOUuYaL0KDuRhi6YHIwTxClnIrPYhCjkyFJOBG+xTJP45g6QsClNRc/SvjeuRWmqwUOVfO8s9jm0JDFYitktRNyeFidawqFQrEX2U/it+87iZp79vBzrVAeLTnXHat1Q/749atcXO2QAY1uyAuPVXlrrsGPrzSpuyFzDZ9npyocGyvxyceGWGj6aEOfIX3vKyBjUgzcA5/icCVPzjRoewmNbkxKRtOPeftqizjJCOKUhhdh6fpN16jOP4VCoVDcLn3fMdTcs4eb7YQysHnsat2j5oYksucV8+bVDjOjJYaKNhOVHCMli7fnWpxfdnHDhH/w6Wmu1Dz8+FN8Rf5rnkre4XVxkrD6cQ6Xba42PIo5g3ygk0qNoYJN3tSouRHHRouEScZIydk05txpjeocVCgUCsXt0PfdYqN2Rs09ezi5VigvND0u11wKlkEme/5OEogSSdOLyFk6RVuQt3TGSzan05RLqx5CgGP1jpuGxmdPjjFb8/izxrN8Sz7DXN3n+ZxJBtiGxkQljxDgRxljZZtLay4XVjqstEOGCianDld2XKMS8wqF4na415TTfkpdKW5O33cLNffs4WZDKC80Pd6aazBb9xgqmNi6zpn5JiB4dqrMzEiBom0wXnY4NFSgWrSZa/qkWcZ7CzFX6x7tIMUQgrxlMFS0ODlRph3EuFHKeNlBIgnjjPGyzWDe5LlDFTphQhCnzDd8TD3G0A28+CP7Ay9K8LcYeF4r5lXK79649vNTKO4EJTAU+4U9cXVTc88eXjaE8puzTeabPk0/4Z35mJ8+PspYyWGmWsSNEl7aMsYlkzBctMibBgVb59BwgclKnkT2xjRuCLS5uk/bi3h3vsWTkwOb4mm0ZFMtOnhRwtn5Nn6c4sYpmRsyW3PRxPUpPZCbBp4b56JK+d0bWz8/hLjnSQoKhUKx11C7guK+0xMfkijNKGuw2o1YbAVICQM5k5ylowk4PdsEJCA4PJznBx/WubTmUnNjRss976eSbbDSCSjaBsfGy7hRyidmBql1Y6aG8lxac8kkrHVCTk70BNnUUA5DCMq2wUDeZKLisNIJKVjpdSm9nKVvRqm6YS9SlUkoWAbzTV91+t0h16V1Wx7CsOxbP0uhuB6VAlPsddSuoLjveFFCvRuiIbi45lJyDI6OFVlqhRi6YKZa4NWLNc4udOgGEaWcyaXVLjlLYzBvoQtwTI3nDlWuE1vPHaowVLBIMxgq9ASZH6XkTIPltsdszcM2Bd/9YI0gSWkFCTlb4/UrTYaLNjlTB8R1Kb1royhxkhHGCWfmWyAlBUu/LmKl2JlrU6YLLY/zy12EroSUQqF4+FA7guK+M1vzmGv4VAomQvQiPKevNJlvelQLNrM1j5W2z0rb4/2lDjPVIuNlm7Yfs9zyMAwdP844M98kiiXNIKaSNwF48cgwbpjiRykNL+StuQbtIGWh4XF4pICj62QyY7Bg0w0SqgWHoYLFQN5Cyt48vl6Rey+9vNIJrotSDRfzJBImB3O4YaoK0W+DrSnTkaINEmQaR/1em0KhUOw2fdkRoiTjSs1Vd/ePAGvdgK+/Nc+rF2qsdkMODRfImwZumGAIDT/JOLfcwdA1Einx44y6G9ENYp6eqtAOYhxTJ04zFlsBIyWHbhAzVnawDQ03TDm31Ob95S7NbsR8y0NKSc2LiJczqkUbmUkqeQM3tHisWsCPe55SOfPGCNO1XaRxkpG3DAqWgRumqqv0NtnaBTlc7A2OFpquPjzFA+drp+c5PdskSjNe+vK398S4mb24JsXd05fiz6Yf8fW3FvjBh6t4UdKPJSgeECudkCTNmBkpULANHF3QDmL8KGWx45NlkkreZLLiMJi3ODyUp5QzsG2dw8NFDg3myTLJSN7GDTMGHIPhok0lZ66PnElZbIVYmkbBMYiSDImg6UbM1T3OrXQ4Nl7m+UOD/OTRKuOVPEdHC5wYL29bPL5RHD9TLQCSpXbARiG6Kja/NR+l9NLNlOlIyebU9BBZHHr9Xp/i0eJrp+f50lfPEKUZAPNNny999QxfOz2/p9b0j/7oTX72X363b2tS3Bt92RV0ISjZBm6UqVTJQ85oySZnGiRpwGDOZKhoUS3afPzwIK9cqDMx4DA24HA8fo/23Hf4rnaM98QJOkHCbMPFNHQODuY4OFigG8ZMVHJEaYYEwjjjw5UOa52A+ZbPWMnh8ZEiS62AnKWja4KSZfLufJuSYxIkGUfHBnDXxXs37P1/OzGVsxJMQ7+hEH0DZYtwI9en9MQNXZBkibprUjxQfv2bH+DH6XXH/Djl17/5Qd8iQNutKZMw1/D7sh7FvdOXHSCVkk6YMFlxVKrkISdvGfz8UweoeyF508AxdebqHqau8fGpAYZKNtmVH/L4D34JkUY8o5l85xP/J2/IIzxxYAAExHGGaWgUA53vf7hG3Q0p2AZPTJYpWiZPHqwwUnb42ESZkZLNn721wEonoO5GCAdsU2O87HC57jLf9ClYGueWOpiGdp2twbXi6GZGscoWYXu2pvS2ik+F4k651xTYQnN7cbLT8QfBTr97I0Kl2H/05SpXyVn8wjMTm3er6u7+4eT6DriUyUqekZLNoeE8q52Qc0sZK+2Q0dmX0dIIQYbMYnKLr2IfPI5jauRMnZOPl3HDlG+9t0TdjVjtRjTcCE1AteiwbAWkaUbeGqLWjagHCR8/NMR7Sy0+NjGAoQksU+PEWJlj4yWgN9T4WidzL0p4+XwN29DIWTqnpod2NIpVTujbo6YUKHaTndJywG2LqYlKjvlthMtEJbd7C71DdlqTpSubtf1KX650lqFxeLhnwKju7h9eNgSHLgSvXmlS92IODDicnChT64bUvYiJisN85eOkmomWJaCbyOmf4OefmGCoaG1uxiudkJLTM+dc7UqKjsVj1SKDRYu8ZfDWXJMfXa5zYaXDQtNnqGBzdLTIxw8PcmyshG32vKo2zrOPCspT6m7IO1dbzDcDKnmT8QGHbpgwWnK2PReVYNiZ0VLPBPXQcF79O1bcE7uRlvvi547zpa+eue51cqbOFz93fFfXeidstyZNwNTggxV3yp9r9+j7lU7d3T+8FG2DOEl5dbbJctunWrRwLJ1vvrPMueUWS+2Qx6p5Hp96gbNj/w5z7gf4ky/CyHNUSxaZ7Ant01ca1L2YhYbPxyYGaHgxh4fy+HHGiCZ4c7bBQitEIFntBBiGzmLLx9DyJKnk0prLyYkyZxfam0Lq5ER5veOvw/uLHWbrPmXHoOXHDOatbcXRtZHTR3Ws0U7RYy9K+MGHq7y/3AUpmW8WePpgRXXmKu6a3UjLbQiuf/LHbxOlGZOVXN875LauqZIz6YYJF9bcfdfBp8RYj75f4dTd/cNL3uq5j9e9iGrRxItSau2Qy3WX2XpAHKfM1QJGSznqI88Sl5/m2HiJQ7a+KXpWOwFzdZ/BvIVEcGg4z98qTjCQtwjiFD+MudoMEDLjg+UumhBUizphpKFrMFS0cMOUlU54nWDPJOTWhyBPFnLMNTxKOZPRssOLR4Zv2Py3i5yOlpw+fbL3n+0E082ix90wwY0yLE3DjRLenG3iRymOqfPikWGqxYf3s1LcH3YrLfeLz03yh6/NAntnw99Y01onZKEVkPQ6NO4qfanoP31Pym60m5+c2L4dXbG/GSnZHBjIMTVYYGIgx8xIgSTNCKKYlp/QCWOuNnwKdk/U5CydTLIpejIJLT9kuR3Q8kPCOEMDpISuH/Onby9zabXD+0sdZJYx4JiYms5TBwcYyFssNAM00esejJOUC6td4iTdLCiPk4yFZsD0cIFPPTbMZ0+ObbvpXxs5zeRHHX8PIxuC6exCm9cv1zctSm72GRRtA0NIzq20Ob/SYants9YJubTm8vL5mrI5UdwxX/zc8fXpAx/R77TcbjPX8HdMX94PvvCbr2xGkRS7x55RLX7Uc6ceKd3Yjq7YZ8y9Bpe/D9OfIT/1Aqemh9aLy9t0gpi2n2DoGpYheXJyAF0TLDQD8mZvxMtA7Q0mz3ybxaFTpPknWevE+LHPQjNE0zTKtsnPP1UmzTJ0HQ5UcszVfcYH8ww4OlEKmZQcHsozNZjjsfBd8q+/yoD2JG7pKUBsLrXthyy1IwbzH0VdtktdPUqR053S7Vs/A02wOfcwbxk8NTWIG6fkLYM35xo0/ZjRko0fp6x2wn6/LcU+Yy+m5XabnTr1+tlVqLhz+r4bbK2tODFe5qWjVSWm9itzr8Hv/QKkEegWwd/7E7ojzwFgGjq2YTBdLTBUqHB+pcv4QI7xss3UcJ65mkft/b/i4Lf/IeU0Ykaz+PrT/zsfy3weD9/iW9lR0uzThGlKhmRqME8nSHDDBCSMF20sy+DZgxUWWj4ZwNUfUfr2P0TLYj4uTIp/7d8xZz7J5ZpLFGe8fLGOjsbLLQ8vlnT8iJGSg21qfOqx4euK1E9OlMkkD31d1E6icSN63A0TNMF1NWenpoc2o4+ZhOcODrLaDZlv+LSDmHNLJgjR9wi44sFyrzU0ezEtt5tYuratmOpnV+FeYL85v/d9N+iGCQ0vRkqJY+p4sZpntq+5/H1kGiFkikwjFt/6c66cfJw4yQC52cXX8mIGchZxmnLiQBnb1FlqhYzWXkOkERoZQsYcuvp1/sban2KQ8FMY/PNL/4KLhaf4/HMHqRQsfv6pcd5d6OCFCZkQTJQtUinRBaSSzdcTZGgSau/8BWcPHgLgatPDD1NyliBNM+I041LNZa4RkCYZHyx1ePrgAHN1nyOjxU1bhIf93LxWMG0VjXlr+5mEq52eb9SG2NQEfOf9FTRNUHJ6jQMI1d/9sLPfi48f9PqnBnMstIIH3lW4l4XKbthePGj6fmEL45Rzyx0uLHf5YLH90KdNHnaCyRfJNJNM6GTCZHnoVK+mBknBNhkp2fzMiVFGSg6WLmi4Ma9erFHvRnSCmFeyJ0g1kxSNVPTOA5MEnQyThM/mznN8vEQiJUXbwDYMumFMO4yZq7m0wwRLFwh6lgmvZE+QaRYZOlI36Yx/monBHBOVPKWcSSYlnSBCCA2BJE0lhga6IcgyCVKQZBJnvXZrv9ZGeVHCSifYrFXa+vNW8paxWUy/3fO2WkicW+pwdqHN2YU2Rdsgk2AbGuWcSTvofbcqIqVQfMRXfuXT/MV/99P82uef2vSQmqzk+LXPP3VfBcNaJ9xzY3Ou5Wa2F3uVviiWTMrNi/Hp2SaOqXO4WmC0ZPH0wcq2HVOPYqv5fmR54Gkuv/g7PO6e5urAx1kZeAa75XF2voVEYOqC4aLF2YUWnSihYBokWYqp68Rpipx8gXdH/i2dD75LY/QTtL2EZ+p/hsxiEkze1p9krRsTJSlelDAx6FC0DByjl4662vDRhUYrSDgxXmQpd4qLf/0PMOdeZnnoFG7hSfKmzkLLI4oyPnOsSsm26IQR45Uc5ZzJbN1DCIGlaSAkhiY2hxzvR5G/tdtuqxXETlG2Wz1vI/rkR+kNBqeagLm6Txin+HFKlGRoVq7Yh7f/SLPfI0SPAg86fTnX8G9IJ/Z7bM617EU3+lvRl12h7fe8gcIkQyIZKdm0/JhyzmKkZF/3WGXYuX/woqTny8RR3i4e40S1yIuHB5mteZQck1T2yrzfW+qgaRq2rmMZGmECBVPHFxIQXDae5OLkYZ48WCEIIt488HuM1n7E+cKzTA0+S7za5YOlLj/4sMbUUI6CbdANE0xDYOkagwWT2brL9z/0qeQszo2e4ODzz3N+oYUpIUxSwjhFCMm55S5Hx0u0vIRqUTJStDl5oEwGjBRtbFPnxSNs1kbVuiFv1Bu9QvahQp8/8dtja/H4ViuInVLpt3peJj8y4LxSc6+rqeqGCUdGiziWzmzNY7bhIzRd/cPtM7sprJRI219cm87bjr0iVPaiG/2t6MuFLUpS5uoejqn35qANOAzmrW39e5Rh5/6hJ2Y0PvX4MAvNgGPjZfJWT3y0g4T5psdgzkKTAtMUaEJSzpmEccLZpTaGJviFZydww5RLq13OzDeZrXkE00+wNPM0jqEh/ZgoyUiSjCBJyRkGOUtndMDBELDQDDh9pcmlmstQ3mIwJzi30mWpE/DeQofj42WW2wFlx+DimkfDC2l0I37ieJWJSp6FlseHK11MXafWCXnp6Mjm+TZXd/mNb50jyXpRql/97LF9Iaa2Fo+PlmzWOuEtOxB3et5Cy+vZUKw3P+Ytg5MTZVY6IaPXGHBqQrDcDuiGMUmagtBUak9xx+yGULvZazwKgnBr3dF27BWhshfd6G9FXxSJEIIfzzaYGHB49tAgJ8bLOzogP0pt5/udje/KDVMG8716qJ640vnM0REurnUZcAz+/VuLBHFMw0/5uZOjpFnGRMXBMnXs9f8eHy2w2g25sJLR8iJafsQnDg9yfrXLUjtgrRsCgrGSg6lrPDtV4eKqSyZ94iyj4YaYusaZ+SYHB3OMDTisdkPsmkvLj1lopJxZ7DCUt2j7McvNkNFSSNuPmat7DOZtLq5GDBd7swEzCedWOiSZ5LFqiUtrHa7UvX0hpLYrHj81bdwyXb7d805OsDmT8OxCm1PTveeeXWjjRynnl7u8eGS49+fVDkutEFOTLLUjUBEphaIvbFd3dC17SajsR9uLvlzY4jQjSDKKjoGlazedEn+zDiLF3mKn70oT4EYJBwYcDE2gawLH1Im6EUkiWemEhEnPaPPwYJ7hosVc3afmhrT8GCEgSiTvLrZpuDEl22Sk7DBastfn9kXU3YhuECOAuhvR8VOECMhbJm4Y8/1zqyy3AhpexMSAw/GxEmeXOiRpRgZkwEy1wPiAzVIrIE4z5pse7yy2eOVCjSOjRWpuSBCnnF9tY+kah4fyd/wZ3aze737WAm502+308+0+L5NQzpkMF2wWWh6Xay4Fy8CPUpbaAS0/phPEXFpzudrwCFPJibEigwULmUTBrr6pXUII8TvAfwKsSCmf7Pd6Hmb2cvSnn51s9/vzuFnabi8Klf1me9EXVSKB4XyvqHdqsHDLKNPtXvQV/Wfju7q2u+vwcAE/SslZOuGlV/jZtf/A9+MnaMaP89qlOlPDDmGcstgO+fpbCzxxoMTUUI7DQ3mCOCVOJeMlm0LOIIgyrjY8Co7OYK5I3tbRhM3Z+RaWodMOElpeTDFn0vSidRduDVuHT8wM0w5iLF1jzY0pmgYZcGKsxETFodYNyVs9n6umFzM5kGfQsZive0RpxnIr5MR4GU0I/tqT43ccjbpZvd/9qAW8H8JsI+q40PI4v9yF9fWudUMWmj7VokMQp6RZL1Vw5mqLD1e6aAiEYe3VOTG/C/wb4Pf7vI5dY6t79V5zs76fgupuXnunTjbYey33d/P+dqo7snSNH/z3/9Gure1RpS81C1kmiVJJ2TF5cnJAiaSHjA1RcPpKg6+8Nsdbcw3+4r1lFt/5Sx7/s7/P36z/Lv+i+z/y9yaWODiUpxOkvLvQYbHZi0IFSUYnSHhjrknLi+kGCbYp0BGEScr4QI7JSp4wTTnzyp9T+8aXWXjnL5lv+qy0Q3RdUC2a6JrAMjT8KCFDsNj0CeOM6cEcjqnzkydGOFDJcWK8xKU1l//w9gJ/emYBISWfmB7i6HiR8ysd1roRr1+qc3nNJW8ZCA3sLaMr1roBZxdbrHV3DrrcbMTKbo+g2WnMy72yUQ9VsA2mhno2EpkEKSCB2gwAACAASURBVAVJmrHU8siZGmXHACFIJeQMjYYXgcx2LtDoI1LK7wH1fq9D0T8e9KiWB81243Y00fOxuh22Gy2zEcH74aU6L33523vGPqEf9EXBaEIQJymmblMtWbf1HGWBsH/YEAWOpZPKj7yYRluvo62bYxoy4WT0NssDz3C4mGOpHeBFGRdXu5wcL1OyDbwwwdB6XXi6bjAxmMONUwbzFk0/IrfyBv/xmf8aLYv5CQz+N/4lbnyEY+MlojQlk72O0OV2gG0IOkHGaMGhkDeIMknRMXlsWDI2kGN1vokfZ4RpjBdnDBUtnrYqpBmcmh7m3EoboxH0ugzTDD9KN8ejeFHCV16b2yxC/8ILU9vO67tZvd9u1wLeryYNL0o4u9Cm6UW8t9hZPyoo53SGizaLrYBMavydUwd562oTITMKjsVqJwKEuNlr72WEEL8M/DLAoUOH+ryah5PdiFLdbXpuNzrZ9nLacru6I8fQqG7pkr9ddtM0cy9/brdLXxSJoQuOjpUZKdu4YUr1Fu4yygJhf7EhCvw4RRcChCTLJBcLz3JYt9CyGKGZdMY/ycfGS5ydb5NmEoFAQxDGkqGiQAhY6Yb4V3viqWiXWGj4XFztoAuNv956Ay2L0cgwSHies7yhHyOVGWkKZUun6cfEmWSk6GAZOhOVHIamYxopl9a6VPImXpQgBHSDmDDJcKu9VOSG6aQXJRwaLDBWtEmkoFowmat7LLWDTdGTZJJDQwXmGi4rnXBbIXUrx/DdrAW8X00a3TDBj1IaXoKhCdp+wgszQ3z//CrvL7XJmwaX1rqcmqnwzMEK331/hdVOhBAgsyTalUX0ASnlbwG/BXDq1Cn5oH//1s1ma3TgK7/y6YdiQ7oX7mVzfxRGtWytO7qXdO/NTDP3Wir0QdAXNWIZGiBpuhHnljo7duxtsHHxdiwdX42Q2Rds+AvNDBfohglxktLVn+eVl36bsfqPuVJ+jneTI7w0kCNKJE0/IUrSXsxCSDp+ynDRZjBv4YYJpq7x9tVmz9gRjYYfEk19muzc/wFZQiYM3AOf4mOyhGXofLDcodmNGK/YmJrAjRLCJMUNY7wooeGGLLcjxko2lqGRpJIgzkil5MJKFz9O6AYpxw+UaAcxT01WqJasbQ0o85aBoQnmGi66EIze5C7vdur9diP6er+aNIq2QZhkrHWC9Q5LjR9dblDrRrS9BKsoaHgZZ662eGFmmJ88NoJE0AljvpKEe7LY/EGx08Z1P4XP2cU2ACcPlO/b79gr3Mvm3q9RLXfKXhntsh9NM+8nu3J1vdOuF9vQmBzM89hIgbobcbnmMj1c2CxS3q7r6/xKd32GmuD5w4O7sWzFfWCtG/Cd91eYq3kgQCCYGMyx0on41EyJeZ5jvvQUHT9hebHNy+drfOxAiU/MDPK9c6u03JgLWodPzlQ5vG47sNIJmakWeX+pzWLLZ7kdkEnJ/5NO8O7xf83J+Awf5p7h0InPMLPW5b2lNo6pU8jp2KbBUN7ipaNVbEMDARdWutS7EUma8fpsg1aQMDZgM1S0aQUxb882qbkRo623eKx1hfedp3lf/zSFusax8TIFWydOUi6sdilYGoeG83zhhalNH6XtolG3YiPq6kcp51e6uzLb7340aeQtg+cOVXj7ahM3Sllq+jS8iKJt0oliqsLiifEByo4JQCVv4UcpV+sxQuj6LV5esYW9ViS+l9lpE59v+nzhN1+5qWCtlmz+m589esuW+35G/XZ7Bt29vIf9aJp5P9mtq+zvcgddL0kqWeuEOJbO1ZoHEubXPXnm6h6moV2XwsskvY3FNPDjXu2HYu/hRQkvn6/x4XKHtt8Tw1kmGZwcYKUTMN/0KVgaSarhRilHRwpYhs6R8RIAK+2QbpCQtw10XfDCzDAA55ba1N2QxaZHzY1Y7YYcHi6ga3DOPEl68AU6YcKzBYt610AXvZEuectgsGDyxIESSZoh1mvzNCFoBxFulFB3IwwBq22DqeECXpCgC8Gz4kP+y9qXMNdiTmgml8x/yvLyPO8e+BTpxCcQEnr9p72yn2rRuSsBtcFGTVPONG6Y7XcvYuhuo1tbn3ftz5nsCSpT05gLXKJUMpA3GSs5TA3lOTSU2zTr3DDqDJN0zxabCyH+EPhpoCqEuAr8Mynlb+/W6z+qYmi76MlWdlOY3Kwz7Xa4l5b7BxEpul/ptLv5Dvajaeb9ZFeElJTye0KI6dt9vK4JyjmTNM0YLVtoCN6YrXNuuUsYp3zm+Ahu+FEKr2j33KszKclZ+3Pe2aNAN0ywDY1K3uLd+TYF2yBvafhJysSAQ9HROT5WImfpfOPMMpfWuhTsjPNLHYI0o+5GnF/uMFK20ZAU7FHyloEfFZhvegwXbY4dKBIkGUmaYRs6T02W0XWdyYpDztIxDZ3BnEWtGzJatvmbT0/gmAbvL3WYzNu4wMcmBji70KbejSjaBgeHili64MnJAXKWzivn1zi+9Cam7NVfkcUcef2f87jMyK7+Dr/7+G8gDr3AM1OV687Te2GzrixKdm22393UFnpRwmon5NxSZ/OG5toZe3GSUbANkCA0kFIynDcQAo6OFvn8xw+SSfjR5Rp/+cEKQ3mLZw5VuLDiIux86a7fzH1ESvl3+72Gh4FrxcSz/9P/hxslxGnvrncjejIx4Nx1gfOt2GlznxjY+QZnqwC6mwLs3Y4U7cRupNN2K5K2H00z7yd9USTdMOHN2QaVgsVaJyRnapxbdnlqssyaG1LOmxweKqAJNjujlCnn3mdD8JYck6nhPKcOD2IZGtWixQ8v1EnXPH58ucHffm4SISRhkjJUsGgECfN1j8G8Sc7WOXmgzEQlhxumvHp+jXcWOyRJylI7JE4zkiRlsOJwbLzIxyYr5Cx9c0bjX30Y8srFGkGcstAMMHWNw0M9M82Lqx2mhvJU8gYnDpQwdcGbsw3m6i7HxorrgkPjmYODHBj7OcTS70IaI4SALNsUVbnFV3jLPEHb79VObSd2ro3gAHfkIv789ODmbL97OdfvtHNvQ3jV3Zi5usunHh/GDdPNGXsFW+fV+SajJQc/Tji37GGZGmMlh586Nsqx8SL1bsR3z63w2sU6o2WHMEkpODpSSsiyna2VH1K+8JuvcHaxfU81SvuhzmmrmGj68Q2P8eOUuYZ/34TUdpv7z5wY4Y9+dJULa+4NkaLtBJB2k77SnaJOD6rweq+l0/abaeb95IEpkmvbh4fHJjg2XiZKMqIkJYhS3DDhg6UOmZTESUbbj/jO+yuUHXOzVmS0tFf9/BTQEwMz1QLnVzoM5i0W2wEnxso4Zs8GIU4lK22fr785T82LaHsRK52QY2MlojTDMXTCOGOlE5JmcGiowDsLHdpBTBinFB0NDYMolVRyJlfrPnmrw1DB3GxYmKkWKTg6jqFzpdHlrbkWc/WAnzha5fSVBjlT52rdI0olS+2AnG0wlDeZHMjjxwlBnBHEKc6xT9P6z/6Y8Pz3yA+MUPj2PyVLI1Jh8lr2MQpGLz05NZTf1qF8IxIUJxkgMQ39llGhrTVNG6amdyuo7rRzb0N4TVZyzDU8FpoBg3nzoxl7zQAQPFYt0g1j/DjjyfW0rWNqnF/u8hvf+pC2H7PUCfjE9CBZBk0/wjR1kI+ekHpUuNUIkg12shnYDcEJ12/uf/eFQzeNFG235kz2PKW2crOo04MqvFbptL3LAxNS17YPHz35tGy4EUJkyAz8OGEwbzKQN5ESDg7leWu2QTdMGR9wmBktstoJyVn7LyL1KPlfeVHC6dkmnSClZBtYpsZIycaxNIIopeaG2KbOSjfg/LJLlGboQlDJmQwXLep+yBMHSnxyukqGJJMSSxdomiDJMmwMcqaOAOabPeuBg4MOCw0PQ9N49lCFY+NFHq8WefNqC1PXmKkWWOmEfO30PBoQpZIDAzZjJYc3ZxvkbAPbNAiSlB+8U2OtG6Lp4IcxbjRKpv2nJG3J3/i536f2zrc5YzzN2doEj3VDPDMFIW/4jq+NBF1c6yIlPD6Qvy4qdKvzYjcsP263c29jLRt2D26UcGKsyLFrZmCenIDZmkc3jHlvqYWOhqEJzi13cAzBQsPvjfDxQh4fLbHQ9HhnvkW1aNNyU2aGcsgs3R1XUMU9cXaxfcvi6zvldkXD7dYr3Qsb7+ulL3/7ppGinda8ndi7WdTpQUWKVDpt79KnnV1QLVokMsOPA8IEYpmRt3SqJZumF7PY8umEKVfqHg0vQgNKjrmvfKQeNf+rjRqpvKXz3mKbwbzFd/wVTk6UOTFRYrVjI6VkqRVwoJKj1glxLA1Ng8nBntCwNZ0MuZmumxrOU/FjJHlOTQ9x5moLhCTN4NJalz98bY4olby70KTeneBTR4Z55mCFgq3z9lwvJRKmGY6p4Zg6cZpSd2Pmm34vImXqeEFKJWfiJwl+kiESyRtzTUbWByIvNj3+wpwmnPwvWO6E6PUWhug1oAVRdsN3XLQN4iTj4loXTYBjatdFhW7nvNgtQ81bde5tXcvJiTJr3RAvSinYvRmYa92Al8/XCJKE759bRRc9h/nJwRxJCgOlHG6cMDXkkEnBpdUupqlRtHQ0IXh3scnf/+RhstDr7LgQxb5ia5qrkjdpeDem867lVvVKu82tIkV3Upx+s9f6X7/w7AOLFKl02t5kV24P1rteXgGOCyGuCiF+6WaP9+OUVMJCM0Qg+MyxKsMFm+nhAifGShRtjYJtUC1YTFQcxsoOXtTzGNrwkdoP7PbYj73ORo3UwHo66NmpCpmUBEmGlGDrGmXHpGgbjBRMUiSapnFhxeN7H67yyvk1Zhsua92QmWqBS2sutqGx2o0YKdm0/JgXZoawDINMSmpuTJpmmJogiDPOr3Y5t9QhkfD84WGOHyhSKZgcHswx4JgkWUbR0ZkacmgHCZbe6/CrFEwmhxwa3Yi2H+FHCXlDEMYJZ+dbtIOY5VbIgUqejx8aZLySo5w3mKg4ZFLu8B1LpATH0Hju8CAnJ8qbgul2zov7Zai5ldVOSN2NKNi9LsG1bsifvb3EN84s8W9fvsK5pTbfOLPE2fkWP7pc5+JKl7ofc6XmIaXg4GCBlU7Eu/MtfvBhjScnChwdLfLkgTJx2tto3lloc3q2CXKHvI5iX7FdmqsbJJj69QVGpiYw1ouOJis5fu3zT923+qjt2CkitHF8u7EpOVPnf/k7T9/Ra/3ic5P82uef2hRgG+/1YYkUqVEwt2a3uvbuqOslTjPenmui61CyDa7UXExd8PhogXPLXVp+Qs2NEQiOjRUZK9nM1nv+QfvJR+pBbYZ7hY1U0monJGfqNP2ID1c6LLR86t2YycEc08N5ThwoUbANHh8r45g9o804zQiiFD/KmKv7rHZD/ChF0rsYD+R7ZpiJlDw7VWG1G/L21TbtMKHph7hRjKFruOtjZdw4oWBqPH94mD954yotP8bQBUeGS+Qcg5GCxUrTp+bGGIbg1Ys1MilB9iwNRgfyPHeowh+9fpWio1N3Q7wwZrUb0fUTVrSQoYJF0Ta4uOqy0gnIGzp+lFJ3Q9yoF7FxwxQ37A1s3ppCu9l5cb8MNbcWwZ9bajPX8JlreJwYK1N3I2brHpWcxeVamyBJaAcJl9dc0jSjGyUc0AWaEHSDhKsND8sQFG2ThVaIoesMlyyW2xHNICZMUgxP8N5iG2E4qsjxIWC7NFec9eoWvSi9Lu10bfTk2iiWpWtMDebuq7C6VU3RnaTKbue1HsZI0YPqSNzv9GVnz6Tk7GKTom1ybKxIMdeLPr12uUmSpmQZVHImmZR8YmaQx0ZKFJ32DT5Se73+6H5thvsBISUdP2GlHZK3EhabPnlLw40S8maFY2NlsgwyJEPrs/N6lgYatiHQ6JmwBknKYsun6ccM5j4qfJZIcqZGycmD8HjmYIWp4QIdPyZvaXQjSStIuFLziDPJYN6k6BjkHR0hQNN6A3URgrYfs9qJ0DXwopQwzXh/qcNoKcfUupmovh716oYJh6s5BBrDBZu3rja5uNKl6ccUbYMUycUVF4lkruExM1Tg3HoTxbVGmzPVAp0wYfQmrv67bah5fRF8SsExqHsxz04NUOvGTA3lqXshUZJxud6l5Ub4UUwQS9bciKlKjuPjZco5k/GyzeRAEd2E1Zbfm624PiA6TU2GSyadyKYbJFiGzkonQJjWo+nW95CxU5prI2IMH4mJDXGxdUOO0oxLNRfggXbxbRVKtyuA9lJ90m4KtVv5X6lRMLdHn4QURLFEcwTvXG0zNZzfrGGRmcRPMoIko2QZrHUjnjqo3+AjtV/qj+6Hu/Re4doIy8b3cHahzWIr4IOlDgVLoxv26ibCpGfamLMMpBAstQNAcmK8zPOHB1nrRLx+pd5Lh5kajmlsmrAuD3kMFyxOjJeoFh1OTkCcZRwdL1LrxDQto3dxl5K8YxAmGiAoOzrdMObAgE0nSPGinilnOWdyeLjA5ZqLpmkYGoRpRJhIJL0RRoYGdTeg1g1prYs4y9QoWDqGptEJYrpBTJz27AH8OGW57TNdLaDrgifGBnDjlInBHO2gN8svlbJ3M7BelG8bvU63F48M35OZ5+2ykVIsWAbfvrhG0+uNy5mo5JisFDi/3CFD4pgaqdQ4OlZkqRMCKY6pYxgan5weIkpSal7EG1drnBgfQMqe79VgweL9pQ4HBiRJKhnO22gI8pbO4aE8Moke6RExDwt3U1y9U4fchbX7L6Z2K1J0q9faemyvzz68nWiTGgVze/Rlhzc1gRsnRM2EIJYIHbphyjOTZaolm7Lo3f2PlBxsQyOT3BDZWekE92W6veL22G6kSZhkSCSLLZ8zV5toQtL2E0xDJ2cJRssOk5UcZcfc/N5ylr7pCl4tWbx8voZtaHy40qHtx0gZsNAMkFLQ7IYcGS9xfqmDF2dEUc/EEyRLzQBT0ziStxjIm4SZJMl6RYBBnJJmEl1AwdJp+RGVvMVkxcE2YkwdKnmTNTcCIsIoZbUdkcoWSSpZ7UT4UUq15PDkwTJ+lHFlrUu1ZPH9D9eodSK8OMHQNc7Ot6iWnF6krWByaDjP6SsNGn5Mmkr8ONn8nJbaAa11v53Pnhy7L+fvtVHbjVTzxbUuszWPom1i6hpNP2K44HCu4+OYOg0/ZqHhUS05ICWGLjg0lGO87HB4OMe/f2uB2brP1bpH200ZLBgcP9BL0za7JsMlG9PQKVoGc/UuByp5MkBouvoH+hBwszTXhtDYys023o3I1H5gr4qiu+F2ok17zbtqt9htkduXC5sESrZJzhQ0/YTBgkWchJRzFk0v4eCgw1w9QNc0LqzAi0eq10V2vKg3xDhO0kem/mivsRHdcCx9M9ICvZqZniu2AKkxkDMp2QaaYzCYN6kWLdwg2fZ7yySUcyYFW+f751Zxw5TS6hscbv6Y8OCL/JCjXGn4XFztMjNcZLkbUHcD4gwQMDrgYJs608N53pxr0nJjgnUfp5GSzZWahxenXFjpYhk6IPnJo1U+c2wUTYM/f2cFL4mZq/t8crrKQqv3XqTMiDPB+IDNJ6aHcaMExxQEsWS0bJM3dc4tdxkvO+Qsg585McLEYL5nJRCmtIMYP045NJzjxIHerL6Xz9do+TEDOQvb0O7LjcB2HXmjJYdukDBeyREmKZ1uRtWyGCvbLLZ9Fho+zW5EnGastn0KttGrPQsSfDuhFfQKzVfXx740gxDHEry32OJq3ceNUi7WPMYHcjw7VWbNSxgZkNi6hkzTm7d1KfYFO6W5/vC12R29oHbakOEj7ybbvD/WCP2ai7cXhgvfjNuJNinvqtujL+pDCBgu2gRRSs6SkEkGCybDBYu236UTJrS8kImKgxvGrHWizdTHtZsDCGaqhU2vm/vJ7dRj7fWard1kc6RJnKILgR8n5Cyd5w8PMVt3ccOUlh8RJilxktH0Yv7kjXkODeU5OJTnF56ZpFqyNjvWvCjhat2j68e0/JimF3GgfYb/avYfYZCQnP0Dlif/JZVDP0UmJd0wwRQ609UidS9ktR2x0PIZLlgM5C3Gir1zotEJWfMiojSj6UfM1TXWOiGaJsiZBu8tdfhbHz/IoaECHT9lqR1QMA3GKxZXm13aQW89cSq5WvMp2DphnPLdD9bIsoyml3BgwMSx9J7giBJYF4ivX66z1Ar44cUax8fLtNejT9Wiw4tHenMEbUO7YezRdq7o16ZPd3I9v5mf1aW1Dl8/vUDTjzZ/7/RwAW8w5uKqy8sX19CBME3phDGr3ZC8adDwYqarBVY7Maap8fKHNZpeTCdMiVPoBAlDuYxqwUYXIXlTIoRgoeWz1AxoBTFpmnLiQHljLKHiIWC7NNdO0SjYfkO+lijN7puQetDslwLt24k29bM2bD9F//qy20dJhm0IwkRwbKzEgXIOoUkeGy1wqeay2Axp+DGzdRdT13n9coNDw/kbNoeN1NC1kar7IWSuTWOFSbZtTYsXJfzgwzXcKKFgGbx0tHqDS/XDJLKuG2ly+KORJgBPTlbQhEaa5bnScEkTiRun6/VJFpmU1N2IM/NNMgm6BnM1D03TyLKM5w5VqLsxMwuvopPQc5ZKOBG9TZOf4kA5x4FKbzBulKYUTB1LMzgwYAOSKElZaAesrqd/DV3QdCPafkKW+jSDFFvvNT0kqcSLehf3qaE8TT/i2HiZtW5EsxvS9GKCKCZv6cRZysKZvyQ6/z0OBcdYHXgGx9QYLtoMlxzenG0RpRbfeW8Vx9RpehGdICGIe+cN4iMlUS06fPbk2A2DgVc7Ia9fquFGGQVLUM7Zm4XqU0M55ur+ZsH6tXWBa92A77y/QiZhKG/x0tEqmoC2H///7L1nsGRnft73e9/3pE433zs5YBJmBhhkYLHYwLAWZdIyucUlixJZLJKSLcukVaZlfVAtM2UtWWL5A+kSbctMLpYosSiKLAYzSSR3sYvFIg0GwERg8s23b+c+8Q3+cPpezAADLPIMsPNUoe6gw+nT3e/p85z/8/yfP70k54X5LoNE4wTsny1jIQ7M1vnKy01OLfYIA4XnBIFfVhF7qWa6HrLWT1nupuTacCSq040LCmOJfAEIxkKPQa45vdxlqZshgemaY6oRUgk8apHCuNLEL6QXfMDL9DbeAd6u7PFWhjJvnHh//Hefv+H9H0RQ5weFD4tB+61Wmz6qHYnvJW6a2dxXkrEQ0sKinaPbz2kNcrKilGKywrLcS8s27OaAK60hU7UQKdgMO6z6anMe34bR+a2Yz98uqRlkpZS43Etp9jN6ScE33TlLOBoqWw081voZZ5Z7NCKfq+2YQ1sb7Jl+leB9GIzxbxdvZKR/dN80WWEJRmGbz1/pkGtDLykYjzwObm3wwkKbZy91qAaKUElqkc+d2ypcag5pxQU7JyNW4wfRC/8B0Fjho3c9hq8kOyYrPLh3klowy5MX1nlZ9FkdZGwZj7jajvnKK+ssd1LCQOILwb27J2j2M1rDgvGKzzDrU/EVSgimqh6z9ZCvvLzGUjdjaTQc+cJqn1PLA9LCkGjopQW1teMcOf0vkLbgmPD5+fgLnFKHMRbWBzlCwMf2ThMXmotrA/7i5ArWWTrD0oc1XQ+vmx95LTbWyMsrA/78pSV2TlbJteXR/VPsmaqjrcMh0NYRBYqkMFxaH7J3ugbA355Z44lXmtQin+mqz2wj4OXVcgj4IDU4B07AlfUhUsBjEzNIIXj2coteUmDigqTQRJ5CSqh4ionIZ32QYZwjKSznVwYsdRPSQpMZCKWg4iu0Lcf/eBKEkASeRArB1fUBIIh8j28+VMWZIv8AluVt3KL47P07+D/+y8tcXB9udl5D+Zu4a7LC6iC7eTv3HuKNJLOFTvKeJ8q/G7wX1aZb5b3cbNyUs7kUcKUVM1uPmAgU1UBycS3nxYUu8+2EufGIKJAIJxAIrrbLYbc7JqsU2pAVZcDjQjvm9FKXbRMRoecRepLtE9U3NZ+/EanZqAYAr5MK66FHpi3NfkZzkNKOCy6vD3lo79RmZQC4puJwvYbx2irah3XczVtBeuGr6BN/xYGph1it3MunD86ijSPyFGeWexzc0mCmHnBmscdyNwEhmKn5CAl/fWoVIWFrI8RYWK4d4xemf5EHOEm6/eNUdn+Mfi9jbZARepK7d46TFJYr7YQLzSFX2wmzdR8cLHRjaoGPJ0At9KgGCgeEvmSsGjBV9XHAnpkG64OcMysDAim50k5IjaMWeoAg9ASFLeOl9g+Po2xRDi92BR9Tp7noH+HMcp9OUiARPH15nf0zdU5c7bLaS5iuh9wxW8e5shvwPz4zz47JClVfEvoevieRAubGQpa6CReafdpxTjX0qAeKwjg6SUZ7mIGDQV5wZT2m2U/BQbOfsWe6hnGWWujjnGOQG5483+LsSp9q6OFJ8KRg11SFTpyza6qKcI5LzSG5KUfx9FIN1hEoSWuQUw88ChcxWfXBybJyV2gcjlrko+OCqXrIzqkK59di2nFBmlsqgWCyHlAJPLqJwhqHdZaTix2EH364Haq38a6x0Z13tZ1sSl8bHikh3Js99UODt5OYfrNxu9r03uCmmc2NdYSBYLruk+SGKPS4Y7bOfDvBWUeWW7q6wDrYOu7jqTK35/zagExrlBD88YuLOCeYqAR86uAUvlJk2gDiDad432j0BsBXXm5yZrkHQnDHVJVjuyY2CVU18HjswDTNQcaFtT4OAzgelmwmU882Qu6YqtCKNXdMVZhthJvkrDTGW9aHGYU2IzO2/MhUp+Jcc6U1RMw/zcE/+wH2mJzdKuAvHvy3yH2P0k8NV5Mhg6xgx2TEhbW4DNwsNKGnRvlQEcudjHqkWOnnfObIFk4t9rhSe4Rn3SMc3jpGs1/w3JUOjcgbrQPL6cUu51f7TNUClBDMNCLiwiCFQFAGeE5WPB7aN8O28Qo7JyNOzndZ6mcUxrLUTfjPp5fL9VgNGa94BAom6xEHt9S43IwpbM5kNeC4OMb3sVAvtwAAIABJREFUCB/PFRR4vBzeQzc2GGPL4dq+5Jg9y6c7Z/n3q7u51NvDyysDDs3V+UzjErtfeY5L/QOsyI+xNkiZqvk8uGeaONe8ON/l3PKAc0t9Qk/RSwp2TUYc3TbGqcU+y92Ei80hgRJMVgO29l/krgtnWZ1+mKTxcZSQNCKFQ7BrsoJzjmqo0MZirWDv1ipKSu7c2uDYjgmevrTO1VbMhbU+aa4JA8UwMeyJX+K7Oc1ZcQ8X+ncRKUE7y0lzTSNS5MZhrCXwFbNjAbONsIyy6JcSqhlVpyIHoVIsDBICKbnQTG537d0G8CqZurYytUGqmv0Pf1XqRpKZLwXWuc108FvRfH4z8F6b8m9W5MTNIVKuNLsudVIKbZioBqx1M14EJqsBnhJIJcnSgmY/Za0Xom0p5xljuNyMubw+pDXI2ToWsdZLeGG+y8cPTPPCfJc9o5bzTxyc3ZwVttrPmGuEN0wbH2SaYa5pRD65sRyfb9OKy/yhTxycBUrCtGM84q8Tw1hF0E0KVnsZu6eqmzJN6HuEviHVjvVBxpmlPmeWexTGsmUs4sE9ZQXrYnP4kYltuNoa8icvLHHiSptva/8pB0yOwmJMTnrub/hivBvnLGlWSlx/dWqVii+RCPqpQXuOdlqwVwkQpXdKW8d33beDxw5Os/pcylQtZKWXluNmck0goSNgxwQ8uGeSU0tdytol3DFd5cLqkJqv8DzBXD1iohrQSwq2jEU8cscMC52EC+tDVvsZ/USzZ7rKTD1kuZvRjjM8KZhtRHz64BaKfYa/PrPK9okKi927+ef257hXv8RT7iiTE/exv1aSuzw37I3P8o/bP49yBT+Dx4r8aY6Lg+xJT/Hpr/4kyhb8j8Ln98f+DU80d4xk4IQHdk8yUy9H6ix2E3ZMRAxzw/7ZBsu9jNNLXTpxge95tBLN/vQU33ryx1C2YJsKeNb8Fo2p+9gxUWPfbJVdU1XOLPVY6KZcWB2wdTwiVJK9s3WmKj7rw5xunHO5FZNpR2YdIYJHg/P8qv0CntNo/Qf8uPxZTuSHSunOOnwpmGuERIHHbMMnLxzDXBNno4sXCZEveHjvFNpa1vultyqK5BuajG/jGxNX28l18t61t3/QeK9Puq+VzCYqPsNco035hm9V8/kHjffLlP9+DOX+erhpXXtCgLWw1i9Y7eU4AWEs+aHHdvOfT62RjhLMo0iRGsPpxS47JqpEvqCT5AzSgtVeQmeYIaXk4JYaVc9jrZuhpGCtn7FrqgbC8fvPzhMXhvHI5x9+8o4bpo3XAo+r7ZhuXNDsF0xWChY7CbONiF5adpH9fy8ukRQaBxyYq7JvpsZUPdiUBO1Gmnc/o5vk1COP0FOs9DPSwtKIfB47MP2RGRvTHKT8zteu8NJil6uthNnwbr6DMgahwONMdB+nlvpkhS3NxoAUjkxbcuN4zD/PZ2ovsxg8RNdMEBell0cIyu+fUg4LPMlyT6ONYaoWcKE5ZM90hcVOzB2zNbaOlaNYPCVY7eWk2jJVD8m0447ZBvfvHueO2QazjZCrrZilboanJIUBh2OYWaqhphXnJLmlnxabFZ2/c3QLJxe7xLlGABeiu3jZHqGXaO6KMxqhz93bx1nspjzWP40yxaY5/q7iRU54B7g7fwE5kgQ9p9nde47Zxj4eumOK04t9hllB6Esaoc/OiQq10GP7eI3puo8A1oYZzWFB5FmiQDC+8rVNidHZgtrSk/Rm7me+MyTwSh/V4W1j5MbSjTMqnscXzzWZ76aMh6VHbbzql+N0lCB0CmMd99uTeGg8YQHNp4KzXPKOMkzL0TD93HBgrsGx7WOcX+3z8nKX1V4pdYe+JMktAsGZ5T737BjHU5J66GERFNoglH97RMxtAK9WoN7q7R82XCuZzbfLyQzX4lY0n3/Q+LCY8t8Kbg6RgnI8Rj1isZuQ5BpPKay1zLcSksJgjSMvDIkULHVihlnBmZUhU1XFK6tDBqkmLcoTYU0J5jspz15Z52o3xvNBa8dXL6yx2En561Mr7J6pc6kY8uJ8h7t2TFxHouJcM10PeDiapDXM+dqFdU4vd1kf5gzzgsNbG9iRX+vQlgbdpKARhTx5fp3FbspsI+CubWP0Es0LCx18KbHWMTsWcHqpTy/R3LNjnKQo5659VMbGrPZL0rplLOSVlQFfdnv5x+4necw7yxezQyx2dyGlZtt4heYgQwiBFJIdkxH70tP84PJP4cUa2/xtntj6a5wK5vCVAgcnF/s0IkWcG+YasH+2xnw7KetOAraMV/CVYqoW8smDM7Tjgjg3vLTQIclt2cxQVTy2f5rAK70JZ5Z6tOMCX0mmaiG9RDNR9amGisiTzNQiTvY6NHs5GyZp6+DhO6boxZpOmvHifI9mP8c5Rzsuq2cSQTfJOO7dxQ/kHg6Nweekf4xaGPCkOcL3Ud6uUZyr3MuWRsSl9ZjjVzosd8u8pv/2nm0Mc017kCNVzHg3QAhBzVd84uA0w9QQKEGn+gj2yq8jXIGVPs+Jo/zt167gKQhUOQKnE5fV3FOLfUJvwOVWwnI3pZcW7J+tUfMloZIoAZWKz87JCqea91Jk/4kNIvySOkZrUDDMCrSBmi/ZPhaxc7LCl19u0k8N3VRjjGWqGjCwjnro0xxkLHXLysJsIyLThomqj7Pmttn8NoDSL3Qj0vRe+YhupVTx2+ngN8ZH6XO5KWfxwJPUQsVU3Weln2AtZNYwEPCnLyyz1E3oJQYLFMYgC1i3OZ1Ys9YT+EoQeApHQaEdA1d6keZbHoW2dGPNZD3g4lrCQiemnRR47SGBp/jS2TX6qabiKw5tHUMK+KPnF+mnOZfWYw5vG+Pccp+LrSFZ4bi6HvPc5Q67JyMurMfsnqrhqXII6+mFLsbBzqmIsUqZg9Ua5kzXynDDdpyx0k1oDcuTTWYMU1Wf2cYsc40P/8X5XCOk4isCWX4nkac44x+lO/EA64OM7eMR1dBjquphrGWmFnBs9wSztYCZEyfwRtEGuILq4tcYZN/BRFVS2LIipW3ZCbY+zLl31zivrPRZ7qYIAfWwNIvvmKhibZk+3upnxKMBwYNMs6tWIdWa00sxgS/JCsNj+2fZPVXBWMtYxWPnRBUnHNO1kJdXekgnmKoFTFR8xqsevTjnq+dbpIWhHefMNULi3DDdCNg5EXFxPaEeKnLtuNI4xk+of8XdxYuc8I5yTh/k6NYG891j/JP2T3O/fYmn3RHWl7bxzWOCii8IPEHoe6z0U/7mzAonV/pkucWXcGT7ON90aA4loRH5FBXLTCNgvrib37nzVziSv0C64+M09QFc3KU1NJxe7rHez7hvzziX1xNWuylIQZIZWl5OZ5CTFBbjDBNRQOgpto9H7Jup83j7Tn7YfJ6H3WmeFUdYd/sJlSUREl9Y+onBOMuVVkJ7lCgvYFMKHeaWyFOkuaYbF4S+whaGA7MNZsuU9I+Gm/g23jV2TVZe1723cTt8METo/X6Nje1+4hf/+iOZDv5u8VFKTb9p5RAlJFlhEZT+I+sg04bVflp6Z0aPy0zpi0FB5Amm/QhtHOuDFEvpt1KuzALqJZok01xtDan6Eq0dnbjAWWjH5cXw8/OC5iBl51SNYW4YpJp+UtAcal5ZHdCNNVc7ZVUszcvuwEE6ZLkTY5xjtZeyrRGx1E9JC4txjng5J/QVwkEv1eSFY7zqsdbLiHNDN83R1nBwa2PTnP5hrkRtYKYe8Z33bec3H7/IlrGI8aqP76eEStIIPOqBx907xlnpJnhKERflUOL9Wxosbn8Ut/AbWKuxwqM18wgqFnhSoWRpWO7EKZ1Ys9jNWGjHXFqPmR7ldyWFZt90Kd3ONiLmGjmRr+hmBdO1gMVuwrbxCq+sDji/OqDqeyz1Uiq+hzaOuc4L3N97jmdbd5FufYhBUuBJwUQtwBhLLVRY63jywjqL3YStY6XEOzcW0ksLHHChOSTyvJLYBYqJis9CeDdr6l6agxRpywypQEqeLPbxJb0PC0z3E/7y9AoP7JpgqRvTifOyW7Hus7iekFtH4An2G8vVTkxrmLPezxir+KwOUq6sx8Aenq8f4g5boxVnWAdJbpiq+SAcpxZ7nFnuIqSgFnkYY+gOMzLtsGmONg6jBUI6LrViCmvpJQXL7hBPmUNEEnYZiAtNYSxCloOehRCcXe6TG4u1ZQefJx2hJxirKsarPrnxyKxl23iFXlrwXfdt58R8F5Tyb+Z6/UZDs58xSEsrwvErHXZNVt7VTLs3Ix7NfrbZiTffTvjD4wt89v4db0hSXtu9FyiJEO49mbl3q6WK3+rp4Deranerfy5vBzfJI1V21aW5IS0cSoGwpWeqPdSbJAoY+VggdJAVjrQwVHyBdaVE6CgDHdOs4OW1AdY6Jqo+mXFcWe0y347p545QgrbgXMyV9Zj7Csu+2TrtOOfEfIeL60PS3GK1IdcGYxzl96vRBqQE4UAqyK0l1xbnHMbA0MDltSGFdcw1QpqDlFpUozlISbQjyQ21oJwft2OiHBMS5x8NMjXMDIu9hNn2CY41X+Ry437aE/fRTnKacc4zl9r4sjxAlnspz1/tEkpJY/vD/ME9v8rW1rMsjD9AMXkvWzrr7JmpIxw8un+K5y51WB1kVD1F4JczF8cjj85Q4iycXxtweT0htxbhHPu31PGEIM4NcWZY6CQstWPWBjnbJyqs9jPmO0MOZGf4ofM/jnIF34LHL4tf4vFsH4EnSEZZBxXfY6WXI4RgtZtR9RVJbljqlAT68NYKUAZldhJNd5BztZ0Q+RIpJNVQjWQKwbaJkIVuwPowx1rIDcRpwQtX26S5RduCeuBxsV2as4NAoRC045znLrVYaMV4nmLLWEBaOF6c72CdIww8FrtD1gc5uS4v7XPtc3Kxhy8F59cG+J6gF1s8pXC5Jregc4cEtNPYHHqFBgeJNlhbHlOJhavdhHzjYDQwzDTPnF8nt5AVBk8pGqFHXhjiwpLmhvVhTiPy8aRk11SVJNe04oJhXoBzHw0DzIcAzX7GxfUhGwWf3Nj3babdxmttVJfeqml4phFeR5xOLfXe8LFvtXp0K6aK38x08FsZH6XP5aacyTNtWetlZe6NgMiTpIVFyZIUKQv56AfdV2BNeXusYZgWtIeOfESwAOICfOmIpCXJLNY5zq/1qfge9chnkOdl5ALQTTTGwjMXWyy2U9aHGcO0YDg6YQxSjTciXRbQpnyecpA7YBTOWI88hmlBOiLTC728rK7h0NpycEuduUZIJzEYa6kEPsY4zq8O2D5R4fK6+tBFH7w2yDTONQudmJ3Dk/zz/GfKUS6d/8g/S36OtWwfW8YlgTJoU3borQ8yto6FnF3t87F90xx88FuZrH47R0LFMDNUfclKt/RSVX2PyFcMU003Kdg+HjJe8Ym1Yazis2OiwitrQ6ZqAozkpYVeKeFKxb07x+klmoVWwlBb4rzgwprBWkd7WLBr+AzKvWoKn2w+xYrdTiUs85Kqo7Tufq9g60QFJCgB45FPJ85pDTO+er5J6CkKa9Hako0WmHEW4SzDXOMcdJKcscgnLywlrYJcG9ZH5ENbR4AoQzNtTm4cnnEEgeDqegyuDPD0lWS+FYOAOLNYICkKnhl0kbK8CJkbD2gOcubbQzLtiItSegtlaeBPRmvVjv4bpmWQB8BSv6zYqmu+78KUj1OUFxJSOKQStIc5cWbwFGTao5MWdJKCYW5Jck1SC/EVJJlmsZswyDTHL3duJ5t/gLhRV9z7NdPuRq91s0zDt6qB+bP37/jQEYRryev7VbX6qORY3ZSzuDGWTBuqoWKqUk6LX+4kzI1FrHRjBrlBaUdqoOpB15QkCmCYO157WbtxDBttKRzozJKsJ4xFJUFzlMQISoIG0M0sRWtA+poxqprypLTxGhuHZH7ND0U/M8zWfVIloHj1DgekmSYp4Cvn1pCeoBp4ZLlhbZAyUw/oZQVCiOuSqT8MZOpGA3CPX+6w3Es5MDhednuV1JOj+Qn+MttDvDZgbizk3p2TrPVTlBRY4OlLHeLM4HuS77xvO/fsnKQ1yDm3NODcao9GxedSc8hid0g/Naz1Mpy1PBZe4O7hi3y5OMKF5kOs9VPSwtAaZviqDP1UUtAcZPhS0BxmhEpQaIcQFk9JFlsJL0/cz2eEj3MFGo/nvbtJE4NOy+iK6XrI1rGIJDcstGK0KUfcRIFHP9dI2ByaXZgyMXwDhS3JUk1BkkOWGda1Za4e0k/1ZhXVUyCFwOIYZmW3oqVcqxZHJfBY65cVocxA6CzVQJEbg2G0PkdLT9lynbYHOZMVn2Fu0aOF64D0DepArw0kEIA/ImUOGBW5kBKMg6SwrPQyhHMIUZLLXlqQa1s+BhjmkOsMY2HvdI1eWqAEpEaXsee38YHgzbricmM5fqWzKb+9GzT72Ru+1tsxDW/IkBs5S5En35HM92E2MN9KBvnbeHu4KWfwwjoyDYPMcGxHnY/dMclfnVpBScgKS1Y4jAUJjNcjMp2SjipQb6QN3O3O8ag7zZPiCM+5Qxxz5/hEfpoL1Xv5a/YigeFrSNO1hOm627/O/meFox8XZLl73X2H9DkeVad5hiOcyA8RKocDstzQiQsiXzF4+QnmWk/T2/8pntn20IeiMnVtkOliJ+bE1Q4vzncIfMkVcRefHcUeaDyecXcReVANfBqBz3wrZlhoMl36cKSA82tD2nHOxbWYTxyY5qmL68y3E5Lc8ODeCV5Z6bHYTcmMIS0s9zTP8Y9GVa9Hhc+/m/gVxPjd1EMfKCs7q/2UxW7CVDUgKQz10GP3dJXCOpqDHIdjdZgxvPN+fjv8FbJXvsiz4igvFPvAOaqBh3Glz2jnVIVeqmkNUmqhpOIrZlrP81jvOF8x5RprKEHgOTwpuMee5RF5mqfsEU5wiCQviUVhy0pOOyk2Cb8SkGuo+BBIgZKCYWoptMNaMNqVJ5asPAYeEOd4lNOc4hjPsh8x2tIGKdsgRImBy60Ya77+Gr4RKh5UQ0k6vP7ZnqTcoBTUAo/CWlJdxlPUAkUlVBS5JhldtDgBxjmWewkvLfSQUuCsA2v1DV72Nt4HvFFX3AZyY9+15PUPHtm9uY0b4a2ahi+uDVgdvNrQudBJ3jBQ+c3wff/3V/Hf4H1/GA3Mt/HhwU0L5Ax8aIQKJSX/5cwag8yik4JE201ZDWB+fWQqf5PtPSDO8e+CL+BT4JD8P/rb+RHvL/HRFNkf8CPu87wcHGVYXP87nprXb+dReZonbXmifLP72tnr9+jV/Sjbx3/AfJ7nh4ewQCMUBEpytz3Ld77wz5C2wC38Bk984te51PjULV+Z2ggyXezEvLI6YKzic745pJfmnEv28E/kT/OYOs0J724uy8MEcU6j4pEby+ogRVuHsY44s2hTNhUU2uErwZ+cWKQd52TakhaGZy51SuO6EgxGJcSHOHldl9/O7rPM1+5m/2yd1jDHYZmoStJCs228SqoL1gelX2e2HmGcY7wSInDk2nLcHUB763xCnEZawdNiP0IIAiW4f9cUj+ybYqYW8bdnV7nQHLBreJJfyH8GT2l+VHn8QP55TuSHeMQ7x/f5T/Dt4d+gnEELj39ofoLj7iCZBkS53pPcgCjlFSFhzFdM13wutzQYRwH4oxEZ2jlCIXHA/eIcv72xpuwf8KPeT/GM20+/uPExMbgBuX+rCDxJxffxyBjtOopSFjQWbO7ouZy7d5QdgUoJpqsBWycqXFwb0EvjMgfMAc6VjxGCqVpYNozcrkh9YHijrrhr8W4lrxvJaBt4q6bhZj+7jkRtYEOGfLvYNVlhsZt+XQPzrWZIv9X25zbeHm7SmdtRaGjHBReaA2ZqAb6SdOOcVF9/gtjwlbz2UjYANg6/R+VpfAo84XDO8D94fwqAEg6c5gF3mueSQ2xcWN8IryNB+ec3ydSb3Xctyv0YBRo6zaPyNM+Z8nFp4RhkBY/WT2+GMxqT0z75N7Sn7qPZz27pylQ18Hho7xSX1od005xCQ2eYcaE5pJ8ZnjT7eCE6SN3z2Tdd5ZzWTNYCCuNItCZPDA5LN8m5Z2eDhXZGJy7K+/MyA6qwlkbFZ/dUBa1daYAuNJ6SnK3ciy5+D0bZSZcbDyCF4Nxyn9Ywo17xca6MxRikBQjHx/fPcGzHOEoKfvMrF7EGfE+ybbxCfe04P5b/LB6a78fjnwY/zQl9iNxojl9pUQ0Vz8+3aQ4ztHF8S+Usfq5R13y3kYRfl18gNAUCNxq1qPmEd4aXzEG0gMiXCAHalpEGroCJKEBbx0o/JTPgja6+rSllU2PLSlUg4JHXrKkH7CmedPuBkuRYGHnzrsdGteqtIpQQeYo4M5vPc0AlFNQCRVpolILAUwSeQjvDMC07bSdrfumbCxXOOaQUzNZDJmsBF3U50mmqHmCzYfcdL8DbeFvYkMUuNIdvug7ejeT1Zs/9he8+dkMisCFbbchYb0aW3kk450wj5J9+5uCbGphvNUP6rbY/t/H2cZOGFguEgGpQhnD2RoZiKLuJrr3G8T1IbyAIXKvSPWmP4JA4ZxCijKsRCLQTGCTbRZO7OcdzvJ78bFSa7hHnCcnLkrLTfLd6nEfdaVquzrerpwgoNonZtfdNicF1fws8cCXhetIe2XwdY+HO/DRq6fgotkFQCI+/TQ7y4PknmOs/z2r8X7H3vm99bz7k9wHVwKMRejzxcpNeUrAyGpxrrMO48j1FviTODcYK8gJ6aY5zouyEtNCyGc9ednx83xStOMdYx1I3pRoqFjspjYqHryRrg4RMOxySI9vHuefwt/GXzRm2tp/F7vkkY9W7aDiIfEXhDALJICu4a9sEO6ciOnHOdD2kOch57MA0P/HfHOVyK8YTgqcvtbhXv3Sdr+thTvNEvp+sgONXu3SSYjPSwljLF7M7+ewoVHPju/3kiORI3Eh2Fmg8vmoOb0Z6SFFKY9pAUUA1EIxVfHpJRjPdqECVn++GLFbY0pMmZLm2r11TT+g78VV5THjq1WaI1+K1J88HxDm+Wz0OwH8yn3rdhcBYRVGLFP2slOckECnYNVEhKSxCaHxP4nkS35NkeemdSnLLy2uDESE2KCnY2QjZM1Whnxm2jFeohx6f2D/DH1vzDS/tXRsTECi5mZv0XuLa6sbXw5tJXl/Ps/NGOUA7JipvmQC82T6+03DOr2dgvtUM6bfa/tzG28dNSzZ3lLk0E9WAmXrAMNNIpV5tLRo9LlCSVF9/sEmuP1E85w7xU8UP8y/930I4M7pSL08HAsffV3/D59Tjr6skXSsJqtEWS9Ov4HvVF1EYFKW5vWwXL4nZje6zCHJ8fq74QabE4HXy4H3iHL8Y/28ElCU3g+Rf6R9kqZ/yuRd/DB+NvfxrdKq/z8ShT72Hn/Y7x2u79ADWBhnOCiarIefXhiSFQZeqFWmhGWaKYVZ6ndaGMb6UPLZvisdfaTLINHNjEQ7YM1PjkbFp1vo5f3t2hVxbpms+eyfrVHxYGygaFUmWW2q+T5obFtURXt5yF74W3OEpQl8Ra0M/NnSShHrk88paH6WgOchIitIMnmnDI3dM88DuSdb6GXGueTy/k0+OfF0GjxfV3YBAiDJiY6mXMFWJWB9mZMbyRLGPf1H7ee5MX+CL+Z2cVgeZCTyM+QMEGo3kz73PcHLuOzi5OIfWZYUq1WW3Xj3yMbZgrBJgrKMb69dVkYprJO1hGZ3GixziB/LPb8rKJ9whhC4/b1x5AH+9KXaPqHP8tjdae8D3qi/y/flP8uxofQqgEXrEmSEeDfGOFMyOhWyZqLLezwlUirEW6RQ7Jyq84HfR1qFG5vTAkzSqPllucEKUhMyBLyW10KceKqzO03e1ID+EuNZErUbS7msjCd4L0/e1r3dtdePN8G4ze26UAyQFb2ubb+blejsk8+1IY7eaIf1W25/bePu4KZ4FT0nGAoUQ5Vyw/qjkJIWgFklqPgQCpms+tYpP9Bq6p8T1RMoDftd+hr+f/xRfscewCJQAgcPD4AmLTynHXItXpbiReVeUhOiU3YvCbt6uBBgEX7F383vmm153X/nX4VPw7eqpG3qsHpWn8Vw5b06M9q3hBjxgT+G50vsjXcHw7BdffdLVp+Dx/738+wFjo0vv1GKPZy61iEeBQlOt5/m2zu+wI34JT0nGKz5KlPEUoSeJ05xOnJHqMsvJOkeuDVMVH1+W8q0vJTsmK3hKcveOMSarIb6UhMrjwlqfc6t9OklGZ5hTGEugoDAWTwkevmOSLWMR26cqHN7WQDjYP1fDIcowShztYTYav9LGl4KrrfLfz1xqIQV0k4In9X5+VP40v+V/P1+Y+UXy7Q8TeRJjy+phlrtSVpSCeuAxUwt5Sh/kN+V380xxEGvgOAf5wswv8idTP8L/Ev5Lfkb/I/5gdRvDvJSuoZQSG5HPeCXAuLLLbbWX4nlyU9ITwJg/IkcjeKPbC8oLhV8138Vz7tAmaXKAs6+XvG+ETwdn8TCbMy59DJ/yT5exBqNtNYc5rbjA2DIywfdlmRCfFCy04zJuxJaNIgudlDCQ+ApqvkcjVAgEnhBsHY94YNckk/UAOTo4fCUxCISU6k128yOHPzy+cF2Wk3GvrxRaV1Yk3itcbSdveUD0G8lvbxWfvX8Hv/DdxzYrR4GS3DFde0vb3Gip3zVZeZ2xXABz9eAtd+29kTT2h8cXbvj4N6rC3SxD+q22P1BWI98s1+s2rsdNqUh5SnBo6xhRIJEILjQHxLkB56iFivGJiNVOjqdKg3YtVDgMiYZglOkjHWR2Y3ulv+QlcYhf1p/jkeAsvtOYEU9UzlLg8TV75DrvyKuySVmR0k5Q4PN77pu5k6ubt5vR7b+sPwfA59Tj4HIUZQULyh9JheOT8iUeCc6+rvr1pD2CRiFdeerTqE3p738aSTdGeBS7HiufcPUp+H+/E0wOKoAf+iPY9cj7+bVch2u79NaHWZnGvvIce/7k+9mtM75N+qSZzeUOAAAgAElEQVRz/5r5+jEK22KQaEKl6OdF+Zk4UFIwU48IPMnMeIWjO8ZYHxYc3T7GQjvl/GofKUuHz0wjoDkoW6kn/QBfObaNVcmNwfcVnpJQGBY7KYvt0tR8fnXAWORzbOcknaRgrOIT55aFTkrFG1UyRXki3zFZDjZe62f4SjBRCWj59/F45WE+eXCaQ4GHJ+Fvzq6WHaNS4HuSLY06K4N0tB1BXpSji5SEKPBpTt7Dv20eIPEtY8IS+hJfSArnGOaahu+NSJTBk+W++EqgnUNlFk25pnuv6Sg1wHhUSnivlbY3mi8MN44wgFezn6qBYm36YXTz95CjaqgWHldqD1KNIR7ViNLCjUbveHStJvIVtcBnz0yVy624lMxFOZjbOseuySr9VNMIFEGguHf7BKeXe0zWAqbrARebQ/qjqQG1yGOpEyNU8O5jqz9E+KW/OPumZu8NvJeVh7fqKwqUfFck6lrZb0NGeye4UcL5v/6ee97WNt+uNHarJWrfavvzUcf7Yey/KUQq8hV7piOUUpxfHWCsoxZIAl+xb7bG9vEKx+mQFoY7tzQ4udQdRSZYqoFkmFk8BeOBoJ86apGkGJbdfs+NpJCPy9N8zR7BUlaDnrJHeIFD110RnnCvyiYbPqcn7RFe4hDL4R3cVbxIX44xTZ8vFYc57srn/3DxeX5U/T6flC+hhEM7uOK2sEeslpWpa4zmGwb359wh/kH+k6/zqSjgv+cneVicJNn+Cf67AyMidenxkkQ5U/699PgHSqQ2uvTWhxlydILd2CeJBav5lvAsv+vdw86JChfNEIfAmJIoO+EQQuIrSScxNHsJ07WQA1vqTNdDkswyyDVZYSmsZf9MDSfKuW2DVKOdJdWGtDD0U00vLdg9VWXHRJWFdkKcGdpxxq6pKqGvOLZjnENbx1jqxvz2E5ephR6eMhzeOoaxjsVuSj8uODBXpxp4VHxJWhg6w5zza0P6iUEJxXgU0M801jmGmWZuPOLotjH2TFe5sh7zwnyXSmBxOKyxnFkajPbTYoxlQvhUQsWOakA/Lbhr5ziTVZ9nL7YJPI8k06jIZ0s9wpiYGoLCGHR+PSnyBCjpMVNXrHSza+PKNvGqrPwqNh6mKacFTFZ9lur38L8O/yV/V3+RXBu6hz7Hzqn7mDg+j9E5xlF+Hrp8D1LAVDUkM5ZXVvtYaxASQjWqOmZllMUg0XhSkhnNcj9hphGwd6bGsZ0T7JiolnEWRSkNb5mIwNlvqGTzG/mHbgQpxDuS9671W933c3/JIHvrFrT3w5v1TrGRcH5qqcfRbWPXeZzeCt6uNHarJWrfavvzUcb7Zey/OS1iDrqpYf9sRCPyGI98roxSm1d7GVIIxis+SW643IrR1rFrssbptI8QAk+VV95JPkpu9jzqkcZYR64dz9lDnBJl7EDu2Oyce200iQSOu0Ob9288puYLVhr38GL/EJPVgFgXrOtXSwYnvcP8X/ZzfEycxY1MwL9h/x4/oX4bf/T/T9sjVNT1EQvPuUM8p6+X/CoKXhR3cs4/ym4dcee5Jp86NMPM3k+VlaiNitTeD9Y3tdGld61HKt3xGL70wTi08Pjj3j50UPrE7tk5gScEJxY6WFv607ZOBGUHHaCkJFCwb6bOeOTxldV1Bqnm0JYG7WFOLynY2ogQoswHUKL8gR1mmum6z5ZGxLccnmOYGXxVNiv4nuTYznGmaiFy5D+ZqITsn6vje5JCW/bP1ZECfudrV1BSsNZPmWtENBsZFmgNMjypKGzO3ESFreMR8dqgHGGkNa1+htaWe3dNcGTbGHFhWB/ktOOcA3N11rplbIOzFl8Jju0co5sYZhsB2pTDkM+t9EFIIl9gjMCXgsVuTJJbjAWluM4/I4B6tTwuclNeNGgNkxVJN3mVi4wKf9eRKUVJyEr3VzlFoBIInhd38qJ3J12tOdKr019ZQwpRVn2lYLoa0k1ypJKIwtKJc8YrAWGk2DJeo5YXSAQztYB65NNtxehR6TFUkqVuyvbxKqcWBxycK4eBX23HKFn6p5bbCcKPqh/M6r35+MPjC2+5c9I4d8Mf82uvnAUlcdq4/ZlLLcw1G+8krylpvglCT7wnM+1uFbyT4be3WqL2zdyfb6Qg0PfL2H9zuvZGsslUPaQaJoxVfcK+Yvt4lWro4SvFtgkfNTphxoXBWEclUEzXQjJtiXxBNyk9LNvGKyz3Y9a6xWZyeapfbwALvfKEtJGRs2emisVxpZlsek0EEHgCBHhSMigMw9TiCYknyuR05ywviEP8z8HPcq9+ia+YI5z0D3HF7eHj6gxPmMMsT9xFEOfko462G34OgOcpoqBkhudWBvz7py5zqTnkex66i9r3/h71pScJ9n/TB1qNeiP0Zu/nq/f9n+Tnv8SL3jGeSPZyYDSfbX2QMzcWcXjrGJXR+Jgd4xXOLPdxQC3y2DfX4O/evRWA1X5Ba5gzzDVbxyvsnKqSj0zeE1HAdCNguZcigH2zdb7l8Bwz9QhI2ToeYV3ZHbR7qkaca554ZZ3Qk0ghODRXRzux6WNbG2R4QjDdCFkfZEzVfC61EuKsYLWf0Rqk+FJQC332ztTo5xpnHb3U4Kuym/DLLzf5zNEtHNsxwaXmgC1jEWOR5Mp6jBQOnEAoWOnlZWSCcyy2YnbGNZY6ZTp6Ny6wFhJjcAgaFcUgKwckT1Y9ltopI4WbqqdoRB4rvbQkI6oko/VQYJxlkDvKKFKohYK4cDgLgQeJftU7tdjLeeZym1rgk+nSJ7U+zEkKy2w9GgWGCrZOVKhGHjXfY8lLkBL2TlfK7lgL9aiOszAsNEJAZgxjkUcl8Ci0w2jLINOsDhLOLPdoDTImawFjkUcj9GlEClek8Qe9Zt8qhBD/NfDLlFz015xzv/hutvdLf3H2bcVPvPbH/LVXzg443xxy9Kf+jOJNflPeDFLAHdM1VgfZ23/yLYyPijT2jUBkbjbeL2P/TSFSaWFoD3IiTzFVDcuOrUa46Ym6b+ck55s9cLBnqgZO0E8Lpms+c42Iwmi6iaYWKtKinGehhIcQr16V1XxRhi+acjyMACKlKKRFaoevyhl/FsHsWFAOqAVCX7JtooaUksKUc/MApuvhKJ3bMFEJ8KTgkjrK8fQAga+4e7bGRO2T/EXzQVYHKTUpcZWQSmBZ7r8+cA7KH8dBVsomaVG2vnXjgtV+yhfPrrFz6k7k9jt5aMsUG5fyN+qkez/QHKSb5KQSlHMB66HH+sT9/FmwBYQjLxLW+hmN0OeuHXX2ztS52kqw1nK1nbJ7usow12ybqNJLMvZMVUbbzjDO8ulDcySF5qG9k6z1MlZ6Ka1YcWBrnaqveHT/NBXfY7YRbs72O7XYYyzyybTl/j0TxLnmz19cZr49pB4F7JqucufWBnFuWGgnXGwOWR+kXG0POd/sk+ty8PF45HF4a4MXrnaIfMV0PWD3dIXWMGP3VA1nLRebQ4yzjFV89s/VePLCOpEnyY3js/dv50or4Wo749xKn2FmsCkUeQ+LwJeStUEGQqCNxfck07WQXqaRCJJCkxWlsb0eKHZMVjEaUmto9cshxk4IHALf83EYhBD4ylGMlrnnlf4qISSBsljpaIQK58xmJVRSzvabrkVICak2VCMPbQv6Iyno7h1jTFVDhmnG5VbKZC1gqubzyP4ZQl/STwoCVY6nubIec7EZY50gzssuTCkMC52MpU5K4EnGKwHdpCAUjvl2zEw9Ii28clr5LQghhAL+DfB3gHngaSHEHznnTr3Tbb6TH+Zrn/NGYZdx8fbU0Y2q2EbUwkwj/MgRqdvS2G28VbyT6uVbwU0hUtVA8cmDs9wxU0cAg7wg8iWRr9g1FXF4e41LzT5xZji51GO84nHPzgm6cU419OgkOYlOqEhF1Xfct3OCEwttOnGOtgbjyq64WhRQsZa0sFQCxd6pGovdhCQvyqyewhB6igf3TPH0pXV8JZkbq7BnqjLqOitAQJpZ6hWfYaGJPA/fl4RS0IgCaqHPsZ3jHJhr0BrmXFwbMhZ61EIPX9uyS200ku9GfhYNeMKhdVkxW+mnrPZSjm4fozYa5jvI9CaRuHbe3fsV4LlR4bnYHDJRDdg6Fm6St9mxgKPbx+immr0zNQIlWWgn9JIyJ+rw1gYCwXgl4dCWBjsmq3TigqVOwunFPi8tdIkLw7nlPtbC4W1jfPrgLJBjKA3dFV+RFAbr2CRR8KoBfvtEtRw2nRmeurjOmeU+8+0hYxWfbpyjXJnNdLU15NH903QTxZ3bxjhxpUNhNF86t0aiLb3lMitqz3QNYxmtR8HTF1tICXONkMlqiHaWYW7oDgtmtzRoDgas9DNmGyHTjYCwKdGeRRtHqDxy5+ilRdmRmhT4qjTep4VG4JiqhUxUfbLC0Io1cWZ4ZSWmFkha/QInKGf7jQIutdZIIZmq+rTjcnuhNYxXQ45sqSGk5OxyjyQ3zDQitjQEZ5cH5KP1NsgsvoBtMzVognKCPdM1Ht4zxcX1AdvGq2hj2TYeUQk9puohd8zU2D/XYKGdYC28sjpg93SlDAv1BAe31FnqpISexFjLbC1EKYizcjbi+iCj4iuywnB0WwPfVzhT3PiK4ubjEeAV59wFACHEfwC+C3jHROqNfrC/3nM28F4Y0AMlNwcUH902Brx+pt3NJhyvDeiEG5uBvx5uNanuw45bbZ28V3i/qpc3hUglheHl1T6REqz0c8ajgOlqwCcPzNJOck4u9OmmmqM7x+ilmlCWUltzmFMbSUkHZuvkxrHcTcisY6IWsm/GsdzN8D3J/pka33HPNh4/t8rppUHZoithphFwpVlQWEuuLcNcM5gvEMC2sQoHtjR4aO8EvVSz0i27yMaigEf2TnJivs14NSQpNFPVgG0TFebbMZVAsXUsYudkhZMLHUDQzwqObB8bST+w0LnxVeAD4hzfbM/wZXmYV/zD+J5ESljrZ3SSdQ5vGSuN3rxBJ937QKQGmSb0JBNVn26SM1n1kQIurQ8JlOTv3bedC2tDjLa8tNShnRSM1wIW2zHDqs8LV7tIIUgLw/c8tJPLrZjVbsqO6QrPXGyRG8e+2QYX1oekheapiy1mGyH7Zmq8cKXDkxfW6Y1m012b+P5aA3xSaK6uxyS5Js4tB+cidk/X0M4xXQ84u2y4sDZkqhognKCXFoS+R5wVfObwFtYHOYEn2Ddbp3Puy/ClF3D1+1DqEDP1EAF8x7HtVALF+bU+vx/PE+eaiarPka1jHNraYLmb8tJ8l7jYCNR07Jms0EsLssJDypLMVH2Fto5IOrZPRAghaQ5yCpMyWfN5UL7MofQF/sru52kOsj5I6ac5oS9RUhAFkulGRD8rUFZsDg2erPlM1iKutmIir4wh2DpZobBwpTXAwf/P3psHx5mn932f9377vnCDAHiAIAkOOUMO51jOzo5md1ZrySV5tZIllxVFrlS0ihOr7DhRlSpWdFRi64wTSRUrkq0jkixHspRda0sqr6S9Mjuzs3MsZzhDDofkkCBB3Ec3+ni73zt/vOheHN0gSALoBvl+qqZ2SaL7ffHixfv79vP7Pt+HhC4jKxJHexIcSEdQZYnBbITDXbHA8O8EW+eHuqMko0pQTZJloqrExGIF2/W5NlckFVHwBZ/5sokiiTiez7OHc1ybLwa/Y6JEf1plrC/OoZ4oIgJ/+/4811a7K33X2b6RZ28ZBCbX/PkO8MzaLxAE4bPAZwGGh4fv+obNHth1NuZJweaH+f0IsbWIQmAoX1t9WiyZ6+IYdjpBeycEzMYcrPo5DqT0Hfd1hYKrORtjOx6mpPXdql62RUgposihrhiG43F7uUJcC9rWFysmkigw1hfn6lyZ5bJNRBV55kiOgVSU670lMrrK5dkitu1RsV3SUZknhlJcmRU4kNYpdDl8dLSLhK5wvD+J5YDpBG3nFdvB8X0UWUKUgpRm2RdIakGIz2BGZ7QnxtOHciyUTDzPR1MkJhbLGI6LKIqrPoM4jx1IktAVBlJRDndHGetLAPD27RQl00ES4LnRLv749dtIkoQirZ6D5TVu0LWjZ35Mkflx/6eZUk/hItCT1FElkbG+REMsNe2k2wXimhyIw5ROJqpyZjjN5emg4nF9vsyoEKc/pZOMyLw7vUJaVynWHFK6gyAKuD70JnU8fEqmw/G+BG/cWOabN5awHB/X85gvuniORyamoSlCMGfP8RjMRBjKBB1y6aiK59MQjBsN8LeXDMqWSyamUjYdhnMx+pIaNcfj7ckCihSY1s+MpEnoEm9MLBPXJSzHRZGDGII7+Srindf5iamfRPJtTgsKb6f/Jbd7nmRwTYUgG1PpTqiUTZd0JEh5qlouA6nAFza9bBCPyAyuGtavL1SYVmrYrk+lZlGxXcpmMBC5anuc6Isyt1JFEHxGzcv8TC0YyPxDosx/q/wMb7tHEUWBhK4hR4J7NhdXWa7o+JhYroiuiiwbDpJgUaw5nPKv8rR5iUXtHDe8Q0iShOd7xDWFYwMJXhrv4fZSFfCZXK5yR6wiCwJPjGSoOi6CD8mISCaqcn40R8V0g+CpVQu87brMF2skdZVsJKgauh48OZxBFAQcLxBtA+kId5YN8lWbgbTOmeEMvl+30e9PfN//beC3Ac6dO3dXh9LGB/barTVY33HX7GH+k586xj/7k7e3fX5RJdhydjyfwXQEXRY3beNN5qub4hg6LUG7fk3WUrVdJvPVh8og38k0i+3otPukFduppO1G9bItQspe9Z+c6k8xnI2iyBJl0yaqyVi2h+fBJ090E9UUdEVirC9oWXc8D8+Hs0NpupMa790pMV+scXmmxFLZZLQnScksB+3cqkR3QuPvnOpFVwQKVZulUo2q7QfbP4KIKApBIrfrYVgOw16MZEThwu0CmixSsz0OZKPosoTr+zx/tIvZFZOnDmYZzkVZKJmIFLFcn8vTRc4dzPIj50eYL5n0JDQqpsuR7hj5co2ZAiiiT0QWEEQfx4WPSe+jCKtjSgSHjypX+WbPRygaNjfmy+TiGjHt2xmGzTrpdoP6cRZWu4Tq404G0oFTqzuukYtr+Ldf57tXvsgF6THe9o/SHdfpTekUqw55w0RTIvQkNLriOt91qp/Xby5zqCtG3ghE6oXJArIoMF+0+N4nuvD8YEHwfNBViZrtElGkpoLRsBwW33+ZM7e+zHvqabq6nuDUYIoD2Qi3lwwMy+VwV5yK5VAxXUo1l7G+OGXL5ZnDOY73Jbk+XyYVUTl2/X0k324MRH7Mfpe/MR8nbTl861ae6wtlijWLUtWhN6lzabqIKE6jy8HWWzois6BKJCIqo31JFoo1yqZL1bJRZYlUVMX1fQqGhe14VC2X6ZUa/RmdY/1JXpj7z8i11YHMOHxcv4qTOccHc2VKNZua5RLTZPpTgejskrXV8MM4vgBF0+GscI3/3f65ICF//s+IDP5r3skeoWg4nB/t4vGhNGO9ScZ6k0wsVdBkiWxMY64UJMBnogqHumKUTKfxM4uqDsd74+QNG8dNE9dlZEnEqVpcna8ylI2hKxInD6SJ6yqCIHBpaoUP5kpcny8FPyfTpWY7RBS5k4cWTwFDa/58YPXvHoj6A7ve1r+Wess/NH+Yf/rMID//hUvkjeZFPFkMwozrAu1L/+N3rOu+WrtVVqdVxlQnJWi3Osft5GOFFaadYb8mrbezktYWIZXSFZ4+lOPxA2kmlw1milUgztMHcyyvZgP1JDQuTxexXK8hUtaKiLLpUDAcjvXHeed2IUgK12UOpKMc7YlzvD/ZqGJ89+kBFkomV2dLGI6LgE8qoiGJHjPFGgXDJl+xycQUTNsjqSsN0bD2XDwfRnJRhnPRIItIdYJKwZqttp6EvtpdBlHVYSgT4RsfCiQ0mWREoWoHMQ012+eidArH+xzgYPsyr1ij2KbDaE+cEwNJNFkMTMx+rSGc6v/tBbdWp8fbThABuVQxsR2X92eK9BUv8sJrP8awZ/GsoPCHY79OMtNPSlc5dyhLybB54Vg38fkLlF/7/+gbPs/BrmO4vk86qjKSixFR5YZg0hSJnoROTJOYL5mMDyTRVkVU/ftd6xGTpt7gI1//rxA9i08JCr8f+zWuzWdYKNXw/OCTSTqiEFEDIarIIh8/0ct0ocYTQ2m6Vzv45komM5kn8aYU8G0cZL4lnORoV4zetM670wWml2uUTBvbczEdl4oZDFsW8CjWbAQEDmWjRDQZkWBbNhNVcD2doXSUD2ZLLBo1DNPDdi08z2e0J4IkKliOy63kWbzCHyH4Dp4g4w1/lCORBCUz8EbdWCiiKyJ38jV6ExovHOtlYqlEOqJhOQ62JzAwdRmFQIwJvsM58RJm/1lGe+I8dTC3zmt2MBdjatlgqlDlUDbCqaE0MU1q3OOLJZPxgUA8nxnJNDx5V2dLXJpaQZQEPB8O5qJ0J1Twg4riStUOtgldj3cnVxBFAc/z6EtpHMrFEGVN35Mb9955AzgqCMIhAgH1D4B/2N5Tgp/9npP88z99e111QAB++Nlhrs2V7/n9Wo1jeVCj7U4KmFbneL9z9x6l1v6dYrcM2bvNz3/hUtsqae2ZtScI9Kd0hldFSSByilQsh4gicTAXY6FkMrNSIxNV8fEbImWtiBAFqJguI7kYfUkdl6Alvi6i1na4jeRidCcC0/TZoTQl0yGhybxzu8ClmSIHc0Gb/TOHs0GnV8UMDOq52GqFRt5UCdrOVlvV9kjHVFzXw/Q8+iIRVFlkYtHgqjLOP67+NCfsS405audFAU2WuL1kBLPiHJdcIoLtuIz1JdctiLvJRj/Woa4YAF+/VuX1D5f5ZP5rCF4Qzun7Nmf9S1z1zjOUjZKNq4gC2BPfRPncD6K5Fp6kcvqH/gxr4KnGdbq1JOH5NKpOhuVw4VaeiuURU0WeO9q97nstmw5VyyWiyOQWX0dcPb7k2wwXv8WC8ywVS+JId5ya6xFb7cyLqjK3lipUzKDyUr+Gzx3tZigbw7A+wQeDf8TsO3/DW8JjfGgfIer4CL5IueoyV6oG27p+8DPuTeoUqxZxXSamSSxXLFRJ5NxImv50jKWKyexKDcf1sdygUcFyXEzbI6oqJHUZRVJ4fCjF1bkyqeGP8Ovur3Lcush7yins2GPUVu/3mCKxXDHpTelYtkdPUmMgo9MdV8nGtSCPynapHDiPd/NPETwHQVYYffrvMNw/ssX9EmzZaatdkWt/3tMFg1evL5GMKOuaGioZl75UBB8BSRCCrduaw+RyBfC5tVgGITDrVy2X7qSO5wdBuY4v4HtORw4t9n3fEQThnwBfJIg/+F3f9y+1+bT49JlBfuNL17ixWGl03v3yD5zm02cGm1ac7sZQJsLN1Q9HdbZrtN1oAK9vHe40Q5kI0yu1TWbggVSnavCHj5/81LFNAr7T4yQ+f2GqZfV2LyppbRFS9Xb6+gN+JPfth3l9kX13Ms/rN5fwPJ/hbIwnRzKN19cF0vhAEs//toCpb0XVv+bNieCBbjoe50dzjUrRlZnVT96CybOjOU4NpYFvd4jl4tom0dSsEnS3rbay6QQl/KhCvmKS1hSGMxGuzZdxPJ+kLPANc5Sv+KON19zJVzncFUcSBVzf59qCQU8yyrtzZSqWSzam7lq33lo2isT6z6fm2CwaJq9543xyNfbRQeaPZoao+gtEVImPJHJcni6Su/Rlel0LAQ/Bs5Fvv7puIPPGa3drqcKVuTJxVWYy7zDWl2Qkt144X55eoWp7HJYe47ik4rk2niiz3PM0uiwRU0VuLpa4MlPiRH+Cy9P+pmrm2ms3X69gySd4v3+AquNxzPfoS2uIeBiOg+14ZOMKx3rjHO5OoCkiV+dKjGRjFAyLZw93UTBsMjGNmuNQs1wKVQtdFjmQjVCxbG6aLqbro/ke2bjG9z95gEREoSuuIwgCb+jjzKROsVxxOKnI9CR0HLdI3rBJ6ioxReJYb4IffmYETZG4OluiVLOZXDIYyETJnHqBxTP/kdzi66hHXqB/i9yxsukEDRmpBEsVs/F7YzteMKTZ8dBW4xrWNjV0JzTGBxLMrNTIxRVUSWK0J06haqMq0mpFTsCKuKti2iemygxnI8RUEUGU2hMAvA183/8r4K/afR4bWetzqqd+P8h7AQ1hdjejbX3WWrM06I3z8XaKroTGT3zi6CYz8P2ModmNUSCdyk5X3tb+eDNRhZ/9npO7du124py3mle5F5W0HXmw3WuYnSwKWwqBsung+AJjPUks16M3pTXU8drtnXqV5tsVjuDT1q2lCiO5GFXLZbZoUjCCruuXxns3VVqC7brYuuPfy/bZVl8b12SiikxMl8jGNU70JVgsmZiOi6KIrNQcZIl1k2c93+XmYgXT9RjORqnZwZ/x/ca8uN3q1tv4fa0VHwDTeYNLd1YoV22uKsf5pe5f5Jh5kVecY7zHUcZcD8NymS8F11U4+DzuB78JnoMvKsiHP7bpGGu37ZbKJrbtImgyzTKhK6aL7wskdJkZ5TS3v+c/4N/8OnPZc8RTj3N+NAfAf353FlkQyBsOmiw1rWbC+qqbabsc7onz4UIFAYmkpqLIAif7UsgIJKMyjw9nePF4D7eXDGRJYCAd4e3JFQqGzfX5Eq/dWMR2POaKNRRZQpNEZldqjA+kOJCJka+Y1GyPjxzpoli1Sa12Q84Va6xUgwiQqmmzUg1iPApGEB17qDvGwVyUpw9lOZCNcnu5wnSxSjYSmPF1RUJRRKThZ1Afe+GuP9u1Itl2PK7OFlFkCfA51BVrbPNtrLQG90SOquWiKhK6LJKJqRRrNlXbxXF9ZAGO9CQ4dSBNNq42QlMBPNvs2EDOR4W6MBvvTzZdwDamqauy0DTTyvODgNDdECjNzMD3KqR2axTIRrYjYPbT9mL9uq0NfK3dY3ZZO9iq6rQXlbQHXo13IsxuYz7S+ECSmCpieR4IPpmIuikCIKbKvDa10qjSjORi6wQSBOMxCoZFOv4X5OoAACAASURBVKqgKWJDFOxF5xsEC89YX4Jlw2IoY7NYMbHcIBxRkx0KhoWn+ZiOQ70ouViyGbXf4LtWXiM+KzMz8mmimfNkYgoV0931c954/vUt0leuLfKNG0t8uFDBcjx0RUY99BG+UTzD1ekStuUwW6xhOz655bfh0pcp9j7Dhe/4fY5WLxI5+gL64eYPkvr7LxtBInhElRhMR+husnWgKCIJTaZkOghDz9Az/jFipsPp1UrTfKlGd0LD9f1GdEOr61W/F+xbrzE2/02eGHsBVT4UJIsrEqbjMLVioKki6YjKs4dzRFWZ6bzB1dkyV+dKHOmKU12dtVexPLqiCjMrNTzPp+zY6DWJZM0lHVWCbDFJ5GhvgjcmlplZqZGNqhzujnH6QIqoqtCb1Hn+aPdqjlbQlDCZN1go1SjVHL58eY7LMyvMrJhYThDSiQC6LDHen2S+VLtrI8JakVy1AqFe/72JqBJdcb3pVjZATJNYqTq4ho0kCJwf7Wq813h/MmjUUEQiirS5ctqhW3shAc3S1E3H3zKGoVNa4zeKld0aBfKwsbFqZ1jOvrxurXxd6YiyJ+e9EyvyA4fZNasSPXe0m7G+oNNlrc+jvvhNFarrqjTApq2oeoWi/mCvLwp70flWpzuh0Z/SSUcU0lGVoWyUL78/T9myOdwVY75kIgg1FssOZ4WrfD8v8wPWV1FsF0rgLPwl/2751+gZf55DB9J75pFaS9l0qFgOnuexWLbxfJ9ao6U7ys2FCnFdJxWRGHffp+vPf5Ruz8K/olL6B39GeuyngNap7AslkyuzRRK6giwK65oF1r6mO6FxvDdOxfIYTOuNa7H2vTZGN5wfzbW8XlFV5mn5Q7Sv/Jfg2nDp/+TIi3+AMvIsSxWTAT2C4wsNn56mSIGfb77MimGxbFgk9WCsTDHpcGN1Rt9gJkJEkShbLmcOpBnMRRlKR4iqMpPLFaZXqkzlDWKazM1lg9HeBGeHM+QNG0HQGlEaFydXmFgp47o+EUUkG1e5MlPC8eCZwzkuTRfIxTQOdccxXZcLtwubfE2tWCuSby1Vmlafmr3e82G0J05Ekanawe/t2q9tti0esj9olaYuCYHNoBW7vdDezxbdfu0820uaVe1a0enXrVXQ5s9978k9Of5OPOnuGmYH6wPthoaH131yblYliqryOn9MnbWt+TFValRpuhPaOp9V/eFe3867m99pt9go3AzLwXJcrs6W8VyffDUYlfOkcJk/VP8VKjYifmOPWsJBuv0Kb8VOce5Qa1GwE7QSOnEtiJ74YK6M4wYddl0JjZlSle64Rndc42hfAgFwb36hYUL3Vn1RjD3fdEt2nShcnR4S5CV9eyTMxiT35452r/PCbeRehbI+9WogonwX34XM/De51XUGUYDhXLQh8uv3ZdVyqZgO8yUTx/eYX6nRFdfw8Tnen2QgrfOJ4714Prw7FQibiCI1hOFwLsqVmSJTy1U0WcRyXSKqxJmRTGMkT71LtR4Z0Z/SuTJXYrpQIxNRWFRE5ks1EpqM5XjcWqpQtVxO9CfuOaz1Xq9XXah6q5XDjdW+vfzdCtlZWi2Wru83pg3c62sflK226LbKAuqUzrNO9mm1Es7NeNDrttvbm81y237hM6f27Frv2RNvbaDd6Pjj/oVb+XWm83t5mNdF1kbhVP+3jV/b7gf72nMomw6HuhI8NpjmndsFijUby/F53ng/aF8XfHw/GFwLYCPxuj9Owt7dXZFW42cMy2GhZGLaHgdzUWzHxfY8NFnCMD2iWZmYLtOVUEnpKkX5GZwb/xbJt/EEGWf4PIblMLG62GdjGi/fWma6WGMgGeG5o11NK031a7UxyT2uyeu8cM0qL/f0Mz/4PEgquBaCpNL/+CcRU7HG+6zN04JAsA9mI1ycWlmtOjkMpHVimtyojmqKRESVePF4T6MZYu355OIaY71xDMclrsmNUUD16s5SxWKhZDKcizbM8Md74w3x+fhwmlvLBnE1GGocUWQKVRMQ7mvL+l49gffyu7pWnId0Nq3ER93wXV+kWr12N9hqi+5ApvUxO2GQ8V75tO6X7YrfTu/Yq7Mxt20vr/FOPN3uOczOcgITeH2G2/3mI3WCSLpX6tW3iunSl9TpS0WYWDR41T3BZ2UZfAcXka8LT7Dopfm89zxz8ZP816tZQLtFM9EC8ObEMssVC+/2N/mu0gWG4qcoZJ/gSE+ca/MlfGAgE+FEf5JsTOVV8yRv9P8yJ613KfQ8xVPJ08yvdk9eny+TippM5avENYUrc0XG+hKM5GI8d7R7W/ESOz4mZ+hp+NG/gImX4eDzeL1nubUqKG8tVRgfSDaE29XZImN9Sc4OZ7izXFsVBz79qSiSWKNiuuvM2xu32NYOgk4vv82BmddY7H6K164/iS/AjYUSL1+r0hNXsV2XF4/3bBIti+VaIzC2ulqN9QiyudZ2se7m78VWv3cbhdNacd7BgZwhtB5rU6+i/IfXb7NYMpvGE+zWQrvVFt1WQmovBhnfrdrU6T6trXxFhuW2pbJzNzrVuL8TT9t7DrOTROGuRuCHlY1GXxefsmnzhaUx/gvrf+JZ8X3e9E9wST6OCKTiMv/o2SG+87HeXV0cW4mWquXSlX+H737/J5B8m2cFhT9Wfx2h9zyLq915uiyxVA6Sv28tGkwqJ5hJnmYgobNcthp5YAdyUeKaxHwxyNIy13y6vZd4iR1vFhh6OvgPKK9WgOp5SldmS41KWr25IaZKnB1K4/gQU0WGc1G6EirzJRMRgdlirfH6iaVKI+X+jZvLTBWqHLUu8+ybn0XwbPwPFf74+G+gHfoIR3sTzBdNREFkZiUQXXWfH2wcJq3Qtzqepr7F1u4PFRurmvXO2YgiM1c0ENRI7O7vEtIuNoqPetfe2kW0VTzBbi20D7JFt5uDjLdTbdorn9b9bh9u5StqV2Vnv/LAT977CbNTJJGDudiWRuCHmfr3vFy2eG9qhWvzwXDZt/0xLrhjqBKc7EuwWLLIxVUWSjYXbuU3BVTu9DltFC2G5fDOZIGTN77SGKEi+A4jpQsUlefpS+scSMVA8HE9Hz0iEVFFkrqCabtULZeJxQpv3c5jux69CZ2/f24Iz/U3beNtdV5rv+dmsQwbO9Vaeb3qbPXvdUE5XTC4Pl/mQDbCneUqXQl7XXPD2BoBUxc4miwirs6mq7++5rh85f15uhMad/IGyYhM4vZrCF5wPV3PZqR4gUnnWa7lg2wow3Y5PZgC/HXBmCO52OowaXX1g4jaluaDVmysFtarkKWazXtTK4hqLHn3dwlpJ2vFRz1Haquv2e3KwFZbdPeTLbVTbKfatBc+rQfZPtyqatfOa7sf2ZEn8L2G2UW1wATeKQvAXlP/5D67UuP2YoWELoEL9V9LWRBYMSxEEbIxBUHwmCuZTCxVGknru8HGXKersyVuLVWQ9NO8tBq+6QoyH8YeJ1KzWSxZyKKE5/kMZAQEUUBApC+lE9cVUhEFx4OD2TgTSxVc3+fKbInzo7l1W1B3Ez6tzrOZrwto6vWq08oLtva9zx3MMrFUAYL5groi0R3XyEbVdc0N9XNoViWqWMHWqK5K3FoyyERU5komyYhKpf8j+Hd+F88LRsK4I+c5dzDDlVmZUwdSvHUrTyKiAMK6YEwIwmz7khqZqNJxH0Q2VjUjahDYOV8ymSpU8T3Havc5huwvHnSx3y2ht51q0174tB50+3AvRfHDTFuewncL5HzYqX9yz0RURFHAcnyQQHRBFkEUfXriGo4HK1WHb91eYXK5RkKXWSyZu5Js3szbcnWuyGLFRM+c5mdT/5Jnpfd5VznFsniMQ2WTZw5nSUVU3ptaQZEEBB++78wgV2ZLaIqIiIBpO0yvVPHwGEhF0FaHEvck9MZxtxI227mOG31dW3motuOxiqoyB3MxFktmkK202nVXf/1awVc2naZVom40FlcHAkuCQNVx6EtqnDqQInb4JT5v/ia9y28wET/Ldzz5CXJxrWEsf3I4y1hfYlMwZrPO1E6iWbXw1pJEKiKjyZ2dbB7SuXTiYr+datNe+LTCmIfOIHywtYH6J/eq4yCJ4urUM5DEQAS4voAgCZi2y4FshKgiIUsiIFC1dz7ZvJm3pZ74LiIwUzAwtXEq6Sd4706ZjFfFsDwSukyx5lAxXQbSEVwP0jF1XeQEwGhvcl0UwFpf04OYx1uFq27lodpuIGsrf9bGcxMFKNZsJMGnP6WvqxJtDKpM6grzRZORnExi9Dy+8jFSVZOS6ZCLa02P1ywYcztG73aJrI1bseMDSeZLJsf7kvzrWqnQlpMK6Xj+5Mc/cl/zA6E9EQPbrTbttgjslJiHR522CalOeOi3i/oifWWmyFA2AnhoiojgCziuR1yTUUSRfKWK7XnEFBlBFEhFZRKaum7u4E6wUcxAfSC0xxNDaQazMd6aWOLVDxcp14Lk21RUJl8xmVmxWDIs1OsCZ4ez6zK86hzvTzYymZplVN2vebyV2NmqPf9u7fvN7st6pav+/9duSV64lWdy2cByPE4OpjZtE0bVIG1dk8UgxHJ1yy+iSlQth8nlKposNSqN9Urdxve4Gw9S2dstDMvh8nSxcU6+Y7UOAAsJuQ8WS2ZbIgb2otq0HToh5iGkTULK8/09eeh3sliLqsFwZEUK4jcTqtxI9q45Hh8uVjBdlyOJOEPZ6Oova4xURF43lXsnaDaguDuhNUJPPfxgjpooEFElbNfD82ChbDNbCubjLZZrDGWj67KngEZGEtDUGH2vuUQbadXtt9X7bBRIa31hG0cV1YWA7biAgCKLjXs2SHz3yEQ1BCGoJjarqIkCXF8dVC2LAk8ezGzyYdWHB0fU+7sOOx4LsQNsEuiCFMYf7BMWSyblmoMPjSpPM9rdjj6Zr27KttqriIHtVpt289rUv8f//k/e3tYg6nuh1TzGkM20SUht7WPZCTrxE/pGuhMaQ9kopZrDmeEsZdMmqklMLhrENYl8xadk2jieT9VymS/VKFYlzo/urEhsJWbqoacTSxWO9iZ4904B03cREBnJRbFdn0LFRBAFitVALNXn5l2ZLWK7Po7rIovBUN1D2QinhjKbBNVe54GtzXNaGwpbj3vQ1SDFuT58ORfT+HChDPgcSSXWhYPGVJHJvIPjuMS0IDtqI/WxKoIgkDcsKqZLVJWJqTKiIGwaHrzWOF82nSAr6i75UHs5Q3K7bDwn/BZpjiEdxWLJ5OZSpTE2vF7lGUjpdO1ilt390Cog9FHyCH36zCD/8396D4BXfurjbT6bR5O2PG1FYb2PRRQ2t7A/KJ34CX0jUVXmxeM9aLJEsWZzZ9lgKBflzwuTzJUtDNNFMxySmsyZ4RSpiEbesFgsWcyXijsqEluJmbrxuj+tc7A7Rt6w6U5oxHUFz/XJxlUSmsJQVieiSo25fAldoVRzWK5Y9CZlNEnkvekSjg/ZmHrf5/ygArJZl1393qhXjmq2i+P5HMrFGvdpTA08ahvHGD13tJuhbIz3plZI6PLqeJdm25cCV+aKgMC7k3k0RUaRRcDnUFcQr7R2ePBCyWyMfrk+Xw7m26lNBgGv+Tnt5QzJ7bDJfO77oZDaB0zmq5uq3lXbZTJf3VUhdT/VD1USm4qp0CMUspe0SUgJjQesKLDOR7FTlaNO/ITejK64zkvjvcEw3NkSHj6jvQkQypgRmYSmMFuqkY4r3FgwGK68h/3hu6j9zxI78tyeiMSoKvP4gTTX5kvcWaqwWLKZWKxguR7pqEZalxkfSDcyoWKqzGTewHZ9krqE4/pULAtVEho5TPdzzltWGSdfb6ST18M1m9Gsy65+b3g+DGUj3F6u4tvupqiG+us3Gr+zcZWuhNZStEdVmbG+BBXLZXDVHOpYTqO6Vc+jWjs8uH4+uiqtzjqT8Xx/y+u215W97dCJ5xSyNa2qPK3+fjfYrqgaykT2NGk95OFgp7cs2/aEW2vE3Y3KUSd+Qm9FVF0/OzAdUfidl28wla9SMh20is3V6RInvA/4gQ//KZJn41/9Ld6R/wCx+8yui8TFco2/vDjDXN4kbzhoqshAKpg315+K4As+Q9lI4xo/d7SLsb4EEHikFssmyxWLpbLVyGG6n3NuWWWcfB3+7+8F1wrm5v3oX7QUU/Whu82ymIJzEnA9n+6EtimqAZp3zG1HtHcnNLIxhYrltKxubY4OqHw7PsF2mg4J3mvupSLYyR7FkNa0qvKoHWhx2+uk9Y2EHqIQ6ID4g92sHO23T8OG5TB76WsY736ZI/4JbnmHkASfuCZStl36im8hejYiHr5vM1Z9G/HgJ3b1ezQshyuzJYo1m2xCxXBdZgtVPpgvUTRslss1bM/ntQ+XOZCJ0hXXG8Kw/vr5YnF1lIzIoa5YyzTuuy28Le+ViZcDEeW7wf9OvNxSSG0lsKOq3BjJoinipqiGVmxHtNe/Zq0Jf6PvaeP9Wn/PJ0cyezJD727ci++wVVhqyN4y3n/vYfJDmQg3V+dL1okoEgMpvfWL2kgn5kyFfJt2xFPsNW1XGfupcrSbLJZrfPVv/5LvfecfI/k2zwkK4sCv8i1vlFxcZ26lxkz6LN7yHyB4DoKkED/2IuyyiHpzYpm8YTFfNFksB9XD3qTO+dEu3p8pMlesIQCXZ1b4yhWVF4/3rFv0N1aRIqrUUkTdbZFuea8cfD6oRNUrUgef3/L72kpg17da7/V+3K5ovzpbWq1KyTx3tOuu3YWd9PtwL77DVmGpIZ1P3Qd1Y7GyrhOsE8eGXJ4p3nf+VEhAM/G5U4L0QUbY7Cc64indaQtGO5gvmfQsv4HsBxUnfIvvXPg9CrF/yFLlDF0JldjAeb4x8Ht0Lb7OwBMvkd7CC7QT1BfDw10JnhwJpr4f6o4xt1IjF9d4/ECGd+4sY9o+UVWiZnvrZsOdO5jddsVxu4t003tl6OlgO28bHqntsJ378X62rRZKJldmiyR0hcm8wVhfolG52w/cS/V4v3gUQ5rTldCYLwfV03on2FohVV9o2yVi1lY5LtwuMJSJ7IgRvt1xDg8bDzrCZr8QPt06hIQm8yXxMZ4VFETfQsLnMesCx6z3+Dn/XzGffpzjfT6XhWMMjD3OIirnrN01mZu2y51lg4VSjZWqTVSVWa5YjGSjDGUiSDmBD+eLrBg1BCFoIlg7G65sOvQk9G1VHB944R16eksB1Uz43K+H536jNaqWS8V00WQJaJKT0OHcS/U4rDR3Jg+SIN4pbKxyWK7HzdVMtv3CTgu2+9nC3QselRE24dOtQ9AUid6TH+MLmd9k/Oq/4VjlLSR8fByOGm/zqnmYuWINXZZ44XgvmiztarfeYrnGn715h2LNwnQ8zgynOZhLcPFOgTv5Kh4+NxYMVEXkxECSrrjGkwfTzBfNTWLobhWeuqAZH0juihfofoYbb8X9RGsYlsPksoEqCywbFo/1JxpdjvuJe6keh5XmkN2gWZXD84PYhrvxMFacOtmD9KiMsOm8NoxHFFEItn4m9JN8Lvkj2Cg4iDjIvOqewLBcXA/myjUuzxRYLJtULRfD2h3vye0lg9vLFRQpSCYv1RxmVgy+dTvPldkiX/1gkbJlc6h6mTO3fpeB8kWGszHOHcwyPpDctjCpi5zL00UuTxd3pXqxVvjU08eb/d12uZ/qWdl0UGSR58e6OdGf5NRQJhQZjwgbE6L/5Mc/smcL+V4ea69oVc2413iGH/qtb6yrztUFyTdvLvPcL36Zz1+YeqDz3AtaeZA65dx/8lPHiCjSur97GOMpwid5h1BPv7Ycj/93+gT/TP15jtXe5pJ2itvaODlRQFUl3KqN5/lMFapcmSlya6l1SOODEFVlRFGgYjnIUjBHr2w59CY1ZldMFks1+ovv8Jn5n0Lybfzp32Oh/z+SGntu07y4rdiL4NT7GW68Ffe6bWVYQWK67XhUgExU2ZfVqJD9y26IqXZVQlpVOe4Wz7D2fJ/4+b+mbDo4ns9zv/hlXjzezZ+/NbXvTNGd7kHayZmE273f2jHaJhRSHUI93+hOwWB+xSQvjPGKcBjFFTgYVYjIEh5wIB3h+aNdLJRtdFVqOd/tQelKqJwdzlC1XfoSGmN9icb4F0mEoWyUj5WuIvk2Eh6uZ3Pnwl9zTT1xT8JuL0zJrYTPTs/4a8babcV6inmr+IeQkE7hbrP22tmN1WxQrygEsQ2t2Hi+hard+LepQpV//9pt/A2v6SRB0or94EHaiXiKTu/+C5/mHUJ9sXdcD1EEy/GwXA9JFNBlEUmUcF0Hy/N5b6ZIWleo2e62s47uBcNyuHArjyKJqLLIs6NdDeHwfWcG+dL7c+SSGiszT+Pe+V3wbWxk3lNOka3aTCxVOJiLbUss7JUpuV1+ne3GP4SE7CQP8om81ay9X/jMqcai1c5KyMYqhyqJ/PIPnN7yuM3Ody2t5sB3kiBpxqPiQer0ylv4RO8goqrMkZ4EZ4czXJgsMLNikNAVXF+gXLM40Z9gulDDtD0G+mIc70sS06SGv2enFuiFksm7UyvIkojjelRMl674amBoscbh7jim43Hs3Ev8QfHXSM5+k/fVUwjxU8xPF9ElkcWSua4ytVWHXDtEzl4NtQ5jAB5tNnqj9gOtZu39yhc/aFQW2l0JqVc5Ls8UGe9P3nUxvd/z2g1BspNbos2qcw+jB6nd99vdCJ/qHUZMk3B8n2XDpGa7eB50xzQyMZWFkonjehzujtOdVIHdmVNYtR2m8lV0VaJmuVTtQKgtlExmV2pIQlAaH85GyRx7jneij6HKAv1pne64xkA6ui6EMZgjWESRJUSBXevOuxd2ypt1twiFMAYgZL/RyrQ9XahyYHX7bL9VQlqd71oE1lem6oJkJzv9dnqLaic9SJ3Mdu+3dn1YCbv2OoyK6VIxHVRJJKoqeJ5PoebQG9f45IleXhrvY7Qn3uiEuN/Os62IKDLn5Ov83ZX/h3PydSKKjGE5XLxT4GsfzPHvvj7B31ya5XMXplAkke95YoCjvUlO9KWIKFKj+iIKQcTAhdt5rsyViakyVcvl1etLXJ4u8ubE8q51Hd6NnagUre043Op7iaoyPQk9FFEh+4JWpu21i9Z+68Zqdr5riSgSP/zscON7H0xH1m1l7hRbbVHdL58+M8iZ4TTPHMryyk99/KETUdD591v4ZO9AkrqC4Ae/YDFFCsIvJYGS6RJRZEayMYZzUSAYbLvT20a9Kxf5wcv/BMG1eEZUmT/2pyxwDsNySEZUkrrFwe44oihQrDpoisV0vgr4RFWZsb5kYwCz58NgJsJk3mCqUEUU2BTa2Q6BsROVor3oOAx5eOnUrb5Ws/bWjolp5lPaDeGxU2w833REaXTtra3iXJsrA7v3s+n0LapOpdMrb+FTv8OIaRJxTWIgHQilVEQjooksl008DwzbwXJduhIDdMW3lxp+r+hTr+J7NgIenmdTvfo1LjtHuLVksFwxyRsO6bJFb7/OC8e6WSiZ+PhULY/5UoXTB9KNcxGFoMp2vDfJWF+CmCZxebrYEZ6hB/Vmra1q2Y7byPUKxVTITnG3USz3s+Df7TWtZu2t7b6Ce/cptZtPnxls+znuty3RTqKTh1OHT/wOw/NhpCuOLErkYgoLhoVhebieh+l4lGsud/IGr15f4qXx3sai/aCG83Ven4PPI0gqvmvhiwr+yEcxHA9VFulPRVBFkVRU4fxoDk2pd6EJlGoOtvvtj7GtYwfkfe8ZWpvGXjFdrs6WuLlY4dZSZdeM6yEhW7GTi0uzWXvtpF0L505nZT0q5vBHjfBp32HENZlsVOHydIG5ogkipHSZlaqDj09MExlIRdAUkbLpYFgOr15fQpNFIur9hXNu7mA7S/RH/wL7w6/xtniSCX+Uy1Mr3FqqsGI42J5Hv6vze6/c5KOjXUQUCcd1V+MYRGLat/eym1V99vvokI3XayQXQ+mA7cqQe0cQhL8P/BxwAnja9/0323tG26PTPpG3g3rWVT2JfCe2etZe193ILur0LaqQ+6PtT/v7HRz7sBJVZY73J/nKB/NYnkffykVetG+w0v00XSc+ylShSldSJ6IEHXBfvTLP1bkyPQmNoVz0vhbxutcnpklMF2oslExGhp7G6T3L5LvTXJ0tcWu5QkSVmCtaiAJEVIXZFYPJfBUBgcFMlOFcNOg0bBXKssO0697Z6I2C+09JD2k77wGfAX6r3SeyFaFwWs/nL0w1zbqCnQto3K3sor3aonoY5wp2Km194u9Vls9+Y6FssmJYjDtX+BfVn0WpOvhLf8hrXb9DInEay3Y5NJTm9pLBhwsVDNPlUrlIKqrc1yIe12Rsx+O1qQIgEFVEuhMat5crvHp9CdvxuJOvkomo6IqA5fh4roMAJPTgtRFFwvfZlYDQZrTz3tnY8ded0Brm+vADwf7C9/33AQRBaPephNwDv/LFD1pmXe2UkAqN4SHbpa1P/LDrqTkiMFM0ebz4LWQcpFXTtz/xdfKHTzBXqlGquri+x3S+xtHeOGXL5tRg+r6uX9Bpl6BiuQymI1RWKz3ThSp5w6InqRNTJOJRhSO5WND1ElXpyzh0J3SyUZUzI+k9zYZq573TyvsV3rsPL4IgfBb4LMDw8HCbzyZkL0TOfjKGh1Wn9tLWHKkw9bk5mZjGkZ4Y76qncJBxEXFFmWuRJ5gsGNxZMvjmzQVMxwXBQ1dFnhzONiIR7ofuhEY2plCxHEQBTNvlzYk8hulwba5ETFeIyjIVyyUVUfjoaDfnRrKcOpDiuaNddMX1RlaSYTnMl2q7mhHV7nsnzIbaPwiC8LeCILzX5L+/t9338H3/t33fP+f7/rnu7u7dPN19xUaf0ucvTO3JcVuJmZ0UOZ2eXRTSObR1FQhTn5sT0yRUUeBr9hF+zP9pPqZ+QPdjn8BMPY5SqjHaG+dO3sByfHJxnfH+JE8MZx74+o3kYkAgqiaWKmiyxHc/PsDrN5YoGDZRVWSpbHK0J07F35vC2QAAIABJREFUcoio0qaZenu15daJ907o9+tMfN9/qd3nsJ+oVzdO/dwXt/y6vfApteInP3WMf/6nbzfNutopQmP4zvGwV8za/rTf7x1cu4HnQzKmkdZlluKP8+fuaV6K9DEcV5hdqTKQjQA+FdMhrklUzNbDOLfDWvFjO0GHiiwIlGo28YVv8d3FN7koPUZZOYvmiBzrSzLaG28qGPZyy62T7p3Q7xfyqLEXPqVWfPrMIL/xpWtNs652+jidml0U0jmET/oOJK7JpDWFquMjuR6yKCJ4MF+0iKgiePCdJ/uZWKowmIlQMd0HEixru/ZemyqQNywmFiukF7/Fj9z6H5B9m5cEhV/xfolC4jQHyu8SnXgda/g85f6n1gmqdm+5tYvQ77c/EQTh+4DfALqBvxQE4W3f9z/V5tPaF7TbjF3PuhrvT+47kbPfzjdka8InfQcSVWVeON7NxFIFx/PxHJfbyxXyVYeoJtGd0NFVkWxMpWK6DyxY6uJnulAj6NqTuLZQ5uOLbyL5NiIekm/znPQBN6UIA3/xTxE9C1dUuPLC71PqOtMYC9Nsy+1R2PLaLQH5KFy7duL7/ueAz7X7PPYj+8mM/aix00GiIVsTPpk7lOFsjPOjXVyaXqFQdbgymado2FiuR7nmkNYVXjjesyOdcnXxc3vJwHZcClUbz/V5SzjJZ5ABB1dQuJ08y1DhDUTPQsQDzyG38AZvuEepWC7ZmNrY0qqfT6str50WCO0WHLvh2Qq3C0M6mXandP/Jj3+k5eicR5ndCBIN2ZrwqdyhRFWZ0wfSzBdrFCoWC0UTSRYwbBfLdrk0U+LkYJpsXN2xY86XaiiyyErV5vEDGd6TzvB/JH6VY7V3uCg/xrR1iENCle8QFfAcXFHmdvIM+P6mLca6sKla7qYtL2BHBUKnCI6d9myF24UhncyjYsbeb9twuxUkGtKaB3oq79fxCvuFgmHxVxdnKFQtypZNRtbxfJ+K6TKdN/jGjQUyUY2YKvPc0a4HWmTroqdg2FQsl1xC5Zyc5q3JE1wWxzAsl/N9SSYWT/GXA/8XR413yJ38OKN95/BmS+u2GDeb1/11W147LRAeVsHxqPrNQvYPoRm782i3d+1R5EGfzPtivMJ+xLAcvvLBPCXT4Yxwnb8nvcVV4XFeV0fpTUVwPJ8bCwZHukUm8wZjfQlGcg/mkzIdj5WqTSqi0hXX6EvpLK4Gcn75yjxTeQNZFMge+yjz3kfpGkgyktA3pXrPl2rrhM2hrhgRVVq35bWTAuFhFRydGPEQEhLS2YTetb3ngZ7M4XiF3aNsOiQ0mdP+B/yvpZ9GxsEzP8cvdf8idA1iey4+AiXTbkQWPIhPKKrKnB/NATQGIB/qinFlpkRy8QI/LrxJbOAFZhKncT3WCZb6sepDlKuWi+1460aorD2fnRYID7Pg6KSIh5BHi/H+5Lo/hwbm/UG7vWuP4oy/8AndocQ1mZ6kTlq+grI6Jsb1bU47l7goPMPx3iR38lXyZQtRFFaTyB/MJ9QV13lpvJeyGaSbV0yXF6I3OfHKf4fo2TD7e6z84J+xkju77nX1rbyq5XJ9vsxoTxxRgENdMWKa1PBFbRRTOykQQsERErJ7bGVgDuksHmbvWqeKs7uuPIIg/C3Q1+Sf/oXv+/9puwcKZ1XdG1FV5mhPgpe7n8Yt/HvwbVxBYaHrHAOpCMf7k8R1mdvLVcqmzZfen2MwE6E/FWWqUGWhZN7XVl9djLxybZF3pwocvfrXnHAtBDw8BxYufpmJE6MossTV2SJjfcGnVs8HXZVwfZ+IIuOt5h1fni623QQeEhLyYGxlYD6QCbeMOo3Qu7a33HVV26nxCr7v/zbw2wDnzp3z7/LlIQCCzxXpOL85/L+RW3yD96THeH9pkB5rkXMHMyyUTeaKBo7rg++zVLG4Pl9BkQRiqrRpS20rDMthoWQ2/lyxHGzH53XvBN8pKODbICnMZM9RsTwGoxqvTa1QsVxiqgz4eIAkCFTtYHwM8FCawENCHjW2MjCHQirkUSdc1TqYiCIzmIngZp7iC7XDTK3UiOOxUDb5m8tzlKomNxerqLLAgWyMZETG9+DEQPKe0s4Ny+GVawtcmSuD73OwK4brelydL/FBdYRK/H/hSf8yyeMvUIyeBNPm5pK7Lvagbih/ciTTyLYCuLVU2ZYJvN05UCEhIa0JDcwhIa150PiDcLzCLtKd0Dg1mCJv2HwYL3E7b1CzoWg6XJ5aIR1TMG0XfJHlssVwNooui/ecdl42HSqWR1yVEYSginSoO86pwRQIUJPOclF9hm5Fw12uoMgSvUmdg12xxrFaVb+2YwJfH5fgrktJDwkJaT9bGZjrW0j3yqNoSt4LwpDSvedBu/bC8Qq7SFSVee5oMCqmWHNYKFvcyVcRRZ+5Uo2S6bBStRjKxRnIaJw+kN4URdCMjdWfuCYTU0VuLFhYrktXTKU7oWE5HiJg2i6ZqELBcKjaLodyKjFN4fSB9KZYg2bfw90EUWPWnyo3tgvXpqSHhIS0l60MzA+DiTkk5EEIV6kOJ6rKHMzFuDFXJpdQWalZjPclWDF9FBlkUUQVBRZLNjFNuqtwaVX9OTOSoVC1qdkeNcdjqWwhSwJHeuLMrdToSUSZLRpMLlcoVCz6U+qOVY3qOVBThWrTlPSQkJD2ExqYd4ewMrf/CVepfYLpuuiyRESRsVw40hUhF9cpGCZxXWUoF8XbhoW/VfVnJBcjqSsYlrna9VdjpWph2j6e7zNfqqJIwaDkTFTDR2y851qj+v2Iq3oO1ELJJKZKOzKIOSQk5OEnFB/rWZv1Va8atuv4j1LWWLhS7QMWSibX5so4vk88ItOTUPm7pwdAgLdu5elK6GQiyraER6vqD4DpeBQMi6gqMVessVyxWa7YjA8kGEhFuDhVwHN9fECVhUY+1CvXFrkyWwRB4HhvnOeOdt+XmBrJydvamgwJCel8tqq0PKoL7m7S7mHF7T5+OwlXqn1C1XZ5d3IFy3UplC36UzFM18V2fSKSzbGexLY636KqzPhAktvLBpLIOrP4+dEcxZrNzEqVO3kDEJguVFEksByPpK4iIFC1HEzHa8zNq1gOCV0BoGJ5D7QlFwZrhoQ83DzKC+5u0u5hxe0+fjsJV6x9QHdCIxtV0VUJ3ZfwfHhrcpljvUlUWeR2waDn3Yvkp19DOPRRrP5zLY3ahuU0QjJ1WeRQV6yxHWdYDgtFk6WSyVQ+aHWWJQFREEnoMsWqw+GeOOWazakDqcb7x1SZyVXhNZjWwy25kJCHgN3aNnuUF9yN7GRlrt3Ditt9/HYSrnj7gKgq8+yRHJ+7MEnVCkzmNVPj2nwJURB4VvmQFz/4CQTXwp/8Hd75+B+wkPsoEXVzdarukaqHZEZUqSGirsyWcH2f8YE0l2aK5CvB+JmVmsVyxSKiSkzlDcZ6EnTFtcZ7jvUlGMpFiChyGFsQEhKyJe1acDvN1L3Tlbl2Z321+/jtRLz7l4R0Aooscrg7wWA2QlxXSegSTx3MMtYb55R9EcG1EPEQXJvEzDe4Olvi8nSRNyeWMSyn8T51j9TakMzFco2/vTzHxGKF6/MlLt7Jk4tqnBxIcmogxXA6wlAmii5LLBZr3ClUuXArz2K5xpsTy9xcrDBfNEMRFRISAny70vLNm8s894tf5vMXphr/1mphfRQW3LVsVZm7H37yU8eIKNK6v9vLYcXtPn47CYXUPiGqSiR0mYSqoMsipZrLhdt5bsxXmEw9CZKKL0ggKSijH0ORRXIxDc+nYQoP3ifwSPUkNcYHgjl5r15f4vJ0gTcnlqnZLuWazYFMhMM9CfrSOscHkswVTT6YK2HYHqokMrNicnvZaFS3Nh5nI4blMF+qrRN1ISEhDx+tKi11MfUoL7hr2enK3KfPDPILnzmFKgXL+mA6wi985tSebZe2+/jtJCwf7BO64hrH+1PcWqxwMpakaDqooohhO7wnjhF99t8ykH+L+dxTlDiG5nhNR7Os9UgtlkxGcjHAZ7FsMbFYwTBdDuSiuL7PUyMZlioWvg8LJQtdkig4Fm9OLHNuJEMmFoi6u42AWZtdFQ4vDgl5uLmbB2qrcM9Hid3YCmt31le7j98uwtVsH1AXP0ldIm+YOI4DCKy4PhXLYTibYUY/xRv2KH16hNJyhU+e6CMbV+/qkYIg9kAWBSRJpGAYiIJP1XSxHA9FEsnFVW4uVuhJ6eQSGhXTZnwwRUJXGjP2tuoU3HjMMGgzJOThZTuVlkd1wV3LVmN3QvYX4Wq2DyibDgXD4rUPl3lzIo+ID4LIYCYCPixXLFIRFVUWsRyvkQvVk9A3vddGj1RMkxAEyBs2S8UaVdtDFASimsjNpQqOB5N5gyM9MSzL48ZSiartcXl6hfGBVOM9txJGzXxZISEhDyePsun4XtitytyjKkzbSeiR2gfEtSB6YLZUQwA8RCqWjeP76IqEKks8eTDDWE+c5YqFKgpMLhtN/Uj1FPHxgSTnDmbxfJBFiVxcZcV0sD2PiWWDFcMhF1d59lCOoUyMxwbSHOyO4yNyvDeJ5fgUqzY3FyubDO13O2ZYjQoJeXjpRA/UVub3dvLpM4OcGU7zzKEsr/zUxx+57c2HhXBF2wdEVZkXjnXz8rUF3q45uB74QNGwsRSPFcOmK64RGZKp2B6ZqIqHv24LbWNY51oxIws+M4UqnuuTiqicdK/w/f4tksUXqSSeIqaKXJsvMZk3mFw2yAyqKLIQbPttc7suDNoMCXk06DQPVBgAGrLbhCvbPmEoG+P8kRzvTq0QUSXypRqqJDKUjRBRhUZC+cRihTuiga5IPDmSAVqbvesz8nxB4Fhvgot3VjhqXuLfiP8SZcFByP8hc5/+U27qJ/nWrXkWyiaLJZPbSxWeOZIlqSvhdl1ISMgmtuOB2qstqDAANGS3CVe/fcRILkpcU1AlATuikIjIpCIKecMhX7FWx74IaIrEUDbSGGLczOwNwYy8mZUqMytVNFVmrDfOi0vXUCwHCQ/ftald+xqTBw5yY7FCzXaJazKSJHByIM1wLhrOxQsJCeloOj1x+2HzND1s3892CD1S+wTDcrBdn48cyXAgG2G8N0FMlSjWbCQElg0TTRbpTmh4vg8IjSpRM7P3QsnkymyRmu1xY7HCVN4gF9e5ET+LKyh4SPiiwlz2HJmYSi6mIokCPUmNmBr4H6Kq3Ji3F+ZDhYSEdCJhAGjIbhOWEfYJZdNBkSW+r2uGlcWv8NXaGG84o8zMlhjrSfDmzWXG+1P0pXQyUZXzo7lGlahu9l5fPTJBEBrv73g+qYiC03+WLx7+LY7X3iF54kXemhvAmS6RjqoMZ+PMl2tEVZnJZYOuhNrIpArzoUJCQtbSKZWJMGYgZLcJV719QlyTic69ycmv/iNEz+YZZP4b8We4bh3GsBw8fAYyEXKrM/A2Cpq1Zu/Fco2likl/SqNg2OiyyHA2xvX5Msf74tyRHuPYuU8gxFRG/Ty6KlGzXbrjGreXDQYzESqmy3zJ7Oh8qI0G+5CQkAejU8TRvdBp5veQh49wddknRFWZo9WLSJ6NiIePw2PWu3zdOcytpUDc2K7Hy1cXUGURQYBTg4GPKarKMPk6TLxM4f9v796D4zrLO45/33PZs7va1cW6WLYi25Fs2bKdxE6ME9c4IcMl4VICQ0sphHLJNMNlOtPpDLRxhpYBmimTljAdmHTyBy1QQgMMhJQUaAKkDkkcJ3Z8d2wcXxTfJVvXvZ3b2z927chrOZZlS7tHej4zGq+0R/ZPxzvP++qcZ9939moeO9KKH2rCUHPDvDqGch4ABc9nx5EB/FBzcijPPes6SMRMQl38Da457XCkP8eZkeIGxi1ph77hQlU2nMtq6kJMnWrbELicLAAqJpOMLBGSWHQbetNDhIGHp02eD7rxgUwhYCjv853fHyTUxWUPauM2r50a4ab5s1iXOEj80Q9C4FJr2DQt+zaxBWt4vT9DfTJGV0uara8PMJDzOHw6Q1ONw2DO5+VDp7m2OUUYQnPa4WBfBscyKPghNy1ooCkVZ9UCqyqv+shq6kIIIaaCjCwREu9YQ/5jj7Pt9//Ntw60slV3AqA1jOR9QJNzNSeHcjQkbBa31pL1AvzjGyBwQQeoEFrPvMwLiaUoFM0ph8Yah4GcV2o+D+jNupi5Ak9uP4FjKVrrknQ2JWlMx5lbn+R0pnDuHYHVuj6UrKYuhBBiKsjoEjHxjjXk3A72HN4KFN8pZ5vFW1mZgkemEJB1A0wUu44N0tWSwuq4FTY+hA5cQsPGn/dH9A+41CVtfrH9GG9fMhuAmG3QlHIoBBpCjR8GnDzjYSgFCmzbxInIxGTsBnshzqeUehD4Y8AFXgM+pbUeqGwqIUSUyOgSQa11CTqb04ThEDkvYG5dnNZ0nOFCgOcXKJia2kSMxmSMha0p4q1r4BNPkNn7O3ba13HCWcr+/UfIugEaOHImxzuXtTA3HWcw65FxA2zDwLEsPL+AG2pipuK6tvoxN0KuVtV6tUxUlaeA+7TWvlLq68B9wN9WOJOYBNIbJSaLjDIRlYpbODETNwgZKQQoQxFqTVPaIT+QZ3atQ2t9ArTi8OkMJJdh3LiMzVuP8frRQc6MuNQ4JmBwoG+ErT0OLXUJ6mpiNNU4HB3KkXZsvCCkNm6hUDSlYzT1b4dDz8KCddC+utKnQYgrorX+31GfbgT+pFJZhBDRJBOpCErETGYlHWocmyAobnfQO5wn74W0pOO01sa5qaOB9roku472s/v4MADXNqW5nr3c6m3Cis/jmewCTKXIuIpXjw+hKS4tZShFjW1yXVsdeTcg6Zj4gcY79CI8/mfFfiszBp94QiZTYjr5NPDYWE8ope4F7gWYN2/eVGYSQlQ5mUhFzNkVxBfOTrLr+CBu4JPzQnrOZKlLxFjb2URrg0Nnc5q8H/DrnWc42DeCH0JT/3bWHPprjNDjS9gUar/GmYYVnBjM0Zd18b2A+lSMhpoYbQ0J6pM2ybhF2rEYLvjEj75wrmmdwC1emZKJlKhySqmngdYxnrpfa/3z0jH3U2w6/MFYf4fW+hHgEYBVq1bpSYoqRNUvJSEuJBOpCBm9NlJbXYI11zbwy115sm5IqDU5L+BMrkBjKsbB3mEGcj6HT2foz3goQ9GS2YRRWofKxKO7sI3/7FuIoTReNuTUYIF01iNbOM6ajkaSjoVjKuqTMdrq4yQSt8FL33zjitSCdbLopah6Wut3vNnzSqlPAu8D3q61lkmSqJjHXznKKz0DuEHI2n/6rSwcGhEy8kXI6LWRAN5Z28Ps8EmeZhE7WIzWAXtODHN8ME/eC+hsTqNQ1CVs6hIxjgerCIYeRYcePhbPekvIhgGuH5J0TPJ+wJ12D13929j4Ujc9yeVorfnc7Z2snN9IPDWH/Ed/hn9gA1bHrYSzb5RFL0WkKaXuBL4I3Ka1zlY6j5i5Hn/lKPf9dAduEAJwdCDHfT/dASCTqSono16EjF4bKXlyMzdu+BRvCT3+wrb4nPH3bPYX0tOX4YAfgqHIuSENSZvlbbVk3JABVvBg7kE6Mlv41UgnW8JF1FgawwDXD1ge7uPLQw9g4fNRLB5K/jNP9rexpWeQhpo4S+fCbr+TsL0Tw4f5Vb5FjBDj8C3AAZ5Sxb0nN2qtP1PZSGImevDXe8/bDxCK/a8P/nqvTKSqnIx6ETJ6bST70LZzt+k0PjcEO3k+7CDM+3hB8QpTImbQXBunozlFX8alIWnz28HlbNOd7OofxFcBFEIMZeAHATcbe7C0j6VCwKfp9Ca0+iBLWtPk3IBXTwwzkHWpTzjkSr1asuiliDKt9cJKZxAC4NhA7rK+LqqHjHwRc3ZtpPyi2whffAgdunjaYmOwhCAE21RA8c+sG+D5AVt7+unLuCRjJqEO8bVCmwa1jsJSFilbcXQw4GW68bAAn1DZnGlazS3pRo4PFjgxmKetIcmmg6dprUuQsE1uWtAgi14KIcRVMLc+wdExJk1z6xMVSCMuh4x8ERXvWMPOd32fTb/5Ob8YWcgW3QWArQNsU9GajjM77fBa7wgohVfa9by4nYyHqaB/JCDtQF08jm3Cdt3Fx9313JHaz974DZw0FtNqQtw2aEw5zK6LM6cuwcLZaeK2Qahl0UshZjppkL46vnDHYu776Y7zbu8lbJMv3LG4gqnEeMgIGFFZ12ePtYRH4zYHhrPYCjwNvg+u1jT0b2NVdh9PZRexXXVR8EMc00ChMAyDWckYvWGBxpSD1lCbjJP3fPZ7SxlOrGAo59NZC0cGcnS0pJiVjJH3AuK2SdwySNim3MoTYoaTBumr5+z5+uJPtuOWfvGVSWk0yEgYUSMFn9q4zfK5tRzszeKX3rStQ7he7ePh8AHsnM+HlcVn1Zd4Nujk5GCeGseixjHJ5CFmmBSCkLwXsKAxxUCmQCxl0Jx2CHWempiJrw1WtjdwfXs9IwWfm+Y3EGrkVp4QQhqkr7IPrGyT8xZBMhJGVPEdfAqlDJrTMQIdMpjx0cAtxh5sSk3j2meF3skzdDKU9ziTdWmoiVHwAlABaAsdamzLoCkd54b2OuY2JHj12CAZN6SpJsZwvthY3pKOXzSPrCclxMwjDdJCXOFESnZOr5xkzKJ9VhI/CKlxLIJQE2hNPh+wMSw1jWsfD4vnvW4AAg15L2Q45xKP2eQzGscOcAPNwIiLaRUb1dvqEvhByPGBPC21DqHWYy5tcHbyZCjYfWxI1pMSYoaRBmkhwLjC738KWK61vh7YR3HndDFF8p5P73CBmKkIdEhz0kGZii26i4+56/mG/6fc7a5np9FVOj7ADyHrhvQOF8j74HrFVdEznsdIzmXDvl5+tfM4h3ozFPyQw6dzDOf9C/qhzq6yvvvYEM/vP03ODWiscQh18bajEGL6+8Idi0nY5nlfkwZpMdNc0WUD2Tm9skKgMRVjVtqhvm8Ly7wd/NZexAavAyheHdLA2XmNF8BNxj7ear3KC8ESNusuLNOg29/F6syrvBR2s9teQswy8YIQ21RYpkl3a+qCf3v0KusFL6Dgh7KelBAzjDRIC3F1e6QuunM6yO7pk6E55QCKur4t/EP/eix8PoTF14yPc7/1fWx8Pm9afMxdzxbdxQq1j+/ZD2Djc69pcQ/rqTVifNN8AEv7eMri7ux69gfd1CYsbpw3i7ht4Ifqglt7o1dZT8SKa0pJE7oQM88HVrbxw009gGy0K2amS454V2PndJDd0yeDY5vc3NnIvPxr2P3+uVXO32u/hK3faDa/xdjDlqDr/CZ0fNbYr1KDjTWqMX2ttYfh2pXELZPhvItlOsxKWhdcZRq9yrpMnoQQQsxUlxz9ZOf06pVyLObUxultWkVw7Dvo0AMUPbGFrCi8eq7ZfGNYbDY/14SOj6ctXtTdNFkOvvtGY/pmYylz6+Jc25Ri8Zw0LWmH69vrx5woyWKcQgghZrorfdee7JxeQcmYxdpFTfS2vpvBhq8y65n1oAPucn/BV7yP02KNsDHsZjvFZvMtuot7w/XcbO1lg7eY7WoRxhCcVOv5kPkcKFjcXMd1K68hDAOODbn0DhcIQli7qEkmTUIIIUSZKx0ZZef0CkvGLOY3WmCMoNEYaDQ+9WqEh727MC2IxwCtcCzFPnM5m7JdFILiWzYTjiJpWLxP/x+29nl/7wYOBu3ssbpJxUKUgqwXjLn8gRBCCDHTXem79mTn9GqxYB3atAl9XdzEOOzG1VBrKGbVOBT8EN8POT3scnZxghAIfM1bU3uxCz4mIUr7XDO0mZ7WZbze7wOatvqEvBNPCCGEGIOMjtNF+2rcjz7Oc0//jH8/2sZOtZC4AY5lMVzw0SH4YUBY9m1BAIfTK8H/MTr0UGaMmq7bWTu7ma7WWgCa045cjRJCCCHGIKPjNBJe8xZeXzab/X2vgecC4AY+oVbkveJtuhBQFNeXAjBMOJhcxv6bf8C1mVeIdd4G7atJAs1pWVxTCCGEeDMykZpGRgo+gzkPSxkkYyaWWeyLmp/bzUq1m8108zJdoMDXYAMNSRsdwLP5a+lpXsba2c0keWPlctn2RQghhLg4GRmnkYIX8Pz+PgbzLmiFUrCosIeH1T9iW8XlDT6t1tOTWs5AxiVum8ypTxCLGQxmfXbkBulqrWV+o3XeyuWnMwVpNhdCCCHGcKV77YkqkHV9Tg3n6R0u0N6QpD4ZwzbANhW3x/9ATBUX3LSVzzpnH4Zh0lKbYOncehbPTmMqOJMtcLQ/R84r3sobvXK5bPsihBBCjE1Gx4gbfQtuJOfSUhenoylF30ieFe0NHD21Cv/EY2h8AmWT7nob17lpTowUCEONqUzaZyWpSzh4QUDCLr4kZOVyIYQQ4tJkdIy40bfgAN5/QxtvXdTEwd4MTSmHkXm383jPw7QPbma49Rbeve4O3jLs8vs/9DK7Lo7WGtcPsUyTmphBc9o593fLyuVCiPGQPfbETCajZMSV34Kb15hkyZxaVrQ3MFLwMRS8kljLKXcNNTGDZMxiXqNF13D6XCP50rm1suGwEEIIMQEyakbcxW7Bnb2adGo4j22ZdNYlzzWNt6TjcttOCCGEuApkBJ0G3uwWXMqx8PyQA30jJG3zXNO43LYTUynr+mBY8oITQkw78q69GUGjdfFPIaba2TdEGHY8WeksQghxtclEapobKfjFW3vNKWzLlJXKxZQ7+4YIHfry4hNCTDsykZrmZD0oUWlnX4PKMOXFJ4SYdqSwTXOyHpSotLOvwdArZCudRQghrjYZVWcAaSwXlZaMWSC39oQQ05Dc2hNCCCGEmCCl9dS/k0sp1QscvsxvawL6JiHOZIlS3ihlBck72SYr73ytdfMk/L1TalT9itr/aznJX1mSv7IuN/9F61dFJlIToZR6WWu9qtI5xitKeaOUFSTvZIta3kqJ+nmS/JUl+SvrauaXW3tCCCGEEBMkEykhhBBCiAmK0kTqkUqmly2ZAAAEd0lEQVQHuExRyhulrCB5J1vU8lZK1M+T5K8syV9ZVy1/ZHqkhBBCCCGqTZSuSAkhhBBCVBWZSAkhhBBCTFBVTaSUUncqpfYqpfYrpf5ujOcdpdRjpedfVEotmPqU5+W5VN6/UUrtVkptV0r9Rik1vxI5R+V507yjjvuQUkorpSr61tbx5FVKfbh0jncppR6d6oxlWS71epinlPqdUuqV0mviPZXIWcryHaXUKaXUzos8r5RS/1r6WbYrpW6c6ozVImp1qVzU6lS5qNWtclGrY+WiVNfGMiW1TmtdFR+ACbwGdAAxYBuwtOyYzwH/Vnr8EeCxKs97O5AsPf5stectHZcGNgAbgVXVnBdYBLwCNJQ+b6nyvI8Any09XgocqmDeW4EbgZ0Xef49wC8BBdwCvFiprJX8iFpdmmD+qqlTE8lfOq4q6tYEz3/V1LEJ5q+aunaRn2HSa101XZFaDezXWh/QWrvAfwF3lR1zF/Dd0uOfAG9XSqkpzDjaJfNqrX+ntT67UetG4JopzjjaeM4vwFeBrwP5qQw3hvHk/Uvg21rrfgCt9akpzjjaePJqoLb0uA44NoX5zg+i9QbgzJscchfwPV20EahXSs2ZmnRVJWp1qVzU6lS5qNWtclGrY+UiVdfGMhW1rpomUm3A66M+P1L62pjHaK19YBBonJJ0FxpP3tHuoTjrrZRL5i1d0mzXWj85lcEuYjzntwvoUko9p5TaqJS6c8rSXWg8eb8M3K2UOgL8D/BXUxNtQi739T1dRa0ulYtanSoXtbpVLmp1rNx0q2tjueJaZ13VOGJMSqm7gVXAbZXOcjFKKQP4BvDJCke5HBbFy+Jvo/hb9Aal1HVa64GKprq4Pwf+Q2v9L0qpNcD3lVLLtdZhpYMJEYU6VS6idatc1OpYuRlf16rpitRRoH3U59eUvjbmMUopi+JlxNNTku5C48mLUuodwP3A+7XWhSnKNpZL5U0Dy4FnlFKHKN4rfqKCjZvjOb9HgCe01p7W+iCwj2JBqoTx5L0H+BGA1voFIE5x48xqNK7X9wwQtbpULmp1qlzU6la5qNWxctOtro3lymtdpRvBRjV8WcAB4FreaGpbVnbM5zm/qfNHVZ53JcVGvUVROL9lxz9DZZvNx3N+7wS+W3rcRPHybGMV5/0l8MnS426KvQSqgud4ARdvwHwv5zdgbqpUzkp+RK0uTTB/1dSpieQvO76idWuC579q6tgE81dVXbvIzzGpta7iP2DZD/QeirPx14D7S1/7CsXfkqA40/0xsB/YBHRUed6ngZPA1tLHE9Wct+zYihekcZxfRfGy/m5gB/CRKs+7FHiuVIy2Au+qYNYfAscBj+JvxPcAnwE+M+rcfrv0s+yo9Guhyv9fq6ouTSB/VdWpy81fdmzF69YEzn9V1bEJ5K+aunaR/JNe62SLGCGEEEKICaqmHikhhBBCiEiRiZQQQgghxATJREoIIYQQYoJkIiWEEEIIMUEykRJCCCGEmCCZSAkhhBBCTJBMpIQQQgghJuj/Af8nybB1CgkOAAAAAElFTkSuQmCC" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "ed058c5d", + "metadata": { + "lines_to_next_cell": 2 + }, "source": [ "Left we show the dataset and posterior samples of $y$. On the right we plot the mean and std. deviation of the latent variables corresponding to the datapoints." - ], - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, + "id": "fb6e4233", + "metadata": {}, + "outputs": [], "source": [ "def plot_mean_and_var(ax, samples=None, N_samples=5_000):\n", " if samples is None:\n", @@ -553,50 +426,32 @@ " ax.set_ylim(Y.min() - Y_MARGIN, Y.max() + Y_MARGIN)\n", " ax.set_xlabel(\"time\")\n", " ax.set_ylabel(\"acceleration\")\n", - " return samples\n" - ], - "outputs": [], - "metadata": {} + " return samples" + ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, + "id": "3b7576bc", + "metadata": {}, + "outputs": [], "source": [ "fig, ax = plt.subplots()\n", - "plot_mean_and_var(ax);" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "100%|██████████| 5000/5000 [03:24<00:00, 24.44it/s]\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9eXCcd3rf+fl1N9CN+74aAIGXNymQFEVCooQRIUpzSJoZz9iTtmcSJ4438Wi93lR213Fq4k022dTWrrOuddVWNpXVOLHX9jieSXtmNKORRjM6KFKCRAq8CRIEDzRBAI37Phro690/nre7cTSABtCNg3g/VSgSR7/9dvf7/p7fc30fpes6JiYmJiY7D8tmn4CJiYmJyeZgGgATExOTHYppAExMTEx2KKYBMDExMdmhmAbAxMTEZIdi2+wTWA3FxcV6bW3tZp+GiYmJybbi8uXLg7qulyz8+bYyALW1tVy6dGmzT8PExMRkW6GU6oj3czMEZGJiYrJDMQ2AiYmJyQ7FNAAmJiYmOxTTAJiYbBajnTA9vPjnM2MwO7Hx52Oy49hWSWATk8cG3wjc+yXY7LD38zDuhbRMyCiAu78AiwUOvAqZhZt9piaPMaYBMDHZaEJBePgx2BygLND6JlhsoIchHIKMfNB1uPlDSM+EsjqoOApKbfaZmzxmmAbAxGQjCc5C+zmY7IPcSvlZRn78v7XnQDgAnRflX+dT4hmYmCQJ82oyMdkowmF48AGMdccW/+VQCqzp8rfeq9D2c5gaFO/AxCQJmAbAxGSjGHoAY52QW766x1kskFcFM6Nw6w1o+RFM9KXmHE12FKYBMDHZCPxT8OgTyCpb+zEyCyGvEvQQtP4Uem8l7/xMdiSmATAxSTW6Do8uSEjHlr7+49lzIKccOi9INZGJyRoxDYCJSaoZfSThn8zi5B3TYpMqoo5PJbdgYrIGTANgYpJKQgHoaIKs4uSXcWYWwng3eM7L85iYrJJNMwBKqWql1Fml1G2l1C2l1D/drHMxMUkZQ/chMA1pGak5fm4lDLfD7Z/CWJdZIWSyKjbTAwgCv6/r+mHgFPB7SqnDm3g+JibJZXYSupohc5EMe/JQCnIrpInszlsSbjIxSZBNawTTdb0H6DH+P6GUagUqgdubdU4mJkmj5zp0XRaph2QkflfCni3GoPMzKRm1WFP/nCbbni2RA1BK1QLHgYtxfvdtpdQlpdSlgYGBjT41E5NV0dTUhOfODdn5Z5dAZiGerl6ariRestl05Raert55P0voGOlZ0isw1L6WUzfZgWy6AVBKZQM/BP4HXdfHF/5e1/Xv6rp+Utf1kyUlKXSlTUySgNPpxP2Xf4qnZxQsNjxdvbjf+QhnaVHixygtwv3OR1EjsKpjZBXDo0+l78DEZAU2VQtIKZWGLP5/rev6jzbzXExMkoFWVYHrmV24z7dQPxKkueUurpefR6tKvPtXqyrH9fLzuN/5iPq6/as7hs0BTMDDJlEZNbWDTJZhM6uAFPCfgVZd1/9ks87DxCSpjHejOQuoP3KQc803qa/bv6rFP4JWVU593f61HSO7BEY7xBMIBVf93CY7h83cHjQAfx94USl1zfh6dRPPx8Rk/QzewzPgo7nlLo31R2huubsonp8Inq7e9R0j1wkDrTJzwOwRMFmCTTMAuq5/rOu60nX9qK7rTxpfb2/W+ZiYrJvADJ7Wa7g/uILr5ec5c+pYNJSzmgU8EvN3vfw86ek2Th07uCgnsGJCWFmkR2CyF9o/ND0Bk7iYAUITk2Qx1Y93YBTXK6ejIZtIPN/bP5TwYbz9Q9GYv7O0iAvX73Dq2EG8/UOrTyrnVMCwR5rFTEwWoPRt1Dl48uRJ/dKlS5t9GiYm8Wk/J924SR7jGFn0V50QjhCYFr2gum+YSeEdilLqsq7rJxf+3LwaTEySQXBWdtqOvKQfel0JYZBZwzNjMNWf9HMz2d6YBsDEJBmMdYMeTEkH7roTwiBaRL03zVyAyTxMA2Bikgz6boE9N+mHnZsQXmtSGZC5w6MdcPvHMD2c9PM02Z6YBsDEZL34RqTaxp6T9EPPTQjD2pLKQKwqKBSEB+9D0J/0czVJIeEwhENJP+ymdgKbmDwWDLTJ8PYU0PDUE4t+plWVr6m5DBBPYLwHui5B7XPrPDuTDaP7sniYpQeSeljTAzAxWQ+zkxL+yUxc62fTySmDvhYYuLvZZ2KSCDPj0H1JJL+TjGkATEzWw8AdCa+sJfkbCsSSsgEfBGfE1R/vka8UuPyAnG9OGXjOwURfap7DJHl4r8pGIwWYISATk7Xin5bKmqw1zPoN+EQyOhQARz74hgAloaTgjPF/GzgKRNYhpyy5525Nh4wCuPsOHPpq0nsXTJJE323ZZKTo8zE9ABOTtdJvzC6yrHIf5Z+UmK6uS+I4MC2LcUaBDI/JKoasIkjPlr/1XoP+1uRLPKdnycCae78Ug2SytRh+CA8/hhyneG0pwDQAJiZrwT+V0O5/0XCXsW48F9+h6eYjmeJlsUJ6ptzgSs1PJkd+l1UI414ZLt/5GUz0QjhJ9fyOPGlie/ixhJ9MtgYBHzz8SJRdrakL1JgGwMRkLQx5ZAe/wu5/3nCXqSE815twX3yEs6I08edSFqneySyCkB96b8iCnaz4fVYJjDyEQTMpvGXovio5IJsjpU9j5gBMTNbC4B3IWFn2ITrc5a2z1JfM0vxgENeZJ9EqCtb2vGkZ8hUxBOmn1t9/oBRklYp3UVALaalddExWoOcm9N2U2c4pxvQATExWy8wY+EZFYycBtMoy6ivTOdfSRf2hmrUv/nOxpkv8vud6cuL3tnTZcfa1rP9YJmvHe00G+eSmLu4/F9MAmJislnHvqm5OT9stmm/eofGp/TS3dePpGUnOeaRnSRlpxydG2eg6Y/jZJWJQTKmIzaH/DnRelMV/tYUFa8Q0ACYmq2WgDRyJ6f542j2433gT14vHOXN8D67GOtznWpJnBBw5kijuvSmLx3q8AYtVvBrPeVM0bqMZfSRJ35yKlAgKLoVpAExMVsPMOEwNyu57JcIhvG1XcJ0+hFYldfxaRQGuxjq8Q+PJOydrupSNBmelK3k9nkBGvry+jqbkl52axGd6GO6/LxVl1rQNfeqdYwAmB8zZqCZroqmpCY/HI9+Md4PFsvJYxqAfeq7TsDcXraY6dqyWDgAa6mqiP/P0jER/vi4cOTRduomn5TOpUIocP5ERknPJrZCqoJYfSr7DJHXouuz8bY6UV/zEY+cYgM6L8OjCvBvDxCQRnE4nbrdbjMDAHTwDvuXHMobD0HNVVEIXdHA6i3LnhYA8PSP88Q8+IhSaf12u1Sg4nZW433ofz/WPIRxc/QhJkPxGdimgzHsm1Uz0wmS/eF6bwM4pAw0Hoeea1FKXHd7sszHZRmiahsvlwv1f/or6wnGa24eXH8s4NSBVQnGaxCIhIPe5FuoPVNLc1s03zxzlQmsn1aV5aBUFeHpGcJ9rwdVYt/pzrSzC9fl6GSH5qItmr47rKy+uTT00qxhGOsQbKNRW/3iT5dF1Q+Uz+TLiibJzPACQXc2jT2BqlVrqJjseTdOoP1DFuSv3lh/LGA7BYNuyN7VWUUD9gUrO3XhI/YFKTh+rjRqFs1fbo4v/WstFNWcR9XX7OHftAfWlfrTiBPIVS5FVIk1nplRE8hnrlOqtFIwRTZSdZQAsaZCeA+0fmPkAk1XhuX+X5o9+QeNz9cuPZZwakMXSZl/6WD0jNLd103i0NloWutAorKdXIHr8pw7Q/GAAz6V3YXZibQdLc4hR6/xszedjEgf/NLSfk+T9JrJzQkARHLmSyOu/AxVHNvtsTLYBHo8H91/+qXTwHnqS2l2V0TGNizyBkYfLVgjNDe9oFQXUlhfgPtfCqUPV84xCbXnBmoxA3OO/fwVXmgOt/ktrKzHMLpHSV4Cqeik7NVk74ZBUWelh6ereRHaWBxAhq1Rib6Zba5IA3q5OXM9Uou09CCwzlnFmQspEl7mpvUPj88I7WkUBpw5V8/0Pb+BqrOPM8d3r6hWId3zXS0/h7emHYc/aErrKAnmVYtza3jLvm/Wg65JYH3ko4bVNRunbKMN/8uRJ/dKlS2t78K035lvc8V4or4Pq+uSdoMnjybBH6rTzKpf/u/470iWcgEbQXJpaOnAW5c7b8Xt6RvAOjc8rF10X4RBMj8hrKDmw9nrzqQHILIZ9X0ypSuVjy0CbhH7yKlcn9TDRC9WnoOzQmp5WKXVZ1/WTC3++Mz0AELe294aU6pmYLIWuiz7LSot60C+hRXv2qp+ioW6xPpBWUZC8xR8k9JNVJAuJ98rah8JnlYiR61tFX4GJMDMush05ZRui85MIW+MsNgOLVRJ1nZ+Zdc4mSzPZB9ODMpwlDlG9/4ke8TAt1uQ1diUbpSCzQMYL9lw1Jo+tgZwymVFrbp4SR9elAtGSNn/mwyazcw0AQEYhjHbIjsbEZCGhAHR8umxJp7O0CPfPz+G5fQXsudEkrLMoMa2gRGhq6ViUD1iXkcnIB79PNj9rqQ6y2KRr9cGHUs1isjITvTDauc6qn+RvVHe2AVBKanC7L5legMliuoxd7jJ12lpVOa7PHcR9vpWzNx6tu4Y/HvG6h9dtZByGUeu6tDYjkFko4yrb3oKhB6Y3sBzhsCgRJCggGPfxQ/fFgCSZnW0AAOy50oo90bPZZ2KylRjvgd6WlYexhwJo9hHqn9CSUsMfj7ndw8loFIuSniW7+c5mqWBaLVnFsji1n4PbP5UYt8liBu5I8ty+BgMQDsLwAxhuT94Y0DmYBgDkg3l00ZTANRHCIUnWZeStnKwb68TTPUjzvb55jV3JJpmNYvNIz5RhMF2frW0Bd+SKeJyyxjqGzSbLGJFZztmrlOIIh0Ui2nMeRjwJDx9aLaYBALmIfUPgvbrZZ2KyFRi6D77hlTVagrN4bl/FffFRUmr4lyNe93DSSMuQggjvVZo+u76oyzkhNdGsIvGir/0NXP+BSErvdGYn4N57Ei5bTcmsf1IqtfpbxUvLLJJwdQrYVAOglPozpVS/Umrz59BlV0i538QSLf4mO4NQQEIicYTcFjHuxTs8ieuFo/Mbr5Ks9z+3uzdlRiYtA8JBnNYh3D8/HzUC7nc+4vXvvz1PTXRJg5DrFG/AZoc7P0ve0PrtSDgE7efFg0xk967rMheg75Z4n/5JuQZTPBlsszs5/j/g/wH+cpPPAywWSYx1XYKDX06ZxTXZ4gw/hNAM2Fao1ggFYNhDw/GDi25SrWIVMg7hkMR4RzqkOicclIYti1WSzwU1eAdnF3f3GkYmqfkGRx5aeAhXfQXun5+n/sgBLrfcm3cvROSlXS8/v/Rx7NnymNafwu4zULw3eee4HQjMiMb/ZA/krtA8CFJJ1X8bpofEeDoKZD3aADbVAOi6fl4pVbshTzZ4T3Y5BdrSb64jD8a6xAvIrdiQ0zLZQoRDUhGWkUCp3mQ/6KG179DGuqCrWf4NzshxIkm+9BwgLBO5Hn5EQ04FqGMwPCodpNb01RmZRFEKMovQGKK+As5duMyXPneC2qoykZeu209zy93lpbAjpGdJt/GDDyTHkL8ruee6VQn64e7PwTee2OIfnJFrLhxKzOtMMpvtAayIUurbwLcBdu1ax0XkG4HB+xKXK6tbOiZnz5Eb8+CXN3Q2p8kWYLxbFt2VhnPo+oqib0syPSxaMP235PHF+6BoLxTulsVAWWNKov5pGLwLnRfg7jvyM4tN/rbkoDw22R2lSuEZt9J8y0Pj3mKar1yltupl6uv2c675Jo31RxKfLWBNl87h+x/AE1+DjCQbrK1I1yXx4BLZQIaC0HNDFv+1loiuky1vAHRd/y7wXRAtoHUdLLMApgck2et8Mr4eSsQL8F6FqkXSGSaPM703ExvOMTshi3NW4cp/G0HX4dGnUhGirKKqWfPc/K7QhbHi9Ey5TsuPwOy4jGccui96MoN3oeRQ0sOV0XzDSyfQSrOo7ejm9e/9GNIcfKnhKZpb7lJbVZa4EUhzQGgW7v5CznUTh5+kFF2Xcs++lpU1o0Cun57rEJjetGlgsBOrgDIK5UZabnh2jhO6r8CQZ2PPzWTzmB6Wkr1EFqiJntV7h+0fyuJfehieeQ12v5C4JIDFKrvnglrY+3k49btQ+zkYaJVwQyB53bjz1ESt6bKAT/RyYp+TM6eO4Xr5edzvfLT0PIR4OPLkXmt75/EcxhRR+PScX1nnJxySPNOjT8QwbuLiD9vAA0gJmQWi8TJ0D4r3L95BWYyZqO0fgONrmxKbM9lgBu/JgrfSbjochLFVir4N3pM4r/M47Hlp/Tt2ZYFdz8pi8uhTGLgrHkX102tX+TRYKEDnHfPz2lfr0YosMNaN5qyISmGvasxkZoFsvG79SMZLZpXJaNbHIcw60CbeY17V8ot/cFb+bnpIDPoWeO2bXQb6N8CnwAGlVJdS6h9t2JNnFoolHl5il2+zy27w3ruS1Td5fAnOSs11ZgIhnekhCAcSv3mDs3D/PdlE7D6T3Oqy2s/Bid+WnMCjT+Dynye9/r6hrgatukLuhb4W6G5GK8uj4aknVn8wRx7klMs5dnwCfbeTeq4bTigg1VsdTfK6llv8QwGJKsyOG+Wdm7/4w+ZXAX1r055cWeSGH7wri328uJ09Byb6xb3b3WiWhj6ujHbKzj6Rip7RR4l3ZQZ8Ugrpn4TDX0vNTZ9VDId/RV7DnTfh2n8RT6BAW1nGYjVY0+S5ZiakT6LqxNqS4BabGIL0LElu62GY7JVz3k5J4ulhaPs5BH3gyF/e8wqHJeS8yfH+eOy8HMBcLBZxTftalhazyi4RIzHUvrHnZrIx6LrMhUjkxvRPyXWSyBg//yRc/Z6Ei/a/Ik1SqSS/Gp78e9KR+/AjuPqXUjkUnE3u80RE5NY7Uc9ik4Wz+7KEY+/+Qt7frY6uSwlw29vyGnIrlzeEAZ909U71b+rw96XYmTmAuVhsUnfdfRkqTy5eCJSSfMDDc2IsEgkTmGwffCOym0ukcmOiVyp4VvIEQwGZQOefgqO/kdixk4EjT4yAfxq6m2WnPt4DR/5Ocqtv7FkyU6DzM6lQWus9Yc+O5VKmhuDmD6XqqeSg9A5sJXRdvL/Oi+IF2bNWLt2cGoCem0a0YXOHvy/FzvYAIqTZwZYh9f++0cW/t9nF7b/3rql4+LgxOZBYLX0oKPHeRJK/Dz6QSqGDX1528U+6zn+E9EzQGuGIS2LOV/9axlUmU/Lcnh1TEn30qRFGW6KqLhGyikR8r+sStL4pMhLrOV4yCAUl/zc9LN7U3V/IBiDPufxuPuCTzt6uK+ItOrZu6atpACKkOWSR916N79o68iROfPsn4gKaPB6MPZIFcyUm+xPLEwx7JKRU/bQ0ai1DSnT+51JQA8e+KRucO2/CTfe6wjaLDFaaA8+4haabj2TB67kmu+O1GhprujRQhQPQ+jO49j3ZQc+My/uf7HBWPEJB8UZ6W+D638C1v4aWH0ryP7965WslkuCe6JU+ka3mySzADAHNJc0h7rv3mriiC2O9kSEYt38C2un4JaQm24dwWGr/VwphhMMw0r7y7j/kh3u/EHe/pmHFp5+r819/oJLmtu6kD5Mhuwye+i3pOH3wAVz5K9j3BSnFXCURgxU5R0/PCO7zt3A11kFWgSzUjy7I+1R2SGL8a8GRJ18hv3jlXc2ALp648ym5T+250tOzWs2ccEgSzwuTtoEZ8doefQKBWbmvM1axgAf9Ml1wuF3ObYsv/BFMA7AQR45R6XBR6rYXunrp2bJTaT8nO4Xqp1cn9WqydZgZlQVhiV1905VbOEuL0PItEs/PKsbTM4J3aDz+wPbIiMVj30pYI2iuzn/j0drk6/uAhLicT0ou685b0PK3Ujq669lVJadXNFiRUEdgWuZr5O+S5rVEkubxiHgEEYIzEm5CAXpMNiO7BPKN55kalP6egloxxCG/YSisoqvf1yoGwPmkGKqpYfn7wLR4LpmF8verYXZCIgfR59o+gRVz5YqHI0dc5c7PoOKY3DhzsaZLbHegVfRjqushb9e2+uBNkNjuMsi83/O4jmShlefNk2VexOyE7FRLDkpDUIIs1PmvLU+ByFuEXCec/G0peOhslvDGrmfFW0nQk13JYDW1dOAsykUrKxTvatyLJ1yBd1LCQs7SonkNZJ6uXrz9Q4n1Fdgc8w1WOAx6UPI4Ix2AounGfZyVVWizk7LQA57uAbyDIzQ8eVAKOQC6LgPhWDXSKsozoxuD0hzJsfS14hmYwjseoKFuG5WyYuYAliYtQ3YI3qswFmdovLLIxaiUJIdb/lZKRc3ZwtuH8S4JJyyBVlWO6/QTuM9e42xLz/KjGB9+JJ+9djrhp98Qnf+FWGxQ/Qw8820RRXz0Kdx+Q7p0Ezzn5QbTRPMafWOQkY9naBb3D3+M0zaBszhvnoxERFp67qyB1b0Wi2zGMvKjswictftwv38Jz5gOuRV4xhXupjac2gHpi7CmG55FuTwmu3TV4RpnSSHuH/0Uz4W3oacFz6AP9ycPkpe72UB2hgcw2S+xe6t9dY+LXFy9NyUxlb9r8U4pPUu+AtPS8ZlXBbtObW65qK7HxvIpFaty2SLdh1sCXZfKleXi1LqOljlJ/eGa5UM0E33S6FP19Kpqvefp7pBCnf94WNNh/8sSJulogkt/Bvu+CGVL78TnGiytooDa8oJFRjFumOilejTHBIQ8uBr24X77LPVHDycuLb0KtKryqF7RquSrE8E/BRM9aKFeXE/m4/6sk/qDrD93E/DJ+mHPSVwfKknsDANw/o/hyl/I/ws00U2Jt5jHw2ITt7H/jsQgi/bEj++mZUJ+poQVWn4oreFpWbLryCqRGy1ZC3DQL/HrmTGYHpQEVnBWLqKAz6iWWOCJ6LoYqtwKOa+sYmli2SbJqqQzMybv43L5m9lxPJ4Omu8PLh2i0XVoPyse465TwJwwyJy/i5c7iJdHSInO/1IoJTmskoPS2NT2ttSu73ou7nWRqMFaFCaqNHb4wVk02wD1RT7Ovf8OjadPJ3Xxjz5/Vfna5KsXEg5L5VdwFkYfSkjLYoO0TLQ9e6gfV6vL3YQCsj4EfXKfTvZL0njakO+wpktuZnZc8hu5FZBTIZvX7qtQfWptr2MZdoYBOPpNEWyaGZOu3pv/VVy/ynooObDywmyxSknX6CPpWiw5AJkl8WP+mYWg58tiPDsplQFho+ogtwKySgFdrL0jz9iB6hJDVhb5fzgkxwrOyDkrqySYpgbkApqd04tgs8vvrTa5OO25kLFEs1LIL41B4SD0GJUOFptcZOV1Mrh6p+QxpodX3AB4Wq/j/rQd10snl9zxMtAKY52i0mno+Metllkqd7AVcORKz8D99ySP0XdLcgMVx+bdG4karCXzGjY7noFpmh+O03jYSXPTh9TmBNFKsoykbXK8Zk9XL80td2msP7J6+epwGAJTUuAx4okN6bHY5PwMb3pVuRvfqIQIB9qYtzFTFtmEaY2yORt5KF8ZhRJx6LoUzWPgyEtJD5LSt1HM+uTJk/qlS5fW9uB3/lDezDSHiFB1Ncvg7/Qc0TUpPxobxLEcgVnwT8gHVnJIGlgSIRwUFzI4A6jYIh+paJiLUkYuQZ+zSCljeLcjMdXKRAmHxaD4J+UiqzgOhbXrVpXc8rSfkwT+UvozAR9Nb/8AZ4UztoNlwU7ePyWhk4wCePLvzmsoiyz6KSvvXAWJeiSAlEK2n5f+CEce1D4vHkKC19vCMNFC4zfvd139uD+8gev5Q2hFdmPHWy4L7Ronrc0dWalVlS/6Pi7BGfmanRAJ+JCxObLnxD2P5V6jVp4vnsJwu2wWpwbk3rLYoOJJyTukZ8k9vNLrDAfFS7CmybV2+Ffh0JfX9L4opS7rur5owMnOMwCRWm5dlw+pq1l2cNZ02fE4n0psOk/AB7NT4hkUaGuXd9XDgNoa/QSBadmtWNKk27F4/+M5yk/XpTvWkbvI0EUrPBwT4vFl5C+9WN55S3Z1J34rbqv/2avt0RDBmeO7U/mKlmXZBSueUYpMPPOckwUsu0wMQUHtitfpcsYGWNoQPVFtbJCMsFz+LjFAkQ0PSj6rlZ4/8vktrDTqG6LhSI1UDM1OyLF1XTaB08aMAkWszHs1rzHkx3P7Kl5PGw1Fo4YBMeQfskqlTLXk4PrkOEYemgYgqQZgLhO9YggG2uQCKzkgeYLsFdQUdd1I4Phk8c+rkq+1qCSul3BQYoyhoMyqjXoVRhLYYpEkeCI7+3BQwlezExLXLqt7vEJDM2Nw82/j1sB7unpxv/0hrsNpaDXVePrG4y+WM2Pw2Z+K97j7zOLjbCEPYM3no+vS4fvwY/ESs8ukZLRwd2o3LKGg7Joj4Y8INruER0J+Y3h6vlzbyhLbSc9OwFSfSLugy+dks8u9EZiWcKklDcJ+xKiki2e92tcTnJWeg8leaX4LTEv4taBWOoYLd4vxShYpMgA7IwewEjnlcOirUsLXfVla+ftbZRdSVS87/HgXiFLSGp6eKSGdsS4jhlcgx7SmSXI4PWvtw8MjRCp7Isne2TFZpP1TckMsvFnmRpYip64jF3taptxI2SXy/0UDcWxS/WTPloa40U6oefbxEcJbpuRRqyrH9dxe3D97l/rR8NKLZdcled8qF48NTaRaJh6rCtWskjU1nCklVUElByRs2nlRBrrk7xKjt7A/JllYbfHr8kMB2a1brGKQxpcoz7Y5IGBoeqXZjWY/6wIvbQ2btIjHMHhPeoRChjRFXhXUfBXyqreGJ78KTAMwF0ce7HlRZrX2XJcBDi0/hMximQ9ceijuQj7vxtV1CM7gafmMdy495OUTtWjleeJapmfhGZjGOzxNw4lDsR25HpYYpA4QNnbygVgiOWx8H13kddnFWNOkWiPNkZCgWdPNhzgLgmhlyIU8dA9P3wTeaRsNDQ1GGMsy343Oq4LpYTy/fB2vqqDhq39/+5eTTg0ubZDDYaP0U1t6sQz4pDS45FBct36t5Z2pTB6vq+HMYqNpKA9n5a+i0SFaN1f+Ag81eNN3Q1YxzuK8lBiueVjlmo/db7ENybLPt57Nl65LXmTgjnxFJKsLd0tM35EnhmQQCJsAACAASURBVGWbLfwREvLrlVL7lVJ/qpT6pVLqg8hXqk9u07A5pFnm6W/DgVflw737Dnz2Xbn4x7tjdfbIjfvHP/iI89cfglJ4BmdwX+yisKiYP37zJh5jM+LxPMT98w9x2kZEbKr7sriPnZ+JVkvvDfn54F2aPvkET7sxrcxmB0cunjFFU/uEXHCOXJra+vH0TbAw+biUmqSzOA/3x214BmcgowDPmMJ9oQNndlhGFnZehKkhaXSZ27Az7Mf9cRtOvUfmnoaCKXnbN4zJ/qWHusyM4ukapPle75LNTvRcE6NcXR/3EA11NYsWV62iYMXFcG4N/dmr7Ql5DYng6Rnhj3/wEacOVc9rODt//WHCyqPOolzcH7XiseyG+t/B4zjKH799j9Ddd3H2vY/7R2/gufwBnq4+3OdakitqF+9cUiWiF9H7v/++hPg++ffSMe29JiHD/S/DyX8Edd+QkvCs4m27+EPiHoAb+H+BPwVCK/zt44PFKi5w6WEp5+xqlqaZjibZgRftgZKDaGUa3zxzlD/6/nm6BsboGhzn1KFqLrR28s0XjuI+f2tOU8zJhG5oZ0WkYiJzyd1gdMf4uf1oBRY8j7y4m+7ienY3DIejOybpfrSjlecvbtJ54UjsfAI+6LqElpmP63Qd7p9/RP0Ro5nmlUa0yjLpdg4FZKj5duwh0HWJES+h9+K5cxP3pw9wvXQifvgmHBTPsECT/o4kkwptIO/QON88c5QLrZ1Ul8pO/dShar7/4Q3+4NefT/i85l077Zl88zd+nQvNV6jWx3HVjvP6T7oY8X9GQUUtr33jTMpyHkkV0QsFZFhLwCdJ/6H7sdLrQk08vOwyUXZNZkx/i5CoAQjquv4fU3omWxmlJLlTUCtJpsk+qSAavCtuobJyOqeMriNp/PmbH3OwtoLh8Wle++rTaBUFXH/Qw1/88ion9jujF2k8lzXi2nqHxnEW5eJqrOP1Nz+jssDBxdYOnt2dh7f1InQE0XIC4B1CGx/m9e9+yGxQx25TnKiwwv12GLLgvhUAdE46bXgnwjRoWeDLJmNMce5iP43PPDnvpmlq65fnv9+OM8dKfYWFN95rYm9NJZda7nGp5R6ul58XD+j+e3isu0XHpWFl5cstg39KPJh4YYGgH2+XB9eZ40uHbyKj/ari7/7XSyq0gSLXWHVp3rxF8w9+/flVHXuhcTp9fDfV5cXRY45kfopD7+JL+R60hz0wWiMLZ+GexCS3V8GaDeXspIQ/Z8alAavvFk0PxnHmWPBOKpxVVWj7n8ETKME7HsCZlYt3cJyG8sdv8YfEDcCbSqn/DvgxEBXl1nV9eTWtxxF7jnwV7YU9L8muYfQhHk8HXR1eXnGG+dG1fqrzLFB1F8+9Em7dHKW1axpHYBTPXiukZ+G+8BBXwz5JZM1OQGAap38U9487OVXjwP3uAKecYUbuj9IXDBMKQmfQwpGwDbcXTu3O40IPnNqrMdI3wqzVht3uoOzEEdx3ujlVWsTl4ENGJ31cnrXy2gvVeHyjvH7RA/4pvrS7l+Z371PrO4h28Ajk74p6FKcOVfP6+22Mjo2jKyu3/H5u3X9Efm42J+v2SX1163XcF97E9e0/2OxPZHXMTiz9O98IDYcqFyW7o81Oui5eYHZZcstj9TD4p/F4h3B/3CZeWUkWtbngfq8Z14vH0SpLYjmgRAbYxGG93sVSxqn+QCVvNLVCejZf/9Lfofn6bWptCm2qG4YfyIOzy6Ur3p4rSeV1zv9NyFDqYVnwAz7paxi8uyBxrKBwN86nTuFu7uHU0QO423o5lZ/HhdYHnDpUvbUb+JJAQmWgSilPnB/ruq5vaHFzyspA10kkPHPqoJML19uoygrw3Q8e4MyxkGOZpSDNz6t7bbx9P8iIT6cgQ/HaiXS0gsU3smdEx92mU1WUw3c/m6CiIJPcrEwK8rJ59Zl9XHgwRlVpAd97/wa/+flj3HjQC0rxpZN7+UXzPVAKe5qVczce8u0vn+TGg166Bsc5eaCSS23dVJXk8dqrx2H0EZeu3cDT2YvrkAWtLBe003hCFbz+s2Y6+sfoGZ7g21+s40bHIGSV8+qLz3Lh+h3RWLnZhqthH9rpb9J0pwen04mmxTTmPR4PXq9363kHfa00vfmXOLX9i2vFb12gYX/h0mW8g/dEOO3gV6QgIBn4p2SByimnqW0QZ1E2WqFNwg3ZZXge3MPb8YCGg2VISa8uhkhZjcbAxPWt1lOaulQvwalD1bx9sY2RyRkKsh289tWnASN8efoJtNyAhFVGOmTnHRlIk1MhhtbmEGmS3AoptkigJDPuuZy9hutpJ5p9zNhQ+aSTNzgTe2BWsSTuc8oleWvPjRY0RI5ZVZzL996/zm9+/hhdA+ObXr4bxewD2LoGoKmlg1BI50JrZ/SCOX/9IX/4Z79kr7OI3/rik5ypc3L2+kPe+Pg2e0sc/JNXDsWUQ+054iJb08GSxtlrHt5oauWBd5g9zkK+3nAoGouuKsnlbtcQuZl2OvpGmfEH+c63TkdvhD/6m/PMBoIMjk1HH3u9vZf/8MZFXnlmP//jN54DYh2ZhIN4H96jIeOB1DTnOvn3rcXcHwpSU5bP+PQsjYdKqS0vwGutxh+2RDVWzpw8ALNTeHKfwf2jH+NyudA0DY/Hg9vtjn6/pXhwFs+dG7g/vD6/W/TtD3EdsqFpNUvvsK/9tTT/Pf2P17wLB4yqr1lZnCxpUPnU8k1CRmUZ1nQpafSNiOEYeyS/S6BxcdXNYAuIV6J6/vpDvv/hDc48uZuT+2X05dwdc9yqnNlJqbAb65TXEQrEyikhJsDoyBcjN9knRi6zWPJZ04M0tfbgLMhAK7DJY4OzeIb8EubcZYtV1+XXiGy7zSEGZ4X3KdK4l5tpl+t+kxv45rGZfQBKqTTgd4GI1u2HwOu6rgeWfNAOoqGuhqaWjnk3U3VpHp+rq6WyOIfmNi9Wi5Xm+4N8vfG4VJbMxo/tRlzb/GwHkzN+njlUFXVxI4m7zz1Rw/fev079gUpm/fMrcgpyMnCkW3nYN0r/6BTffesSvcMT/N7Xn+GXl+7z6e1Hi3Y2El54Fvpa8Fz+AN/Dhzyz5xDfu/KA3/z8MZrbR6gty8MZ7sZ9Y3K+xkpuCK3Qhsvlwu12U19fT3Nz89Zc/AGmBtFqa3C9nDtfMfJ0HZq1Z+mF3dC2Z89La1/8QwFJMFqsssDlOA1xvhUGpigV+xuLVUIpIN3avS2iW2OxgD1vyYa99SqPxqtgslrVojxC5JjxKqEA2YDVLvAKAz4ptfSNGF+j0oEcCshuPTgjE9lCQXDk0XBsvzRyKaO50WZHq8lEc+SJts4aNnmR+25/ZVHUA1hS/C8cEAHGcIBYk82c36Pkx8oSm528RUk0BPSfgDTAkNTk7wMhXdf/cQrPbREb5gGEw9JNqyxrqnlfuLs6f/0hf/T983znm6c5fax2yd3XXLf6QmvnvH/fvihdyq/W7+XC7Uec2l/GhTvdnNpfzoW2bk7tr+DC3Z55//7Hn1/nnneUfc58/vd/8Dk+bevlz9+/zW9/6Th/9wtPxT/vs9c4ldXNhdsPOVWncWGyilOHa+T5g7O89o0X0I6/gKe7TzRWzpxAq6mEw1/j7Icfcu7cORobGzlzZnF37KYTDsHlv5BFRSnOXrge82aqwzGNp3jceVsmR5363bVJ9kZUWksOQU5pcheFgM9oQvTEOl1Ds7ILXo/8wA5hyfvuYCUXbnfgatiLVpJJdLFPy5BS7MxCeb+VRbwUm12qxHyj8reBadm5h4Mx4cW0zLX10WxyJ3C9ruvH5nz/gVLq+prOZKsRkT2IJNh0ZBcV+TBDcZychfpt1jSjMcsGyoJ3cAzX6cPRxd1qVXznm6exWuUCWmr3FdmleQdG5aIrzaY6F7xDE5yozQOlYw1O4zp9CK1mF9W1Gt7haVxfP847F1pxuX4D7/AUrqekXNHa1MtXnjjKzKyft+/6mZq28duvnOTja/d5dnc+mrPQELyyxp7/zJN4BzVcVRVoU1eozlV4LTWcOFAp+vhZfpjoRauqwPXy83j7h9CmBvC0NNPc3ExjYyPNzc3U1tZuPQ/AP0VEYG+eYuSNVmqDNrTaJer0AzMw2CaSGGtZ/PWwhI52Pb3u5Gdc0jKk2ibXKbMJZsYgZx9MeKXpLSNfFp9w+PGS9EgS0fsu4iUVO6jO0fGOTOE6fRjveBjtyDGwZ4Etc/n30GKTDUaE/Brp2p8ZF4Mw7pXrwZG7JTyDRD2AK4BL1/UHxve7gb/VdX3xNjKFrNsDiKjyzcWaJm5jVrHsEG12SUpFPuRQIKbMGQ7Jl0KOE+nW9Y3GEk9hf+zYC9/bhfIMS731FpskqdKzwOqQEZU2h+wirPYVb+KFCojnm2/yR3/q5ju/4+J0/RH5/c/ew/XcXrTMKXlNkd3i3ASc9xrcf1dKHne/YLwfhk5LbUO0Ltrz4D7udz/D9Xv/C9refVs3BzDRC3fewjNuma8Y2XYL949/guvzT8cPW3RfloHqT/3W6uUPwkHwjUF+lfSTbCS6LiW7fbcAi1xz1vSU5MG2LSG/IcWuYvejPSc2UzjZqrhBv6EW+kAMsmNl8Tlg0z2APwDOKqXakcuoBvjtNZ3JZmFNh/RcKN4zx2XLWLmZabkLIC1DLHnOEjKz4WDM9ddD4iUoY8C4HhaPwWqXn1usRhJ4CS3/VeDtH5onf2u1WvjO77iwWsVwaFXluL7yedm9aydluMy4V2YFOHJj74nzSZgaoOmTT3HO5KAdPhEdoOK5+RlevYiGp57AOzaLq/EwWpZUeGiahsvlwuv1bi0D4J8CffH7o+UGcZ1eIh4eDknjV07F6hf/mQlAN8TB9iTnNawGpUTKw54Xi0l7rxo5A0VTax/OsmLJARmkRMJhlaRSEwkQSfeAIelgSRPDbM8GrGBLk3syVd29tnSRW891SkPi0AOYmYSMzfEIEnpGXdffV0rtAw4YP2rTdX12ucdsOcqPyL9JbkhZFost1jcwl1UMoF4VhsFpqNPEmIQCoCxxB25rVeWxMsisYvnKKRf1x8CUeCDKAntexNnRifvNd3FllYpC5kgI93sXcP26C0COH+mQLTkItnQ0Tdtaiz+Ip2ZLm/9+hIIwNYBWXYEWz7PyXhUDuffzq3uucFAGlu96bmOvuXg45lx/1c8Y4YgpnNUZuN85L+Wau5x4ese2RN170jWRdD1WdaWHZeNWfkTuS5tjc7StbOkiHpddIfmb4XbZJGYUbKi0xLIGQCn1oq7rHyilfm3Br/YqpdB1/UcpPLfkYrEuVszcLoT8cgErZVzIhu2d67aiy+48q9xwa8fl9+FATLtHB6zWmELpwmqW7FKpThm6JxelMcZSe+7ruHx/jvvHP6W+8WWa27xyczomY4+12GTBG++WFvqtyPTQ4rr52XFZIOIt/gGfDE0vqF3da9J1mB6F8ic2f/FfiDXNGGJUhPbkLlw5Fbjf+Cn1PUM03+2V/FKxI6aguQkkRerBaK4jOEtTqxdnRTnanv2y0bHn4Onuw9v/MO7maEOx2sQjyKuUzdfUQGryREuwkgfQCHwAfDXO73Rg+xiArU6kNjySeA4uqI3OLpXf5VXJOMrowmKUCKZlLD0pLFJ+ODsh/070GB2RShb5uWEwW7pUqoBhBAohsxDtqTPUD/+Ccxev0fjMcbRdlXKxzozH6qvtuRJv3qoGwDds6MTPYWYMlIofdrjVjLd9koavN67ueWZGjOlWi+cNrIYlh5v0DyVt4dL27KG+4UXOfXaDxudOo+0rklDZzKgYskjOSdfFeG6QUVh113LQLzv88Jyy6KwS6XB3TOP+5ae4nJloRbnzcmRbBmsalD4B3suyUVFWwwtPrTewrAHQdf1fG//9t7quz+sGVkpt0bt8A4iUekXqgK3pspvWwzFt/uhYR2OSUTi4uJU/klyODG1x5In1t9klMZ2eLY915K+vesOaFgvzAPCk7G6H2uWCmzIGbERkbS0WMQLKYkzFKsTDLpr702is6ae5rUvqowvtkpyqOCqHTc+WyhPfaOrCXGslFJQdoX1BM9DUANgcOIuI0116A9cTRYti/8vGqPfmy3OUHV53xY2ztGjJ8YbJIloN9fRR6e3Ysxet6pAsqBM9YghCAfHwpodinoElLTZkKPI6w8Z1n4RFKyb1UEPznc5YPX44JBuZhd58WobE1R15hoebGY2pa7sKcb38/Py+j+VGREbQw1IBFpiW+zrC3NcXTRwbYx4ja8FasNrAeULKkUc7pQkuxTM4Es06/BBYWPHzt8CJ5J7OFibgk92iHhbrXLxPFCV144IMB4zGj1z50NIy5caZHTdizxmQYYy4m+oHlDEU3rhgN7o8Ly1DQhTFe0X+dvCetOznlBnJaIvE8wFP223cn3XKTTNyntqiLKPV/zBaQZ8sFjbD+1BWuXi3mgEITBmJ0Dk3bzgon48jD60iY37YobUD155ptH1HFx1qyRj1qRr5nCuOJyWhp1WVr23hSpCF1WK1VWXz5+cW1CD1HgahoNwDM6Oy2/ZPy/sXDho5J2ORXE4fPxSpibfOmXtteBh6CHRdNJE+uo2r4QBakZ3afBvu9z7D1bBfZgDkVcl9Y7HGJDFWaKbTqsqpr9sf7ftY9j30T4pnqyzSgZznlH9tDvlcbfbY5xvwiWc52invxWSXMQxqjUq5VptsAu058n5Oj0Bm6kJCK+UADgJPAHkL8gC5wOMpjzeXSI9AZGB6VX2sASRRDZZI1+ZcspMvI7xmbHapUomMs3z4sSzmkZu4cA/e0csiOV1ZApduogXu4Gp8Vapm8rMl5BEZn+nIF4XU8rqtpZPun15cdhvtCzCqo+aGHXZZ0KxKJL8XEDdG3bAXrTQXKo8nVSZ7VQvXKllUDWUYHG//UPznsdokf5A1Z7JWOCRe3+SAbIpGOkTmYSkDaLPHwpt62Agp6fLZ2OygrHhHZnC9eEL6VHKr0fYV4Kq4j3dgDG33iTVtlub1fUS62KvKY6FRZSFam+3Il8FQuZUrf5aOXLnHI9pQg/eh/awYJnvO2ktuLTaZUd7XIhu0FM3gWGmbcgD4CpDP/DzABPA7631ypdTLwP8NWIH/pOv6H633mElhdlJ2ObZ0ydLvbpQywK20oCUbpaBkv1zM7eekgSirGGzpNJz5gvQEKAs4j8ODD9D2zOAdAvcnHk4+YUU7KQbA0zfKpc8u4hzIpuGlVzb5Rc3BP7n4Z7MTzG3ln6cw+dF71O5Jx9sRwFk8EjfcEzUWR2rQCtMkFJZkzfglF651EMktzM0jzM0trOr4FqtUs+RVy/elh405CZEQjREitaYZA97TDOkHI7yRng1Bn6HfI8tRQ+3ip9H27EdbYyXtIk+nogj3G2/JbI79h2Wxd+SJQQoH5/cBrZbivfK6podkwJIvuHZv2JoG5cfEoPbcSEn+ZaUcwE+AnyilntV1/dNkPrFSygr8B+ALQBfQrJT6qa7rt5P5PKtmalB2Ige//Pgv+vGIjMW89WPZIadnQVaphLtmp6CsjqZzH+C0fIJzz4u8+ckdLt/t5tVgIX2jk1xuuQf+aU6+uMVEBqcHF+/mJgdk4WGBfEdpFrXdk7jbczlVFSc3YMgGRI3FzbvU1n4BLclhrxVDNGskpbkFiyW+1zuXtAypfImwlPxGkoh6OhXFUvJboHD9g/8W77QV7XAKJEuMwgmySuDOWzA9vPZYvsUioThrenIlyA0S7QR2AP8ICQdFtzi6rv83a35ipZ4F/o2u618yvv8XxjH/j6Ues65O4FtvxGqA4xHyywdlz4UDr2y98r2NZqIP7vxM3OH0THGVH12AzEI8F9+SBOlv/jbYMvh333uXq52T7K7dRY2zlNe+8aLsmI/+xma/ihitb0roIbLYhMPiqttzZQ7y3MRuz3W490s85V/B68+Mxvwj4Z6IVoyrsQ4tX8lozSuDuF45ndQQTSqrgCKLfipyC1uK4IzE0UF21EV7ZR5BipOrUWbGZbzkzKiESde6oZzohepTULY2GfL1dgL/FXAH+BLwb4G/B7Su6UxiVAKdc77vAp5Z5zHXxvSwJKDK6qR6Y6cv/iC7uINfkR2MxahQyikH3wha3dO4+m7LyMj6enRlpTLfTrrNxpcanhJNnXGvhNK2guyArhu7sDmx64DRjW24+vM6THuuQ2Yx2r6DaMYNO7ck0WpVhqeQA4FptGMNuMqGl46dr5EVG/jWQSpzC1uC4IyEYdIyoeZZyW+lZ2+8R+/IhYOvyuZpoE2iCtbN1wCKkGiga6+u6/8KmNJ1/S+AL7NBi7VS6ttKqUtKqUsDAwPJfwL/tCSyDn9dhnyn2B3dVuSUQe3nJCwGImwV9ENWKVplKfWFY7zR1Mr4TIiSbBsOK/yi6Up0mDy+LTIwLuCT2O7cGOrs5CIlX0B2WpN94DwWXSwWTp+KegqzEzIb2JaOVlW++U1Fq2BhbiH6mW1XQkG5Tsd7YMwrn3nN5+CISxK0C3WuNhKbHbTTop814RXvc4uQqAGISGKOKqXqgDxglcIoi+gGqud8X2X8bB66rn9X1/WTuq6fLClJcvXMzLiUbu3/YkJDNXYkhbvl5gn4DIG6TAj58Vhq+cWNPvqHRugdmuDvvfAE3/n1U6DrvP6Dn+PpHVswfm8T8U8BiqYrt2IL3dQAWOx4ekZoaumI/W1/q1RwGMJtc3MDZ47vjlb/eLxDgA6522/nPDfmf+bUsWip6WYZgXmfy5xzbLpya+kH6WHZ5c+MS/f5zIgxpvWMCKYd/Q0oPZh8Mbe1ohSUPQGVJ2Cie8uoEiRqAL6rlCoA/iXwU+A28O/W+dzNwD6llKaUSge+aRx7Y5gakA/l8NeWFnMzEXe1+hmjO1FBgYbnUTfu20FOVFj5tScy+V//4UtceDAEk3289usvc+KJvXhHZ6SJbCtgVABFkp+eR16YHsQz6MN9rgVnkWH8dV16IfJ3Rat5lhyk0tsn7nySq342guXKPzeD6OdiGIGIgXKWFslnMrcEcnYcxrqlNFIPSyy/thGOfUtCPYWa3M+bJGOxIs6npON33LtYLXgTWDEYpZSyAOO6ro8A54GkzEjTdT2olPrvgV8gZaB/puv6MiY/SQT90oiVWym7hZWmMZnIgpieJTuurBKZOfDiU2h9fcA0HKulujQPb1cXDScz0XYZ1STjXgmxbXZOxTcsmkZVpbLb/dkH1JfO0OxZMPN1ekiSdVX10YfGU5/UyvLQckNGo9T2I5W5hbWwZMNbZZns7i1pRtc90pV98HkZMr8dZxtYLLDrWck5Dt6VdWgTWdEA6LoeVkr9c+C/JvvJdV1/G3g72cddkogY2K5npRJgq7iHWx2LVer/O5ogt5KGk0dlVx3QoOMTCPjQKgrQ8hVMDUvlUATf8OYbgMlBSJOdulZVTv1BJ+fOnafxxKH5GjND9+Xfor3LH292PBYaM0kK0aT0ZzdoPKah5YZlp196UAzyaIe8349DabbFIiqxwRnJV6xUNpvKU0nw795TSv0zpVS1Uqow8pXSM0s2Skmcv+4b0qVqLv6ro3C3MWowKLuW4AwUGM7gyEP5Ny1TklwRLDZJqG42vuHY8JquXpqv36HxyC6ZzdwzEvu7oXuywCxXuRQKyLWUvz13/1uKyLwMwHO3leZLl2k8Uk3zgyE8afvh8K9ATYMY75IDovWz3Rf/CFYbaC/ImjQztmmnkWg9UqSg+/fm/EwnSeGgDaH4oCg0mlU+a8Nml8Ro/21DrlaJpIUtA4Y9UmlhSxcvKxL2sWfDyCNJfG0WoaCIeWXkx5Kfz9aileZSu2s61uRVlC4VQDWfW/54M2NQdiSpcg87hnBQynGVRbpjx3vkc7l9Ffcn7bh+9w/RDh6jtqPDmChXhZazDcM8iWJLB60Rbv9YNk+bsClN6N3VdV2L87V9Fn+A0gNmpc96KaiVHXBEKyjol5+NeOYktJTMQAUxDr7h+dLWG01gOvpfb/8Qri82oBXYwJY2bzYzI0Yl0HJS1gEfpOeYRQNLMTsu18RCwmFJ2k4NyiYi1ylS41X1cMSFN+8pWfwPHweLZd5EuceerCLY1SDKq+HU6P0sR0IegFIqE/ifgF26rn87Mh1M1/WfpfTsTLYWmUWysw/OSqikr0UWzIFWucFzykRaYbJvfqzWN7J5i2ZgOnoeDU89IbX7j+YLwGkVBdLwlpYRE7WLh38aKp9KXfIx6JdFIN1Qkp0aEC/KniuL6PRgbKA4FmOM6CaFMiNKt+GQ5HwiA1hC46KeqSxyzpENQOlh6a1Iz5LNQlmdJHSVouHFlxcdfktOlEsVZYckyd15AXKrts5EsDn8OXAZeM74vhtwA6YB2ElE5gT0XJdFSNfFAwDxAnLKZBGdGpKb32KRhWBycPMMgH9qfrndHI8giq5LHqOgdumbLxyUxTYjyamvcNjIk4SlU9ViE/EvFFSelJ3yuBewGF5svnSUgjGTwjAYkYFAIIvyZJ8EabOL5eez4/K55JRJuAGiI0MXlUyG/IY6Z3qsizrok/6IiKSqJU2Ez6zp0HtTFvkDL4ssct8tiXFbbJBfKyJ5c6UXlNrUxOeWpPyIVDz5RrbURLAIe3Rd/w2l1LcAdF2fVupxycaYrIr8XeC9Iju5tAy50bPLZKbprlPG9Kgw+Cdput2JMy8DLacTKmSeq8fjwev10tDQsDHnOzM6X5p4ZnzxgjfZL4ahYJkd5+ykvPZk7/6n+iXBWXFMdvvhsISj0hySs6o8Lj/Tw2C10dTUhNP5pOyOAzMwdB9P6zW87c00HNsrRso/JeEVaxp0XTJGgwZF5K+jSQwHSGI8OGOM8wzHmpMi8yCmfNItnV8jxwuHpPrLmj5/klzRXtGOKqgVz8+RbwwgKtlSsgdbGmUY/NtvyPu3Qctrop+OXymVgWH+lVJ7gO01FN4kOWQUxCac5VRIeV6B6qk9LgAAIABJREFUJtK3wRlZVJQC34g0+Pz8PK7AFNq+L+LpeGQk91wbd77TI/ObtXwjUQXQKJEqpog3E49waP1eTDgsO/GQ3xiKYpNFtOpkbL6ExQJFCwyRxUIkXed0OqPvoaZpeHxZuJu9uL72d8ExJp9LblXsGPm7RN67ZL8YmqwS0UGy50pn98yodD+n58rw+LQMWdx1XYyTPWd+uevcWQARMvJjksc2O1QcWd/7tFPJLjFyah3iISVhqNBKJPoM/xp4B6hWSv010AD8w1SdlMkWxmKB/F00nT+Ls6QATQ9LHqDzAp62W3j1YhoOlMJkH1r107heOY37jbept79N843W6MK1YfhGYjvVcFhyAAvHQo51ysSnpSrEZifE8KWvQ9hudhxmJqGwRhbYcFhKZitPJD5cCKIJUrfbTX19Pc3Nzcu/p45cODxnlIcxED5KRgHUPLfoYYAka002ltrnJVzWeUkGxae4ozkhA6Dr+rtKqSvAKcQp/Ke6rg+m9MxMti75NTjzHbjPXsN10IpWuwvPuA33jWu4vvFrsuP2jUA4KA0+h2s59+EHNH7pVzZ28Q9LKCo60zc0O08BFJDvx7uj2j+LjxGS5GzVybW75eGwhJ4Of23BfOG1lcdqmkZ9fT3nzp2jsbFx5yRLdwJpDtkUhIIyWS97vZJry7NsQFMp9VTkCxkO2gN4gV3Gz0x2IlnFaJWlaNVOXn+vjbNX7uN+YMe1T9r1m24ZGkD+KWm8au2g8cQTNDc34/F4Nu48gz5jhrNxmRtNR/OY7JeQTF5V/GPMjsuow7Xs/iO6NaMdsaqXJODxeGhubqaxsXHj31OTjaHssITzUqwcupIH8H8t8zsdeDGJ52KyXbDnQHoWJw/W8N4H8MbHt/j6oRqYacF99iquM8cBHY/Hg7vpnui67Kmm9uRL8+LXKcc/zTzN54Bv8S5+zBhJkVdNXHR9bQu3roNvHPYat8hSBmaVeDyeee9hbW3txr6nJhuDPUeaKwfuprRiaqWRkCmYl2byWFBQC48+oaAgn5Hecb5/1UfBtJ/XvmXU1fun8Ho6ZPF3FsNUP9r+L0YbfDZksQpMEZtNizSoLUysjXVJ1UU8+YdwSEofbWsQDPSNQMGuuEPl14PX65232M9tmjINwGNGxZMw9ECqvVJEQjVtSqlMpdS/VEp91/h+n1LqKyk7K5Mtj2ckiPuXn/Dat77KN08fwJGVy4hPh3FD+8fmoGFPjig6WmwSZglMo2naBpaAjs9f8H1j8yuAdF0MQL7s/ptaOuZrAwVn8IxB07VVDr/zT0ldfOWiCXzrpqGhYdFCv6HvqcnGkZ4J2vPSEJii+QGJFjX/OeBnfiPY/5aSMzLZFniHfbheOgm2dJrbh/j6cwcoyMvh0h0jHm2xyg460nillFTTbCRTg7EFP5IQnluXPj0opatGeCYy+zdiBDxd/bg/bRdd+kTwT4q+TcAnYwDjlUyamKyG/BoZJJOi6XqJGoA9uq7/nxiTwXRdXxBcNdlpNJz5AqRl4H77Q1xfeI4zT5Tz2hcO4vGO4OmOjO7Uo8NYQMUakDYK33BUBpqg4UarOZf8gvh/RBvIfa6Fs1fbcTfdxfXKC4np5E8Pi8GrfV4UZ029IJNkoJQ0WFY8Of/aTRKJHtFsBDNZhHc6DdeZJ9H27IVwAG3PXlyHrXg7DWE1S1ps0U/L3NgJYZGa/4g8QrwKoNGuWEOUgVZRIAPgr3uo31eeWFx9akie5+BXoGTf5s8/MHm8sFhhz0sivZHsQyf4dwsbwd4H/nnSz8ZkW9Hw0itoZTmSQFUKcp1oBRYaKkPyBzaHyEODlFGOd88f75dKAtMS44+WgE7NrwDSdfEAFlTnRAfAP+Gk+eE4Hu8AyzI1CGl2OPDK8nMETEzWgyWOZlMSMBvBTNZOZC6ALQOwyIKfUSC179WIZIRvRETHrGmSyPKNSMt7qvFPzf/eNxzzBkDOIzA9zwDMHQCv5enUHnwyOjw9bhgo5Ad0OPCqOWfCZFuSaBXQrwJBXdffMiSgg0qpr6f21Ey2PPbcWJVNZpFUvuRWxgZeR3bckcVYWURfZiMITNF0/V500Di+MbDZ8fSM0NTSEbf+PzoAviwPFGh79y8/LH16GMqPmYu/ybYl4RCQruvRuWW6ro8iYSGTnYzFYoiLTYv6Y8gwAEHf/KqF2XH5Nz0bhh9uzLlNj+AsK8b9zkd4HnZCyI+nbwL3uRacRblS/pmWOU96t6GuRnoYAtOQVQa2dLSq8rhD1NF1Gey93AAZE5MtTqJicPEMhanzaiKCYT3XJP6t67GQyli3IRnskCRp/i5Jjk72Sx4g1TLBvmG0mipcLxfifvNd6ksDND80dvjl+fCoU+r/4+n7hPwrC6HNjELeLnPKnMm2JlEP4JJS6k+UUnuMrz9BBsSY7HQyi6T8Mc1IgDryRVJ4vFu+tznEGwgbmjy6HvMIUsn0MFgdIkZ3oIpzLV3UH6iUHf7suFQIxZNniAyPWagYOhejqY1KUw7LZHuTqAH4J0gj2A+A7wMzzB8Qb7JTiZRQWm0SUgkHIcdpTLEiNiAmUoc/rzcgRYQCRtNXmojR3bhN49Eamtu6pclrdBn9n6BfdFiWGvquh2X4Se1pCXuZmGxjEq0CmgK+k+JzMdmOGGMMmy7dxGnT0XJnZZLV8AM8Xb14R2dp2J0jO+b0TCllmx6RkFCqmJ0AFJ7uPqnieVZDKy+gtnpKqnyqh9BsDpkBsJCgDwp3L33siT7pzCzZn7LTNzHZKBKtAnpXKZU/5/sCpdQvUndaJtsGiwWyy3AWZOJuuounexByKvCMhHG/e1ESrsoSk4GwZaS+Esg/Ceh4+4dwfbEBrTANrLZop6+3u0vCP/Hi/3o4Nt1qITNj4vFU1af09E1MNopEQ0DFRuUPALqujwCpnVRgsn3IqUAry8b1ciPuprucfTCL+3YA15Esibnb7FJ3D5IfmEyxAfCNgrLQ8NQTaOVGiMpoCNOK0mko9y0R/zcEt+aOQJyLf0qmZy0VHjIx2WYkagDCSqmoz66UqsWQhTAxIbMQwiE0rYb6feWScN1dgOYwKoet6WIAdF0awoIzKZW4ZbJfDA0sfp6JHvk3p2Lx44KzksSON4s15BdhuezUabObmGw0iRqA/xn4WCn1V0qp7wHngH+RutMy2VbYcwCFxztIc/sIjXXVNPfb8XR0yaJvsc5PBCuV2kTw9OAcAzBFU6s3JvM80SvnOpEuDWFzCc5A1hKOrW8USg7OHydpYrLNSehq1nX9HeAk0Ab8DfD7QBx1LZMdiT0Hj3cQ98/P4/riKc7UleN6/hDuGxN4Hs5ZZP2GNLRO6kpBg0aJZkT2YWYcZ3FBTOZ5ogfPbD7uj9vwDk3M1//XdTzDszRdubX4uOEgFNam5pxNTDaJRJPA/xgRgPt94J8BfwX8m9Sdlsm2wmLFOxHG9VI92u49Mgx+7z5ch9PwdhkGQKnYom9Nk2qaVOCfnB+cnBlFqyoxZJ5vcvZ6B+42hauxjpP7K2OGQdfx9I7h/uDqYv1/36h0DGea+v4mjxeJ+rP/FKgHOowxkceBDRZ3N9nKNJx+QZRB0wwp5KxitCI7DRWG+qctQyYbgXTPDrfHmq6SyewE0TGQ4ZDIQFvSROa5No9z7T7qD4nkwzz9/8t3cV98hOvVxvnCb0G/lIbufiF+1ZCJyTYmUQMwo+v6DIBSyq7r+h3gQOpOy2TbkV0qDVgRA4CSYdaRpKvNLmWU4aCEZ4IzIqeQbHyjsSRu0IhSKiUyzy33aKyx0dwdjIZ+ovr/1x5Qf/TwYtXP6QHY1SCJbhOTx4xEDUCX0QfwBvCuUuonQMcKjzHZSdhzAd3oCM6AcEA6gif7ZdGP7J5n50wIm1xBa38tTPXHhrgbFUBRmec6B2f2ZOD6fH009BPV/69z0ny/P6YeGsUihszE5DEk0STwr+q6Pqrr+r8B/hXwnwFTDtokhj0nJr+cUSChk9wKPMMBmpqvy98oJbLMIBLKIw+Tfx5TA7ExkLOToFRM5tk+CjnlaM4iXI11XLrbHdX/P3N0F65Xz4h6aMQI6GE556X6AkxMtjmrrmnTdf2crus/1XXdv9YnVUq5lFK3lFJhpdTJtR7HZAths+N0OnG//SGe4YDIL09l4L4dwGk35gHYHLEu4OiEsEDyziE4KzH/aAXQKFjtNNTV4B0YxdPZE53Vq1UURP/VSrMgLQOttnq+/n9gRhK/KZjEZGKyFdgsSecW4NeA1zfp+U1SgLb/MC7A/csLMfnlY/lodqP6x2YXDyAcMhZVXZK2yYqvz04iA+sMZkajOQlnZgD3rVlce3LQIBr+cTXWieHINgxDVXksDxCYhrx9yTk3E5MtyKYYAF3XWwGUWVXxeJFVjlbaSf2xw5x79y0aTxxCsw/DhKEMqiyALgurPQfCuiSGk2UA/HMqgIKz4l3YZfeuZU7hOpyG++ow9f52mtu6JSxUUSBzi+cMhokSCkhy28TkMWXLtzUqpb6tlLqklLo0MJCCpKFJ8sjIw9PVT/PtdhqPajTf6cQzkyeLfKQJDGIjIm3psdLQZDA9BJY0+X9gQZ/iRM//3969B0d13Qcc/57V+/1C6IEEujyNkcFgZCAyyBQSMHXjxOmm7tRJ3GYSTzLNH32kkzZ/9DWdaSdt/+g0ndiZZpombWNvMk5wEsePGIMjG7xgDBYYAeby0AskISSh92pP/zh3dXelFUiyVitpf58ZjVe7q91zLXF/e3/nnN8Pa2k2NRtWcvj0Zbc3ADh5/kkauqflzd74hJhnYhYAlFKvKaUaonw9Np3X0Vo/q7XeqrXeWlw8B83ExYzZrTfx/fo43v072b19E96PrTa7gbuC7nJQT7K7/DMlY3Y3hN0OLwHRH/lYbxv2cCH+xhbqNla5vQGCo0CSu3IoRGtAS8cvsajFLAWktd4bq9cW81NLRzfePVuxli2FngDW0ht49z5IyxtnsHpboWiVmQjuH1cZNBicnRo7/R1uJ6/BW2bHMUBgCLu5HV/rMryPmbRPVakpD+H92CqsFZUT3z8waMpCh15DiEVo3qeAxMJRu3MX1uo1Jv+emgVaY1WUULu+zO0QlpRiyjUEA06LyKCTu/+IhvvMxq9Qr+GBblO9E6C31ZSqeHhTxOofb101Le03o5d4GOyGIpkAFotbXAKAUurTSqkmYAfwC2kus4hkFZv8e0qWe19umUkBae1uCBsOy9EPzkJhuIEuxlYABQNjLSEB6GmmdnkK1urILl5WWQG168ujr/PX+u6N4YVY4OISALTWL2itK7TWaVrrEq31vniMQ8RA1lJTOz851XwCD/UIHh02zeFDRpwdwZ4k08D9o7rd7q7XD004h4JNT4vp35ucFvkzoVpEqVmR948Om+dmSPkHsbhJCkjMrvRc98SakQ8jTo9gCEsDpbplIFKzoPvqR3/f7mtmcxlE9hrQ2rxvtE/zoyPm/cfn+Qe7oWi11P4Xi578hYvZlZrl7sXKLITgsPkknZTmrgRKyTTLP4OjJlV0+4YpHTFToyPQF7YCqL/Tzf/3d5hP9LnLJv5cYDB6/n90BPIrZz4eIRYICQBidqVmA8pM7qZmu3n/nFLoCS0FNZO/9ceOYzc7y0D7TfkF27apr6+f3nsOdgN6rO8v/V2Q4gSA0FVHtCuA4Ej0DWAg6R+RECQAiNnl8TjF4AbDSkNjTsB97ebTOIDyUJ6lTfG11ptmnb5t4/P5KC+f5uRrX6d7OzAIo0NuSeieZjOO9PwoP6gm5v8Dg2YpaaignBCLWLxqAYnFLKvYVPrMLAybCC4DtNn4lV8JqVlYScN49z2E75eHqOkYwN+RjtfrxbKs6b3f7TY3/TM8bgNYKP8/vuxIcNQEifAgBaae0JLV03t/IRYouQIQsy+r2OwFAHciOMeZCA7NAySlwMggVkmeqR309nFqNm+c/sk/9JqhZaeDPe7JfqTfLA+Nlv4ZGTQBanxgGA0bqxCLnAQAMfvSchhrzBuaCE7NhPQ8NwCA6dRl26ZT15a1+N98Hdu2p/deI4NmE1iyUwK6r93sNoaw/H+UCeDRIchcEuUF1eTzAkIsMhIAxOwLL6wWmggG80m8p3nse/vGbXwvHTa1gx7agXfHCnzPPz+9IDAUtoksOGomhJPDJoCVB7In6eiVNq7OTzBgrkzG3y/EIiUBQMy+lCyTXw8GI3PseZXm07pTDK6lewTvjhWm/n5qJlZRGt5H99LS0jL19xrsdm8P9xOxGqin2Zz8x6/z10GiTgAP95v0j5QpFwlCAoCYfR6PWV8f6DepmeR0s7Y+r8I8fusaALUbV2IVprp7AJJTsbKHqa2tnfp79baGTQCHbQALjkJvW/T8f2AIMgsmbvQa7oueLhJikZIAIGIju9StyZ9V5FTXLDRXBN0mALiTtU5/gPR86LxgrhymqrfNvcro74CkNOobrmB/eNGkdJwTut3aRX3DFfO8wBBkRMv/69lrTiPEAiABQMRG9hJ3zX9GobkCUMpcBXQ3uc9TCgacPH5SCgRGImsG3Un/TTfnr7XZTJaSRnlRLr7XT5g+BHnLsFu78B1uoLzIye3rIGRMkudPlwYwInFIABCxkZbL2J9XeK49r9JM3IZy98kZZh1/iMfjrt65k+AoXP6N+9ojfSZ4eJJNqed7FL4LKRw6c8PU/Q+1fwxNSKeM3wA2bF4rZVxjGCEWMQkAIjbScgFN/Ykz2Nd7zcRsMAj5ldhdQeqPnTDPS06DoW43XZSeC50X7/767Y3Qd8NdsjnQ5aaUdBAruZ2adcsmtn8MFYALLRsNGb4t5Z9FwpEAIGIjKRkyCigvzMb3Sj32LQ2jQ9jdSfjOJ1Gunf0ASoFWYW0iM00q5249AtrPRa7X72l1yzf0tWN3DOC/nhTZ/hFM05isKPn/0SFTtlqIBCIBQMROdglWSQ7e/TvxvWVz6N0L+I6cwbtrA1ZSm5kYBnPi7gnbIJaUCm0Nk7/u0G3ziT80+RsYcuYCTACwLzbiOzuC9+Pb2L15Jd66anyHG9wewNE2emnMrmUhEogEABE72SUQGMSqKKVm03oOv3/NpGPWbzITsZ0fmuelpJtP/aHloFlL4MYZ6L9JfX39hI1h9pkT1J+64N4RunpwUkAt167g3Vxoev0S1v6x07mqSM0mQqgBvGwAEwlGAoCIHWdFjd3Uhr+xibrqCpOOuZ1uTsId583zQhu3+q6736dkwtWjlJeW4PP5xoKAbdv4fvTDyIqht2+YqwaAYIDagk6sVZEF3cbaPyanTZzoDQyZT//j5wWEWOSkGqiInbQc7OZ2fPXn8R54GGvkAlUVt00aqKoCq8tZq+9JNuUjOi9BzjJnI1khdDdhlS7B6/Xi8/moqanBf/QtvLVrsKxV5j2CQdNdLFR+orvJ1PkvXDVxPCP97ma08fcXVMXsf4MQ85VcAYjYSUmn5dYQ3r3bTbmHrCKs4gyTjgkuMSd/Z1cwSanmk3h/h/vzOeXQ+h5W+m1qtm7l8KHXqSn3YFWUuLt4h3tBj7r9gG9eMgElf/nE8QRHozd6GR2avF6QEIuYBAARU7U7d2GV5JhvnA1hVlkBtdu2mhN1V1h+PzXD9BEI8Xggtwz76Iv4X/ohdcvBf/xd7FthO4X7w5d/ajOvkL88Sv0fZ/2/k+evf/cMdlOb+1hG/sy6kQmxgEkAELGVU+au8Q/fEJaUYjaF3bzk3peSaVb3DPWO3WW3dOCrv4C1NIeqygq8nzpguog1tWE3tVF/7B1IdlYDDdw0E8KFKyeOIzBsylQ7ef7ypUXmda61mrLULZ0z60YmxAImAUDEVnqe2yQ+JcvdEAbmRD3QZb5CPMmmvo+j5UYn3kd2sXXL/fheOwaAd/9OjjdcwHfwFcqzcCdv2xvNf4uidPQKDESkeayKUrM89aU3OPTeFXwv/Gxm3ciEWMAkAIjYSs910y8ejykMF7oiKHROth1hSzrTsk2xuGAAgNotG7AqSt0T9q/e5HLTdezL1/BuzMZaUeFsJtNw4wNzVZGWM3EcwaCpABrGqiilZl0lhxuuUVNTIyd/kXAkAIjYSsl0y0EDZJfBqLMBLKPAVOtsOx0WJJIjJ4fDWBWl1FSv5bD/fWoq00x5h1Dzl74bJgVUfM/EMQSDJvikRq7zt5va8L9/nrrde/D7/dPvRibEAicBQMRWqALoUK+ZeO0cBJTbFSxpJfWN7ZEVQtPzzR6Bwd6Il7Kb2vA3nKdu8xr8p89h3wy4D7afA+Wh/kamW/Yh9HPXWqm/1BdR/99uasP3qzfx7tnK7o8/MrbUVIKASCQSAETs5VVAYMhMvL72DnbXKAQGTZnmhkHKC9Kh9ZT7fE+S2azVdmqsXMTYCXv/TnavK8C7cz2+I2ec8g4BUzqioIrykmK37AOmD4DvSAPlyyMnhltudOLdV4u1bCmk52FZFl6vd3rdyIRY4GQjmIg9p8nKWB7/hRepKU/Cf7kH78ObsPqD0HISBne69fhTs8wVQNO7sGyzOWHv34lVnAXX3sdaUYE3PZeWzh4s1WQ2cy17AKugYKz2T826Zfgbm/HWrsVaFbkxrHbLBlMBNDl9bA+BZVkyDyASilwBiNhLyzMn2WDA5PE3bzQTr6EyzRU1gIKrRyN/Lj0HgsNw5W1qV+VjZQ6Y5ySZk7ZVVkDthuXQ5IespZC/AjBlH8ZKQa9eYmoCRavzP9wHuVF2BguRICQAiNjzeMxk73CfyeOfu0rd5nX4P7hsUjVpOVC2Ea43UP/u2cgcfloOducg9YdehZsfmo1cqWGN5m9+aArJVWwd2xBmt3bhb2w2paA/uIrdlxZ9XMFRyC6O4YELMb9JABBzI68S277i5vHrduHdZrn5+sptgKK87+zEHP5vzlNeUWFWDSWFZS21hitvmbSRs/on1P7RW1fN7vstvLVr8B35wN31O166lIAWiUsCgJgbmYW0dHSZPH6oLlBZHt5d95oyzWk5UPkglrbxbi7Ed7iBQycvRbZzHK/LhtvXoXL7WB6/pbPHfX5gEKtqBd7f3k3Ljc7Inx0dNvn/1KyJrytEgpBJYDE30vOo3bQGcpzduMlpkFWMldSDVV5k7qvcBtfPYPUeo2bNQxw+fZm6jVXRT/4j/XDpDTO/ULJh7O7a6hWRzylYiZVnNpJFGLptVieF6ggJkYDicgWglPqWUuqcUuq0UuoFpZRchy92yamm5n5gyL0vr8LtCgamPtCaT2A3t+Ovf526jSsi2zmGjPTD6edMF7C1+9xKoOF0EFDR2z+CGUe00tBCJJB4pYBeBaq11huB88BfxmkcYi7llMNIH+BU4+wcMid9p+yD73AD3z5yHV9bJd4V3ezObGT7PWV86/k3sVu7qG+4gt3cAWd/BgO3YMPj+E734jsc2T7Sbu2i/uR5yCq+c5OXaK0hhUggcQkAWutXtNahbZxHAfkolghySmHEXAGULy3C98pb2P2ZMHQbu7WLE43NvH32Gtu37cDa9BD2ufc5+uuf88SD5bR09lCeDb6f/AT78lVYuw97MJcTjc2cON8SOWl8uIHy/NTJP+EHg6YoXWjPgRAJaj7MAfwR8NxkDyqlvgx8GWD58ihNPsTCkZ43lnMf2xT289epKR7Eb3fz9O88CJgrgdF1y/B3rsO7/irWyCHoToWbw3hXaHyty6hpTsff2BDxM2Mbvx66B2tJ2uSf8Ef6TDCKljoSIoHELAAopV4DSqM89E2t9c+c53wTCAD/M9nraK2fBZ4F2Lp1q47BUMVcCQUApzibVVFKRlYWPz15lU9tWTo22VtRnMsPXnuPz+29H+u+A9B60mzaSsnCenA9NR90TJggDm38qttYhVWYZEpNT3aCH+6Dkuq5Omoh5q2YBQCt9d47Pa6Uegp4FNijtZYTeyLwOCfm7muQWYTd1Ebz9U4GVTovn7hCVUU51zr6+OFrp3hy7yb8jc1UlRZgVW4be4mITV7O48fPN3OisZl9NWvwn7tGVX4SpHhosc+Ykg/j6aCZHxAiwcUlBaSU2g/8BVCnte6PxxhEnBTfA50Xx4q7Pf3EAQCe+f5z/NV3f0VbzzB/+4U97NpUFbGpyyormPB9VWkBz7z4Drf6hsjPTqeqtICqPMUzb9iQ0c3Tv/fIxPfXQcAjE8BCEL9VQP8O5ACvKqXeU0p9J07jEHMtqxjScmhpaR3bFGZVlLLvt3YR0Iq66gp2baoCTE0fb1212SjGuE1ezuMPrFvGni0refrRGnyvn+TyzUFIy+OBDasnrv0HUwAutyxyR7EQCSou/wq01lF69omE4PFA6X3UDt2GXHOCtpva8J+5yFeffBz/20ewm65jVZgNY1ZZwdgJP2KTl8Nb5+TyRwPUrCri8KVh9u2sYff2TdHff7gfSu6b/eMSYgGSUhBi7jlVOwkGI+v8P7wT72c+je/QKexr06jLHxzFvnwFf5uibscD+BvOT177R2vIXvrRj0GIRUACgJh7qZlmMniwy63z76RrrLX34vV+lpb2burf/WBidy9nQxhg8vkDt7AvX8V3qgfvY4+we/umsd7BE4LA6LApQZFROBdHKcS8JwFAxMfSe2BkaKzpezhr1Spq93+G8tISfK8ew77SDFq7m7wKs82Gsr6bkFNKS9oqvJ9+1A0izh6DCQXg+rtg6b0RrSGFSGQyEybiI2sppOeaNfnRKnKmZGBt3Ys3vxLfwZepae3Af6HNdPcqSDL7Cco2Qk4ptSUTC7qFJpcjBEehYOI8ghCJSgKAiA+PB1bsgMZfQkpm9KqcSmGtuYea2iEOv3OKul37sLZtMvsJplvFc7gfMgvG2lMKISQFJOIpd5mZEO7vnPQpdlMb/obz1D24Cf/ZS9itHTMr4TzYDUujbAoTIoFJABDxoxQs325W5gz3TXg4YoXQnSZ370YHAQ35lbMzbiEWCQkAIr7S80xN/8FbZpVOmAlwxYf3AAAGyklEQVQrhCab3L2bgVtQaEX2EhZCyByAmAdySmDFQ2Afhny34mu0Oj5RJ3fvZmRgrGewEMIlVwBifliy1qzQ6W6C2zcmXA3M2MAt0xUsu2R2Xk+IRUSuAMT84PFA1U5oP286hF0/Y+YGMnLNKqGZCAZM7f/J2kYKkeAkAIj5IzULlm02t0s2QOcl6GyEnlBZCGV28qblmFaSd9PbZhrNy9JPIaKSACDmp9RMKKs2X8P9ZhnnSL8JBp0XzVVBSgYM9ZrSzuOXhvZ1mmWmJbL0U4jJSAAQ819qpruCp2iVOamffwkGuiCzCHpbzMk+ZOAWKMDaKakfIe5AAoBYeDILYcPjgAJPMlw6BLeuOg8qSMuGNY+YVJEQYlISAMTClJLh3l69FwJDZsNXYNBJD6XHb2xCLBASAMTCp5R7wpfNXkJMmewDEEKIBCUBQAghEpQEACGESFASAIQQIkFJABBCiAQlAUAIIRKUBAAhhEhQEgCEECJBKa11vMcwZUqpduBKHN56CdARh/eNJTmm+W+xHQ8svmNaKMezQmtdPP7OBRUA4kUpdVxrvTXe45hNckzz32I7Hlh8x7TQj0dSQEIIkaAkAAghRIKSADA1z8Z7ADEgxzT/LbbjgcV3TAv6eGQOQAghEpRcAQghRIKSACCEEAlKAkAYpdR+pVSjUuqiUuobUR5PU0o95zx+TClVNfejnJ4pHNOfKqXOKqVOK6V+rZRaEY9xTtXdjifseZ9RSmml1LxfojeVY1JKfdb5PZ1RSv3vXI9xOqbwN7dcKXVIKXXS+bs7EI9xTpVS6ntKqRtKqYZJHldKqX9zjve0UmrLXI9xxrTW8mXmQZKAD4GVQCpwCrh33HO+CnzHuf0E8Fy8xz0Lx7QbyHRuf2U+H9NUjsd5Xg5wBDgKbI33uGfhd7QGOAkUON8vjfe4P+LxPAt8xbl9L3A53uO+yzHtArYADZM8fgB4CVDAduBYvMc81S+5AnA9CFzUWl/SWg8DPwIeG/ecx4DvO7d/DOxRSqk5HON03fWYtNaHtNb9zrdHgYo5HuN0TOV3BPD3wD8Bg3M5uBmayjF9Cfi21roLQGt9Y47HOB1TOR4N5Dq384CWORzftGmtjwA37/CUx4D/1sZRIF8pVTY3o/toJAC4lgHXwr5vcu6L+hytdQDoBormZHQzM5VjCvdFzCeZ+equx+NcfldqrX8xlwP7CKbyO1oLrFVK1Suljiql9s/Z6KZvKsfzN8CTSqkm4JfA1+ZmaDEz3X9n84Y0hRcAKKWeBLYCdfEey0wppTzAvwJPxXkosy0ZkwZ6GHOFdkQpdZ/W+lZcRzVzvw/8l9b6X5RSO4AfKKWqtdbBeA8s0cgVgKsZqAz7vsK5L+pzlFLJmMvXzjkZ3cxM5ZhQSu0Fvgl8Ums9NEdjm4m7HU8OUA28oZS6jMnHHpznE8FT+R01AQe11iNaaxs4jwkI89FUjueLwPMAWuu3gXRMUbWFakr/zuYjCQAuP7BGKWUppVIxk7wHxz3nIPAF5/bvAq9rZxZonrrrMSmlNgPPYE7+8zm3DHc5Hq11t9Z6ida6SmtdhZnT+KTW+nh8hjslU/m7+ynm0z9KqSWYlNCluRzkNEzleK4CewCUUusxAaB9Tkc5uw4Cn3dWA20HurXWrfEe1FRICsihtQ4opf4YeBmzkuF7WuszSqm/A45rrQ8C/4m5XL2ImRR6In4jvrspHtO3gGzA58xnX9VafzJug76DKR7PgjLFY3oZ+IRS6iwwCnxdaz0vrzyneDx/BnxXKfUnmAnhp+bzByml1P9hAvASZ97ir4EUAK31dzDzGAeAi0A/8IfxGen0SSkIIYRIUJICEkKIBCUBQAghEpQEACGESFASAIQQIkFJABBCiAQlAUCISSil8pVSX3VulyulfhzvMQkxm2QZqBCTcMp9/1xrXR3noQgRE7IRTIjJ/SOwSin1HnABWK+1rlZKPQV8CsjClGT4Z0zp488BQ8ABrfVNpdQq4NtAMWaD0Je01ufm/jCEiE5SQEJM7hvAh1rr+4Gvj3usGngcqAH+AejXWm8G3gY+7zznWeBrWusHgD8H/mNORi3EFMkVgBAzc0hr3Qv0KqW6gRed+98HNiqlsoGP4ZbYAEib+2EKMTkJAELMTHjV1GDY90HMvysPcMu5ehBiXpIUkBCT68WUmJ42rXUPYCulvDDWN3bTbA5OiI9KAoAQk3AqbtY7zcC/NYOX+APgi0qpU8AZorevFCJuZBmoEEIkKLkCEEKIBCUBQAghEpQEACGESFASAIQQIkFJABBCiAQlAUAIIRKUBAAhhEhQ/w/SCiEQglN1QwAAAABJRU5ErkJggg==" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "metadata": {} + "plot_mean_and_var(ax)" + ] }, { "cell_type": "markdown", + "id": "efb2db26", + "metadata": {}, "source": [ "The deep GP model can handle the heteroscedastic noise in the dataset as well as the sharp-ish transition at $0.3$." - ], - "metadata": {} + ] }, { "cell_type": "markdown", + "id": "9dd1cf83", + "metadata": {}, "source": [ "## Conclusion\n", "\n", @@ -604,37 +459,22 @@ "\n", "\n", "[1] Silverman, B. W. (1985) “Some aspects of the spline smoothing approach to non-parametric curve fitting”. Journal of the Royal Statistical Society series B 47, 1-52." - ], - "metadata": {} + ] }, { "cell_type": "markdown", - "source": [], - "metadata": {} + "id": "c252fac0", + "metadata": {}, + "source": [] } ], "metadata": { - "orig_nbformat": 4, - "language_info": { - "name": "python", - "version": "3.7.4", - "mimetype": "text/x-python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "pygments_lexer": "ipython3", - "nbconvert_exporter": "python", - "file_extension": ".py" - }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.7.4 64-bit ('gpflow2': conda)" - }, - "interpreter": { - "hash": "f5f45e34fe652d34c37da17ab3d28157e258abad05dd1da4f29284863e79e5cf" + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 5 } diff --git a/docs/notebooks/gpflux_features.py b/docs/notebooks/gpflux_features.py index 909ea7fd..7c2b0cac 100644 --- a/docs/notebooks/gpflux_features.py +++ b/docs/notebooks/gpflux_features.py @@ -61,15 +61,22 @@ def motorcycle_data(): """ # %% -import gpflux +from gpflow.kernels import SquaredExponential -from gpflux.architectures import Config, build_constant_input_dim_deep_gp +import gpflux +from gpflux.architectures.config import GaussianLikelihoodConfig, ModelHyperParametersConfig +from gpflux.architectures.factory import build_constant_input_dim_architecture from gpflux.models import DeepGP -config = Config( - num_inducing=25, inner_layer_qsqrt_factor=1e-5, likelihood_noise_variance=1e-2, whiten=True +config = ModelHyperParametersConfig( + num_layers=2, + kernel=SquaredExponential, + likelihood=GaussianLikelihoodConfig(noise_variance=1e-2), + inner_layer_qsqrt_factor=1e-5, + whiten=True, + num_inducing=25, ) -deep_gp: DeepGP = build_constant_input_dim_deep_gp(X, num_layers=2, config=config) +deep_gp: DeepGP = build_constant_input_dim_architecture(config, X) # %% [markdown] """ @@ -164,9 +171,7 @@ def plot(model, X, Y, ax=None): prediction_model.save_weights("weights") # %% -prediction_model_new = build_constant_input_dim_deep_gp( - X, num_layers=2, config=config -).as_prediction_model() +prediction_model_new = build_constant_input_dim_architecture(config, X).as_prediction_model() prediction_model_new.load_weights("weights") # %% diff --git a/docs/notebooks/plotting_functions.py b/docs/notebooks/plotting_functions.py new file mode 100644 index 00000000..f7a03694 --- /dev/null +++ b/docs/notebooks/plotting_functions.py @@ -0,0 +1,195 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module contains experimental code for plotting. As such, the disclaimer is clear. +""" +import numpy as np +import matplotlib.pyplot as plt +import tensorflow as tf +import io + + +def get_classification_detailed_plot( + num_layers, X_training, Y_training, where_to_save, f_mean_overall, f_var_overall, name_file +): + + xx, yy = np.mgrid[-5:5:0.1, -5:5:0.1] + grid = np.c_[xx.ravel(), yy.ravel()] + grid = grid.astype(np.float32) + + indices_class_1 = np.where(Y_training == 1.0) + indices_class_0 = np.where(Y_training == 0.0) + + fig, axs = plt.subplots( + nrows=2, ncols=num_layers, sharex=True, sharey=True, figsize=(20 * num_layers, 40) + ) + + for current_layer in range(num_layers): + + current_mean = f_mean_overall[current_layer] + current_mean = current_mean.reshape((100, 100)) + current_var = f_var_overall[current_layer] + current_var = current_var.reshape((100, 100)) + + ################### + ##### F mean ##### + ################### + + axis = axs[0, current_layer] + contour = axis.contourf(xx, yy, current_mean, 50, cmap="coolwarm") + cbar1 = fig.colorbar(contour, ax=axis) + + cbar1.ax.tick_params(labelsize=60) + + axis.set(xlim=(-5.0, 5.0), ylim=(-5.0, 5.0), xlabel="$X_1$", ylabel="$X_2$") + axis.set_title(label="Predictive Mean", fontdict={"fontsize": 60}) + axis.tick_params(axis="both", which="major", labelsize=80) + + axis.scatter( + X_training[indices_class_0, 0], + X_training[indices_class_0, 1], + s=100, + marker="X", + alpha=0.2, + c="green", + linewidth=1, + label="Class 0", + ) + axis.scatter( + X_training[indices_class_1, 0], + X_training[indices_class_1, 1], + s=100, + marker="D", + alpha=0.2, + c="purple", + linewidth=1, + label="Class 1", + ) + + # NOTE -- don't need this + # axis.scatter(Z_np[current_layer][:,0], Z_np[current_layer][:,1], + # s=750, marker="*", alpha=0.95, c = 'cyan', + # linewidth=1, label = 'Inducing Points') + + axis.legend(loc="upper right", prop={"size": 60}) + # axis.text(-4.5, 4.5, 'LL:'+"{:.2f}".format(total_nll_np)+'; Acc:'+"{:.2f}".format(precision_testing_overall_np), size=50, color='black') + + ################################# + ##### F var Distributional ##### + ################################# + + axis = axs[1, current_layer] + contour = axis.contourf(xx, yy, current_var, 50, cmap="coolwarm") + cbar1 = fig.colorbar(contour, ax=axis) + cbar1.ax.tick_params(labelsize=60) + + axis.set(xlim=(-5, 5), ylim=(-5, 5), xlabel="$X_1$", ylabel="$X_2$") + + axis.set_title(label="Predictive Variance", fontdict={"fontsize": 60}) + axis.tick_params(axis="both", which="major", labelsize=80) + + axis.scatter( + X_training[indices_class_0, 0], + X_training[indices_class_0, 1], + s=100, + marker="X", + alpha=0.2, + c="green", + linewidth=1, + label="Class 0", + ) + axis.scatter( + X_training[indices_class_1, 0], + X_training[indices_class_1, 1], + s=100, + marker="D", + alpha=0.2, + c="purple", + linewidth=1, + label="Class 1", + ) + + # axis.scatter(Z_np[current_layer][:,0], Z_np[current_layer][:,1], + # s=750, marker="*", alpha=0.95, c = 'cyan', + # linewidth=1, label = 'Inducing Points') + axis.legend(loc="upper right", prop={"size": 60}) + + plt.tight_layout() + plt.savefig(where_to_save + name_file) + plt.close() + + +def get_regression_detailed_plot( + num_layers, + X_training, + Y_training, + where_to_save, + mean, + var, + name_file, + x_margin, + y_margin, + X_test, +): + + figure, axs = plt.subplots( + nrows=1, ncols=num_layers, sharex=True, sharey=True, figsize=(10 * num_layers, 10) + ) + + for current_layer in range(num_layers): + current_mean = mean[current_layer] + current_var = var[current_layer] + + ################### + ##### F mean ##### + ################### + + X_test = X_test.squeeze() + lower = current_mean - 2 * np.sqrt(current_var) + upper = current_mean + 2 * np.sqrt(current_var) + + axis = axs[current_layer] + axis.set_ylim(Y_training.min() - y_margin, Y_training.max() + y_margin) + axis.plot(X_training, Y_training, "kx", alpha=0.5, label="Training") + axis.plot(X_test, current_mean, "C1") + + axis.fill_between(X_test, lower, upper, color="C1", alpha=0.3) + axis.legend(loc="upper right", prop={"size": 60}) + + axis.set_title(label=f"Layer {current_layer+1}", fontdict={"fontsize": 60}) + axis.tick_params(axis="both", which="major", labelsize=80) + + plt.tight_layout() + # plt.savefig(where_to_save+name_file) + # plt.close() + return figure + + +def plot_to_image(figure): + """Converts the matplotlib plot specified by 'figure' to a PNG image and + returns it. The supplied figure is closed and inaccessible after this call.""" + # Save the plot to a PNG in memory. + buf = io.BytesIO() + plt.savefig(buf, format="png") + # Closing the figure prevents it from being displayed directly inside + # the notebook. + plt.close(figure) + buf.seek(0) + # Convert PNG buffer to TF image + image = tf.image.decode_png(buf.getvalue(), channels=4) + # Add the batch dimension + image = tf.expand_dims(image, 0) + return image diff --git a/docs/notebooks/sparse_orthogonal_deep_gps.py b/docs/notebooks/sparse_orthogonal_deep_gps.py new file mode 100644 index 00000000..8032f371 --- /dev/null +++ b/docs/notebooks/sparse_orthogonal_deep_gps.py @@ -0,0 +1,240 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.14.1 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# # Sparse Orthogonal Variational Inference for Deep Gaussian Processes +# +# In this notebook, we explore the use of a new interpretation of sparse variational approximations for Gaussian processes using inducing points, which can lead to more scalable algorithms than previous methods. It is based on decomposing a Gaussian process as a sum of two independent processes: one spanned by a finite basis of inducing points and the other capturing the remaining variation . +# +# Sparse orthogonal VI is based on decomposing the GP prior as the sum of a low-rank approximation using inducing points, and a full-rank residual process. It's been observed how the standard SVGP methods can be reinterpreted under such decomposition. By introducing another set of inducing variables for the orthogonal complement, we can increase the number of inducing points at a much lower additional computational cost. + +import gpflow +import gpflux +import numpy as np +import matplotlib.pyplot as plt +import tensorflow as tf + +# ## Load data +# +# The data comes from a motorcycle accident simulation [1] and shows some interesting behaviour. In particular the heteroscedastic nature of the noise. + + +def motorcycle_data(): + """Return inputs and outputs for the motorcycle dataset. We normalise the outputs.""" + import pandas as pd + + df = pd.read_csv("./data/motor.csv", index_col=0) + X, Y = df["times"].values.reshape(-1, 1), df["accel"].values.reshape(-1, 1) + Y = (Y - Y.mean()) / Y.std() + X /= X.max() + return X, Y + + +# + +X, Y = motorcycle_data() +num_data, d_xim = X.shape + +X_MARGIN, Y_MARGIN = 0.1, 0.5 +fig, ax = plt.subplots() +ax.scatter(X, Y, marker="x", color="k") +ax.set_ylim(Y.min() - Y_MARGIN, Y.max() + Y_MARGIN) +ax.set_xlim(X.min() - X_MARGIN, X.max() + X_MARGIN) +# - + +# ## Orthogonal Deep Gaussian process +# +# GPflux provides provides a class `OrthGPLayer`, which implements a Sparse Orthogonal Variational multioutput Gaussian Process as a `tf.keras.layers.Layer`. In the following, we build a 2-layer orthogonal deep GP model using this new layer type, with a Gaussian likelihood in the output layer. A standard squared exponential kernel is used throughout the layers. + +# + +from typing import Type +from gpflow.kernels import SquaredExponential, Stationary +from gpflow.mean_functions import Zero +from gpflow.likelihoods import Gaussian +from scipy.cluster.vq import kmeans2 + +from gpflux.helpers import ( + construct_basic_inducing_variables, + construct_basic_kernel, + construct_mean_function, +) +from gpflux.layers import OrthGPLayer +from gpflux.layers.likelihood_layer import LikelihoodLayer +from gpflux.models import OrthDeepGP + + +def build_kernel(input_dim: int, is_last_layer: bool, kernel: Type[Stationary]) -> Stationary: + """ + Return a :class:`gpflow.kernels.Stationary` kernel with ARD lengthscales set to + 1.0 and a small kernel variance of 1e-6 if the kernel is part of a hidden layer; + otherwise, the kernel variance is set to 1.0. + + :param input_dim: The input dimensionality of the layer. + :param is_last_layer: Whether the kernel is part of the last layer in the Deep GP. + :param kernel: the :class:`~gpflow.kernels.Stationary` type of the kernel + """ + assert input_dim > 0, "Cannot have non positive input dimension" + + variance = 1e-6 if not is_last_layer else 1.0 + lengthscales = [1.0] * input_dim + + return kernel(lengthscales=lengthscales, variance=variance) + + +def build_orthogonal_deep_gp( + num_layers: int, num_inducing_u: int, num_inducing_v: int, X: np.ndarray +) -> OrthDeepGP: + """ + :param num_layers: the number of (hidden) layers + :param num_inducing_u: The number of inducing points to use for the low-rank approximation + :param num_inducing_v: The number of inducing points to use for the full-rank residual process + :param X: the data + """ + num_data, input_dim = X.shape + X_running = X + + gp_layers = [] + centroids, _ = kmeans2(X, k=min(num_inducing_u + num_inducing_v, X.shape[0]), minit="points") + + centroids_u = centroids[:num_inducing_u, ...] + centroids_v = centroids[num_inducing_u:, ...] + + for i_layer in range(num_layers): + is_last_layer = i_layer == num_layers - 1 + D_in = input_dim + D_out = 1 if is_last_layer else input_dim + + inducing_var_u = construct_basic_inducing_variables( + num_inducing=num_inducing_u, + input_dim=D_in, + share_variables=True, + z_init=centroids_u, + ) + + inducing_var_v = construct_basic_inducing_variables( + num_inducing=num_inducing_v, + input_dim=D_in, + share_variables=True, + z_init=centroids_v, + ) + + kernel = construct_basic_kernel( + kernels=build_kernel(D_in, is_last_layer, SquaredExponential), + output_dim=D_out, + share_hyperparams=True, + ) + + if is_last_layer: + mean_function = Zero() + q_sqrt_scaling = 1.0 + else: + mean_function = construct_mean_function(X_running, D_out) + X_running = mean_function(X_running) + if tf.is_tensor(X_running): + X_running = cast(tf.Tensor, X_running).numpy() + q_sqrt_scaling = 1e-5 + + # NOTE: here we're using the specialised GPLayer + layer = OrthGPLayer( + kernel, + inducing_var_u, + inducing_var_v, + num_data, + mean_function=mean_function, + name=f"orth_gp_{i_layer}", + num_latent_gps=D_out, + ) + layer.q_sqrt_u.assign(layer.q_sqrt_u * q_sqrt_scaling) + layer.q_sqrt_v.assign(layer.q_sqrt_v * q_sqrt_scaling) + gp_layers.append(layer) + + # NOTE: here we return an instance of a DeeGP type specialised for sparse orthogonal VI + return OrthDeepGP(gp_layers, LikelihoodLayer(likelihood=Gaussian(variance=1e-2))) + + +# - + +# ### Create the model +# +# We now instantiate one model using the above utility function. Note how we can use substantial more inducing points compared to model defined in other notebooks, for both the low-rank approximation and the full-rank residual process. + +# + + +orthogonal_dgp = build_orthogonal_deep_gp(num_layers=1, num_inducing_u=50, num_inducing_v=50, X=X) +gpflow.utilities.print_summary(orthogonal_dgp, fmt="notebook") +# - + +# ### Model training + +# + +# Fit the model on the training data + +BATCH_SIZE = 32 +NUM_EPOCHS = 1000 + +model = orthogonal_dgp.as_training_model() +model.compile(tf.optimizers.Adam(5e-2)) + + +callbacks = [ + # Create callback that reduces the learning rate every time the ELBO plateaus + tf.keras.callbacks.ReduceLROnPlateau("loss", factor=0.95, patience=10, min_lr=1e-6, verbose=0) +] +history = model.fit( + {"inputs": X, "targets": Y}, + batch_size=BATCH_SIZE, + epochs=NUM_EPOCHS, + callbacks=callbacks, + verbose=0, +) +gpflow.utilities.print_summary(orthogonal_dgp, fmt="notebook") +# - + +fig, ax = plt.subplots() +ax.plot(history.history["loss"]) +ax.set_xlabel("Epoch") +ax.set_ylabel("Loss") + +# + +fig, ax = plt.subplots() +num_data_test = 200 +X_test = np.linspace(X.min() - X_MARGIN, X.max() + X_MARGIN, num_data_test).reshape(-1, 1) +model = orthogonal_dgp.as_prediction_model() +out = model(X_test) + +mu = out.y_mean.numpy().squeeze() +var = out.y_var.numpy().squeeze() +X_test = X_test.squeeze() + +for i in [1, 2]: + lower = mu - i * np.sqrt(var) + upper = mu + i * np.sqrt(var) + ax.fill_between(X_test, lower, upper, color="C1", alpha=0.3) + +ax.set_ylim(Y.min() - Y_MARGIN, Y.max() + Y_MARGIN) +ax.set_xlim(X.min() - X_MARGIN, X.max() + X_MARGIN) +ax.plot(X, Y, "kx", alpha=0.5) +ax.plot(X_test, mu, "C1") +ax.set_xlabel("time") +ax.set_ylabel("acc") +# - + +# ## Conclusion +# +# In this notebook we have shown how to create a variant of the deep gp model using the recently introduced sparse orthogonal variational inference of Gaussian processes in GPflux. +# +# +# ## References +# +# [1] Shi, J. et al. (2020) “Sparse Orthogonal Variational Inference for Gaussian Processes”. Proceedings of the 23rdInternational Conference on Artificial Intelligence and Statistics (AISTATS), 109. + +# diff --git a/docs/refs.bib b/docs/refs.bib index 824f8d65..e9c611ba 100644 --- a/docs/refs.bib +++ b/docs/refs.bib @@ -83,3 +83,14 @@ @article{yu2016orthogonal pages={1975--1983}, year={2016} } + +@inproceedings{shi2020sparseorthogonal, + title = {Sparse Orthogonal Variational Inference for Gaussian Processes}, + author = {Shi, Jiaxin and Titsias, Michalis K. and Mnih, Andriy}, + booktitle = {Proceedings of the 23rdInternational Conference on Artificial Intelligence and Statistics (AISTATS)}, + year = {2020}, + volume = {109}, + series = {Proceedings of Machine Learning Research}, + publisher = {PMLR}, + url = {http://proceedings.mlr.press/v108/shi20b/shi20b.pdf} +} diff --git a/docs/tutorials.rst b/docs/tutorials.rst index fd348eff..a0d51063 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -8,6 +8,7 @@ Tutorials notebooks/intro notebooks/gpflux_features notebooks/deep_cde + notebooks/sparse_orthogonal_deep_gps .. toctree:: :caption: Advanced diff --git a/gpflux/architectures/__init__.py b/gpflux/architectures/__init__.py index d2ab53fe..e69de29b 100644 --- a/gpflux/architectures/__init__.py +++ b/gpflux/architectures/__init__.py @@ -1,4 +0,0 @@ -""" -Pre-specified architectures -""" -from gpflux.architectures.constant_input_dim_deep_gp import Config, build_constant_input_dim_deep_gp diff --git a/gpflux/architectures/config.py b/gpflux/architectures/config.py new file mode 100644 index 00000000..d9b9ee89 --- /dev/null +++ b/gpflux/architectures/config.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""This module declares configurations for building various types of architectures""" +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Type + +import tensorflow_probability as tfp + +import gpflow.likelihoods +from gpflow.kernels import Stationary + + +class LikelihoodConfig(ABC): + """Config for the model likelihood""" + + @abstractmethod + def create(self) -> gpflow.likelihoods.Likelihood: + """Create a likelihood instance with the parameters of the config""" + + +@dataclass +class GaussianLikelihoodConfig(LikelihoodConfig): + """Config for a :class:`~gpflow.likelihoods.Gaussian` likelihood""" + + noise_variance: float + """The variance of the likelihood""" + + def create(self) -> gpflow.likelihoods.Gaussian: + return gpflow.likelihoods.Gaussian(variance=self.noise_variance) + + +@dataclass +class StudenttLikelihoodConfig(LikelihoodConfig): + """Config for a :class:`~gpflow.likelihoods.Studentt` likelihood""" + + df: float + """The number of degrees of freedom""" + + scale: float + """The scale parameter""" + + def create(self) -> gpflow.likelihoods.StudentT: + return gpflow.likelihoods.StudentT(df=self.df, scale=self.scale) + + +@dataclass +class HeteroSkedasticLikelihoodConfig(LikelihoodConfig): + """Configuration for a :class:`~gpflow.likelihoods.HeteroskedasticTFPConditional`""" + + distribution_class: Type[tfp.distributions.Distribution] = tfp.distributions.Normal + """The distribution class""" + + def create(self) -> gpflow.likelihoods.HeteroskedasticTFPConditional: + return gpflow.likelihoods.HeteroskedasticTFPConditional( + distribution_class=self.distribution_class + ) + + +@dataclass +class HyperParametersConfig(ABC): + """Configuration of the hyperparameters of a model""" + + num_layers: int + """The number of GP layers in the model, excluded the likelihood one""" + + kernel: Type[Stationary] + """The (stationary) kernel to use in the layers""" + + likelihood: LikelihoodConfig + """Configuration for the model likelihood""" + + inner_layer_qsqrt_factor: float + """ + A multiplicative factor used to rescale the hidden layers' + :attr:`~gpflux.layers.GPLayer.q_sqrt`. Typically this value is chosen to be small + (e.g., 1e-5) to reduce noise at the start of training. + """ + + whiten: bool + """ + Determines the parameterisation of the inducing variables. + If `True`, :math:``p(u) = N(0, I)``, otherwise :math:``p(u) = N(0, K_{uu})``. + .. seealso:: :attr:`gpflux.layers.GPLayer.whiten` + """ + + def __post_init__(self) -> None: + assert self.num_layers > 0, "Cannot have non-positive number of layers" + assert self.whiten, "Non-whitened case not yet supported" + + +@dataclass +class ModelHyperParametersConfig(HyperParametersConfig): + """The configuration used to build a DeepGP model""" + + num_inducing: int + """ + The number of inducing variables, *M*. The Deep GP uses the same number + of inducing variables in each layer. + """ + + +@dataclass +class OrthogonalModelHyperparametersConfig(HyperParametersConfig): + """The configuration used to build an OrthDeepGP model""" + + num_inducing_u: int + """ + The number of inducing variables for the `main` process. The Deep GP uses the same number + of inducing variables in each layer. + """ + + num_inducing_v: int + """ + The number of inducing variables for the process capturing the remaining variation. + The Deep GP uses the same number of inducing variables in each layer. + """ diff --git a/gpflux/architectures/constant_input_dim_deep_gp.py b/gpflux/architectures/constant_input_dim_deep_gp.py deleted file mode 100644 index 50c07418..00000000 --- a/gpflux/architectures/constant_input_dim_deep_gp.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# Copyright (c) 2021 The GPflux Contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -This module provides :func:`build_constant_input_dim_deep_gp` to build a Deep GP of -arbitrary depth where each hidden layer has the same input dimensionality as the data. -""" - -from dataclasses import dataclass -from typing import cast - -import numpy as np -import tensorflow as tf -from scipy.cluster.vq import kmeans2 - -import gpflow -from gpflow.kernels import SquaredExponential -from gpflow.likelihoods import Gaussian - -from gpflux.helpers import ( - construct_basic_inducing_variables, - construct_basic_kernel, - construct_mean_function, -) -from gpflux.layers.gp_layer import GPLayer -from gpflux.layers.likelihood_layer import LikelihoodLayer -from gpflux.models import DeepGP - - -@dataclass -class Config: - """ - The configuration used by :func:`build_constant_input_dim_deep_gp`. - """ - - num_inducing: int - """ - The number of inducing variables, *M*. The Deep GP uses the same number - of inducing variables in each layer. - """ - - inner_layer_qsqrt_factor: float - """ - A multiplicative factor used to rescale the hidden layers' - :attr:`~gpflux.layers.GPLayer.q_sqrt`. Typically this value is chosen to be small - (e.g., 1e-5) to reduce noise at the start of training. - """ - - likelihood_noise_variance: float - """ - The variance of the :class:`~gpflow.likelihoods.Gaussian` likelihood that is used - by the Deep GP. - """ - - whiten: bool = True - """ - Determines the parameterisation of the inducing variables. - If `True`, :math:``p(u) = N(0, I)``, otherwise :math:``p(u) = N(0, K_{uu})``. - .. seealso:: :attr:`gpflux.layers.GPLayer.whiten` - """ - - -def _construct_kernel(input_dim: int, is_last_layer: bool) -> SquaredExponential: - """ - Return a :class:`gpflow.kernels.SquaredExponential` kernel with ARD lengthscales set to - 2 and a small kernel variance of 1e-6 if the kernel is part of a hidden layer; - otherwise, the kernel variance is set to 1.0. - - :param input_dim: The input dimensionality of the layer. - :param is_last_layer: Whether the kernel is part of the last layer in the Deep GP. - """ - variance = 1e-6 if not is_last_layer else 1.0 - - # TODO: Looking at this initializing to 2 (assuming N(0, 1) or U[0,1] normalized - # data) seems a bit weird - that's really long lengthscales? And I remember seeing - # something where the value scaled with the number of dimensions before - lengthscales = [2.0] * input_dim - return SquaredExponential(lengthscales=lengthscales, variance=variance) - - -def build_constant_input_dim_deep_gp(X: np.ndarray, num_layers: int, config: Config) -> DeepGP: - r""" - Build a Deep GP consisting of ``num_layers`` :class:`GPLayer`\ s. - All the hidden layers have the same input dimension as the data, that is, ``X.shape[1]``. - - The architecture is largely based on :cite:t:`salimbeni2017doubly`, with - the most notable difference being that we keep the hidden dimension equal - to the input dimensionality of the data. - - .. note:: - This architecture might be slow for high-dimensional data. - - .. note:: - This architecture assumes a :class:`~gpflow.likelihoods.Gaussian` likelihood - for regression tasks. Specify a different likelihood for performing - other tasks such as classification. - - :param X: The training input data, used to retrieve the number of datapoints and - the input dimension and to initialise the inducing point locations using k-means. A - tensor of rank two with the dimensions ``[num_data, input_dim]``. - :param num_layers: The number of layers in the Deep GP. - :param config: The configuration for (hyper)parameters. See :class:`Config` for details. - """ - if X.dtype != gpflow.default_float(): - raise ValueError( - f"X needs to have dtype according to gpflow.default_float() = {gpflow.default_float()} " - f"however got X with {X.dtype} dtype." - ) - - num_data, input_dim = X.shape - X_running = X - - gp_layers = [] - centroids, _ = kmeans2(X, k=config.num_inducing, minit="points") - - for i_layer in range(num_layers): - is_last_layer = i_layer == num_layers - 1 - D_in = input_dim - D_out = 1 if is_last_layer else input_dim - - # Pass in kernels, specify output dim (shared hyperparams/variables) - - inducing_var = construct_basic_inducing_variables( - num_inducing=config.num_inducing, input_dim=D_in, share_variables=True, z_init=centroids - ) - - kernel = construct_basic_kernel( - kernels=_construct_kernel(D_in, is_last_layer), - output_dim=D_out, - share_hyperparams=True, - ) - - assert config.whiten is True, "non-whitened case not implemented yet" - - if is_last_layer: - mean_function = gpflow.mean_functions.Zero() - q_sqrt_scaling = 1.0 - else: - mean_function = construct_mean_function(X_running, D_in, D_out) - X_running = mean_function(X_running) - if tf.is_tensor(X_running): - X_running = cast(tf.Tensor, X_running).numpy() - q_sqrt_scaling = config.inner_layer_qsqrt_factor - - layer = GPLayer( - kernel, - inducing_var, - num_data, - mean_function=mean_function, - name=f"gp_{i_layer}", - ) - layer.q_sqrt.assign(layer.q_sqrt * q_sqrt_scaling) - gp_layers.append(layer) - - likelihood = Gaussian(config.likelihood_noise_variance) - return DeepGP(gp_layers, LikelihoodLayer(likelihood)) diff --git a/gpflux/architectures/factory.py b/gpflux/architectures/factory.py new file mode 100644 index 00000000..f492ba27 --- /dev/null +++ b/gpflux/architectures/factory.py @@ -0,0 +1,230 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""This module defines factories for various Deep GP architectures""" +from functools import singledispatch +from typing import Type, cast + +import numpy as np +import tensorflow as tf +from scipy.cluster.vq import kmeans2 + +from gpflow import default_float +from gpflow.kernels import Stationary +from gpflow.mean_functions import Zero + +from gpflux.architectures.config import ( + HyperParametersConfig, + ModelHyperParametersConfig, + OrthogonalModelHyperparametersConfig, +) +from gpflux.helpers import ( + construct_basic_inducing_variables, + construct_basic_kernel, + construct_mean_function, +) +from gpflux.layers import OrthGPLayer +from gpflux.layers.gp_layer import GPLayer +from gpflux.layers.likelihood_layer import LikelihoodLayer +from gpflux.models import DeepGP, OrthDeepGP + + +def build_kernel(input_dim: int, is_last_layer: bool, kernel: Type[Stationary]) -> Stationary: + """ + Return a :class:`gpflow.kernels.Stationary` kernel with ARD lengthscales set to + 1.0 and a small kernel variance of 1e-6 if the kernel is part of a hidden layer; + otherwise, the kernel variance is set to 1.0. + + :param input_dim: The input dimensionality of the layer. + :param is_last_layer: Whether the kernel is part of the last layer in the Deep GP. + :param kernel: the :class:`~gpflow.kernels.Stationary` type of the kernel + """ + assert input_dim > 0, "Cannot have non positive input dimension" + + variance = 1e-6 if not is_last_layer else 1.0 + lengthscales = [1.0] * input_dim + + return kernel(lengthscales=lengthscales, variance=variance) + + +@singledispatch +def build_constant_input_dim_architecture( + model_config: HyperParametersConfig, X: np.ndarray +) -> DeepGP: + r""" + Build a Deep GP consisting of a number of :class:`GPLayer`\ s. + All the hidden layers have the same input dimension as the data, that is, ``X.shape[1]``. + + The architecture is largely based on :cite:t:`salimbeni2017doubly`, with + the most notable difference being that we keep the hidden dimension equal + to the input dimensionality of the data. + + .. note:: + This architecture might be slow for high-dimensional data. + + :param model_config: The configuration for (hyper)parameters. + :param X: The training input data, used to retrieve the number of datapoints and + the input dimension and to initialise the inducing point locations using k-means. A + tensor of rank two with the dimensions ``[num_data, input_dim]``. + + :return: an instance of a DeepGP model + :raises ValueError: If the config type is not registered. + """ + raise ValueError( + f"Don't know how to create model from config of type: {type(HyperParametersConfig)}" + ) + + +@build_constant_input_dim_architecture.register +def build_constant_input_dim_deep_gp( + model_config: ModelHyperParametersConfig, X: np.ndarray +) -> DeepGP: + if X.dtype != default_float(): + raise ValueError( + f"X needs to have dtype according to gpflow.default_float() = {default_float()} " + f"however got X with {X.dtype} dtype." + ) + + num_data, input_dim = X.shape + X_running = X + + gp_layers = [] + centroids, _ = kmeans2(X, k=model_config.num_inducing, minit="points") + + num_layers = model_config.num_layers + for i_layer in range(num_layers): + is_last_layer = i_layer == num_layers - 1 + D_in = input_dim + D_out = 1 if is_last_layer else input_dim + + # Pass in kernels, specify output dim (shared hyperparams/variables) + + inducing_var = construct_basic_inducing_variables( + num_inducing=model_config.num_inducing, + input_dim=D_in, + share_variables=True, + z_init=centroids, + ) + + kernel = construct_basic_kernel( + kernels=build_kernel(D_in, is_last_layer, model_config.kernel), + output_dim=D_out, + share_hyperparams=True, + ) + + assert model_config.whiten is True, "non-whitened case not implemented yet" + + if is_last_layer: + mean_function = Zero() + q_sqrt_scaling = 1.0 + else: + mean_function = construct_mean_function(X_running, D_out) + X_running = mean_function(X_running) + if tf.is_tensor(X_running): + X_running = cast(tf.Tensor, X_running).numpy() + q_sqrt_scaling = model_config.inner_layer_qsqrt_factor + + layer = GPLayer( + kernel, + inducing_var, + num_data, + mean_function=mean_function, + name=f"gp_{i_layer}", + ) + layer.q_sqrt.assign(layer.q_sqrt * q_sqrt_scaling) + gp_layers.append(layer) + + likelihood = model_config.likelihood.create() + + return DeepGP(gp_layers, LikelihoodLayer(likelihood)) + + +@build_constant_input_dim_architecture.register +def build_constant_input_dim_orthogonal_deep_gp( + model_config: OrthogonalModelHyperparametersConfig, X: np.ndarray +) -> OrthDeepGP: + if X.dtype != default_float(): + raise ValueError( + f"X needs to have dtype according to gpflow.default_float() = {default_float()} " + f"however got X with {X.dtype} dtype." + ) + + num_data, input_dim = X.shape + X_running = X + + num_inducing_u = model_config.num_inducing_u + num_inducing_v = model_config.num_inducing_v + gp_layers = [] + centroids, _ = kmeans2(X, k=min(num_inducing_u + num_inducing_v, X.shape[0]), minit="points") + + centroids_u = centroids[:num_inducing_u, ...] + centroids_v = centroids[num_inducing_u:, ...] + + num_layers = model_config.num_layers + for i_layer in range(num_layers): + is_last_layer = i_layer == num_layers - 1 + D_in = input_dim + D_out = 1 if is_last_layer else input_dim + + # Pass in kernels, specify output dim (shared hyperparams/variables) + + inducing_var_u = construct_basic_inducing_variables( + num_inducing=num_inducing_u, + input_dim=D_in, + share_variables=True, + z_init=centroids_u, + ) + + inducing_var_v = construct_basic_inducing_variables( + num_inducing=num_inducing_v, + input_dim=D_in, + share_variables=True, + z_init=centroids_v, + ) + + kernel = construct_basic_kernel( + kernels=build_kernel(D_in, is_last_layer, model_config.kernel), + output_dim=D_out, + share_hyperparams=True, + ) + + assert model_config.whiten is True, "non-whitened case not implemented yet" + + if is_last_layer: + mean_function = Zero() + q_sqrt_scaling = 1.0 + else: + mean_function = construct_mean_function(X_running, D_out) + X_running = mean_function(X_running) + if tf.is_tensor(X_running): + X_running = cast(tf.Tensor, X_running).numpy() + q_sqrt_scaling = model_config.inner_layer_qsqrt_factor + + layer = OrthGPLayer( + kernel, + inducing_var_u, + inducing_var_v, + num_data, + mean_function=mean_function, + name=f"orth_gp_{i_layer}", + num_latent_gps=D_out, + ) + layer.q_sqrt_u.assign(layer.q_sqrt_u * q_sqrt_scaling) + layer.q_sqrt_v.assign(layer.q_sqrt_v * q_sqrt_scaling) + gp_layers.append(layer) + + likelihood = model_config.likelihood.create() + + return OrthDeepGP(gp_layers, LikelihoodLayer(likelihood)) diff --git a/gpflux/conditionals/__init__.py b/gpflux/conditionals/__init__.py new file mode 100644 index 00000000..39743565 --- /dev/null +++ b/gpflux/conditionals/__init__.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from . import conditionals, multioutput +from .dispatch import conditional + +# from .util import base_conditional + + +__all__ = [ + "conditional", + "conditionals", + "dispatch", + "multioutput", +] diff --git a/gpflux/conditionals/conditionals.py b/gpflux/conditionals/conditionals.py new file mode 100644 index 00000000..ec89fef1 --- /dev/null +++ b/gpflux/conditionals/conditionals.py @@ -0,0 +1,89 @@ +# Copyright 2017-2020 The GPflow Contributors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import tensorflow as tf + +from gpflow.base import MeanAndVariance +from gpflow.inducing_variables import InducingVariables +from gpflow.kernels import Kernel + +from gpflux.conditionals.dispatch import conditional +from gpflux.posteriors import get_posterior_class + + +@conditional._gpflow_internal_register( + object, InducingVariables, InducingVariables, Kernel, object, object +) +def _sparse_orthogonal_conditional( + Xnew: tf.Tensor, + inducing_variable_u: InducingVariables, + inducing_variable_v: InducingVariables, + kernel: Kernel, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + full_output_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False +) -> MeanAndVariance: + """ + Single-output distributional orthogonal GP conditional. + + The covariance matrices used to calculate the conditional have the following shape: + - Kuu: [M, M] + - Kuf: [M, N] + - Kff: [N, N] + + Further reference + ----------------- + - See `gpflow.conditionals._dense_conditional` (below) for a detailed explanation of + conditional in the single-output case. + - See the multiouput notebook for more information about the multiouput framework. + + Parameters + ---------- + :param Xnew: data matrix, size [N, D]. + :param f: data matrix, [M, R] + :param full_cov: return the covariance between the datapoints + :param full_output_cov: return the covariance between the outputs. + NOTE: as we are using a single-output kernel with repetitions + these covariances will be zero. + :param q_sqrt: matrix of standard-deviations or Cholesky matrices, + size [M, R] or [R, M, M]. + :param white: boolean of whether to use the whitened representation + :return: + - mean: [N, R] + - variance: [N, R], [R, N, N], [N, R, R] or [N, R, N, R] + Please see `gpflow.conditional._expand_independent_outputs` for more information + about the shape of the variance, depending on `full_cov` and `full_output_cov`. + """ + posterior_class = get_posterior_class(kernel, inducing_variable_u, inducing_variable_v) + + posterior = posterior_class( + kernel, + inducing_variable_u, + inducing_variable_v, + f_u, + f_v, + q_sqrt_u, + q_sqrt_v, + whiten=white, + mean_function=None, + ) + + return posterior.fused_predict_f(Xnew, full_cov=full_cov, full_output_cov=full_output_cov) diff --git a/gpflux/conditionals/dispatch.py b/gpflux/conditionals/dispatch.py new file mode 100644 index 00000000..d7725e30 --- /dev/null +++ b/gpflux/conditionals/dispatch.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from gpflow.conditionals.dispatch import conditional # noqa: F401 diff --git a/gpflux/conditionals/multioutput/__init__.py b/gpflux/conditionals/multioutput/__init__.py new file mode 100644 index 00000000..a6dcada5 --- /dev/null +++ b/gpflux/conditionals/multioutput/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from . import conditionals + +__all__ = ["conditionals"] diff --git a/gpflux/conditionals/multioutput/conditionals.py b/gpflux/conditionals/multioutput/conditionals.py new file mode 100644 index 00000000..dab8f437 --- /dev/null +++ b/gpflux/conditionals/multioutput/conditionals.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional + +import tensorflow as tf + +from gpflow.base import MeanAndVariance +from gpflow.inducing_variables import SharedIndependentInducingVariables +from gpflow.kernels import SharedIndependent + +from gpflux.conditionals.dispatch import conditional +from gpflux.posteriors import get_posterior_class + + +@conditional._gpflow_internal_register( + object, + SharedIndependentInducingVariables, + SharedIndependentInducingVariables, + SharedIndependent, + object, + object, +) +def shared_independent_orthogonal_conditional( + Xnew: tf.Tensor, + inducing_variable_u: SharedIndependentInducingVariables, + inducing_variable_v: SharedIndependentInducingVariables, + kernel: SharedIndependent, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + full_output_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False +) -> MeanAndVariance: + """Multioutput conditional for an independent kernel and shared inducing inducing. + Same behaviour as conditional with non-multioutput kernels. + The covariance matrices used to calculate the conditional have the following shape: + - Kuu: [M, M] + - Kuf: [M, N] + - Kff: N or [N, N] + + Further reference + ----------------- + - See `gpflow.conditionals._conditional` for a detailed explanation of + conditional in the single-output case. + - See the multioutput notebook for more information about the multioutput framework. + Parameters + ---------- + :param Xnew: data matrix, size [N, D]. + :param f: data matrix, [M, P] + :param full_cov: return the covariance between the datapoints + :param full_output_cov: return the covariance between the outputs. + Note: as we are using a independent kernel these covariances will be zero. + :param q_sqrt: matrix of standard-deviations or Cholesky matrices, + size [M, P] or [P, M, M]. + :param white: boolean of whether to use the whitened representation + :return: + - mean: [N, P] + - variance: [N, P], [P, N, N], [N, P, P] or [N, P, N, P] + Please see `gpflow.conditional._expand_independent_outputs` for more information + about the shape of the variance, depending on `full_cov` and `full_output_cov`. + """ + + posterior_class = get_posterior_class(kernel, inducing_variable_u, inducing_variable_v) + + posterior = posterior_class( + kernel, + inducing_variable_u, + inducing_variable_v, + f_u, + f_v, + q_sqrt_u, + q_sqrt_v, + whiten=white, + mean_function=None, + ) + return posterior.fused_predict_f(Xnew, full_cov=full_cov, full_output_cov=full_output_cov) diff --git a/gpflux/conditionals/util.py b/gpflux/conditionals/util.py new file mode 100644 index 00000000..eb881c7b --- /dev/null +++ b/gpflux/conditionals/util.py @@ -0,0 +1,564 @@ +from typing import Optional, Tuple + +import tensorflow as tf + +from gpflow.base import MeanAndVariance +from gpflow.conditionals.util import rollaxis_left +from gpflow.config import default_float + + +def base_orthogonal_conditional( + Kmn: tf.Tensor, + Kmm: tf.Tensor, + Knn: tf.Tensor, + Cmn: tf.Tensor, + Cmm: tf.Tensor, + Cnn: tf.Tensor, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False, + Lm: Optional[tf.Tensor] = None, +) -> MeanAndVariance: + r""" + + #TODO -- this needs to be updated to suit sparse orthogonal GPs + Given a g1 and g2, and distribution p and q such that + p(g2) = N(g2; 0, Kmm) + + p(g1) = N(g1; 0, Knn) + p(g1 | g2) = N(g1; Knm (Kmm⁻¹) g2, Knn - Knm (Kmm⁻¹) Kmn) + + And + q(g2) = N(g2; f, q_sqrt q_sqrtᵀ) + + This method computes the mean and (co)variance of + q(g1) = ∫ q(g2) p(g1 | g2) + + :param Kmn: [M, ..., N] + :param Kmm: [M, M] + :param Knn: [..., N, N] or N + :param f: [M, R] + :param full_cov: bool + :param q_sqrt: If this is a Tensor, it must have shape [R, M, M] (lower + triangular) or [M, R] (diagonal) + :param white: bool + :return: [N, R] or [R, N, N] + """ + + # NOTE -- this is now passed from _get_Cnn method of posterior class + # hence the Kmm argumnent is a bit redundant now + # Lm = tf.linalg.cholesky(Kmm) + return base_orthogonal_conditional_with_lm( + Kmn=Kmn, + Lm=Lm, + Knn=Knn, + L_Cmm=tf.linalg.cholesky(Cmm), + Cmn=Cmn, + Cnn=Cnn, + f_u=f_u, + f_v=f_v, + full_cov=full_cov, + q_sqrt_u=q_sqrt_u, + q_sqrt_v=q_sqrt_v, + white=white, + ) + + +def base_orthogonal_conditional_with_lm( + Kmn: tf.Tensor, + Lm: tf.Tensor, + Knn: tf.Tensor, + L_Cmm: tf.Tensor, + Cmn: tf.Tensor, + Cnn: tf.Tensor, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False, +) -> MeanAndVariance: + r""" + Has the same functionality as the `base_conditional` function, except that instead of + `Kmm` this function accepts `Lm`, which is the Cholesky decomposition of `Kmm`. + + This allows `Lm` to be precomputed, which can improve performance. + """ + # compute kernel stuff + num_func = tf.shape(f_u)[-1] # R + N = tf.shape(Kmn)[-1] + M = tf.shape(f_u)[-2] + V = tf.shape(f_v)[-2] + + # get the leading dims in Kmn to the front of the tensor + # if Kmn has rank two, i.e. [M, N], this is the identity op. + K = tf.rank(Kmn) + perm = tf.concat( + [ + tf.reshape(tf.range(1, K - 1), [K - 2]), # leading dims (...) + tf.reshape(0, [1]), # [M] + tf.reshape(K - 1, [1]), + ], + 0, + ) # [N] + Kmn = tf.transpose(Kmn, perm) # [..., M, N] + Cmn = tf.transpose(Cmn, perm) # [..., M, N] + + shape_constraints = [ + (Kmn, [..., "M", "N"]), + (Lm, ["M", "M"]), + (Knn, [..., "N", "N"] if full_cov else [..., "N"]), + (f_u, ["M", "R"]), + (Cmn, [..., "V", "N"]), + (L_Cmm, ["V", "V"]), + (Cnn, [..., "N", "N"] if full_cov else [..., "N"]), + (f_v, ["V", "R"]), + ] + if q_sqrt_u is not None: + shape_constraints.append( + (q_sqrt_u, (["M", "R"] if q_sqrt_u.shape.ndims == 2 else ["R", "M", "M"])) + ) + + if q_sqrt_v is not None: + shape_constraints.append( + (q_sqrt_v, (["V", "R"] if q_sqrt_v.shape.ndims == 2 else ["R", "V", "V"])) + ) + + tf.debugging.assert_shapes( + shape_constraints, + message="base_orthogonal_conditional() arguments " + "[Note that this check verifies the shape of an alternative " + "representation of Kmn. See the docs for the actual expected " + "shape.]", + ) + + leading_dims = tf.shape(Kmn)[:-2] + + ################################################################################### + + fmean_u, fvar_u = conditional_GP_maths( + leading_dims=leading_dims, + Lm=Lm, + Kmn=Kmn, + Knn=Knn, + num_func=num_func, + M=M, + N=N, + f=f_u, + q_sqrt=q_sqrt_u, + white=white, + full_cov=full_cov, + just_parametric=True, + ) + + fmean_v, fvar_v = conditional_GP_maths( + leading_dims=leading_dims, + Lm=L_Cmm, + Kmn=Cmn, + Knn=Cnn, + num_func=num_func, + M=V, + N=N, + f=f_v, + q_sqrt=q_sqrt_v, + white=white, + full_cov=full_cov, + just_parametric=False, + ) + + ################################################################################### + + shape_constraints = [ + (Kmn, [..., "M", "N"]), # tensor included again for N dimension + (f_u, [..., "M", "R"]), # tensor included again for R dimension + (fmean_u, [..., "N", "R"]), + (fvar_u, [..., "R", "N", "N"] if full_cov else [..., "N", "R"]), + (fmean_v, [..., "N", "R"]), + (fvar_v, [..., "R", "N", "N"] if full_cov else [..., "N", "R"]), + ] + tf.debugging.assert_shapes( + shape_constraints, message="base_orthogonal_)conditional() return values" + ) + + return fmean_u + fmean_v, fvar_u + fvar_v + + +def base_heteroskedastic_orthogonal_conditional( + Kmn: tf.Tensor, + Kmm: tf.Tensor, + Knn: tf.Tensor, + Cmn: tf.Tensor, + Cmm: tf.Tensor, + Cnn: tf.Tensor, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False, +) -> MeanAndVariance: + r""" + + #TODO -- this needs to be updated to suit sparse orthogonal GPs + Given a g1 and g2, and distribution p and q such that + p(g2) = N(g2; 0, Kmm) + + p(g1) = N(g1; 0, Knn) + p(g1 | g2) = N(g1; Knm (Kmm⁻¹) g2, Knn - Knm (Kmm⁻¹) Kmn) + + And + q(g2) = N(g2; f, q_sqrt q_sqrtᵀ) + + This method computes the mean and (co)variance of + q(g1) = ∫ q(g2) p(g1 | g2) + + :param Kmn: [M, ..., N] + :param Kmm: [M, M] + :param Knn: [..., N, N] or N + :param f: [M, R] + :param full_cov: bool + :param q_sqrt: If this is a Tensor, it must have shape [R, M, M] (lower + triangular) or [M, R] (diagonal) + :param white: bool + :return: [N, R] or [R, N, N] + """ + Lm = tf.linalg.cholesky(Kmm) + return base_heteroskedastic_orthogonal_conditional_with_lm( + Kmn=Kmn, + Lm=Lm, + Knn=Knn, + L_Cmm=tf.linalg.cholesky(Cmm), + Cmn=Cmn, + Cnn=Cnn, + f_u=f_u, + f_v=f_v, + full_cov=full_cov, + q_sqrt_u=q_sqrt_u, + q_sqrt_v=q_sqrt_v, + white=white, + ) + + +def base_heteroskedastic_orthogonal_conditional_with_lm( + Kmn: tf.Tensor, + Lm: tf.Tensor, + Knn: tf.Tensor, + L_Cmm: tf.Tensor, + Cmn: tf.Tensor, + Cnn: tf.Tensor, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False, +) -> MeanAndVariance: + r""" + Has the same functionality as the `base_conditional` function, except that instead of + `Kmm` this function accepts `Lm`, which is the Cholesky decomposition of `Kmm`. + + This allows `Lm` to be precomputed, which can improve performance. + """ + # compute kernel stuff + num_func = tf.shape(f_u)[-1] # R + N = tf.shape(Kmn)[-1] + M = tf.shape(f_u)[-2] + + # get the leading dims in Kmn to the front of the tensor + # if Kmn has rank two, i.e. [M, N], this is the identity op. + K = tf.rank(Kmn) + perm = tf.concat( + [ + tf.reshape(tf.range(1, K - 1), [K - 2]), # leading dims (...) + tf.reshape(0, [1]), # [M] + tf.reshape(K - 1, [1]), + ], + 0, + ) # [N] + Kmn = tf.transpose(Kmn, perm) # [..., M, N] + Cmn = tf.transpose(Cmn, perm) # [..., M, N] + + shape_constraints = [ + (Kmn, [..., "M", "N"]), + (Lm, ["M", "M"]), + (Knn, [..., "N", "N"] if full_cov else [..., "N"]), + (f_u, ["M", "R"]), + (Cmn, [..., "V", "N"]), + (L_Cmm, ["V", "V"]), + (Cnn, [..., "N", "N"] if full_cov else [..., "N"]), + (f_v, ["V", "R"]), + ] + if q_sqrt_u is not None: + shape_constraints.append( + (q_sqrt_u, (["M", "R"] if q_sqrt_u.shape.ndims == 2 else ["R", "M", "M"])) + ) + + if q_sqrt_v is not None: + shape_constraints.append( + (q_sqrt_v, (["V", "R"] if q_sqrt_v.shape.ndims == 2 else ["R", "V", "V"])) + ) + + tf.debugging.assert_shapes( + shape_constraints, + message="base_orthogonal_conditional() arguments " + "[Note that this check verifies the shape of an alternative " + "representation of Kmn. See the docs for the actual expected " + "shape.]", + ) + + leading_dims = tf.shape(Kmn)[:-2] + + ################################################################################### + + fmean_u, fvar_u = conditional_GP_maths( + leading_dims=leading_dims, + Lm=Lm, + Kmn=Kmn, + Knn=Knn, + num_func=num_func, + M=M, + N=N, + f=f_u, + q_sqrt=q_sqrt_u, + white=white, + full_cov=full_cov, + just_parametric=True, + ) + + fmean_v, fvar_v = conditional_GP_maths( + leading_dims=leading_dims, + Lm=L_Cmm, + Kmn=Cmn, + Knn=Cnn, + num_func=num_func, + M=tf.shape(f_v)[-2], + N=N, + f=f_v, + q_sqrt=q_sqrt_v, + white=white, + full_cov=full_cov, + just_parametric=False, + ) + + ################################################################################### + + shape_constraints = [ + (Kmn, [..., "M", "N"]), # tensor included again for N dimension + (f_u, [..., "M", "R"]), # tensor included again for R dimension + (fmean_u, [..., "N", "R"]), + (fvar_u, [..., "R", "N", "N"] if full_cov else [..., "N", "R"]), + (fmean_v, [..., "N", "R"]), + (fvar_v, [..., "R", "N", "N"] if full_cov else [..., "N", "R"]), + ] + tf.debugging.assert_shapes( + shape_constraints, message="base_orthogonal_)conditional() return values" + ) + + return tf.concat([fmean_u, fmean_v], axis=-1), tf.concat([fvar_u, fvar_v], axis=-1) + + +def conditional_GP_maths( # noqa: C901 + leading_dims: tf.Tensor, + Lm: tf.Tensor, + Kmn: tf.Tensor, + Knn: tf.Tensor, + num_func: int, + M: int, + N: int, + f: tf.Tensor, + q_sqrt: tf.Tensor, + just_parametric: bool, + white: bool = True, + full_cov: bool = False, +) -> MeanAndVariance: + + # Compute the projection matrix A + Lm = tf.broadcast_to(Lm, tf.concat([leading_dims, tf.shape(Lm)], 0)) # [..., M, M] + A = tf.linalg.triangular_solve(Lm, Kmn, lower=True) # [..., M, N] + + if just_parametric: + pass + else: + + # compute the covariance due to the conditioning + if full_cov: + fvar = Knn - tf.linalg.matmul(A, A, transpose_a=True) # [..., N, N] + cov_shape = tf.concat([leading_dims, [num_func, N, N]], 0) + fvar = tf.broadcast_to(tf.expand_dims(fvar, -3), cov_shape) # [..., R, N, N] + else: + fvar = Knn - tf.reduce_sum(tf.square(A), -2) # [..., N] + cov_shape = tf.concat([leading_dims, [num_func, N]], 0) # [..., R, N] + fvar = tf.broadcast_to(tf.expand_dims(fvar, -2), cov_shape) # [..., R, N] + + # another backsubstitution in the unwhitened case + if not white: + A = tf.linalg.triangular_solve(tf.linalg.adjoint(Lm), A, lower=False) + + # construct the conditional mean + f_shape = tf.concat([leading_dims, [M, num_func]], 0) # [..., M, R] + f = tf.broadcast_to(f, f_shape) # [..., M, R] + fmean = tf.linalg.matmul(A, f, transpose_a=True) # [..., N, R] + + if q_sqrt is not None: + q_sqrt_dims = q_sqrt.shape.ndims + if q_sqrt_dims == 2: + LTA = A * tf.expand_dims(tf.transpose(q_sqrt), 2) # [R, M, N] + elif q_sqrt_dims == 3: + L = tf.linalg.band_part(q_sqrt, -1, 0) # force lower triangle # [R, M, M] + L_shape = tf.shape(L) + L = tf.broadcast_to(L, tf.concat([leading_dims, L_shape], 0)) + + shape = tf.concat([leading_dims, [num_func, M, N]], axis=0) + A_tiled = tf.broadcast_to(tf.expand_dims(A, -3), shape) + LTA = tf.linalg.matmul(L, A_tiled, transpose_a=True) # [R, M, N] + else: # pragma: no cover + raise ValueError("Bad dimension for q_sqrt: %s" % str(q_sqrt.shape.ndims)) + + # NOTE -- very ineffective at the moment + if just_parametric: + + if full_cov: + fvar = tf.linalg.matmul(LTA, LTA, transpose_a=True) # [R, N, N] + else: + fvar = tf.reduce_sum(tf.square(LTA), -2) # [R, N] + + else: + + if full_cov: + fvar = fvar + tf.linalg.matmul(LTA, LTA, transpose_a=True) # [R, N, N] + else: + fvar = fvar + tf.reduce_sum(tf.square(LTA), -2) # [R, N] + + if not full_cov: + fvar = tf.linalg.adjoint(fvar) # [N, R] + + #################################################################### + + return fmean, fvar + + +def separate_independent_orthogonal_conditional_implementation( + Kmns: tf.Tensor, + Kmms: tf.Tensor, + Knns: tf.Tensor, + Cmns: tf.Tensor, + Cmms: tf.Tensor, + Cnns: tf.Tensor, + f_u: tf.Tensor, + f_v: tf.Tensor, + *, + full_cov: bool = False, + q_sqrt_u: Optional[tf.Tensor] = None, + q_sqrt_v: Optional[tf.Tensor] = None, + white: bool = False, + Lms: Optional[tf.Tensor] = None, +) -> MeanAndVariance: + """ + Multi-output GP with independent GP priors. + + Number of latent processes equals the number of outputs (L = P). + + Further reference: + + - See `gpflow.conditionals._conditional` for a detailed explanation of + conditional in the single-output case. + - See the multioutput notebook for more information about the multioutput framework. + - See above for the parameters and the return value. + """ + fs_u = tf.transpose(f_u)[:, :, None] # [P, M_u, 1] + # [P, 1, M_u, M_u] or [P, M_u, 1] + + fs_v = tf.transpose(f_v)[:, :, None] # [P, M_v, 1] + # [P, 1, M_v, M_v] or [P, M_v, 1] + + if q_sqrt_u is not None and q_sqrt_v is not None: + q_sqrts_u = ( + tf.transpose(q_sqrt_u)[:, :, None] + if q_sqrt_u.shape.ndims == 2 + else q_sqrt_u[:, None, :, :] + ) + q_sqrts_v = ( + tf.transpose(q_sqrt_v)[:, :, None] + if q_sqrt_v.shape.ndims == 2 + else q_sqrt_v[:, None, :, :] + ) + + base_conditional_args_to_map: Tuple[tf.Tensor, ...] = ( + Kmms, + Kmns, + Knns, + Cmms, + Cmns, + Cnns, + fs_u, + fs_v, + q_sqrts_u, + q_sqrts_v, + Lms, + ) + + def single_orthogonal_gp_conditional( + t: Tuple[tf.Tensor, ...] + ) -> MeanAndVariance: # pragma: no cover - tf.map_fn is invisible to codecov + Kmm, Kmn, Knn, Cmm, Cmn, Cnn, f_u, f_v, q_sqrt_u, q_sqrt_v, Lm = t + + return base_orthogonal_conditional( + Kmn, + Kmm, + Knn, + Cmn, + Cmm, + Cnn, + f_u, + f_v, + full_cov=full_cov, + q_sqrt_u=q_sqrt_u, + q_sqrt_v=q_sqrt_v, + white=white, + Lm=Lm, + ) + + else: + base_conditional_args_to_map = (Kmms, Kmns, Knns, Cmms, Cmns, Cnns, fs_u, fs_v, Lms) + + def single_orthogonal_gp_conditional( + t: Tuple[tf.Tensor, ...] + ) -> MeanAndVariance: # pragma: no cover - tf.map_fn is invisible to codecov + Kmm, Kmn, Knn, Cmm, Cmn, Cnn, f_u, f_v, Lm = t + return base_orthogonal_conditional( + Kmn, + Kmm, + Knn, + Cmn, + Cmm, + Cnn, + f_u, + f_v, + full_cov=full_cov, + q_sqrt_u=q_sqrt_u, + q_sqrt_v=q_sqrt_v, + white=white, + Lm=Lm, + ) + + rmu, rvar = tf.map_fn( + single_orthogonal_gp_conditional, + base_conditional_args_to_map, + (default_float(), default_float()), + ) # [P, N, 1], [P, 1, N, N] or [P, N, 1] + + fmu = rollaxis_left(tf.squeeze(rmu, axis=-1), 1) # [N, P] + + if full_cov: + fvar = tf.squeeze(rvar, axis=-3) # [..., 0, :, :] # [P, N, N] + else: + fvar = rollaxis_left(tf.squeeze(rvar, axis=-1), 1) # [N, P] + + return fmu, fvar diff --git a/gpflux/covariances/__init__.py b/gpflux/covariances/__init__.py new file mode 100644 index 00000000..5fbe730d --- /dev/null +++ b/gpflux/covariances/__init__.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# flake8: noqa +from . import cvfs, cvvs, multioutput +from .dispatch import Cvf, Cvv + +""" +__all__ = [ + "dispatch", + "multioutput", + "Cvf", + "Cvv", + "cvfs", + "cvvs", +] +""" diff --git a/gpflux/covariances/cvfs.py b/gpflux/covariances/cvfs.py new file mode 100644 index 00000000..ea7f8056 --- /dev/null +++ b/gpflux/covariances/cvfs.py @@ -0,0 +1,39 @@ +from typing import Optional + +import tensorflow as tf + +from gpflow.base import TensorLike, TensorType +from gpflow.config import default_jitter +from gpflow.inducing_variables import InducingPoints +from gpflow.kernels import Kernel + +from gpflux.covariances.dispatch import Cvf + + +@Cvf.register(InducingPoints, InducingPoints, Kernel, TensorLike) +def Cvf_kernel_inducingpoints( + inducing_variable_u: InducingPoints, + inducing_variable_v: InducingPoints, + kernel: Kernel, + Xnew: TensorType, + *, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + Kvf = kernel(inducing_variable_v.Z, Xnew) + + if L_Kuu is None: + Kuu = kernel(inducing_variable_u.Z) + jittermat = tf.eye(inducing_variable_u.num_inducing, dtype=Kuu.dtype) * default_jitter() + Kuu += jittermat + L_Kuu = tf.linalg.cholesky(Kuu) + + Kuv = kernel(inducing_variable_u.Z, inducing_variable_v.Z) + Kuf = kernel(inducing_variable_u.Z, Xnew) + + L_Kuu_inv_Kuv = tf.linalg.triangular_solve(L_Kuu, Kuv) + L_Kuu_inv_Kuf = tf.linalg.triangular_solve(L_Kuu, Kuf) + + Cvf = Kvf - tf.linalg.matmul(L_Kuu_inv_Kuv, L_Kuu_inv_Kuf, transpose_a=True) + + return Cvf diff --git a/gpflux/covariances/cvvs.py b/gpflux/covariances/cvvs.py new file mode 100644 index 00000000..060db5b6 --- /dev/null +++ b/gpflux/covariances/cvvs.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional + +import tensorflow as tf + +from gpflow.config import default_jitter +from gpflow.inducing_variables import InducingPoints +from gpflow.kernels import Kernel + +from gpflux.covariances.dispatch import Cvv + + +@Cvv.register(InducingPoints, InducingPoints, Kernel) +def Cvv_kernel_inducingpoints( + inducing_variable_u: InducingPoints, + inducing_variable_v: InducingPoints, + kernel: Kernel, + *, + jitter: float = 0.0, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + Kvv = kernel(inducing_variable_v.Z) + + if L_Kuu is None: + Kuu = kernel(inducing_variable_u.Z) + jittermat = tf.eye(inducing_variable_u.num_inducing, dtype=Kuu.dtype) * default_jitter() + Kuu += jittermat + L_Kuu = tf.linalg.cholesky(Kuu) + + Kuv = kernel(inducing_variable_u.Z, inducing_variable_v.Z) + + L_Kuu_inv_Kuv = tf.linalg.triangular_solve(L_Kuu, Kuv) + Cvv = Kvv - tf.linalg.matmul(L_Kuu_inv_Kuv, L_Kuu_inv_Kuv, transpose_a=True) + + Cvv += jitter * tf.eye(inducing_variable_v.num_inducing, dtype=Cvv.dtype) + + return Cvv diff --git a/gpflux/covariances/dispatch.py b/gpflux/covariances/dispatch.py new file mode 100644 index 00000000..2b85a407 --- /dev/null +++ b/gpflux/covariances/dispatch.py @@ -0,0 +1,4 @@ +from gpflow.utilities import Dispatcher + +Cvv = Dispatcher("Cvv") +Cvf = Dispatcher("Cvf") diff --git a/gpflux/covariances/multioutput/__init__.py b/gpflux/covariances/multioutput/__init__.py new file mode 100644 index 00000000..30193266 --- /dev/null +++ b/gpflux/covariances/multioutput/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from . import cvfs, cvvs + +__all__ = ["cvfs", "cvvs"] diff --git a/gpflux/covariances/multioutput/cvfs.py b/gpflux/covariances/multioutput/cvfs.py new file mode 100644 index 00000000..5ba597f0 --- /dev/null +++ b/gpflux/covariances/multioutput/cvfs.py @@ -0,0 +1,211 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional + +import tensorflow as tf + +from gpflow.base import TensorType +from gpflow.experimental.check_shapes import check_shapes +from gpflow.inducing_variables import ( + InducingPoints, + SeparateIndependentInducingVariables, + SharedIndependentInducingVariables, +) +from gpflow.kernels import MultioutputKernel, SeparateIndependent, SharedIndependent + +from gpflux.covariances.dispatch import Cvf + + +@Cvf.register(InducingPoints, InducingPoints, MultioutputKernel, object) +@check_shapes( + "inducing_variable_u: [M_u, D, 1]", + "inducing_variable_v: [M_v, D, 1]", + "Xnew: [batch..., N, D]", + "return: [M_v, P, batch..., N, P]", +) +def Cvf_generic( + inducing_variable_u: InducingPoints, + inducing_variable_v: InducingPoints, + kernel: MultioutputKernel, + Xnew: TensorType, + *, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + return Cvf( + inducing_variable_u, + inducing_variable_v, + kernel, + Xnew, + L_Kuu=L_Kuu, + ) # [M, N] + + +@Cvf.register( + SharedIndependentInducingVariables, + SharedIndependentInducingVariables, + SharedIndependent, + object, +) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "Xnew: [batch..., N, D]", + "return: [M_v, batch..., N]", +) +def Cvf_shared_shared( + inducing_variable_u: SharedIndependentInducingVariables, + inducing_variable_v: SharedIndependentInducingVariables, + kernel: SharedIndependent, + Xnew: tf.Tensor, + *, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + return Cvf( + inducing_variable_u.inducing_variable, + inducing_variable_v.inducing_variable, + kernel.kernel, + Xnew, + L_Kuu=tf.unstack(L_Kuu, axis=0), + ) # [M_v, N] + + +@Cvf.register( + SeparateIndependentInducingVariables, + SeparateIndependentInducingVariables, + SharedIndependent, + object, +) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "Xnew: [batch..., N, D]", + "return: [L, M_v, batch..., N]", +) +def Cvf_separate_shared( + inducing_variable_u: SeparateIndependentInducingVariables, + inducing_variable_v: SeparateIndependentInducingVariables, + kernel: SharedIndependent, + Xnew: TensorType, + *, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + return tf.stack( + [ + Cvf( + ind_var_u, + ind_var_v, + kernel.kernel, + Xnew, + L_Kuu=l_kuu, + ) + for ind_var_u, ind_var_v, l_kuu in zip( + inducing_variable_u.inducing_variable_list, + inducing_variable_v.inducing_variable_list, + tf.unstack(L_Kuu, axis=0), + ) + ], + axis=0, + ) + + +@Cvf.register( + SharedIndependentInducingVariables, + SharedIndependentInducingVariables, + SeparateIndependent, + object, +) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "Xnew: [batch..., N, D]", + "return: [L, M_v, batch..., N]", +) +def Cvf_shared_separate( + inducing_variable_u: SharedIndependentInducingVariables, + inducing_variable_v: SharedIndependentInducingVariables, + kernel: SeparateIndependent, + Xnew: TensorType, + *, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + return tf.stack( + [ + Cvf( + inducing_variable_u.inducing_variable, + inducing_variable_v.inducing_variable, + k, + Xnew, + L_Kuu=l_kuu, + ) + for k, l_kuu in zip(kernel.kernels, tf.unstack(L_Kuu, axis=0)) + ], + axis=0, + ) + + +@Cvf.register( + SeparateIndependentInducingVariables, + SeparateIndependentInducingVariables, + SeparateIndependent, + object, +) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "Xnew: [batch..., N, D]", + "return: [L, M_v, batch..., N]", +) +def Cvf_separate_separate( + inducing_variable_u: SeparateIndependentInducingVariables, + inducing_variable_v: SeparateIndependentInducingVariables, + kernel: SeparateIndependent, + Xnew: TensorType, + *, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + n_iv_u = len(inducing_variable_u.inducing_variable_list) + n_iv_v = len(inducing_variable_v.inducing_variable_list) + n_k = len(kernel.kernels) + assert ( + n_iv_u == n_k + ), f"Must have same number of inducing variables and kernels. Found {n_iv_u} and {n_k}." + + assert ( + n_iv_v == n_k + ), f"Must have same number of inducing variables and kernels. Found {n_iv_v} and {n_k}." + + return tf.stack( + [ + Cvf( + ind_var_u, + ind_var_v, + k, + Xnew, + L_Kuu=l_kuu, + ) + for k, ind_var_u, ind_var_v, l_kuu in zip( + kernel.kernels, + inducing_variable_u.inducing_variable_list, + inducing_variable_v.inducing_variable_list, + tf.unstack(L_Kuu, axis=0), + ) + ], + axis=0, + ) diff --git a/gpflux/covariances/multioutput/cvvs.py b/gpflux/covariances/multioutput/cvvs.py new file mode 100644 index 00000000..4bd96b4d --- /dev/null +++ b/gpflux/covariances/multioutput/cvvs.py @@ -0,0 +1,213 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional, Union + +import tensorflow as tf + +from gpflow.experimental.check_shapes import check_shapes +from gpflow.inducing_variables import ( + FallbackSeparateIndependentInducingVariables, + FallbackSharedIndependentInducingVariables, + InducingPoints, +) +from gpflow.kernels import ( + IndependentLatent, + MultioutputKernel, + SeparateIndependent, + SharedIndependent, +) + +from gpflux.covariances.dispatch import Cvv + + +@Cvv.register(InducingPoints, InducingPoints, MultioutputKernel) +@check_shapes( + "inducing_variable_u: [M_u, D, 1]", + "inducing_variable_v: [M_v, D, 1]", + "return: [M_v, P, M_v, P]", +) +def Kuu_generic( + inducing_variable_u: InducingPoints, + inducing_variable_v: InducingPoints, + kernel: MultioutputKernel, + *, + jitter: float = 0.0, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + _Cvv = Cvv( + inducing_variable_u, + inducing_variable_v, + kernel, + L_Kuu=L_Kuu, + ) # [M, M] + jittermat = tf.eye(inducing_variable_v.num_inducing, dtype=_Cvv.dtype) * jitter + return _Cvv + jittermat + + +@Cvv.register( + FallbackSharedIndependentInducingVariables, + FallbackSharedIndependentInducingVariables, + SharedIndependent, +) +def Cvv_shared_shared( + inducing_variable_u: FallbackSharedIndependentInducingVariables, + inducing_variable_v: FallbackSharedIndependentInducingVariables, + kernel: SharedIndependent, + *, + jitter: float = 0.0, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + _Cvv = Cvv( + inducing_variable_u.inducing_variable, + inducing_variable_v.inducing_variable, + kernel.kernel, + L_Kuu=L_Kuu, + ) # [M, M] + jittermat = tf.eye(inducing_variable_v.num_inducing, dtype=_Cvv.dtype) * jitter + + return _Cvv + jittermat + + +@Cvv.register(FallbackSharedIndependentInducingVariables, SeparateIndependent) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "return: [L, M_v, M_v]", +) +def Cvv_fallback_shared( + inducing_variable_u: FallbackSharedIndependentInducingVariables, + inducing_variable_v: FallbackSharedIndependentInducingVariables, + kernel: Union[SeparateIndependent, IndependentLatent], + *, + jitter: float = 0.0, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + _Cvv = tf.stack( + [ + Cvv( + inducing_variable_u.inducing_variable, + inducing_variable_v.inducing_variable, + kernel.kernel, + L_Kuu=l_kuu, + ) + for k, l_kuu in zip(kernel.kernels, tf.unstack(L_Kuu, axis=0)) + ], + axis=0, + ) + + jittermat = tf.eye(inducing_variable_v.num_inducing, dtype=Cvv.dtype)[None, :, :] * jitter + + return _Cvv + jittermat + + +@Cvv.register( + FallbackSeparateIndependentInducingVariables, + FallbackSeparateIndependentInducingVariables, + SharedIndependent, +) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "return: [L, M_v, M_v]", +) +def Kuu_fallback_separate_shared( + inducing_variable_u: FallbackSeparateIndependentInducingVariables, + inducing_variable_v: FallbackSeparateIndependentInducingVariables, + kernel: SharedIndependent, + *, + jitter: float = 0.0, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + + _Cvv = tf.stack( + [ + Cvv(ind_var_u, ind_var_v, kernel.kernel, L_Kuu=l_kuu) + for ind_var_u, ind_var_v, l_kuu in zip( + inducing_variable_u.inducing_variable_list, + inducing_variable_v.inducing_variable_list, + tf.unstack(L_Kuu, axis=0), + ) + ], + axis=0, + ) + + jittermat = tf.eye(inducing_variable_v.num_inducing, dtype=Cvv.dtype)[None, :, :] * jitter + + return _Cvv + jittermat + + +@Cvv.register( + FallbackSeparateIndependentInducingVariables, + FallbackSeparateIndependentInducingVariables, + SeparateIndependent, +) +@check_shapes( + "inducing_variable_u: [M_u, D, P]", + "inducing_variable_v: [M_v, D, P]", + "return: [L, M_v, M_v]", +) +def Kuu_fallback_separate( + inducing_variable_u: FallbackSeparateIndependentInducingVariables, + inducing_variable_v: FallbackSeparateIndependentInducingVariables, + kernel: SeparateIndependent, + *, + jitter: float = 0.0, + L_Kuu: Optional[tf.Tensor] = None, +) -> tf.Tensor: + n_iv_u = len(inducing_variable_u.inducing_variable_list) + n_iv_v = len(inducing_variable_v.inducing_variable_list) + n_k = len(kernel.kernels) + assert ( + n_iv_u == n_k + ), f"Must have same number of inducing variables and kernels. Found {n_iv_u} and {n_k}." + + assert ( + n_iv_v == n_k + ), f"Must have same number of inducing variables and kernels. Found {n_iv_v} and {n_k}." + + lista = [] + for ind_var_u, ind_var_v, l_kuu, k in zip( + inducing_variable_u.inducing_variable_list, + inducing_variable_v.inducing_variable_list, + tf.unstack(L_Kuu, axis=0), + kernel.kernels, + ): + + lista.append(Cvv(ind_var_u, ind_var_v, k, L_Kuu=l_kuu)) + + """ + lista = [ +>>>>>>> 53dae0f597e5aa12eea4bc13839a3c94807acd36 + Cvv(ind_var_u, ind_var_v, k, L_Kuu=l_kuu) + for ind_var_u, ind_var_v, l_kuu, k in zip( + inducing_variable_u.inducing_variable_list, + inducing_variable_v.inducing_variable_list, + L_Kuu, + kernel.kernels, + ) + ] + """ + + _Cvv = tf.stack( + lista, + axis=0, + ) + + jittermat = tf.eye(inducing_variable_v.num_inducing, dtype=_Cvv.dtype)[None, :, :] * jitter + + return _Cvv + jittermat diff --git a/gpflux/helpers.py b/gpflux/helpers.py index 45bfc6b6..585c53fa 100644 --- a/gpflux/helpers.py +++ b/gpflux/helpers.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 The GPflux Contributors. +# Copyright (c) 2022 The GPflux Contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -r""" +""" This module contains helper functions for constructing :class:`~gpflow.kernels.MultioutputKernel`, :class:`~gpflow.inducing_variables.MultioutputInducingVariables`, :class:`~gpflow.mean_functions.MeanFunction`, and :class:`~gpflux.layers.GPLayer` objects. """ - import inspect import warnings from dataclasses import fields @@ -33,7 +32,8 @@ SeparateIndependentInducingVariables, SharedIndependentInducingVariables, ) -from gpflow.kernels import SeparateIndependent, SharedIndependent +from gpflow.kernels import SeparateIndependent, SharedIndependent, SquaredExponential, Stationary +from gpflow.mean_functions import Identity, Linear, MeanFunction, Zero from gpflow.utilities import deepcopy from gpflux.layers.gp_layer import GPLayer @@ -64,8 +64,10 @@ def construct_basic_kernel( the different outputs, but the kernel can have different hyperparameter values for each. """ if isinstance(kernels, list): - mo_kern = SeparateIndependent(kernels) - elif not share_hyperparams: + return SeparateIndependent(kernels) + + assert output_dim, "With single kernel, you must specify the number of outputs" + if not share_hyperparams: copies = [deepcopy(kernels) for _ in range(output_dim)] mo_kern = SeparateIndependent(copies) else: @@ -125,7 +127,7 @@ def construct_basic_inducing_variables( z_init_is_given = z_init is not None - if isinstance(num_inducing, list): + if isinstance(num_inducing, List): if output_dim is not None: # TODO: the following assert may clash with MixedMultiOutputFeatures # where the number of independent GPs can differ from the output @@ -136,18 +138,25 @@ def construct_basic_inducing_variables( inducing_variables = [] for i, num_ind_var in enumerate(num_inducing): if z_init_is_given: + assert z_init is not None assert len(z_init[i]) == num_ind_var z_init_i = z_init[i] else: - z_init_i = np.random.randn(num_ind_var, input_dim).astype(dtype=default_float()) + z_init_i = np.random.uniform( + low=-0.5, high=0.5, size=(num_ind_var, input_dim) + ).astype(dtype=default_float()) assert z_init_i.shape == (num_ind_var, input_dim) inducing_variables.append(InducingPoints(z_init_i)) + return SeparateIndependentInducingVariables(inducing_variables) - elif not share_variables: + if not share_variables: + assert output_dim, "When num_inducing is a number, the number of outputs must be given" + inducing_variables = [] for o in range(output_dim): if z_init_is_given: + assert z_init is not None if z_init.shape != (output_dim, num_inducing, input_dim): raise ValueError( "When not sharing variables, z_init must have shape" @@ -155,28 +164,28 @@ def construct_basic_inducing_variables( ) z_init_o = z_init[o] else: - z_init_o = np.random.randn(num_inducing, input_dim).astype(dtype=default_float()) + z_init_o = np.random.uniform(-0.5, 0.5, (num_inducing, input_dim)).astype( + dtype=default_float() + ) inducing_variables.append(InducingPoints(z_init_o)) + return SeparateIndependentInducingVariables(inducing_variables) - else: - # TODO: should we assert output_dim is None ? + # Share the same inducing variables across the outputs + z_init = ( + z_init + if z_init_is_given + else np.random.uniform(-0.5, 0.5, (num_inducing, input_dim)).astype(dtype=default_float()) + ) + shared_ip = InducingPoints(z_init) - z_init = ( - z_init - if z_init_is_given - else np.random.randn(num_inducing, input_dim).astype(dtype=default_float()) - ) - shared_ip = InducingPoints(z_init) - return SharedIndependentInducingVariables(shared_ip) + return SharedIndependentInducingVariables(shared_ip) -def construct_mean_function( - X: np.ndarray, D_in: int, D_out: int -) -> gpflow.mean_functions.MeanFunction: +def construct_mean_function(X: np.ndarray, D_out: int) -> MeanFunction: """ Return :class:`gpflow.mean_functions.Identity` when ``D_in`` and ``D_out`` are - equal. Otherwise, use the principal components of the inputs matrix ``X`` to build a + equal. Otherwise, use the principal components of the input matrix ``X`` to build a :class:`~gpflow.mean_functions.Linear` mean function. .. note:: @@ -186,14 +195,15 @@ def construct_mean_function( :param X: A data array with the shape ``[N, D_in]`` used to determine the principal components to use to create a :class:`~gpflow.mean_functions.Linear` mean function when ``D_in != D_out``. - :param D_in: The dimensionality of the input data (or features) ``X``. - Typically, this corresponds to ``X.shape[-1]``. :param D_out: The dimensionality of the outputs (or targets) ``Y``. Typically, this corresponds to ``Y.shape[-1]`` or the number of latent GPs in the layer. + :return: a GPflow mean function """ + + D_in = X.shape[-1] assert X.shape[-1] == D_in if D_in == D_out: - mean_function = gpflow.mean_functions.Identity() + mean_function = Identity() else: if D_in > D_out: _, _, V = np.linalg.svd(X, full_matrices=False) @@ -202,7 +212,7 @@ def construct_mean_function( W = np.concatenate([np.eye(D_in), np.zeros((D_in, D_out - D_in))], axis=1) assert W.shape == (D_in, D_out) - mean_function = gpflow.mean_functions.Linear(W) + mean_function = Linear(W) gpflow.set_trainable(mean_function, False) return mean_function @@ -213,7 +223,7 @@ def construct_gp_layer( num_inducing: int, input_dim: int, output_dim: int, - kernel_class: Type[gpflow.kernels.Stationary] = gpflow.kernels.SquaredExponential, + kernel_class: Type[Stationary] = SquaredExponential, z_init: Optional[np.ndarray] = None, name: Optional[str] = None, ) -> GPLayer: @@ -252,7 +262,7 @@ def construct_gp_layer( kernel=kernel, inducing_variable=inducing_variable, num_data=num_data, - mean_function=gpflow.mean_functions.Zero(), + mean_function=Zero(), name=name, ) return gp_layer diff --git a/gpflux/layers/__init__.py b/gpflux/layers/__init__.py index 34f7e073..245a05eb 100644 --- a/gpflux/layers/__init__.py +++ b/gpflux/layers/__init__.py @@ -21,4 +21,5 @@ from gpflux.layers.gp_layer import GPLayer from gpflux.layers.latent_variable_layer import LatentVariableLayer, LayerWithObservations from gpflux.layers.likelihood_layer import LikelihoodLayer +from gpflux.layers.orth_gp_layer import OrthGPLayer from gpflux.layers.trackable_layer import TrackableLayer diff --git a/gpflux/layers/gp_layer.py b/gpflux/layers/gp_layer.py index a9324555..c4297664 100644 --- a/gpflux/layers/gp_layer.py +++ b/gpflux/layers/gp_layer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 The GPflux Contributors. +# Copyright (c) 2022 The GPflux Contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,13 +27,13 @@ from gpflow import Parameter, default_float from gpflow.base import TensorType -from gpflow.conditionals import conditional from gpflow.inducing_variables import MultioutputInducingVariables from gpflow.kernels import MultioutputKernel from gpflow.kullback_leiblers import prior_kl from gpflow.mean_functions import Identity, MeanFunction from gpflow.utilities.bijectors import triangular +from gpflux.conditionals import conditional from gpflux.exceptions import GPLayerIncompatibilityException from gpflux.math import _cholesky_with_jitter from gpflux.runtime_checks import verify_compatibility diff --git a/gpflux/layers/orth_gp_layer.py b/gpflux/layers/orth_gp_layer.py new file mode 100644 index 00000000..7a46d5fb --- /dev/null +++ b/gpflux/layers/orth_gp_layer.py @@ -0,0 +1,263 @@ +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""This module provides the implementation of the sparse orthogonal variational GP layer""" +from typing import Optional, Tuple + +import numpy as np +import tensorflow as tf + +from gpflow import Parameter, default_float +from gpflow.base import TensorType +from gpflow.conditionals import conditional +from gpflow.functions import MeanFunction +from gpflow.inducing_variables import MultioutputInducingVariables +from gpflow.kernels import MultioutputKernel +from gpflow.kullback_leiblers import prior_kl +from gpflow.utilities import triangular + +from gpflux.layers import GPLayer + + +class OrthGPLayer(GPLayer): + """ + A sparse orthogonal variational multioutput GP layer. This layer holds the kernel, + inducing variables and variational distribution, and mean function. + """ + + q_mu_u: Parameter + r""" + The mean of ``q(v)`` or ``q(u)`` (depending on whether :attr:`whiten`\ ed + parametrisation is used). + """ + + q_mu_v: Parameter + r""" + The mean of ``q(v)`` or ``q(u)`` (depending on whether :attr:`whiten`\ ed + parametrisation is used). + """ + + q_sqrt_u: Parameter + r""" + The lower-triangular Cholesky factor of the covariance of ``q(v)`` or ``q(u)`` + (depending on whether :attr:`whiten`\ ed parametrisation is used). + """ + + q_sqrt_v: Parameter + r""" + The lower-triangular Cholesky factor of the covariance of ``q(v)`` or ``q(u)`` + (depending on whether :attr:`whiten`\ ed parametrisation is used). + """ + + def __init__( + self, + kernel: MultioutputKernel, + inducing_variable_u: MultioutputInducingVariables, + inducing_variable_v: MultioutputInducingVariables, + num_data: int, + mean_function: Optional[MeanFunction] = None, + *, + num_samples: Optional[int] = None, + full_cov: bool = False, + full_output_cov: bool = False, + num_latent_gps: int = None, + whiten: bool = True, + name: Optional[str] = None, + verbose: bool = True, + ): + """ + :param kernel: The multioutput kernel for this layer. + :param inducing_variable: The inducing features for this layer. + :param num_data: The number of points in the training dataset (see :attr:`num_data`). + :param mean_function: The mean function that will be applied to the + inputs. Default: :class:`~gpflow.mean_functions.Identity`. + + .. note:: The Identity mean function requires the input and output + dimensionality of this layer to be the same. If you want to + change the dimensionality in a layer, you may want to provide a + :class:`~gpflow.mean_functions.Linear` mean function instead. + + :param num_samples: The number of samples to draw when converting the + :class:`~tfp.layers.DistributionLambda` into a `tf.Tensor`, see + :meth:`_convert_to_tensor_fn`. Will be stored in the + :attr:`num_samples` attribute. If `None` (the default), draw a + single sample without prefixing the sample shape (see + :class:`tfp.distributions.Distribution`'s `sample() + `_ + method). + :param full_cov: Sets default behaviour of calling this layer + (:attr:`full_cov` attribute): + If `False` (the default), only predict marginals (diagonal + of covariance) with respect to inputs. + If `True`, predict full covariance over inputs. + :param full_output_cov: Sets default behaviour of calling this layer + (:attr:`full_output_cov` attribute): + If `False` (the default), only predict marginals (diagonal + of covariance) with respect to outputs. + If `True`, predict full covariance over outputs. + :param num_latent_gps: The number of (latent) GPs in the layer + (which can be different from the number of outputs, e.g. with a + :class:`~gpflow.kernels.LinearCoregionalization` kernel). + This is used to determine the size of the + variational parameters :attr:`q_mu` and :attr:`q_sqrt`. + If possible, it is inferred from the *kernel* and *inducing_variable*. + :param whiten: If `True` (the default), uses the whitened parameterisation + of the inducing variables; see :attr:`whiten`. + :param name: The name of this layer. + :param verbose: The verbosity mode. Set this parameter to `True` + to show debug information. + """ + + super().__init__( + kernel=kernel, + inducing_variable=inducing_variable_u, + num_data=num_data, + mean_function=mean_function, + num_samples=num_samples, + full_cov=full_cov, + full_output_cov=full_output_cov, + num_latent_gps=num_latent_gps, + whiten=whiten, + name=name, + verbose=verbose, + ) + + self.inducing_variable_u = inducing_variable_u + self.inducing_variable_v = inducing_variable_v + + """ + try: + num_inducing, self.num_latent_gps = verify_compatibility( + kernel, mean_function, inducing_variable + ) + # TODO: if num_latent_gps is not None, verify it is equal to self.num_latent_gps + except GPLayerIncompatibilityException as e: + if num_latent_gps is None: + raise e + + if verbose: + warnings.warn( + "Could not verify the compatibility of the `kernel`, `inducing_variable` " + "and `mean_function`. We advise using `gpflux.helpers.construct_*` to create " + "compatible kernels and inducing variables. As " + f"`num_latent_gps={num_latent_gps}` has been specified explicitly, this will " + "be used to create the `q_mu` and `q_sqrt` parameters." + ) + + num_inducing, self.num_latent_gps = ( + inducing_variable.num_inducing, + num_latent_gps, + ) + """ + num_inducing_u = self.inducing_variable_u.num_inducing + num_inducing_v = self.inducing_variable_v.num_inducing + + ######################################################## + # Introduce variational parameters for q(U) # + ######################################################## + + self.q_mu_u = Parameter( + np.random.uniform( + -0.5, 0.5, (num_inducing_u, self.num_latent_gps) + ), # np.zeros((num_inducing, self.num_latent_gps)), + dtype=default_float(), + name=f"{self.name}_q_mu_u" if self.name else "q_mu_u", + ) # [num_inducing, num_latent_gps] + + self.q_sqrt_u = Parameter( + np.stack([np.eye(num_inducing_u) for _ in range(self.num_latent_gps)]), + transform=triangular(), + dtype=default_float(), + name=f"{self.name}_q_sqrt_u" if self.name else "q_sqrt_u", + ) # [num_latent_gps, num_inducing, num_inducing] + + ######################################################## + # Introduce variational parameters for q(V) # + ######################################################## + + self.q_mu_v = Parameter( + np.random.uniform( + -0.5, 0.5, (num_inducing_v, self.num_latent_gps) + ), # np.zeros((num_inducing, self.num_latent_gps)), + dtype=default_float(), + name=f"{self.name}_q_mu_v" if self.name else "q_mu_v", + ) # [num_inducing, num_latent_gps] + + self.q_sqrt_v = Parameter( + np.stack([np.eye(num_inducing_v) for _ in range(self.num_latent_gps)]), + transform=triangular(), + dtype=default_float(), + name=f"{self.name}_q_sqrt_v" if self.name else "q_sqrt_v", + ) # [num_latent_gps, num_inducing, num_inducing] + + self.num_samples = num_samples + + def predict( + self, + inputs: TensorType, + *, + full_cov: bool = False, + full_output_cov: bool = False, + ) -> Tuple[tf.Tensor, tf.Tensor]: + """ + Make a prediction at N test inputs for the Q outputs of this layer, + including the mean function contribution. + + The covariance and its shape is determined by *full_cov* and *full_output_cov* as follows: + + +--------------------+---------------------------+--------------------------+ + | (co)variance shape | ``full_output_cov=False`` | ``full_output_cov=True`` | + +--------------------+---------------------------+--------------------------+ + | ``full_cov=False`` | [N, Q] | [N, Q, Q] | + +--------------------+---------------------------+--------------------------+ + | ``full_cov=True`` | [Q, N, N] | [N, Q, N, Q] | + +--------------------+---------------------------+--------------------------+ + + :param inputs: The inputs to predict at, with a shape of [N, D], where D is + the input dimensionality of this layer. + :param full_cov: Whether to return full covariance (if `True`) or + marginal variance (if `False`, the default) w.r.t. inputs. + :param full_output_cov: Whether to return full covariance (if `True`) + or marginal variance (if `False`, the default) w.r.t. outputs. + + :returns: posterior mean (shape [N, Q]) and (co)variance (shape as above) at test points + """ + mean_function = self.mean_function(inputs) + mean_cond, cov = conditional( + inputs, + self.inducing_variable_u, + self.inducing_variable_v, + self.kernel, + self.q_mu_u, + self.q_mu_v, + q_sqrt_u=self.q_sqrt_u, + q_sqrt_v=self.q_sqrt_v, + full_cov=full_cov, + full_output_cov=full_output_cov, + white=self.whiten, + ) + + return mean_cond + mean_function, cov + + def prior_kl(self) -> tf.Tensor: + r""" + Returns the KL divergence ``KL[q(u)∥p(u)]`` from the prior ``p(u)`` to + the variational distribution ``q(u)``. If this layer uses the + :attr:`whiten`\ ed representation, returns ``KL[q(v)∥p(v)]``. + """ + return prior_kl( + self.inducing_variable_u, self.kernel, self.q_mu_u, self.q_sqrt_u, whiten=self.whiten + ) + prior_kl( + self.inducing_variable_v, self.kernel, self.q_mu_v, self.q_sqrt_v, whiten=self.whiten + ) diff --git a/gpflux/models/__init__.py b/gpflux/models/__init__.py index 479c4c57..d2541f88 100644 --- a/gpflux/models/__init__.py +++ b/gpflux/models/__init__.py @@ -16,4 +16,4 @@ """ Base model classes implemented in GPflux """ -from gpflux.models.deep_gp import DeepGP +from gpflux.models.deep_gp import DeepGP, OrthDeepGP diff --git a/gpflux/models/deep_gp.py b/gpflux/models/deep_gp.py index 529e2ca7..0ebbb118 100644 --- a/gpflux/models/deep_gp.py +++ b/gpflux/models/deep_gp.py @@ -179,6 +179,26 @@ def _evaluate_deep_gp( features = layer(features, training=training) return features + def _evaluate_layer_wise_deep_gp( + self, + inputs: TensorType, + *, + training: Optional[bool] = False, + ) -> tf.Tensor: + """ + Evaluate ``f(x) = fₙ(⋯ (f₂(f₁(x))))`` on the *inputs* argument. + """ + features = inputs + hidden_layers = [] + + for count, layer in enumerate(self.f_layers): + + features = layer(features, training=training) + moments = features.mean(), features.variance() + hidden_layers.append(moments) + + return hidden_layers + def _evaluate_likelihood( self, f_outputs: TensorType, @@ -291,6 +311,12 @@ def as_prediction_model( return model_class(self.inputs, outputs) +class OrthDeepGP(DeepGP): + """ + Orthogonal Deep Gaussian Processes + """ + + def sample_dgp(model: DeepGP) -> Sample: # TODO: should this be part of a [Vanilla]DeepGP class? function_draws = [layer.sample() for layer in model.f_layers] # TODO: error check that all layers implement .sample()? diff --git a/gpflux/posteriors.py b/gpflux/posteriors.py new file mode 100644 index 00000000..5b7f241b --- /dev/null +++ b/gpflux/posteriors.py @@ -0,0 +1,459 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional, Tuple, Type, Union + +import tensorflow as tf + +from gpflow import kernels +from gpflow.base import MeanAndVariance, TensorType +from gpflow.conditionals.util import expand_independent_outputs +from gpflow.config import default_jitter +from gpflow.covariances import Kuf, Kuu +from gpflow.experimental.check_shapes import check_shapes +from gpflow.inducing_variables import ( + InducingVariables, + SeparateIndependentInducingVariables, + SharedIndependentInducingVariables, +) +from gpflow.kernels import Kernel, SeparateIndependent, SharedIndependent +from gpflow.mean_functions import MeanFunction +from gpflow.posteriors import ( + AbstractPosterior, + PrecomputedValue, + _DeltaDist, + _DiagNormal, + _MvNormal, + get_posterior_class, +) + +from gpflux.conditionals.util import ( # expand_independent_outputs, duplicate of gpflow? + base_orthogonal_conditional, + separate_independent_orthogonal_conditional_implementation, +) +from gpflux.covariances import Cvf, Cvv + +""" +#NOTE -- I don't think we need this +def create_posterior( + kernel: Kernel, + inducing_variable: InducingVariables, + q_mu: TensorType, + q_sqrt: TensorType, + whiten: bool, + mean_function: Optional[MeanFunction] = None, + precompute_cache: Union[PrecomputeCacheType, str, None] = PrecomputeCacheType.TENSOR, +) -> BasePosterior: + posterior_class = get_posterior_class(kernel, inducing_variable) + precompute_cache = _validate_precompute_cache_type(precompute_cache) + return posterior_class( # type: ignore + kernel, + inducing_variable, + q_mu, + q_sqrt, + whiten, + mean_function, + precompute_cache=precompute_cache, + ) +""" + + +class AbstractOrthogonalPosterior(AbstractPosterior): + def __init__( + self, + kernel: Kernel, + inducing_variable_u: Union[tf.Tensor, InducingVariables], + inducing_variable_v: Union[tf.Tensor, InducingVariables], + cache: Optional[Tuple[tf.Tensor, ...]] = None, + mean_function: Optional[MeanFunction] = None, + ) -> None: + """ + TODO -- add documentation here + """ + super().__init__(kernel, inducing_variable_u, mean_function=mean_function) + + self.kernel = kernel + self.inducing_variable_u = inducing_variable_u + self.inducing_variable_v = inducing_variable_v + self.cache = cache + self.mean_function = mean_function + + +class BaseOrthogonalPosterior(AbstractOrthogonalPosterior): + # TODO -- introduce a suitable check_shapes here + # @check_shapes( + # "inducing_variable_u: [M, D, broadcast P]", + # "q_mu_u: [N, P]", + # "q_sqrt_U: [N_P_or_P_N_N...]", + # ) + def __init__( + self, + kernel: Kernel, + inducing_variable_u: InducingVariables, + inducing_variable_v: InducingVariables, + q_mu_u: tf.Tensor, + q_mu_v: tf.Tensor, + q_sqrt_u: tf.Tensor, + q_sqrt_v: tf.Tensor, + whiten: bool = True, + mean_function: Optional[MeanFunction] = None, + ): + + super().__init__( + kernel, inducing_variable_u, inducing_variable_v, mean_function=mean_function + ) + self.whiten = whiten + self._set_qdist(q_mu_u, q_sqrt_u, q_mu_v, q_sqrt_v) + + @property + def q_mu_u(self) -> tf.Tensor: + return self._q_dist_u.q_mu + + @property + def q_mu_v(self) -> tf.Tensor: + return self._q_dist_v.q_mu + + @property + def q_sqrt_u(self) -> tf.Tensor: + return self._q_dist_u.q_sqrt + + @property + def q_sqrt_v(self) -> tf.Tensor: + return self._q_dist_v.q_sqrt + + def _set_qdist( + self, q_mu_u: TensorType, q_sqrt_u: TensorType, q_mu_v: TensorType, q_sqrt_v: TensorType + ) -> None: + + if q_sqrt_u is None: + self._q_dist_u = _DeltaDist(q_mu_u) + elif len(q_sqrt_u.shape) == 2: # q_diag + self._q_dist_u = _DiagNormal(q_mu_u, q_sqrt_u) + else: + self._q_dist_u = _MvNormal(q_mu_u, q_sqrt_u) + + if q_sqrt_v is None: + self._q_dist_v = _DeltaDist(q_mu_v) + elif len(q_sqrt_v.shape) == 2: # q_diag + self._q_dist_v = _DiagNormal(q_mu_v, q_sqrt_v) + else: + self._q_dist_v = _MvNormal(q_mu_v, q_sqrt_v) + + def _precompute(self) -> Tuple[PrecomputedValue, ...]: + """ + #TODO -- needs to be implemented + """ + raise NotImplementedError + + +class IndependentOrthogonalPosterior(BaseOrthogonalPosterior): + @check_shapes( + "mean: [batch..., N, P]", + "cov: [batch..., P, N, N] if full_cov", + "cov: [batch..., N, P] if not full_cov", + "return[0]: [batch..., N, P]", + "return[1]: [batch..., N, P, N, P] if full_cov and full_output_cov", + "return[1]: [batch..., N, P, P] if (not full_cov) and full_output_cov", + "return[1]: [batch..., P, N, N] if full_cov and (not full_output_cov)", + "return[1]: [batch..., N, P] if (not full_cov) and (not full_output_cov)", + ) + def _post_process_mean_and_cov( + self, mean: TensorType, cov: TensorType, full_cov: bool, full_output_cov: bool + ) -> MeanAndVariance: + return mean, expand_independent_outputs(cov, full_cov, full_output_cov) + + @check_shapes( + "Xnew: [N, D]", + "return: [broadcast P, N, N] if full_cov", + "return: [broadcast P, N] if (not full_cov)", + ) + def _get_Kff(self, Xnew: TensorType, full_cov: bool) -> tf.Tensor: + + # TODO: this assumes that Xnew has shape [N, D] and no leading dims + + if isinstance(self.kernel, kernels.SeparateIndependent): + # NOTE calling kernel(Xnew, full_cov=full_cov, full_output_cov=False) directly would + # return + # if full_cov: [P, N, N] -- this is what we want + # else: [N, P] instead of [P, N] as we get from the explicit stack below + Kff = tf.stack([k(Xnew, full_cov=full_cov) for k in self.kernel.kernels], axis=0) + elif isinstance(self.kernel, kernels.MultioutputKernel): + # effectively, SharedIndependent path + Kff = self.kernel.kernel(Xnew, full_cov=full_cov) + # NOTE calling kernel(Xnew, full_cov=full_cov, full_output_cov=False) directly would + # return + # if full_cov: [P, N, N] instead of [N, N] + # else: [N, P] instead of [N] + else: + # standard ("single-output") kernels + Kff = self.kernel(Xnew, full_cov=full_cov) # [N, N] if full_cov else [N] + + return Kff + + # TODO -- check_shapes has to be updated + # @check_shapes( + # "Xnew: [N, D]", + # "return: [N, N] if full_cov", + # "return: [N] if (not full_cov)", + # ) + def _get_single_Cff( + self, + Xnew: TensorType, + kernel: Kernel, + inducing_variable_u: InducingVariables, + full_cov: bool, + ) -> tf.Tensor: + + # TODO: this assumes that Xnew has shape [N, D] and no leading dims + + Kff = kernel(Xnew, full_cov=full_cov) + # NOTE calling kernel(Xnew, full_cov=full_cov, full_output_cov=False) directly would + # return + # if full_cov: [P, N, N] instead of [N, N] + # else: [N, P] instead of [N] + + Kmm = Kuu(inducing_variable_u, kernel, jitter=default_jitter()) + L_Kmm = tf.linalg.cholesky(Kmm) + + Kmf = Kuf(inducing_variable_u, kernel, Xnew) + L_Kmm_inv_Kmf = tf.linalg.triangular_solve(L_Kmm, Kmf) + + # compute the covariance due to the conditioning + if full_cov: + # TODO -- need to add broadcasting capability + Cff = Kff - tf.linalg.matmul( + L_Kmm_inv_Kmf, L_Kmm_inv_Kmf, transpose_a=True + ) # [..., N, N] + # num_func = tf.shape(self.q_mu_u)[-1] + # N = tf.shape(Kuf)[-1] + # cov_shape = [num_func, N, N] + # Cff = tf.broadcast_to(tf.expand_dims(Cff, -3), cov_shape) # [..., R, N, N] + else: + # TODO -- need to add broadcasting capability + Cff = Kff - tf.reduce_sum(tf.square(L_Kmm_inv_Kmf), -2) # [..., N] + # num_func = tf.shape(self.q_mu_u)[-1] + # N = tf.shape(Kuf)[-1] + # cov_shape = [num_func, N] # [..., R, N] + # Cff = tf.broadcast_to(tf.expand_dims(Cff, -2), cov_shape) # [..., R, N] + + return Cff, L_Kmm + + # TODO -- need to update check_shapes + # @check_shapes( + # "Xnew: [N, D]", + # "return: [broadcast P, N, N] if full_cov", + # "return: [broadcast P, N] if (not full_cov)", + # ) + def _get_Cff(self, Xnew: TensorType, full_cov: bool) -> tf.Tensor: + + # TODO: this assumes that Xnew has shape [N, D] and no leading dims + + if isinstance(self.kernel, SeparateIndependent): + # NOTE calling kernel(Xnew, full_cov=full_cov, full_output_cov=False) directly would + # return + # if full_cov: [P, N, N] -- this is what we want + # else: [N, P] instead of [P, N] as we get from the explicit stack below + + # TODO -- this could probably be done in a smarter way + # NOTE -- at the moment it's incurring a double computation + Cff = tf.stack( + [ + self._get_single_Cff(Xnew, k, ind_var, full_cov)[0] + for k, ind_var in zip( + self.kernel.kernels, self.inducing_variable_u.inducing_variable_list + ) + ], + axis=0, + ) + + L_Kmm = tf.stack( + [ + self._get_single_Cff(Xnew, k, ind_var, full_cov)[1] + for k, ind_var in zip( + self.kernel.kernels, self.inducing_variable_u.inducing_variable_list + ) + ], + axis=0, + ) + + elif isinstance(self.kernel, kernels.MultioutputKernel): + # effectively, SharedIndependent path + Cff, L_Kmm = self._get_single_Cff( + Xnew, self.kernel.kernel, self.inducing_variable_u.inducing_variable, full_cov + ) + + else: + # standard ("single-output") kernels + Cff, L_Kmm = self._get_single_Cff( + Xnew, self.kernel, self.inducing_variable_u, full_cov + ) # [N, N] if full_cov else [N] + + return Cff, L_Kmm + + def _conditional_with_precompute( + self, + cache: Tuple[tf.Tensor, ...], + Xnew: TensorType, + full_cov: bool = False, + full_output_cov: bool = False, + ) -> MeanAndVariance: + """ + #TODO -- need to implement this + """ + raise NotImplementedError + + +class IndependentOrthogonalPosteriorSingleOutput(IndependentOrthogonalPosterior): + + # could almost be the same as IndependentPosteriorMultiOutput ... + # TODO -- @inherit_check_shapes results in an error atm + # @inherit_check_shapes + def _conditional_fused( + self, Xnew: TensorType, full_cov: bool = False, full_output_cov: bool = False + ) -> MeanAndVariance: + # same as IndependentPosteriorMultiOutput, Shared~/Shared~ branch, except for following + # line: + + Knn = self._get_Kff(Xnew, full_cov=full_cov) + Cnn = self._get_Cff(Xnew, full_cov=full_cov) + + Kmm = Kuu(self.inducing_variable_u, self.kernel, jitter=default_jitter()) # [M_u, M_u] + Kmn = Kuf(self.inducing_variable_u, self.kernel, Xnew) # [M_U, N] + + Cmm = Cvv( + self.inducing_variable_u, self.inducing_variable_v, self.kernel, jitter=default_jitter() + ) # [M_v, M_v] + Cmn = Cvf(self.inducing_variable_u, self.inducing_variable_v, self.kernel, Xnew) # [M_v, N] + + fmean, fvar = base_orthogonal_conditional( + Kmn, + Kmm, + Knn, + Cmn, + Cmm, + Cnn, + self.q_mu_u, + self.q_mu_v, + full_cov=full_cov, + q_sqrt_u=self.q_sqrt_u, + q_sqrt_v=self.q_sqrt_v, + white=self.whiten, + ) # [N, P], [P, N, N] or [N, P] + + return self._post_process_mean_and_cov(fmean, fvar, full_cov, full_output_cov) + + +class IndependentOrthogonalPosteriorMultiOutput(IndependentOrthogonalPosterior): + # TODO -- @inherit_check_shapes results in an error atm + # @inherit_check_shapes + def _conditional_fused( + self, Xnew: TensorType, full_cov: bool = False, full_output_cov: bool = False + ) -> MeanAndVariance: + + if isinstance(self.inducing_variable_u, SharedIndependentInducingVariables) and isinstance( + self.kernel, SharedIndependent + ): + # same as IndependentPosteriorSingleOutput except for following line + + Knn = self._get_Kff(Xnew, full_cov=full_cov) + Cnn, L_Kuu = self._get_Cff(Xnew, full_cov=full_cov) + + Kmm = Kuu(self.inducing_variable_u, self.kernel, jitter=default_jitter()) # [M_u, M_u] + Kmn = Kuf(self.inducing_variable_u, self.kernel, Xnew) # [M_U, N] + + Cmm = Cvv( + self.inducing_variable_u, + self.inducing_variable_v, + self.kernel, + jitter=default_jitter(), + L_Kuu=L_Kuu, + ) # [M_v, M_v] + Cmn = Cvf( + self.inducing_variable_u, self.inducing_variable_v, self.kernel, Xnew, L_Kuu=L_Kuu + ) # [M_v, N] + + fmean, fvar = base_orthogonal_conditional( + Kmn, + Kmm, + Knn, + Cmn, + Cmm, + Cnn, + self.q_mu_u, + self.q_mu_v, + full_cov=full_cov, + q_sqrt_u=self.q_sqrt_u, + q_sqrt_v=self.q_sqrt_v, + white=self.whiten, + Lm=L_Kuu, + ) # [N, P], [P, N, N] or [N, P] + + else: + + # Following are: [P, M, M] - [P, M, N] - [P, N](x N) + Kmms = Kuu(self.X_data, self.kernel, jitter=default_jitter()) # [P, M, M] + Kmns = Kuf(self.X_data, self.kernel, Xnew) # [P, M, N] + Knns = self._get_Kff(Xnew, full_cov=full_cov) # [P, N](x N) + + Cnns, L_Kuus = self._get_Cff(Xnew, full_cov=full_cov) # [P, N](x N) + Cmms = Cvv( + self.inducing_variable_u, + self.inducing_variable_v, + self.kernel, + jitter=default_jitter(), + L_Kuu=L_Kuus, + ) # [P, M_v, M_v] + Cmns = Cvf( + self.inducing_variable_u, self.inducing_variable_v, self.kernel, Xnew, L_Kuu=L_Kuus + ) # [P, M_v, N] + + # TODO -- this fails in tests + fmean, fvar = separate_independent_orthogonal_conditional_implementation( + Kmns, + Kmms, + Knns, + Cmns, + Cmms, + Cnns, + self.q_mu_u, + self.q_mu_v, + full_cov=full_cov, + q_sqrt_u=self.q_sqrt_u, + q_sqrt_v=self.q_sqrt_v, + white=self.whiten, + Lms=L_Kuus, + ) + + return self._post_process_mean_and_cov(fmean, fvar, full_cov, full_output_cov) + + +@get_posterior_class.register(kernels.Kernel, InducingVariables, InducingVariables) +def _get_posterior_base_case( + kernel: Kernel, inducing_variable_u: InducingVariables, inducing_variable_v: InducingVariables +) -> Type[BaseOrthogonalPosterior]: + # independent single output + return IndependentOrthogonalPosteriorSingleOutput + + +@get_posterior_class.register( + (kernels.SharedIndependent, kernels.SeparateIndependent), + (SeparateIndependentInducingVariables, SharedIndependentInducingVariables), + (SeparateIndependentInducingVariables, SharedIndependentInducingVariables), +) +def _get_posterior_independent_mo( + kernel: Kernel, inducing_variable_u: InducingVariables, inducing_variable_v: InducingVariables +) -> Type[BaseOrthogonalPosterior]: + # independent multi-output + return IndependentOrthogonalPosteriorMultiOutput diff --git a/tests/experiment_support/__init__.py b/tests/experiment_support/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/experiment_support/test_ci_utils.py b/tests/experiment_support/test_ci_utils.py new file mode 100644 index 00000000..4ba9ecdf --- /dev/null +++ b/tests/experiment_support/test_ci_utils.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +from typing import Union + +import pytest + +from gpflux.experiment_support.ci_utils import ( + is_continuous_integration, + notebook_list, + notebook_niter, + notebook_range, +) + + +class CIEnviroment: + """Context manager to simulate a set up where a CI env variable is used""" + + def __init__(self, flag: Union[str, bool]) -> None: + """ + :param flag: the CI env variable value + """ + if isinstance(flag, str): + self._flag = False if flag == "" or flag == "0" else True + else: + self._flag = flag + + try: + self._ci = os.environ["CI"] + except KeyError: + self._ci = "false" + + def __enter__(self): + os.environ["CI"] = str(self._flag).lower() + + def __exit__(self, exc_type, exc_val, exc_tb): + os.environ["CI"] = self._ci + + +@pytest.mark.parametrize("ci", [True, False, "1", "0", ""]) +def test_is_continuous_integration(ci: bool) -> None: + with CIEnviroment(ci): + if isinstance(ci, str): + ci = False if ci == "" or ci == "0" else True + assert is_continuous_integration() == ci + + +@pytest.mark.parametrize("ci,niter", [(True, 2), (False, 10)]) +def test_notebook_niter(ci: bool, niter: int) -> None: + with CIEnviroment(ci): + assert notebook_niter(10) == niter + + +@pytest.mark.parametrize("ci,niter", [(True, 2), (False, 10)]) +def test_notebook_range(ci: bool, niter: int) -> None: + with CIEnviroment(ci): + assert notebook_range(10) == range(niter) + + +@pytest.mark.parametrize("ci,niter", [(True, 2), (False, 10)]) +def test_notebook_list(ci: bool, niter: int) -> None: + with CIEnviroment(ci): + assert list(range(niter)) == notebook_list(list(range(10))) diff --git a/tests/gpflux/architectures/test_config.py b/tests/gpflux/architectures/test_config.py new file mode 100644 index 00000000..fd422caf --- /dev/null +++ b/tests/gpflux/architectures/test_config.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any + +import pytest +import tensorflow_probability as tfp + +from gpflow.kernels import SquaredExponential + +from gpflux.architectures.config import ( + GaussianLikelihoodConfig, + HeteroSkedasticLikelihoodConfig, + LikelihoodConfig, + ModelHyperParametersConfig, + StudenttLikelihoodConfig, +) + + +@pytest.fixture( + name="likelihood_config", + params=[ + GaussianLikelihoodConfig(noise_variance=1e-3), + StudenttLikelihoodConfig(df=3, scale=1.0), + HeteroSkedasticLikelihoodConfig(distribution_class=tfp.distributions.Normal), + HeteroSkedasticLikelihoodConfig(distribution_class=tfp.distributions.StudentT), + ], +) +def _likelihood_config(request: Any) -> LikelihoodConfig: + return request.param + + +def test_likelihood_create(likelihood_config: LikelihoodConfig) -> None: + try: + likelihood_config.create() + except: + pytest.fail(f"Could not create likelihood with config: {type(likelihood_config)}") + + +@pytest.mark.parametrize("num_layers, whiten", [(0, True), (-1, True), (2, False), (-1, False)]) +def test_hyperparameters_model_config__raises_with_invalid_parameters( + num_layers: int, whiten: bool +) -> None: + with pytest.raises(AssertionError): + _ = ModelHyperParametersConfig( + num_layers=num_layers, + kernel=SquaredExponential, + likelihood=GaussianLikelihoodConfig(noise_variance=1e-2), + inner_layer_qsqrt_factor=1e-3, + whiten=whiten, + num_inducing=7, + ) diff --git a/tests/gpflux/architectures/test_constant_input_dim_deep_gp.py b/tests/gpflux/architectures/test_constant_input_dim_deep_gp.py deleted file mode 100644 index 5a6d660a..00000000 --- a/tests/gpflux/architectures/test_constant_input_dim_deep_gp.py +++ /dev/null @@ -1,37 +0,0 @@ -import numpy as np -import pytest -import tensorflow as tf - -from gpflux.architectures import Config, build_constant_input_dim_deep_gp -from gpflux.helpers import make_dataclass_from_class - - -class DemoConfig: - num_inducing = 7 - inner_layer_qsqrt_factor = 1e-3 - between_layer_noise_variance = 1e-3 - likelihood_noise_variance = 1e-2 - whiten = True - - -@pytest.mark.parametrize("input_dim", [7]) -@pytest.mark.parametrize("num_layers", [3]) -def test_smoke_build_constant_input_dim_deep_gp(input_dim, num_layers): - config = make_dataclass_from_class(Config, DemoConfig) - X = np.random.randn(13, input_dim) - Y = np.random.randn(13, 1) - dgp = build_constant_input_dim_deep_gp(X, num_layers, config) - model_train = dgp.as_training_model() - model_train.compile("Adam") - model_train.fit((X, Y), epochs=1) - model_test = dgp.as_prediction_model() - _ = model_test(X) - - -@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.int32]) -def test_build_constant_input_dim_deep_gp_raises_on_incorrect_dtype(dtype): - config = make_dataclass_from_class(Config, DemoConfig) - X = np.random.randn(13, 2).astype(dtype) - - with pytest.raises(ValueError): - build_constant_input_dim_deep_gp(X, 2, config) diff --git a/tests/gpflux/architectures/test_factory.py b/tests/gpflux/architectures/test_factory.py new file mode 100644 index 00000000..ec5d1b6d --- /dev/null +++ b/tests/gpflux/architectures/test_factory.py @@ -0,0 +1,181 @@ +# +# Copyright (c) 2022 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Any, Type + +import numpy as np +import pytest +import tensorflow_probability as tfp + +from gpflow.kernels import Matern12, Matern32, Matern52, SquaredExponential, Stationary + +from gpflux.architectures.config import ( + GaussianLikelihoodConfig, + HeteroSkedasticLikelihoodConfig, + HyperParametersConfig, + ModelHyperParametersConfig, + OrthogonalModelHyperparametersConfig, + StudenttLikelihoodConfig, +) +from gpflux.architectures.factory import build_constant_input_dim_architecture, build_kernel + + +@pytest.fixture( + name="kernel_type", + params=[ + SquaredExponential, + Matern12, + Matern32, + Matern52, + ], +) +def _kernel_type(request: Any) -> Type[Stationary]: + return request.param + + +@pytest.fixture(name="is_last_layer", params=[False, True]) +def _is_last_layer(request: Any) -> bool: + return request.param + + +@pytest.mark.parametrize("input_dim", [-1, 0]) +def test_build_kernel__raises_with_invalid_input_dim(input_dim) -> None: + with pytest.raises(AssertionError): + build_kernel(input_dim, False, SquaredExponential) + + +def test_build_kernel(kernel_type: Type[Stationary], is_last_layer: bool) -> None: + kernel = build_kernel(3, is_last_layer, kernel_type) + + assert isinstance(kernel, kernel_type) + expected_variance = 1.0 if is_last_layer else 1e-6 + expected_lengthscales = [1.0] * 3 + + np.testing.assert_allclose(kernel.variance.numpy(), expected_variance) + np.testing.assert_allclose(kernel.lengthscales, expected_lengthscales) + + +MODEL_CONFIGS = [ + ModelHyperParametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=GaussianLikelihoodConfig(noise_variance=1e-2), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing=7, + ), + ModelHyperParametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=StudenttLikelihoodConfig(df=3, scale=1e-2), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing=7, + ), + pytest.param( + ModelHyperParametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=HeteroSkedasticLikelihoodConfig(), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing=7, + ), + marks=pytest.mark.xfail, + ), + pytest.param( + ModelHyperParametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=HeteroSkedasticLikelihoodConfig( + distribution_class=tfp.distributions.StudentT + ), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing=7, + ), + marks=pytest.mark.xfail, + ), + OrthogonalModelHyperparametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=GaussianLikelihoodConfig(noise_variance=1e-2), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing_u=7, + num_inducing_v=7, + ), + OrthogonalModelHyperparametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=StudenttLikelihoodConfig(df=3, scale=1e-2), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing_u=7, + num_inducing_v=7, + ), + pytest.param( + OrthogonalModelHyperparametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=HeteroSkedasticLikelihoodConfig(), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing_u=7, + num_inducing_v=7, + ), + marks=pytest.mark.xfail, + ), + pytest.param( + OrthogonalModelHyperparametersConfig( + num_layers=3, + kernel=SquaredExponential, + likelihood=HeteroSkedasticLikelihoodConfig( + distribution_class=tfp.distributions.StudentT + ), + inner_layer_qsqrt_factor=1e-3, + whiten=True, + num_inducing_u=7, + num_inducing_v=7, + ), + marks=pytest.mark.xfail, + ), +] + + +@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.int32]) +@pytest.mark.parametrize("model_config", MODEL_CONFIGS) +def test_build_constant_input_dim_architecture__raises_on_incorrect_dtype( + dtype, model_config: HyperParametersConfig +) -> None: + X = np.random.randn(13, 2).astype(dtype) + + with pytest.raises(ValueError): + build_constant_input_dim_architecture(model_config, X) + + +@pytest.mark.parametrize("model_config", MODEL_CONFIGS) +def test_build_constant_input_dim_architecture__does_not_smoke( + model_config: HyperParametersConfig, +) -> None: + X = np.random.randn(13, 2) + Y = np.random.randn(13, 1) + + model = build_constant_input_dim_architecture(model_config, X) + model_train = model.as_training_model() + model_train.compile("Adam") + model_train.fit((X, Y), epochs=1) + model_test = model.as_prediction_model() + _ = model_test(X) diff --git a/tests/gpflux/layers/test_likelihood_layer.py b/tests/gpflux/layers/test_likelihood_layer.py index a978aca2..465c7a28 100644 --- a/tests/gpflux/layers/test_likelihood_layer.py +++ b/tests/gpflux/layers/test_likelihood_layer.py @@ -18,7 +18,14 @@ import tensorflow as tf from gpflow.kernels import Matern52 -from gpflow.likelihoods import Bernoulli, Beta, Gaussian, Poisson +from gpflow.likelihoods import ( + Bernoulli, + Beta, + Gaussian, + HeteroskedasticTFPConditional, + Poisson, + StudentT, +) from gpflow.mean_functions import Zero from gpflux.helpers import construct_basic_inducing_variables, construct_basic_kernel @@ -26,7 +33,14 @@ from gpflux.layers.likelihood_layer import LikelihoodOutputs from gpflux.losses import LikelihoodLoss -TEST_GPFLOW_LIKELIHOODS = [Bernoulli, Beta, Gaussian, Poisson] +TEST_GPFLOW_LIKELIHOODS = [ + Bernoulli, + Beta, + Gaussian, + Poisson, + StudentT, + pytest.param(HeteroskedasticTFPConditional, marks=pytest.mark.xfail), +] def setup_gp_layer_and_data(num_inducing: int, **gp_layer_kwargs): diff --git a/tests/gpflux/layers/test_orth_gp_layer.py b/tests/gpflux/layers/test_orth_gp_layer.py new file mode 100644 index 00000000..e626510b --- /dev/null +++ b/tests/gpflux/layers/test_orth_gp_layer.py @@ -0,0 +1,201 @@ +# +# Copyright (c) 2021 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import numpy as np +import pytest +import tensorflow as tf +import tensorflow_probability as tfp + +from gpflow.kernels import RBF +from gpflow.mean_functions import Zero + +from gpflux.helpers import construct_basic_inducing_variables, construct_basic_kernel +from gpflux.layers import OrthGPLayer +from gpflux.types import unwrap_dist + + +def setup_orth_gp_layer_and_data(num_inducing_u: int, num_inducing_v: int, **gp_layer_kwargs): + input_dim = 30 + output_dim = 5 + num_data = 100 + data = make_data(input_dim, output_dim, num_data=num_data) + + kernel = construct_basic_kernel(RBF(), output_dim) + inducing_vars_u = construct_basic_inducing_variables(num_inducing_u, input_dim, output_dim) + inducing_vars_v = construct_basic_inducing_variables(num_inducing_v, input_dim, output_dim) + mean_function = Zero(output_dim) + + gp_layer = OrthGPLayer( + kernel, + inducing_vars_u, + inducing_vars_v, + num_data, + mean_function=mean_function, + **gp_layer_kwargs + ) + return gp_layer, data + + +def make_data(input_dim: int, output_dim: int, num_data: int): + lim = [0, 20] + sigma = 0.1 + + X = np.random.random(size=(num_data, input_dim)) * lim[1] + cov = RBF().K(X) + np.eye(num_data) * sigma ** 2 + Y = [np.random.multivariate_normal(np.zeros(num_data), cov)[:, None] for _ in range(output_dim)] + Y = np.hstack(Y) + return X, Y + + +def test_build(): + num_inducing_u = 5 + num_inducing_v = 10 + + gp_layer, (X, Y) = setup_orth_gp_layer_and_data(num_inducing_u, num_inducing_v) + output_dim = Y.shape[-1] + + gp_layer.build(X.shape) + assert gp_layer.q_mu_u.shape == (num_inducing_u, output_dim) + assert gp_layer.q_sqrt_u.shape == (output_dim, num_inducing_u, num_inducing_u) + + assert gp_layer.q_mu_v.shape == (num_inducing_v, output_dim) + assert gp_layer.q_sqrt_v.shape == (output_dim, num_inducing_v, num_inducing_v) + + +def test_kl_change_q_mean(): + gp_layer, (X, Y) = setup_orth_gp_layer_and_data(num_inducing_u=5, num_inducing_v=5) + + gp_layer.build(X.shape) + assert gp_layer.prior_kl() > 0.0 + + q_mu_u_random = np.random.random(size=gp_layer.q_mu_u.shape) + gp_layer.q_mu_u.assign(q_mu_u_random) + + q_mu_v_random = np.random.random(size=gp_layer.q_mu_v.shape) + gp_layer.q_mu_v.assign(q_mu_v_random) + + assert gp_layer.prior_kl() > 0.0 + + +def test_kl_change_q_sqrt(): + gp_layer, (X, Y) = setup_orth_gp_layer_and_data(num_inducing_u=5, num_inducing_v=5) + + gp_layer.build(X.shape) + assert gp_layer.prior_kl() > 0.0 + + gp_layer.q_sqrt_u.assign(gp_layer.q_sqrt_u * 3) + gp_layer.q_sqrt_v.assign(gp_layer.q_sqrt_v * 3) + assert gp_layer.prior_kl() > 0.0 + + +def test_call_shapes(): + gp_layer, (X, Y) = setup_orth_gp_layer_and_data(num_inducing_u=5, num_inducing_v=5) + gp_layer.build(X.shape) + + output_dim = Y.shape[-1] + batch_size = X.shape[0] + + samples = tf.convert_to_tensor(gp_layer(X, training=False)) + assert samples.shape == (batch_size, output_dim) + + assert not gp_layer.full_cov and not gp_layer.full_output_cov + + distribution = gp_layer(X, training=False) + assert isinstance(unwrap_dist(distribution), tfp.distributions.MultivariateNormalDiag) + assert distribution.shape == (batch_size, output_dim) + + gp_layer.full_cov = True + distribution = gp_layer(X, training=False) + assert isinstance(unwrap_dist(distribution), tfp.distributions.MultivariateNormalTriL) + assert distribution.shape == (batch_size, output_dim) + assert distribution.covariance().shape == (output_dim, batch_size, batch_size) + + gp_layer.full_output_cov = True + gp_layer.full_cov = False + distribution = gp_layer(X, training=False) + assert isinstance(unwrap_dist(distribution), tfp.distributions.MultivariateNormalTriL) + assert distribution.shape == (batch_size, output_dim) + assert distribution.covariance().shape == (batch_size, output_dim, output_dim) + + gp_layer.full_output_cov = True + gp_layer.full_cov = True + with pytest.raises(NotImplementedError): + gp_layer(X) + + +def test_call_shapes_num_samples(): + num_samples = 10 + gp_layer, (X, Y) = setup_orth_gp_layer_and_data( + num_inducing_u=5, num_inducing_v=5, num_samples=num_samples + ) + gp_layer.build(X.shape) + + output_dim = Y.shape[-1] + batch_size = X.shape[0] + + samples = tf.convert_to_tensor(gp_layer(X, training=False)) + assert samples.shape == (num_samples, batch_size, output_dim) + + gp_layer.full_cov = True + samples = tf.convert_to_tensor(gp_layer(X, training=False)) + assert samples.shape == (num_samples, batch_size, output_dim) + + +def test_predict_shapes(): + gp_layer, (X, Y) = setup_orth_gp_layer_and_data(num_inducing_u=5, num_inducing_v=5) + gp_layer.build(X.shape) + + output_dim = Y.shape[-1] + batch_size = X.shape[0] + + mean, cov = gp_layer.predict(X) + assert mean.shape == (batch_size, output_dim) + assert cov.shape == (batch_size, output_dim) + + mean, cov = gp_layer.predict(X, full_cov=True) + assert mean.shape == (batch_size, output_dim) + assert cov.shape == (output_dim, batch_size, batch_size) + + mean, cov = gp_layer.predict(X, full_output_cov=True) + assert mean.shape == (batch_size, output_dim) + assert cov.shape == (batch_size, output_dim, output_dim) + + +def test_losses_are_added(): + gp_layer, (X, Y) = setup_orth_gp_layer_and_data(num_inducing_u=5, num_inducing_v=5) + gp_layer.build(X.shape) + + # to make KL non-zero + gp_layer.q_mu_u.assign(tf.ones_like(gp_layer.q_mu_u) * 3) + assert gp_layer.prior_kl() > 0.0 + + assert len(gp_layer.losses) == 0 + + _ = gp_layer(X, training=True) + assert gp_layer.losses == [gp_layer.prior_kl() / gp_layer.num_data] + + # Check loss is 0 when training is False + _ = gp_layer(X, training=False) + assert gp_layer.losses == [tf.zeros_like(gp_layer.losses[0])] + + # Check calling multiple times only adds one loss + _ = gp_layer(X, training=True) + _ = gp_layer(X, training=True) + assert len(gp_layer.losses) == 1 + assert gp_layer.losses == [gp_layer.prior_kl() / gp_layer.num_data] + + +if __name__ == "__main__": + test_call_shapes() diff --git a/tests/gpflux/models/test_deep_gp.py b/tests/gpflux/models/test_deep_gp.py index b5fc8a42..a2c361d8 100644 --- a/tests/gpflux/models/test_deep_gp.py +++ b/tests/gpflux/models/test_deep_gp.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from typing import Tuple + import numpy as np import pytest import tensorflow as tf @@ -23,14 +25,32 @@ from gpflow.mean_functions import Zero from gpflux.helpers import construct_basic_inducing_variables, construct_basic_kernel -from gpflux.layers import GPLayer, LikelihoodLayer -from gpflux.models import DeepGP +from gpflux.layers import GPLayer, OrthGPLayer +from gpflux.models import DeepGP, OrthDeepGP MAXITER = int(80e3) PLOTTER_INTERVAL = 60 +Dataset = Tuple[np.ndarray, np.ndarray] + + +@pytest.fixture(name="dataset", scope="module") +def _dataset() -> Dataset: + input_dim = 2 + num_data = 1000 + + lim = [0, 100] + kernel = RBF(lengthscales=20) + sigma = 0.01 + X = np.random.random(size=(num_data, input_dim)) * lim[1] + cov = kernel.K(X) + np.eye(num_data) * sigma ** 2 + Y = np.random.multivariate_normal(np.zeros(num_data), cov)[:, None] + Y = np.clip(Y, -0.5, 0.5) + + return X, Y -def build_deep_gp(input_dim, num_data): + +def build_deep_gp(input_dim, num_data) -> DeepGP: layers = [input_dim, 2, 2, 1] # Below are different ways to build layers @@ -62,7 +82,54 @@ def build_deep_gp(input_dim, num_data): return DeepGP(gp_layers, Gaussian(0.1)) -def train_deep_gp(deep_gp, data, maxiter=MAXITER, plotter=None, plotter_interval=PLOTTER_INTERVAL): +def build_orth_deep_gp(input_dim, num_data) -> OrthDeepGP: + layers = [input_dim, 2, 2, 1] + # Below are different ways to build layers + + # 1. Pass in Lists: + kernel_list = [RBF(), Matern12()] + num_inducing_u = [25, 25] + num_inducing_v = [10, 10] + l1_kernel = construct_basic_kernel(kernels=kernel_list) + l1_inducing_u = construct_basic_inducing_variables( + num_inducing=num_inducing_u, input_dim=layers[0] + ) + l1_inducing_v = construct_basic_inducing_variables( + num_inducing=num_inducing_v, input_dim=layers[0] + ) + + # 2. Pass in kernels, specify output dims (shared hyperparams/variables) + l2_kernel = construct_basic_kernel(kernels=RBF(), output_dim=layers[2], share_hyperparams=True) + l2_inducing_u = construct_basic_inducing_variables( + num_inducing=25, input_dim=layers[1], share_variables=True + ) + + l2_inducing_v = construct_basic_inducing_variables( + num_inducing=10, input_dim=layers[1], share_variables=True + ) + + # 3. Pass in kernels, specify output dims (independent hyperparams/vars) + # By default and the constructor will make indep. copies + l3_kernel = construct_basic_kernel(kernels=RBF(), output_dim=layers[3]) + l3_inducing_u = construct_basic_inducing_variables( + num_inducing=25, input_dim=layers[2], output_dim=layers[3] + ) + l3_inducing_v = construct_basic_inducing_variables( + num_inducing=10, input_dim=layers[2], output_dim=layers[3] + ) + + # Assemble at the end + gp_layers = [ + OrthGPLayer(l1_kernel, l1_inducing_u, l1_inducing_v, num_data), + OrthGPLayer(l2_kernel, l2_inducing_u, l2_inducing_v, num_data), + OrthGPLayer(l3_kernel, l3_inducing_u, l3_inducing_v, num_data, mean_function=Zero()), + ] + return OrthDeepGP(gp_layers, Gaussian(0.1)) + + +def train_deep_gp( + deep_gp, data, maxiter=MAXITER, plotter=None, plotter_interval=PLOTTER_INTERVAL +) -> None: optimizer = tf.optimizers.Adam() @tf.function(autograph=False) @@ -82,17 +149,6 @@ def step(): plotter() -def setup_dataset(input_dim: int, num_data: int, dtype: np.dtype = np.float64): - lim = [0, 100] - kernel = RBF(lengthscales=20) - sigma = 0.01 - X = np.random.random(size=(num_data, input_dim)) * lim[1] - cov = kernel.K(X) + np.eye(num_data) * sigma ** 2 - Y = np.random.multivariate_normal(np.zeros(num_data), cov)[:, None] - Y = np.clip(Y, -0.5, 0.5) - return X.astype(dtype), Y.astype(dtype) - - def get_live_plotter(train_data, model): from matplotlib import pyplot as plt from mpl_toolkits import mplot3d @@ -133,11 +189,9 @@ def plotter(*args, **kwargs): return fig, plotter -def run_demo(maxiter=int(80e3), plotter_interval=60): - input_dim = 2 - num_data = 1000 - data = setup_dataset(input_dim, num_data) - deep_gp = build_deep_gp(input_dim, num_data) +def run_demo( + deep_gp: DeepGP, data: Dataset, maxiter: int = int(80e3), plotter_interval: int = 60 +) -> None: fig, plotter = get_live_plotter(data, deep_gp) train_deep_gp( deep_gp, @@ -148,19 +202,20 @@ def run_demo(maxiter=int(80e3), plotter_interval=60): ) -def test_smoke(): +@pytest.mark.parametrize("model", [build_deep_gp(2, 1000), build_orth_deep_gp(2, 1000)]) +def test_smoke(dataset: Dataset, model: DeepGP) -> None: import matplotlib matplotlib.use("PS") # Agg does not support 3D - run_demo(maxiter=2, plotter_interval=1) + run_demo(deep_gp=model, data=dataset, maxiter=2, plotter_interval=1) +@pytest.mark.parametrize("model", [build_deep_gp(2, 1000), build_orth_deep_gp(2, 1000)]) @pytest.mark.parametrize("dtype", [np.float16, np.float32, np.int32]) -def test_deep_gp_raises_on_incorrect_dtype(dtype): - input_dim = 2 - num_data = 1000 - X, Y = setup_dataset(input_dim, num_data, dtype) - model = build_deep_gp(input_dim, num_data) +def test_deep_gp_raises_on_incorrect_dtype( + dataset: Dataset, model: DeepGP, dtype: np.dtype +) -> None: + X, Y = dataset[0].astype(dtype), dataset[1].astype(dtype) with pytest.raises(ValueError): model.predict_f(X) diff --git a/tests/gpflux/test_helpers.py b/tests/gpflux/test_helpers.py index 04517deb..3980b661 100644 --- a/tests/gpflux/test_helpers.py +++ b/tests/gpflux/test_helpers.py @@ -19,7 +19,6 @@ import pytest import gpflow -from gpflow import mean_functions from gpflow.inducing_variables import ( InducingPoints, MultioutputInducingVariables, @@ -142,14 +141,14 @@ def test_construct_inducing_shared_independent_duplicates(z_init): def test_construct_mean_function_Identity(): num_data, input_dim, output_dim = 11, 5, 5 X = np.random.randn(num_data, input_dim) - mean_functions = construct_mean_function(X, input_dim, output_dim) + mean_functions = construct_mean_function(X, output_dim) assert isinstance(mean_functions, gpflow.mean_functions.Identity) def test_construct_mean_function_Linear(): num_data, input_dim, output_dim = 11, 5, 7 X = np.random.randn(num_data, input_dim) - mean_functions = construct_mean_function(X, input_dim, output_dim) + mean_functions = construct_mean_function(X, output_dim) assert isinstance(mean_functions, gpflow.mean_functions.Linear)