feat2vec_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feat2vec example\n",
    "\n",
    "This script shows a minimum working example for the feat2vec library. feat2vec learns dense vectors for all types of features used in data science- categories, real-valued variables, even images or text. It is an answer to word2vec's widespread usage for vector representation of things that are not really words. It can group \"similar columns\" into single features that have a more sensible interpretation (for example, a 10-dimensional matrix indicating what genres a movie might belong to can be aggregated to a single \"genre\" embedding). We use Factorization Machines and a novel sampling algorithm to generate these data. Please see the PDF in the repo for detailed technical explanation of our learning method.\n",
    "\n",
    "It is built primarily on pandas, numpy, and keras. Below we show a minimum working example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import os,sys\n",
    "import keras\n",
    "import feat2vec\n",
    "import matplotlib.pyplot as plt\n",
    "from feat2vec import DeepFM\n",
    "from feat2vec.implicitsampler import *\n",
    "# reload(implicitsampler)\n",
    "# from implicitsampler import ImplicitSampler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set some important hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "samplesize = 100000 #number of records\n",
    "batch_size=2000 #number of positive labels per batch\n",
    "neg_samples=5 #number of negative labels per batch\n",
    "embed_dim = 100 #dimensionality of embeddings to learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate some fake data in a pandas df, including a latent variable that generates a binary variable to create a correlation structure "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "attrs = np.random.multivariate_normal(mean = np.array([0.,0.]), cov = np.array([[1.,.1],[.1,1]]),size=samplesize)\n",
    "data = pd.DataFrame({ 'id': np.random.randint(0,50,size=samplesize),\n",
    "                          'attr1':attrs[:,0],\n",
    "                          'attr2':attrs[:,1],\n",
    "                          'attr3': np.random.uniform(size=samplesize),\n",
    "\t\t\t\t\t\t  'offset_':np.zeros(samplesize)\n",
    "\t})\n",
    "\n",
    "data['item'] = np.round(data.id%10 + np.random.normal(0,.5,size=samplesize))\n",
    "data.loc[data.item >=9,'item'] = 9\n",
    "data.loc[data.item <=0,'item'] = 0\n",
    "\n",
    "data['latenty'] = ( (data.attr3)**4/600. + np.exp(data.attr1*data.attr2) \\\n",
    "                       + (data.item<=3)*np.pi - ((data.item==2) | (data.item==4))*data.attr3 \\\n",
    "                       + ((data.item==8) | (data.item==7) )*data.attr2 \\\n",
    "                       + (data.item>=6)*data.attr1 + (data.id%3)*data.attr3 \\\n",
    "                       + np.random.normal(size=samplesize) - 5 )\n",
    "data.loc[np.abs(data['latenty']) >= 10,'latenty'] = 0\n",
    "data['y'] = (data[\"latenty\"] > 0).astype(int) #  np.floor(data.latenty)\n",
    "dim_y = len(pd.unique(data.y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    79897\n",
       "1    20103\n",
       "Name: y, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"y\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>attr1</th>\n",
       "      <th>attr2</th>\n",
       "      <th>attr3</th>\n",
       "      <th>offset_</th>\n",
       "      <th>item</th>\n",
       "      <th>latenty</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35</td>\n",
       "      <td>-0.794268</td>\n",
       "      <td>-1.615025</td>\n",
       "      <td>0.127423</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.654092</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>1.111472</td>\n",
       "      <td>-0.328067</td>\n",
       "      <td>0.139565</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>-4.826574</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>0.902116</td>\n",
       "      <td>-2.185723</td>\n",
       "      <td>0.334804</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>-5.791192</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28</td>\n",
       "      <td>-0.783354</td>\n",
       "      <td>-1.804620</td>\n",
       "      <td>0.778878</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>-2.534848</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>-0.069323</td>\n",
       "      <td>-0.403888</td>\n",
       "      <td>0.179108</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-3.803285</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id     attr1     attr2     attr3  offset_  item   latenty  y\n",
       "0  35 -0.794268 -1.615025  0.127423      0.0   5.0  0.654092  1\n",
       "1   9  1.111472 -0.328067  0.139565      0.0   9.0 -4.826574  0\n",
       "2  17  0.902116 -2.185723  0.334804      0.0   8.0 -5.791192  0\n",
       "3  28 -0.783354 -1.804620  0.778878      0.0   8.0 -2.534848  0\n",
       "4  13 -0.069323 -0.403888  0.179108      0.0   4.0 -3.803285  0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define what features we are going to use and how we are going to use them\n",
    "Some important things to note here:\n",
    "* Each item in the model_features list will be represented as a dense vector that learns similarities by interacting with all other model features.At the moment, we can pass multiple columns as a single model feature, but this is only available to for real valued features (not categories)\n",
    "* Each item in the sampling_features list represents a unique feature of data that should be sampled together for the oversampling method . For example, one might want to sample car model and car manufacturer together because other combinations are not only not observed in the data, they are entirely nonsensical. sampling_features do not need to coincide with model features (i.e. a researcher might still feel its important to learn separate embeddings for make and model of a car)\n",
    "* features is a list of what we call the model_features in the keras model layers\n",
    "* We pass most arguments to feat2vec as a list of lists, and often each sub-list only has one element. this is necessary because it is possible to have multiple elements in a single \"feature\". \n",
    "* We aggregate attr1,attr2,attr3 into a single \"feature\" called attrs. The idea is these features all represent at a high level one characteristic of each observation, and so should be grouped together. One might imagine in a real world setting, aggregating latitude and longitude into a single \"location\" feature\n",
    "*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['item','id','attrs','y','offset']\n",
    "model_features = [['item'],['id'],['attr1','attr2','attr3'],['y'],['offset_']]\n",
    "sampling_features = [['item'],['id'],['attr1','attr2','attr3'],['y']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define how we are going to use the features in the model; specifically, whether to learn biases or embeddings only. typically, we want do not want to learn bias terms since these are not straightforward to import to external applications. Here we only learn embeddings for everything except the intercept term (offset\\_) which we keep to \n",
    "modify the average level of the score function. both bias_only and embeddings_only except an ordered boolean list where positionally they correspond to each feature listed in model_features\n",
    "\n",
    "also pass a realvalued bool list which tells our model whether we should treat the inputs as discrete categories or real-valued scalar numbers\n",
    "\n",
    "finally, feature\\_dims refers to the dimensionality of each feature. for discrete categories, this is the total # of categories. for realvalued features, this is the number of columns (i.e. for attrs it is 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bias_only = [False]*len(model_features)\n",
    "bias_only[ features.index('offset') ] =True\n",
    "embeddings_only = [True]*len(model_features)\n",
    "embeddings_only[ features.index('offset') ] =False\n",
    "realvalued = [False]*len(model_features)\n",
    "realvalued[ features.index('attrs') ] =True\n",
    "feature_dims = [10,50,3,dim_y,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# build the feat2vec model in keras\n",
    "note we specify noise contrastive estimation as the objective; negative sampling is also available but disencouraged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/quy/anaconda3/envs/py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/quy/anaconda3/envs/py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4115: The name tf.random_normal is deprecated. Please use tf.random.normal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/quy/risk_inspection/Feat2Vec/feat2vec/model.py:35: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "WARNING:tensorflow:From /home/quy/risk_inspection/Feat2Vec/feat2vec/model.py:499: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "item (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "id (InputLayer)                 (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "y (InputLayer)                  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_item (Embedding)      (None, 1, 100)       1000        item[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_id (Embedding)        (None, 1, 100)       5000        id[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "attrs (InputLayer)              (None, 3)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_y (Embedding)         (None, 1, 100)       200         y[0][0]                          \n",
      "__________________________________________________________________________________________________\n",
      "offset (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_item_flattened (Resha (None, 100)          0           embedding_item[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "embedding_id_flattened (Reshape (None, 100)          0           embedding_id[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_attrs (Dense)         (None, 100)          300         attrs[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "embedding_y_flattened (Reshape) (None, 100)          0           embedding_y[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bias_offset (Embedding)         (None, 1, 1)         1           offset[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "interaction_itemxid (Dot)       (None, 1)            0           embedding_item_flattened[0][0]   \n",
      "                                                                 embedding_id_flattened[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "interaction_itemxattrs (Dot)    (None, 1)            0           embedding_item_flattened[0][0]   \n",
      "                                                                 embedding_attrs[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "interaction_itemxy (Dot)        (None, 1)            0           embedding_item_flattened[0][0]   \n",
      "                                                                 embedding_y_flattened[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "interaction_idxattrs (Dot)      (None, 1)            0           embedding_id_flattened[0][0]     \n",
      "                                                                 embedding_attrs[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "interaction_idxy (Dot)          (None, 1)            0           embedding_id_flattened[0][0]     \n",
      "                                                                 embedding_y_flattened[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "interaction_attrsxy (Dot)       (None, 1)            0           embedding_attrs[0][0]            \n",
      "                                                                 embedding_y_flattened[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "bias_offset_reshaped (Reshape)  (None, 1)            0           bias_offset[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "factors_weight0 (Add)           (None, 1)            0           interaction_itemxid[0][0]        \n",
      "                                                                 interaction_itemxattrs[0][0]     \n",
      "                                                                 interaction_itemxy[0][0]         \n",
      "                                                                 interaction_idxattrs[0][0]       \n",
      "                                                                 interaction_idxy[0][0]           \n",
      "                                                                 interaction_attrsxy[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "output_layer (Add)              (None, 1)            0           bias_offset_reshaped[0][0]       \n",
      "                                                                 factors_weight0[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "noise_probs (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "nce_obj (Lambda)                (None, 1)            0           output_layer[0][0]               \n",
      "                                                                 noise_probs[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 6,501\n",
      "Trainable params: 6,501\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "feat2vec_obj = DeepFM(model_features, feature_dims,  obj='nce' ,\n",
    "                      feature_names=features,realval=realvalued)\n",
    "feat2vec_fm = feat2vec_obj.build_model(embed_dim, deep_out=False,bias_only=bias_only,embeddings_only=embeddings_only)\n",
    "print(feat2vec_fm.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Implicit Samplers that Produce Negative Labels\n",
    "\n",
    "To sample negative labels, we need to choose hyperparameters that determine both how often we sample a given feature, and conditional on sampling a given feature, how often we sample the most popular ones. this parameter is the same as the flattening hyperparameter in word2vec, $Pr(w)$ $\\propto$ $|\\#w|^\\alpha$. Below we show graphically how this parameter changes sampling probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEaCAYAAAA7YdFPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3yeVfn/3yd7dmR0J01HupKmg7SltOy9pzL8KtO6EPTnAr8yvogKiixBtLJUpAgoCAgyW7tHupI2bZM0TZs0XRlNs9dz/f44d5KnacbzZI/r/Xrdr3ud+5zrnp/7rOsYEUFRFEVROopPbxugKIqi9G9USBRFUZROoUKiKIqidAoVEkVRFKVTqJAoiqIonUKFRFEURekUKiQ9jDEmzhgjxhi/Dhx7pjFmT3fY1VUYY84xxuR18Ng2r40x5qfGmBdbCmuM+cgYc2sbcf/BGPNAR+zqyTj7E8aYRcaYTGNMmTHmmt62pyWMMa8aYx51lnvl/enMO+Fh/GXGmIlt7M8xxlzQXekDeP0x626MMTnASKDebfMUEcnvRJznAK+JyLjOWde7iMgqYGpv29FbiMgv29h3acOyMeY24C4RWey2/5vdYE+Xx9mS7V0c/5eB7wGzgY0ick4nonsEeE5EnukK27qbgfr+iEhYw7Ix5lUgT0R+1pM29NUcyZUiEuY2dVhEuoKO5B4GKnot+j1FwNPAY10Q13hgZxfEo3SAPvUuikifmoAc4IJW9p0OrAWOA9uBc9z23Q7sAkqBbOAbzvZQoBJwAWXONAZ4FXjU7fhzsErubsdPgFSgGpt7GwP8AzgG7APuaeM8goHfAvuBEmC1sy0OEOBW4ABQAPyv23GB2Bc935meBgJbsTEG+KdjTyH277Bh3x3O9SgGPgbGt2Jngz1LnPQOAT9w2/8w8DbwGnACuMsTG4GfOueWA3zFLb7Lga1OXLnAw17a8lqzsH7O+grHtulAFTZHWwYcd/Y3v99XANuwz9JaIMlt30+Ag9hnaQ9wfivXrjFOt/P+AXDUsf32Np6P27DPaSn2WfpKG7YHAk9gn5cjwB+AYE+udxvp3wWs8CDc14EsrAC9B4xxtu/FvlOVjq2BLRzb4nUE5gPrnGt/CHgOCHA7ToBvA5nOsT8HJjnHnADebAjf3vm3dI+aveM/xL7jJcDfgSC3/T927Mt3rpcAk1u5Ti1+f1pJdy72HSgF3nLSfbS9a+52bb7jXJt9btsmY9+bWqDGuSfvt3eebtfvxzQ9t9cAlwEZjg0/bfc58fQD31MTrQgJMBb7sbwMm5O60FmPdvtATQIMcDZQAcxt6Ua28mFp6SHbhv1YBztpbgYeBAKAic4Dc3Er5/E89uM2FvAFzsB+EOKcG/8nJ95ZWKGa7hz3CLAeGAFEYz9yP29uoxPnduAprFgGAYudfdc4D+J0rAD+DFjbip0N9ixz4pmJFaYLnP0POw/nNc41CPbAxjrgSed8zwbKgalu+2c6cSVhP4zXeGFLm0LiLN8GrG7tfmNf5KPAAuc63urc70Bs0UcuTR/MOGBSK9fOPc6G834E8Mc+pxXA8BaOC8V+EBuuyWggoQ3bn8Z+UCKAcOB94FeeXO823rN2hQQ4D/txnuvE/TtgZXvvqrOv1esInIb9KfRztu8Cvud2rDjnOwRIwL4fn2PfuaFAOnCrh89b83vU/B3fiP1BjHDs+Kaz7xLgsJN+CPBX2hYSj74/2G/HfuBe5zm5Dvvhf9TDay7Ap469wW7bJrf0XfPgPBuu34OOPV/HvnOvY5+1BOzPzcQ2n5Wu+Ph35eScdBn2b+U48K6z/SfAX5uF/bjhgWohnneBe1t6gFq64K08ZHe4rS8ADjSL437glRbS9sH+qc1qYV+cc+PHuW3bCNzkLO8FLnPbdzGQ08IDudC54X4tpPERcGczeypoIVfiZs80t22/Bl5ylh92f5A9tLEOCHXb/ybwQCv36WngKS9s6QoheQFH+Nz278F+ACZjReYCwL+dZ9U9znOce+7ntv8ocHoLx4Vin+3rcT4GbvtOsh37YSrHTcyce7+vI9fbLYwnQvIS8Gu39TDsT0Wc2zvSmpB4cx2/B7zjti7AIrf1zcBP3NZ/Czztyfm3cI+av+P/0+xZ+4Oz/DKOWLudT6tC0sI5tfj9Ac7C5tKMW9jVbja2d80FOK9ZWp4ISWvneQ72ufV11sOd+BY0u/7XtHW+fbWO5BoRGeZMDa1BxgNfMsYcb5iAxdi/OYwxlxpj1htjipx9lwFRnbQj1215PDCmWfo/xTYMaE4UNoewt424D7stV2AfGLB/Dfvd9u13tjUnBtgvInUt7BsPPONmZxH2gzS2DXvcz7V5mrnNwrZnY7GIlLe03xizwBiz3BhzzBhTAnyTU+9TW7Z0BeOBHzS7lzHYv+cs7IftYeCoMeYNY4yn6Rc2ux/u97UR59rciD33Q8aYfxtjprUSZzT2j3izm63/cbY30Or17iQn3WcRKcOWArT1HDWEbfU6GmOmGGM+MMYcNsacAH7Jqc/AEbflyhbW3a9rZ86/rffQ/Tls/g6chBffnzHAQXG+0C3E7ck1b9OWVmjtPME+tw2NmyqdeVvX+xT6qpC0RC42RzLMbQoVkceMMYHYuosngJEiMgz4EPvxBKuwzSnHvqANjGohTPObva9Z+uEiclkLxxVgs4OTvDtFwJbHjndbj3W2NScXiG2lwi0XW0brbmuwiKxtI92YNtJsfv3as3G4MSa0lf2vY4stYkRkKLa833AybdniCS3db3dygV80uz4hIrIMQEReF9tqarwT1+Nept++gSIfi8iF2B+h3diizpZsL8C+yAlutg4Vt5Y6tH29O8NJ99lJIxL7R90ubVzHF7DnHC8iQ7A/ZM2fAW/ojvM/BLi38oxpLaAH35/m8Y41xrjvc4/bk2ve1vPd3rPfLfQnIXkNuNIYc7ExxtcYE+S0zx6HLXcMxBb11BljLgUucjv2CBBpjBnqtm0bcJkxJsIYMwr799QWG4ETxpifGGOCHRsSjTHzmgcUERc2a/ykMWaME3ah88C1xzLgZ8aYaGNMFLbs8rVW7DkEPGaMCXWuxyJn3x+A+40xCQDGmKHGmC+1k+4DxpgQ55jbsRVynbHx/4wxAcaYM7EV228528OBIhGpMsbMB27ppC0tcQQYZ4wJaGX/n4BvOrkj41y/y40x4caYqcaY85x7VYX9iNe3Ek+HMMaMNMZc5XwkqrFFuQ1pnGS78yz9CXjKGDPCOX6sMebiZtG2dr2bp+1rjAnC1k/4OM+Nfyumvg7cboyZ7VyPXwIbRCTHg3Ns6zqGY+uIypyc2Lfai88DPDp/L3gTe+7TjTEh2Ge8Ndr7/rizDnsd7jbG+BljrsY2Pmigw9fc4Qi2LqlH6TdCIiK5wNXYv5dj2L/KHwE+IlIK3IO9+cXYj9N7bsfuxn78sp3igTHYyrPt2PLDT2jnY+Vk/a7Etr/fh/1TfBFb+dcSPwTSgE3YoqXH8ex6PwqkYFtYpAFbnG2t2TMZ25onD1tcgoi846T3hlN0sAO4tHkczfgvtoL+c+AJEfmkEzYext6HfOBv2Iq93c6+bwOPGGNKsS/nm520pSW+wDZLPWyMKWi+U0RSsJWKzzl2ZmHrJsB+EB7D3t/D2AYFP/Uy/fbwwbbuysc+G2djr0trtv/EsXG9cz8/4+T+EG1d7+Z8FftRfwE401n+U0sBReRz4AHs3/YhbA77Jg/Psa3r+EPsO1rqpO3tj0JzvDl/jxCRj4BngeXYa7/O2VXdQtg2vz/NwtZgK9jvxNaT/Q/wQUO8nbzmYOtYZjjfuXe9OK5TmJOL6pTBhjEmDiuM/q3Utyh9GDNAOtt2lJ46f2PMdOwPWWBXvyfGmA3Yyu9XujLenqTf5EgURVF6EmPMtU5x2XBsDv/9rhARY8zZxphRTtHWrdhm8P/pbLy9iQqJoihKy3wDW4y+F1uv0RV1OWCLJbdjOwf+ALhBRA51Udy9Qo8WbRljLgGewXYCe1FEHmu2Pxb4MzDMCXOfiHzYYwYqiqIoXtNjQmKM8cV2ub8QWzG8CbhZRNLdwiwFtorIC8aYGcCHIhLXIwYqiqIoHaInnX7NB7JEJBvAGPMGthVWulsYwbpFANsaqt224FFRURIXF9e1liqKogxwNm/eXCAi0e2HbJ+eFJKxnNwjMw/rdsSdh4FPjDHfxbqRaNGHvjFmCdZBGbGxsaSkpHS5sYqiKAMZY8z+9kN5Rk9WtrfUy7N5udrNwKtOU77LgL8aY06xUUSWikiyiCRHR3eJoCqKoigdpCeFJI+TXQGM49SiqztxOqiJyDqsv6rO+stSFEVRupGeFJJNQLwxZoLj/uEmTu39eQA4Hxo7AAVhm98piqIofZQeqyMRkTpjzN1Y1+++wMsistMY8wiQIiLvYdtU/8kY831ssddt0oFmZbW1teTl5VFVVdWVpzAoCAoKYty4cfj7t+Z+SVEU5WT6vYuU5ORkaV7Zvm/fPsLDw4mMjORkJ5tKW4gIhYWFlJaWMmHChN42R1GUbsQYs1lEkrsirgHZs72qqkpFpAMYY4iMjNScnKIoXjEghQRQEekget0URfGWASskiqIoAxERIeNIKU9/lsHuwyd62xygZzskDirCwsIoKysjJyeH6dOnM23aNKqqqggPD+c73/kOt956a2+bqChKP0FE2HWolI92HOLDtEPsPVaOMRAZFsi0UUPaj6CbUSHpASZNmsTWrVsByM7O5rrrrsPlcnH77bf3smWKovRVGsTj32n5fJh2mH0F5fgYWDgpktsXTeCihJGMCA/qbTMBFZIeZ+LEiTz55JP84Ac/UCFRFOUU9hwu5YPUfP6deojsgnJ8fQwLJ0by9TMncnHCSCLDPBmxu2cZ8ELyf+/vJD2/a8sRZ4wZwkNXJnT4+Llz57J7d6dGAlUUZQCx91gZH2w/xAep+WQeLWvMedzVh8XDnQEvJH2R/t53R1GUzpNXXMEHqYd4f3s+O/NPYAzMi4vg51cncEniaKLD+7Z4uDPghaQzOYfuYuvWrUyfPr23zVAUpYcpKKvmw7RDvLctn5T9xQDMjhnGA1fM4PKZoxk1tG/UeXjLgBeSvkZOTg4//OEP+e53v9vbpiiK0gOUV9fxSfph3t2az+qsAupdwpSRYfzo4qlcmTSG2MiQ3jax06iQ9AB79+5lzpw5jc1/v/vd72pFu6IMYGrrXazOLOCdrQf5NP0IlbX1jB0WzJKzJnLVrDFMH937TXa7EhWSbqKsrAyAuLg4Kisre9kaRVG6GxEhNa+Ed7Ye5P3t+RSW1zAsxJ/r5o7lmjljOS12OD4+A9NzhAqJoihKJzh4vJJ3tx7kH1vyyD5WToCfDxdOH8nVs8dwztQRBPgNfAciKiSKoiheUlZdx0dph/jnloOsyy4EYP6ECJacOZFLZ45maPDgGoZBhURRFMUDXC5hfXYhb2/O46Mdh6msrScuMoTvXzCF6+aOJSai/1eadxQVEkVRlDY4UFjB25tz+ceWgxw8Xkl4kB/XzBnLDaeNZW7scPWYjQqJoijKKVTU1PFh2mHeTMll474ijIHFk6P4yaXTuGjGSIL8fXvbxD6FComiKAq21dWWA8d5KyWX97fnU15ji65+dPFUrps7ltFDg3vbxD5LjwqJMeYS4BnsmO0vishjzfY/BZzrrIYAI0RkWE/a2FW4u5Ffu3Ytt9xyS2+bpChKCxSUVfPOloP8PSWXrKNlhAT4cvnM0XwpOYZ5cVp05Qk9JiTGGF/geeBCIA/YZIx5T0TSG8KIyPfdwn8XmNNT9nUXOTk5vP766yokitKHqHcJq7MK+PumA3yafoTaemFu7DAev34mlyeNISxQC2u8oSev1nwgS0SyAYwxbwBXA+mthL8ZeKiHbOs27rvvPnbt2sXs2bO59dZbueeee7jvvvtYsWIF1dXVfOc73+Eb3/gGK1as4KGHHmLkyJFs27aN6667jpkzZ/LMM89QWVnJu+++y6RJk3r7dBSlX3OopJI3N+XxZkouB49XMjzEn68tjOOmeTHEjwzvbfP6LT0pJGOBXLf1PGBBSwGNMeOBCcAXrexfAiwBiI2NbTvVj+6Dw2neW9sWo2bCpY+1Hw547LHHeOKJJ/jggw8AWLp0KUOHDmXTpk1UV1ezaNEiLrroIgC2b9/Orl27iIiIYOLEidx1111s3LiRZ555ht/97nc8/fTTXXseijIIqKt3sWLPMZZtPMDyPUdxia04v/+yaVw4YySBflpx3ll6UkhaKmhszZ/6TcDbIlLf0k4RWQosBUhOTu5XPtk/+eQTUlNTefvttwEoKSkhMzOTgIAA5s2bx+jRowE7qmKDwMycOZPly5f3ms2K0h85VFLJGxtzeTMll0MlVUSHB/KtcyZxY3LsgHCU2JfoSSHJA2Lc1scB+a2EvQn4Tpek6mHOoacQEX73u99x8cUXn7R9xYoVBAY2jT/g4+PTuO7j40NdXV2P2qko/ZF6l7Ay4xh/23CAL3YfQYAz46N56MoZnD99JP6+A99dSW/Qk0KyCYg3xkwADmLF4pQaaGPMVGA4sK4Hbes2wsPDKS0tbVy/+OKLeeGFFzjvvPPw9/cnIyODsWPH9qKFitL/OVZazZspuSzbeIC84kqiwgL45tmTuHl+7KDucd5T9JiQiEidMeZu4GNs89+XRWSnMeYRIEVE3nOC3gy8IQNkGMGkpCT8/PyYNWsWt912G/feey85OTnMnTsXESE6Opp33323t81UlH6HiLBxXxGvbTjAf3YcorZeWDgxkvsuncZFM0YNCmeJfQXT37/XycnJkpKSctK2Xbt26QiEnUCvn9KXKauu450tefx1/X4yjpQxJMiPG06L4ZYFsUweEdbb5vUbjDGbRSS5K+LSxtKKovQLMo6U8td1+/nnljzKa+qZOXYov74+iStnjSE4QFte9SYqJIqi9Fnq6l18mn6EP6/LYX12EQF+PlyZNIavLhzP7Jh+6fRiQKJCoihKn6OwrJplGw/wtw0HOFRSxdhhwfzkkmncOC+GiNCA3jZPaYYKiaIofYa0vBJeXZvD+9vzqal3sXhyFP93VQLnTx+J7wAdpnYgoEKiKEqvUlvv4uOdh3l1TQ4p+4sJCfDlxnkx3HrGeCaPULcl/QEVEkVReoWi8hqWbTzAX9ft5/CJKsZHhvDgFTO4IXkcQ4IG11C1/R1taN2NvPPOOxhj2L17N9DkCbiBbdu28eGHH7Z6fEpKCvfccw8ADz/8ME888YRX6T/99NNUVFR0wHJF6T72HC7lvn+ksvBXn/Obj/cweUQYL92azPIfnMMdiyeoiPRDVEi6kWXLlrF48WLeeOMNwDshqaurIzk5mWeffbbD6auQKH0Fl0v4YvcRvvLiei5+eiXvbD3IdXPH8cn3z+K1uxZw/vSR+GgdSL9Fi7a6ibKyMtasWcPy5cu56qqrePjhh09yKX/zzTfz/PPPU1lZyerVq7n//vvZtWsX+fn55OTkEBUVxZIlS07yHLx9+3bOO+88cnNz+fGPf8zXv/51VqxYcVKYu+++m+TkZE6cOEF+fj7nnnsuUVFRLF++nE8++YSHHnqI6upqJk2axCuvvEJYmHbgUrqPipo6/rE5j1fW5JBdUM6oIUH8+JKp3DwvluHa+mrAMOCF5PGNj7O7aHeXxjktYho/mf+TNsO8++67XHLJJUyZMoWIiAi2bNlyikv5kSNHkpKSwnPPPQfY4qvNmzezevVqgoODWbFixUlxpqamsn79esrLy5kzZw6XX355q+nfc889PPnkkyxfvpyoqCgKCgp49NFH+eyzzwgNDeXxxx/nySef5MEHH+zcxVCUFjhcUsVf1uXwtw0HKKmsZda4oTxz02wumzlaHScOQAa8kPQWy5Yt43vf+x4AN910E8uWLWvzw9/AVVddRXBwy2NDX3311QQHBxMcHMy5557Lxo0bGTbMs05Z69evJz09nUWLFgFQU1PDwoULPTwbRfGMnfklvLRqH++n5lPvEi5OGMWdiydw2ngdsnYgM+CFpL2cQ3dQWFjIF198wY4dOzDGUF9fjzGGyy67rN1jQ0NDW93X/EU0xuDn54fL5WrcVlVV1eKxIsKFF17IsmXLPDwLRfEMl0v4b8Yxlq7MZl12IaEBvvzP6eO5Y9EE9bw7SNA8Zjfw9ttv87WvfY39+/eTk5NDbm4uEyZMwMfH5ySX8s1dzLfHv/71L6qqqigsLGTFihXMmzeP8ePHk56eTnV1NSUlJXz++ectxn/66aezZs0asrKyAKioqCAjI6OLzlgZjFTV1vP3TQe4+OmV3P7qJnIKy7n/0mmsvf98HroyQUVkEDHgcyS9wbJly7jvvvtO2nb99dfzxhtvnORS/tZbb+Wxxx5j9uzZ3H///e3GO3/+fC6//HIOHDjAAw88wJgxYwD48pe/TFJSEvHx8cyZM6cx/JIlS7j00ksZPXo0y5cv59VXX+Xmm2+muroagEcffZQpU6Z04Zkrg4HjFTW8tn4/r67dT0FZNTNGD+HpG2dzeZLWfwxW1I28cgp6/ZSWyC2q4KXV+/j7plwqa+s5e0o0S86ayBmTIrX+ox+ibuQVRekxdhwsYenKbP6ddggfA1fOGsOSsyYybdSQ3jZN6SOokCiKcgoiwuqsAv7432xWZxUQFujHnYsncPuiOEYPbblVoTJ4USFRFKWRunoXH+04zB/+u5ed+ScYER7IfZdO45YFseq6RGmVHhUSY8wlwDPYMdtfFJHHWgjzZeBhQIDtInJLT9qoKIORqtp63t6cx9KV2RwoqmBidCiPXz+Ta+aMJdBPRx9U2qbHhMQY4ws8D1wI5AGbjDHviUi6W5h44H5gkYgUG2NG9JR9ijIYKams5bX1+3llzT4KymqYFTOMn142nYtmqO8rxXN6MkcyH8gSkWwAY8wbwNVAuluYrwPPi0gxgIgc7UH7FGXQcLS0ipdW7+Nv6w9QVl3H2VOi+ebZkzh9YoS2wFK8xmshMcaEAlUiUu/loWOBXLf1PGBBszBTnDTWYIu/HhaR/7RgwxJgCUBsbKyXZijK4OVAYQV/XLmXtzbnUVfv4rKZo/nm2ZNIHDu0t01T+jHtCokxxge4CfgKMA+oBgKNMceAD4GlIpLpQVot/eY078TiB8QD5wDjgFXGmEQROX7SQSJLgaVg+5F4kLaiDGr2HC7lhRVZvJ96CF9juP60sXzjrEnERbXukkdRPMWTHMly4DNs3cUOEXEBGGMigHOBx4wx74jIa+3EkwfEuK2PA/JbCLNeRGqBfcaYPVhh2eSBnX2GBx54gKioKO69914A/vd//5eRI0c2DlKlKD3FttzjPPdFFp/tOkJIgC93LIrjrjMnMnJIUG+bpgwgPBGSC5wP+0mISBHwD+AfxhhP2gVuAuKNMROAg9hcTvMWWe8CNwOvGmOisEVd2R7E3SqHf/lLqnd1rRv5wOnTGPXTn7a6/8477+S6667j3nvvxeVy8cYbb7Bx48YutUFRWkNEWJddyPPLs1iTVcjQYH++d0E8ty6M0zFAlG6hXSFpSUQ6GKbOGHM38DG2/uNlEdlpjHkESBGR95x9Fxlj0oF64EciUthe3H2NuLg4IiMj2bp1K0eOHGHOnDlERkb2tlnKAEdE+GL3UZ5bnsXWA8eJDg/kp5dN45YF4wkL1C5jSvfR6afLGPMTEXnck7Ai8iG2XsV924NuywL8P2fqEtrKOXQnd911F6+++iqHDx/mjjvu6BUblMGByyX8Z+dhnvsii/RDJxg7LJifX5PIl04bR5C/9gFRup+OtNp6030VmA14JCSDiWuvvZYHH3yQ2trak8ZpV5Suoq7exfup+Ty/fC9ZR8uYGBXKE1+axdWzx6gXXqVH6UiO5ISI3NWwYox5oQvtGTAEBARw7rnnMmzYMHx99a9Q6Tpq6ly8szWP36/Yy/7CCqaNCud3N8/hspmj8dVOhEov4LGQGGNOE5HNwC+a7frfrjVpYOByuVi/fj1vvfVWb5uiDBCqaut5KyWXP/w3m4PHK5k5dihLv3oaF0zXXuhK7+JNjuRWY8z9wBPAPgBjzJMi0mX1GQOF9PR0rrjiCq699lri4+N72xyln1NZU8+yjQf448q9HDlRzdzYYTx6bSLnTInWXuhKn8AbITkKXAX80xhTCgQA67rFqn7OjBkzyM7uVKtlRaGipo7X1u9n6cp9FJRVM39CBE9+ebYOJKX0ObwRkv8BpopItTFmDPArYGv3mNV5RERftg7Q30fMHAiUVdfxl3U5vLhqH0XlNSyeHMV3z5vDgonahFzpm3gjJLnABGC3iORji7p2AU93i2WdICgoiMLCQiIj9c/NG0SEwsJCgoK013NvUFpVy1/W7edPq7I5XlHL2VOiuef8yZw2PqK3TVOUNvFGSO7F9mLfAmzBujgp7xarOsm4cePIy8vj2LFjvW1KvyMoKIhx48b1thmDihNVtby6JoeXVu+jpLKW86aN4J7z45kdM6y3TVMUj/BYSEQk3RgzF7gAmAMcxrqB73P4+/szYcKE3jZDUdqkpLKWV9bs4+XV+zhRVccF062AJI1TAVH6F171IxGRauDfzqQoSgdoEJCXVu+jtKqOC2eM5N7z49WVu9Jv8cSNvKcDfhwXkROdtEdRBizNBeSiGSO594J4EsaogCj9G09yJH/GjhvSVq21AK8Cf+kCmxRlQNFcQC5OGMk956uAKAMHT7z/ntsThijKQEMFRBkseOMixWBHSZwoIo84RV6jREQH2lAUN05U1fLK6hxeWp3NCS3CUgYB3lS2/x5wAecBjwCl2IGt5nWDXYrS7yh1mvG+6DTjvXDGSL6nAqIMArwRkgUiMtcYsxVARIqNMTrcmjLoKauu489rcxo7El4wfQTfu2CKtsJSBg3eCEmtMcYXW7GOMSYam0NRlEFJRU0df1m3nz/+dy/FFbWcOzWa7184RfuBKIMOb4TkWeAdYIQx5hfADcDPusUqRenDVNbU87cN+3lhxV4Ky2s4a0o0378gnjmxw3vbNEXpFTwSEqeifSWwGTgf2xT4GhHZ5U1ixphLgGewY7a/KCKPNdt/G/Ab4KCz6TkRedGbNBSlu6iqtTKLnXkAACAASURBVO7cf79iL8dKq1k8OYrvXxivvrCUQY9HQiIiYox5V0ROA3Z3JCGnWOx54EIgD9hkjHlPRNKbBf27iNzdkTQUpTuorqvnzU25PL98L4dPVLFgQgTP3azeeBWlAW+KttYbY+aJyKYOpjUfyBKRbABjzBtYX13NhURR+gS19S7e3pzHc19kcfB4Jcnjh/PkjbM4Y1JUb5umKH0Kb4TkXOCbxpgcrNdfg82sJHl4/FisK/oG8oAFLYS73hhzFpABfF9EcpsHMMYsAZYAxMZ66sFFUTyjrt7FO1sP8uwXmeQWVTIrZhi/um4mZ8ZH6bAEitIC3gjJpZ1Mq6U3sPkoSu8Dy5zBs76Jdc9y3ikHiSwFlgIkJyfrSExKl1DvEt7fns8zn2eyr6CcmWOH8shtiZwzVYe0VZS28GrM9la2P+Lh8XlAjNv6OCDfPYCIFLqt/gl43GPrFKWDuFzCRzsO8/RnGWQeLWPaqHD++NXTuGjGSBUQRfEAb4TEfRCrIOAKwJtWW5uAeGPMBGyrrJuAW9wDGGNGi8ghZ/UqL+NXFK8QET5NP8KTn2aw+3Apk0eE8fwtc7k0cRQ+PiogiuIp3gxs9Vv3dWPME8B7XhxfZ4y5G/gY2/z3ZRHZaYx5BEgRkfeAe4wxVwF1QBFwm6fxK4qniAgrMo7x1KcZpOaVEBcZwtM3zubKWWPwVQFRFK8xIh2rYjDGDAc2ikh815rkHcnJyZKSktKbJij9iLVZBfz20ww27y9m7LBg7j0/nuvmjsXP16e3TVOUHsUYs1lEkrsiLm+8/6bRVDnuC0QDP+8KIxSlu9mUU8RvP9nD+uwiRg0J4tFrEvlycgwBfiogitJZvKkjucJtuQ44IiJ1XWyPonQp23OP89tPM1iZcYyosEAeunIGN8+PJcjft7dNU5QBgzdC8m0R+Yn7BmPM4823KUpfID3/BE9+msFnu44wPMSf+y+dxtcWxhEcoAKiKF2NN0JyIdBcNC5tYZui9BqZR0p5+rNM/p12iCFBfvzwoinctmgCYYHePOqKonhDu2+XMeZbwLeBicaYVLdd4cDa7jJMUbwhp6CcZz7P5N1tBwnx9+We8yZz55kTGRrs39umKcqAx5PftNeBj4BfAfe5bS8VkaJusUpRPCS3qILffZHJP7YcxN/XsOTMiXzj7ElEhOqYa4rSU7QrJCJSApQANztNfuOxHRIxxiAiK7vXREU5lcMlVTy3PJO/b8rFYPjawvF865xJjAgP6m3TFGXQ4U3z37uAe7GuTbYBpwPraMEXlqJ0F8dKq3lhxV5e27Afl0u4cV4Md583mdFDg3vbNEUZtHhTA3kvMA9YLyLnGmOmAf/XPWYpyskUl9fwx5XZ/HltDjX1Lq6bM5Z7zo8nJiKkt01TlEGPN0JSJSJVxhiMMYEistsYM7XbLFMUoKSylpdWZfPymhzKa+q4atYY7j0/nonRYb1tmqIoDt4ISZ4xZhjwLvCpMaaYZt57FaWrKKuu49U1+1i6MpsTVXVcmjiK7184hSkjw3vbNEVRmuGN08ZrncWHjTHLgaHAf7rFKmXQUllTz1/W5fDHldkUlddw/rQRfP/CKSSOHdrbpimK0god6qUlIv/takOUwU1VbT3LNh7g9yv2cqy0mjPjo/h/F05hTuzw3jZNUZR28KbVlgG+AkwUkUeMMbHAKBHZ2G3WKQOemjoXb6bk8twXWRw+UcXpEyN4/pa5zJ8Q0dumKYriId7kSH4PuLDNfR8BSoF/YFtyKYpX1Na7+OeWPJ79PIuDxys5bfxwnvzyLM6YHNXbpimK4iXeCMkCEZlrjNkKICLFxhjtPqx4Rb1L+Ne2gzz7eSY5hRUkjRvKL65N5OwpOi66ovRXvBGSWmOML86YJMaYaGwORVHaxeUS/p12iKc/y2DvsXKmjx7Cn76WzAXTR6iAKEo/xxsheRZ4BxhhjPkFcAPws26xShkwuFzCxzsP8/Rnmew5UsqUkWG88JW5XJyg46IrykDBE++/sSJyQET+ZozZDJwPGOAaEdnlTWLGmEuAZ7AjLL4oIo+1Eu4G4C1gnojoOLr9EBHhs11HeerTDNIPnWBidCjP3DSbK5J0XHRFGWh4kiN5F5jrLP9CRK7vSEJOsdjz2HFN8oBNxpj3RCS9Wbhw4B5gQ0fSUXoXEWHFnmM8+WkGaQdLGB8ZwpNfnsVVs8bouOiKMkDxREjcfx8ndiKt+UCWiGQDGGPeAK4G0puF+znwa+CHnUhL6WFEhJWZBTz1aQbbco8TExHMb25I4to5Y1VAFGWA44mQSCvL3jIWyHVbzwMWuAcwxswBYkTkA2NMq0JijFkCLAGIjY3thElKZxER1mQV8tRnGWzeX8zYYcH86rqZ3HDaOPxVQBRlUOCJkMwyxpzA5kyCnWWcdRGRIR6m1VLBeKMwGWN8gKeA29qLSESWAksBkpOTOyNuSidYu9fmQDblFDN6aBCPXpPIl5NjCPBTAVGUwYQnA1v5dlFaeUCM2/o4Tnb6GA4kAiuc5qCjgPeMMVdphXvfYn12IU99msGGfUWMHBLII1cncOO8GAL9uupRURSlP9EhX1sdZBMQb4yZABwEbgJuadjpjMTY2K3ZGLMC+KGKSN9hQ7YtwlqfXcSI8EAeunIGN8+PJchfBURRBjM9JiQiUmeMuRv4GNv892UR2WmMeQRIEZH3esoWxTs27iviqU8zWJddSHR4IA9eMYNbFqiAKIpi6ckcCSLyIfBhs20PthL2nJ6wSWmdjfuKePqzDNbuLSQqLJAHrpjBLfNjCQ5QAVEUpYkeFRKlf9BcQH52+XS+smC8CoiiKC2ibuSVRtZnF/LMZ5msy9YciKIonqNu5Ac5IsI6R0A27CsiOlwFRFEU71A38oOUho6Ez36eycYc2wpLK9EVRekI6kZ+kNHgyuTZzzPZvL+YUUOC+L+rbD8QFRBFUTqCupEfJIgIX+w+yrNfZLE99zhjhgbx82sS+dJp41RAFEXpFB4LSTM38gBXi8ju7jFL6SpcLuHTXUd49vNMduafYNzwYH55rfWFpa5MFEXpCrxptfUl4D8i8rwx5gHgl8aYR0VkS/eZp3SUepfwYdohnl+exe7DpcRFhvBrxxuvOlNUFKUr8aZo6wERecsYsxg7pshvgRdo5sFX6V1q6128ty2f51dkkX2snEnRoTx14yyuTNLxQBRF6R68EZJ6Z3458AcR+Zcx5uGuN0npCNV19fxj80Fe+G8WuUWVTBsVzvO3zOWSxFE6IqGiKN2KN0Jy0BjzR+AC4HFjTCCgv7i9TGVNPcs2HmDpymwOn6hiVswwHroigfOnj8DxoqwoitKteCMkXwYuAZ4QkePGmNHAj7rHLKU9TlTV8td1+3l59T4Ky2uYPyGCX9+QxJnxUSogiqL0KN602qowxuwFLjbGXAysEpFPus80pSUKy6p5ZU0Of16XQ2lVHWdNiebucyczf0JEb5umKMogxZtWW/cCXwf+6Wx6zRizVER+1y2WKSeRf7ySP63KZtnGA1TXubgkYRTfPmcyM8cN7W3TFEUZ5HhTtHUn1k1KOYAx5nFgHaBC0o1kHyvjD//dyztbD+ISuGb2WL51zkQmjwjvbdMURVEA74TE0NRyC2dZC+O7iR0HS/j9iiw+2nGYAF8fbp4fy9fPnEhMREhvm6YoinIS3gjJK8AGY8w7WAG5GnipW6wapIgI6/YW8sJ/97Iqs4DwID++fc4kbl80gaiwwN42T1EUpUW8qWx/0hlHfbGz6XYR2epNYsaYS4BnsEPtvigijzXb/03gO9jcThmwRETSvUmjP1LvEj7ZeZg//Hcv2/NKiAoL5L5Lp3HLgliGBPn3tnmKoiht4u0IifVY77+Cl55/Hc/Bz2N7xecBm4wx7zUTitdF5A9O+KuAJ7FNjgckVbX1vLP1IEtXZrOvoJzxkSH84tpErp+rjhQVRek/dKTV1j+wRVvettqaD2SJSLYT3xvY4rFGIRGRE27hQ3Fc1g80Sipr+duG/byyJodjpdXMHDtUe6EritJv6clWW2OBXLf1PFrw02WM+Q7w/4AA7GiMA4b845W8vHofyzYeoLymnrOmRPPMjRNZOClSOxEqitJv6clWWy2FPSXHISLPA88bY27Bjndy6ykRGbMEWAIQGxvrhQm9Q3r+CV5clc172/MR4Iqk0Sw5ayIJY7QPiKIo/Z/Ottp62Yvj84AYt/VxQH4b4d/Aehc+BRFZCiwFSE5O7pPFXyLCqswC/rQqm1WZBYQE+PK1hXHcsTiOccO1Ca+iKAOHnmy1tQmIN8ZMAA4CNwG3uAcwxsSLSKazejmQST+jps7Fe9vzeXFVNrsPlxIdHsiPL5nKV+aPZ2iItsBSFGXg4U1leyAwDQhzjrvSGHOliDziyfEiUmeMuRv4GNv892UR2WmMeQRIEZH3gLuNMRcAtUAxLRRr9VWKy2t4feMB/rw2h6Ol1UwdGc5vbkjiqtljCPTTFliKogxcvCna+hdQAmwGqjuSmIh8CHzYbNuDbsv3diTe3iT7WBmvrMnhrc25VNW6ODM+it98aRZnqRdeRVEGCd4IyTgRGbB9OrxBRFiXXchLq/bx+e6jBPj6cPXsMdx55gSmjRrS2+YpiqL0KN4IyVpjzEwRSes2a/o4VbX1vLc9n5dX72P34VIiQwO45/x4vnr6eKLD1YWJoiiDk3aFxBiThm2m6wfcbozJxhZtGUBEJKl7Tex9jpZW8dr6A7y+YT8FZTVMHRnOY9fN5Jo5Y7UHuqIogx5PciRXdLsVfZTtucd5dW0OH6TmU+cSzps6gjsWT+AM7UCoKIrSiCdC4g+MFJE17huNMWfSdj+QfklNnYv/7DzMq2v2seXAcUIDfPnKgvHcdkYccVGhvW2eoihKn8MTIXka+GkL2yudfVd2qUW9xNHSKl7fcIDXNxzgaGk1E6JCefjKGVx/2jjCO+uB1+WCukqoq4baSqirsst1VVBfY5fra9ym2qZlV71dd9U5U72dSz2Iy66LCxAQscvSQh9NYwBj58aA8QHja+c+vnbZxxd8/Jy5P/j6OfMA8PV3pgDwCwTfQDv3CwS/ILd5EPgH27CKogwKPBGSOBFJbb5RRFKMMXFdblEPIiJsOVDMn9fu56Mdh6itF86ZEsVvrp7MmTEB+NSUQsF2qD4B1aXOVAY1DfNyqClzpnKoqYDacisWNRVQW2GX6zvUWrodjCMAPk0TDQLhiEbTmTri4jZ31TeJkXjlyNkzfPzBP8SKin8wBITa9YAQCAiz6wGhznIYBIZDoLMcNAQCnalhOSDUOS9FUfoanghJUBv7grvKkJ7m3d9+m7rVqzkxyofzRgg/Dq9lNBX45p6AA/XtR+Ab0PQRDAhp+lAGj236YPoFN839g+x2v0C77hfQ9CfvG+D84Qc4y04OwMfJBTTmFPybcgxd+VFtyMk05HZctW45oYbcUZ0zr4a6hrn7VNU01Va6TRVN85pyOy8vsKJcU26nusr2bTS+VlSChkHwMAga2rQcPLzZFAEhEU1zzR0pSrfiiZBsMsZ8XUT+5L7RGHMntnNiv6TmSD4T99USugtcBjaNhf2Th1E3PZ7RE8aTMCyOqUMnERwc6fwVhzf9OQeE2Y/+QMGYpqIteuG86utsrq661Mn9NSyXQNUJqCqxU7WzXHkcqo5DyUGoLLbLrrrW4w8cagUlJBJCoyAkCkIjnXm0ncKim5ZVeBTFK4y0VJ7uHsCYkcA7QA1NwpGM/eJcKyKHu9XCdkhOTpaUlBSvj3O5bPHOoU0ryf/sA1zrNjMk+ygAxaGwfaJh2yQfTsyayKSYJBKiEkiITGBqxFQCfbXPSJ9CxApR5XGoLIKKoqZ5w3J5AVQUQkUBlDvz+pqW4wseDqEjIGwEhI10m4+E8JEQNgrCR9lwWtym9FOMMZtFJLlL4mpPSNwSPRdIdFZ3isgXXWFAZ+mokLREXUEBZatXU/jFp1StW49PaQUuA9kx/myKq2fbREPeaD8mRcSTEJlAQlQCiZGJTB4+GX8f/YvtV4jYXE/5MSsy5ceg/CiUNcyP2m1lR6D0iK37ao5voBWW8NFWWMJHN01DxtgpfLQt3lSUPkaPCokxxkg7gTwJ0110pZC4I3V1VKamUbZqJeUrV1G1cycA1UOD2Ts1nFUx5ayPqaI82BDgE8C0iGlWWKISSYxKJG5IHD7Gp8vtUnqJ6jJHVA5D2WErLqWHnG2H4MQhu6+m9NRjg4fDkLFN4jJknJ0PHWenIWNsgwRF6UF6WkhWYIfX/ZeIHHDbHoB1KX8rsFxEXu0Kg7ylu4SkOQ25lfKVqyhfs4b6khLw8aFqaiz7Z0Swfnw1y0P2U1FfBUCYfxgzImeQGJXIzKiZJEYlMjJkpHZkHOhUlzqicghO5ENpvp2fyIcTB229TkXBqceFRDUJy9AYGBbjthxr63f02VG6kJ4WkiDgDuArwATgOLa1lg/wCfC8iGzrCmM6Qk8JiTtSX09lairlq1ZTtmoVVTt2gAi+w4fjmp/EwYQRbI5zsbkmk4ziDOqciuCo4KiThCUxKpEhAerkcdBRW2UFpiTPCktJHpTkOkKTB8dzTy1K8w9pEpWTpvEwfLwKjeI1PV5HYozxAe4Hfg1EAZUicrwrDOgsvSEkzakrKqJ8zRrKVq2ifPUa6ouKwBiCEhIIWnwGBUnjSB1Rxc7iXaQVpJFzIqfx2LghcY3CkhSdxJThUwjwHUAtwhTvEbGt0Upyrag0zI/vd5YP2P3u+IdaYRk+3hGXOLs8PM6uB4b1xpkofZjeqmz/XETO74pEu5K+ICTuiMtF1c50ylevomzlKiq3bweXC58hQwg94wzCzlyMzJ/Fbt9j7CjYQWpBKjsKdlBQaYs7/H38mR4x3eZcomeSFJVETHiMFokpJ1N1wopK8X4rMMcPNC0X59hWbO6ERjviEgfDJ9h5xAS7HD5KczODkN4Skt8CJ4Cfi3RHV+iO0deEpDn1JSWUr1tH2cpVlK9eTd1R28Q4MD6e0DPPJGzxIoJOO41jdcWkHkslrSCNtII00gvTqXQ66g0NHMrMKCsqM6NnMjNqJkMDh/bmaSl9GRHb7Lk4B47n2HnRviaRKck72ZuBX3CTsERMdJYn2mlojHWVoww4ektI3gRmAsOBDUAqkCoib3WFIR2lrwuJOyJCdUamza2sXk1lymakthYTHEzI/HmELVpM6OLFBEyIo17q2Xt8L6kFqaQds+Ky9/heBHu/4obENRaHJUXZIjF/7UineEJdjZOb2WcFpkFoirLttrqqprA+frZoLGIiRE5yBGaSFZ1h41Vk+jG9IiRuiQcCCVhRSRSRH3lx7CXAM9gx218Ukcea7f9/wF1AHXAMuENE9rcVZ38Skua4Kioo37CB8tVrKF+9mpr99lT9x4wh9MwzCV28iNDTT8c3PByAspoydhbuJK0gjdRjqaQeS6WwqhCAQN9ApkdMJyna5lpmRc1iVOgoLRJTvMPlss2bi/ZB0d4mgWlYdi8yaxCZyElWXBqEJnKybXHmo2P19GV6utXWrcBvsa203gfuFpEWGsu3k5AxvkAGcCGQB2wCbhaRdLcw5wIbRKTCGPMt4BwRubGtePuzkDSnJjeX8tWrKVu9hop163BVVICvL8GzZhG6eBFhixYRlJiI8bUvqIhwqPxQY64l9Vgqu4p2Ue04iYwOjrZFYtFJJEUnkRCZQIi/do5TOoiI7aRZuNcKS+M824qNe0sz34AmUYmc5MydKTRa62T6AD0tJFnAl4CDwHeBYSLyXa8TMmYh8LCIXOys3w8gIr9qJfwc4DkRWdRWvANJSNyR2loqt22jbM0aylevsR0iRfAZOpTQhQsJXXQGYYsW4T9mzEnH1dbXklGcwfZj2xtzLgdKbfcfH+ND/LD4RmFJik7SjpNK1yBiO2Q2CExhlhWXhrm7O5rAIW7iEm+Xo+JtrkZbl/UYPS0kW0Rkrtv6BhFZ4HVCxtwAXCIidznrXwUWiMjdrYR/DjgsIo+2Fe9AFZLm1BUXU752rS0GW7OmsdI+YOJEQhctIvSMhYTOn49P6KmDbxVXFTdW4m8/up0dBTsorbWZyvCA8MZK/KQoKy5aka90Ka56WydTmAUFWXbeMJXkAW7foPAxEOUITFS8M59sK/21qKxL6WkhOQQ8BOwCdgMfuwuLxwkZ8yXg4mZCMr+l3I0x5n+Au4GzReSUwTyMMUuAJQCxsbGn7d/fZjXKgENEqMnKasytVKSkIFVV4O9PiFMMFnrGGQQlJDQWg7njEhc5JTlsP7ad1AJb15J1PAuX05InbkhcYyV+UnQS8cPj8fPRSlWlG6ittDmWgswmcSnIhMJM6+m5Ad/AplxMo8BMsSITpD8+HaGnhWQJkIStXJ8JhAGfAduxrbaWeZSQh0VbxpgLgN9hReRoe/EOlhxJW7iqq6ncutXWr6xdS3X6LgBbDLZgAaFnnEHoojMIiIlpNY7y2nLSC9PZfmy7FZhjqRRVFQEQ7BfMjMgZJEUnMStqFknRSUSHRPfIuSmDFBHrTLMws0lYCrLsvGifHZStgbCRVlQaRCZqip1rLqZNervV1jiahCVRRL7q4XF+2Mr287H1LZuAW0Rkp1uYOcDb2CKwTE/iVSE5lbqiIsrXrbNFYWvXUXfoEAD+MTG2fuWMhYQsWIDf8OGtxiEi5Jfns/1oU11LelF6o7uX0aGjT8q1TI+cru71lZ6hrsY2WS7MhIIMKzAFGXaqcnO44RvYTFwcgYmcrHUx9LKQdCoxYy7DjvPuC7wsIr8wxjwCpIjIe8aYz7ACdcg55ICIXNVWnCokbSMi1OzLsaKybh0VGzbgKiuzLlxmzLB1KwsXEjx3Lj5BbQ2GCdX11ewu2t3Y9Hj7se0cKre3ys/Hr6n5sdNSbFzYOG1+rPQcInbMmYIMm4tpmBdmWuFx74Q5ZOypAhM1xbr9HyTPbL8Vku5AhcQ7pK6OyrS0xhxL5fZUqK3FBAQQPHcuoaefTujC0239il/79SLHKo6RWmBFJe1YGjsLdzb2yI8Iijip+XFiZCJhAfonqPQCddVNdTENuZcCp9jM3fV/QLitd3EXl6gptimz38DKcauQuKFC0jlc5eVUbN5M+Zq1lK9fT/WePQD4hIcTMn9+o7AETJrkUe6izlVH1vGsxlxLakEq+0r2AWAwTBo26aRcy6Shk/DVcmylt2hottxQTHbMTWRO5DWFMz7WdUxzgYmaYodx7oeokLihQtK11BUWUr5+PRXr11O+fgO1ubkA+EZHEbrgdEJPX0DI6acTMG6cx3GWVJews2BnYyuxtII0Sqpti5wQv5BG1/oNTZC1Il/pE1SXNbUiK9jTlIMpzIJ6t8akIZEtCEy87fXfh3+SVEjcUCHpXmryDlKxYT3l69ZTvmE99cccL8VjxhBy+umELphPyIIF+I8a5XGcIsKB0gONuZa0gjT2FO2hTpoq8t2LxKZHTCfIr+36G0XpMVz11gFmQSYc29PUsuzYHqgsagrXxyv7VUjcUCHpOUSEmuxsJ8eygfKNG3GV2JxFwPjxhCxYQMiC+YQuWIBfVJRXcVfVVTVW5De0EssvzwfAz/gRPzz+pFxL3FDtka/0QcoLrbC4C0xBhmeV/ZHxdtjlHqrsVyFxQ4Wk9xCXi+o9eyhfv4GKDRuoSEmxLcKAgEmTCJk/j9D58wmZN89rYQEoqCxo9HycWpDKzoKdlNXa+MP8w0iISiApKqnRC3JUsPdpKEqPUFdt+78UZDjFZFlNdTEnVfaHOR0v3frDNORi/IO71CQVEjdUSPoOUldH1a5dVGywuZXKlM3W8STWlctJwhLtfT2IS1zsK9lH6jE7GFhaQRoZxRnUO53TRoWOsrkWZ8RJdVKp9HmaV/a794kpyXULaGwHy4YWZQ1FZqOSOlzZr0LihgpJ30Xq6qjauZOKTZtOFZYJEwhJTiZk/jxC5s3zqo7Fncq6SnYX7W7MuaQVpHGw7CBgnVROHDqxUVgSoxKJHx6Pv4+O26L0A2oqrBNMd4Fp6OHf4Gn5sidg/tc7FL0KiRsqJP2HxhzLxo1UbNxExebNjUVh/jExVliSTyMkORn/2NgOd2YsqipiZ4Edt6Uh53K82vZ4DvAJYFrEtEZhSYhKUA/ISv9CBE7kW1FpGPulA6iQuKFC0n+R+nqq9+yhYtMmKlJSqNiUQv1x+8H3i44mZF4ywadZYQmMj8f4dOxjLyLkleWxs2Bno7DsKtrV2HEyzD+MGZEzSIhKIDHSCszo0NHaK18Z0KiQuKFCMnAQl4uavXup2LyZipTNVKSkUHf4MGA7SAbPnUPIacmEnDaXoMREfAI73tO43lVPdkk2Owp2sLPQCsye4j2NvsQigiKsuEQmkBCZQGJUovZvUQYUKiRuqJAMXESE2oP5VG5OscKyZQs1e/cCYPz9CZo5k5DT5hI8Zy7Bc2a36YTSE2rqa8gozmBHwQ52FOwgvSidvcf3NrrXHxE8ghlRMxoFZkbkDG0ppvRbVEjcUCEZXNQVF1O5ZQsVW7ZQuXkLlTt3Qm0tYFuGBc+ZTchcKy4BE+I6XTxVUVvB7qLdpBemk16Yzo7CHeSU5CDOYEwjQkY0ikrDpOKi9AdUSNxQIRncuKqqqEpLo2LrNiq3bKFy61bqnU6SvsOGETx7NsFz5tj5zER8QjrfHLi8tpxdhbvYWbizUWD2n9h/krjMiLCiMj1yOjMiZxAdHK11LkqfQoXEDRUSxR1xuajJyXFyLVup3LaNmuxsu9PXl6CpU62ozJ5F8OzZ+MfEdMkHvkFcdhXtIr0wnV2Fu9h3Yl9jsVhEUIQVlQgrLtMjpjM2bKyKi9JrqJC4oUKitEf98eNUbNtG5bZtVG7bTmVqKuL0Z/EdPpzgWbOssMyaRdDMmfiGdY0fpIraCvYU7zlJYLKPZzf6FAv3D2dqxFSmRUxjeuR0pkVM7Df3VwAAE9xJREFUY8LQCdrPRekRVEjcUCFRvEXq66nOyrKism0blampjZX4GEPApIkEJ80iOCmJ4KSZtumxf9d83Kvrq8kqziK9yOZadhftJqM4g2rHm2yATwDxw+OZFjGtUWSmDJ9CqH9ol6SvKA2okLihQqJ0BfUnTlCZmkZl6nYqt2+nantqY58WExhI0IwZBCclEZQ0k+CZM7usSAzsGC45JTnsLt7NnqI97CqyAtPgah8gNjyWqRFTmTp8auN8VOgoLRpTOowKiRsqJEp3ICLU5uVRmZpKVWoqlWk7qEpPR6qqAPAdOpSgxESCZiYSnJhIUGIifiNHdtmHXUQ4UnGE3UVWXPYU72F30W5yS5v8Lw0JGMKU4VOYGjHVzodPZdKwSepyX/GIfiskxphLgGewY7a/KCKPNdt/FnZM9yTgJhF5u704VUiUnkLq6qjOzKQyLY2qtB1UpqVRnZkJ9dZppG9UFEEJMwhOSCQoMYGghET8R47oUhvKa8vJLM60AlO8h4ziDDKLMxt76fsYH2LDY5kyfApThk8hfng88cPjGRs2Vt3AKCfRL4XEGOMLZAAXAnnAJuBmEUl3CxMHDAF+CLynQqL0dVxVVVTv3m1zLDt3UrVzB9V7s8FlW2v5RkcRPCOBoIQZBM2wk9/ornW/4hIXeaV5ZBRnsKd4D5nFmWQUZ5yUewnxC2HysMnED48/aR4ZHNlldij9i/4qJAuBh0XkYmf9fgAR+VULYV8FPlAhUfojrvJyqvbsoWrHTisu/7+9O4+N7a4OOP49nsXbeN+X8fr2pBKkgQKqgCq0okElRUorlohSRY0EKkhtqUTVVSBEW0RbtQLRQCMWsUcBAoIGSkFQSlLeY038Fu9+491+tp/3mbFP//hdz5vn52WeZ/Ey5yNd3XtnfnPnnBnbx3f7/Xp6WO/vv1VcKioouucChefPU3T+AkXnzxHs6EB8mR2WdSW2Qt98X2KvpXe+l9653kQHluAuSz5deZpTVac4Vemm7spuyoJlGY3FHD2ZLCT+TGwkRS1Acgf7EeDXDrIhEXkMeAygra0t/ciMyaCC0lJK7ruPkvvuSzy2ubrK+tWrrF2+zFpPD2s9l5n71KdR7658KS6m6MwZCs+fo+jceYrOnaXwzJm0bqAsCZQkhiveoqrMrM7QO99L31wfvfO99M/381TvU4nDY+BuqjxVeYquiq5Ecemq7KI8WH7geMzJlctCstO+/IF2h1T1ceBxcHsk6QRlTC4UFBd7N0K+KPGYxmKsDwyw1nOZtcs9rF+5ys1vfJP5z3/BNRAh2NZG4blzrrCcPUfR2TP4m5sPfGhMRKgrqaOupI5XNL8i8fimbjK2NEb/fD99832J+ZOTT7K2sZZoV19cnygqXRXeVNlFddHBBlcyJ0MuC0kECCettwJjOXx/Y44UCQQoOnuWorNn4Q2/C9zqqHL96hXWrlxh/coV1np6WHzmmcTrCkIhCs+epfDMabcXc+YMhadP4ys/+N5CgRTQWtZKa1krrwq/KvH4xuYGY8uuwPTP9zOwMEDffN8dezCVhZV0VXTRWdFJZ0VnYrmptAlfQWYP2ZmjJ5fnSPy4k+0PAKO4k+1vVtUXdmj7CewciTEJG0vLrF+75k1XWbtylfVr1xIDgwH4GxspPHOawtNJU1cXBcWZHesb3B7M5PIk/Qv9DMwPMLAwwODCIIMLg8ytzyXaFfoKaS9vp6O8g86KTjoqOugs76S9vJ1QMDM9CJiDOZYn2wFE5EHc5b0+4AlVfb+IvBe4qKpPi8hLgC8DVcAaMKGq9+y1TSskJl+pKvHxcdauXWO9t5f1a72sX7tGdGAgce4FEQLhMIWnTlHY3U3h6VMEu7uzVmAA5tbmGFwYZOjmEIMLgwwsDDC0MERkKZLoewygtriWjvIO2svb6azopK2sjfaKdsKhMAGfdROTbce2kGSDFRJjbqfxONGREVdY+vq8qZfo0DDEXT9fiBBoaSHY3UVhVzeF3V0EvbmvoiIrccU2YowsjjC0MMTgzUGGFoYYWRxh+OYwN9ZuJNoVSAHNpc20l7fTVt7m5mVttJW30Rxqtr7IMsQKSRIrJMakRmMxV2B6+1gf6Cfa1896fz/RwUE0Gk2089XWUtjZSbCzk2BXZ2I50NKS8UuUtyysLzByc4Shm0MM3xxOLI8sjrAcW74Vm/hoDjXTVtZGuCxMuCxMW7lbbgm12F39d8EKSRIrJMakRzc2iI2OuqIyMMj64ADRgUGi/f2JsV3AXRwQaG9zhaWjk2BHB8HODoLt7fiqq7PS75eqMrs2y/XF64kCs7UcWYywGFu8rX19ST3hsjCtoVY39y4gaAm1UFNUY32TJbFCksQKiTHZE5+bIzo4SHRggOjQEOuDQ279+vXEyJTgriQLtrcTbG8n0N6WWA62t+OrqspakVlYX2Bk0RWX64vXiSxGEvOp1anb2hf7i2kJtdAaaqWlrIWWUAvNoWZaQ600h5rz7iZMKyRJrJAYk3sajxMbH3dFZWiY6PAw0aEhoiMjxEZHE3fxgysygbYwwXAbwbYwgXCYYFsbwXAYf2Nj1g6XrcXXGF0aZXRpNFFcttZHl0ZvO2QGrhPMllALTaVNNIeab02lbl4eLD9RezRWSJJYITHmaNFolGhklNj1EVdghkeIXh8hNjxCdGzstj0ZAgECzU0EW8MEwq0EW1sJtIYJtLYSaGnGV1mZ1b2Z0aVRIksRxpbGEgVmfGmcseWx2+6TAddfWVNpE42hRppLm91yaWNi3lDScKyuNrNCksQKiTHHh25sEBufIBa57vZeIqOuyIxcJxaJ3HZOBlx3M4GWltun5mZvObuFZn59nrHlMcaXxhldGmVieYKxpTHGl8eZWJ647X4ZAEGoKa6hsaSRxtJbU0NpA40lrtDUltQemavOrJAksUJizMmxsbhILBIhNjpKNBIhFhklNjbmHotE2PSGSN4ixcUEmppccWlqItDchL+piUCjt9zYSEEwmJVYV+OriaIyuTyZWJ5YnmBixc2379VsFZuGkgbqS+oT8+1TKBDK+mE0KyRJrJAYkx9Ulc2FBVdYtqbRpOWJCTZmZ+94na+mhkBjI/6mRgINjQSaGvE3NBJobMDf4KaCwsKsxLsYW2RyeZKJ5QmmVqaYXJm8NS1PMrUyxc3ozTteW+wvpq7Y9YlWX1xPbUltYl5XXEddcR21JbWUBcrS6XfNCskWKyTGmC2ba2vEJyaITUwQGxsnNj7m1scniE2ME5+cYnNx8Y7X+SorvaJST6ChAX9dPf76evwNbh6or3eXOGfhwoDV+CrTK9NMrkwyvTLN9KpbnlmZYWp1iqmVKWZWZ+7YuwF4z0vfw1vOv+VA73tcu5E3xpisKigqcve3dHTs2mZjaZn45ATxyUliE5PEJyeITU4Sn5wiPjnJ2uXLbMzMwvZ/sgsK8NfUuAJTV4e/rtab1+GrrcVf663X1NxV9zPF/mLayt2d+7tRVZZjy0ytTjGzMsPM6gzTq9Pc35CROpA2KyTGmLziC5XiC3VT2N29axuNx4nPzBCfmiI+NUVsaor49LRbn54mNjnJ6vPPu0NpOxzVKSgtxV9b6wpMTQ3+2hp8NTX4a7bmtfhrqvHV1FBQWrrv4SkRIRQMEQqG6KroSvszyDQrJMYYs434/QQaGwk0Nu7ZTuNx4jdusDEz4wrNzKwrQNPTxGdn2JiZZb2vj+XnnmNz2xVpifcKBPBVV+OrqcZfVY2vuhp/dRW+qmp8VVX4qqvwV1V5y9X4ysuzdu/NQVkhMcaYAxK/n4B3DmU/Go0Sn5tnY3aG+I05N5+9wcbcDTefnXU9CQwNEZ+bQ7ddoXbrTQVfeTm+ykpq3/VOKl73ugxndfeskBhjTA5IMEigoZ5Aw/5FB9yFAxvz82zcuEF8bo6NObe8MT/vTXP4KiuzHHVqrJAYY8wRVFBUREEKh9eOgoLDDsAYY8zxZoXEGGNMWqyQGGOMSUtOC4mIvFZEropIn4i8Z4fnC0XkC97zz4lIRy7jM8YYc/dyVkhExAd8GPht4ALwJhG5sK3Zo8Ccqp4C/hn4h1zFZ4wx5mByuUfyUqBPVQdUNQp8HnhoW5uHgE96y08CD8hJGknGGGNOoFwWkhbgetJ6xHtsxzaqGgcWgJrtGxKRx0TkoohcnJ6ezlK4xhhjUpHLQrLTnsX2TmpSaYOqPq6q96vq/XV1dRkJzhhjzMHk8obECBBOWm8FxnZpExERP1AB3Nhro5cuXZoRkeEDxlQLzBzwtcdZPuadjzlDfuadjznD3efdnqk3zmUh+TFwWkQ6gVHgjcCbt7V5GvgD4EfAw8B/6z4DpqjqgXdJRORipvrjP07yMe98zBnyM+98zBkON++cFRJVjYvIHwPPAD7gCVV9QUTeC1xU1aeB/wA+LSJ9uD2RN+YqPmOMMQeT0762VPUbwDe2PfY3SctrwO/lMiZjjDHpyfc72x8/7AAOST7mnY85Q37mnY85wyHmfezHbDfGGHO48n2PxBhjTJqskBhjjElLXhSSfOwsMoWc/1REekTkFyLyHRHJ2DXlh2m/vJPaPSwiKiLH/jLRVHIWkd/3vu8XROSzuY4xG1L4GW8Tke+KyE+9n/MHDyPOTBKRJ0RkSkSe3+V5EZF/9T6TX4jIfTkJTFVP9IS71Lgf6AKCwM+BC9vavAP4qLf8RuALhx13DnL+DaDEW377cc851by9dmXA94FngfsPO+4cfNengZ8CVd56/WHHnaO8Hwfe7i1fAIYOO+4M5P1K4D7g+V2efxD4Jq6XkJcBz+UirnzYI8nHziL3zVlVv6uqK97qs7ieBo67VL5rgPcB/wis5TK4LEkl5z8CPqyqcwCqOpXjGLMhlbwVKPeWK7izJ41jR1W/z969fTwEfEqdZ4FKEWnKdlz5UEgy1lnkMZJKzskexf0Xc9ztm7eIvBgIq+rXcxlYFqXyXZ8BzojID0XkWRF5bc6iy55U8v474BERieDuX3tnbkI7VHf7u58ROb0h8ZBkrLPIYyTlfETkEeB+4FVZjSg39sxbRApw49y8LVcB5UAq37Ufd3jr1bg9zx+IyL2qOp/l2LIplbzfBHxCVT8kIi/H9Zpxr6puZj+8Q3Mof8vyYY/kbjqLJNXOIo+4VHJGRF4D/CXwelVdz1Fs2bRf3mXAvcD3RGQIdwz56WN+wj3Vn++vqmpMVQeBq7jCcpylkvejwBcBVPVHQBGuY8OTLKXf/UzLh0KS6CxSRIK4k+lPb2uz1VkkpNhZ5BG3b87eIZ5/xxWRk3DMHPbJW1UXVLVWVTtUtQN3buj1qnrxcMLNiFR+vr+Cu7gCEanFHeoayGmUmZdK3iPAAwAich5XSE76AEZPA2/1rt56GbCgquPZftMTf2hL87CzyBRz/iAQAr7kXVcwoqqvP7SgMyDFvE+UFHN+BvgtEekBNoA/V9XZw4s6fSnm/WfAx0TkT3CHd952zP9BREQ+hztEWeud+/lbIACgqh/FnQt6EOgDVoA/zElcx/xzNcYYc8jy4dCWMcaYLLJCYowxJi1WSIwxxqTFCokxxpi0WCExxhiTFiskxhhj0mKFxBhjTFqskJicEZENEflZ0tQhIkv7vKZSRN6x2/oB4/jfdF6/bVtbOT0vIl8SkZJMbTuNmNL+jIy5G3ZDoskZEVlS1dB+j217vgP4uqreu9P6YUuOX0Q+A1xS1X9K4XWC+/3LeAeCB/mMshmPOflsj8QcGSLyFRG55I3i95j38N8D3d5//R/cYR0ReURE/s977N9FxOft7VwWkY952/uWiBR77Ze8+a5tvOf/WkSuiMi3ReRzIvLufVL4AXBqt1yS3u8jwE+A8C45b7W9IiIf9/Z2PiMirxHXFXyviLx0t9wP8Bkl4tnn+7lHRP5LRK55n82/ichL9v9mzYmX6xG+bMrfCdfP08+86cveY0tJz1d782LgedyYMB0kjQa3w/p54GtAwFv/CPBWr10ceJH3+BeBR5Lfc58293txFuN6De4F3r1DTlvb8gNf5daIfLvlsgm8bK+ct8X2K7h/+C4BT+C6CX8I1xHjXrmn+hndFs8e310R0APc48U6DDx12D9TNh2N6cR32miOlFVVfdEez79LRN7gLYdxXZ1P7LPNB4BfBX7sdT5ZDEzhhtIdVNWfee0u4f5wbrdbm1/Hdb2+CiAiX9vl/YtFZOv1P8B1ALpXLsPqRq5jj3ZbHSoOquovvfd/AfiOqqqI/NKLc6/ck+3Vbns8u3kN8FNVfcGLJwh8yFvuwg1HUKGqD6ewLXPCWCExR4KIvBr3x+rlqroiIt/D/Re870uBT6rqX2zbXgeQPMbKBu4P6Ha7tUl1qOU7iuM+uSyn2G57bJtJ65u43929cr/toT3aLZOaF+MOfyEizbg9sR8CqOoA8KiIPJnitswJY+dIzFFRAcx5f1DP4QadAljEHVpil/XvAA+LSD2AiFSLSHsG4vkf4HdEpEhEQsDr7uK1u+Vy0Ha72S33bHxG67hBkgA+AATv8vXmBLNCYo6K/wT8IvIL4H24QadQN27GD70Tzh/cYb0H+CvgW95rvw00pRuMqv4YN0jQz4GngIvAQjq5pNFutxh3zD1Ln9FngVeKyFXcZ/IjEfmXu9yGOaHs8l9jdiEiIVVdEndvyPeBx1T1J4cd11EjIjXA+4HfBD6uqh845JBMjlkhMWYXIvJZ4ALuvMUn7Q+kMTuzQmKMMSYtdo7EGGNMWqyQGGOMSYsVEmOMMWmxQmKMMSYtVkiMMcakxQqJMcaYtFghMcYYk5b/B8YZn/vdmOjqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pr_item=[]\n",
    "pr_id = []\n",
    "pr_attr=[]\n",
    "pr_y = []\n",
    "stdout = sys.stdout\n",
    "sampled_feature_names = ['item','id','attrs','y']\n",
    "sys.stdout = open(os.devnull, 'w')\n",
    "for a in np.arange(0,1,.01):\n",
    "    tempprobs = gen_step1_probs(feat2vec_fm,sampled_feature_names,a)\n",
    "    pr_item.append(tempprobs[sampled_feature_names.index('item')])\n",
    "    pr_id.append(tempprobs[sampled_feature_names.index('id')])\n",
    "    pr_attr.append(tempprobs[sampled_feature_names.index('attrs')])\n",
    "    pr_y.append(tempprobs[sampled_feature_names.index('y')])\n",
    "sys.stdout = stdout\n",
    "plt.plot(np.arange(0,1,.01),pr_id,label = 'ID')\n",
    "plt.plot(np.arange(0,1,.01),pr_item,label = 'Item')\n",
    "plt.plot(np.arange(0,1,.01),pr_attr,label = 'Attribute')\n",
    "plt.plot(np.arange(0,1,.01),pr_y,label = 'y')\n",
    "plt.legend()\n",
    "plt.xlabel(r'Flattening Parameter  $\\alpha_1$')\n",
    "plt.ylabel(r'$Pr($Choose Feature$ | \\alpha_1)$')\n",
    "plt.title(r'Feature choice probabilities in step 1 of sampling algorithm')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "create the probabilities for step1 of our sampler which determines how likely a feature is resampled for producing a negative label. here we choose \n",
    "\n",
    "Pr(choose feature $j$) $\\propto$ $|j|^{.25}$\n",
    "\n",
    "where $|j|$ is the number of parameters associated with  of feature $j$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Original Probs--\n",
      "[0.15384615 0.76923077 0.04615385 0.03076923]\n",
      "--New Probs with alpha=0.25--\n",
      "[0.25613626 0.38301305 0.18956204 0.17128864]\n"
     ]
    }
   ],
   "source": [
    "feature_alpha=0.25\n",
    "step1_probs = gen_step1_probs(feat2vec_fm,sampled_feature_names,feature_alpha)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split data into validation and training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "pct_train = .90\n",
    "train_sample = np.random.uniform(size=samplesize)\n",
    "train_sample = (train_sample <=np.percentile(train_sample,pct_train*100) ).astype('bool')\n",
    "train_data = data[train_sample==True]\n",
    "valid_data = data[train_sample==False]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create samplers from implicitsampler class . Here we pass the second main hyperparameter, $\\alpha_2=.25$ (sampling_alpha) to determine how much we flatten our empirical distribution that samples values for negative labels.\n",
    "\n",
    "* sampling_bias adds a minimum count to each unique value in the data to ensure very low frequency values get sampled enough\n",
    "\n",
    "* negative_samples is the number of negative samples per observed record\n",
    "\n",
    "* keep_noise_probs necessary if we use NCE, otherwise should not use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no sampling_items passed; generating unique item sets from df passed\n",
      "[['item'], ['id'], ['attr1', 'attr2', 'attr3'], ['y']]\n",
      "generating probs for ['item']\n",
      "generating probs for ['id']\n",
      "generating probs for ['attr1', 'attr2', 'attr3']\n",
      "generating probs for ['y']\n",
      "calculating prob lookup dict for expedited sampling of negative labels...\n",
      "no sampling_items passed; generating unique item sets from df passed\n",
      "[['item'], ['id'], ['attr1', 'attr2', 'attr3'], ['y']]\n",
      "generating probs for ['item']\n",
      "generating probs for ['id']\n",
      "generating probs for ['attr1', 'attr2', 'attr3']\n",
      "generating probs for ['y']\n",
      "calculating prob lookup dict for expedited sampling of negative labels...\n"
     ]
    }
   ],
   "source": [
    "trainsampler = ImplicitSampler(train_data,model_features=model_features,sampling_features=sampling_features,\n",
    "                          batch_size=batch_size,                 \n",
    "                          sampling_alpha=.25,sampling_bias=10,\n",
    "                          negative_samples=neg_samples,\n",
    "                          init_probs = step1_probs,keep_noise_probs=feat2vec_obj.obj=='nce')\n",
    "validsampler = ImplicitSampler(valid_data,model_features=model_features,sampling_features=sampling_features,\n",
    "                          batch_size=batch_size,                 \n",
    "                          sampling_alpha=.25,sampling_bias=10,\n",
    "                          negative_samples=neg_samples,\n",
    "                          init_probs = step1_probs,keep_noise_probs=feat2vec_obj.obj=='nce')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, fit the model!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/quy/anaconda3/envs/py36/lib/python3.6/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/quy/anaconda3/envs/py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3376: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/quy/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/quy/anaconda3/envs/py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
      "\n",
      "Epoch 1/100\n",
      "45/45 [==============================] - 4s 91ms/step - loss: 2.5145 - acc: 0.2839 - val_loss: 2.0254 - val_acc: 0.3102\n",
      "Epoch 2/100\n",
      "45/45 [==============================] - 3s 66ms/step - loss: 2.1013 - acc: 0.4132 - val_loss: 1.4934 - val_acc: 0.4518\n",
      "Epoch 3/100\n",
      "45/45 [==============================] - 3s 57ms/step - loss: 1.6181 - acc: 0.6154 - val_loss: 1.1101 - val_acc: 0.7068\n",
      "Epoch 4/100\n",
      "45/45 [==============================] - 3s 58ms/step - loss: 1.3708 - acc: 0.7088 - val_loss: 1.0022 - val_acc: 0.7054\n",
      "Epoch 5/100\n",
      "45/45 [==============================] - 3s 68ms/step - loss: 1.3076 - acc: 0.7072 - val_loss: 0.9767 - val_acc: 0.7050\n",
      "Epoch 6/100\n",
      "45/45 [==============================] - 3s 57ms/step - loss: 1.2790 - acc: 0.7068 - val_loss: 0.9728 - val_acc: 0.7168\n",
      "Epoch 7/100\n",
      "45/45 [==============================] - 3s 58ms/step - loss: 1.2622 - acc: 0.7073 - val_loss: 0.9972 - val_acc: 0.7388\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fdd181149e8>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss', patience=0)]\n",
    "feat2vec_fm.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer=keras.optimizers.TFOptimizer(tf.train.AdamOptimizer()))\n",
    "\n",
    "feat2vec_fm.fit_generator(generator = trainsampler.keras_generator(),\n",
    "                          epochs=100, \n",
    "                          steps_per_epoch = len(train_data)/batch_size,\n",
    "                          validation_data=validsampler.keras_generator(),\n",
    "                          validation_steps =  len(valid_data)/batch_size,\n",
    "                          callbacks=callbacks,\n",
    "                          verbose=1,\n",
    "                          max_queue_size=1,\n",
    "                          workers=1,\n",
    "                          use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example Usage : Look at cosine similarity of embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#collect embeddings\n",
    "embeddings = []\n",
    "embed_names = ['dim_'+str(i) for i in range(1,embed_dim+1)]\n",
    "for j in feat2vec_obj.feature_names:\n",
    "    for l in feat2vec_fm.layers:\n",
    "         if l.name=='factor_{}'.format(j) or l.name=='embedding_{}'.format(j):\n",
    "                embedding = feat2vec_fm.get_layer(l.name).get_weights()[0]\n",
    "                embedding = pd.DataFrame(embedding,columns=embed_names)\n",
    "                embedding['value'] = [j +'_' + str(i) for i in embedding.index]\n",
    "                embeddings.append(embedding)\n",
    "embeddings = pd.concat(embeddings,ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kNN_embeddings(value,k):\n",
    "    embedding = embeddings.loc[embeddings['value']==value,embed_names]\n",
    "    scores = np.dot(embeddings[embed_names],embedding.T).flatten()\n",
    "    scores = np.array([s/(np.linalg.norm(embedding)*np.linalg.norm(embeddings.loc[i,embed_names])) for i,s in enumerate(scores)]).flatten()\n",
    "    temp = np.argsort(-scores)\n",
    "    ranks = np.empty(len(scores), int)\n",
    "    ranks[temp] = np.arange(len(scores))\n",
    "    for r in range(1,k+1):\n",
    "        print(embeddings.loc[ranks==r,'value'].values,scores[ranks==r])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check out what the closest embeddings are to ID#5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['id_35'] [0.9448017]\n",
      "['id_25'] [0.9348221]\n",
      "['id_24'] [0.9337117]\n",
      "['id_15'] [0.9302186]\n",
      "['id_45'] [0.9301508]\n"
     ]
    }
   ],
   "source": [
    "kNN_embeddings('id_5',5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['id_35'] [0.9448017]\n",
      "['id_25'] [0.9348221]\n",
      "['id_24'] [0.9337117]\n",
      "['id_15'] [0.9302186]\n",
      "['id_45'] [0.9301508]\n"
     ]
    }
   ],
   "source": [
    "kNN_embeddings('id_5',5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It learned that $mod(10)$ IDs by the same items and have similar y!"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}