neural.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.style as style\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "style.use(\"fivethirtyeight\")\n",
    "import gc\n",
    "import math\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import h5py\n",
    "import pytorch_lightning as pl\n",
    "import torch\n",
    "from pytorch_lightning import seed_everything\n",
    "from pytorch_lightning.metrics.functional import accuracy\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from torch import nn as nn\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import DataLoader, Dataset, random_split\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "pd.set_option(\"display.max_rows\", 100)\n",
    "\n",
    "SEED = 69\n",
    "seed_everything(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading lectures arrays\n",
      "Loading questions arrays\n",
      "CPU times: user 32.9 ms, sys: 1.1 s, total: 1.13 s\n",
      "Wall time: 1.14 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# This code is needed for dataset\n",
    "train = pd.read_pickle(\"riiid_train.pkl.gzip\")\n",
    "questions_df = pd.read_csv(\"data/questions.csv\")\n",
    "lectures_df = pd.read_csv(\"data/lectures.csv\")\n",
    "\n",
    "folder_path = \"data\"\n",
    "print(\"Loading lectures arrays\")\n",
    "lectures_ids = np.load(f\"{folder_path}/lectures_ids.npy\")\n",
    "lectures_parts = np.load(f\"{folder_path}/lectures_parts.npy\")\n",
    "lectures_types = np.load(f\"{folder_path}/lectures_types.npy\")\n",
    "lectures_tags = lectures_df.tag.values\n",
    "\n",
    "print(\"Loading questions arrays\")\n",
    "questions_parts = np.load(f\"{folder_path}/questions_parts.npy\")\n",
    "questions_lectures_parts = np.concatenate([questions_parts, lectures_parts])\n",
    "\n",
    "\n",
    "# process tags\n",
    "def split_tags(t):\n",
    "    try:\n",
    "        return [int(i) for i in t.split(\" \")]\n",
    "    except AttributeError:\n",
    "        return list()\n",
    "\n",
    "\n",
    "# Get tags to be 2D array of shape (Q, T), where Q is question_idx, and T is the max number of tag possible (6)\n",
    "questions_df[\"tags\"] = questions_df.tags.apply(split_tags)\n",
    "questions_tags = pd.DataFrame(questions_df[\"tags\"].tolist(), index=questions_df.index)\n",
    "\n",
    "# map lecture id to new id\n",
    "\n",
    "lectures_mapping = dict(\n",
    "    zip(lectures_df.lecture_id.values, (lectures_df.index + 13523).values)\n",
    ")\n",
    "lectures_df.lecture_id = lectures_df.index + 13523\n",
    "lectures_tags = pd.DataFrame(\n",
    "    lectures_df.tag.values, index=lectures_df.lecture_id.values\n",
    ")\n",
    "\n",
    "questions_lectures_tags = pd.concat([questions_tags, lectures_tags])\n",
    "# pad with max tag + 1\n",
    "questions_lectures_tags = (\n",
    "    questions_lectures_tags.fillna(questions_lectures_tags.max().max() + 1)\n",
    "    .astype(np.int)\n",
    "    .values\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions_content_mean = (\n",
    "    train[~train.content_type_id]\n",
    "    .groupby(\"content_id\")\n",
    "    .answered_correctly.mean()\n",
    "    .reset_index()\n",
    ")\n",
    "questions_lectures_mean = np.concatenate(\n",
    "    [questions_content_mean.answered_correctly.values, np.zeros(len(lectures_df))]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_df(df):\n",
    "    \"\"\"\n",
    "    Converts the lecture ids to proper content_ids\n",
    "    Adds the answered_correctly column if not exists\n",
    "    \"\"\"\n",
    "    df.content_type_id = df.content_type_id.astype(bool)\n",
    "\n",
    "    # prior information\n",
    "    df.prior_question_elapsed_time = (\n",
    "        df.prior_question_elapsed_time.fillna(0).clip(upper=300000) / 300000\n",
    "    )  # normalizes to 0-1\n",
    "\n",
    "    # map lecture ids to new content_ids\n",
    "    df.loc[df.content_type_id, \"content_id\"] = df[df.content_type_id].content_id.map(\n",
    "        lectures_mapping\n",
    "    )\n",
    "    # if not answered correctly then add column with\n",
    "    # y = 3 (padding) for all questions and y = 4 for lectures\n",
    "    if \"answered_correctly\" not in df.columns:\n",
    "        df[\"answered_correctly\"] = df.content_type_id.map({False: 3, True: 4})\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Time Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps = 0.0000001\n",
    "def get_time_elapsed_from_timestamp(arr):\n",
    "    arr_seconds = np.diff(arr, prepend=0) / 1000\n",
    "    return (np.log(arr_seconds + eps).astype(np.float32)-3.5) / 20\n",
    "\n",
    "# ## Show effects of transformation \n",
    "# times = train.groupby(\"user_id\").timestamp.apply(get_time_elapsed_from_timestamp)\n",
    "# times = np.concatenate(times.tolist())\n",
    "# sns.boxplot(x=times)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Agg Feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_parts_agg_feats(q_idx, parts, answered_correctly):\n",
    "    parts_aggs = np.zeros((len(parts), 7))\n",
    "\n",
    "    # select parts for only questions\n",
    "    p = parts[q_idx]\n",
    "\n",
    "    # get running counts of parts answered\n",
    "    count_parts = np.zeros((p.size, 7))\n",
    "    count_parts[np.arange(p.size), p - 1] = 1\n",
    "    count_parts = count_parts.cumsum(axis=0)\n",
    "\n",
    "    # get running sum of answered correctly for each part\n",
    "    answered_correctly_parts = np.zeros((p.size, 7))\n",
    "    answered_correctly_parts[np.arange(p.size), p - 1] = answered_correctly[q_idx]\n",
    "    answered_correctly_parts = answered_correctly_parts.cumsum(axis=0)\n",
    "\n",
    "    parts_aggs[q_idx, :] = answered_correctly_parts / (count_parts + 1)\n",
    "\n",
    "    return parts_aggs\n",
    "\n",
    "\n",
    "def get_agg_feats(content_ids, answered_correctly, parts):\n",
    "\n",
    "    # Calculate agg feats\n",
    "    # question idx\n",
    "    q_idx = np.where(answered_correctly != 4)[0]\n",
    "\n",
    "    # user mean\n",
    "    m_user_mean = np.zeros(len(content_ids))\n",
    "    m_user_mean[q_idx] = answered_correctly[q_idx].cumsum() / (\n",
    "        np.arange(len(q_idx)) + 1\n",
    "    )\n",
    "    # prevent leak\n",
    "    m_user_mean = np.roll(m_user_mean, 1)\n",
    "    m_user_mean[0] = 0\n",
    "\n",
    "    # content mean\n",
    "    m_content_mean = np.zeros(len(content_ids))\n",
    "    m_content_mean[q_idx] = questions_lectures_mean[content_ids][q_idx].cumsum() / (\n",
    "        np.arange(len(q_idx)) + 1\n",
    "    )\n",
    "\n",
    "    # part mean\n",
    "    parts_mean = get_parts_agg_feats(q_idx, parts, answered_correctly)\n",
    "\n",
    "    # prevent leak\n",
    "    parts_mean = np.roll(parts_mean, 1, axis=0)\n",
    "    parts_mean[0] = 0\n",
    "\n",
    "    return np.concatenate(\n",
    "        [m_content_mean[..., np.newaxis], m_user_mean[..., np.newaxis], parts_mean],\n",
    "        axis=1,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Hdf5\n",
    "\n",
    "Only run this once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.15 s, sys: 320 ms, total: 1.47 s\n",
      "Wall time: 1.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train = preprocess_df(train) #convert lecture ids to be sequentially following the question_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['row_id', 'timestamp', 'user_id', 'content_id', 'content_type_id',\n",
       "       'task_container_id', 'user_answer', 'answered_correctly',\n",
       "       'prior_question_elapsed_time', 'prior_question_had_explanation'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.answered_correctly.replace(-1, 4, inplace=True) # set lecture to token 4 for answered correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d41257e0404542f990c1242fc1f0e279",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=338170.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# ignore lectures for now\n",
    "hf = h5py.File(\"feats.h5\", \"w\")\n",
    "\n",
    "for user_id, data in tqdm(train.groupby(\"user_id\")):\n",
    "    processed_feats = data[\n",
    "        [\n",
    "            \"content_id\",\n",
    "            \"answered_correctly\",\n",
    "            \"timestamp\",\n",
    "            \"prior_question_elapsed_time\",\n",
    "        ]\n",
    "    ].values\n",
    "\n",
    "    hf.create_dataset(f\"{user_id}/content_ids\", data=processed_feats[:, 0], maxshape=(None,))\n",
    "    hf.create_dataset(f\"{user_id}/answered_correctly\", data=processed_feats[:, 1], maxshape=(None,))\n",
    "    hf.create_dataset(f\"{user_id}/timestamps\", data=processed_feats[:, 2], maxshape=(None,))\n",
    "    hf.create_dataset(f\"{user_id}/prior_question_elapsed_time\", data=processed_feats[:, 3], maxshape=(None,))\n",
    "\n",
    "hf.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pytorch Stuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data\n",
    "\n",
    "Here we define the pytorch Dataset object and a custom collate function. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RIIDDataset(Dataset):\n",
    "    \"\"\"RIID dataset.\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, user_mapping, user_history, hdf5_file=\"feats.h5\", window_size=100,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            user_mapping (np.array): array of user_ids per row\n",
    "            user_history (np.array): array of length of history up till this row\n",
    "            hdf5_file (string): location of hf5 feats file\n",
    "            window_size (int): size of window to lookback to max\n",
    "        \"\"\"\n",
    "        # np array where index maps to a user id\n",
    "        self.user_mapping = user_mapping\n",
    "        self.user_history = user_history\n",
    "        self.hdf5_file = f\"{hdf5_file}\"\n",
    "        self.max_window_size = window_size\n",
    "        self.cache = {}\n",
    "\n",
    "    def open_hdf5(self):\n",
    "        # opens the h5py file\n",
    "        self.f = h5py.File(self.hdf5_file, \"r\")\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.user_mapping)\n",
    "\n",
    "    def load_user_into_cache(self, user_id):\n",
    "        \"\"\"\n",
    "        add a user to self.cache\n",
    "        \"\"\"\n",
    "        # LOAD from file\n",
    "        content_ids = np.array(self.f[f\"{user_id}/content_ids\"], dtype=np.int64)\n",
    "        answered_correctly = np.array(\n",
    "            self.f[f\"{user_id}/answered_correctly\"], dtype=np.int64\n",
    "        )\n",
    "        timestamps = np.array(self.f[f\"{user_id}/timestamps\"], dtype=np.float32)\n",
    "        prior_question_elapsed_time = np.array(\n",
    "            self.f[f\"{user_id}/prior_question_elapsed_time\"], dtype=np.float32\n",
    "        )\n",
    "\n",
    "        self.cache[user_id] = {\n",
    "            \"content_ids\": content_ids,\n",
    "            \"answered_correctly\": answered_correctly,\n",
    "            \"timestamps\": timestamps,\n",
    "            \"prior_question_elapsed_time\": prior_question_elapsed_time,\n",
    "        }\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        # open the hdf5 file in the iterator to allow multiple workers\n",
    "        # https://github.com/pytorch/pytorch/issues/11929\n",
    "        if not hasattr(self, \"f\"):\n",
    "            self.open_hdf5()\n",
    "\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "\n",
    "        user_id = self.user_mapping[idx]\n",
    "        length = self.user_history[idx]\n",
    "\n",
    "        window_size = min(self.max_window_size, length)\n",
    "\n",
    "        # index for loading larger than window size\n",
    "        start_index = 0\n",
    "        if length > window_size:\n",
    "            # randomly select window size subset instead of trying to cram in everything\n",
    "            start_index = length - window_size\n",
    "\n",
    "        if user_id not in self.cache:\n",
    "            self.load_user_into_cache(user_id)\n",
    "        \n",
    "        ## load in full array from first element\n",
    "        content_ids = self.cache[user_id][\"content_ids\"][\n",
    "            : start_index + window_size\n",
    "        ].copy()\n",
    "        answered_correctly = self.cache[user_id][\"answered_correctly\"][\n",
    "            : start_index + window_size\n",
    "        ].copy()\n",
    "\n",
    "        # Get parts:\n",
    "        # get question parts\n",
    "        parts = questions_lectures_parts[content_ids]\n",
    "        \n",
    "        # Get agg feats\n",
    "        agg_feats = get_agg_feats(content_ids, answered_correctly, parts)\n",
    "        agg_feats = agg_feats[start_index:]\n",
    "\n",
    "        ### Select proper length:\n",
    "        parts = parts[start_index:]\n",
    "        content_ids = content_ids[start_index:]\n",
    "        answered_correctly = answered_correctly[start_index:]\n",
    "        \n",
    "        \n",
    "        ## Load in timestuff\n",
    "        timestamps = self.cache[user_id][\"timestamps\"][\n",
    "            start_index : start_index + window_size\n",
    "        ].copy()\n",
    "        prior_q_times = self.cache[user_id][\"prior_question_elapsed_time\"][\n",
    "            start_index : start_index + window_size\n",
    "        ].copy()\n",
    "\n",
    "        # convert timestamps to time elapsed\n",
    "        time_elapsed_timestamps = get_time_elapsed_from_timestamp(timestamps)\n",
    "\n",
    "        # get question tags\n",
    "        tags = questions_lectures_tags[content_ids, :]\n",
    "\n",
    "        # shift by one the answered_correctly sequence\n",
    "        answers = np.roll(answered_correctly, 1)\n",
    "\n",
    "        # set start token if start_index is actually first element\n",
    "        if start_index == 0:\n",
    "            answers[0] = 2\n",
    "        # else replace first element of sequence with actual previous element\n",
    "        else:\n",
    "            answers[0] = self.cache[user_id][\"answered_correctly\"][start_index - 1]\n",
    "\n",
    "        return {\n",
    "            \"parts\": torch.from_numpy(parts).long(),\n",
    "            \"tags\": torch.from_numpy(tags).long(),\n",
    "            \"content_ids\": torch.from_numpy(content_ids),\n",
    "            \"answered_correctly\": torch.from_numpy(answered_correctly),\n",
    "            \"answers\": torch.from_numpy(answers),\n",
    "            \"timestamps\": torch.from_numpy(time_elapsed_timestamps),\n",
    "            \"prior_q_times\": torch.from_numpy(prior_q_times),\n",
    "            \"agg_feats\": torch.from_numpy(agg_feats).float(),\n",
    "            \"length\": window_size,\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "\n",
    "# The collate function is used to merge individual data samples into a batch\n",
    "# It handles the padding aspect\n",
    "def collate_fn(batch):\n",
    "    # collate lenghts into 1D tensor\n",
    "    items = {\"length\": torch.tensor([batch_item[\"length\"] for batch_item in batch])}\n",
    "\n",
    "    # find shape that the batch will have\n",
    "    max_length = items[\"length\"].max()\n",
    "    num_items = len(batch)\n",
    "\n",
    "    # padding list\n",
    "    for (key, padding) in [\n",
    "        (\"parts\", 0),\n",
    "        (\"content_ids\", 13942),\n",
    "        (\"answered_correctly\", 3),\n",
    "        (\"answers\", 3),\n",
    "        (\"timestamps\", 0.0),  # note timestamps isnt an embedding\n",
    "        (\"tags\", 188),\n",
    "        (\"agg_feats\", 0),\n",
    "        (\"prior_q_times\", 0),\n",
    "    ]:\n",
    "        items[key] = pad_sequence(\n",
    "            [batch_item[key] for batch_item in batch],\n",
    "            batch_first=False,\n",
    "            padding_value=padding,\n",
    "        )\n",
    "\n",
    "    # mask to weight loss by (S, N)\n",
    "    items[\"loss_mask\"] = (\n",
    "        (\n",
    "            torch.arange(max_length).expand(num_items, max_length)\n",
    "            < items[\"length\"].unsqueeze(1)\n",
    "        )\n",
    "        .transpose(1, 0)\n",
    "        .float()\n",
    "    )\n",
    "    items[\"loss_mask\"] *= items[\"answered_correctly\"] != 4  # mask the lectures\n",
    "    items[\"answered_correctly\"] = items[\"answered_correctly\"].float()\n",
    "\n",
    "    return items"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hyperparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1024\n",
    "max_window_size = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dataset + Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.81 s, sys: 1.01 s, total: 3.83 s\n",
      "Wall time: 3.82 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "101230332"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Create Dataset with all rows\n",
    "user_mapping = train.user_id.values\n",
    "user_history = train.groupby(\"user_id\").cumcount().values + 1\n",
    "\n",
    "dataset = RIIDDataset(\n",
    "    user_mapping, user_history, hdf5_file=\"feats.h5\", window_size=max_window_size\n",
    ")\n",
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18.1 s, sys: 1.42 s, total: 19.5 s\n",
      "Wall time: 19.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## if not using lectures\n",
    "# q_train_indices = np.load(\"data/train_90000000_lec_True.npy\")\n",
    "q_valid_indices = np.load(\"data/val_2500000_lec_True.npy\")\n",
    "# q_valid_indices = np.load(\"data/val_2500000_lec_True.npy\")\n",
    "q_train_indices = (\n",
    "    train[(~train.row_id.isin(q_valid_indices)) & (~train.content_type_id)]\n",
    "    .sample(90000000)\n",
    "    .row_id.values\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# if using lectures\n",
    "# question_ids = train[~train.content_type_id].row_id.values\n",
    "# q_train_indices = np.intersect1d(question_ids, cv_train_ids)\n",
    "# q_valid_indices = np.intersect1d(question_ids, cv_valid_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Subset\n",
    "\n",
    "# Init DataLoader from RIIID Dataset subset\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    dataset=Subset(dataset, q_train_indices),\n",
    "    batch_size=batch_size,\n",
    "    shuffle=True,\n",
    "    collate_fn=collate_fn,\n",
    "    num_workers=6,\n",
    "    pin_memory=torch.cuda.is_available(),  # if GPU then pin memory for perf\n",
    ")\n",
    "val_loader = torch.utils.data.DataLoader(\n",
    "    dataset=Subset(dataset, q_valid_indices),\n",
    "    batch_size=batch_size,\n",
    "    collate_fn=collate_fn,\n",
    "    num_workers=6,\n",
    "    pin_memory=torch.cuda.is_available(),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_lightning.core.decorators import auto_move_data\n",
    "from pytorch_lightning.metrics.functional.classification import auroc\n",
    "\n",
    "\n",
    "def init_weights(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        torch.nn.init.xavier_uniform_(m.weight)\n",
    "        torch.nn.init.zeros_(m.bias)\n",
    "\n",
    "\n",
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, d_model, max_len=1000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(\n",
    "            torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)\n",
    "        )\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer(\"pe\", pe)\n",
    "\n",
    "    def forward(self, sequence_length):\n",
    "        # returns embeds (sequence_length, 1, d_model)\n",
    "        return self.pe[:sequence_length, :]\n",
    "\n",
    "\n",
    "class RIIDDTransformerModel(pl.LightningModule):\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate=0.001,\n",
    "        n_content_id=13943,  # number of different contents = 13942 + 1 (for padding)\n",
    "        n_part=8,  # number of different parts = 7 + 1 (for padding)\n",
    "        n_tags=189,  # number of different tags = 188 + 1 (for padding)\n",
    "        n_correct=5,  # 0,1 (false, true), 2 (start token), 3 (padding), 4 (lecture)\n",
    "        n_agg_feats=9,  # number of agg feats\n",
    "        emb_dim=64,  # embedding dimension\n",
    "        dropout=0.1,\n",
    "        n_heads: int = 1,\n",
    "        n_encoder_layers: int = 2,\n",
    "        n_decoder_layers: int = 2,\n",
    "        dim_feedforward: int = 256,\n",
    "        activation: str = \"relu\",\n",
    "        max_window_size=100,\n",
    "        use_prior_q_times=False,\n",
    "        use_prior_q_explanation=False,\n",
    "        use_agg_feats=False,\n",
    "        lr_step_frequency=2500,\n",
    "    ):\n",
    "        super(RIIDDTransformerModel, self).__init__()\n",
    "        self.model_type = \"RiiidTransformer\"\n",
    "        self.learning_rate = learning_rate\n",
    "        self.lr_step_frequency = lr_step_frequency\n",
    "        self.max_window_size = max_window_size\n",
    "\n",
    "        self.use_prior_q_times = use_prior_q_times\n",
    "        self.use_prior_q_explanation = use_prior_q_explanation\n",
    "        self.use_agg_feats = use_agg_feats\n",
    "\n",
    "        # save params of models to yml\n",
    "        self.save_hyperparameters()\n",
    "\n",
    "        #### EXERCISE SEQUENCE\n",
    "        self.embed_content_id = nn.Embedding(n_content_id, emb_dim, padding_idx=13942)\n",
    "        self.embed_parts = nn.Embedding(n_part, emb_dim, padding_idx=0)\n",
    "        self.embed_tags = nn.Embedding(n_tags, emb_dim, padding_idx=188)\n",
    "\n",
    "        # exercise weights to weight the mean embeded excercise embeddings\n",
    "        self.exercise_weights = nn.Parameter(\n",
    "            torch.tensor([0.35, 0.55, 0.1]), requires_grad=True\n",
    "        )\n",
    "        self.register_parameter(\"exercise_weights\", self.exercise_weights)\n",
    "\n",
    "        ### RESPONSE SEQUENCE (1st time stamp of sequence is useless)\n",
    "        self.embed_answered_correctly = nn.Embedding(\n",
    "            n_correct, emb_dim, padding_idx=3\n",
    "        )  # 2 + 1 for start token + 1 for padding_idn_inputs\n",
    "\n",
    "        self.embed_timestamps = nn.Linear(1, emb_dim)\n",
    "\n",
    "        # embed prior q time and q explanation\n",
    "        self.embed_prior_q_time = nn.Linear(1, emb_dim)\n",
    "        self.embed_prior_q_explanation = nn.Embedding(2, emb_dim)\n",
    "\n",
    "        self.embed_agg_feats = nn.Linear(n_agg_feats, emb_dim)\n",
    "\n",
    "        # response weights to weight the mean embeded response embeddings\n",
    "        w = [0.5, 0.5]\n",
    "        if use_prior_q_times:\n",
    "            w.append(0.5)\n",
    "        if use_agg_feats:\n",
    "            w.append(0.5)\n",
    "\n",
    "        self.response_weights = nn.Parameter(torch.tensor(w), requires_grad=True)\n",
    "        self.register_parameter(\"response_weights\", self.response_weights)  ###\n",
    "\n",
    "        # Transformer component\n",
    "        self.pos_encoder = PositionalEncoding(emb_dim)\n",
    "        self.transformer = nn.Transformer(\n",
    "            d_model=emb_dim,\n",
    "            nhead=n_heads,\n",
    "            num_encoder_layers=n_encoder_layers,\n",
    "            num_decoder_layers=n_decoder_layers,\n",
    "            dropout=dropout,\n",
    "            dim_feedforward=dim_feedforward,\n",
    "            activation=activation,\n",
    "        )\n",
    "\n",
    "        self.out_linear = nn.Linear(emb_dim, 2)\n",
    "        init_weights(self)\n",
    "\n",
    "    def generate_square_subsequent_mask(self, sz):\n",
    "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
    "        mask = (\n",
    "            mask.float()\n",
    "            .masked_fill(mask == 0, float(\"-inf\"))\n",
    "            .masked_fill(mask == 1, float(0.0))\n",
    "        )\n",
    "        return mask\n",
    "\n",
    "    def get_random_steps(self, lengths, max_steps=10):\n",
    "        \"\"\"\n",
    "        for x return integer between 1 - 10 or\n",
    "        between 1 - x if x < 10\n",
    "        \"\"\"\n",
    "        m = torch.distributions.uniform.Uniform(\n",
    "            0,\n",
    "            (\n",
    "                torch.minimum(\n",
    "                    torch.ones(lengths.shape, device=self.device) * max_steps, lengths\n",
    "                )\n",
    "            ).float(),\n",
    "        )\n",
    "        return torch.floor(m.sample()).long() + 1\n",
    "\n",
    "    def forward(\n",
    "        self, content_ids, parts, answers, tags, timestamps, prior_q_times, agg_feats\n",
    "    ):\n",
    "        # content_ids: (Source Sequence Length, Number of samples, Embedding)\n",
    "        # tgt: (Target Sequence Length,Number of samples, Embedding)\n",
    "\n",
    "        # if data is flat then expand to get Batch dim\n",
    "        if len(content_ids.shape) == 1:\n",
    "            content_ids = content_ids.unsqueeze(1)\n",
    "            parts = parts.unsqueeze(1)\n",
    "            answers = answers.unsqueeze(1)\n",
    "            tags = tags.unsqueeze(1)\n",
    "            timestamps = timestamps.unsqueeze(1)\n",
    "            prior_q_times = prior_q_times.unsqueeze(1)\n",
    "            if self.use_agg_feats:\n",
    "                agg_feats = agg_feats.unsqueeze(1)\n",
    "\n",
    "        # sequence that will go into encoder\n",
    "        embeded_content = self.embed_content_id(content_ids)\n",
    "        embeded_parts = self.embed_parts(parts)\n",
    "        embeded_tags = self.embed_tags(tags).sum(dim=2)\n",
    "        e_w = F.softmax(self.exercise_weights, dim=0)\n",
    "\n",
    "        embeded_exercises = (\n",
    "            torch.stack([embeded_content, embeded_parts, embeded_tags], dim=3) * e_w\n",
    "        ).sum(dim=3)\n",
    "\n",
    "        # sequence that will go into decoder\n",
    "        embeded_answered_correctly = self.embed_answered_correctly(answers)\n",
    "        embeded_timestamps = self.embed_timestamps(timestamps.unsqueeze(2))\n",
    "        r_w = F.softmax(self.response_weights, dim=0)\n",
    "\n",
    "        exercise_sequence_components = [embeded_answered_correctly, embeded_timestamps]\n",
    "        if self.use_prior_q_times:\n",
    "            embeded_q_times = self.embed_prior_q_time(prior_q_times.unsqueeze(2))\n",
    "            # zero embedding - if start token\n",
    "            embeded_q_times[0, torch.where(answers[0, :] == 2)[0], :] = 0\n",
    "            exercise_sequence_components.append(embeded_q_times)\n",
    "\n",
    "        if self.use_agg_feats:\n",
    "            embeded_agg_feats = self.embed_agg_feats(agg_feats)\n",
    "            # progressively increase user experience\n",
    "            # embeded_agg_feats[:, torch.where(answers[0, :] == 2)[0], :] *= (\n",
    "            #             torch.arange(answers.shape[0], device=self.device, dtype=torch.float)[\n",
    "            #                 :, None, None\n",
    "            #             ]\n",
    "            #             / self.max_window_size\n",
    "            #         )\n",
    "            embeded_agg_feats[0, torch.where(answers[0, :] == 2)[0], :] = 0\n",
    "            exercise_sequence_components.append(embeded_agg_feats)\n",
    "\n",
    "        embeded_responses = (\n",
    "            torch.stack(exercise_sequence_components, dim=3) * r_w\n",
    "        ).sum(dim=3)\n",
    "\n",
    "        # adding positional vector\n",
    "        sequence_length = embeded_responses.shape[0]\n",
    "        embedded_positions = self.pos_encoder(sequence_length + 1)\n",
    "        # add shifted position embedding ( start token is first position)\n",
    "        embeded_responses = embeded_responses + embedded_positions[:-1, :, :]\n",
    "        embeded_exercises = embeded_exercises + embedded_positions[1:, :, :]\n",
    "\n",
    "        # mask of shape S x S -> prevents attention looking forward\n",
    "        top_right_attention_mask = self.generate_square_subsequent_mask(\n",
    "            sequence_length\n",
    "        ).type_as(embeded_exercises)\n",
    "\n",
    "        output = self.transformer(\n",
    "            embeded_exercises,\n",
    "            embeded_responses,\n",
    "            tgt_mask=top_right_attention_mask,  # (S,S)\n",
    "            src_mask=top_right_attention_mask,  # (T,T)\n",
    "        )\n",
    "\n",
    "        output = self.out_linear(output)\n",
    "        return F.softmax(output, dim=2)[:, :, 1]\n",
    "\n",
    "    def process_batch_step(self, batch):\n",
    "        # return result\n",
    "        return self(\n",
    "            batch[\"content_ids\"],\n",
    "            batch[\"parts\"],\n",
    "            batch[\"answers\"],\n",
    "            batch[\"tags\"],\n",
    "            batch[\"timestamps\"],\n",
    "            batch[\"prior_q_times\"],\n",
    "            batch[\"agg_feats\"] if self.use_agg_feats else None,\n",
    "        )\n",
    "\n",
    "    @auto_move_data\n",
    "    def predict_n_steps(self, batch, steps, return_all_preds=False):\n",
    "        \"\"\"\n",
    "        Predicts n steps for all items in batch and return predictions\n",
    "        only for those steps (flattened)\n",
    "        steps: tensor of length B where each item is the number of steps that need to be taken\n",
    "        \"\"\"\n",
    "        seq_length, n_users = batch[\"content_ids\"].shape\n",
    "        lengths = batch[\"length\"]\n",
    "\n",
    "        users = torch.arange(n_users)\n",
    "\n",
    "        user_indexes = []\n",
    "        sequence_indexes = []\n",
    "\n",
    "        for i in range(steps.max().int(), 0, -1):\n",
    "            preds = self.process_batch_step(batch)\n",
    "\n",
    "            sequence_indexes_at_i = lengths[steps >= i] - i\n",
    "            user_indexes_at_i = users[steps >= i]\n",
    "\n",
    "            # get index for which to update the answers\n",
    "            # since answers is shifted we want to map preds 0..98 -> answers 1:99\n",
    "            answers_idx = torch.where(sequence_indexes_at_i + 1 != seq_length)\n",
    "            a_seq_idx = sequence_indexes_at_i[answers_idx] + 1\n",
    "            u_seq_idx = user_indexes_at_i[answers_idx]\n",
    "\n",
    "            # set answer to either 0 or 1 if not lecture\n",
    "            batch[\"answers\"][a_seq_idx, u_seq_idx] = torch.where(\n",
    "                batch[\"answers\"][a_seq_idx, u_seq_idx] != 4,\n",
    "                (preds[sequence_indexes_at_i[answers_idx], u_seq_idx] > 0.5).long(),\n",
    "                batch[\"answers\"][a_seq_idx, u_seq_idx],\n",
    "            )\n",
    "\n",
    "            user_indexes.append(user_indexes_at_i)\n",
    "            sequence_indexes.append(sequence_indexes_at_i)\n",
    "\n",
    "        if return_all_preds:\n",
    "            return preds\n",
    "\n",
    "        user_indexes = torch.cat(user_indexes)\n",
    "        sequence_indexes = torch.cat(sequence_indexes)\n",
    "\n",
    "        return (\n",
    "            preds[sequence_indexes, user_indexes],\n",
    "            batch[\"row_ids\"][sequence_indexes, user_indexes],\n",
    "        )\n",
    "\n",
    "    @auto_move_data\n",
    "    def predict_fast_single_user(\n",
    "        self,\n",
    "        content_ids,\n",
    "        parts,\n",
    "        answers,\n",
    "        tags,\n",
    "        timestamps,\n",
    "        prior_q_times,\n",
    "        n=1,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Predicts n steps for a single user in batch and return predictions\n",
    "        only for those steps (flattened)\n",
    "        \"\"\"\n",
    "        length = len(content_ids)\n",
    "        out_predictions = torch.zeros(n, device=self.device)\n",
    "        for i in range(n, 0, -1):\n",
    "            preds = self(\n",
    "                content_ids,\n",
    "                parts,\n",
    "                answers,\n",
    "                tags,\n",
    "                timestamps,\n",
    "                prior_q_times,\n",
    "            )\n",
    "            out_predictions[n - i] = preds[length - i, 0]\n",
    "\n",
    "            # answers are shifted (start token) so need + 1\n",
    "            answer_idx = length - i + 1\n",
    "            # don't update if at end of answers\n",
    "            if answer_idx + 1 < len(answers):\n",
    "                # don't update if true is lecture\n",
    "                if answers[answer_idx] != 4:\n",
    "                    answers[answer_idx] = (preds[length - i, 0] > 0.5).long()\n",
    "\n",
    "        return out_predictions\n",
    "\n",
    "    def training_step(self, batch, batch_nb):\n",
    "        result = self.process_batch_step(batch)\n",
    "        loss = F.binary_cross_entropy(\n",
    "            result, batch[\"answered_correctly\"], weight=batch[\"loss_mask\"]\n",
    "        )\n",
    "        self.log(\"train_loss\", loss.cpu())\n",
    "        return loss\n",
    "\n",
    "    def validate_n_steps(self, batch):\n",
    "        \"\"\"\n",
    "        Predicts max_steps steps for all items in batch and return predictions\n",
    "        only for those steps (flattened)\n",
    "        steps: tensor of length B where each item is the number of steps that need to be taken\n",
    "        \"\"\"\n",
    "        n_users = batch[\"content_ids\"].shape[1]\n",
    "        seq_length = batch[\"answers\"].shape[0]\n",
    "        lengths = batch[\"length\"]\n",
    "        steps = batch[\"steps\"]\n",
    "        users = torch.arange(n_users)\n",
    "        user_indexes = []\n",
    "        sequence_indexes = []\n",
    "        for i in range(steps.max().int(), 0, -1):\n",
    "            preds = self.process_batch_step(batch)\n",
    "            sequence_indexes_at_i = lengths[steps >= i] - i\n",
    "            user_indexes_at_i = users[steps >= i]\n",
    "\n",
    "            # get index for which to update the answers\n",
    "            # since answers is shifted we want to map preds 0..98 -> answers 1:99\n",
    "            answers_idx = torch.where(sequence_indexes_at_i + 1 != seq_length)\n",
    "            a_seq_idx = sequence_indexes_at_i[answers_idx] + 1\n",
    "            u_seq_idx = user_indexes_at_i[answers_idx]\n",
    "\n",
    "            # set answer to either 0 or 1 if not lecture\n",
    "            batch[\"answers\"][a_seq_idx, u_seq_idx] = torch.where(\n",
    "                batch[\"answers\"][a_seq_idx, u_seq_idx] != 4,\n",
    "                (preds[sequence_indexes_at_i[answers_idx], u_seq_idx] > 0.5).long(),\n",
    "                batch[\"answers\"][a_seq_idx, u_seq_idx],\n",
    "            )\n",
    "\n",
    "            user_indexes.append(user_indexes_at_i)\n",
    "            sequence_indexes.append(sequence_indexes_at_i)\n",
    "\n",
    "        user_indexes = torch.cat(user_indexes)\n",
    "        sequence_indexes = torch.cat(sequence_indexes)\n",
    "        return (preds, sequence_indexes, user_indexes)\n",
    "\n",
    "    def val_test_step(self, batch, log_as=\"val\"):\n",
    "        batch[\"steps\"] = self.get_random_steps(batch[\"length\"], max_steps=10)\n",
    "        result, sequence_indexes, user_indexes = self.validate_n_steps(batch)\n",
    "\n",
    "        step_mask = torch.zeros(batch[\"loss_mask\"].shape, device=self.device)\n",
    "        step_mask[sequence_indexes, user_indexes] = 1\n",
    "\n",
    "        batch[\"loss_mask\"] *= step_mask\n",
    "\n",
    "        loss = F.binary_cross_entropy(\n",
    "            result, batch[\"answered_correctly\"], weight=batch[\"loss_mask\"]\n",
    "        )\n",
    "        self.log(f\"{log_as}_loss_step\", loss.cpu())\n",
    "\n",
    "        select_mask = batch[\"loss_mask\"] > 0\n",
    "        return (\n",
    "            torch.masked_select(result, select_mask).cpu(),\n",
    "            torch.masked_select(batch[\"answered_correctly\"], select_mask).cpu(),\n",
    "        )\n",
    "\n",
    "    def val_test_epoch_end(self, outputs, log_as=\"val\"):\n",
    "        y_pred = torch.cat([out[0] for out in outputs], dim=0)\n",
    "        y = torch.cat([out[1] for out in outputs], dim=0)\n",
    "        auc = auroc(y_pred, y)\n",
    "        if log_as == \"val\":\n",
    "            self.log(f\"avg_{log_as}_auc\", auc, prog_bar=True)\n",
    "        else:\n",
    "            self.log(f\"avg_{log_as}_auc\", auc)\n",
    "\n",
    "    def validation_step(self, batch, batch_nb, dataset_nb=None):\n",
    "        return self.val_test_step(batch, log_as=\"val\")\n",
    "\n",
    "    def validation_epoch_end(self, outputs):\n",
    "        self.val_test_epoch_end(outputs, log_as=\"val\")\n",
    "\n",
    "    def test_step(self, batch, batch_nb, dataset_nb=None):\n",
    "        return self.val_test_step(batch, log_as=\"test\")\n",
    "\n",
    "    def test_epoch_end(self, outputs):\n",
    "        self.val_test_epoch_end(outputs, log_as=\"test\")\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
    "        scheduler = {\n",
    "            \"scheduler\": torch.optim.lr_scheduler.ReduceLROnPlateau(\n",
    "                optimizer, mode=\"max\", patience=5\n",
    "            ),\n",
    "            \"monitor\": \"avg_val_auc\",\n",
    "            \"interval\": \"step\",\n",
    "            \"frequency\": self.lr_step_frequency,\n",
    "            \"strict\": True,\n",
    "        }\n",
    "\n",
    "        return [optimizer], [scheduler]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "   | Name                      | Type               | Params\n",
      "------------------------------------------------------------------\n",
      "0  | embed_content_id          | Embedding          | 892 K \n",
      "1  | embed_parts               | Embedding          | 512   \n",
      "2  | embed_tags                | Embedding          | 12.1 K\n",
      "3  | embed_answered_correctly  | Embedding          | 320   \n",
      "4  | embed_timestamps          | Linear             | 128   \n",
      "5  | embed_prior_q_time        | Linear             | 128   \n",
      "6  | embed_prior_q_explanation | Embedding          | 128   \n",
      "7  | embed_agg_feats           | Linear             | 640   \n",
      "8  | pos_encoder               | PositionalEncoding | 0     \n",
      "9  | transformer               | Transformer        | 365 K \n",
      "10 | out_linear                | Linear             | 130   \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9bf45be8878a4add84a1285754d7c0a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-8:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/threading.py\", line 870, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/site-packages/torch/utils/data/_utils/pin_memory.py\", line 25, in _pin_memory_loop\n",
      "    r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/multiprocessing/queues.py\", line 116, in get\n",
      "    return _ForkingPickler.loads(res)\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/site-packages/torch/multiprocessing/reductions.py\", line 282, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/multiprocessing/connection.py\", line 502, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/home/gautier/miniconda3/envs/kaggle/lib/python3.8/multiprocessing/connection.py\", line 629, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "/home/gautier/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "ename": "MisconfigurationException",
     "evalue": "ModelCheckpoint(monitor='avg_val_auc') not found in the returned metrics: ['train_loss']. HINT: Did you call self.log('avg_val_auc', tensor) in the LightningModule?",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    493\u001b[0m                 \u001b[0;31m# run train epoch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 494\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    495\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    560\u001b[0m             \u001b[0;31m# ------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m             \u001b[0mbatch_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_training_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataloader_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    562\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mrun_training_batch\u001b[0;34m(self, batch, batch_idx, dataloader_idx)\u001b[0m\n\u001b[1;32m    727\u001b[0m                         \u001b[0;31m# optimizer step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_step_and_backward_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure, *args, **kwargs)\u001b[0m\n\u001b[1;32m    468\u001b[0m             \u001b[0;31m# optimizer step lightningModule hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 469\u001b[0;31m             self.trainer.accelerator_backend.optimizer_step(\n\u001b[0m\u001b[1;32m    470\u001b[0m                 \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_step_and_backward_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, batch_idx, opt_idx, lambda_closure, *args, **kwargs)\u001b[0m\n\u001b[1;32m    113\u001b[0m         \u001b[0;31m# model hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         model_ref.optimizer_step(\n\u001b[0m\u001b[1;32m    115\u001b[0m             \u001b[0mepoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1379\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1380\u001b[0;31m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1381\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtrain_step_and_backward_closure\u001b[0;34m()\u001b[0m\n\u001b[1;32m    717\u001b[0m                         \u001b[0;32mdef\u001b[0m \u001b[0mtrain_step_and_backward_closure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 718\u001b[0;31m                             result = self.training_step_and_backward(\n\u001b[0m\u001b[1;32m    719\u001b[0m                                 \u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtraining_step_and_backward\u001b[0;34m(self, split_batch, batch_idx, opt_idx, optimizer, hiddens)\u001b[0m\n\u001b[1;32m    812\u001b[0m         \u001b[0;31m# lightning module hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 813\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhiddens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    814\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_curr_step_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, split_batch, batch_idx, opt_idx, hiddens)\u001b[0m\n\u001b[1;32m    319\u001b[0m             \u001b[0margs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild_train_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhiddens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m             \u001b[0mtraining_step_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    321\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_training_step_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_step_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_accelerator.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__training_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_accelerator.py\u001b[0m in \u001b[0;36m__training_step\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m     79\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-14-3ecc073db4e5>\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, batch, batch_nb)\u001b[0m\n\u001b[1;32m    317\u001b[0m         )\n\u001b[0;32m--> 318\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"train_loss\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    319\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mMisconfigurationException\u001b[0m                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-cceb53413772>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;31m# Train the model ⚡\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m trainer.fit(\n\u001b[0m\u001b[1;32m     59\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, train_dataloader, val_dataloaders, datamodule)\u001b[0m\n\u001b[1;32m    443\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'on_fit_start'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    444\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 445\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    446\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mteardown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/accelerators/gpu_accelerator.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;31m# train or test\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_or_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36mtrain_or_test\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m                 \u001b[0;31m# hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 533\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    534\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    535\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrun_evaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_mode\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_batches\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mon_train_end\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    188\u001b[0m         \u001b[0;31m# when a checkpoint was saved at the last step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_step\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 190\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_checkpoint_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshould_save\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_last\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    191\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_step\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36mcheck_checkpoint_callback\u001b[0;34m(self, should_save, is_last)\u001b[0m\n\u001b[1;32m    218\u001b[0m                 \u001b[0mrank_zero_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Saving latest checkpoint...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m             \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m             \u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_validation_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcheckpoint_callbacks\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_train_epoch_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    218\u001b[0m                 \u001b[0mrank_zero_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Saving latest checkpoint...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m             \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m             \u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_validation_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcheckpoint_callbacks\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_train_epoch_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py\u001b[0m in \u001b[0;36mon_validation_end\u001b[0;34m(self, trainer, pl_module)\u001b[0m\n\u001b[1;32m    184\u001b[0m         \u001b[0mcheckpoints\u001b[0m \u001b[0mcan\u001b[0m \u001b[0mbe\u001b[0m \u001b[0msaved\u001b[0m \u001b[0mat\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mend\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mval\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m         \"\"\"\n\u001b[0;32m--> 186\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpl_module\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_save_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpl_module\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py\u001b[0m in \u001b[0;36msave_checkpoint\u001b[0;34m(self, trainer, pl_module)\u001b[0m\n\u001b[1;32m    215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_backward_monitor_support\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_monitor_key\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;31m# track epoch when ckpt was last checked\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/kaggle/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py\u001b[0m in \u001b[0;36m_validate_monitor_key\u001b[0;34m(self, trainer)\u001b[0m\n\u001b[1;32m    479\u001b[0m                 \u001b[0;34mf\"HINT: Did you call self.log('{self.monitor}', tensor) in the LightningModule?\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    480\u001b[0m             )\n\u001b[0;32m--> 481\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mMisconfigurationException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    482\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    483\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_metric_interpolated_filepath_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mckpt_name_metrics\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMisconfigurationException\u001b[0m: ModelCheckpoint(monitor='avg_val_auc') not found in the returned metrics: ['train_loss']. HINT: Did you call self.log('avg_val_auc', tensor) in the LightningModule?"
     ]
    }
   ],
   "source": [
    "from pytorch_lightning.callbacks import (\n",
    "    EarlyStopping,\n",
    "    LearningRateMonitor,\n",
    "    ModelCheckpoint,\n",
    ")\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "from torch.utils.data import random_split\n",
    "\n",
    "seed_everything(SEED)\n",
    "\n",
    "\n",
    "# Params\n",
    "learning_rate = 0.004  # 0.0001\n",
    "emb_dim = 64  # 256\n",
    "dropout = 0.0\n",
    "n_heads = 2  # 2\n",
    "n_encoder_layers = 2\n",
    "n_decoder_layers = 2\n",
    "dim_feedforward = 512\n",
    "use_prior_q_times = True\n",
    "\n",
    "# Init our model\n",
    "model = RIIDDTransformerModel(\n",
    "    learning_rate=learning_rate,\n",
    "    emb_dim=emb_dim,  # embedding dimension - this is for everything\n",
    "    dropout=dropout,\n",
    "    n_heads=n_heads,\n",
    "    n_encoder_layers=n_encoder_layers,\n",
    "    n_decoder_layers=n_decoder_layers,\n",
    "    dim_feedforward=dim_feedforward,\n",
    "    max_window_size=max_window_size,\n",
    "    use_prior_q_times=use_prior_q_times,\n",
    "    use_agg_feats=True\n",
    ")\n",
    "\n",
    "logger = TensorBoardLogger(\"lightning_logs\", name=\"use_aggs_512\",)\n",
    "\n",
    "# Initialize a trainer\n",
    "trainer = pl.Trainer(\n",
    "    gpus=1,\n",
    "    max_epochs=5,\n",
    "    progress_bar_refresh_rate=1,\n",
    "    callbacks=[\n",
    "        EarlyStopping(monitor=\"avg_val_auc\", patience=10, mode=\"max\"),\n",
    "        ModelCheckpoint(\n",
    "            monitor=\"avg_val_auc\",\n",
    "            filename=\"{epoch}-{val_loss_step:.2f}-{avg_val_auc:.2f}\",\n",
    "            mode=\"max\",\n",
    "        ),\n",
    "        LearningRateMonitor(logging_interval=\"step\"),\n",
    "    ],\n",
    "    logger=logger,\n",
    "    val_check_interval=2500, # check validation every 1000 step\n",
    "    limit_val_batches=0.10, # run through only 25% of val every time\n",
    ")\n",
    "\n",
    "# Train the model ⚡\n",
    "trainer.fit(\n",
    "    model,\n",
    "    train_dataloader=train_loader,\n",
    "    val_dataloaders=[val_loader],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "# load best\n",
    "model = RIIDDTransformerModel.load_from_checkpoint(\n",
    "    \"model.ckpt\"\n",
    ")\n",
    "\n",
    "model.freeze()\n",
    "model.cuda()\n",
    "print(\"done\")\n",
    "# trainer.test(test_dataloaders=[val_loader])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "### NOTE THIS IS NOT TEST SET\n",
    "train = pd.read_pickle(\"riiid_train.pkl.gzip\")\n",
    "train[\"uid_count\"] = train.groupby(\"user_id\").row_id.cumcount()\n",
    "train = train[~train.content_type_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = train.sample(10000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Subset\n",
    "\n",
    "# Init DataLoader from RIIID Dataset subset\n",
    "loader = torch.utils.data.DataLoader(\n",
    "    dataset=Subset(dataset, t.row_id.values),\n",
    "    batch_size=1500,\n",
    "    collate_fn=collate_fn,\n",
    "    num_workers=6,\n",
    "    pin_memory=torch.cuda.is_available(),  # if GPU then pin memory for perf\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b4ef6bbf40a4a299d36a6f35698efab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6667.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "for b in tqdm(loader):\n",
    "    b[\"steps\"] = torch.ones(b[\"length\"].shape, device='cuda')\n",
    "    result, sequence_indexes, user_indexes = model.validate_n_steps(b)\n",
    "    preds.append(result[sequence_indexes, user_indexes].cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8069748981960596"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "predictions = torch.cat(preds).numpy()\n",
    "t[\"predictions\"] = predictions\n",
    "y = t[\"answered_correctly\"].values\n",
    "y_pred = t[\"predictions\"].values\n",
    "\n",
    "roc_auc_score(y, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pd.merge(\n",
    "    train[[\"uid_count\", \"row_id\", \"answered_correctly\"]],\n",
    "    t[[\"row_id\", \"predictions\"]],\n",
    "    how=\"inner\",\n",
    "    on=\"row_id\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAy0AAAINCAYAAAAzy5CEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAACR2ElEQVR4nO3dd5wTZf4H8E/qJtvYhWXpyyIsZVFElGJBTiwoHhbUw3IWPASB8/TOAp6e2Mt5YuUQxYYNy3Eih+1+wimoCIiIsgJLWfrCsr1kN/X3R5iQzE6SmWSSTJLP+/XiBSSTmWeemew+33me7/Po6urqPCAiIiIiItIofaILQEREREREFAqDFiIiIiIi0jQGLUREREREpGkMWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJqW0KDlm2++wZVXXolBgwYhLy8Pb7/9dtjPbN68GePHj0fXrl0xaNAgPPHEE/B4uNQMEREREVGqSmjQ0tzcjNLSUjz++OOwWq1ht29oaMCll16KwsJCrFixAo8//jief/55vPDCC3EoLRERERERJYIxkQc/77zzcN555wEAZsyYEXb7Dz74ADabDfPnz4fVakVpaSm2bduGf/7zn/jjH/8InU4X6yITEREREVGcJVVOy9q1a3HqqacG9MqcffbZOHjwIHbv3p3AkhERERERUawkVdBy+PBhdO7cOeA14f+HDx9ORJGIiIiIiCjGkipoISIiIiKi9JNUQUthYSGqqqoCXhP+X1hYmIgipZXy8vJEFyGlsD7VxzpVF+tTXaxP9bFO1cX6VBfrU11JFbSMGDEC3333HVpbW32vrVy5Et26dUPv3r0TWDIiIiIiIoqVhAYtTU1N2LRpEzZt2gS32419+/Zh06ZN2Lt3LwDggQcewEUXXeTb/vLLL4fVasWMGTNQVlaGjz/+GM888wxmzJjBmcOIiIiIiFJUQoOWH3/8EWeeeSbOPPNM2Gw2PPbYYzjzzDPx6KOPAgAqKyuxa9cu3/YdOnTAv//9bxw8eBBnnXUW7rzzTsycORN//OMfE3UKREREREQUYwldp2X06NGoq6sL+v78+fPbvTZ48GB8+umnMSwVERERERFpSVLltBARERERUfph0EJERERERJrGoIWIiIiIiDSNQQsREREREWkagxYiIiIiItI0Bi1EREREpMiyChvGf1KFZRW2RBeF0gSDFiIiIiJSZH5ZE3Y0OPFiWVOii0JpgkELERERESkyvTQb/XKNuLk0O9FFoTTBoIWIiIiIFJlQbMXy8Z0xodiq+LORDi3jkLT0xqCFiIiIiOIm0qFlHJKW3hi0EBEREVHcRDq0jEPS0psx0QUgIiIiovQxodga0bAy4TPzj/a0RLIPSl7saSEiIiKipMAhYumLQQsRERERJQUOEUtfHB5GREREREkh0qFllPzY00JERERERJrGoIWIiIiI6CiuB6NNDFqIiIiIiI5isr82MWghIiIiIjqKyf7axER8IiIiIqKjmOyvTexpISIiIiIiTWPQQkREREREmsaghYiIiIiINI1BCxERERERaRqDFiIiIiIi0jQGLURERESkKVzgkcQYtBARERGRpnCBRxJj0EJEREREmsIFHkmMi0sSERERkaZwgUcSY08LERERESUc81goFAYtRERERJRwzGOhUBi0EBEREVHCaT2PRUlP0LIKG6ZtymCvkYoYtBARERFRwk0otmL5+M6q57KoNexMSU/Q/LIm7LHp2GukIgYtRERERJSy1Bp2pqQnaHppNoqsHs32GiUjzh5GRERERClremk2XixrijqAUDKj2YRiKwY62lDCGdBUw6CFiIiIiFIWp09ODRweRkREREREmsaghYiIiIiINI1BCxERERFpkpyZv5TODua/PRe0TB4MWoiIiIjSlNYb7XJm/lI6O5j/9lzQMnkwaCEiIiJKU3Ia7YkMbORMM6x0UUr/7bW+oCUdw9nDiIiIiNJUwHTAjjrJbfwDm3jPwiVn5i+ls4OJt+fMYsmBPS1EREREaUrOKvTsjSAtYE8LEREREQXFdU5IC9jTQkRERERpQesTD1BwDFqIiIiIKC1wtrDkxaCFiIiIiOIqUT0ezM9JXsxpISIiIqK4StSMZMzPSV7saSEiIiKiuGKPBynFnhYiIiKiNLaswob5ZU24JN+AkpL4HJM9HqQUe1qIiIiI0pgwVGvxgdg9y+asXRQtBi1EREREaUwYqnVld2fMjsFZuyhaDFqIiIiI0tiEYiuWj++MswpcMTsGc1goWsxpISIiIqKYYg4LRYs9LURERESkSLLkqMSynFqoAy2UIV4YtBARERGRIvPLmrC5xo6Zq2s13WCOZS6NFvJ0tFCGeGHQQkRERESKTC/Nhgc66HXQdIM5lrk0WsjT0UIZ4oU5LURERESkiJCf8mJZk6YbzLHMpdFCno4WyhAvDFqIiIiISLF0ajDHi7DQ5/TSbNatCIeHEREREaWZdErgTibplKOiFIMWIiIiojTDxrE2pVOOilIcHkZERESUZqaXZms+HyUdcchdcAxaiIiIiNIMG8eUbDg8jIiIiIiINI1BCxERERGFxeT99lgn8cOghYiIiIjCYvJ+e6yT+GHQQkRERERhcWar9lgn8cNEfCIiIqI0JixoeEm+ASUlwbdTI3k/1RZP5IQG8cOeFiIiIqI0JgxxWnwg+mfZ4XI8OJyKIsWghYiIiEjjlCZ8K9leGOJ0ZXdntMUMG5Qk23CqYPXIBPz4Y9BCREREpHFKeyiUbD+h2Irl4zvjrAJXtMUMG5QIx9L6kCohKHlkQ4NkPUrVLwOZ2GLQQkRERKRxSnsoEtWjkSxBSThCUAJ4JOtRqn459C22mIhPREREpHFKE76ZIB6d6aXZeLGsCTcHmTBAqn79P0PqY9BCREREROQnkqCPgWJscXgYERERERFpGoMWIiIiogRg4jaRfAxaiIiIiBKAidtE8jFoISIiopSk9Z6MeM3wlch60NI1iOVaNxR7DFqIiIgoJWm9J0OYHhhATBvHiaiHcOucJEIs17qh2GPQQkRERCkpWVZfj3XjOBH1EG6dk0RIlrVuSBqnPCYiIqKUlCxT0MZ6fY9E1EO4dU4SgWvdJDcGLUREREQJlIqN41DntKzChvllTZh+NEgT/p1qdUDq4vAwIiIiShmJSp5m0rZ8/sPh1Bwax2uQ2hIetCxcuBBDhgxBly5dMGbMGHz77bcht3/55ZcxYsQIdO3aFaeccgrefffdOJWUiIiItC5RydNKj5vODWz/XBE180aYOJ/aEhq0LFmyBLNnz8btt9+Or7/+GiNGjMAVV1yBvXv3Sm7/yiuv4P7778ddd92FNWvW4O6778add96JTz/9NM4lJyIiIi1KVPK00uMmcwM72oBLmDVNGEIm/DtaTJxPbQkNWubNm4err74a119/PQYMGIAnn3wSXbp0wauvviq5/XvvvYfrrrsOl19+OYqLi3HZZZfh+uuvx7PPPhvnkhMREZEWqdkIjuVxk7mBrdWAK5Jrv6zChpFLKjFqyaGgQVg694ppScIS8e12OzZu3Ihbbrkl4PWxY8fi+++/l/xMW1sbLBZLwGtWqxU//PADHA4HTCaT5OfKy8vVKTSxLlXG+lQf61RdrE91sT7Vl6x1OhDAMyUAHHXQ0inIqc9L8g1YbDPi4nwbysvrYl+oGHp6UwZ2NHif4T/9QxUGOtokt9lj0+HpH2yS74eSrPdnopSUlAR9L2FBS3V1NVwuFzp37hzweufOnXH48GHJz5x99tl48803MWHCBJx00knYuHEjFi1aBIfDgerqanTt2lXyc6EqgOQrLy9nXaqI9ak+1qm6WJ/qYn2qT1yn/rNSxbunJZHHVovce7SkBJgqY39q10ks6vjPJhse2VAPQIc/D8tFicR+/2yy+aZvlno/GH7n1ZXwRHwl7rzzTpx33nk477zzUFBQgKuvvhpXXXUVAECvT6pTISIiohAiGZKTyGFLiR4ypcUhTGrXSSzqeEKxFWsmdsWaiV2CBkKJGnJIgRLW0u/UqRMMBgOqqqoCXq+qqkJhYaHkZ6xWK+bNm4eDBw9i06ZN+OWXX1BUVIScnBwUFBTEo9hEREQUB5E0UBOZJzK9NBv5Zh2qW90JCRwSHTRJUft6JHMeEEUvYUGL2WzG0KFDsXLlyoDXV65ciZEjR4b8rMlkQo8ePWAwGPCvf/0L48aNY08LERFRComkgZrIJ+ITiq3oaDGg1u5OSOCg1Qa9R8V9xfv6arH3Kp0ltKU/c+ZMvPPOO1i0aBG2bt2KWbNmobKyEpMnTwYATJs2DdOmTfNtv337dixevBg7duzADz/8gBtvvBG//vor/va3vyXqFIiIiCiESBt+yTgkJ56Bg7hehfoCoJmGthZ7f5RI9vKnmoQl4gPAxIkTUVNTgyeffBKHDh3CoEGD8P7776OoqAgAsG/fvoDtXS4X5s2bh+3bt8NkMuGMM87AF198gd69eyei+ERERBSGf8NPCEBSIWldirDuSDxI1Wuo1xNhemm2L4E9GSV7+VNNQoMWAJgyZQqmTJki+d7y5csD/j9gwACsWrUqHsUiIiIiFUg1/LTUsA5Fy8FVsAa1lhra8QziYiHZy59qmAhCREQUA8k6Hl7tcksN80p0/oXcc9Ty8KBgw+cmFFtxc2k25pc1Jd295y9Zvz8UOwxaiIiIYkDLDd5QxOWOReMx0fkqcq9NooOrSCXrvecvHucQyb2t5DMrjxhU/+6kczDHoIWIiCgGkrXBKy53KjSAxeRem0QHV5FK1nvPXzzOIZJ7W8lnFh8wqv7dScXvo1wJz2khIiJKRck6Hl5c7khzJLScD5Ks10auVDi/eJxDJPe2ks9c2d2JpbVWVQMvLeUsxRuDFiIiIgoq0sZjsiTbq0HLAVowyVhmtUVybyv5zFkFLkw9tXMkRVPl+KmGw8OIiIhIdckyREmNHAG5Q3a0lI+QzsOMBFq6HhQegxYiIiJSXbLkg6jReJcboCk5Vqwb1OIyL6uwYdqmjJg34OMZKIQ7FgO35MKghYiI0g6fsJLAv/Ee6X0hN0BT0vuktEGttOziMs8va8Iemy7mDfh4BgrhjpUsvYHkxaCFiIjSDp+wKpPKQZ5/4z2W94WQQ3KzzBwSpQ3qaMs+vTQbRVZPxA14ufdIPAOFcMdKlt5A8mLQQkREaYdPWJVJVJAX72AplvdFuDqUOlePgv1HW/YJxVYsGNIWcQNe7j0Sz0CBQUlqYdBCRERpJ10aM3Ib/eG2S1SQF+9gKZb3Rbg6FJ+r0nNP9D0tlSMTi4AzlXv9KDQGLURERClKbsM33HaJahCnUo9YuDoUn6tWzz1Y0CCVIxOLgJNDO9MX12khIiJKUXIXohO2O6XAhPGfVGF4gQnrjjgSvoZHOq1JIT5XrZ673PV3YrUIYjovrpjuGLQQERGlCPGCgXIbvsJ24z+pwo4GJ36pccBq1KXFwpCkjNygQe2gy//eXj5e3QUbKTlweBgREVEcxXJMvhozSPXLNWJy/0xNDk3SAq3lVMS7PPEcKuh/blL3diTnHo/6Eo6x8oghIfeL1u5RtTBoISIiioLSBkIsx+QPLzDB5vTglAJTRJ8XGqQPjMhL2YkKxNdLyfVbVmHDzNW1KKt1xDWnIlQZtbRgpdr8z00qxyeS71I8cmKEYyw+YExIDk6q5v0waCEiUkGyNQZIPXKnsl15xAAgtgnW6454h3WtP+JQfd+pIppZuuaXNUEHD9wexLUXKlQZY7lgZaL5n5tUD08k36V4THAgHOPK7k5ML81GvlmH6lZ3SkzdnUgMWoiIVJBsjQFSj9ypbBcf8KaRJnJaXYpulq7ppdk4vqMZ887Ij3kvlP+DkFBlVHI/Jdv9Ee7cIvkuxWN4m3CMswpcmFBsRUeLAbV2d0pM3Z1ITMQnIlIBZ7RJX+ESjoV74+L82D9l1dqMU+KJAbQgmlm6wm2r5vn6PwhRqwGqtfsjXfD3gzrY00JEpIJUfbJF0fN/6hoL4YYmxnroorD/OWvr2h0nkT2QUue9rMKGkUsqMWrJIU1OhOBP7V4Rte4DDoVVjr8f1MGghYiIKImFayjHOnAQ9v/atpZ2x4nHcKRgjWip855f1oSdDa6o6iNUo13N85Xb0A1WHv/Xl1XYMPWrGnx/2I5HNtRHVa5HNjRg3WE7HtnQENV+iJRi0EJERJqSCk9yY3UOUvsN11COdeAQaprkeDxhDhaUSZ339NJs9M01oG8U9REqCEzEE3X/8vj3JD2yocH3+vyyJtjdgMsDNNo9Ud2bHnh8/5KSCt9f0iYGLUREpClam9QgkkZYrM5Bar+xSFZWItHTJAcLyqTOe0KxFWsmdsWaiV3alVPudU5k71G48vj3JAEe3+vTS7NR0sGAgR2MyDHro7o37x3WASMKzbhnWAfJ9yO599M10EnX844UgxYiIlKFWr+AI20URnr8cJ+T0wgT70PNhq3cWaSU7CeWn1GDkuOqFZTJbWwnsvcoXHn8e5LuGdbB97p/sHbPsNyo7s1w5x/JPaq1BxXxkq7nHSkGLUREpAq1fgHLaRRKNWojPX64z8lphIn3oWbD1n/f0ew3EQvxRRr0JKIxp6XpgCMtS6ieJP9tYt3zJvT6yL3uWqr7eErX844UgxYiIlJFPH8BSzVqIz1+uM/JaeTF8tzV2nciFuKLNPhIRGNOSzM8aakskVB63ZP9fCOVrucdKa7TQkREqojnGhBS6x5Eenw1yh3Lcxf2O9+vFyfS/Sj9bLTnFen6FFpbTyTW681ocT0bwFuuhzfUQwcd7hmWK7tsXJeEYoE9LURElBBKhg6Jt023J5TJOvY9EdcpFnk4atW/kumZtSDSKaL9r7t46uVw14bJ6RQMgxYiIkoIJQ01tRt1ydYw4tj3Y9SYOEEptepfyfTMSsTqflZ7img510arARwlHoMWIiKKu2UVNlS3upBv1stqDIkbddE20pKtYRSvHotkCOaEa/fIhnpM25SBOWvrYjZzmyDahR4FSqZnViJW97OcxP5w/M9ZzrVhgE7BMGghIoqTeDYItd74nF/WhDq7B50selmNIXGjLtJGmlAvwwtMMWsYJfN1TobVzoVGbaPDg00Nery0pTlgccX5R3MpYpV7Ekkvj/A5ADEJPoU6OaXApLnvvf93V05wlm5DP0k+Bi1ERHESz6f7WuxJUGu9ESDyp7FCvaw/4gjZMJJqnIZrsArvP7KhAZtr7Ji5ujbmjUe1r7P/audaDXyFRm22SQcA6GjWBSyuGG19hDrvSKfHjvX3UaiTdUccAcfR6jVUU7Bz1OK5a7FMmuHxAE5nyE0YtBARxUk8n4ZqcYiFWuuNAJE/jZVbL1KNzHBP0R/Z0IAdDU402l1ocgBOtydg25VHDAHXXY3FMNW+zv6rnSttaMeyQSa173uHdcCQXDceH5UfsLii3PqIJCk+0umx47UoqPg4Wnx4obZg56jFc9dimVQnBB92O2CzAU1NQH09UFcHXXU1dIcPQ1dZCd3+/dDt2QNdRQX0O3dCv2MH9BUVIXfNKY+JiGLMfzrT5eM7Y/wnVQGN91iI95SxcqZs1cI0qHLrRaqswcovNETyzTrkm3XY3eRGhgEw6nUB2y4+YMR+x7HrLm7AyJ3y1v9z4gZytFPniutHyfUSB6VyyC2v1L4nFFsx0NGGEr/PKbnvg5U31H2aiGm1ldSr+Dha+M7FWrBz1OK5a7FMQXk8gMsFuN3ev4U/Hg90wr/933O7oXO7va/FCHtaiIhiTNw4nV6ajXyzDtWt7oim+42GGvuKZDX6WOcaqE3qiXm4p+j3DOuAjhYDzHpvwDLvjHwA8NXVld2dAU/B/Z+Kh+vF8a/rUE/t1XySKz7fSJPMQ5Fb3lj0HEabFB+voT7RnHs65IcEO0ctnntCyhRlz4d+zx7o9++HvrIS+qoq6I8cga62FrqGBuiamqCz2aCz26FzOmMasACArq6uzhN+MyKgvLwcJSUliS5GymB9qk+rdbqswuZ7uib8shJ6W/rlGrF8fGffdlJPnaW2jZSSfQWrz2Bl9z9H8bmoeQ6xIJR3eIEJ6444Iu6pENeD/3k/U1IX9P6UukcA5dc+2H7UEItrGG154/Gdj8f3UivKy8uxxdRTkwtdhqPFBTpVvT+Fng9xD0cCez5iwd2vX9D32NNCRJqSiomKUk/XpJ6exmoNB7X3JbWPcLN7aTHHBmifj/LatpaoeirE9RDuvMPNKqW03tR6kiu3hyfaxQJDlVfpz4JY/eyIx/dSS5I17yJpyi30fLS1Be/5OHgwsOdjx45jPR9792qm5yPe2NNCsmn1KXayYn1Ki+bpZbLXaSyfkkdCqj7lPs3UwrnIKatwv+WbdehkMeCUAhPWH3GoUm7x8aWeYovvd608LZb7PZSzXaTfaf9r09FikKyTl77biY9qczDdb4id2j0fsbqXtXKt/Qn3aKK/u5GI+88cGT0fu7ZvR5/evY/1fBx9n4IL1dPCRHwi0pSkSlSMAa3/OpObFBzviQCkyCmr//0Wj+Rx/4URheFoOkBytqdE1p/c76Gc7YJtE64ehc9VNDixo8GOmtb6dtsJkxs8sqEeHkD2YqVKqHEvS52rnGudiMBGC9/dSERc7iDBh29oldSwK5nBh76lBTpb6owaSDQODyOimFI6ZEOLyZPxIieZPdFD5xIxJCbS85YzxfSEYqsvEV7uVMTRJI8LrwE6yfVixJ+Re+5q3xtyv4fRLBYoVY/+5yFcmxq7G95BLrp2+xYmNwB0ihYrjQf/c5E6VznfpVD32rIKG0YuqcSoJYdSajhtxPyHXbW0+IZd6Wproauuxn9/2IPr3v4FX36/Q9awK111dfBhV+wtSQgGLUQUU0kzzlgDwjVitFCXQgMUQNwCqEjPW7zg3iMb6mWtyxGuoVjd6kK+WY9TCkwhG41SAZFQpnuG5QaduUr4zJy1dZi5uhZltY6gZQnVKA62rVbIyeuaX9YEsx7IMupwz7Bc33biXKBg9ZlI/uciJw9MSriZ4nY2uBL+M0F1YYIPX87Hvn3Bg48DB9oFHx9urMS+yno8vPoQVu5qYPCRhBi0EFFMpWqyaiyEa8SosXCeWts/sqEB6w7b8ciGBlnbyzVnbR2K3jqAOWvrfK9Few+JezekEqr9p6AO11AUnuivO+II22gMFkyEutbzy5qwucaOFzY3w+n2wO1B0LKEahTLKUc4sQx25ExQMb00G8d3NGPeGfkB2wnns/iAMei+Es3/XCItX6jPTS/NRt9cA/pq9edrjIIPXUMDdM3N0LW2Kur5uKpfJjzQQa8DFm9viUMFkNqYiE+yJXuSs9ZopT61mAwaKa3UaaL4X0ulSclSydKh6nPkkkrsbHChb64BayZ2lVUmOfdX0VsH0OjwIMekw57fdw+7vZLjhkrUlZss7r8PAHhkQz0Aby9AsBmwhO0HOvbJuj+XVdgwc3UtnG6Pb72XcPuWs6ZIuG2l6ixcMvyyChse3lAPXYg6iAXhfC7Ob8TUU4+LyzGjFcnP2nj/fJb8zvvnfPjldwTkfPi/p+GE8xX7W7F4ewuu7JeJsT0sMT9eRUUFiouLY36cVMIpj4koKC0MOSJ1SD15D5XP4U9pb8a9wzqgb64BgC7kvuXeX8IT/dFdTcgx6TC5f6ascog/L5RF6rjhnlqHO3/xApkTiq1HF5QM/qs0kifsE4qtmHdGPoqyDeieaQh6jkr2LWfbUHkXwXqp5AxRikVvjXA+ZxW4It5HPIbMKRnCJ0XVn8+hej6OHIHu8GGYqqri1vMRjRX7W3HTVzVYsb9V0efG9rDgpTEd4xKwUATC3DecPYwozaX7bF2pRDwTlv+Uukpm+xIaWJfkGxCsY2BCsVXx7Fyh+PcMhethkTMTk9L7OtTMQ8LxalrdqLW7JWcDi3S2L/9zEfYnnJd437GeWUyqzoR6Efcw+X+mptXb2yRnaFoienOD9VZIlUvtng3xgwShDoMdR/y61DVZtqsFL/1Sj5sHWHFhT3Ngz4fETFeKZ7tqVRYIiK3Y34p3t7fgqhj2Zry7vQV7mpxYvL0lZQKQeNSbqpxOwGaDrqXF+7f/v1taAv7te81mA/z/Fr9ms6GhtjboIRm0EKW5ZJ3eMphEDHfTyhA7/wbm+E+qML0029foEXpc5JTRly9gM2KqxPv+q8f7T9cbqkzhhAsy/OtYWAiyprXBV15xWaSCMOHcw/1fqj4219hhdwG9c9rPBhZN0O/fqPUAIQMvJXUkZ7iYUG/rjjh8nwn2uWDvybm+iX4wEixokiqX3ABLbl1LPUgAvMPudtbZ8fLPdZjQTe8LMhatr0J1fRsWN1TjosyOuCjDhYuGuABXC1DhXf/jP/+rhrPJiWWHjZgwpqMKNaSueAQUV/XL9A3zShUxqzePB7DbAwKFdkFEqMBCCDzEr9nt6pVRJua0kGzpni+gNtan+srLy3FreV5MFpgLRSofIpIGpFQjGoDigEiqPEoW+AuXLxBqX7EK4MZ/UoXNNXZ4oEOOUYfDrW70zTWgo8UQtu7F5Q33f6n6mLm6FnodMDjfFHSbcOcttbikuAcjmsXxlFxjoT6bHECGASFzZ7RMzs9RtXN/AFFdX1AQmNchrOUh1fNxtMdj5e5mLN7e3C63Qk7ORbR5GaGe6KuRg7Fifyte3NwIQIebB2cnR69BjCipzxX7W/HetkZc09OAMflu6Z4I8WutrcG38+/lcEU+jDIYj14PWK3wZGb6/vb9W3g9MxMeq9X3t+81i8X7t+h9WK1wDxgQ9JgMWkg2NrLVxfpUX6JWc5Zq6Pg3ssM1BkM1ooWn70qCsGUVtnYJ4nPW1uG1bS2Y3D8TD4zIk7WfYPeoGgntUvsM1ej3Dxy6WvXoZDEEbehLrTLvv024/wcrX6ht5Jx3rINqoYyniHpPgm0rJPu3uYAcsy5oQKZFx4YwqpiIHySxXCr4+HJ3C97d1ohrjrNgbI8Myd1pdbjPTV/VYE+TE72zjXhJ1FOjVuJ4qGPIodW6C+BwhO6JaGlB7f796GixhB8qJbwW5dC8YDxmc2BwICeIsFiArKygAQgyMgBd+7WTohUqEZ9BC8nGRra6kr0+4zUkSs5xQjVgEjV0K9zTeXFvSrBGtPi9UMcL1rtwc2l22J4CKZHco0qeavuT0+iXu+9IyxBqf3LuwXDHjFdQrXQmtFMKTFh/xBHXQF9cjkh7E3uYHFhxWVHgm6LZrj7d1YxXf23ElBILzu9hbt/74fG0y/lQo9EcbcM9VkL11KgVtETbG6Rq3Xk8QGtr+HwL8Wui99sFG05ndOUKVlz/3otQPRZH3w8ILETbC5+BMXmyQRi0kCqSvZGtNdHWZ6LzKCJ9oh6L44RqwMgZ+hPNUKxg+5IKPkINWQq3r3CkeheE3hYAONjilNXr4y+ae1Rp+dUONNSk1r3uX59qfH9DJXOHqstE/+wQ8++VnNw/81gvUW9L0OFVX+xqwqItjbggpwG/H9ojZMJ5JA1gNRrNz21qwL92teKyPhb8aUhu+A8oEKueiPd+3Iv/a8hKXA/H0eTub3fW4outNbioEBie7ZQXRIRI7tbFYDYzj8FwLGAQAgrRUKkGhwM5Xbq0Hz7lH5T4f9ZiAfTpPbFvqKAleUIvIgqQ6Bl54pVcK+c4wjYX57efujTc50MlQislvibixFtxkvUjG+p9CxqKjyfs65EN9YoTfoHA2b3yzToc39GsakAgJ3ldSV36z8bl/3+1yxWJWNzrwepHzjA58Vo84n0ES44PNQtazInW+PAfenVb11bM2l4Nh8uFjw650cHowbJKPS4+M8/3cXEj/fwc4PzhJlRUNIYdUhNJ0rbwmePzjbjpq5qIGvE/1zphMQK/1Kr/RD5WSdvLDxtwyCljvx4P0NYWPmAIM3xKCEbEyd3jjv5RiycjI6BXQrLHQjw8SjwcSvQaTKawxz1YUYEMrtOiGgYtRBqipMEVq6BBSaNJjR6WcMeTMzuRsE15eZ3iz4vrMZo6DXZNllXYUN3qQr5ZLxlUSE25KsyGVd3qDtgmVNK++HqIZy7yn1Us2sZquKAqkvsz0kBcTkM+GrGYYS9Y/YQr/8Mb6lFe78KPR2owdWBW2NnbpPZthAc2J3BKQfhGFyD6jhZlSAYgUgsNfrnHhne3NRzN+wjeAD4/F3jV2IodNhd0AAzQ4eq+gT0TQiP9xc2NinsYhO3ePboKupzPje3hLbPQ4xJJcBCLGa6E4O2EfCN0gLx9u1ztgwOpIMJmw5xdlTjY6EI3Txt+WdSCPkY7Cjxt0sOn3G7VzksQLLl7k82IGn0GjJmZOK1vfvC8DPFQqaPJ3TAYwh+cNI9BC5FG+OdAyGlwxWqq4nCNJjmNQiXBV6J7jPyf8Isb/nLPI1wgN7+sCXV2D/rlGgICDakpg4X6qGl1oaPFgHE9M3z5Bv7vy1m7Q3yPiBeqi6ZHQmh0i4OqYMdWsk+lQWOwtTDiJZLeHfGUzMKq8uN6ZoQMRHTQwQ3A7gbWH3EoenDgf81cbid+rGwBWi2BwYfE6uf//qoKzgY7/nNQj4vH5Ic8hn+PyHvbW7CvyYnF272N21DBxs2Ds0POOCUEAHVt7oiCCLk9E+IenWgCDyHwUSRMcvfhjVU4rbEZ3WDHnYUe4DsZQ6Xa2mQffujRP3L4ei/E+RbBEr79ggjfUCmZyd17/XJk2rSamE8xx6CFSCPmlzVBBw/cnuCLtMVDuAXQ5DQKlQQi8WhkKhnKJPxf6ql9sBm4xL0O4rUvxOcobK8DgvaOCMGAsI3QSyIOdIYXmPBLjSPkU3OpICnaYFFodAdbdND/uHLXQxEHOpGshQEA8U7UjLYuhVXlAWD9EX3IiQg88KCHVY8coddOKudDYsar/9vdjH9va8Idx1mgy3J7p9vtkQn9vpaw5buu2ITF2x24sl/guUnlVfgHB/4N/nBBQ7gGvvC+f4K3EsGCD/E5iMsZtFxud/vk7mBrXsiZNUpmcvdNis76GPFQp2BDpWrb2pDXvTt+bTXi63oDRvbJw5CeHSSnplWS3O1fz8DRALaX37/7WTE2P/z1j+R48c7N8T/2cWHKEm05JevVb19JMQubAkzEJ9mYiK8ucX1qMRFZ7nocANo1TiM5l2jzEYLdo8J55Jt16GgxhExc9l+Z/WZRY/iGlTVwA8gx6bDn990DgoH1RxyoPporYHN6YDXqQibZh6sfcYAU7FoomahAvJaJnNmuxPUpdyVvqXoXcinkJLTPWVuHFzY3I8MAnFRglr3+jdxZyGI14UKo/YWaQvqx9TUwuN2YfWK2d5VzcfDhcuHa/6vCvgY7jsvU4dUx+d7XIa9houZsTMLx6tvcqHe4A/YZLPk82tmkgpE125Xfyt3iwOLlHw6jpaEZPXR2TOoG7DnciJ2HGjEow4FuaJPOwRD2EwNOvQEtGVbYM6zIz8uSTO72DZ+yWPDWAWC/JwPZuVm4+ZTOgdtZLPBkZXl7L2Qmd6s1e5iY//3nAST/reYMa4mcuc3/2H/t3YBHducGLUu05QxWr8K+tDqDXShMxCdKAKUNo1gN9wonVDnFT7Clcgce2VCPAy3ugGFtkZ6L0uRzuecmJz9E6M3wzz3xHzpW0+qGWQ+0ur1Bi/BZcW+IeApZKXLqZ90RB6xGHdYfcUjmxAhBzeiuJl+AFUyo66i0foP1Kki9LvQCGXVArd0bvPTLNeKUAlNAbo1UeV7b1gIXgBZXYO5FuCFuoXruhGFYe5pcyDDo1J9wwePxNpDF63ocndXKWFMDXWWl77Uv97Tg3a2NuKavBWuGCg35NuCg9HFv7KXD4u0uXNkn0xewAPKGPqmZXyEcr4NJh97ZxoB9hko+D/uENILk7u6HD8NiMEgGFr4Zp0Ks3H2r6P8lR//I4bFY2idyi/IxPj8CVOosaDBa4LRYkZmbhT8M6xzQY+Hx6/FYcdgla3FJIUgtAPB/R7d3afhJuvj+C/bvWB0vngKO7WgIWZZoyxmqXtXYv9awp4VkY0+LMuGe+GqlPsP1QgTb1gPv+HpA/nS6/kGE1OJ3QkM8x6RDk8Pdbp/hGtviOh25pBI7G1zom2vAmold2z0Rl7Myuv85d7IYfL0p/nUgLOCopmC9P0K5it46gEaHx9fro4TcKXyD9bRI9SqEWmDT7gJ65xh99SRnDZlQPS2h6iXcea87bIfLA2SbQtyvUiuZ+/V6fFHRjDe3NuL6vhac090cdKpdMfFTbLWegsaqFyPU8d7f2oire+owJs8dEDBs2leP7yvqcWaeC4MynL7gYUV5DVzNLejkth+bwjbeyd2i4VDhhkq1m5rWPwfDYpGV3C1cm+Pzjfil1hn0GikZxiN136g1DChWPS3pivWpHHtaiBIgEQnBgPKn6OESqoNtW2d3I9/sfXraLdMoq+EuNDJ/qfH2IoiPJfQu5Jh0aHTo4HR7MHN1LYDgs22Fqoc9TS54m0A63z6CzXAl9GYYgYBpiMXX8eEN9cg3e4da1NndvuR6tYnLKs4z6perx0/VLozueqwXItIcEDnCTTYg1Xs0/WhAkmEEOln0kr0hD2+oR5PDgwxDYPL5AyPycEphRrtyBqsXSaLg45bewAOHmtDQ6sQNPSy4yGoADta1nwErTPAxLg8w9QTmb6jCyxukE8flUOspaMjx/h5P++TuUAvsBcvLOPq3o6kFF7S04GKndO/FqKN/xC6QeS7tkruF4U5+//YPLKpbWtCxVy/p4VPC58zmmKzcLYfcXAwlUxhL3TexmgKZSEsYtBDFSLyHe0W6BoNQTqGXI1hCt9QK7kLwIrfhLjRWgw2fEjemxbOpKWlszy9rglkPmPTenhCp8/BPJheO5fYATrgDhqgJQ7/8ewTE+S6xIBUoCL0UNqcHBVY96vzajvPLmrC5xh4Q6EmJ5N6MJNlc2C5U4PHIhoaQn59QbPUGEg5Hu96Pi3LduGiEDnA1AAdqfe8FCz4uzAA+ctdjj92JnXuboTtO/kADqYTtvU3ekDjShmK7Bq1ayd3i2af8hpJFK0Moqk4HnTDNrETvxEFPBsraTDiuaw56F+YEzctot7CewpW7ayoqkKuxJ9mR9HooWRdGKhBKtWFApCKd7tgfAB4hgPf/W7RNu9fCvO4Rvy93v8FeC4JBC1ECLKuw4elNGfizqf2igpHyX0hQGHbjP5VquJ4QoZfj832tWHek/Voe/o3W5eM7h505Skq4xrJ4+uF5Z+S3WzBRbn2J1yiROg//4wkzt03un+lLqhdPLew/u1s8glKpsoYK/ISeDbnTZospyW+Sy7+O4XYHrvPhcuH+EmDWt/Uwetx4/9t6XJSRL6vnQ2nDMKL1LfyIn2RfXWzGm3V1sLS14YYMHfSbD7Zb80IcbPSsqoJFr28XWNibvMGFxR56kcRIOQwmtJgtsFus6JiffSxB+2hORsihUKKhVKvr9Xj3ANC/MAs/17mC1v9dSZgAHIlwM6jJnS0q2nVhxIFMqs0apQl+DWuPf+M6RCPcnZkJT3Z2++1Ef4LuL1jjXmq7YK+lEAYtRAkwv6wJe2yRJwJLkWqkj/+kyjeVarhjhRomJpUIDoRfxTxUIzjYdLj+PUVC70KoVdL99zPQ7zUhWAtVT6HqThyQiT8ntUjjsgobZq2pRa3dg7Hdzai1I6rJBKTKGipYCtazIZfsNV/81/IIs8AgXC4sXXkE7gY7lh804GJRA3a8FbAMdHufEvfIhK65WVZZww2H8TXa+loxtkCH5T/uh7O2CQ0HHFhYaoJurw26rS3BE7hbWnCkpgnVtU3oobPjpVYb2ppa0MHZClOrDRc5HLhIYf2ag7zuX/qA5O4gPRJSQ6We3+nCHrcZHTpk469ndA3o+ZCT3C3X6T2B0wcjbOM6mif/iWhwR3pMqfvwqn6ZeHFzI+ra3FixvzVgf+HuW7V6TDQ/XExJg13uE/9gT/2V9h4Eey0CjuZmeLp2jfjzFIiJ+CSbVhLHYy3a6VDlHuPpH6rw55M7x/Rp/bIKGx7ZUA8oSBYPlVAtlWwvTuT3T7KXSpQWByfiJHgh4T1YsrzAP2nf6QH65RrxTEkdSkpKMHJJJbbWu2DQAaMKzYoW4AtWJ/73RKjph7895B2vZQBQYNXLThKXc9xItw+6H1Hw8emuZrxW1oA/9Lfi/B5m7Cwvx3HFxe2T0Y8mS6/Y34r5mxvR7PAg26QPmtshO1Hc6Tw2PErcYyHqndh9uBE7KxswKMOB7jp7u6FSdXVNyGizwWpvhT5MjkokPAaDZIL2Eb0FO50m9CjIQbeC7ID3qlpaUNCrl++1BzfbscNlhinLCnNONi4dlI+xvZQ1Vv17kPwTvdVo+IdbXyJWyf9KJigIlujsXy8/1zrDrpEhNPKVJrcHq4dg5xDTSRP8Gthf7m/DO9tbcFX/bJzd0xK+sX/0/Z27duG4vn2D9wTICRz8txGmWo5BD0Aki//Ge4bOdGk3xQuDFpItXb58cmdWUkLqh6bS+lTSa+H/uv9K2+IZu+Q2csX5HOJV44UemoMtTjQ5gByzTjLvwz9YEQcnSmalAo7NnmUEYDXpMLqrCQfrbfjzyZ3xyIYGbKt3IsMATB2YJTlTmRLie0LcC+Of6zN7TS1qjva01Nkha62aYNdB9r3oduM/O5rw59XVMMKNwbl6fHh2vm9182u+OIw9DQ50NLqRb9Lhur4WnN3NFDLhfMX+Vjy3sQZmkxk3D/aem68B1z0DsNvxl/87gIpDjTDbbch1tGKwxYnbSwwRJXfrWlpgDpLcHS1PRobk6tu+tSxEyd1CL8YmmxH/V63D6L75OLl3XkAvR7Dk7lANbnEDW9yAjSTQCHa8m76qQXmdAx7oMOeUXFkzVon/H+s1HoKdr5KGfbCgRSh7qxOwGIEOJh06ZBgCjuV/flf69XCM7en9Dv7hq1rsbnahd7YRC8/qpOjp/Bf72vD6thZcPyAL44qs0Q0FEr3+n92teHFLM6aV5mBCn8xj20dB+Bl0SX4jpp56nKKhxYki9+djLH6ny5Uu7aZ4YdBCsqXLly8WizxK/dB86bud+Kg2R3ZjOtQP3lBP/tcd9jYErUZduwUPQ31O6FmZ3D8T6444fAsoBqsXIbBxuj0w6o99zr93Qmr6W7nEDfs5a+vw0q/NcAHINOrg9gAelws6g8GXlyI1La5UgBDuSZzUPRGsxygSwnXon63DsnGdfD0bn1a04PWyBtxYYsH5PcxBp+AFQjRS3W58vaMO/9lSDXtjC1wtLSg22HFXf6P0itxH/16/uw72xhZkOlpR4G5FRpsNpjYbsuytyHS0qprcLRCSu9vMFtQaLLDmZuEgzKjSWWAzZSAjJxunHZfXfthUiBwMWK2ypqaVItWoDhdYhGpwh5v+NFSQoLSBv2J/Kx5Y3wC9DijpIB10iI8n/r/UvhUFVqIG+Jf72/DW9hZcU5KNs3tm4Ib/HQ0Kckx4fWwnxcN4PDoddu7Y0b5nAMCne9uwcEszTiow44caJ6rbPKi1e9A314CPLij0Nf4XbGnGtEHZ+G2fzHa9APFa8FdpT0AsGuHCPnuYHFhxWRHGf1KFNYfscAMY2ME7ZbzWyL0+iVy4OV3aTfHCoIVk0+KXT63hM7Em9WT+YEMrapx6WeubiPch3tZ/oUH/PAr/4WHjemZg/REHTvEbvgVAcviYeDYt/2AnXI+PeA0N/2Fjr21rCeitUXI9gq2h4h9cvbKlCUaDPqA3SGptFvFQt0gaAcGGswFol+/x2dFhVzceHXYlzvtYsdeG97Y14qreJpzV0dMuiNi0tw5rK+rR19iG6tpm9NbbUV/fjBMsDvTU24GWFtTWNqO2rgld0YYcR+uxYKQ1NsndHpOpfe+EeKpZ/5W5QwQWq+oNeG+/G5eWdsTYntZ2T8Bf3NwIQN7UwuJGdbAV2uWQCiKi6X0IF7QEDXh0Okz5Xw12N7tQlGPEwrMKAABf7rfjze0tuLYkE2f3srZruP93XxsWlbfg2gHZOE942n90f9Dp8NneVry2rQWTB2ZjXJEVn+1pxavbWnDjwGyc39sasK3w55LPq7Gj0RXQ+BcHFst2t2L+r81hew7VaEzK/b2UyIZrOEp//sTiXIR9XuzX0zL16xrY3UBJrhFrJnZR5TiRlCtRQ7vUoMV2UzJj0EKyafHLF+qHvdQPu3h0E4f7ISuUIQsOHHEaJYdcKeU/Da64N0VcNvEwr2B1IrW6u38DP9RilOJEeKEnwn+42IRiq2/xx0KLDr1zTSF/MQXr7fAPBuesqYLZnBGyF2dZhQ1Tv6qB3QOUBFl0EoA3sBD1bODoAoI6pxNfbjmCf/9chWu6unB6tgO6pibompu9ieSiqWn/b1sNnM0tKHC3YaTfwnrtejmc7VcRV0PYwMJ/RW+LxbfQ3qGmJhQWF0suugfTsamx1U6cDtVbEe5Y4qBizNLDaHZ6kGXU4atLurQboiM8/f99SZa34Q/g//Z7G/sndjLjpxoHru2fjXOPNvq/2NeGN7Y147oB2e2G/QChp/7csXMn+vbrpzwRGMFzzdT6eaYkPyBcgznczxQ1G9uR/l7SUmNYSwGVf33KKVes6zGRQ7vUoMV2UzJj0EKyafHLF+qHqlTjGkBMnlD5j/0Nt0q3/xOtbt26hS2PMAyqo0WPoZ2MWFXpwOT+mXhgRF67fYqnwZUK2sRDtJTWiX8OS7BhUeIVz6XyPyYUWzFqySHsaHDCqAdyzYFJ60LvkfhchTJIndu2mlYM6Gg59hT3lwbM6GfGhZ2c0DU2QtfYCDQ24s4v9qG+pgE5bc0Yn+/E+Z3cgF/Q4Z8ILuRiBAyfstm8+SIq8xgMgcGBL7k7AzsdZlhzM3EQGcjNzcQejwXH98hFSbfcwOFRwnS2fjkbvmRYP3ICDaFnoN22ogb1jf+rwe5mN4pyjHj1rE7eD4dL0g0yBChcwu+Yjw9jV6MLfXKN+OqSru228R/y49HpMOv7OtQ6vLlN4vsICN6DF4uGktIhoeGo2dhVOwCKVyM80txANYZ1ainwUYvS+ox14KylgC4SWmw3JTMGLSRbsn35QjWug023G8kvH/+8kRGF5na9CcGEq0+hTD8escN2NH3AAMANIMekw57fdw9ZbqlfJnICDqkyhErwl+rVEA9LCzabWECwVWXHjH4m/LbAG0Cc914F9E2N6OJswVsjzNA1NeGzXw7jl3116OhoQWZrM7q6W3FOvgtHjjTgQFUDrK0t6OxuA2w2mNtssLbZkBHL5G6r1dcrEdBjcfTv3e4MbGwxYlCPXPTrmovNrSasqNPjtD55GNqrA5CZiW8aDHj/IHBxaUf8pncOoNNhxf42vL3DhqtLsjC2l98wHeHvcDP2BHnPPyC4/L/V+LXOCbdOh9/3y8K6WhdOLjBhfbUTU0tzcGFxJsq3b8dWcy/M/KYOOr0OgztKz8SmtGERzfdNCHT7yhiyIrdXMFQPnpJpvMXvA4GTNEz/urrd8EU1qNF4TtbGYaSNbMlhnQrFsxcgXgFSqPqMJqiQU/5k6lWRez2Srd2kdQxa0pjSH4Ja+vIpKbvUtL/iH45yfliGasAHm1ZY3LgHjjViBjr2haxPoUxGeFDT5kFHix5drDr8VO3C+b3MePuc9uWWWrU+WMNLzlTIwerlt/+pxIEjTTje1IYbewL/3lyFq7p6cEaW3dtj0dSEl9dVoq2uEebWFnSw25DvbEFmawsy21rQ39iGTq5W4GivRixW7ha4dHrYzBbYM6zIy8vyLaa3y2nGLqcJnQtyUNqjA5CVhe0OM75rMuKkog5wWjPxeY0eY/p1wsl98r3BydE/yMrC5/vteG1bC24YmI3ze2dKrg1w2X+rsbPRhT65Jiy5oBAXf34EOxpdOC7XhI/He/MBfvtpFbY3uNC3w9GGrE4Xt2GM4rwl8fDC8vJy3Fqehx+r7LC7gT8Olu6tkHMsOVNGy92X3Ma10iBdjnBl93/fAwT8++eqVjihj2giCjH/Og3Xu6slSoI+Oe+LhzOF+72gZnDm/9Al2hkKw4n0O6Pm7/lovrdyf8cmS+Asty601G5KBQxa0pjSH0Ba+vIF60WQ2+sg/uEobsQDwYdWBUucF44/vMCEz/a1+vI5djR48xRGFJoDGjHCmiLCZ8U9F1JlEuejSCWZ72hwol+OAcvP7+Sd0tZu9w6LOhpM6JqbgaYmPL7qIJrqGlEMG8bkuvDz3lqMyHTgOEMbDhz29lwUulvhaGxGd7QhxyGaaSoGPGZzYLJ2VhaQmYmVdQbUmaxozshEszkTbRYrcvJzcFLvfHxZp0e3zrnY7MjASQV65PbqhbcO6nHZkM44Z2AB/nMIeGlrC24anOudHtQvRyBYzlO+WYcDLe6Q+UbiIXDBAlrxfSbnqX40Ty+VBvQvljUhzwysqnRgdFcTdjS4INxzAx37sMXU03ffdbXqg/ZYhKIkAdv/exRuiu5o8zDUmkku2PtA4EKfT/9QhRZdhioBVLAhmIlq7Mm9FkqCPqnvlfjz/r+XEvWkPl4PGSK5xmr+no8mqEimgEQOueejpXZTKmDQksaU/hBR+kQr3LGj/bycpFT/BlCo6XrFn/cPLm4uzcbDG+rRZPegxu5Ghv5oQ/b8o2P3PR7A48FFnx7GrnonWp1uNNndADwwejzoYtUj16TDnSfmQOfx4MmfGqDzANd2acb1w3tD5/Fg0hdV2Hi4DZkOGwaZ2tDV04pJXTz477YaNNQ2oljfBktrCxrrGpHZZsOFBW70MdhxsKoR+6oa0ddoR2d3Kxrrm9Hc0IyOrlZY7a2xT+7OykKL2YrDegtyO2Sh3GXBYYM3uMjKy0ZJjzx812zC0KI8OLOy8VmtEYcMmag0WJFf0AHPnNczoPfCP7nb35y1dXhhczMyDEC+WYdGJ3xTKgdrwEgJ9XRaPHnAwRZnyJndpGZKkzshhFr8e+OC1YmS/YjvfSGwFs61osGJw61u9M1VNgWqkp81oSaViKSH1L8McibmkDOkLFLl5eXYYuqpSuNNac5XrMm9FsGCdKkeYv/vlfB/8c9ypYnjsRDP4ZGxLluwn6FazN/RYpnEGLSoy5joAlBiRPJlX3nEgFvLqwIafC+WNcl+Erz+cJvvl+y6Iw7sqHdgwS8NmNDLfGyRu6MBgPD/5RUtePnXJkwdkInxRd5k4U932/DGr424ZUAWzu/oAOrtgMeDW3s68dqWJkzunglddTUA4N01VahvcGJHvQGfnN0J8DQAlfWSx7s934Z5+xqABmBMFzN+sdlxdY9MvLNiN/TVduQCOCFLh7wMA66ymKHftD8gQfuRPXX4dlcd+pkc+GFPPaz2VmTabchztuIEqwPNr7Wgi6cVz9Y2I9Peig7OFuQYnUBLC5Y1t0Df1n7l7nPDXJPio38E+Uf/+HMbDGgxZ6LJbIHTYkW3zrnwZGXhsM6CX+wmtGRkwZSThe2uDJT0yMNZAwrwU6sZ8/YAteYsdOjYAS9e2Ms7JKrWiBf3eHDD0M6YcFyW7xjfHr3OuxucOGhzI8MAvHRmR9wkaswPRODQtKXmXEzoEX4IyAMj8nBKYYZvqI8Tbqw/+hTe/0l2KP7DoV48um/xBAF1dg/65Rp8T65PKTDh4Q31eGRDQ7vhPMK/hYaUDpAsh9zvSiSGF5jwS40DrS4PHG7gtW0tmHdGvuw6Abz1Ut3qQr5Z3753wFEHwHuuwsQJh1vdAJQtZCd8XjhesJ4UAL7rIp5Uwv894TUl11/qOog/718XAGJy3fzrIhg5P5/XHXHAatRh/RFHu/diec8FI/daSJ2/f3mXj+8c8L44gNEBQfOqYtmADbV/8TmFK0u8rk+kdSL1uUTcU+FosUwUW+xpSVaixn3A/4MEAP7/v/yLI9jV4ETfHAPeP+fYWgO6EJ+9bNk+VDoM6JNjxPUlmVi0rQnX9cvEuT0zAj7z5V4b3ipvxu/7ZeHt8ibsaXSiOFuP8joHmh0eZBuAOafkylrxWLxew7vbW1Df5sZhmwsON9AjyxBy7YYV+2z415Y6XNUdMLba8MW2Wkzo7MEpWU7obDZ8sbUam/c3YmS2A2fkeXs2nE0tKHC3oq/JgeqaJuS7WuFotsFqtyHP1QZzmw06e+ySu+0ZVtQZLbDkZCG7QxY2txpRY7BAl5WJU4/rCE9mJra7LFjTbEKnTjn4st6I2ows5OTl4Mlze+KWTS5scVlw2JQFZ3YOenXKws2Dc9oNU5q5uhbNTg/0kF54UpiO2P+perAk6GDJrcGe8ikZIiLVcxEsHyfceOxww/vEQUxNqztgeJ/cRT3D7VethpVUT4t4VrlIk1+XVdjw9A9V+PPJnQMCjnBDo+Qer8HuhtMNWQn1apDzxDnWQ67kPnWNdAIBOe/JEe8n2HLKG2wboU5jPURLyf7DbRuvHqFI6kTIYws2iYuWhndpsUxi7GlRV3oHLcEa/Epf99uXTmkwEW7bYPuMUqh1EIJ578e9+LIhK+hnhGlR69vcqHe40cGkgweA7uiicL9U20Mv8uZyeXMm/Fbk/s+v1XA2tsBob0Undyssbd7Awd18dHVuhw3dYUcfox01tU3ormtDrt/CerDZvKuGq8yt06NFIrnbf5pZT1YWKpwmbGgxIj8vCzvdFuzzWHBQb0WrxQq7NROdCvPxj3N7wpOdDU92tnd4lF7vXcH76N/L9trx4q/NQYfCVR9tWPfN9Sb3CkOchMUkg00H/WOVHa1uoHumHpf3sUouPCn+hSAVyADKG7NSuR6h3hcL9stYrfHY/o1Gr+CzpCmZZUrpukLByB32GE3yq3gKaTnkHE8Y1mTUedDogKyhZvFqRMe6EaR0IUQ1JxDw33cqzeIk1KlUUjyg3kOCSCaAiHVjOtr8KymRDGFMhmFaiaIkaFFaj+lY72kRtOgqKrx/BwkKSJ73ftyL/2vIwgn5RpRV2XBNDx3OzHNj7a5afLm9Fq6mFrhbWtDJ1Qqr3Ya2JhtyHK3ooWvDuAJPwGJ6DXXNaGxoRoHbm3vhbm6B0d4Wk3K7zGY0m6xwW61oMFqQ3SELuR2y8GOzAYd0GWgyWVFcmI2TivLwwUHgAMxoNFhgM2WgOSMTVw0pwMm9845Na5uVhae3OfHGzjY4dAYUZBlw9ykdMa44Cx5RwOH/97LdrQEBxaq9DRjdK1dyiIxUYr4///HswrAp8Sr0QsAYrLEtTuoHwk8nq7SRLoi2Z0XuNJtqrYOhRqMjWI5VsF4iNZ/kqnEeUj0tcj4T7v4Q1sdQMuVsIhvRkQSTwbaNZE0RtRu/0fbiaI24ToPlZ2k9+IpELL4XkfQMRJtblsqU1KfS65lMDxfUkhY5LbFKRNYKoYfjhHwjfq51Bi4W53YDra3Sq2+LXwu2SvfR9y6uacTv7K2wtrXC5D5Wp2OP/lGi49E/AmHpO1uGFRnZ3tmjGowWVHrMcFqt2O/JQIvZG3jU6C1oNFnQZrbCYbGg3mhFk8nb65HdIQvV+gwcRAY65mfh9fE9ccWKWuxtcqMoW4/3z/N+sZfvb8ULv7bArTdg5gm5KO2dhWUHHPj7hgbUuPQ4vVsGdjQDbr0Bo4flYZ9ej/lbbTjZasG6Cgd2tzqxP9/bg7MXwLOVZpw3rKPoB7L3Gvg31oSciQdG5KE8vwq3lnuTph/ZUB8wzr+m1Y2dDd7pf18sawIQ+MRw3REHdPDgtW0tOKUwI+AHlvCEdufRYU3+4339yyeV++CBx/cvKaHG44caXyzkXpxSIJ1oH2o8vNR+g5Vj8QEj9juiH+MsJ+8gHKlzmlBsDVpPSvIzoskfkGtCsRUDHW0oUfB5OfdHvlnnG34lvC58NhgldaM2JePm1RhjL27UqbUfgVCXQg+tVFmV5mhoifheSdR9Ew+J/F5EWo5g3xEt3WOJKovS66mV6x9PadHTot++Peh7claFjhunE7DZ8O3OWny+tQYXFwLDs52+IGLLgQZs2lePEdlOHGd0+AKLDXvq4GmxIaPNhixHK3IdrchzeWeOsthbY1NUvQHuzEwYszLRbLLisM6M3Lxs5B8dKvWLzYjNbSb0LsxBx/wsrGky4qReHbCyzoB9bhP0mVYYs7NwTklHnFrcAf+rN+CdPS5M6u/NTxFflzFLD6PZ6UGGHuiRpUeTw4Nskx5ndDVjdWUbXHojXHo9DrW6UecywGPQo3duBj67uCvOXl6N7c1udLKa0CPPjKmDO+CfW22Sswb5LxIpXmNBmNFIBw8aHMea9T0y9Xh8ZB4mFFslh09J5XsA3ulPR/fKxfojDt8wEGHGJPGwJKmZrqR6SgRz1tbhpV+b0dFyrGz+ZYnFWOtQnw03s1uoNQ7CTVfr//mDBw9iaW2Opp8QR9pbFW+R9LSE25+cGf+0Rs1hQXKeuqpVJ2p+12N5naK959M1ZyBWPytiPXtYpHmO8aRmWdL1/oyVtA9apBK9wwYwHg/Q1nZsQbzWVvywuw6rdtTi3I5unGB1BvRS+LYTvybu0XC0nwVGDc4MC5pMFngsVhzRe3ssmk0WNJu8yd1n9s335WJ4rFZssZuwqt6AEb3z4LRa8ckRHYoKsrC+Abj0xG4Y3TffNzWtOLgQ57X0zvY2+P3r2D+XRtxLdGVJNsYWZeKGr+uxq9mNolwz3jyvEE9sbMRbu9pwZUk2/jqiY+DQK4MBwLEhT063B0b9sWRrqfHhUkm24qE7AAJWdRdyPl7b1hJwDODYDFIvbWmG3Q2U+A2vkhprPb+syZcvIEypLAwb+3xfK8TDh6SSuUPlMwRbdHJ4gUly/7EWqsEqNbWtHFKf91/7RknZ5CxIp8aK4/5BllSwFm6yALXOKdiQO/ECkP73qP+UtPHME5B7fUINp4yFSO4JOQ0YtYZmqTnEK5J9ya2faBuIoYaEaukhgJRg5Utk3pH4HhUP64xVUCH1ezJR10zN7w6DFnWlR9CycaMvcFi/uw5f7ajDQJMd2w81Qm87NiuUpc0bPFjtrSgxO9DW2IxeejvyXa2wNTbD3tSCHEcrjK22Y7Nsqcil18NtscJpzUSNPgPNZivqDBa0mC0wZ2eiR0EOytqMOK5rLva7zfimwQi7xYJWcyacVgs6dMjCnad2wXeNBjy6xYk6gwVuSwZyLEZU2bwDfywGwKDTIcsE1LR6UJRtwPvnFQSU43dfHMGeJheKsg3okOGd9avFCRh1HpgMesw5JdcX1N30VQ3KahxwuIHfl1jxc60Tu5tcyM4wooPViKsH5GBjrQtvbrchx2rCb4uz8EOtC5NLc3FBcSYu+W8typvdaHbp4NHr4YEOk/tn4oOdLai1ezB1YJas2ZCE11udLvxU7cKJnQywGI+N1w7VqJFKAhf+/8iGhnY5HuIfaP6NZx08QRuc4pmJhKfY4rUIQvWgiPcjN7navzHcPdOg6JdPrH7x+/+S8p/aVunCiP6fH+jYFzIRX2rfcupTjQaCOMiSCtbC9aApPZaSHCKpQHfOmiqYzd7ZAYV7Rk6egJr3jNzrs+6wHS4PkG2KLuBTs1xiqdaAiSaPzX8f0TQQx/5rD/Y7TJLH0dLTeynBypfI1eOD5QiFy0FT6zuv9WumVKp95xMtLXJacn7zG9+/f3P0j1LiEYNCcrchKxPV+gxfcJGfl4XDugz0LMxFj8452Ok0YV2TEQV5mdjhMmN47zyc0MO7TsaczXbsdJpRZ7CgWm+BzWBCF6sePbKNOCHfiFWVbd4pgk163Dw4Gyf0sGDDpgbcs6sVdrcHdjdggDdY+KXWid/0y4S7hwULv6pBfa4DNqf3/UO2ozObwRuwnNLZiDWHHOho0eGMrmbc9FUNruqXCQB4d3sLmhzeXA23TodJAzvg3h8a0WbQoQ56ZJgMeOmgCb85sQCf7W3DNqsO23MBt16Px+v06JFnhi5fj5lHeyqeOvqEZn+OtwZ3V+qggwdrfnRgXpYefxjaydfwfGlLM1pdHjy3uRl6nTfPxX8NAv8GnXg8rNDwP2Jzww3gx2oXjDoXdjfUtssnEf9AFY+x9f+/Bx44PMDWeifmrK3DKYUZ7X4we4ObehgB5JgNQZ/0+o8/FecL+CfS6+CB26MLOk41XH4I0H5M+nS/YAjwBOQU+JP6xRNqnH40v6ikxusLv7Dk5ARIfb68PPj20eSSqDF2WLz+iNQ6JBNE94M/JXUdrrz+ZRn/iXftJfFnJhRb8fQPwP6jSfP+90yofYf6nkZC7vWpaa3H7iaXasdVo1wC4dpdkm9AKrVfHt5Qj50NLtS01rer73jkXQHAld2dWFprlTxOtN/bWPfUBCufnHKrkXcnh/j3VjBqrZmSjnkaJF9a9LTk9uiBOn0GGowWNJstaDFZ0WrOQKPZihaTFS1mC1rMVtjMFrgyrKgxWtBisqDZbEWr2YpGkwWGTCssOZk4oregXp+BGrcBegBuAB1MQCeLAb2y9fjqoAM6AJ0tOvTINmJ/kxM1bR7o4M2B8O/ZEKYdzjHB9zmzwfuksMnuDUrMBuCh4R0AeAOKsloHWr352TDpgI4WHe44MTdgOJuw3+PzjXir3AaHTgePTo+CTCPcegP2t+ng1utgNRsxoGMGtjd7UNzBjCq7B9ubPOiUaURRhwyc0tmMz/a1+laC97g9MBv1vqeY/mtE1LR54AJg1ME3bMo/8RYAGu3eW63G7kaGQYeuVn3A7DUjl1Ria7335HTwTsUrzscINnRGeOqUZwZWVTpg1AGNDg+MeiDXrPc9JZLqehavLC0einXDyhq4AeSYdDi+oynkkzElq2gLT2CCzQQm7vWJNg9AzpM5qX2HWnlb7adi0T49VDrlcaSNkkiGX4XbDxC6l0mtuvY/pjhXSuyl73YqzhESptO2u4E/Ds5qd8/EkpIhJvEcOiRcux4mB1ZcVqT481od5hRutsF4iOWT7FR76i9HpPWZTDPOxRN7WtSVFkGLfvt2PLepAW+U22RtLwQYFgPgcgOOozVk1gH2ILXVN8eAQzY3mpzHejUsBqDVFTgPk8UAjCo0YX2VE5f1seD4TmY8sL4BLo8HHg/QwaxDtkmPJqcbh2weGHRAQYYOR1o98Oh0sOv08Oj1MBn10BmMMJkNKM414Z1xXfDpPjue+LkJbr0Rdw3rgAuPy8acHxrxWrkNo7uasKrSgWanBy6Ptwfm/F5mbG9woenoSVW1uuGCNx/jnmG5AYsP9s01INNj9yXlCsOtmuwe5BxdPXp3oxM2N5Bp8AZnwpS743pm4LN9rdjT5PIFK50sBlS3unGwxQm7C+idY8S4nhn4cGcLDtg8MOkAi7F9zoh/DkeoRG6poUPiYVLi4CvYLyZxUCE1ZW2odRWCNTiEH2ZFbx1Ao8ODHJMOe37fPeDYwRLYQyWmR5t3oSRZWmu/qIKNb49maJiUUBMLyAkGpPYTbtiVWnWtZAHFSH7hRjPELZ5DTEItqql2gCBcu4vzGzH11OMUfz6aBxVKg2clOSvxziPyP7ZQ3lBDQtU4jpZ+vsUDG9nqYn2qSx9+k9hauHAhhgwZgi5dumDMmDH49ttvQ27/wQcf4IwzzkC3bt3Qv39/TJ06FYcOHQr5mRX7W/GvXcdm0dLBe+IGAHkmbyBhNQR+pm+OAaMKTYDn2Pb+AYvu6B/B/hYXemfroIO3t8GkA2yu9hPH6gCsrHSizq3Hq7uceLjMjkqDFYdNWUB+Hqqy8+EuLMRpQ4pQ3bEL6gq7Y01WD2wtKMLWzr1RUdALhwp6oLCkNy4YUYy8nl1x5aje8OTn47m9evzUZsFPNiMe+qUVMBrxwMh87Pl9d9TaAR08MOuAgR2MeP2sjqi1A5UtLlTa3KhsdcPp8QZmAPDIhgY4XB64PUCOyVvyK7t7p9Ad/0kVHtnQgDq7B41OD2rtbjTaXd6eoaM9LYAOdXYPcHRa3u0NLrS5ALcHuGdYBywf3xnjemagyQG0ub2NtfVHHOida0KuCXB54BvmITQAheBjflmT75fWjgYnXtvW4uuW9nek1YXP97UF/MIZXmBCkwNwuj2+7aeXZrcbLrWswobxn1RhWYUND4zIw57fd/cNDQN0qLW7A443odiK5eM7455hue325d9tLmVy/0zkmLy5PGJSZROOFWqYmz//c/H/txSpfUuVIVxZ5BDKMmdtXcgyKbH4gFGyDoLVTahzC0Xqc/7HkLtf/+3CfSaaug52TLX26W9CsRXzzsjH4HyT4noN912RS079B9tGrTL4E+r5rAJXRJ+P9D6VOhepnwGRnPP8sibU2T3oZPE2I9T6Dss9ttrXSIrc70e4n6ux+my6YB2RIKE9LUuWLMHUqVPx1FNPYdSoUVi4cCHeeecdrFmzBr169Wq3/Zo1azB+/Hg89NBDuPDCC1FVVYXbb78deXl5+Pjjj4MeZ/Q/vkGbyxtMFGV7oxNhZquXxnQMGKYl9ID8aUiub5pdIwCD3tuItrm8QY7dBbh0Ouh1Ojj0BhiMeugNBphMBuRkGHGgDahz6eHS6WEx6tEMA3rmGLCjWQe33vtDPtPgnYjM5gYMOqCDEah1APlmHQqtBt8UuA6XB61HezDcAKYOzArIrQC8P8TzzcAne+3QAehm1aN3rjEguVxqaMzM1bVodXpg9wBmPdDZoofT4y3DgRY39DqgzemB0wP0srjQNdeKzTV2tDgAowEY292MOjt8vSZCDwZwbL0Q/94UYVYuIc9E6j2p5Gzx4on+T4nFvSlA++mL/ZONxT0tcodgyU1IFAv2tC7cUKZInnyGm0oy0sXWIi2PnOFOkc4eJiXYcKZYzn4U7hhaHdrjL1xvYDzLkegn27EsQyLqU9wzrKT3NtR+/XtZ5PYsqsW/vLHsaZErmmFkWhuCpsWeAa3VkRJarM9kltCg5eyzz8bgwYPx3HPP+V4bNmwYLr74YsyZM6fd9s8//zwWLFiAX375xffaW2+9hVmzZmH//v1BjzP8iW8AAH1zvIsLCkGKMO0udDpArw9YzfyLAw488GMjapw6dLAY0aYzoNmtg1unh0uvx2ndLPj6sAs5Jh2aHG7frFdCgq2QCyIMnRICkGaHN/ejo8kbAFmNgNWgR45Zj91NTtiOPoiz6o8NmfJfx0P40vo3oA+0uOF0e9DmAjIM3p4OuTNECYGLMAys0KJDoxMB+RUVjU4ctrnRy+LGhONy8MLmZnjgDbSEgCBUgy3YcKNQeSb+nxfP7CXnl6vUL2txeYBjwZPUkC6pIE/NRmmoH2Zq/pAWn3MkDbJIyiN3nQj/xPRop7pU8xdEPNfLUGvF9XCUDpMrLy/HFlNPzQddkYp3QBlNvkCk5ZSaFS7aoEyNfapV91poFEZTp1oI1P2FmkI63vyHgAeb3l/tY6l93nLrM9RwdzomYUGL3W5Ht27d8Morr+CSSy7xvX7HHXegrKwMn3zySbvPrF27FhdeeCEWLVqE888/HzU1NbjpppuQm5uL119/Peix+j+/CQa9AbP7O3FGZw9gMMCj03n/1uu9632ITNuUgT02HYqsHlzZ3YnFB4wYnOPG5kY9ruzu9HXzrzxiaPfe4gNGlDfp4NHpcF+J92m/sM03NXp4B2oBe23esVhDct1YMKQNz+0y4f0DRrgBWPRASZb3df/jCEO0Xtxt9O3nSBvQ6Pbuy6IDHhxg923jgQ7TeztCDktYecQQsL9GJ1Bk9UgeWzg3u0eHHhYPbg6z72DHE/Z3VoEroK6FY0pdB/F70Vh5xIAHy83eSRPMbuSZEHBdlYi0jOJ6kPteIkRSHqWfidW1jlSw8is9r3DbKznvaOso2OfFZRR/57V0XdQkrg8tfe/UuAb+P9sj+Vktp2yR7lNr33c51PyZraV7zZ+Wrks8yxKrY8ndr7Bdq1sHi96jifpPlFAPIRIWtBw8eBCDBg3C8uXLcfrpp/tef+KJJ/DBBx9g/fr1kp/7+OOPMXPmTNhsNjidTpx11ll45513YLUGj0gv/KRKcYSu9OmHOLlVKhHVP5Jff7hNcsVyOcf2P5aw2KEwW1bfo0n0/j0IwoxWciJ48Sxc/jNGCU9d1Vr4zH94W7AejFgkegq9S1ILREbydCPSJ2Wh1hdQMjtVNGXXEjWeOIZ76qrG07REzpgWbR3J/bz/bFd/Prmzpp4Eq0lcH7EehiLcn3LuQyUTJgSj5WE1oe5FJd9TtXpalF6TaNeE0eq1iWTGwFiJZy9UrI4ltz6DTR5EgZIqaNmyZQsuvfRSTJ8+HWPHjsWhQ4fwt7/9DSeccAIWLFgQ9FjCOgSxvAHEN3y4WZiU5BaEGiLlP62wkGchfk0YWqYkd0BqRqtYDb0RyiwOqpQsTqakIRrJDGKxEOqHmZLFASPNU0lF4e5RNa6z1Hc9FYJGf9HOdiXsI9nqJZaNpGUVNskFZWO5eKDWhh7JpeR7qtbvpWgXdIwkL0iL10YLw+1SCetTXQmbPaxTp04wGAyoqqoKeL2qqgqFhYWSn5k7dy6GDRuGP/3pTzj++ONx9tln46mnnsJ7770XMqclEbOMhJuFSclsMOKZUvz3LexHmJFL6jVhRqvJ/TMljyk1M0eoGa3CkTPTh//5B5sFTG4dKZ1JZnppNo7vaA5Y6yXS2XmicVaBK+jMNOHKE+m9lGzUnjUm0rryL4f4ux2vmYziKdrZroD41ovUfRLJvROL2dQE88uasMemkz27nBpliXYfiZq1KVj9xLI8cn82BHvKq7SuY3mvEaWqhCfiH3/88Xj22Wd9r5188sm46KKLJBPxr732Wuh0OixatMj32tq1a3Heeefh559/lpxxDIhseJiWxPqJjNyFEcVPDJQm9QYTbbdoIp9YRfM0mU9gwlN6L8WqTpNprRo1RbpOS7ySZwWhZvuLtFdN7Z4i/54Wrd0nav0sjzWp8sTqOx9sVslgCxynCv5eUhfrU10JXadl5syZeOedd7Bo0SJs3boVs2bNQmVlJSZPngwAmDZtGqZNm+bb/vzzz8cnn3yCV155BRUVFVizZg1mzZqFE088MWjAAiDpn2bE+omM8IQJ0Cl6MqrW2hfC+T0wIi+i80zkE6tUfMquJVrpRYrVWjVao8aTbP91leJVL1LXJ9p7RziPRzbUq/J0f0KxFQuGtGnyPlHjZ3m0947SHvpYk6qT6aXZ8EDnW0NMrnRdZyRdz5tix5jIg0+cOBE1NTV48skncejQIQwaNAjvv/8+ioqKAAD79u0L2P6aa65BU1MTXn75Zdx7773Izc3FmWeeifvvvz/kcYQhHSRtQrG1Xa6MHNP9kkOl9pcOgtUBqUMr95JWyhFr/g21SM83Ed8JqesT7TUTzqO61R11nWidGj/Lo7135Hw+nt9DqToRji3n/vbvqVHjexULsc470+p5U/JK6PCweLnwkypNdG0nO3Zzqov1qb50rVO1Fv8UD3VLhvqMdcNLzeF/yVCfkYrXzHZiWq1TNWZ/i7VYD7dLh0lLwtHq/ZmsEjo8LF74FJyIUlkkwxSlhj8l41C3WA/RVKNOhGEyK48YVCyZtgj1BCCiIUFav/eUDnXyH8qmxXNbVmFDdasL+Wa9qm2kdJu0hOIrLYIWLf2gICJSWyRj/SPNZZOSyLHrWsl7CkVorC0+ENsR2VrIIUjmhmmo+hOfV7i61mKg4m9+WRPq7B50suhVLWOo66/176oWvj8UWloELUREqSySBpLwGWFK9GgaEolsqGq9cQgca6xd2d0Z0+NoIWAI1zDVcsNQSYNbC3XtL5qeIDUl86QlWrum1B6DFiKiNKZGQyKes0wlIzXWvZFDC0+yw91PWm4YKmlwi7dN9H2ttF5jFUBoPTAJRQvfHwqNQQsREUVFSUNFy43WZCfnOiS6cR2vhmGw8wx1/kruY63layS6wZ3o+0oNyRxwpQsGLURECZYKv/DlSnTjKt0lsnEtzB4VzSxacr8rwc5TzvlH8n0MdV/H4/vt3+BOxM+TcPWaTj/jKHYYtBARJViin9LGE59mJlY8g0ZxQ1WN+1zuPoKdp5zzj6Scoe7reH+/43U8/+s7vMAEm9ODUwpMCS0Tpba0CFoY2RORlrH3geIl2qBRyRNzcUNVjfs82D7E5Qp2nnLOX+3vY7y/3/E6nv/1XXfEAatRh/VHHAktE6W22M6/qBFcjZWItCyeK30TRUPJKufiVeWV3OfBFj8Ntjihmquvq/19jPf3O17HE1/fRzbUo7rV7VujJRFlotSWFj0tjOyJiOTh2HMKRckT82h6daSGEyXzGiByxOq7F6v9+l/fCcVWdLQYUGt3cwgYxUxaBC2M7omI5OHY89SjZqM1ljlJ/uWUCkKSeQ0QOWL13VO630jvl1QIHEnb0iJoISIiedjwSD3JEoiKh3iJg5B4BUyJkogFH6VEer+kQuBI2saghYiIfNjwSD1qNYbVatgH208iAmahLI9saEh4YKeVBR/54EKeWAe6WgiktYZBCxERUQoL1mhV2ihSq8dG2M8jG+plzfgVS0JZAA8b6kfxwYU8se7BTJYe0nhi0EJERKRQKjwFnV/WhM01dsxcXSvrPNR6Ai/sB9AlvFEmlOWeYR3YUCdFYt0jxR6v9tJiymMiIiI1qTnFbqJML83GzNW10OvkLQ2g1rS1wn6WVdgCpsxNBE7FS5GK9b3De7M9Bi1EREQKideoSEZCgyhR58FGmbaFWheHKBE4PIyIiEihVBn3nyrnEUtqDwVMlqGFzKkgrWHQQkRERBSE2o33ZAkGmFNBWsOghYgohSXLU12iWIrme6B24z1ZggH2wpHWMGghIkphyfJUlyiWovkehGq8RxIMMRggigyDFiKiFJYsT3WJYilW34NYPBRg7yiRNM4eRkSUwjhDE1HsvgexmEUuFabTJooFBi1EREREEYhFMJQK02kTxQKDFiIiIiKNSKfeUa4FQ0owp4WIiIjihjkbgdK5PjhRCCnBoIWIiIjiJlkaqvEKJpKlPmKBE4WQEgxaiIiIKC6WVdhQ3epCvlmv+YZquGBCraAmnRvuqTL9czr3lsWToqDl6aefxrhx44K+f8EFF+D555+PulBERESUeuaXNaHO7kEni17zDdVwwYRaPSSp0nBPZ+ncWxZPioKWDz74AMOHDw/6/vDhw7F48eKoC0VERESpJ5l6FcIFE8l0LmLsGVBXMt8LyUTR7GEVFRUoKSkJ+n7fvn3x6quvRl0oIiIiSj2pNDNWvM9FzZm2uBaMulLpvtYyRT0tGRkZqKysDPr+wYMHodczTYaIiEguPvUmOeaXNWFzjR0zV9cyj4bSkqIIY8SIEXjrrbdQX1/f7r26ujq89dZbGDlypGqFIyIiSnUcD09yTC/Nhgc66HVgHg2lJUXDw2bPno0LLrgAp59+OqZPn45BgwYBAMrKyvDiiy+iqqoKr7/+eizKSURElJK4AjrJIQQYvFfa4yKV6UFR0HLSSSfhvffew6233op7770XOp0OAODxeFBcXIz33nsPp5xySkwKSkRElIo4Hp7kSuS9ouXAgDk66UFR0AIAY8aMwY8//oiffvoJu3btAgD06dMHJ554oi+IISIiIiJ1aCFg0HJgwN7K9KA4aAEAnU6HoUOHYujQoSoXh4iIiIj8aSFg0HJgwN7K9KAoaPnmm29kbXf66adHVBgiIiIiCqSFgIGBASWaoqDlt7/9rawhYDU1NREXiIiIiIiOiVXAoIVhZ2pQ8zzmrK3Da9taMLl/Jh4YkdfuGMMLTFh3xIHpRwNI8Wtyj+9fZmE/kZY/Va5jOIqClmXLlrV7zeVyYc+ePXjjjTfgdrsxZ84c1QpHRERERLGhhWFnalDzPF7b1oJGhwevbWsJCFqEY/xS44DVqMOLZU3wAO1ek3t8/zIL+4m0/KlyHcNRFLScccYZQd+75pprcMEFF2D16tUYM2ZM1AUjIiIiotjRwrAzNah5HpP7Z/p6WqSOcUqBCeuPOHzHknotkjJHU/5UuY7h6Orq6jxq7WzBggV4+umnsWXLFrV2SRpSXl6OkpKSRBcjZbA+1cc6VRfrU12sT/WxTtXF+lQX61NdejV3Vltbi/r6ejV3SUREREREaU7R8LC9e/dKvl5fX49vv/0Wzz//PE499VRVCkZERERE6kmXhG1KTYqCliFDhgSdPczj8WD48OF4+umnVSkYEREREaknXRK2KTUpClpeeOGFdkGLTqdDXl4e+vTpg4EDB6paOCIiIiJSR7okbFNqUhS0XHPNNbEqBxERERHFEBeIpGSmaiI+ERERERGR2hT1tADA4cOH8eabb2Ljxo1oaGiA2+0OeF+n0+Hjjz9WrYBERERElHixSOTn5AAkl6KgpaysDL/97W/R0tKCfv36oaysDAMHDkRdXR0OHjyIPn36oEePHrEqKxERERElSCwS+Tk5AMmlaHjYAw88AIvFgu+//x5Lly6Fx+PBY489hrKyMrz88suoq6vDQw89FKuyEhEREVGCTC/NRr9co6qJ/LHYJ6UmRUHLmjVrcMMNN6B3797Q670f9Xg8AIDLL78cEydOxN/+9jf1S0lEREQUhWUVNoz/pArLKmyJLkrSmlBsxfLxnVXtEYnFPik1KQpaHA4HunbtCgCwWCwAvAtLCk444QT8+OOPKhaPiIiIKHr+w5CIKPkoClp69eqFffv2AQCsViu6du2KtWvX+t4vKytDVlaWuiUkIiIiihKHIRElN0WJ+KNHj8by5cvx17/+FQBwxRVX4J///KdvFrH33nsP1157bUwKSkRERBQprlFCauGMZ4mhKGi59dZbMXr0aLS1tSEjIwP33HMP6urqsHTpUhgMBkyaNImJ+EREREQawka2ujjjWWIoClp69eqFXr16+f6fkZGB5557Ds8995zqBSMiIiKi6GmxkZ3MgdT00my8WNbEoYZxpiinhYiIiIiSS6h8nkTNqhariRHicT6c8SwxGLQQERERpbBgjexlFTbMXF2LslpH3GdVi9XECJwlLnUpGh5GRERERKlhflkTdPDA7dF5gwdHXdyOHauJETh0K3UxaCEiIiJKQ/4N/AnFVpSXJ7pE0eMscamLQQsRERFRGmIDn5IJc1qIiIhINYlK7KboLKuwYdqmDFnXzf8a83pTvDBoISIiItUwETo5zS9rwh6bTtZ187/GvN4ULwxaiIiISDWxmhWKYmt6aTaKrB5Z183/GmvperPXJ7Uxp4WIiIhUwzyJ5DSh2IqBjjaUyLh24muslestZxHNZF7UMt2xp4WIiIgoTaRyb4ScXh8OZ0teDFqIiIiI0kQqN9rlrFSvpeFspAyDFiIiIiINiWVvSLo32uUENqRNzGkhIiIi0hA5uRmRYs4RJSv2tBAREVHSUbKuSLJJ994QIinsaSEiIqKk47+uSKr1HLA3hKg99rQQERFR0lGyrggRJT/2tBAREVHSUbKuCBElP/a0EBEREREdlcpr2SQzBi1EREREREel8lo2yYxBCxERERFFLVV6KDh7mzYxaCEiIiKiqKnRQ6GFwIcLUGoTgxYiIiIiipqSHopgwUk8hmZpITAi5Ri0EBEREVHUlPRQBAtO4jE0izkryYlBCxERERHFVbDgJB5Ds5izkpy4TgsRERERSVpWYcP8siZML81WNZCYUGxNWM5IIo9NkUt4T8vChQsxZMgQdOnSBWPGjMG3334bdNvp06cjLy+v3Z/u3bvHscRERERE6YFDqUgrEhq0LFmyBLNnz8btt9+Or7/+GiNGjMAVV1yBvXv3Sm7/+OOPY+vWrQF/iouLcckll8S34ERERERpgEOpSCsSGrTMmzcPV199Na6//noMGDAATz75JLp06YJXX31VcvsOHTqgS5cuvj+7du1CRUUFrr/++jiXnIiIiCj1cfpf0oqEBS12ux0bN27E2LFjA14fO3Ysvv/+e1n7eOONNzBo0CCMHDkyFkUkIiIiIiINSFgifnV1NVwuFzp37hzweufOnXH48OGwn6+vr8dHH32E++67L+y25eXlEZeTArEu1cX6VB/rVF2sT3WxPtXHOlUX61NdrE9lSkpKgr6XtLOHvf/++3C73bjyyivDbhuqAki+8vJy1qWKWJ/qY52qi/WpLtan+lin6mJ9qov1qa6EDQ/r1KkTDAYDqqqqAl6vqqpCYWFh2M+/8cYbuOiii5Cfnx+rIhIRERERkQYkLGgxm80YOnQoVq5cGfD6ypUrw+ao/PDDD/jll19w3XXXxbKIRERERESkAQmdPWzmzJl45513sGjRImzduhWzZs1CZWUlJk+eDACYNm0apk2b1u5zr7/+Ovr27YvRo0fHu8hERESUhJZV2DD+kyosq7AluihEFIGE5rRMnDgRNTU1ePLJJ3Ho0CEMGjQI77//PoqKigAA+/bta/eZxsZGLFmyBHfddVe8i0tERERJyn+RRE7fS5R8Ep6IP2XKFEyZMkXyveXLl7d7LScnB/v37491sYiIiCiFTC/NxotlTVwkkShJJTxoISIiIoq1CcVW9rAQJbGE5rQQERERERGFw6CFiIiIiIg0jUELERERESUMZ3YjORi0EBEREVHC+M/sBjCIIWkMWoiIiIgoYaaXZqNfrtE3s5s4iCECOHsYERERESWQeGY3Tk9NUhi0EBEREZFmcHpqksLhYUREREQaxfwOIi8GLUREREQaxfwOIi8GLUREREQaJU5SJ0pXzGkhIiIi0ijmdxB5saeFiIiIiFIG84BSE4MWIiIiIkoZzANKTQxaiIiIiChlMA8oNTGnhYiIiIhSBvOAUhN7WoiIiIiISNMYtBARERGRJjCJnoJh0EJEREREmqA0iZ5BTvpg0EJEREREmqA0iZ4zhaUPJuITERERkSYoTaKfXpqNF8uaOFNYGmDQQkRERERJiTOFpQ8ODyMiIiIiIk1j0EJEREREKWtZhQ0jl1Ri1JJDMUnY52QA8cGghYiIiIhS1vyyJuxscMUsYZ+TAcQHgxYiIiIiSlnTS7PRN9eAvgpmJVO6fyUznlFkmIhPRERERCkr1sn6nAwgPhi0EBEREVFSW1Zhw/yyJgwvMGHdEQeml2YzkEgxHB5GRERERElNyCt5bVsL80tSFIMWIiIiIkpqQl7J5P6ZzC9JURweRkRERJSihGFTqT5cinklqY89LUREREQpitPxUqpg0EJERESUojgdb3xwgcnYY9BCRERElKImFFuxfHxnDp2KkNxghD1asceghYiIiIhIgtxghD1ascdEfCIiIiIiCdNLs/FiWVPYYIQTAcQegxYiIiIiIgkMRrSDw8OIiIiIiEjTGLQQEREREZGmMWghIiIiIiJNY9BCRERERESaxqCFiIiIiIg0jUELERERERFpGoMWIiIiIiLSNAYtRERERESkaQxaiIiIiIhI0xi0EBERERGRpjFoISIiIiIiTWPQQkREREREmsaghYiIiIiINI1BCxERERERaRqDFiIiIiIi0jQGLUREREREpGkMWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJrGoIWIiIiI0s6yChvGf1KFZRW2RBeFZGDQQkRERERpZ35ZE3Y0OPFiWVOii0IyMGghIiIiorQzvTQb/XKNuLk0O9FFIRmMiS4AEREREVG8TSi2YkKxNdHFIJnY00JERERERJrGoIWIiIiI0h4T87WNQQsRERERpRylQQgT87WNQQsRERERpRylQQgT87WNifhERERElHKml2bjxbIm2UEIE/O1jUELEREREaUcBiGphcPDiIiIiCjlMLE+tTBoISIiIqKUw8T61MKghYiIiIhSDhPrUwtzWoiIiIgo5TCnJbWwp4WIiIiIiDSNQQsREREREWkagxYiIiIiItI0Bi1ERERERKRpDFqIiIiIKGlw/ZX0xKCFiIiIiJIG119JTwxaiIiIiChpcP2V9MR1WoiIiIgoaXD9lfTEnhYiIiIiItI0Bi1ERERERKRpDFqIiIiIiEjTEh60LFy4EEOGDEGXLl0wZswYfPvttyG3t9vteOSRRzBkyBAUFhbi+OOPx4svvhin0hIRERERUbwlNBF/yZIlmD17Np566imMGjUKCxcuxBVXXIE1a9agV69ekp+58cYbceDAATz77LM47rjjUFVVBZuN83QTEREREaWqhAYt8+bNw9VXX43rr78eAPDkk0/iyy+/xKuvvoo5c+a0237FihX4+uuv8eOPP6JTp04AgN69e8e1zEREREREFF8JGx5mt9uxceNGjB07NuD1sWPH4vvvv5f8zPLly3HSSSdh3rx5KC0txbBhw3DXXXehqYmLCxERERERpaqE9bRUV1fD5XKhc+fOAa937twZhw8flvxMRUUF1qxZg4yMDCxatAj19fW46667UFlZiUWLFgU9Vnl5uaplT2esS3WxPtXHOlUX61NdrE/1sU7VxfpUF+tTmZKSkqDvJdXikm63GzqdDi+//DI6dOgAwDukbOLEiTh8+DAKCwslPxeqAki+8vJy1qWKWJ/qY52qi/WpLtan+lin6mJ9qov1qa6EDQ/r1KkTDAYDqqqqAl6vqqoKGnx06dIF3bp18wUsANC/f38AwL59+2JXWCIiIiIiSpiEBS1msxlDhw7FypUrA15fuXIlRo4cKfmZUaNGobKyMiCHZceOHQAQdLYxIiIiIiJKbgldp2XmzJl45513sGjRImzduhWzZs1CZWUlJk+eDACYNm0apk2b5tv+8ssvR8eOHTFz5kz8+uuvWLNmDWbPno2LL764XW4MERERERGlhoTmtEycOBE1NTV48skncejQIQwaNAjvv/8+ioqKALQf8pWdnY2PPvoId911F8aOHYu8vDxceOGFktMjExERERFRakh4Iv6UKVMwZcoUyfeWL1/e7rWSkhL8+9//jnWxiIiIiIhIIxI6PIyIiIiIiCgcBi1ERERERKRpDFqIiIiIiEjTGLQQEREREZGmMWghIiIiIiJNY9BCRERERESaxqCFiIiIiIg0jUELERERERFpGoMWIiIiIiLSNAYtRERERESkaQxaiIiIiIhI0xi0EBERERGRpjFoISIiIiIiTWPQQkREREREmsaghYiIiIiINI1BCxERERERaRqDFiIiIiIi0jQGLUREREREpGkMWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJrGoIWIiIiIiDSNQQsREREREWkagxYiIiIiItI0Bi1ERERERKRpDFqIiIiIiEjTGLQQEREREZGmMWghIiIiIiJNY9BCRERERESaxqCFiIiIiIg0jUELERERERFpGoMWIiIiIiLSNAYtRERERESkaQxaiIiIiIhI0xi0EBERERGRpjFoISIiIiIiTWPQQkREREREmsaghYiIiIiINI1BCxERERERaRqDFiIiIiIi0jQGLUREREREpGkMWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJrGoIWIiIiIiDSNQQsREREREWkagxYiIiIiItI0Bi1ERERERKRpDFqIiIiIiEjTGLQQEREREZGmMWghIiIiIiJNY9BCRERERESaxqCFiIiIiIg0jUELERERERFpGoMWIiIiIiLSNAYtRERERESkaQxaiIiIiIhI0xi0EBERERGRpjFoISIiIiIiTWPQQkREREREmsaghYiIiIiINI1BCxERERERaRqDFiIiIiIi0jQGLUREREREpGkMWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJrGoIWIiIiIiDSNQQsREREREWkagxYiIiIiItI0Bi1ERERERKRpDFqIiIiIiEjTGLQQEREREZGmMWghIiIiIiJNY9BCRERERESaxqCFiIiIiIg0jUELERERERFpGoMWIiIiIiLSNAYtRERERESkaQxaiIiIiIhI0xIetCxcuBBDhgxBly5dMGbMGHz77bdBt121ahXy8vLa/dm2bVscS0xERERERPFkTOTBlyxZgtmzZ+Opp57CqFGjsHDhQlxxxRVYs2YNevXqFfRza9asQX5+vu//BQUF8SguERERERElgK6urs6TqIOfffbZGDx4MJ577jnfa8OGDcPFF1+MOXPmtNt+1apVmDBhAnbs2IFOnTrFs6hERERERJQgCRseZrfbsXHjRowdOzbg9bFjx+L7778P+dnf/OY3GDBgAC666CJ8/fXXsSwmERERERElWMKGh1VXV8PlcqFz584Br3fu3BmHDx+W/EzXrl0xd+5cDBs2DHa7He+99x4uvvhiLF++HKeddlo8ik1ERERERHGW0JwWpUpKSlBSUuL7/4gRI7Bnzx4899xzDFqIiIiIiFJUwoaHderUCQaDAVVVVQGvV1VVobCwUPZ+Tj75ZOzcuVPt4hERERERkUYkLGgxm80YOnQoVq5cGfD6ypUrMXLkSNn7+fnnn9GlSxe1i0dERERERBqR0OFhM2fOxLRp03DyySdj5MiRePXVV1FZWYnJkycDAKZNmwYAWLBgAQDgn//8J4qKijBo0CDY7Xa8//77WL58ORYtWpSwcyAiIiIiothK6OKSEydOxGOPPYYnn3wSo0ePxpo1a/D++++jqKgIALBv3z7s27fPt73D4cB9992H008/HRdccIFv+4suuihgv0oWrEwnc+fOxVlnnYVevXqhb9++mDRpEsrKygK28Xg8eOyxxzBw4EB07doVF154IX799deAberq6jB16lQUFRWhqKgIU6dORV1dXcA2mzdvxvjx49G1a1cMGjQITzzxBDyehM2uHRdz585FXl4e7rzzTt9rrE9lKisrcfPNN6Nv377o0qULRo4cidWrV/veZ33K53K58PDDD/t+Fg4ZMgQPP/wwnE6nbxvWZ2jffPMNrrzySgwaNAh5eXl4++23A96PZ/0tXboUI0eORGFhIUaOHIlly5bF5JxjKVR9OhwOzJkzB6eddhq6d++OAQMGYMqUKdi7d2/APtra2nDnnXfiuOOOQ/fu3XHllVdi//79Advs3bsXkyZNQvfu3XHcccfhrrvugt1uD9hm9erVGDNmDLp06YITTzwRr776auxOPEbC3Z/+brvtNuTl5eH5558PeJ31GUhOnW7fvh2///3vUVRUhG7duuHMM8/E1q1bfe+zTmMnoUELAEyZMgU///wzDh8+jK+++gqnn366773ly5dj+fLlvv/feuut2LBhAyorK1FRUYFPP/0U5513XsD+hAUrb7/9dnz99dcYMWIErrjiinY/+NLR6tWr8Yc//AGff/45Pv74YxiNRlxyySWora31bfPss89i3rx5eOKJJ7BixQp07twZl156KRobG33bTJkyBZs2bcKHH36IDz/8EJs2bfL1igFAQ0MDLr30UhQWFmLFihV4/PHH8fzzz+OFF16I6/nG07p16/D6669j8ODBAa+zPuWrq6vDuHHj4PF48P777+P777/H3//+94AZBlmf8j3zzDNYuHAhnnjiCaxduxaPP/44Xn75ZcydO9e3DesztObmZpSWluLxxx+H1Wpt93686m/t2rW48cYbccUVV2DVqlW44oorcMMNN2D9+vWxrQCVharPlpYW/PTTT7jjjjvw1Vdf4Z133sH+/ftx+eWXBwTad999N5YtW4ZXXnkFn3zyCRobGzFp0iS4XC4A3mB90qRJaGpqwieffIJXXnkFH3/8Me655x7fPioqKvC73/0OI0aMwNdff42//OUvuOuuu7B06dL4VIRKwt2fgqVLl+KHH35At27d2r3H+gwUrk4rKiowbtw49O7dGx9//DG+++473HvvvcjKyvJtwzqNnYQuLhkLShesTGdNTU0oKirC22+/jQsuuAAejwcDBw7ETTfdhDvuuAMAYLPZUFJSgoceegiTJ0/G1q1bMXLkSHz22WcYNWoUAOC7777DBRdcgHXr1qGkpASvvPIK7r//fmzbts33pX/yySfx6quvoqysDDqdLmHnHAv19fUYM2YMnnvuOTzxxBMoLS3Fk08+yfpU6MEHH8Q333yDzz//XPJ91qcykyZNQn5+Pl588UXfazfffDNqa2vx3nvvsT4V6tGjB/7+97/jmmuuARDf+3Hy5Mmora3FRx995CvPxRdfjIKCArzyyivxrQiViOtTypYtWzBq1Ch88803GDx4MOrr69GvXz/MmzcPv/vd7wB4R2SccMIJ+PDDD3H22Wfjv//9L373u9/h559/Rs+ePQEA7733Hv70pz+hvLwcubm5mDNnDpYtW4YNGzb4jnXLLbdgy5Yt+O9//xvbE4+RYPW5Z88ejBs3Dh999BEuv/xyTJ06FbfccgsAsD7DkKrTKVOmQKfT4eWXX5b8DOs0thLe06KmaBasTEdNTU1wu93Iy8sDAOzevRuHDh0KqD+r1YrTTjvNV39r165FdnZ2wGQJo0aNQlZWVsA2p556asBTirPPPhsHDx7E7t2743Bm8XXbbbfh4osvxplnnhnwOutTmeXLl+Pkk0/G5MmT0a9fP5xxxhl46aWXfMNkWJ/KjBo1CqtXr8a2bdsAeBuAq1atwrnnnguA9RmteNbfunXr2v1eO/vss1P+95rQYyX8jtq4cSMcDkdAXfTs2RMDBgwIqM8BAwb4GoOAt67a2tqwceNG3zZS9fnjjz/C4XDE8Iziy+l0YsqUKbjjjjswYMCAdu+zPpVxu9347LPPMGDAAFx22WXo27cvzjrrLCxZssS3Des0tlIqaIlkwcp0Nnv2bJxwwgkYMWIEAODQoUMAELL+Dh8+jE6dOgU8PdXpdCgoKAjYRmofwnup5I033sDOnTtx7733tnuP9alMRUUFXnnlFRQXF+Nf//oXbr75ZjzwwAO+J1qsT2Vuu+02TJo0CSNHjkRBQQFGjRqFq666ClOmTAHA+oxWPOvv0KFDafd7zW63495778X555+PHj16APDWh8FgQKdOnQK2Fde5uK6EJRbC1bnT6UR1dXWsTinuHnvsMXTs2BF/+MMfJN9nfSpTVVWFpqYmX37wv//9b1x22WW46aabfCMEWKexlVSLS5J6/vrXv2LNmjX47LPPYDAYEl2cpFReXo4HH3wQn332GUwmU6KLk/TcbjdOOukk3zDOE088ETt37sTChQsxderUBJcu+SxZsgSLFy/GwoULMXDgQPz888+YPXs2ioqKcN111yW6eERBOZ1OTJ06FfX19Xj33XcTXZyktGrVKrzzzjtYtWpVoouSMtxuNwBg/Pjx+OMf/wgAGDJkCDZu3IiXX34Z48aNS2Tx0kJK9bSotWBlqrv77rvxr3/9Cx9//DGKi4t9rwvr3YSqv8LCQlRXVwfMbOPxeHDkyJGAbaT2IbyXKtauXYvq6mqMGjUKnTp1QqdOnfDNN99g4cKF6NSpEzp27AiA9SlXly5d2g1h6N+/v28GQd6fytx333344x//iMsuuwyDBw/GlVdeiZkzZ+Lpp58GwPqMVjzrr0uXLmnze83pdOIPf/gDNm/ejKVLl/p+jgLe+nC5XO2eNIvrXFxXwiiMcHVuNBrbPSFPVqtXr0ZlZSUGDBjg+/20d+9ezJkzB6WlpQBYn0p16tQJRqMx5O8p1mlspVTQotaClals1qxZvoClf//+Ae/17t0bXbp0Cai/1tZWfPfdd776GzFiBJqamrB27VrfNmvXrkVzc3PANt999x1aW1t926xcuRLdunVD7969Y3l6cXXhhRfi22+/xapVq3x/TjrpJFx22WVYtWoV+vXrx/pUYNSoUdi+fXvAa9u3b0evXr0A8P5UqqWlpV0vqsFg8D0tZH1GJ571N3z48LT4veZwODB58mRs3rwZy5Yta7dw9NChQ2EymQLqYv/+/b4JDwBvfW7dujVgitmVK1ciIyMDQ4cO9W0jVZ8nnXRSyvSaT5kyBd98803A76du3bphxowZvhmoWJ/KmM1mDBs2DOXl5QGv+/+eYp3GVkoFLYB3wcp33nkHixYtwtatWzFr1qyABSvT2R133IF33nkHL7/8MvLy8nDo0CEcOnQITU1NALxjradPn45nn30WH3/8McrKyjBjxgxkZWXh8ssvBwAMGDAA55xzDv785z9j7dq1WLt2Lf785z9j3LhxKCkpAQBcfvnlsFqtmDFjBsrKyvDxxx/jmWeewYwZM1JmJiHAmxxaWloa8CczMxP5+fkoLS1lfSo0Y8YMrFu3Dv/4xz+wc+dOfPTRR3jppZd8ORisT2XOP/98PPPMM/j888+xe/duLFu2DPPmzcNvf/tbAKxPOZqamrBp0yZs2rQJbrcb+/btw6ZNm7B379641t/NN9+Mr7/+Gk8//TS2bduGuXPnYtWqVZg+fXrC6iYSoerT6XTi+uuvx/r167Fw4ULodDrf7yibzQYA6NChA6699lrMmTMH//vf//DTTz9h2rRpGDx4MH7zm98A8E68M2jQINx888346aef8L///Q/33XcfrrvuOuTm5gIAJk+ejIMHD2L27NnYunUrFi1ahHfeecc35CdZhKrPzp07t/v9ZDQa0aVLF9+9x/psL1SdAsCf/vQn/Pvf/8brr7+OnTt34o033sCSJUt8v6dYp7GVclMeA97FJZ999lkcOnQIgwYNwqOPPhqw/ku6EmZgEZs1axbuvvtuAN6hC48//jhef/111NXV4eSTT8Y//vEPX3cy4F1P46677sKnn34KALjgggvw97//PWD/mzdvxh133IENGzYgLy8PkydPxqxZs5K+ERPOhRde6JvyGGB9KvX555/jwQcfxPbt29GzZ0/cdNNNmDZtmu88WZ/yNTY24pFHHsF//vMfHDlyBF26dMFll12Gu+66CxaLBQDrM5xVq1ZhwoQJ7V6/6qqrMH/+/LjW39KlS/Hwww+joqICffr0wb333ttuYWWtC1Wfs2fPxoknnij5uXnz5vmmnW1ra8O9996LDz/8EK2trTjzzDPx1FNPBczEtHfvXtxxxx34+uuvYbFYcMUVV+Chhx5CRkaGb5vVq1fjr3/9K7Zs2YKuXbvitttuw4033qjyGcdWuPtT7IQTTgiY8hhgfYrJqdO3334bc+fOxf79+3HcccfhL3/5i+9BBcA6jaWUDFqIiIiIiCh1pNzwMCIiIiIiSi0MWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJrGoIWIiIiIiDSNQQsRUQKsWrUKeXl5+Ne//pXoosg2f/58DB06FB07dsQZZ5yR6OKo6u2330ZeXh52796d6KIkjHBPrlq1KtFFISJqh0ELERGF9d133+Huu+/GySefjBdeeAH33XdfooukGQcPHsRjjz2GTZs2Jboosjz11FP4z3/+k+hiEBEpYkx0AYiISPtWr14NAJg7dy46dOiQ4NKo78orr8Rll10WsCK1XJWVlXjiiSdQVFSEIUOGxKB06po7dy4uuugi/Pa3vw14/fTTT0dlZSXMZnOCSkZEFBx7WoiIUlhzc7Mq+6mqqgKAlAxYAMBgMMBisUCn0yW6KD5qXTu59Ho9LBYL9Ho2DYhIe/iTiYhS3mOPPYa8vDyUl5dj+vTpKCoqQlFREWbMmIGWlhbfdrt370ZeXh7efvvtdvvIy8vDY4891m6fW7duxdSpU1FUVITjjjsODz74IDweDw4cOICrr74avXr1QklJCZ577jnJsrlcLjz66KMYOHAgunXrhokTJ2LHjh3tttu+fTtuuOEG9OnTB126dMHo0aOxdOnSgG2EvIyvvvoKd911F0pKStCjR4+QdeNyufCPf/wDJ510EgoLC3H88cfjvvvug81mCzj3l156yffvYHUkOHz4MG655RYMHjwYhYWFKCkpweWXX45ff/01YLsVK1Zg/Pjx6NGjB3r06IHLLrtMcoiVknNfvXo1/vrXv6Jv377o3r07rrnmGhw5ciRkHfh/3j+n5cILL8Tw4cOxZcsWTJgwAd26dcOgQYPw7LPP+rZZtWoVzjrrLADAzJkzffXjf69Ee+1qa2vxt7/9Daeddhp69uyJHj164MILL8S3337b7jw8Hg9efvllnHHGGejatSuOO+44XHLJJb5t8/Ly0NzcjHfffddX1gsvvNB3LlI5LatXr8b48ePRvXt3FBUVYdKkSSgrKwvYRu53DAC++uorXHDBBejduze6deuGoUOH4s477wx7jYgovXF4GBGljRtvvBHFxcWYM2cOfvrpJyxatAidO3fGAw88EPE+//CHP6B///6YM2cOvvjiC8ydOxf5+fl46623cNppp+H+++/HBx98gPvuuw8nnngixowZE/D5Z555Bm63G3/84x9RV1eHBQsWYMKECfjmm2+Qn58PANi6dSvOO+88dOnSBbfeeiuysrLwn//8B9dffz0WLFiASZMmBexz1qxZyMvLw+23346GhoaQ5b/tttvw5ptvYsKECZg5cyZ+/PFHPPfcc/j111/x/vvvQ6fTYcGCBVi8eDFWrlyJBQsWAABGjhwZdJ/XX389Nm/e7Avmqqur8c0332D79u0YNGgQAOCDDz7A1KlTcdZZZ+G+++6D3W7H66+/jvHjx2PFihXo379/ROd+9913Iz8/H7NmzcKePXswf/583HnnnXjttddkXM32GhoacPnll+O3v/0tLrnkEixduhRz5sxBaWkpzj33XAwYMAB//etf8eijj+KGG27AqaeeCgAYPHhwROWXunYVFRVYunQpLr30UhQXF6O+vh5vvvkmLrnkEqxYsQLHH3+87/O33norFi1ahLPPPhtXX301PB4P1q5di2+//RannXYaFixYgD/96U8YNmwYbrjhBgBAYWFh0PP/+uuvMXHiRPTu3RuzZ89Ga2srFi5ciPPPPx8rVqxAv379ArYP9x3bsmULfve736G0tBSzZ89GZmYmdu3ahS+//DKi60NE6YNBCxGljSFDhmDevHm+/9fU1ODNN9+MKmgZOnQoXnjhBQDADTfcgCFDhuC+++7DPffcgzvuuAMAcNlll2HQoEF4++232wUtVVVVWLduHfLy8gAAo0ePxsUXX4x58+bh3nvvBQDMnj0b3bp1w8qVK2G1WgEAN910Ey699FI88MAD+N3vfhcwrEloGBuNoX/E//LLL3jzzTdx9dVX45///Kfv9Z49e+KJJ57A559/jvPPPx+TJk3C+vXrsXLlynaNbLG6ujp89913eOihh3DLLbf4Xv/zn//s+3dzczPuvPNOXH311QHX49prr8Upp5yCv//971i4cGFE596xY0d89NFHvtfcbjcWLFiA+vr6iIa2HTp0CPPnz8dVV13lK+MJJ5yAN998E+eeey4KCwtx7rnn4tFHH8Xw4cPb1Y8a1660tBQbN24MGLZ1ww03YPjw4ViwYAGef/55AN6ekkWLFmHKlCn4xz/+4dt25syZ8Hg8AIBJkybhL3/5C4qLi8NeSwC49957kZubi//+97/o2LEjAO/9PGrUKDz44INYtGhRwPbhvmMrV65EW1sbPvzwQ3Tq1Mm33f333x+2LESU3jg8jIjSxvXXXx/w/1NPPRU1NTVheyNCue6663z/NhgMGDp0KDweD6699lrf63l5eejXrx8qKiraff7KK6/0BSwAMGbMGAwaNAifffYZAO/QoP/973+45JJL0NLSgurqat+fs88+GwcOHMD27dvbnWe4gAUAvvjiCwDeRq2/GTNmwGAw+N5Xwmq1wmw2Y/Xq1aitrZXcZuXKlairq8MVV1wRcD4ulwunnnqqb3hSJOd+7bXXBgQBp556KlwuF/bu3av4XITz8W/cm81mDBs2TPJaiql17TIyMnwBS2trK2pqauByuTBs2DBs3LjRt93HH38MwNvbJBZJrk5lZSU2bdqEq666yhewAEDfvn1xwQUX4Msvv4TL5WpXfn/i71hubi4AYPny5XC73YrLRETpiz0tRJQ2evbsGfB/IVioq6vzNaai3Wdubi5MJhO6dOnS7nUhmd1f3759JV/7+uuvAQA7d+6Ex+PB448/jscff1yyDFVVVSgpKfH9v7i4WFbZ9+7dC51O126IT4cOHdC1a1fs2bNH1n78ZWRk4P7778ff/vY3lJSU4JRTTsG5556LSZMm+epKyNm55JJLJPchNNAjOfdQ1zgS3bp1a5eYnpeXh82bN4f9rFrXzu1249lnn8Xrr7/ebh2Z3r17+/69a9cuFBYWBvRgREMI9PzLJ+jfvz8+/vhjVFdXBwwvC/cdmzhxIt566y386U9/wv33348zzzwTF154IS699FJZgTYRpS/+hCCitGEwGCRfF4bOBHsaLX6aHG6fwWZfEo6jhPA0esaMGTjvvPMktyktLQ34vzAMKVFmzJiB8ePH45NPPsH//vc/PPnkk5g7dy4WL16M0aNH+87pn//8J7p37x50P5Gce7hrrFQ0+1Pr2s2dOxcPP/wwrrrqKtx7773o2LEjDAYD5s6di127doUtRzyFqy+r1Yrly5fjm2++wX//+198+eWXuOmmmzBv3jx8+umnCb93iUi7GLQQER0lPBWur68PeD3SoUVySM0UtmPHDhQVFQE49uTdaDTiN7/5jarH7tWrFzweD7Zv3+5LHAe8yeeVlZUYN25cxPsuLi7GjBkzMGPGDOzfvx+jR4/GU089hdGjR6NPnz4AgIKCgpDnFMtzV1OwYFet8n/00Uc444wzMH/+/IDX/WcoA4A+ffrg//7v/3DkyBEUFBQoLq9Yr169AADl5eXt3isvL0dWVlZEvTp6vR6jR4/G6NGj8eCDD+KVV17B7bffjmXLluF3v/ud4v0RUXpgTgsR0VG5ubno1KlTu6lkhaTwWFi8eHHA0KWvvvoKv/76qy9g6Ny5M0aPHo033ngDBw4caPd5OdP5BiM8/Rc3hl988UW4XK6IgpaWlpaA6ZIBoEePHujcubMvGBw7diw6dOiAuXPnwm63t9uHcE6xPHc1ZWZmAmg/BE2t8hsMhnY9O99//z3Wrl0b8NpFF10EAJJD0fw/n5mZKWu4XNeuXXHiiSdi8eLFAflJu3btwqeffopzzjknaM9KMDU1Ne1eO/HEEwG0f1hAROSPPS1ERH6uu+46PP3007jllltw0kkn4dtvv22XLK2mzp074/zzz8fvf/971NfX48UXX0TXrl0DkuPnzp2LcePG4fTTT8f111+PPn36oKqqCuvXr8fWrVvx448/RnTs448/Htdeey3efPNNNDQ04Mwzz8RPP/2Et956C+ecc07QIU2hbN++HRdddBEuueQSDBw4EBkZGfjiiy+wdetWPPTQQwC8weHTTz+Nm266CWeeeSYuu+wyFBYWYu/evfjyyy8xcOBAXyAVq3NXU58+fZCXl4dXX30V2dnZyM7OxqBBg1BaWqpK+S+44AI8/vjjmDZtGk477TTs2LEDr7/+OgYOHIimpibfdqNHj8bVV1+NhQsXYteuXTjnnHMAAOvWrcPgwYNx++23AwBOOukkfPXVV3j++efRvXt3FBQUtJvVTvDQQw9h4sSJOPfcc3H99df7pjy2WCz429/+priu/v73v2P16tUYN24cioqKUFdXh1dffRVZWVlR9ewRUepj0EJE5Oeuu+7CkSNHsHTpUnz00Uc455xz8OGHH7ZLVlfLbbfdhvLycjz//POor6/Hqaeeir///e8BszWVlJRg5cqVeOKJJ7B48WJUV1ejoKAAxx9/PO65556ojv/MM8+gd+/eeOutt/Dpp5+isLAQt9xyC+6+++6IZpzq2bMnrrjiCnz99df48MMPodPp0LdvXzz//PMBM6pNnDgRXbt2xdy5c/HCCy+gra0NXbt2xciRIzF58uS4nLtaTCYTFixYgAceeAB33HEHHA4HZs2ahdLSUlXK/5e//AU2mw0ffPABli5dikGDBuHVV1/Fv/71L6xevTpg2xdeeAGDBw/Gm2++iTlz5iA7OxsnnngiTj/9dN82jz76KG677TY8/vjjaG5uxumnnx40aDnzzDPx73//G48++igeffRRGI1GnHrqqZgzZ05E34nx48dj3759ePfdd3HkyBF07NgRw4cPx1133eUbEklEJEVXV1cXWXYiERERERFRHDCnhYiIiIiINI1BCxERERERaRqDFiIiIiIi0jQGLUREREREpGkMWoiIiIiISNMYtBARERERkaYxaCEiIiIiIk1j0EJERERERJrGoIWIiIiIiDTt/wH75Inlhw3GEgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "def roc_score_per_interaction(x):\n",
    "    try:\n",
    "        return roc_auc_score(x.answered_correctly.values, x.predictions.values)\n",
    "    except ValueError:\n",
    "        return np.nan\n",
    "\n",
    "t[\"uid_count\"] = (t[\"uid_count\"] // 10) *10\n",
    "interactions_auc = t.groupby(\"uid_count\").apply(roc_score_per_interaction)\n",
    "interactions_auc = interactions_auc[~interactions_auc.isna()].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "seq_length, n_users = batch[\"content_ids\"].shape\n",
    "lengths = batch[\"length\"]\n",
    "\n",
    "users = torch.arange(n_users)\n",
    "\n",
    "user_indexes = []\n",
    "sequence_indexes = []\n",
    "\n",
    "for i in range(steps.max().int(), 0, -1):\n",
    "    preds = self.process_batch_step(batch)\n",
    "\n",
    "    sequence_indexes_at_i = lengths[steps >= i] - i\n",
    "    user_indexes_at_i = users[steps >= i]\n",
    "\n",
    "    # get index for which to update the answers\n",
    "    # since answers is shifted we want to map preds 0..98 -> answers 1:99\n",
    "    answers_idx = torch.where(sequence_indexes_at_i + 1 != seq_length)\n",
    "    a_seq_idx = sequence_indexes_at_i[answers_idx] + 1\n",
    "    u_seq_idx = user_indexes_at_i[answers_idx]\n",
    "\n",
    "    # set answer to either 0 or 1 if not lecture\n",
    "    batch[\"answers\"][a_seq_idx, u_seq_idx] = torch.where(\n",
    "        batch[\"answers\"][a_seq_idx, u_seq_idx] != 4,\n",
    "        (preds[sequence_indexes_at_i[answers_idx], u_seq_idx] > 0.5).long(),\n",
    "        batch[\"answers\"][a_seq_idx, u_seq_idx],\n",
    "    )\n",
    "\n",
    "    user_indexes.append(user_indexes_at_i)\n",
    "    sequence_indexes.append(sequence_indexes_at_i)\n",
    "\n",
    "if return_all_preds:\n",
    "    return preds\n",
    "\n",
    "user_indexes = torch.cat(user_indexes)\n",
    "sequence_indexes = torch.cat(sequence_indexes)\n",
    "\n",
    "return (\n",
    "    preds[sequence_indexes, user_indexes],\n",
    "    batch[\"row_ids\"][sequence_indexes, user_indexes],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simulate Kaggle Sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InferenceRIIDDataset(Dataset):\n",
    "    \"\"\"RIID dataset.\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, hdf5_file=\"feats.h5\", window_size=100,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            user_mapping (np.array): array of all unique user ids \n",
    "            hdf5_file (string): location of hf5 feats file\n",
    "        \"\"\"\n",
    "        self.hdf5_file = hdf5_file\n",
    "        self.max_window_size = window_size\n",
    "        self.cache = {}\n",
    "\n",
    "    def open_hdf5(self):\n",
    "        # opens the h5py file\n",
    "        self.f = h5py.File(self.hdf5_file, \"r\")\n",
    "\n",
    "    def __len__(self):\n",
    "        if not hasattr(self, \"f\"):\n",
    "            self.open_hdf5()\n",
    "        return len(self.f.keys())\n",
    "\n",
    "    def load_user_into_cache(self, user_id):\n",
    "        \"\"\"\n",
    "        add a user to self.cache\n",
    "        \"\"\"\n",
    "\n",
    "        if not hasattr(self, \"f\"):\n",
    "            self.open_hdf5()\n",
    "\n",
    "        if f\"{user_id}\" in self.f:\n",
    "            content_ids = np.array(self.f[f\"{user_id}/content_ids\"], dtype=\"int64\")\n",
    "            answers = np.array(self.f[f\"{user_id}/answered_correctly\"], dtype=\"int64\")\n",
    "            timestamps = np.array(self.f[f\"{user_id}/timestamps\"], dtype=\"int64\")\n",
    "            prior_q_times = np.array(self.f[f\"{user_id}/prior_question_elapsed_time\"], dtype=\"float32\")\n",
    "\n",
    "            self.cache[user_id] = {\n",
    "                \"content_ids\": content_ids,\n",
    "                \"answers\": answers,\n",
    "                \"timestamps\": timestamps,\n",
    "                \"prior_q_times\": prior_q_times,\n",
    "                \"row_ids\": np.zeros(len(timestamps)),\n",
    "                \"steps\": 0,\n",
    "            }\n",
    "        else:\n",
    "            self.cache[user_id] = {\n",
    "                \"content_ids\": np.array([], dtype=\"int64\"),\n",
    "                \"timestamps\": np.array([], dtype=\"int64\"),\n",
    "                \"answers\": np.array([], dtype=\"int64\"),\n",
    "                \"prior_q_times\": np.array([], dtype=\"float32\"),\n",
    "                \"row_ids\": np.array([], dtype=\"int64\"),\n",
    "                \"steps\": 0,\n",
    "            }\n",
    "\n",
    "    def update_user_rows(self, user_rows):\n",
    "        if not hasattr(self, \"f\"):\n",
    "            self.open_hdf5()\n",
    "\n",
    "        user_id = user_rows.user_id.values[0]\n",
    "        num_rows = len(user_rows)\n",
    "        new_content_ids = user_rows.content_id.values\n",
    "        new_timestamps = user_rows.timestamp.values\n",
    "        new_answered_correctly = (\n",
    "            user_rows.answered_correctly.values\n",
    "        )  # should be 3 (3) for every question and 4 for lectures\n",
    "        new_prior_q_times = user_rows.prior_question_elapsed_time.values\n",
    "        \n",
    "        new_row_ids = user_rows.row_id.values\n",
    "\n",
    "        if user_id not in self.cache:\n",
    "            self.load_user_into_cache(user_id)\n",
    "\n",
    "        self.cache[user_id][\"content_ids\"] = np.concatenate(\n",
    "            [self.cache[user_id][\"content_ids\"], new_content_ids]\n",
    "        )[-(self.max_window_size + 1) :]\n",
    "\n",
    "        self.cache[user_id][\"timestamps\"] = np.concatenate(\n",
    "            [self.cache[user_id][\"timestamps\"], new_timestamps]\n",
    "        )[-(self.max_window_size + 1) :]\n",
    "\n",
    "        self.cache[user_id][\"answers\"] = np.concatenate(\n",
    "            [self.cache[user_id][\"answers\"], new_answered_correctly]\n",
    "        )[-(self.max_window_size + 1) :]\n",
    "\n",
    "        self.cache[user_id][\"prior_q_times\"] = np.concatenate(\n",
    "            [self.cache[user_id][\"prior_q_times\"], new_prior_q_times]\n",
    "        )[-(self.max_window_size + 1) :]\n",
    "\n",
    "        self.cache[user_id][\"row_ids\"] = np.concatenate(\n",
    "            [self.cache[user_id][\"row_ids\"], new_row_ids]\n",
    "        )[-(self.max_window_size + 1) :]\n",
    "\n",
    "        self.cache[user_id][\"steps\"] = len(new_row_ids)\n",
    "\n",
    "    def update_answered_correctly(self, answered_correctly_rows):\n",
    "        user_id = answered_correctly_rows.name\n",
    "        new_answered_correctly = (\n",
    "            answered_correctly_rows.values\n",
    "        )  # this is only the answers\n",
    "        \n",
    "        # update the correct answers for the questions (keep 4 - lectures)\n",
    "        n = self.cache[user_id][\"steps\"]\n",
    "        l = len(self.cache[user_id][\"answers\"])\n",
    "        idxs = np.where(self.cache[user_id][\"answers\"][-n:] != 4)[0] + l - n\n",
    "\n",
    "        self.cache[user_id][\"answers\"][idxs] = new_answered_correctly\n",
    "        self.cache[user_id][\"steps\"] = 0\n",
    "\n",
    "\n",
    "    def __getitem__(self, user_id):\n",
    "        if user_id not in self.cache:\n",
    "            self.load_user_into_cache(user_id)\n",
    "\n",
    "        length = len(self.cache[user_id][\"content_ids\"])\n",
    "        window_size = min(self.max_window_size, length)\n",
    "\n",
    "        content_ids = np.array(self.cache[user_id][\"content_ids\"][-window_size:]).copy()\n",
    "        answers = np.array(self.cache[user_id][\"answers\"][-window_size:]).copy()\n",
    "        timestamps = np.array(self.cache[user_id][\"timestamps\"][-window_size:]).copy()\n",
    "        prior_q_times = np.array(self.cache[user_id][\"prior_q_times\"][-window_size:]).copy()\n",
    "        row_ids = np.array(self.cache[user_id][\"row_ids\"][-window_size:]).copy()\n",
    "        \n",
    "        # convert timestamps to time elapsed\n",
    "        time_elapsed_timestamps = get_time_elapsed_from_timestamp(timestamps)\n",
    "\n",
    "        # get tags\n",
    "        tags = questions_lectures_tags[content_ids, :].astype(np.int64)\n",
    "\n",
    "        # get parts\n",
    "        parts = questions_lectures_parts[content_ids].astype(np.int64)\n",
    "        \n",
    "        # shift by one the answered_correctly sequence\n",
    "        answers = np.roll(answers, 1)\n",
    "\n",
    "\n",
    "        if length > self.max_window_size:\n",
    "            answers[0] = self.cache[user_id][\"answers\"][-window_size - 1]\n",
    "        else:\n",
    "            answers[0] = 2\n",
    "\n",
    "        return {\n",
    "            \"parts\": torch.from_numpy(parts).long(),\n",
    "            \"tags\": torch.from_numpy(tags).long(),\n",
    "            \"content_ids\": torch.from_numpy(content_ids).long(),\n",
    "            \"answers\": torch.from_numpy(answers).long(),\n",
    "            \"prior_q_times\": torch.from_numpy(prior_q_times).float(),\n",
    "            \"timestamps\": torch.from_numpy(time_elapsed_timestamps).float(),\n",
    "            \"length\": window_size,\n",
    "            \"row_ids\": torch.from_numpy(row_ids).int(),\n",
    "            \"steps\": self.cache[user_id][\"steps\"],\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn.utils.rnn import pad_sequence\n",
    "\n",
    "\n",
    "# The collate function is used to merge individual data samples into a batch\n",
    "# It handles the padding aspect\n",
    "def inference_collate_fn(batch):\n",
    "    # collate lenghts into 1D tensor\n",
    "    items = {\n",
    "        \"length\": torch.tensor([batch_item[\"length\"] for batch_item in batch]),\n",
    "        \"steps\": torch.tensor([batch_item[\"steps\"] for batch_item in batch]),\n",
    "    }\n",
    "\n",
    "    # find shape that the batch will have\n",
    "    max_length = items[\"length\"].max()\n",
    "    num_items = len(batch)\n",
    "    if num_items > 1:\n",
    "        # padding list\n",
    "        for (key, padding) in [\n",
    "            (\"parts\", 0),\n",
    "            (\"content_ids\", 13942),\n",
    "            (\"answers\", 3),\n",
    "            (\"timestamps\", 0.0),  # note timestamps isnt an embedding\n",
    "            (\"prior_q_times\", 0),\n",
    "            (\"tags\", 188),\n",
    "            (\"row_ids\", 0),\n",
    "        ]:\n",
    "            items[key] = pad_sequence(\n",
    "                [batch_item[key] for batch_item in batch],\n",
    "                batch_first=False,\n",
    "                padding_value=padding,\n",
    "            )\n",
    "    else:\n",
    "        for key in [\n",
    "            \"parts\",\n",
    "            \"content_ids\",\n",
    "            \"answers\",\n",
    "            \"timestamps\",\n",
    "            \"tags\",\n",
    "            \"row_ids\",\n",
    "            \"prior_q_times\",\n",
    "        ]:\n",
    "            items[key] = batch[0][key].unsqueeze(1)\n",
    "    return items"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict for DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "model = RIIDDTransformerModel.load_from_checkpoint(\"nn-model/model.ckpt\")\n",
    "model.freeze()\n",
    "model.cuda()\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "cv_valid_ids = pd.read_pickle(f\"cv5_valid.pickle\")[\"row_id\"].values\n",
    "valid = train[train.row_id.isin(cv_valid_ids)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min, sys: 477 ms, total: 1min\n",
      "Wall time: 1min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "valid_groups = np.array_split(valid, len(valid) // 10) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inference_dataset = InferenceRIIDDataset(hdf5_file=\"train_no_lec_feats.h5\",)\n",
    "\n",
    "\n",
    "def predict_for_df(df):\n",
    "    # select questions only\n",
    "    unique_users_with_questions = df[~df.content_type_id].user_id.unique()\n",
    "    df[\"answered_correctly\"] = 0.5  # set useless value for column\n",
    "\n",
    "    # batch with only lectures\n",
    "    if len(unique_users_with_questions) < 1:\n",
    "        pass  # return empy\n",
    "\n",
    "    # case of single user with questions in batch\n",
    "    elif len(unique_users_with_questions) == 1:\n",
    "        item = inference_dataset[unique_users_with_questions[0]].copy()\n",
    "        predictions = model.predict_fast_single_user(\n",
    "            item[\"content_ids\"],\n",
    "            item[\"parts\"],\n",
    "            item[\"answers\"],\n",
    "            item[\"tags\"],\n",
    "            item[\"timestamps\"],\n",
    "            item[\"prior_q_times\"],\n",
    "            n=item[\"steps\"],\n",
    "        ).cpu()\n",
    "        df.loc[\n",
    "            df.user_id == unique_users_with_questions[0], \"answered_correctly\"\n",
    "        ] = predictions.numpy()\n",
    "    # case of multiple users with questions in batch\n",
    "    else:\n",
    "        batch = inference_collate_fn(\n",
    "            [inference_dataset[u].copy() for u in unique_users_with_questions]\n",
    "        )\n",
    "        predictions, row_ids = model.predict_n_steps(batch, batch[\"steps\"])\n",
    "        df[\"answered_correctly\"] = df[\"row_id\"].map(\n",
    "            dict(zip(row_ids.cpu().numpy(), predictions.cpu().numpy()))\n",
    "        )\n",
    "    return df[~df.content_type_id]\n",
    "\n",
    "\n",
    "previous_test_df = None\n",
    "predictions = []\n",
    "row_ids = []\n",
    "for current_test in tqdm(valid_groups):\n",
    "    if previous_test_df is not None:\n",
    "        previous_test_df[~previous_test_df.content_type_id].groupby(\n",
    "            \"user_id\"\n",
    "        ).answered_correctly.apply(inference_dataset.update_answered_correctly)\n",
    "    # add current to cache\n",
    "    current_test = current_test.copy()\n",
    "    previous_test_df = current_test.copy()\n",
    "\n",
    "    current_test.drop(columns=[\"answered_correctly\"], inplace=True)\n",
    "    current_test =  current_test[~current_test.content_type_id].copy()\n",
    "\n",
    "    # preprocessing code heree\n",
    "    current_test = preprocess_df(current_test)\n",
    "    current_test[\n",
    "        [\"row_id\", \"user_id\", \"content_id\", \"timestamp\", \"answered_correctly\", \"prior_question_elapsed_time\"]\n",
    "    ].groupby(\"user_id\").apply(\n",
    "        lambda user_rows: inference_dataset.update_user_rows(user_rows)\n",
    "    )\n",
    "\n",
    "    # your prediction code here\n",
    "    current_test = predict_for_df(current_test)\n",
    "    predictions.append(current_test[\"answered_correctly\"].values)\n",
    "    row_ids.append(current_test[\"row_id\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation auc: 0.7889184570559918\n"
     ]
    }
   ],
   "source": [
    "all_preds = np.concatenate(predictions)\n",
    "all_row_ids = np.concatenate(row_ids)\n",
    "df = pd.DataFrame({\"predictions\": all_preds, \"row_id\": all_row_ids})\n",
    "df = df.merge(valid[[\"row_id\", \"user_id\", \"answered_correctly\"]], on=\"row_id\", how=\"left\")\n",
    "print('validation auc:',roc_auc_score(df.answered_correctly.values, df.predictions.values))\n",
    "\n",
    "#validation auc: 0.7704724230724678"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAy0AAAINCAYAAAAzy5CEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABuO0lEQVR4nO3deXxU9b3/8ffMZIUQAlnZQhCiEBVxxaW44FawWNe6XUV7VQpUba97taL1XpdScUXkilzrUnGprVC0y0+pO6KtlmoUAxJZzB4SCCRkmfP7YzLDTHImySQzc87MvJ6PBw+SmTMz3/Odk5nzOZ/v9/N1NDQ0GAIAAAAAm3Ja3QAAAAAA6AlBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAALZmadDy/vvv68ILL9SkSZOUlZWl559/vtfHfPHFF5o5c6YKCgo0adIk3X///TIMlpoBAAAA4pWlQcvu3btVUlKi++67T+np6b1uv3PnTp199tnKy8vTW2+9pfvuu0+PPvqoHnvssSi0FgAAAIAVkqx88dNOO02nnXaaJGnevHm9bv/yyy+rublZS5YsUXp6ukpKSvT111/r8ccf109/+lM5HI5INxkAAABAlMXUnJZ169bpmGOOCcjKnHzyyaqoqNC3335rYcsAAAAAREpMBS3V1dXKzc0NuM37e3V1tRVNAgAAABBhMRW0AAAAAEg8MRW05OXlqaamJuA27+95eXlWNAlBlJWVWd2EhEXfW4e+txb9bx363jr0vXXo++iKqaDlqKOO0ocffqiWlhbfbWvWrNGIESM0duxYC1sGAAAAIFIsDVqampq0fv16rV+/Xm63W9u2bdP69eu1detWSdJdd92lM88807f9eeedp/T0dM2bN0+lpaVauXKlHnroIc2bN4/KYQAAAECcsjRo+fTTT3X88cfr+OOPV3Nzs+69914df/zxuueeeyRJlZWV2rx5s2/7oUOH6g9/+IMqKip00kkn6cYbb9T8+fP105/+1KpdAAAAABBhlq7TMm3aNDU0NAS9f8mSJd1uO/DAA/XGG29EsFUAAAAA7CSm5rQAAAAASDwELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAALZG0AIAAADA1ghaAAAAANgaQQsAAAAAWyNoAQAAAGBrBC0AAAAAbI2gBQAAAICtEbQAAAAAsDWCFgAAAAC2RtACAAAAwNYIWgAAAADYGkELAAAAAFsjaAEAAABgawQtAAAAAGyNoAUAAACArRG0AAAAALA1ghYAAAAAtkbQAgAAAMDWCFoAAAAA2BpBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAALZG0AIAAAAgbFaVN2vm6zVaVd4ctuckaAEAAAAQNktKm7RpZ7ueKG0K23MStAAAAMSRSFzlBkIxtyRDEzKT9JOSjLA9Z1LYngkAAACW87/KPaso3ermIAHNKkoP+7FHpgUAACCOROIqN2A1Mi0AAABxJBJXuQGrkWkBAAAAYGsELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAALZG0AIAAADA1ghaAAAAANgaQQsAAAAAWyNoAQAAAGBrBC0AAAAAbI2gBQCABLaqvFkzX6/RqvJmq5sCAEERtAAAkMCWlDZp0852PVHaZHVTACAoghYAABLY3JIMTchM0k9KMqxuCgAElWR1AwAAgHVmFaVrVlG61c0AgB6RaQEAAABgawQtAAAAQAKJxQIcBC0AAABAAonFAhwELQAAAEACicUCHEzEBwAAABJILBbgINMCAAAAwNYIWgAAAADYGkELAABxLhYrBQGAP4IWAADiXCxWCgIAfwQtAADEuVisFGR3ZK+A6KJ6GAAAcS4WKwXZnX/2ir4FIo9MCwAAQIjIXgHRRaYFAAAgRGSvgOgi0wIAAADA1ghaAACWYCIzAKCvCFoAAJagDC8AoK8IWgAAlmAiMwCgr5iIDwCwBBOZAQB9RaYFAAAAgK0RtAAAAACwNYIWAACABEHVPsQqghYAAIAEYZeqffEePFm9f1a/fiQQtAAAACQIu1Tts0vwFClW75/Vrx8JBC0AAAAJYlZRulbPzLW8cp9dgqdIsXr/rH79SKDkMQAAAKIq3kueW71/Vr9+JJBpAQAAAGBrBC0AAAAAbI2gBQAAAICtEbQAAAAAsDWCFgAAAAC2RtACAAAiIh4XuANgDYIWAAAQEfG4wB0AaxC0AACAiIjHBe6sFOuZq57aH+v7Fk70hTmCFgAAEBF2WX09XsR65qqn9sf6voUTfWHO8qBl2bJlmjx5svLz83XCCSfogw8+6HH7J598UkcddZQKCgp0xBFH6IUXXohSSwEAGLiBXkXlKmziivXMVU/tj/V9Cyf6wlySlS/+6quv6pZbbtEDDzygo48+WsuWLdP555+vtWvXasyYMd22f+qpp3TnnXfq4Ycf1hFHHKF//OMfuu6665SVlaUZM2ZYsAcAAITG/ypqfzIQA308YtesovSYfs97an+s71s40RfmLM20LF68WBdffLFmz56tAw44QAsXLlR+fr6WL19uuv2LL76oyy67TOedd56Kiop07rnnavbs2Xr44Yej3HIAQFdkAPpmoFdRuQoLIBFZlmlpbW3VZ599pmuuuSbg9unTp+ujjz4yfczevXuVlpYWcFt6err+8Y9/qK2tTcnJyaaPKysrC0+jERL63Tr0vXUSue8fXJ+qLc0OPfiPZk1s22tJG2Kh/ydKeqhYUluD+tPcgT4+UmKh7+MVfW8d+j68iouLg95nWdBSV1enjo4O5ebmBtyem5ur6upq08ecfPLJevbZZzVr1iwdeuih+uyzz/TMM8+ora1NdXV1KigoMH1cTx2AyCgrK6PfLULfWyfR+/7nyc16orRJPynJULEFQxsSvf+tRN9bh763Dn0fXZbOaQnVjTfeqKqqKp122mkyDEN5eXm66KKL9PDDD8vptLymAAAkNMZhx55V5c1aUtqkuSUZvHcAbM2yM/3s7Gy5XC7V1NQE3F5TU6O8vDzTx6Snp2vx4sWqqKjQ+vXr9fnnn6uwsFBDhgxRTk5ONJoNAEDcoLQqgFhhWdCSkpKiKVOmaM2aNQG3r1mzRlOnTu3xscnJyRo1apRcLpd+//vf6/TTTyfTAiDuMdE9dtn1vWNSP4BYYenwsPnz52vOnDk6/PDDNXXqVC1fvlyVlZW64oorJElz5syRJC1dulSStHHjRn3yySc68sgj1dDQoMWLF+vLL7/UkiVLLNsHAIgWSt3GLru+dwzpAxArLA1azjnnHNXX12vhwoWqqqrSpEmT9NJLL6mwsFCStG3btoDtOzo6tHjxYm3cuFHJycn63ve+p7/+9a8aO3asFc0HgKiaW5Lhm+iO2MJ7BwADY/lE/CuvvFJXXnml6X2rV68O+P2AAw7Qu+++G41mAYDtcFU8dvHeAcDAMBEEAAAAgK0RtAAAAMQwuxZ6sEoi90c87ztBCwAAQAyjdHWgRO6PeN53ghYAAAYonq9uwv4oXR0okfsjnvfd8on4AADEOruWNA6HVeXNWlLapLklGXG3b/GCQg+BErk/4nnfybQAADBA8Xx1M56HmwCIHWRaAAAYoHi+uskaMwDsgKAFAPqIYTJIRPEckAGIHQwPA4A+YpgMEgFFBQDYEUELAPRRPM9bgH1FO4ggOAdgRwQtANBHs4rStXpmLkNl4pzdMg3RDiIIztFfdvvbiYRE2Ee7ImgBAMCP3TINkQgiejrxIjhHf9ntbycSEmEf7YqgBQAAP3bLNEQiiODEC5Fgt7+dSEiEfbQrqocBAOAnEaplUcYYkZAIfzv++1hWZnFjEgyZFgAAEky0h4B5h6MtWNfAfADYjt3nqdi9fdFC0AIAACLKOxzt/77ew7A02I7dh0vavX3RQtACAAAiyjsP4Ir9BzEfALZj93kqdm9ftDCnBQAAaFV5s5aUNmluSUbYh40lwlwHxC67H592b1+0kGkBAAAMQQFgawQtAACAISgAbI3hYQAAgCEoAGyNTAsAAOgVZVetQ98DBC0AAKAPmPNiHfoeIGgBAAB9wJwX69D3AHNaAABAHzDnxTr0PUCmBQAQYYzHBwAMFEELACCiGI8PABgoghYAQEQxHh8AMFDMaQEARBTj8QEAA0WmBQAQc5gnAwCJhaAFAGyAk/DQME/GI9hxw/EEIN4QtABAFAU7meQkPDTMk/EIdtxwPAGIBaFcYCFoAYAoCnYyyUl4aGYVpWv1zNyEnysT7LjheAIQC0K5wMJEfCABrCpv1pLSJs0tyUj4kzyrzS3J0BOlTd1OJpmsjv4IdtxwPAGIBcG+E80QtAAJwP9KBicy1uJkEgAAj1C+ExkeBiQAhoogXjHhHAASA5kWIAFwdR/xyj+L+FCx1a0BAEQKmRYAQMwiiwgAiYGgBQAQs6JVRcwOw9Ds0AYAsApBCwAAvbDDuid2aAMAWIWgBQCAXthhGJod2gAAVmEiPgCEkXdNnCNzkvVxbRtr48QJOxSzsEMbAMAqZFoAIIy8Q3j+7+s9DOUBACBMCFoAIIy8Q3iu2H8QQ3kAAAgThocBQBgxhAewlneIJkMzgfhCpgUAEDMo+4veUGUNiE8ELQASDie+wdm9bzghRW+osgbEJ4IWAAmHE9/g+tM30Qx0OCFFb6K14CgQ6+x+kaorghYACYcT3+D60zfRDAI5IQWA8Ii1C3gELQDCKhau3HDiG1x/+iYeg8BYOI4BYCBi7bOb6mEAwsr/yg1BQWKIx4ppHMcA4l2sfXaTaQEQVrF25QYww3EMAPZCpgVAj0Jd8yDWrtwAZjiOAcBeyLQACNB1LH+sTdQDEB3M+0lcvPfhEY/9GMl9ImgBEKBrkBLrw2Ti8UsBsAMuaCQu3vvwiMd+jOQ+EbQACNA1SIn1Slvx+KWA+BRrAXasX9BA//Heh0c89mMk94k5LQACxNtY/rklGXqitCmuvhQQn2KtYpn/Z0VZmcWNQVTF2/eEVeKxHyO5T2RaEBNi7QpkOLDP/d/GX6xnitA3q8qbNWd9akz/vcTjVVcACBeCFsSERBziwz73fxskniWlTdrS7Ijp44IAGwCCI2hBTEjEK5Dsc/+3QWTYOfs3tyRDhekGxwUAxKmECFocFRVy1NVJO3dKzc1SR4fVTUKIEvEKZNd9tvMJY7j05X32biMpLvojlt7Xvma5etunSOzzrKJ0LZ28N6E+I2CdWPq7BeJFYgQtu3fLsWOHnNXVcm7fLufmzXJu3izHtm1y1NRIDQ0EM4i6UL/0Ym1YVKS/1GOtP4Kx636YvX99zXL1tk923WegrziGgehLiKDFVEeHHC0tcjQ2yllb6wtmHOXlcmzfLkdtrdTYKLW0SG631a1FHAr1Sy/WhkVF+ks91vojmJ72w8qruWbvX18znr29N/Hy3iFxcQwD0UfJ4y4c7e1Se7vU3CyH3+1GUpKUkiKlpspITvb9LIcj6HMBPQm1FG+slUaMdKnhWOuPYHraDytL4A7k/Qu2T6vKm7WktElzSzJ8Q/yAWBQvnz9ALCFo6SNfMLNnj3kwk5LiCWZSUz2/OxM3iYW+ifcvvXjfv2gIZ+DnDRiOzEnWx7VtmluS0evcoXC/f7G2Dgnin38gzTEJ2Btn1gPkaG+XY88eORoa5KypkXPbNjm/+cYzzOy77zzDzHbuZJgZkKAGMsQrnAUovAHD/329x7Kx+PEwpMbuE7Dt3j67YW4KEDsIWiIkIJipru4ezHirme3dKxmG1c0FECF2OSnyBgxX7D/IssDBLAiLtZNsu7yfwdi9fXYTD4E0kCgYHhZlQYeZeefJeIeZdf7MMLP4wBCExBXpuT19ZdfherE2ZMwu72cwdm+f3fT174LPcMB6BC024Whrk9rapN27u8+Z6QxiAoKZJN66WBJrJ2YIH7sGC3YRayfZdn8/7d6+WMVnOGA9znxtLlg1Mzmd3YsAeAMaKprZTqydmAF9EY6rz3Y/yeYKOyQ+wwE7IGiJVW63HK2tUmurJJkPNfOWZ/ZWNINl7H5ihvgQ7RPsRLj6nAj7iN7xGQ5YLyEmTLj+8Q85tm/3DL9KAI62Njl275ajvl7Oqio5t2yRc9MmObZskaOyUo4dO6SmJl/AAyA+RHsSdiJMYk6EfQSAWJAQmZZBc+ZIkgyHQ0ZuroyCArnz8z3/jxghd0GB72dlZMTn8CrD2JeZaWral5lxOIIXAYhiPzAEA/EuGsd4tIewJMLV50TYRwCIBQkRtHRMnixHVZUcNTVyVldL1dVyBdnWGDzYE9B0CWaMggLP7zk58TUJ3j+YkSxbODPWhmAQZCFUkTzGWWkeABDv4ujsO7g9y5d7fmhvl6O6Ws7KSjkqKjz/V1bKWVXl+b2iQo7du+X65hvpm29Mn8twuWTk5gYGNPn5nt87Ax0NHhzFvYucoOWZvRXNvHNmvJkZV7BQsHexNskx1oIsWC+SxzjHIwAg3iVE0OKTlCRj5Eh1jBxpfr9hSI2NclZW7gtouv5fW+v7PRhjyJDAYKbLMDQjO3tAJ/hWC1rRzOXyBTGuxkbPvJmUFE+A08tQMzsMwQglexJrQRasF8ljnOMxNoSaoSWjCytw3MGuEito6Y3DIWVlyZ2VJffEiebbtLbuy9Z4gxn/rE1lpRy7dsm1a5f09demT2G4XIHzavyyNN7flR6DHxQdHXJ0dEgtLUpqaNgX2DkcnuxM5/Ayw5uZsVlFs1CuVtshyAK8OB5jQ6gZMTJosALHHeyKoCVUKSkyRo9Wx+jR5vcbhhwNDcEDmooKOXfskGP7djm3bw/6Mu6hQ7vPp/H73xg+PCJzSyLCMPYtnil1LwLgDWb8h5pZIJ6uVnOlDLCfUD9j4ukzKZ7F2+ctxx3siqAl3BwOGcOGyRg2TO5Jk8y3aWmRo6oqIJjx/VxR4bmvsVFqbJRrwwbTpzBSUgKHn5kENkpNjeCOhkGwIgAWVTTzXq1eVd6sma/XxPQXkBVXyuLtiztR8L5FT6gZsUhk0Hi/wy/eMhNkbmFXBC1WSEuTMXasOsaONb/f7Zajvr77nJqKin1FAxob5di6Vc6tW4O+jHv48H3D0EyqoRlZWf0KAt7a3qIXNu7RRRMGafqotJAf36ueKpoFC2bCmHWKhy8gK66UxUO/JSLet8TC+x1+ZCaA6CBosSOnU0ZOjoycHLkPOsh8m+bm7gGNN0tTVeW5vb5eqq+Xq7TU9CmM1NTA7EyXeTVGfr5nEn0XL2zcoy1N7VqxcU9kgpYerCnfFRAweQOoCyYO1cnjhuyraOYtz9yPoCwevoCsuFIWD/2WiOL5fWPie3fx/H5bhcwEEB0ELbEqPV3uceOkcePUYXZ/R4ccdXW+IWem1dB27ZLr22+lb781fQnD4ZCRnR0Y1OTn6+cp2fpDS5aOHz/WU3EtiotQdg2YvL+/9FWjTsl3mZdn9mZm/H7uaa0dvoD6h36zp95OxHt732L5RJ6J793xdwogVlketCxbtkyPPPKIqqqqNHHiRN1777069thjg27/8ssv6+GHH9amTZs0ZMgQnXjiibr77ruVn58fxVbHAJdLRl6ejLw8uSdPNt+mqWlfVsabpfEvHlBTI2dtrVRbK9fnn/seNrXznyQZ6ekBAY13+NkgSY6UFBl5eWFdjPOiCYO0YuMeXThhkOnvXQVba8aqeTOIL5E6oQ/n8w70RDyWT+RjZeJ7JAPDWA46AcCfo6GhwbDqxV999VVdffXVeuCBB3T00Udr2bJl+t3vfqe1a9dqzJgx3bZfu3atZs6cqbvvvltnnHGGampqdP311ysrK0srV64M+jrOjRsjuRvxq71djtra4FXQKivl2LOnx6cwnE4ZubmeogEm82rcBQVShr2GKQRkZ/q4eKadTgzKyspUXFxsaRu6slP/hNPM12u0aWe7JmQmafXM3LD1fdfnHYhV5c2+E/H+9P1AHx9Nwfrf7sdfON/vaD63Pzt+7iQK+t469H10WZppWbx4sS6++GLNnj1bkrRw4UK9+eabWr58uRYsWNBt+48//lgjR47U/PnzJUlFRUW6+uqrdfPNN/f8Qg6HZxhTnIj4RHivpCQZBQXqKCiQpkzpfr9heLI1FRWBQ88qKtT67bdKb2jwZGuqqqSqKrnWrzd9GWPw4KClnd0FBTJyc6O6GGfQ7IzTuS+ISU72/Nz5L5avRkdDqP1j95NMr0hdmQ/n8w50OFA8DCeyezW9SGZ4mMMCIF5YlmlpbW3ViBEj9NRTT+mss87y3X7DDTeotLRUr7/+erfHrFu3TmeccYaeeeYZff/731d9fb2uuuoqZWZm6umnnw76WmVlZZ4TbLdbDsPwzPfo/N13m9vtqVrVdTvvNp3/28EdG1L0XYtDI9MM/eqAVqub07P2diXX1SmltlbJnf9SamoCfnbu3dvjUxhOp9qys9WWk6PW3Fy15eR0+9lt8WKcaxuT9Me6dJ0xSjo2zykjKUnupCRP1iaMw+PCZU2tSyu+S9KFI9t1Uo7prChLX2/O+lRtaXaoMN3Q0sk9Hx+IT+E8RqN9vEscwwDQHz1lriw7m6qrq1NHR4dycwPT1bm5uaqurjZ9zFFHHaWnnnpKV199tZqbm9Xe3q6TTjpJS5Ys6fG1wpq66+jwBC8dHfv+eYMa/9u8t3d0hD3Lc3lyi28eR1GUq3f1VXl5uYqKijy/TJhguk2bpDbDkKOxsXuRAO88m8pKOevqlFJTo5SaGg3+8kvT5zIyM03XqvHOtzGys31lkSORqSqSdGGwO53OfUPOkpMDfo5EQNOXdPV1ZTXa3tau13ak6+pjIjdkxKu4WLo6hO1/nrxvSFJxjFzlX1XerAf/UaOfH54b85kJO+jPMRrs2A/1+AuHWDyGB4JhMtah761D30eX/S4B9+Crr77SzTffrBtvvFHTp09XVVWVfvnLX+pnP/uZli5dGp1GuFyef11KAQcLSwzJE7SYBTTeQMfk/54yO9NHpUW91HDEOBwysrJkZGXJPXGi+TatrXJUVwcOQ+ucU+OsrJSjqkqOnTvl2rlTrq+/Nn0KIynJtxhnlmOYTh+Uo6r8EXKdMH7f3Jq0CPWp2x103RlfQNM57CzSAY2X3YeMxOKQpCWlTdrS7GCIYJjY/RjtTSwew7BGrAyHBaxmWdCSnZ0tl8ulmpqagNtramqUl5dn+phFixbpsMMO07XXXitJOuiggzRo0CDNmDFDd9xxh0aNGhXxdveLw+E5Ae1yEtpT/sV3nzeY6RrYGIYni2Nyny/g8cvwRG0eTCSkpMgYPVodo0eb328Yeq+0Uu9++q3OSmvU5L21+7I13sU4d+yQY/t2Obdv1wmSTvA+9uV9T+POyupW3jlgMc7hw8NfWcw/oNm9u3uFM29A451D4xfgDKQtnFCF39ySDD34j+aYPcm2G45RJArmRAJ9Y1nQkpKSoilTpmjNmjUBc1rWrFmjM8880/Qxzc3NcnWZkO393W2T+Sa9CfmKijezY6LH7I4UMIzt4X9X61tXmhqqHTrpoKxumR9f1ieIUIKet7a36OkNKbo8uSU6AZLDod/WpmrLsP30abJDQ1NduujELu1safGVdPaVd+78v2Xrd0qtqVZyQ4PU0CDXV1+ZvoyRkuLL1niDm4BqaPn5nkUtw8Uw5Ghrk9raPLvZ9W7/IWf+Vc5giVlF6ZrYtjchhgIBCJ9YzyoC0WLp8LD58+drzpw5OvzwwzV16lQtX75clZWVuuKKKyRJc+bMkSTf0K/vf//7uu666/TUU0/p5JNPVmVlpW699VYdcsghpiWS7ai3KyphTRM7nZ5/ycn68aG5eqK0Sf9RkiEju/vzdhvG1t7uCWY6f3/qoyptbU3WM+XtOmlsUo9zdV7YuEfftTh8C0BGg3e9loa97oDFJ33S0mSMHauOsWO7LcZ51dv12rqzVQerSQ+MazEt8eysrPTMvdm6Vc6tW4O2wz18uOmcGu/vRlZW2LI1vipnCgxoUrdskcN/iJl3Ho232lkUK7EhdAwVQSJLxOOfrCLQN5YGLeecc47q6+u1cOFCVVVVadKkSXrppZdUWFgoSdq2bVvA9pdccomampr05JNP6vbbb1dmZqaOP/543XnnnRa0vn96u6LiH9R4fw/Hh3efPhT9h7F1Zgy8YcmPjhusJ0qbdHZJhoyidM/tQYoPnHtYmv730xqdd1CWjLTkoJmccA5Z887zeWt7S4+LTZq9/sHDkuSQNGPCaLlHpcl90EHmD9izZ99wsy7ZGu/cGmd9vVRfL1dpqelTGKmpQQMad0GBJ1vTZb5UyLpkaCSZDzvzBjL+Vc68gU4IgVUinmREEkNFkMg4/gEEY+nikujOfyE374d3pBcFCxf/k9eJbdvMK2r4BTjnvlGlbxtbVTxIWjF9WGCGp5fhauFw1dv12tLUrrEZSfrfE4YP/Ak7OuSorfVVPvMFMv6Zm6amHp/CcDhkZGd3C2b8f1ZmZregwj8A3K+tcl/ltn4ykpL2FZxITpbh/dkb3Phla6K1eF0sCEclmVhazNFu+tv/BN4DF64qShz/fdOX71uO68gzO+6j1e+J+P7GVPWwWDDQg6hrRiSWxrn6XyF7KNh3l98cncsPy9cTpU26uCRDRm5gX3mHq/1pU5Oe/KJRc/ZP18xRyfvKSPsFOAHFB0LgHVLWl6xMn7hcMvLzPfNeDjnEfJumpuClnSsrPYtx1tZKtbVy/fvfpk9hDBq0L4jJz5cxYoS2NmRoeGq2/l/9SF198AAzNfIbeta5hk63vIvT6QlkkpL087x2/d/OFl02JkvavXtfxqazxDRCw1CR6OPqvn3E8vEfzZPIvnzfclxbI1r9nojvb0JnWsL1AeP/PLGWHQkn/ytkQTMtIQrpKr53To5fMBPOICcq2ts9gYtZQOP9f8+eHp/CcDpl5OZ2X7Omswqau6BAyohCIOwX2PiGoflnbJKTw1+NzWLU7LfWQDIt/b26n4hXO82E+9iPxX6NZta5L9+3ZK0iL1imJRr9nojvb0IHLX39gOntw3Pm6zX6or5Vhhy6Yv9B+qS2LaEOIjMxMVTAMALn5XRdJydKi4WG3OadO+WsqtLjfyuTs6pS+++p1feTG3zD0Bx1dXL00kYjI8N8IU5vcJOTE50J+y7Xvvk0ncPOfIGO//8xEtwQtFjLiv5neKRHuPs+FvvVqpNIPnesQ99HV0IPD+ttUrw3WKlvcWtHqztoCm5uSYbmv7dDTof0SW1bzHzAxoJwDBUIGnQ6HEFLSvdYTto/oOlSZe1v5bv13Fc7ddl+aTplRFL4szkOh95qStULFTk6eFqBPt/RrgkTBqnFr5DBt2VlKkpP7z4MrcvcGtfGjdLGjeb76XLJyMvrOVszKAzD6rx91zkMTTIZiqYuc2zMCgdQEQ0WoVxtZMRiv8by0DYgFiR00NLbB4x3qNewFIcmZCYF/fD0PkesfcAmCu/7+D//bAzPcAO/UtJe3iBn0cfSpuRU1TQmafpxuaZD1gKGq/kHPX0McF7YuEdbmtrlkEwLCBjJyb7FOH0T9Gf4VWgzDE/55mABTWWlnHV1vippwRiZmd2KBAQsxpmdHbZ5Lb3OsXE4PBka/3/e4Wh+vyspyTZzbew2/MVu7bE7//7iQlX4EQAA6Cqhg5be+F/pmVWUrlXlzZr5eo3pl3qsf8DG8wmL932sa3FHfNJa16uDq75t8evXwZLMszi+dXK8J+ddMjj+Qc9FEwZrxcbdfSog4A1wAtatcThkZGXJyMqSe+JE8we2tu6rfNYlqPHe5ti5U66dO+X6+mvTpzCSkgIX4/TL0nhvU1qY1vExjIB1a7xMB5X5Bzh+Q9MMpzOqw9LsNonSbu2xO/oLAxXP37tAJBC09KBrIBIrX1L9+SCMlX3rD+/76D/eONKv5RVSvzoc+xaA7GQW4JxYJJ34PU8g4+4MaBx+gY07NVVGkmcB0H5XSEtJkTFmjDqCLdpqGLp+9Tfq+K5CB+2t09xhuzxr2PgHOA0NcmzfLuf27UFfxj1sWPB5NQUFMoYNC1vg0G1dIL/MjRQkwHG5PMGM0xmYxXE6Pbd3CXxCYbfhL3Zrj93RX9ERqRN7OwQM8fy9C0QCQUsI+vMlZcUHY18+CLu2KxG+gK3IhkWsX7vMxfEPbtp27ZJRVCRD0olj23XiVE8Gwt3e7slGdC486Whv7/+cG4dDZxw6SiuGDNN+EwapzWxx0JYWXxBjOgytqkrOHTukHTvk+vJL05cxUlN92RqzjI2Rny+lpPSpyaZZp954s11mXWByW+rWrXKkpOzL1HiHqXUNelwu22Vn7dYeu6O/oiOUE/tQvm/tEDAkwvcuEE4ELer7B11/vqSs+GDsywdh13Z13Tc7XIWKB5af2HTJAPgHN76iAp1BjNrb5fD+7A1qeqhCNn1UWs8n/2lpMoqK1FFUJNPTfrfbU+msosKTpfGfU+MNcHbulGPLFjm3bAn6Mu7s7H1BjMminBo6VHI4wr8uT7B98vahn2C5IsM7x8ZvWFq3uTnefxZXUOMzAVYI5cQ+lO9bOwQMln8/ADEmYYKWrl+4ZmurRCKwsOKDsS8fhL21Kxx90p+TnL4+JtonUNF8vYG81qryZj24PlU/T27u22OdTik11fNPJkFN18xMZ3CjtraBl3/uXE/GyM1V0HzP7t37sjL+a9Z459tUVclZVyfV1cn1xRemT2GkpcldUKAzCgo0Y8QIGWX5gQFOXl7AkLxocnjn4LS27rst2MbeIWn+w9I6AxqzTI6czrAGOna4Mo3EE8qJfSjftwQMQOxJiKBl5us13coW+38Bd5s83ctJYygnlVZ+MHrbeWROsj6ubQtob2/tCuXDP1h/9Ockp6+PifYJVDRfbyCvtaS0SVuaHeFrpzdTk57ebX5Nt6Cmo2Nf1sYb5AzU4MFyjx8vjR9vnq3p6JCjtnZfdqaiIrCAQEWFp7xzeblUXm76EobDISMnxxPE5OcHFgvoDGw0ZIjlmQ7f+kEmeg10vOW9uwQ3AcUH/IMdE3a4Mt1XZIUSE4EIEN8SImgxK1vctTJYKCfbsXLF0dvOz+vblJ4U2olsKB/+wfqjPyc5fX1MtE+govl6A3mtuSUZevAfzX1+7IBP7noKarzV0Lpmarw/B5krEhKXS0Z+vmfeyyGHmG/T1NR9zRr/rE1NjZw1NVJNjYKt9mIMGhRYLMCbpfH+n5MT8kT8qPAPdLoMWZOCV1cLKDTQ+f+Zg50682iXDOdeaVe77Yav+YuVz2gAQN85GhoaLFzeOzrOeL2mTyvU+mcmelrVPpyr3kbyiqC3nUf0sj/heh3/52eV2OgwO35C6XtLV512u7sHNd7fOzrCk6kxYVZFzFFTYx7QeP/fs6fH5zQ6h7o1DxumlKKi7tXQCgqkDPtnKAaka0W1rnN0/DM6IayVE8pnpPfYt2pl8kQW6mc+2bDwSdTvWzscQ4na91ZJiKClryJ5AhfsjyuU1+xpXo7Z71aK1z/kcPRxON8ns+MnlL639cmdWaamtXXAlc+uerteW5raNTYjyXRxTtN27NplulaNb82a2lo5epnfY2RkBFY+67JmjZGTE1ARLq5518oxKyXtvd2b4flbvcqaDO03NEWrz8jr8Wnj9XMnFoTa95ZeMIkziXrc2+EYStS+t4oNxzNYJ5JDgMIxhKrrc/T2O8IvHH0czvdpoMesrceAd1m3pluRgI6OwCDGvwpaD1makKuIORxSZqbcmZnS/vubz61pbZWjulrVn32mEdK+uTX+WZumJrnKyqSyMtOXMTqHunlLOZtVQ9OgCFY+iybvYqAmug4y+6+hLVpRs0cXjhok5zdNwdfNcbnkbGqSdu/u0xwdWCuW5kjBnjiGEg9Bi59InsAF++Pqy2v6D1tzSKbzcnp6DbuxU0YoVOHo43C+T7YOOiLN5ZLS04PPp/EvCuD38/TRjr6v09JXKSkyRo/W7vZ2tRcVdb/fMORobOy+Vo1/8YC6Ojm++07O774L+jLG0KFy5+d3z9h4F+McPtxWJ+ndhuL1Q0Bp7V6KESTX1clZUdHljs6sTudcnYCCBN7MjlmhAm9whIhI6M8uhAXHUOJJuKAlWifMXV+ntz+untrlvTLvkAJSoF2fsz9/wHZd/NKuwvEhaccP2lgOJE05HJ5FJzsXnuyWpekS0Pxt006tKN2h/9gvNfwBTWd7jKwsGVlZck+caL7N3r37Kp/5D0Pzq4bmaGyUq7FRrq+/Nn0KIynJk6XpYRia0iKwf0H0a0HPcOshqyP1UHlNCgx4/IMZ/+prJoFQwgzzg6m4+zwFbCLhgpZwnjD3JdDo6+v0tH04r8x3bXOo7QzHh3GsZISixQ5fcLEcSPZLl2Fni9YZ2pSerspdTp04eqhnuFdnMPP8l42aXZSs6aNSI9um1FQZhYXqKCw0v9/tlmPHjsAsTVVVYPGAhgY5tm+Xc/v2oC/jHjasW0ATkK0ZNixslcCisqBnJPkHPH2tvublcsnomt3xz/B4f/f/57+djaqxITQJ93kKREnCBS3hPGEOZ6DR0/bhvDLftc2htrM/H8ahZp0Gyg5BQCi8ffo//2y0rN1zSzL0P/9sVF2LW6vK+7gwZRzx/h3MKcnwZCLS0jzBzEdubUpNU/XuJJ1YNGzfHJq2tsD5NANZZLOvnE4Z2dkysrPlPugg821aWgKLBPgHON7/d+yQduyQ68svTZ/CSE31zanpGtB459t4M1i9CRjalWg6OnoManoNScyKFXT+3LVYQUCGh2DHclyYAyKD6mEDYOvKS0EMtM19fbx/RY1oV/jwvt6wFIeGp7lsH7x4+7SucwHUgfZTf6uZ2KESi930+e+lc6jZN199pfGFhZ7fW1ujF9D0VUeHHHV1clRVadmbZXJVVap4T51OdtbvKxywa1evT+POzu6+Zo3fMDQNHWrJyXN5ebmKzOYU2VQ45vx04x/smGRwTOfw+G/XT4lSRcmOF8USpe/tiL6ProTLtIRTJDMGkfpgHGib+/N4qxaCrGtxx0SK3tun/ifIVojE+2THL/hQ9Pl47xxu1jFkiKd0cSdD8pRt9s/QWBnQuFwy8vJk5OWpaHixVmzcozETBqnZ/4R59+7u2Rn/ogE1NXLW1Ul1dXJ98YXpyxjp6YHDz7rOs8nL8w3PS2ThnPPT1wCo11DSbOha16DHLMtjp+A8ghj6BViHoMWmQvlgtPuJYbQnntslCAiV1RP0I/H6fMHLs6hikuejtltBgNZW3/wZtbZ6JuO3tUXlBDDo0K3Bg+UeP14aP968vHN7uxy1teYBjff33bvl2rxZ2rzZ9LUNh0NGbq4niOka0HRmbpSREfdDncI55ydsAVAPFdqk4EFP6pYtcnqHrnnn8pgEPoZ3vo7JPJ9QMz1WfPcx9AuwDkGLTQ1k/RZ4WB0EgC/4XnVWODOtbtbaKkdnUOP72Q5Xs5OSZBQUqKOgIPg2TU2BQUxnUOOthuaoqZGzulqqrlaw01Rj8OCgxQLc3sU4k2L7Kyycc35sUfSgl0pt0gAzPd6KbZ1Bz/JP67W1ydCT69s0a0xKVKq28b0CWCe2P/HjWCgfjJwYwq74gu+nzuFmxuDBvpu6BTPen9vaPAtt2klGhtzFxVJxcfBsTXX1vmIB3rVqqqrUvOU7JVdXKG33brk2bZI2bTJ9CcPlkpGbGxDQDE9KkuvAA32/y6//4l3cFD0IIdNz3fAWrajfowuHDpJz8x7Pjf5Bj8lQNsOsfDVlqoGYQNASB2LlxDCUVL7dh7wBlggWzLjdgUPN/AMaO2RnukpKkjFypDpGjux211Vv12vLrjaVuJr18LiW4OWda2t9mRyvUV2eyxgyxDRb4/3ZyM7mZDWGmQZq/RzeJikwqOkyZ6fbXB7/++N8GCNgFwQtiJpQhrHFypA3givYgtMZUKrZK6AQgHfyv//PNuQd5jRrwlC5R6UFX4yztXVftqYzeNldVqbM3bv3lXretUuuXbuksjLTp/Atxpmf3334mbcSWjp/1wnDW6ba5G+jT4uQ9lSe2nu7//0AQkLQgqgJZRhbrAx5i5XgCgnMWwhg0KDAgMYwzIOZtjZP5sYifR7mlJIiY/RodYwe7btpe3m5kr0ljw1DjsbGwLVqus6zqa/vfTHOoUODBjRGQYGM4cM5AU10PczlCRrsdBYrMEwCmmABEFlBJDqCFvQonJmEUIaxxcqQt1gJroBuHA4pNVVKTe29EEAUq5qFjcMhIytLRlaW3JMmmW/T0uKZS+OXrelWFa2xUWpslGvDBtOnMFJSfItx+q9V4x/YKDU1gjsa2yKyVk0sMAxPZqejw/P35qfXIWxdApqkHTvkqK8PnMvTda0eAmvEgZCClgcffFB//vOf9Ze//MX0/hkzZmjmzJm65pprwtI4WI9MQs9iJbiCOYb3BRFs7ox/NmbvXnvPm+mLtDQZY8eqY+xY8/vdbjnq6wODmKqqfdmbigo5Gxvl2LpVzq1bg76Me/hwX3lns6yNMWxYws6LCOdaNQnBJNBx7dzpOU57epw3s+MtR22yAKmvMlvXeTvebRL0GIV9hBS0vPzyy5o+fXrQ+4888kitWLGCoCWGdT2JI5MQX+L5JL0/+xYsKI/nfhoQsxLNhuE5gfJmY7zBjBWLZ4ab0ykjJ0dGTo7cBx1kvk1zc/ciAV2zNfX1Un29XKWlpk9hpKb6sjX+a9X4Apy8PE/f+4mXDIUtSjUnAm9mp4dNQi5H7RfY/Hl7q576ullXlGRqxrjBVGZDRIQUtJSXl6u4uDjo/ePHj9fy5csH3ChYp+tJXE+ZBE7sYk88Z876s2/BgvJY6Sdb/A06HMGDGf81ZrzDzGxaAKDf0tPlLiqSiorMyzt3dHiugndmZ0wDm1275NiyRc4tW4K+jDsnJ2ABzvpdQzQ2PVsfbh+p6WfsL2VmxuSV8Lgp1ZwIeqjM9uon9drV1K7XdtbpjNTh3Tfwn7/Tde0dbwU2szk8MXhMI3JCClpSU1NV6VdesquKigo5GTfZJ7Y42TDBopbxzW6Zs3D+HfRn37oG5d72HJmTLIdkm34KxtZ/g8HmzJgFM95hZvGocz0ZIzdX7smTzbdpagoMYqqqAoObmho5a2ul2lq5Pv9cknSF/+MXSUZ6evfyzl2zNTG+GCfsq9eMmTfg6W9ltiBBTbAS1WR34lNIn2BHHXWUnnvuOf3kJz/R0KFDA+5raGjQc889p6lTp4a1gfHKricbLGoZ3+w2Byecfwfh2DdvexySVs/MHdBzRUNM/g32FsyYDTOLdxkZck+YIE2YEHwxztrawMDGvypaZaUcu3fLtXmztHmz6UsYDocnePIvFtBlGJoyYug4gq1ELGPmX5nN5MJGjwEP2Z24E1LQcsstt2jGjBk67rjjNHfuXE3qrMhSWlqqJ554QjU1NXr66acj0c64E5MnG13Y7QQYscOuGY1Y+7uMq7/BYMGMd+FM/2CmrS0xghmvpCQZBQXqKCgwv98w9mVrKio8WZrOqmjewMZRUyNndbVUXS3X+vXmTzN4cPdgxv//nByyNYgd4crudF1o1C8AcnReXKEkdXSE9Olz6KGH6sUXX9R1112n22+/XY7OaNQwDBUVFenFF1/UEUccEZGGxpu4OtkAQmTXjAZ/lzYUbOFMt3tfNsY7zCxRTxocDmnIELmHDJGKi4Nna7yLcXbN0njn2uzeLdemTdKmTaYvY3QOdQsYdpafr0+Ts/VCY7pOb8vU8cUm8xmAWBJCdielslLONL8Mk3+xApMKbMEKGVCWum9CvmRywgkn6NNPP9W//vUvbe5MQ48bN06HHHKIL4gBgJ7EWkYDNuR0elarT0/3BTN79+yRe9y4bsGMo7XV0gUzbSEpScbIkeoYOdL8fsOQGhvN16rxBjp1db7f/R3T+U+PSMaQIaZzanzlnbOzEze4RPzroViB1IcKbd55Or0FPSZD3hJhWFu/8rwOh0NTpkzRlClTwtwcAImgrxkNuxasgI25XNKgQdKgQb5gxpA8Q0S8Q8v815hJ9GDGy+GQsrLkzsqSe+JE821aWz3DzvzXqqmsVEP5drVtr1DBzhq5du2Sa9cuqazM9CmMpCRPeWezNWtGjJA7P98TjAKJaCBlqb1r8fSQzekx0xMDQU9IQcv777/fp+2OO+64fjUGQHeJcOIebB/tWrACMSgpyTcfIyCYaWvrXpY5lhfMjKSUFBljxqhjzJiAm9MlVZWXK3PsWDl27Og+7Mx/Uc4dO+TYvl3O7duDvox76FDTRTh92ZrhwxlKA3TlXYunw3SAqKQQ5vKYBTxd5/lYsBZPSEHLD37wgz4NAauvr+93g4B4Fs4FEONJsH30H0aWCMEbLJCcLCUnyxg82HeTIQVfY4ZgJjiHQ8bw4TKGD5e7pMR8m5aWfUUC/AoG+AoIVFfL2dgoNTbKtWGD6VMYKSm+xTgDqqH5/a/U1AjuaGLxX8hUUtQXNY2XhVT7I6r77j+Xx8RAFh+V06k/b9urJ79u1n+aLUDax4sQIQUtq1at6nZbR0eHtmzZot/+9rdyu91asGBBKE8JxKVwZg4SYf5HsH30H0Y28/WauA/eYCPBFszsrGDm6MzQEMyEKC1Nxtix6hg71rxggNvtWYzTL1uz5pPNyqip0uimWhU11cjR2CjH1q1ybt0a9GXcw4d3z9L4z63JyoqJ4TB28MLGPdrS1K4VG/fIkHw/RyuA8H/9RAtaYmrfe5nP8+rH9drd1K7XGmt1RopJwY7O+TxGUVHQ5wgpaPne974X9L5LLrlEM2bM0HvvvacTTjghlKcF4k5fMgd9lQgVrfqyj4kQvMHmHI6egxnvPBn/BTOZMxMap1NGTo6MnBy5DzpIktR6Uose71y4MGdUmrRnj2e4mf/wM/+sTVWVnPX1Un29XKWlpi9jpKYGDWjcBQUy8vM9WTh0Wziyx0Uko/D6iSSe9r3Xfemcz9PT5R9HQ0ND2C4PLV26VA8++KC++uqrcD0lYlRZWZmKi4utboZlVpU3+06wox1wJHrfW8kufZ+oQ+ns0v8BOjr2BTHt7YE/x1F2pry8XEU9XCGNqo4OOerqfMUCTMs7NzX1+BSGwyEjOztwbo1f8QB3QYGUmWmLbI2t+j7B0Pfh554wIeh9YV0laseOHWpsbAznUwIxKRGyI7CvRJgHFTNcrm6lmaXAIgBqa9uXoYmThTMtnYfgcsnIy5ORlyf3IYeYb+NdjNOstLN3Mc7aWqm2Vq7PPzd9CmPQoH3Zmfz87tmavDwW4wTCKKS/pq1Bxo82Njbqgw8+0KOPPqpjjjkmLA0DrJaoV6sR+xhKFyM6iwBI6j7crGshAO9wsxhh+7H4GRmeK7oTJgRfjLO2Vr95Y4McFRU6oKVO56Q1BgY4e/bI9c030jffmL6E4XTKyM31FA3oGtB4szUZ/I0CfRVS0DJ58uSg1cMMw9CRRx6pBx98MCwNA6zG1WrEKjJ9Mc7h8FS/Sk3tPZjxFgKwmZgfi5+UJKOgQIeelqUVG/doyoRB2usffBmGtGvXvrk0fuvWeP931NbKWVUlVVXJtX696csYgwcHLe3sLiiQkZvLYpxAp5CClscee6xb0OJwOJSVlaVx48ZpYrAFqYAYxNVqALbSWzDjrWrmXTzTwmBm+qg0e2ZYQhR0PxwOKTNT7sxM6YADzLM1bW37igN4gxn/3ysq5Ni9W66NG6WNG01f3+gc6mYa0HizNUCCCCloueSSSyLVDsB2uFodiOFy9sF7Ed9Cfn+DBTNud/dgJk7mzMSE5GQZo0erY/Ro8/sNQ2psNC8W4J1f4y0oUFER9GUmZWTIMXJk8PLO2dksxom4wAwxAH0SjuFy8Xiy7b9P0co1M3QxvoXt/XU6pbQ0z9oofjcb3opmfsGMo63NU+kM0eNwSFlZcmdlyR1spEprq2fBTW9pZ79iAd7MTVJTk/T113J9/bXpUxhJSfsW4+wyt8a3GGda7GfFEP9CDlqqq6v17LPP6rPPPtPOnTvl7lKD3uFwaOXKlWFrIAB7CMdwuXg82fbfp4eiVG2XoYvxLeLvr0lFM0OS/Esy+wc1rDVjnZSUXrM1W9ev19jk5MDqZ5WVnrVsKirk3LFDju3b5dy+PejLuLOyAgOaLtXQjOHDbVHeGYktpKCltLRUP/jBD7Rnzx5NmDBBpaWlmjhxohoaGlRRUaFx48Zp1KhRkWorAAuFY7hcrJ5s95QhCtintoaotCcehy7GYxauvyx7f5OSPP8GDQoMZrwLZ/pP/m9ri6t1ZiIhKmWfHQ51DB0qd1GR3CUl5tu0tAQWC+j8OWCeTUOD1NAg15dfmj6FkZISsPBmQDW0ESM85Z1TUyOzj0CnkIKWu+66S2lpaVqzZo0yMjI0YcIE3XvvvTrhhBP0yiuv6KabbtLy5csj1VYAMS5WT7Z7yhD571NZmRWtiw/xmIWLG52lmY3Bg303+YKZzgDGXVMjIyWFYMaPbco+p6XJGDtWHWPHmhcMcLvlqK/vvgBn5+/Oyko5Ghvl2LJFzi1bgr6M27sYZ7C5NUOHkq3BgIQUtKxdu1bz58/X2LFjtWPHDkmdVUsknXfeeVq7dq1++ctfatWqVeFvKWABrv7aXzTeo1jNEMUS+jgGeYMZSW05OTIKCz3nBP7zZLyVzGJojRkpPFmSmCn77HTKyMmRkZMj90EHmW+zZ8++4WZ+ZZ591dCqquSsq5Pq6uT64gvTpzDS0syroPlnazrXLQLMhBS0tLW1qaCzvF5a56StxsZG3/0HH3ywVqxYEcbmAdbi6q/9ReM9itUMUSyhj+OEwyGlpHjmYvjdbMeyzD0JR5YkXso+S5IGDZJ73Dhp3DjzbE1Hhxy1tb4sjWl556YmucrLpfJy05cwHA5P8NQlqPFfx0ZDhpCtSWAhBS1jxozRtm3bJEnp6ekqKCjQunXr9MMf/lCSZ87LYL/0MRDruPrbnd2yT7xHQAzoa1lmbyEAi4OZmMmS2IXLJSM/3zPf5ZBDzLdpagpe2rmyUo6aGjlraqSaGrn+/W/TpzAGD/YUCTAZfuYeMUJGTo5nXhbiUkjv7LRp07R69Wr94he/kCSdf/75evzxx31VxF588UVdeumlEWkoYAWu/nZnt+xTsPfIbsEVABPByjJbHMzEVZbELjIy5J4wQZowwTxb097uCVzMAhrv77t3y/XNN9I335i+hOF0ysjNDQxkumRulMEFrlgVUtBy3XXXadq0adq7d69SU1N12223qaGhQa+99ppcLpcuuOAC3X333ZFqKwAbiJXMht2CKwAh6CmY2bvXE8z4l2emLHPsS0qSMWKEOkaMML/fMKSdO/fNrTHJ2Djq6uSsqpKzqkr617/MnyYjo+e5NTk5nrLgsJ2Qh4eNGTPG93tqaqoeeeQRPfLII2FvGAB7ipXsU6wEVwBC4HQGX2PGL5ChLHMccjikoUPlHjpU2n9/82xNW1vgXBqzrE1Tk1wbN0obN5q+jOFyycjLMw9ovNmaQQwbtAID/2A5hvEgEmIluAIQBp1rzHQry+zNxnStZEYwE5+Sk3tdjNPR2Ng9kPEvHlBX56uSFowxdKjc+fkam5mp1P32CxyGNmKEZzFOpzNCO5m4CFpgOYbxAAAiIlglszgoy4x+cDhkZGXJyMqSe+JE82327u2WrQko79y5bo2rsVGZkvTJJ92ewkhK8hQl6FoswC9zozTmTIWKoAWWYxgPACBq4qQsMyIkNVVGYaE6CgvN73e75dixQ47KStX++9/Kb2/3zLPxX5SzoUGO7dvl3L496Mu4s7K6BTQB2Zphwyjv3AVBCyzHMB4AgOV6K8vszcb4VzJjmFnicTplZGfLyM7WzsGDNbyoqPs2LS3dszTegMY7DK2hQWpokOvLL01fxkhJ6ZadCcjc5OV5jtcEQtACAAAQTLBKZt5hZv7BzN69nmDGpJrZW9tb9MLGPbpowiDKKce7tDQZRUXqKCoyLxjgdstRVxeYnfEfjlZZKcfOnXJs2SLnli1BX8adnR20WIBRUCBj6NC4ytYQtAAAAIQq2DAzyVPNzL8kc1ubnvumUdub2rVi4x6ClkTXuZ6MkZsr98EHm2+ze7d5FTTv/9XVctbVSXV1cn3xhelTGGlpgQGNd56N9/e8PCk5OYI7Gl4ELQAAAOHUWc1Mgwb5ApofnjRcSz/fqVnFKXLnuQKHmlEEAF0NHiz3+PHS+PHBF+Osre25vPPu3XKVl0vl5aYvYTgcMnJyPEPO8vN9wUzAYpxDhtgmW0PQAgAAEGFd52/2WNHMP5hh3gzMJCXJKChQR0FB8G2amgKDmM7haL4qaLW1ctbUSDU1CracpjF4cEBAY7oYZ1J0wgmCFgCmWD8HAKKkp6FmXYMZFs5EX2VkyD1hgjRhQvBsTXV1QEDjqKz0VEPzz9Z88430zTemL2F0DnXrFtD4Z2sywlMdlqAFgCnWz7EPAsjIoF8RE5KTPYsm+t3kC2ZaW9Xe0CBjyJB9mRmTIgDov7guoJCUJGPkSHWMHGl+v2FIO3d2LxLg97+jtlbOqio5q6qCvowxZIhpMOMr75ydLbmC5Xr8mtvf/QQQ31g/xz4IICODfkVM6wxmOoYOlZGfLykwmFFbmxz+C2cSzPTLCxv3aEuiFlBwOKShQ+UeOlQ64ADzbE1bW7fFOB0VFYHZml275Nq1SyorM30Zw+XyFQnY/dZbQZtD0ALAFOvn2AcBZGTQr4hLncGMtG/ejK+i2d69BDMhumjCIK3YuEcXThhkdVPsKTlZxujR6hg92vx+w5CjsXHf8LOuAU1VlZx1dXJ8952c333X40sRtACAzRFARgb9ioTirWimHoIZ/zkzBDOSpOmj0hIvwxJODoeMrCwZWVlyT5xovs3evb5sTU8IWgAAABJVT8GM/1oz3sxMh+kgIaD/UlNlFBaqo7Cwx82cUWoOAAD9tqq8WTNfr9Gq8marmxKSWG034F1nRllZMvLyZIweLfe4cXIXFck9cqTcOTkyMjNlpKX1aRJ1pLy1vUVXvV2vt7a3WNaGnpi1z+5ttiuCFgCA7flPmo8lsdpuIKhgwcy4cfuCmaFDoxbM+E+UtyOz9tm9zXZF0AIAsL25JRmakJk0oEnzVmQ9wtFuICa4XPuCmdzcqAUzF00YpLEZSbadKG/WPru32a6Y0wIAsL1wTJq3osQxk/2R8LzBzKBBvVcza20NedFMu0+UN2uf3dtsVwQtAICEQIljxJq4XgA1WAGALpP/fdXMkPAIWgAgTsX1CU8/kPVArEnIBVBTUqSUFPnnWwzD2BfAeIOZ1lY52tstayaijzktABCnmAQOxLZQ50RFat6WFfPBAl7T4ZBSU6XMTK1sGqwZnyVrpfLl3m8/uUeN0ut7MnThx+36W60kJ6e2/vpSqSxWqpnxzgJAnGISOGC9gZzwzypK1+qZuX3OskTqQoUVF0CCvWbA7U6nlJ6uR7a59A/HUC2qH+oJZIqK5B4xwjP5f8gQGSkpnsAnAfWlUlmsVDMjaAGAOBXqCQ+A8IvmCX+kLlRYcQEk2Gua3d7ttqQkafBgTyWz/HwZhYVyjx8vd2Gh3AUFMoYPlzF4sIzk5Kjtj1X6UqksVqqZMacFAAAgQqJZACJS87asmA8W7DXNbu9z+xJwvkxfKpXFSjUzghYAAIAIoQCEzXnny6SmBgYzHR2B1cu8QY3bbVlTE53lw8OWLVumyZMnKz8/XyeccII++OCDoNvOnTtXWVlZ3f6NHDkyii0GEG1WTAIFACQwl0tKTw9cLLPLfJmOwYMTer5MtFkatLz66qu65ZZbdP311+udd97RUUcdpfPPP19bt2413f6+++7Thg0bAv4VFRXprLPOim7DAUQVVbDshSAS8YZjGn3mN1+mPScncL5Mfn5CzZeJNkuDlsWLF+viiy/W7NmzdcABB2jhwoXKz8/X8uXLTbcfOnSo8vPzff82b96s8vJyzZ49O8otBxBNVMGyF4JIxBuOaQxYSoo0ZIgnaBkxQsbYsZ5gZswYufPyZGRlyRg0SEYSMzP6y7Kea21t1WeffaZrrrkm4Pbp06fro48+6tNz/Pa3v9WkSZM0derUSDQRgE0wJtxeWFke8YZjGhHBfJmwsixoqaurU0dHh3JzcwNuz83NVXV1da+Pb2xs1B//+EfdcccdvW5bVlbW73ai/+h369D31kmEvp8o6aFiSW0NstvuJkL/21Us972dj+m+iOW+j3Vh7fv2djnb2uRoa5OjtdX3swyj98fGicIJE4LeF7M5qpdeeklut1sXXnhhr9sWFxdHoUXwV1ZWRr9bhL63Dn1vLfrfOvR9eK0qb9aS0ibNLcnoNctM31snan3fJSuj1lbPz3Gop1yTZUFLdna2XC6XampqAm6vqalRXl5er4//7W9/qzPPPFPDhg2LVBMBxKBQvuwBwI7859jwOYag68t4A5jOoEZtbXGzvowZyybip6SkaMqUKVqzZk3A7WvWrOl1jso//vEPff7557rssssi2UQAMYgJtUBkUWkr8ig+gl45HFJampSZKSMnR8aoUTKKiuQeN07uUaPkzsmRkZkpIy1Nclq+wklYWDo8bP78+ZozZ44OP/xwTZ06VcuXL1dlZaWuuOIKSdKcOXMkSUuXLg143NNPP63x48dr2rRpUW8zAHuLxoRasjlIZGQBIo/iI+g37/oy6em+zIwhSe3t+yb8+w83i6H5MpYGLeecc47q6+u1cOFCVVVVadKkSXrppZdUWFgoSdq2bVu3x+zatUuvvvqqbrrppmg3F0AMiMaXPSdtSGRU2gJiUFKSlJQkY/Bg302G1H2+zN69th1iZvlE/CuvvFJXXnml6X2rV6/udtuQIUO0ffv2SDcLAILipC22kSkbGLIAQByJofkylgctABBrOGmLbWTKAKAH3vkyaWnB15fpHGYWzfVlCFoAAAmFTFnfkJECEMDi+TIELQCAhEKmrG/ISAHoE7P5MoYhdc6TCdd8mfiogQYAAMKKsruJJRFLWSfiPg9ESP3lcHjmywwZImP4cBkjRnhKMo8fL/fo0XLn5cnIypKRni4jqW85FIIWAEBcWFPr4gQkjGYVpWv1zFyyLAkiEde4SsR9Hoiw9FdP68uMHNnjQwlaAABxYcV3SZyAAP2UiJm1RNzngYhof7lc0qBBPW7CnBYAQFy4cGS7XtuRzgkI0A+JONcrEfd5IKzuL4IWAEBcOCmnQ1cfk2t1MwAAEcDwMAAAAAC2RtACAAAAwNYIWgAAAIAIobRyeBC0AAAAABFCaeXwIGgBAAAAIoTSyuFB9TAAAAAgQqwuFRwvyLQAABAmjF2PPvocSAwELQAAhAlj16OPPgcSA0ELAABhwtj16KPPgcTAnBYAAMKEsevRR58DiYFMCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAi7FIpr2E+n5Y9f5Fo50D2be+Pta7XU8IWgAAACzGIpn2Eur7YdX7F412DmTf+vpY73Y9IWgBAACwGItk2kuo74dV71802jmQfevrY73b9cTR0NBghNwCoBdlZWUqLi62uhkJib63Dn1vLfrfOvS9deh769D30UWmBQAAAICtEbQAAAAAsDWCFgAAAAC2RtACAAAAwNYIWgAAAADYGkELAAAAAFsjaAEAAABgawQtAAAAAGyNoAUAAACArRG0AACAuLOqvFkzX6/RqvJmq5sCIAwIWgAAQNxZUtqkTTvb9URpk9VNARAGBC0AACDuzC3J0ITMJP2kJMPqpgAIgySrGwAAABBus4rSNaso3epmAAgTMi0AACBmMFcFSEwELQAAIGYwVwVITAQtAAAgZjBXBUhMzGkBAAAxg7kqQGIi0wIAAADA1ghaAAAAANgaQQsAAAAAWyNoAQAAAGBrBC0AAACwRDTX3WGNn54NtH8i3b8ELQAAALBENNfdYY2fng20fyLdvwQtAAAAsEQ0191hjZ+eDbR/It2/rNMCAAAAS0Rz3R3W+OnZQPsn0v1LpgUAAACArRG0AAAAALA1ghYAAAAAtkbQAgAAAMDWCFoAAAAA2BpBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAkhFXlzZr5eo1WlTdb3RQAISJoAQAACWFJaZM27WzXE6VNVjcFQIgIWgAAgKWilQGZW5KhCZlJ+klJRkRfB7CbeMgyErQAAABLRSsDMqsoXatn5mpWUXpEXwewm3jIMhK0AAAAS5EBASIrHv7GkqxuAAAASGyzitLJfgARFA9/Y2RaAAAAANgaQQsAAEAMWlXerDnrU2N6cjXQVwQtAAAAMWhJaZO2NDtienI10FcELQAAADFobkmGCtONmJ5cDfQVE/EBAABi0KyidE1s26viGJ9gDfQFmRYAAAAAtkbQAgAAAMDWCFoAAAAA2BpBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAM2KryZs18vYbV2QFEBEELAAAYsCWlTdq0s912q7MTTAHxgaAFAAAM2NySDE3ITLLd6ux2DaYAhCbJ6gYAAIDYN6soXbNsuDL73JIMPVHaZLtgCkBoCFoAAEDcsmswBSA0lg8PW7ZsmSZPnqz8/HydcMIJ+uCDD3rcvrW1Vf/zP/+jyZMnKy8vTwcddJCeeOKJKLUWAAAAQLRZmml59dVXdcstt+iBBx7Q0UcfrWXLlun888/X2rVrNWbMGNPH/PjHP9Z3332nhx9+WPvtt59qamrU3MzkOgAAACBeWRq0LF68WBdffLFmz54tSVq4cKHefPNNLV++XAsWLOi2/VtvvaV33nlHn376qbKzsyVJY8eOjWqbAQAAAESXZcPDWltb9dlnn2n69OkBt0+fPl0fffSR6WNWr16tQw89VIsXL1ZJSYkOO+ww3XTTTWpqoiIIAACIbZRnpg8QnGWZlrq6OnV0dCg3Nzfg9tzcXFVXV5s+pry8XGvXrlVqaqqeeeYZNTY26qabblJlZaWeeeaZoK9VVlYW1rajb+h369D31qHvrUX/W4e+H7gH16dqS7NDD/6jWRPb9vb5cfHU9/3tA6vEU9/bQXFxcdD7Yqp6mNvtlsPh0JNPPqmhQ4dK8gwpO+ecc1RdXa28vDzTx/XUAYiMsrIy+t0i9L116Htr0f/Woe/D4+fJzb7yzMV9rHgWb33fnz6wSrz1vd1ZFrRkZ2fL5XKppqYm4PaampqgwUd+fr5GjBjhC1gkaf/995ckbdu2LejjAABAdKwqb9aS0ibNLcmg1HCIKM9MHyA4y+a0pKSkaMqUKVqzZk3A7WvWrNHUqVNNH3P00UersrIyYA7Lpk2bJClotTEAABA9rEAPIBIsXadl/vz5+t3vfqdnnnlGGzZs0M0336zKykpdccUVkqQ5c+Zozpw5vu3PO+88DR8+XPPnz9eXX36ptWvX6pZbbtEPf/jDbnNjAABA9M0tydCEzCRWoAcQVpbOaTnnnHNUX1+vhQsXqqqqSpMmTdJLL72kwsJCSZ4hX/4yMjL0xz/+UTfddJOmT5+urKwsnXHGGablkQEAQPQxvAdAJFg+Ef/KK6/UlVdeaXrf6tWru91WXFysP/zhD5FuFgAAAACbsHR4GAAAAAD0hqAFAAAAgK0RtAAAAACwNYIWAAAAALZG0AIAAADA1ghaAAAAomRVebNmvl6jVeXNVjcFiCkELQAAAFGypLRJm3a264nSJqubAsQUghYAAIAomVuSoQmZSfpJSYbVTQFiiuWLSwIAACSKWUXpmlWUbnUzgJhDpgUAAACArRG0AAAAALA1ghYAAABR2QuwM4IWAAAAUdkLsDOCFgAAAFHZC7AzqocBAACIyl6AnZFpAQAAAGBrBC0AAAAAbI2gBQAAAICtEbQAAAAAsDWCFgAAAEQM698gHAhaAAAAEDGsf4NwIGgBAABAxLD+DcKBdVoAAIDtrSpv1pLSJs0tyWAtlRjD+jcIBzItAADA9hhiBCQ2ghYAAGB7DDECEhvDwwAAgO0xxAhIbGRaAAAAANgaQQsAAADQA9aasR5BCwAAANADCkFYj6AFAAAAIUuk7AOFIKzHRHwAAACEzD/7EO9FEigEYT0yLQAAAAgZ2QdEE5kWAAAAhIzsA6KJTAsAAABgkUSaGzQQBC0AAACARahM1jcELQAAAIBFmBvUN8xpAQAAACzC3KC+IdMCAAAAwNYIWgAAAADYGkELAAAAAFsjaAEAAABgawQtAAAAAGyNoAUAAACArRG0AAAAALA1ghYAAAAAtkbQAgAAAMDWCFoAAAAQdqvKmzXz9RqtKm+21XPBev15PwlaAAAAEHZLSpu0aWe7nihtstVzwXr9eT8JWgAAABB2c0syNCEzST8pybDVc8F6/Xk/kyLYHgAAACSoWUXpmlWUbrvngvX6836SaQEAAABgawQtAAAAAGyNoAUAAACArRG0AAAAALA1ghYAAAAAtkbQAgAAAMDWCFoAAAAA2BpBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAALZG0AIAAOLeqvJmzXy9RqvKm61uCoB+IGgBAABxb0lpkzbtbNcTpU1WNwVAPxC0AACAuDe3JEMTMpP0k5IMq5sCoB+SrG4AAABApM0qStesonSrmwGgn8i0AAAAALA1ghYAAAAAtkbQAgAAAMDWCFoAAAAA2BpBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFAAAAgK0RtAAAAACwNYIWAAAAALZG0AIAAADA1ghaAAAAANgaQQsAAAAAWyNoAQAAAGBrBC0AAAAAbI2gBQAAAICtWR60LFu2TJMnT1Z+fr5OOOEEffDBB0G3fffdd5WVldXt39dffx3FFgMAAACIpiQrX/zVV1/VLbfcogceeEBHH320li1bpvPPP19r167VmDFjgj5u7dq1GjZsmO/3nJycaDQXAAAAgAUcDQ0NhlUvfvLJJ+vAAw/UI4884rvtsMMO0w9/+EMtWLCg2/bvvvuuZs2apU2bNik7OzuaTQUAAABgEcuGh7W2tuqzzz7T9OnTA26fPn26Pvroox4fe+KJJ+qAAw7QmWeeqXfeeSeSzQQAAABgMcuGh9XV1amjo0O5ubkBt+fm5qq6utr0MQUFBVq0aJEOO+wwtba26sUXX9QPf/hDrV69Wscee2w0mg0AAAAgyiyd0xKq4uJiFRcX+34/6qijtGXLFj3yyCMELQAAAECcsmx4WHZ2tlwul2pqagJur6mpUV5eXp+f5/DDD9c333wT7uYBAAAAsAnLgpaUlBRNmTJFa9asCbh9zZo1mjp1ap+f59///rfy8/PD3TwAAAAANmHp8LD58+drzpw5OvzwwzV16lQtX75clZWVuuKKKyRJc+bMkSQtXbpUkvT444+rsLBQkyZNUmtrq1566SWtXr1azzzzjGX7AAAAACCyLF1c8pxzztG9996rhQsXatq0aVq7dq1eeuklFRYWSpK2bdumbdu2+bZva2vTHXfcoeOOO04zZszwbX/mmWcGPG8oC1aib+69995ui3ruv//+vvsNw9C9996riRMnqqCgQGeccYa+/PLLgOdoaGjQ1VdfrcLCQhUWFurqq69WQ0NDlPfE/t5//31deOGFmjRpkrKysvT8888H3B+uvv7iiy80c+ZMFRQUaNKkSbr//vtlGJZVQLeF3vp+7ty53f4OTjnllIBt9u7dqxtvvFH77befRo4cqQsvvFDbt28P2Gbr1q264IILNHLkSO2333666aab1NraGvH9s7NFixbppJNO0pgxYzR+/HhdcMEFKi0tDdiGYz8y+tL3HPuR8eSTT+rYY4/VmDFjNGbMGJ166qn6y1/+4rufYz5yeut7jnn7sTRokaQrr7xS//73v1VdXa23335bxx13nO++1atXa/Xq1b7fr7vuOv3zn/9UZWWlysvL9cYbb+i0004LeD7vgpXXX3+93nnnHR111FE6//zztXXr1qjtU7wqLi7Whg0bfP/8g8GHH35Yixcv1v3336+33npLubm5Ovvss7Vr1y7fNldeeaXWr1+vV155Ra+88orWr1/vy6Zhn927d6ukpET33Xef0tPTu90fjr7euXOnzj77bOXl5emtt97Sfffdp0cffVSPPfZYVPbRrnrre8lTct3/7+Dll18OuP/WW2/VqlWr9NRTT+n111/Xrl27dMEFF6ijo0OS1NHRoQsuuEBNTU16/fXX9dRTT2nlypW67bbbIr5/dvbee+/pP//zP/WXv/xFK1euVFJSks466yzt2LHDtw3HfmT0pe8ljv1IGDlypO666y69/fbbWrNmjY4//nhdcskl+vzzzyVxzEdSb30vcczbjaWLS0ZCqAtWom/uvfderVy5Uh9++GG3+wzD0MSJE3XVVVfphhtukCQ1NzeruLhYd999t6644gpt2LBBU6dO1Z///GcdffTRkqQPP/xQM2bM0McffxxQFQ77jBo1Sr/+9a91ySWXSApfXz/11FO688479fXXX/tOzhcuXKjly5ertLRUDofDmh22ka59L3muvNXX1+vFF180fUxjY6MmTJigxYsX60c/+pEkT8b44IMP1iuvvKKTTz5Zf/vb3/SjH/1I//73vzV69GhJ0osvvqhrr71WZWVlyszMjPzOxYCmpiYVFhbq+eef14wZMzj2o6hr30sc+9FUVFSkBQsW6PLLL+eYjzJv319xxRUc8zZkeaYlnAayYCV6V15erokTJ2ry5Mn68Y9/rPLycknSt99+q6qqqoB+T09P17HHHuvr93Xr1ikjIyOgyMLRRx+twYMH896EIFx9vW7dOh1zzDEB2YSTTz5ZFRUV+vbbb6O0N7Hpww8/1IQJE3T44Yfr2muvDaiA+Nlnn6mtrS3g/Rk9erQOOOCAgL4/4IADfF9gkqfv9+7dq88++yxq+2F3TU1NcrvdysrKksSxH01d+96LYz+yOjo69Pvf/167d+/WUUcdxTEfRV373otj3l5iap2W3vRnwUr0zRFHHKHHH39cxcXFqq2t1cKFC3Xaaadp7dq1qqqqkiTTfq+oqJAkVVdXKzs7O+CKjsPhUE5ODu9NCMLV19XV1Ro5cmS35/DeV1RUFKldiGmnnHKKZs2apbFjx2rLli367//+b5155pn6+9//rtTUVFVXV8vlcik7Ozvgcf6fQdXV1d3eP28JeP4W9rnlllt08MEH+04gOPajp2vfSxz7kfTFF1/otNNOU0tLiwYPHqznnntOBx54oO/El2M+coL1vcQxb0dxFbQgck499dSA34844ghNmTJFv/vd73TkkUda1Cogus4991zfzwceeKCmTJmigw8+WH/5y1+6FQRB//3iF7/Q2rVr9ec//1kul8vq5iSUYH3PsR85xcXFevfdd7Vz50699tprmjt3rv70pz9Z3ayEEKzvS0pKOOZtKK6Gh4VrwUr0LiMjQxMnTtQ333zjWyenp37Py8tTXV1dQLUSwzBUW1vLexOCcPV1Xl6e6XN470PfjBgxQiNHjvQtcJuXl6eOjg7V1dUFbNf1/ena994sMX3vmdj6+9//XitXrgy4AsyxH3nB+t4Mx374pKSkaL/99tOUKVO0YMECHXzwwXr88cc55qMgWN+b4Zi3XlwFLeFasBK9a2lpUVlZmfLz8zV27Fjl5+cH9HtLS4s+/PBDX78fddRRampq0rp163zbrFu3Trt37+a9CUG4+vqoo47Shx9+qJaWFt82a9as0YgRIzR27Ngo7U3sq6urU0VFhe/kYsqUKUpOTg54f7Zv3+6bLCt5+n7Dhg0BZTHXrFmj1NRUTZkyJartt5ubb77Zd9LsX1Jd4tiPtJ763gzHfuS43W61trZyzFvA2/dmOOatF1dBi+RZsPJ3v/udnnnmGW3YsEE333xzwIKV6J/bb79d7733nsrLy/XJJ59o9uzZ2rNnjy666CI5HA7NnTtXDz/8sFauXKnS0lLNmzdPgwcP1nnnnSdJOuCAA3TKKafo5z//udatW6d169bp5z//uU4//XQqh3XR1NSk9evXa/369XK73dq2bZvWr1+vrVu3hq2vzzvvPKWnp2vevHkqLS3VypUr9dBDD2nevHkJXUmmp75vamrS7bffrnXr1unbb7/Vu+++qwsvvFC5ubn6wQ9+IEkaOnSoLr30Ui1YsEB///vf9a9//Utz5szRgQceqBNPPFGSpzDIpEmT9JOf/ET/+te/9Pe//1133HGHLrvssoSuJHPDDTfod7/7nZ588kllZWWpqqpKVVVVampqkiSO/Qjqre859iPnzjvv1AcffKBvv/1WX3zxhe666y699957Ov/88znmI6ynvueYt6e4K3kseRaXfPjhh1VVVaVJkybpnnvuCVj/BaH78Y9/rA8++EB1dXXKycnREUccodtuu00TJ06U5ElH33fffXr66afV0NCgww8/XL/5zW9UUlLie46GhgbddNNNeuONNyRJM2bM0K9//etuFWoS3bvvvqtZs2Z1u/2iiy7SkiVLwtbXX3zxhW644Qb985//VFZWlq644grdfPPNCf0l1lPfL1q0SJdcconWr1+vxsZG5efna9q0abrtttsCKsPs3btXt99+u1555RW1tLTo+OOP1wMPPBCwzdatW3XDDTfonXfeUVpams4//3zdfffdSk1Njcp+2lGwz4Gbb75Zt956q6Twfc5w7Afqre+bm5s59iNk7ty5evfdd1VdXa3MzEwdeOCBuvbaa3XyySdL4piPpJ76nmPenuIyaAEAAAAQP+JueBgAAACA+ELQAgAAAMDWCFoAAAAA2BpBCwAAAABbI2gBAAAAYGsELQAAAABsjaAFACzw7rvvKisrS7///e+tbkqfLVmyRFOmTNHw4cP1ve99z+rmhNXzzz+vrKwsffvtt1Y3xTLeY/Ldd9+1uikA0A1BCwCgVx9++KFuvfVWHX744Xrsscd0xx13WN0k26ioqNC9996r9evXW92UPnnggQf0pz/9yepmAEBIkqxuAADA/t577z1J0qJFizR06FCLWxN+F154oc4999x+rVJdWVmp+++/X4WFhZo8eXIEWhdeixYt0plnnqkf/OAHAbcfd9xxqqysVEpKikUtA4DgyLQAQBzbvXt3WJ6npqZGkuIyYJEkl8ultLQ0ORwOq5viE673rq+cTqfS0tLkdHJqAMB++GQCEPfuvfdeZWVlqaysTHPnzlVhYaEKCws1b9487dmzx7fdt99+q6ysLD3//PPdniMrK0v33ntvt+fcsGGDrr76ahUWFmq//fbTr371KxmGoe+++04XX3yxxowZo+LiYj3yyCOmbevo6NA999yjiRMnasSIETrnnHO0adOmbttt3LhRl19+ucaNG6f8/HxNmzZNr732WsA23nkZb7/9tm666SYVFxdr1KhRPfZNR0eHfvOb3+jQQw9VXl6eDjroIN1xxx1qbm4O2Pf//d//9f0crI+8qqurdc011+jAAw9UXl6eiouLdd555+nLL78M2O6tt97SzJkzNWrUKI0aNUrnnnuu6RCrUPb9vffe0y9+8QuNHz9eI0eO1CWXXKLa2toe+8D/8f5zWs444wwdeeSR+uqrrzRr1iyNGDFCkyZN0sMPP+zb5t1339VJJ50kSZo/f76vf/yPlYG+dzt27NAvf/lLHXvssRo9erRGjRqlM844Qx988EG3/TAMQ08++aS+973vqaCgQPvtt5/OOuss37ZZWVnavXu3XnjhBV9bzzjjDN++mM1pee+99zRz5kyNHDlShYWFuuCCC1RaWhqwTV//xiTp7bff1owZMzR27FiNGDFCU6ZM0Y033tjrewQgsTE8DEDC+PGPf6yioiItWLBA//rXv/TMM88oNzdXd911V7+f8z//8z+1//77a8GCBfrrX/+qRYsWadiwYXruued07LHH6s4779TLL7+sO+64Q4cccohOOOGEgMc/9NBDcrvd+ulPf6qGhgYtXbpUs2bN0vvvv69hw4ZJkjZs2KDTTjtN+fn5uu666zR48GD96U9/0uzZs7V06VJdcMEFAc958803KysrS9dff7127tzZY/t/9rOf6dlnn9WsWbM0f/58ffrpp3rkkUf05Zdf6qWXXpLD4dDSpUu1YsUKrVmzRkuXLpUkTZ06Nehzzp49W1988YUvmKurq9P777+vjRs3atKkSZKkl19+WVdffbVOOukk3XHHHWptbdXTTz+tmTNn6q233tL+++/fr32/9dZbNWzYMN18883asmWLlixZohtvvFH/93//14d3s7udO3fqvPPO0w9+8AOdddZZeu2117RgwQKVlJTo1FNP1QEHHKBf/OIXuueee3T55ZfrmGOOkSQdeOCB/Wq/2XtXXl6u1157TWeffbaKiorU2NioZ599VmeddZbeeustHXTQQb7HX3fddXrmmWd08skn6+KLL5ZhGFq3bp0++OADHXvssVq6dKmuvfZaHXbYYbr88sslSXl5eUH3/5133tE555yjsWPH6pZbblFLS4uWLVum73//+3rrrbc0YcKEgO17+xv76quv9KMf/UglJSW65ZZbNGjQIG3evFlvvvlmv94fAImDoAVAwpg8ebIWL17s+72+vl7PPvvsgIKWKVOm6LHHHpMkXX755Zo8ebLuuOMO3XbbbbrhhhskSeeee64mTZqk559/vlvQUlNTo48//lhZWVmSpGnTpumHP/yhFi9erNtvv12SdMstt2jEiBFas2aN0tPTJUlXXXWVzj77bN1111360Y9+FDCsyXtinJTU80f8559/rmeffVYXX3yxHn/8cd/to0eP1v3336+//OUv+v73v68LLrhAn3zyidasWdPtJLurhoYGffjhh7r77rt1zTXX+G7/+c9/7vt59+7duvHGG3XxxRcHvB+XXnqpjjjiCP3617/WsmXL+rXvw4cP1x//+EffbW63W0uXLlVjY2O/hrZVVVVpyZIluuiii3xtPPjgg/Xss8/q1FNPVV5enk499VTdc889OvLII7v1Tzjeu5KSEn322WcBw7Yuv/xyHXnkkVq6dKkeffRRSZ5MyTPPPKMrr7xSv/nNb3zbzp8/X4ZhSJIuuOAC/dd//ZeKiop6fS8l6fbbb1dmZqb+9re/afjw4ZI8x/PRRx+tX/3qV3rmmWcCtu/tb2zNmjXau3evXnnlFWVnZ/u2u/POO3ttC4DExvAwAAlj9uzZAb8fc8wxqq+v7zUb0ZPLLrvM97PL5dKUKVNkGIYuvfRS3+1ZWVmaMGGCysvLuz3+wgsv9AUsknTCCSdo0qRJ+vOf/yzJMzTo73//u8466yzt2bNHdXV1vn8nn3yyvvvuO23cuLHbfvYWsEjSX//6V0mek1p/8+bNk8vl8t0fivT0dKWkpOi9997Tjh07TLdZs2aNGhoadP755wfsT0dHh4455hjf8KT+7Pull14aEAQcc8wx6ujo0NatW0PeF+/++J/cp6Sk6LDDDjN9L7sK13uXmprqC1haWlpUX1+vjo4OHXbYYfrss898261cuVKSJ9vUVX/m6lRWVmr9+vW66KKLfAGLJI0fP14zZszQm2++qY6Ojm7t99f1bywzM1OStHr1arnd7pDbBCBxkWkBkDBGjx4d8Ls3WGhoaPCdTA30OTMzM5WcnKz8/Pxut3sns/sbP3686W3vvPOOJOmbb76RYRi67777dN9995m2oaamRsXFxb7fi4qK+tT2rVu3yuFwdBviM3ToUBUUFGjLli19eh5/qampuvPOO/XLX/5SxcXFOuKII3Tqqafqggsu8PWVd87OWWedZfoc3hP0/ux7T+9xf4wYMaLbxPSsrCx98cUXvT42XO+d2+3Www8/rKeffrrbOjJjx471/bx582bl5eUFZDAGwhvo+bfPa//999fKlStVV1cXMLyst7+xc845R88995yuvfZa3XnnnTr++ON1xhln6Oyzz+5ToA0gcfEJASBhuFwu09u9Q2eCXY3uejW5t+cMVn3J+zqh8F6Nnjdvnk477TTTbUpKSgJ+9w5Dssq8efM0c+ZMvf766/r73/+uhQsXatGiRVqxYoWmTZvm26fHH39cI0eODPo8/dn33t7jUA3k+cL13i1atEj//d//rYsuuki33367hg8fLpfLpUWLFmnz5s29tiOaeuuv9PR0rV69Wu+//77+9re/6c0339RVV12lxYsX64033rD82AVgXwQtANDJe1W4sbEx4Pb+Di3qC7NKYZs2bVJhYaGkfVfek5KSdOKJJ4b1tceMGSPDMLRx40bfxHHJM/m8srJSp59+er+fu6ioSPPmzdO8efO0fft2TZs2TQ888ICmTZumcePGSZJycnJ63KdI7ns4BQt2w9X+P/7xj/re976nJUuWBNzuX6FMksaNG6f/9//+n2pra5WTkxNye7saM2aMJKmsrKzbfWVlZRo8eHC/sjpOp1PTpk3TtGnT9Ktf/UpPPfWUrr/+eq1atUo/+tGPQn4+AImBOS0A0CkzM1PZ2dndSsl6J4VHwooVKwKGLr399tv68ssvfQFDbm6upk2bpt/+9rf67rvvuj2+L+V8g/Fe/e96MvzEE0+oo6OjX0HLnj17AsolS9KoUaOUm5vrCwanT5+uoUOHatGiRWptbe32HN59iuS+h9OgQYMkdR+CFq72u1yubpmdjz76SOvWrQu47cwzz5Qk06Fo/o8fNGhQn4bLFRQU6JBDDtGKFSsC5idt3rxZb7zxhk455ZSgmZVg6uvru912yCGHSOp+sQAA/JFpAQA/l112mR588EFdc801OvTQQ/XBBx90mywdTrm5ufr+97+v//iP/1BjY6OeeOIJFRQUBEyOX7RokU4//XQdd9xxmj17tsaNG6eamhp98skn2rBhgz799NN+vfZBBx2kSy+9VM8++6x27typ448/Xv/617/03HPP6ZRTTgk6pKknGzdu1JlnnqmzzjpLEydOVGpqqv76179qw4YNuvvuuyV5gsMHH3xQV111lY4//nide+65ysvL09atW/Xmm29q4sSJvkAqUvseTuPGjVNWVpaWL1+ujIwMZWRkaNKkSSopKQlL+2fMmKH77rtPc+bM0bHHHqtNmzbp6aef1sSJE9XU1OTbbtq0abr44ou1bNkybd68Waeccook6eOPP9aBBx6o66+/XpJ06KGH6u2339ajjz6qkSNHKicnp1tVO6+7775b55xzjk499VTNnj3bV/I4LS1Nv/zlL0Puq1//+td67733dPrpp6uwsFANDQ1avny5Bg8ePKDMHoD4R9ACAH5uuukm1dbW6rXXXtMf//hHnXLKKXrllVe6TVYPl5/97GcqKyvTo48+qsbGRh1zzDH69a9/HVCtqbi4WGvWrNH999+vFStWqK6uTjk5OTrooIN02223Dej1H3roIY0dO1bPPfec3njjDeXl5emaa67Rrbfe2q+KU6NHj9b555+vd955R6+88oocDofGjx+vRx99NKCi2jnnnKOCggItWrRIjz32mPbu3auCggJNnTpVV1xxRVT2PVySk5O1dOlS3XXXXbrhhhvU1tamm2++WSUlJWFp/3/913+publZL7/8sl577TVNmjRJy5cv1+9//3u99957Ads+9thjOvDAA/Xss89qwYIFysjI0CGHHKLjjjvOt80999yjn/3sZ7rvvvu0e/duHXfccUGDluOPP15/+MMfdM899+iee+5RUlKSjjnmGC1YsKBffxMzZ87Utm3b9MILL6i2tlbDhw/XkUceqZtuusk3JBIAzDgaGhr6NzsRAAAAAKKAOS0AAAAAbI2gBQAAAICtEbQAAAAAsDWCFgAAAAC2RtACAAAAwNYIWgAAAADYGkELAAAAAFsjaAEAAABgawQtAAAAAGzt/wMQfLat77jCrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def roc_score_per_interaction(x):\n",
    "    try:\n",
    "        return roc_auc_score(x.answered_correctly.values, x.predictions.values)\n",
    "    except ValueError:\n",
    "        return np.nan\n",
    "\n",
    "df[\"num_iteractions\"] = (df.groupby(\"user_id\").cumcount() // 10) * 10\n",
    "interactions_auc = df.groupby(\"num_iteractions\").apply(roc_score_per_interaction)\n",
    "interactions_auc = interactions_auc[~interactions_auc.isna()].copy()\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(12, 8))\n",
    "sns.regplot(\n",
    "    y=interactions_auc.values,\n",
    "    x=interactions_auc.index,\n",
    "    line_kws={\"color\": \"red\", \"linewidth\": 2},\n",
    "    scatter_kws={\"s\": 3},\n",
    ")\n",
    "ax.set_ylim(0.5, 1)\n",
    "ax.set_xlim(0, interactions_auc.index.max())\n",
    "ax.set_ylabel(\"auc\")\n",
    "ax.set_xlabel(\"number of seen interactions\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Real test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Iter_Valid(object):\n",
    "    def __init__(self, df, max_user=1000):\n",
    "        df = df.reset_index(drop=True)\n",
    "        self.df = df\n",
    "        self.user_answer = df[\"user_answer\"].astype(str).values\n",
    "        self.answered_correctly = df[\"answered_correctly\"].astype(str).values\n",
    "        df[\"prior_group_responses\"] = \"[]\"\n",
    "        df[\"prior_group_answers_correct\"] = \"[]\"\n",
    "        self.sample_df = df[df[\"content_type_id\"] == 0][[\"row_id\"]]\n",
    "        self.sample_df[\"answered_correctly\"] = 0\n",
    "        self.len = len(df)\n",
    "        self.user_id = df.user_id.values\n",
    "        self.task_container_id = df.task_container_id.values\n",
    "        self.content_type_id = df.content_type_id.values\n",
    "        self.max_user = max_user\n",
    "        self.current = 0\n",
    "        self.pre_user_answer_list = []\n",
    "        self.pre_answered_correctly_list = []\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def fix_df(self, user_answer_list, answered_correctly_list, pre_start):\n",
    "        df = self.df[pre_start : self.current].copy()\n",
    "        sample_df = self.sample_df[pre_start : self.current].copy()\n",
    "        df.loc[pre_start, \"prior_group_responses\"] = (\n",
    "            \"[\" + \",\".join(self.pre_user_answer_list) + \"]\"\n",
    "        )\n",
    "        df.loc[pre_start, \"prior_group_answers_correct\"] = (\n",
    "            \"[\" + \",\".join(self.pre_answered_correctly_list) + \"]\"\n",
    "        )\n",
    "        self.pre_user_answer_list = user_answer_list\n",
    "        self.pre_answered_correctly_list = answered_correctly_list\n",
    "        return df, sample_df\n",
    "\n",
    "    def __next__(self):\n",
    "        added_user = set()\n",
    "        pre_start = self.current\n",
    "        pre_added_user = -1\n",
    "        pre_task_container_id = -1\n",
    "\n",
    "        user_answer_list = []\n",
    "        answered_correctly_list = []\n",
    "        while self.current < self.len:\n",
    "            crr_user_id = self.user_id[self.current]\n",
    "            crr_task_container_id = self.task_container_id[self.current]\n",
    "            crr_content_type_id = self.content_type_id[self.current]\n",
    "            if crr_content_type_id == 1:\n",
    "                # no more than one task_container_id of \"questions\" from any single user\n",
    "                # so we only care for content_type_id == 0 to break loop\n",
    "                user_answer_list.append(self.user_answer[self.current])\n",
    "                answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "                self.current += 1\n",
    "                continue\n",
    "            if crr_user_id in added_user and (\n",
    "                (crr_user_id != pre_added_user)\n",
    "                or (crr_task_container_id != pre_task_container_id)\n",
    "            ):\n",
    "                # known user(not prev user or differnt task container)\n",
    "                return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "            if len(added_user) == self.max_user:\n",
    "                if (\n",
    "                    crr_user_id == pre_added_user\n",
    "                    and crr_task_container_id == pre_task_container_id\n",
    "                ):\n",
    "                    user_answer_list.append(self.user_answer[self.current])\n",
    "                    answered_correctly_list.append(\n",
    "                        self.answered_correctly[self.current]\n",
    "                    )\n",
    "                    self.current += 1\n",
    "                    continue\n",
    "                else:\n",
    "                    return self.fix_df(\n",
    "                        user_answer_list, answered_correctly_list, pre_start\n",
    "                    )\n",
    "            added_user.add(crr_user_id)\n",
    "            pre_added_user = crr_user_id\n",
    "            pre_task_container_id = crr_task_container_id\n",
    "            user_answer_list.append(self.user_answer[self.current])\n",
    "            answered_correctly_list.append(self.answered_correctly[self.current])\n",
    "            self.current += 1\n",
    "        if pre_start < self.current:\n",
    "            return self.fix_df(user_answer_list, answered_correctly_list, pre_start)\n",
    "        else:\n",
    "            raise StopIteration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.21 s, sys: 96.8 ms, total: 2.3 s\n",
      "Wall time: 2.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "inference_dataset = InferenceRIIDDataset(hdf5_file=\"train_no_lec_feats.h5\",)\n",
    "iter_test = Iter_Valid(valid,max_user=1000)\n",
    "predicted = []\n",
    "def set_predict(df):\n",
    "    predicted.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inference_dataset = InferenceRIIDDataset(hdf5_file=\"train_no_lec_feats.h5\",)\n",
    "iter_test = Iter_Valid(valid,max_user=1000)\n",
    "predicted = []\n",
    "def set_predict(df):\n",
    "    predicted.append(df)\n",
    "\n",
    "pbar = tqdm(total=len(valid))\n",
    "previous_test_df = None\n",
    "for (current_test, current_prediction_df) in iter_test:\n",
    "    if previous_test_df is not None:\n",
    "        previous_test_df[\"answered_correctly\"] = eval(current_test[\"prior_group_answers_correct\"].iloc[0])\n",
    "        previous_test_df[~previous_test_df.content_type_id].groupby(\n",
    "            \"user_id\"\n",
    "        ).answered_correctly.apply(inference_dataset.update_answered_correctly)\n",
    "        \n",
    "\n",
    "    # your feature extraction and model training code here\n",
    "    current_test = preprocess_df(current_test)\n",
    "    previous_test_df = current_test.copy()\n",
    "    \n",
    "    current_test = current_test[~current_test.content_type_id].copy()\n",
    "\n",
    "    # add current to cache\n",
    "    current_test[\n",
    "        [\"row_id\", \"user_id\", \"content_id\", \"timestamp\", \"answered_correctly\", \"prior_question_elapsed_time\"]\n",
    "    ].groupby(\"user_id\").apply(\n",
    "        lambda user_rows: inference_dataset.update_user_rows(user_rows)\n",
    "    )\n",
    "    \n",
    "    # your prediction code here\n",
    "    current_test = predict_for_df(current_test)\n",
    "    set_predict(current_test.loc[:, [\"row_id\", \"answered_correctly\"]])\n",
    "    pbar.update(len(current_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation auc: 0.8077434878428923\n"
     ]
    }
   ],
   "source": [
    "#validation score\n",
    "y_pred = pd.concat(predicted).answered_correctly\n",
    "y_true = valid[valid.content_type_id == 0].answered_correctly[:len(y_pred)]\n",
    "\n",
    "print('validation auc:',roc_auc_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}