diff --git a/README.md b/README.md index 7a326f3c..1b645d91 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ Users can replace the environment with any real-world problems. ## Tutorials 1. The first tutorial of Pearl focuses on recommender systems. We derived a small contrived recommender system environment using the MIND dataset (Wu et al. 2020). More details in https://github.com/facebookresearch/Pearl/tree/main/pearl/tutorials/single_item_recommender_system_example/demo.ipynb +2. The second tutorial of Pearl focuses on contextual bandit algorithms and their implementation using Pearl library. We designed a contextual bandit environment based on UCI dataset and tested the performance of neural implementations of SquareCB, LinUCB, and LinTS. More details in https://github.com/facebookresearch/Pearl/tree/main/pearl/tutorials/contextual_bandits/contextual_bandits_tutorial.ipynb + More tutorials coming in 2024. ## Design and Features diff --git a/pearl/tutorials/contextual_bandits/cb_experiments_tutorial.png b/pearl/tutorials/contextual_bandits/cb_experiments_tutorial.png new file mode 100644 index 00000000..46536604 Binary files /dev/null and b/pearl/tutorials/contextual_bandits/cb_experiments_tutorial.png differ diff --git a/pearl/tutorials/contextual_bandits/contextual_bandits_tutorial.ipynb b/pearl/tutorials/contextual_bandits/contextual_bandits_tutorial.ipynb new file mode 100644 index 00000000..20e0d835 --- /dev/null +++ b/pearl/tutorials/contextual_bandits/contextual_bandits_tutorial.ipynb @@ -0,0 +1,1197 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8NNfwWXGvn_o" + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YY0f_c_WLNGP" + }, + "source": [ + "## Installation\n", + "If you haven't installed Pearl, please make sure you install Pearl with the following cell. Otherwise, you can skip the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1uLHbYlegKX-", + "outputId": "7b7ce54c-0dde-49bc-cc77-23519fd19027" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[33mWARNING: Skipping Pearl as it is not installed.\u001b[0m\u001b[33m\n", + "\u001b[0mCloning into 'Pearl'...\n", + "remote: Enumerating objects: 4759, done.\u001b[K\n", + "remote: Counting objects: 100% (972/972), done.\u001b[K\n", + "remote: Compressing objects: 100% (161/161), done.\u001b[K\n", + "remote: Total 4759 (delta 877), reused 826 (delta 811), pack-reused 3787\u001b[K\n", + "Receiving objects: 100% (4759/4759), 12.55 MiB | 13.86 MiB/s, done.\n", + "Resolving deltas: 100% (3207/3207), done.\n", + "/content/Pearl\n", + "Processing /content/Pearl\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: gym in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (0.25.2)\n", + "Collecting gymnasium[accept-rom-license,atari,mujoco] (from Pearl==0.1.0)\n", + " Downloading gymnasium-0.29.1-py3-none-any.whl (953 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m953.9/953.9 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (1.23.5)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (3.7.1)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (1.5.3)\n", + "Collecting parameterized (from Pearl==0.1.0)\n", + " Downloading parameterized-0.9.0-py2.py3-none-any.whl (20 kB)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.31.0)\n", + "Collecting mujoco (from Pearl==0.1.0)\n", + " Downloading mujoco-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.3/5.3 MB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.1.0+cu121)\n", + "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (0.16.0+cu121)\n", + "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (from Pearl==0.1.0) (2.1.0+cu121)\n", + "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from gym->Pearl==0.1.0) (2.2.1)\n", + "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from gym->Pearl==0.1.0) (0.0.8)\n", + "Requirement already satisfied: typing-extensions>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (4.5.0)\n", + "Collecting farama-notifications>=0.0.1 (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", + " Downloading Farama_Notifications-0.0.4-py3-none-any.whl (2.5 kB)\n", + "Collecting autorom[accept-rom-license]~=0.4.2 (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", + " Downloading AutoROM-0.4.2-py3-none-any.whl (16 kB)\n", + "Collecting shimmy[atari]<1.0,>=0.1.0 (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", + " Downloading Shimmy-0.2.1-py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: imageio>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (2.31.6)\n", + "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from mujoco->Pearl==0.1.0) (1.4.0)\n", + "Requirement already satisfied: etils[epath] in /usr/local/lib/python3.10/dist-packages (from mujoco->Pearl==0.1.0) (1.6.0)\n", + "Collecting glfw (from mujoco->Pearl==0.1.0)\n", + " Downloading glfw-2.6.5-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl (211 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.8/211.8 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pyopengl in /usr/local/lib/python3.10/dist-packages (from mujoco->Pearl==0.1.0) (3.1.7)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (1.2.0)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (4.47.2)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (1.4.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (23.2)\n", + "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (9.4.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->Pearl==0.1.0) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->Pearl==0.1.0) (2023.3.post1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->Pearl==0.1.0) (2023.11.17)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (3.13.1)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (3.1.3)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (2023.6.0)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch->Pearl==0.1.0) (2.1.0)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (8.1.7)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0) (4.66.1)\n", + "Collecting AutoROM.accept-rom-license (from autorom[accept-rom-license]~=0.4.2->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", + " Downloading AutoROM.accept-rom-license-0.6.1.tar.gz (434 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m434.7/434.7 kB\u001b[0m \u001b[31m22.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->Pearl==0.1.0) (1.16.0)\n", + "Collecting ale-py~=0.8.1 (from shimmy[atari]<1.0,>=0.1.0->gymnasium[accept-rom-license,atari,mujoco]->Pearl==0.1.0)\n", + " Downloading ale_py-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m27.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: importlib_resources in /usr/local/lib/python3.10/dist-packages (from etils[epath]->mujoco->Pearl==0.1.0) (6.1.1)\n", + "Requirement already satisfied: zipp in /usr/local/lib/python3.10/dist-packages (from etils[epath]->mujoco->Pearl==0.1.0) (3.17.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->Pearl==0.1.0) (2.1.4)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->Pearl==0.1.0) (1.3.0)\n", + "Building wheels for collected packages: Pearl, AutoROM.accept-rom-license\n", + " Building wheel for Pearl (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for Pearl: filename=Pearl-0.1.0-py3-none-any.whl size=189176 sha256=c714ccb4fffa67d9face168ac466ca3c003a4058b0c818254fd2909f4bcd2202\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-it8ogr97/wheels/83/80/1d/d9211ba70ee392341daf21a07252739e0cb2af9f95439a28cd\n", + " Building wheel for AutoROM.accept-rom-license (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for AutoROM.accept-rom-license: filename=AutoROM.accept_rom_license-0.6.1-py3-none-any.whl size=446660 sha256=5ada5f9c3d6e5fee0849255d2669da5fbc5011ab81e5edbc01a5192297463a92\n", + " Stored in directory: /root/.cache/pip/wheels/6b/1b/ef/a43ff1a2f1736d5711faa1ba4c1f61be1131b8899e6a057811\n", + "Successfully built Pearl AutoROM.accept-rom-license\n", + "Installing collected packages: glfw, farama-notifications, parameterized, gymnasium, ale-py, shimmy, AutoROM.accept-rom-license, autorom, mujoco, Pearl\n", + "Successfully installed AutoROM.accept-rom-license-0.6.1 Pearl-0.1.0 ale-py-0.8.1 autorom-0.4.2 farama-notifications-0.0.4 glfw-2.6.5 gymnasium-0.29.1 mujoco-3.1.1 parameterized-0.9.0 shimmy-0.2.1\n", + "/content\n" + ] + } + ], + "source": [ + "%pip uninstall Pearl -y\n", + "%rm -rf Pearl\n", + "!git clone https://github.com/facebookresearch/Pearl.git\n", + "%cd Pearl\n", + "%pip install .\n", + "%cd .." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nCSJf8nALNGQ" + }, + "source": [ + "## Import Modules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vcb70ZC_h3OA" + }, + "outputs": [], + "source": [ + "from pearl.utils.functional_utils.experimentation.set_seed import set_seed\n", + "from pearl.action_representation_modules.one_hot_action_representation_module import OneHotActionTensorRepresentationModule\n", + "from pearl.replay_buffers.sequential_decision_making.fifo_off_policy_replay_buffer import FIFOOffPolicyReplayBuffer\n", + "from pearl.utils.functional_utils.train_and_eval.online_learning import online_learning\n", + "from pearl.pearl_agent import PearlAgent\n", + "from pearl.utils.scripts.cb_benchmark.cb_download_benchmarks import download_uci_data\n", + "from pearl.utils.instantiations.environments.contextual_bandit_uci_environment import (\n", + " SLCBEnvironment,\n", + ")\n", + "from pearl.policy_learners.exploration_modules.contextual_bandits.squarecb_exploration import SquareCBExploration\n", + "from pearl.policy_learners.exploration_modules.contextual_bandits.ucb_exploration import (\n", + " UCBExploration,\n", + ")\n", + "from pearl.policy_learners.exploration_modules.contextual_bandits.thompson_sampling_exploration import (\n", + " ThompsonSamplingExplorationLinear,\n", + ")\n", + "from pearl.policy_learners.contextual_bandits.neural_bandit import NeuralBandit\n", + "from pearl.policy_learners.contextual_bandits.neural_linear_bandit import (\n", + " NeuralLinearBandit,\n", + ")\n", + "import torch\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "\n", + "set_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A2756RO8LNGR" + }, + "source": [ + "## Load Environment\n", + "The environment which underlies the experiments to follow is a contextual bandits environment we added to Pearl that allows to use UCI datasets (https://archive.ics.uci.edu/datasets) to built environmens to test contextual bandits algorithms.\n", + "\n", + "The UCI datasets span a wide variety of prediction tasks. We use these tasks to construct a contexual bandit environment in which an agent recives an expected reward of 1 if it correctly label a data point and 0 otherwise. Pearl library currently support the following datasets: pendigits, letter, satimage, yeast. Additional ones can be readily added.\n", + "\n", + "In the following experiment will test different types of contextual bandits algorithms on the pendigits UCI dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "g1VHtmldi3A2", + "outputId": "838f8462-7083-4ce4-ebdf-ed6fc24f436b" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", + " and should_run_async(code)\n", + "/usr/local/lib/python3.10/dist-packages/pearl/utils/instantiations/environments/contextual_bandit_uci_environment.py:50: DeprecationWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)`\n", + " df.iloc[:, 0] = df.iloc[:, 0].cat.codes\n" + ] + } + ], + "source": [ + "# load environment\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "# Download UCI dataset if doesn't exist\n", + "uci_data_path = \"./utils/instantiations/environments/uci_datasets\"\n", + "if not os.path.exists(uci_data_path):\n", + " os.makedirs(uci_data_path)\n", + " download_uci_data(data_path=uci_data_path)\n", + "\n", + "# Built CB environment using the pendigits UCI dataset\n", + "pendigits_uci_dict = {\n", + " \"path_filename\": os.path.join(uci_data_path, \"pendigits/pendigits.tra\"),\n", + " \"action_embeddings\": \"discrete\",\n", + " \"delim_whitespace\": False,\n", + " \"ind_to_drop\": [],\n", + " \"target_column\": 16,\n", + "}\n", + "env = SLCBEnvironment(**pendigits_uci_dict)\n", + "\n", + "# experiment code\n", + "number_of_steps = 10000\n", + "record_period = 400" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UYIoDAGSLNGR" + }, + "source": [ + "\n", + "\n", + "```\n", + "# This is formatted as code\n", + "```\n", + "\n", + "## Contextual Bandits learners\n", + "The following code sections show how to implement the neural versions of SquareCB, LinUCB and LinTS with the Pearl library.\n", + "\n", + "## Contextual Bandits learners: SquareCB\n", + "\n", + "The SquareCB algorithm requires only a regression model with which it learns the reward function. Given the reward model, SquareCB executes the following policy.\n", + "$$\n", + "\\widehat{a}_*\\in \\arg\\max_a\\widehat{r}(x,a)\\\\\n", + "\\widehat{r}_*\\in \\max_a\\widehat{r}(x,a)\\\\\n", + "\\text{If $a\\neq \\widehat{a}_*$}: \\pi(a,x)= \\frac{1}{A + \\gamma (\\widehat{r}_* - \\widehat{r}(x,a))}\\\\\n", + "\\text{If $a= \\widehat{a}_*$}: \\pi(a,x) = 1-\\sum_{a'\\neq \\widehat{a}_*}\\pi(a',x).\n", + "$$\n", + "This exploratiative policy, that balances exploration and exploitation in an intelligent way.\n", + "\n", + "To use the SquareCB algrorithm in Pearl we set the policy learner as NeuralBandit. NeuralBandit is a base class and supports the estimation of the reward function with a neural architecture. With access to an estimated reward model, we then instantiate the exploration module with SquareCBExploration module.\n", + "\n", + "To further highlight the versatility of the modular design of Pearl, we use the OneHotActionTensorRepresentationModule as the action representation module. Namley, when the action set is a finite number of elementes\n", + "$\n", + "\\{1,2,.,,N\\}\n", + "$\n", + "as a one-hot vector.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "kulkpFAvnOQx", + "outputId": "7c93f477-d8ec-4a5a-91d9-1c7fd51f6557" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "episode 100, step 100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.15010957419872284\n", + "episode 200, step 200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.08203082531690598\n", + "episode 300, step 300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.03324655815958977\n", + "episode 400, step 400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0116292238235474\n", + "episode 500, step 500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8727301955223083\n", + "episode 600, step 600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0681545734405518\n", + "episode 700, step 700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.10991505533456802\n", + "episode 800, step 800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.16739219427108765\n", + "episode 900, step 900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.205057144165039\n", + "episode 1000, step 1000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9941626191139221\n", + "episode 1100, step 1100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9581815004348755\n", + "episode 1200, step 1200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.03158580884337425\n", + "episode 1300, step 1300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0983383655548096\n", + "episode 1400, step 1400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.017097746953368187\n", + "episode 1500, step 1500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9298166632652283\n", + "episode 1600, step 1600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.08266790956258774\n", + "episode 1700, step 1700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.11597150564193726\n", + "episode 1800, step 1800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.163917899131775\n", + "episode 1900, step 1900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0052275657653809\n", + "episode 2000, step 2000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0892176628112793\n", + "episode 2100, step 2100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9310503602027893\n", + "episode 2200, step 2200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8945373296737671\n", + "episode 2300, step 2300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9437602758407593\n", + "episode 2400, step 2400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9521269798278809\n", + "episode 2500, step 2500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8708449602127075\n", + "episode 2600, step 2600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9630036354064941\n", + "episode 2700, step 2700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9632565379142761\n", + "episode 2800, step 2800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.170562744140625\n", + "episode 2900, step 2900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9632094502449036\n", + "episode 3000, step 3000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.006435513496399\n", + "episode 3100, step 3100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9676222801208496\n", + "episode 3200, step 3200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9731466770172119\n", + "episode 3300, step 3300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9930073022842407\n", + "episode 3400, step 3400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9663928747177124\n", + "episode 3500, step 3500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8308923840522766\n", + "episode 3600, step 3600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.021346502006053925\n", + "episode 3700, step 3700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0449684858322144\n", + "episode 3800, step 3800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.048456072807312\n", + "episode 3900, step 3900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.963261067867279\n", + "episode 4000, step 4000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.95270836353302\n", + "episode 4100, step 4100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9612334966659546\n", + "episode 4200, step 4200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1417059898376465\n", + "episode 4300, step 4300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.038852766156196594\n", + "episode 4400, step 4400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9254987239837646\n", + "episode 4500, step 4500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1320136785507202\n", + "episode 4600, step 4600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8077775835990906\n", + "episode 4700, step 4700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0377964973449707\n", + "episode 4800, step 4800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0698367357254028\n", + "episode 4900, step 4900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.974791944026947\n", + "episode 5000, step 5000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0970743894577026\n", + "episode 5100, step 5100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.17044997215271\n", + "episode 5200, step 5200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9542230367660522\n", + "episode 5300, step 5300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8773468732833862\n", + "episode 5400, step 5400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0972086191177368\n", + "episode 5500, step 5500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0788975954055786\n", + "episode 5600, step 5600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9238188862800598\n", + "episode 5700, step 5700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.7602724432945251\n", + "episode 5800, step 5800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.881024181842804\n", + "episode 5900, step 5900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9699533581733704\n", + "episode 6000, step 6000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1488981246948242\n", + "episode 6100, step 6100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8956410884857178\n", + "episode 6200, step 6200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1703811883926392\n", + "episode 6300, step 6300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9063166975975037\n", + "episode 6400, step 6400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9786220788955688\n", + "episode 6500, step 6500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9373737573623657\n", + "episode 6600, step 6600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.983406662940979\n", + "episode 6700, step 6700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.995124101638794\n", + "episode 6800, step 6800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1568156480789185\n", + "episode 6900, step 6900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8350116610527039\n", + "episode 7000, step 7000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1857116222381592\n", + "episode 7100, step 7100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0497853755950928\n", + "episode 7200, step 7200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9718841314315796\n", + "episode 7300, step 7300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.053848385810852\n", + "episode 7400, step 7400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9327347874641418\n", + "episode 7500, step 7500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8159596920013428\n", + "episode 7600, step 7600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9617915749549866\n", + "episode 7700, step 7700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9615158438682556\n", + "episode 7800, step 7800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0029709339141846\n", + "episode 7900, step 7900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.060897946357727\n", + "episode 8000, step 8000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.974728524684906\n", + "episode 8100, step 8100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.048927903175354\n", + "episode 8200, step 8200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9603986740112305\n", + "episode 8300, step 8300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0292688608169556\n", + "episode 8400, step 8400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9123027324676514\n", + "episode 8500, step 8500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8954773545265198\n", + "episode 8600, step 8600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0856765508651733\n", + "episode 8700, step 8700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0120528936386108\n", + "episode 8800, step 8800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9945083260536194\n", + "episode 8900, step 8900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0352985858917236\n", + "episode 9000, step 9000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9688308238983154\n", + "episode 9100, step 9100, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0430936813354492\n", + "episode 9200, step 9200, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.839612603187561\n", + "episode 9300, step 9300, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.008146047592163\n", + "episode 9400, step 9400, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0851472616195679\n", + "episode 9500, step 9500, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0283199548721313\n", + "episode 9600, step 9600, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8875988721847534\n", + "episode 9700, step 9700, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8883333802223206\n", + "episode 9800, step 9800, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9826716184616089\n", + "episode 9900, step 9900, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9570529460906982\n", + "episode 10000, step 10000, agent=PearlAgent with NeuralBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.032433271408081\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVGklEQVR4nO3deVxU5f4H8M/MADPsi8CwOIjigigKYhKarSgu17TMzCyNzMr0qtFqlt5rt6i816XyZtcy/bVpltmiYUaaS6SJKy6ooILIKsKwLzPP7w9kdHQ0wZk5M/J5v17zEs6cM/M9Z8z59JxnkQkhBIiIiIjIiFzqAoiIiIhsEUMSERERkQkMSUREREQmMCQRERERmcCQRERERGQCQxIRERGRCQxJRERERCY4SF2Aten1epw9exbu7u6QyWRSl0NERETXQQiBiooKBAUFQS63ThtPmwtJZ8+ehUajkboMIiIiaoXc3Fy0b9/eKu/V5kKSu7s7gKaL7OHhIXE1REREdD20Wi00Go3he9wa2lxIar7F5uHhwZBERERkZ6zZVYYdt4mIiIhMYEgiIiIiMoEhiYiIiMiENtcn6XrpdDo0NDRIXQaZgaOjIxQKhdRlEBGRnWFIuowQAgUFBSgrK5O6FDIjLy8vBAQEcG4sIiK6bgxJl2kOSP7+/nBxceGXqp0TQqC6uhpFRUUAgMDAQIkrIiIieyFpSNq6dSvmz5+P9PR05Ofn49tvv8WoUaOuecyWLVuQlJSEQ4cOQaPR4NVXX8Vjjz1mlnp0Op0hILVr184sr0nSc3Z2BgAUFRXB39+ft96IiOi6SNpxu6qqCr1798aSJUuua/+TJ09i+PDhuOuuu7Bv3z7MnDkTTzzxBDZu3GiWepr7ILm4uJjl9ch2NH+m7GdGRETXS9KWpKFDh2Lo0KHXvf/SpUvRsWNH/Oc//wEAdO/eHdu3b8fChQuRkJBgtrp4i+3mw8+UiIhayq6mAEhLS0N8fLzRtoSEBKSlpV31mLq6Omi1WqMHERER0V+xq5BUUFAAtVpttE2tVkOr1aKmpsbkMcnJyfD09DQ8uLgtERERXQ+7CkmtMWvWLJSXlxseubm5UpdEREREdsCuQlJAQAAKCwuNthUWFsLDw8MwgulySqXSsJjtzb6obXFxMaZMmYKQkBAolUoEBAQgISEBO3bskLq061ZQUIC///3v6NSpE5RKJTQaDUaMGIHU1FTDPqGhoZDJZJDJZFAoFAgKCsKkSZNw/vx5CSsnopuVTi9QVFGL+ka91KWQldnVPElxcXHYsGGD0bZNmzYhLi5Ooopsy+jRo1FfX4+VK1eiU6dOKCwsRGpqKs6dOydpXfX19XBycvrL/U6dOoUBAwbAy8sL8+fPR2RkJBoaGrBx40ZMnToVR48eNew7b948TJ48GTqdDseOHcOTTz6J6dOn49NPP7XkqRDRTag5BOWW1uDM+WqcOd/0Z25pDc6UVSO/rBaNegGZDAjwUEHj7YL23s5ND5+mnzXeLgj0VMFBYfttD0II1DToUF7T0PSoboC2thF6IaByVMDZUQGVoxwqRwVUDgqonC7+7KiQtamBMJKGpMrKSpw4ccLw+8mTJ7Fv3z74+PggJCQEs2bNQl5eHv7v//4PAPD000/j/fffx4svvojHH38cv/76K7766iusX7/eYjU2/2WSgrOj4rr/MpaVlWHbtm3YsmUL7rjjDgBAhw4d0K9fP8M+x48fx6RJk7Br1y506tQJixcvxuDBgw3zU23ZsgV33XUXzp8/Dy8vLwDAvn37EB0djZMnTyI0NBTnzp3DtGnTsHXrVpw/fx5hYWF45ZVXMG7cOMP73HnnnejZsyccHBzw2WefITIyEps3b0ZGRgZeeOEFbNu2Da6urhg8eDAWLlwIX19fAMAzzzwDmUyGXbt2wdXV1fB6PXr0wOOPP250vu7u7ggICAAABAcHY+LEifjyyy9bfpGJ6Kan1wsUV9ZdDD6GIFSD3PPVOFtWgwad+MvXEQLIL69Ffnktdp268nmFXIYAD1VTaPJpDlIuht8DPFRQyFsfMIQQaNQLNOoEGvV6NOouCzsXHtrLfr/8OW1NI+p1rWsVk8twIUQ1P+RGPzs7KqB0VCDMzw1Jg7q2+lxthaQhaffu3bjrrrsMvyclJQEAJk6ciBUrViA/Px85OTmG5zt27Ij169fj2WefxeLFi9G+fXt89NFHZh3+f7maBh0i5phnHqaWOjwvAS5O1/cRubm5wc3NDevWrcOtt94KpVJp9Lxer8f9998PtVqNnTt3ory8HDNnzmxxTbW1tYiJicFLL70EDw8PrF+/Ho8++ijCwsKMAtnKlSsxZcoUw62+srIy3H333XjiiSewcOFC1NTU4KWXXsKDDz6IX3/9FaWlpUhJScEbb7xhFJCaNYc2U/Ly8vDDDz8gNja2xedDRDfmbFkN3k09juySKgyOUOP+Pu3h4/rXLceWVNeoQ+qRIqzbm4fjRZXIO1/zl6HAQS5DkJezoYVI4+2C9j5NIUfj7QI/dyXOV9cjt7TaEK6ag9aZ0mqcKatBfaMeeWU1yCurwc6TpSbfI9BLhQAPFfQCFwKP/mLouSwAGZ7XN/2s0/91kGsJhVwGT2dHeDo7wsPZEQoZUNugR22jDrX1OtQ26lHboENNgw7iwlvrBVBVr0NV/bUbD2I6eN8UIUkmhDDvVbdxWq0Wnp6eKC8vv6J/Um1tLU6ePImOHTtCpVIBAKrrG+0iJAHAN998g8mTJ6OmpgZ9+vTBHXfcgYceegi9evXCzz//jOHDh+P06dMICgoCAKSkpGDo0KEtakky5W9/+xvCw8Px73//G0BTS5JWq8WePXsM+/zrX//Ctm3bjCb+PHPmDDQaDTIzM1FWVobY2FisXbsW99133zXPMzQ0FPn5+XB0dIROp0NtbS1iY2ORkpJy1TBl6rMlotYrr27Af387gU92nDLqq+OokGFwjwCM7avBbZ19Ib+BlpOWOpKvxVe7c7Fubx7OVxtPHKuQyxDoqTK07lx+y+xGW3n0eoGSyjpDeGoOUy1trWoph8uCjudVHkbPuTT96ep0fXcrhBCo1+mbAlSD7sJDjxrDz7rLntPBx02Je3sHmfVcr/X9bSl21SdJCs6OChyeZ7mWqr9675YYPXo0hg8fjm3btuGPP/7ATz/9hHfeeQcfffQRysvLodFoDAEJQKv6cul0Orz55pv46quvkJeXh/r6etTV1V0xS3lMTIzR7/v378fmzZvh5uZ2xWtmZWXBx8enRXW88MILeOyxxyCEQG5uLl555RUMHz4cW7du5bIjdNOqb9TjWGEFQn1d4aaU5p/v2gYdPk07jfc3n0B5TVMQie3og0ERany//ywOnCnH+gP5WH8gH8FezniwrwZj+rZHkJfpwTU3qrymAd/vP4uv/szFwbxyw/YADxUeiGmP/p3bWaW/kFwug7+HCv4eKsR0uPL5S/s9FVfUQSEHHORyOChkl/wpg4NCfuHPpu2OChkUchkcFfKmPy/se+k2S5PJZFA6KKB0UMDT2dHi72dLGJL+gkwma1FrjtRUKhUGDRqEQYMG4bXXXsMTTzyBuXPnGm5lXotc3vQPyKWNi5cv4zF//nwsXrwYixYtQmRkJFxdXTFz5kzU19cb7Xf5LbPKykqMGDECb7/99hXvGxgYiLq6OshkMqPO2dfi6+uLzp07AwC6dOmCRYsWIS4uDps3b75iwlEie3a+qh6bM4vwy5FCbD1Wgsq6Rrg6KTAyOhgP9wtBz2BPq9Sh1wus25eH//x8DHllTfPSdVW7YdbQ7rizmx9kMhmeGNgJh882teas3XMGeWU1WPjLMSxKPYY7uvrhoVs0uDtcDSeHGwsrer1AWvY5fLU7FykZBai70JLlqJBhUIQaY/pqcHsXP6sEiOvV1JLljEBPy4RFsgz7+fanVomIiMC6devQvXt35ObmIj8/H4GBgQCAP/74w2hfPz8/AEB+fj68vb0BNN1uu9SOHTswcuRIPPLIIwCa+jodO3YMERER16yjT58++OabbxAaGgoHhyv/2rm6uiIhIQFLlizB9OnTrwhZZWVl1+yX1Nx6dLVJRYnshRACWcVVSD1SiF+OFCL99Hlc2hVF5ShHVb0OX+zMwRc7c9Bb44XxsSEY0SsIzk6WaUXdeqwYb/10FIfzm1YsCPBQIWlwV4zu0/6KIBIR5IF/3NsDLw8Nx8ZDBVi1Kxdp2eewJbMYWzKL0c7VCaNj2uPBvhp09r+yZfla8spq8PXuM1iTnosz5y/+tx4e4I4H+2owKjpY8v5QdHNhSLpJnDt3DmPGjMHjjz+OXr16wd3dHbt378Y777yDkSNHIj4+Hl27dsXEiRMxf/58aLVazJ492+g1OnfuDI1Gg3/84x944403cOzYMcM6ec26dOmCr7/+Gr///ju8vb2xYMECFBYW/mVImjp1KpYtW4Zx48bhxRdfhI+PD06cOIFVq1bho48+gkKhwJIlSzBgwAD069cP8+bNQ69evdDY2IhNmzbhgw8+wJEjRwyvV1FRgYKCAsPtthdffBF+fn7o37+/+S4qkZU06vTYffo8fjlciNSjRThZUmX0fHiAOwZFqHFPdzUigz2x62QpPt95GhsPFWB/bhn255bh9R8PY3Sf9hgfG4Iuanez1JWRV463fjqK7SdKAADuKgdMuTMMjw/oCNVfdAdQOSowMioYI6OCcaqkCl/tzsWa9DMorqjD/7Zm439bs3FLqDfG3hKCYZEBV22xr23QYdPhQny1OxfbT5QYOhC7qxwwMioID/bVIDLYs00NSyfrYUi6Sbi5uSE2NhYLFy5EVlYWGhoaoNFoMHnyZLzyyiuQy+X49ttvMWnSJPTr1w+hoaF49913MWTIEMNrODo64ssvv8SUKVPQq1cv3HLLLfjXv/6FMWPGGPZ59dVXkZ2djYSEBLi4uODJJ5/EqFGjUF5ebqosg6CgIOzYsQMvvfQSBg8ejLq6OnTo0AFDhgwx3Obr1KkT9uzZgzfeeAPPPfcc8vPz4efnh5iYGHzwwQdGrzdnzhzMmTMHQFML2C233IKff/4Z7dq1M9clJbIobW0Dfsssxi9HCrEls9jQvwdoum10a6d2GBShxt3h/mjvbdznLy6sHeLC2qGksg5rdp/BF7tOI7e0Bit+P4UVv59Cv1AfjL81BEN6BkDp0PLWpdzSavz750x8t+8sAMBJIcejcR0w7a7O8G5FS02oryteHBKOpEFdsTmzGKv/zMXmzCL8eeo8/jx1Hv/4/hDujQrCQ7dcDDwZeeVYszsX6/adNbo2/cPa4cG+GgzpGfCXQY3oRnF02yXa4ggomUxmGN12M2uLny3Znpxz1fjlwm20XSdL0XjJfTRvF0fcFe6P+O5qDOziC3fV9XeQ1esFtp0owRc7T+OXI0WGoeI+rk54IKY9xvULQUffK6fWuNz5qnq8v/kEPk07bRgyPyoqCM8N7gaNj8tfHN0yhdpafJ1+Bl/tzsXpc9WG7d0DPSCXAYfOXlyMPMhThQf6ajAmpr3Z6yD7wdFtREQ3Gb1e4MOt2fh27xkcK6w0eq6zvxvu6d4UjPqEeLe6o7FcLsMdXf1wR1c/FJTXYvWfuVj1Zw7yy2sNt7Zu6+yL8bEhiI9Qw/GyUV61DTos33ESH2zJQkVtIwBgYBdfvDQk3GIdw9UeKky9qzOm3BGGP06ew+o/c/FTRgGOXOj35KSQY3APNR7sq8GAzr421Qmb2g6GJCIiC1q6NQvvpGQCaBrh1C/UxxCMQq+jdaelAjxVmBHfBVPvCsOWzGJ8vvM0thwrxvYTJdh+ogR+7kqM7avBQ/00CPR0xjd7zmDBz8dQoK0F0NSSM2toOG7v6mf22kyRy2XoH+aL/mG++Gd1PTYcLAAADIsMgJcLO2GTtHi77RK8JXPz4mdLUtibcx5jlqahUS/w/OCuePTWUHi6WH+emdzS6gutS7koqawDAMM6ZPnlTeEo2MsZzyd0xcjewVadAJLoevF2m41oY7mxTeBnStamrW3A9FV70agX+FuvQEy9q7NkI7A0Pi54PqEbZsR3wabDhfhiZw62nyhBfnktPJ0dMe2uzng0rgM7QhNdhiHpEo6OTf+HV11dDWdnTvh1M6mubuoY2vwZE1mSEAKvfpuB3NIatPd2xhv3RdrEEHVHhRzDIgMxLDIQJ0uqcCRfiwFhvpK0bhHZA4akSygUCnh5eaGoqAgA4OLiYhP/sFHrCSFQXV2NoqIieHl5cckSsopv9uTh+/1noZDLsPihaJtcyqGjr+t1jXgjassYki4TEBAAAIagRDcHLy8vw2dLZEnZxZWY810GACBpUFfEdPCWuCIiai2GpMvIZDIEBgbC39//inXLyD45OjqyBYmsoq5Rh79/uRfV9TrEdWqHp+8Ik7okIroBDElXoVAo+MVKRC0yPyUTh85q4e3iiIVjozi3D5Gdu7GlmImICACwObMIH20/CQB454HeCPDkVBNE9o4hiYjoBhVV1OL5r/YDACbGdcCgCLXEFRGROTAkERHdAL1e4Lmv9uNcVT3CA9wxa1h3qUsiIjNhSCIiugHLtmVj2/ESqBzleG9cNCdkJLqJMCQREbXS/twyzN/YtC7b3BE90EXtLnFFRGRODElERK1QccmyI8MiA/DQLRqpSyIiM2NIIiJqhTnfHcLpc9UI9nJG8n29ODs/0U2IIYmIqIXW7jmDb/fmQS4DFj8UxbXPiG5SnEySiCS1+1Qp3klp6tejdJTD2VEBlaMCKkc5VI4KODsqoGz+3UEBZ6eLP6ucFE1/XrKvt6uTRddKO1VShdfWNS07MuOerugb6mOx9yIiaTEkEZFkiivq8PRne1BSWWe215TLgKE9AzFpYEf0CTHvumn1jXpMX7UXVfU69Ovog2l3dzbr6xORbWFIIiJJ6PUCSV/tQ0llHbqp3TH9ni6obdChtlGH2gZ908+Ghx41l/xc16hDTb2pffWorGvE+oP5WH8wH31CvDB5YCcM7hFgliVC/vNzJg6cKYensyMWcdkRopseQxIRSeKD37IM8wu9/3C02YbPHy3Q4uNtJ/HdvrPYk1OGKZ/vgcbHGY8P6IgxfTVwU7bun72tx4rx4dZsAMDbo3shyMvZLPUSke2SCSGE1EVYk1arhaenJ8rLy+Hh4SF1OURt0u5TpRj7vz+g0wu880AvPNjX/MPniypq8WnaaXz2x2mcr24AALirHPBwbAge6x+KQM/rDznFFXUYungbSirr8MitIfjXqEiz10tE1ybF9zdDEhFZ1fmqegx/dxvOltfivuhgLHiwt0WHz9fU6/DNnjNYvv0kskuqAAAOchmG9wrE5IGd0DPY85rH6/UCiSv+xG/HitFV7Ybvp93GWbWJJMCQZAUMSUTSEUJg8v/txi9HitDR1xU//P22Vt/+aim9XuDXo0X4aHs2/sguNWyP7eiDJwZ2wj3h/pCb6GP00bZs/Gv9ESgd5Ph+2m3oFsBZtYmkIMX3N/skEZHVLN9xCr8cKYKTQ1M/JGsFJACQy2WIj1AjPkKNjLxyfLQtGz8eyMfOk6XYebIUHX1d8fhtHfFAn/ZwdmpqKTp4phxvpxwFALz2twgGJKI2hi1JRG1ARW0DjhdVIru4Cj2DPRAeYP2/+wfOlGH0B7+jQScwb2QPTIgLtXoNl8svr8HK30/ji52noa1tBAB4uTjikdgOGB3THomf7MKpc9VI6KHG0kdiOKs2kYR4u80KGJLoZlZd34jjhZU4VliB40VNfx4rqMDZ8lrDPgq5DG+P7oUHYtpbrS5tbQP+9u525JRWY0iPAHzwSB+bChxVdY1YszsXy3ecQk5ptdFzgZ4q/DRjILxcnCSqjogA3m4joutU26DDiaJKHC+qQGZBJY4XVuBYUQVyS2uueoy/uxI+rk44WlCB59fsR0llHZ66vZPFw4oQArPWHkROadM6Z28/YHvrnLkqHfDYgI54NC4Umw4X4KNtJ7H79HnIZcDCsVEMSERtFEMSkY07V1mH7SdKjFqITp+rgv4qbcC+bk7o4u+Ormo3dFG7o1uAO7r4u8HLxQl6vcDbKUfx4dZsvPXTURRX1GH2sO4mOyyby5e7crH+QD4c5DK893C0RZcMuVEKuQxDegZiSM9AZOSVA8Bfjn4jopuX5CFpyZIlmD9/PgoKCtC7d2+899576Nevn8l9GxoakJycjJUrVyIvLw/dunXD22+/jSFDhli5aiLr0OsFHliahpMXhq5fysvFEV3VTWGoq9rdEIzauSmv+npyuQyzhnWHr5sSb2w4go+3n0RJZR3mP9AbTg7mX+/6aIEW//zhEADgxSHdzL5MiCUxHBGRpCFp9erVSEpKwtKlSxEbG4tFixYhISEBmZmZ8Pf3v2L/V199FZ999hmWLVuG8PBwbNy4Effddx9+//13REdHS3AGRJb1x8lzOFlSBRcnBUZGBV0IRe7oonaDn5uy1betJt/eCb7uTnhhzQF8t+8sSqvqsfSRGLiacbRZdX0jpn6+B3WNetzZzQ9P3NbJbK9NRGQNknbcjo2NxS233IL3338fAKDX66HRaPD3v/8dL7/88hX7BwUFYfbs2Zg6daph2+jRo+Hs7IzPPvvM5HvU1dWhru7i4plarRYajYYdt8kuPPfVfnyz5wzG9QtB8v3mn+V5S2YRnvl8D6rrdejd3hPLH7vlmi1RLfHCmv1Yk34Gag8lNkwfaLbXJaK2SYqO2+ZvX79O9fX1SE9PR3x8/MVi5HLEx8cjLS3N5DF1dXVQqVRG25ydnbF9+/arvk9ycjI8PT0ND43G/MsfEFlCdX0jfsrIBwCM7hNskfe4s5s/vph8K7xdHLH/TDkeWJqG3MtGd7XG2j1nsCb9DOQyYPFD0QxIRGSXJAtJJSUl0Ol0UKvVRtvVajUKCgpMHpOQkIAFCxbg+PHj0Ov12LRpE9auXYv8/Pyrvs+sWbNQXl5ueOTm5pr1PIgsJSWjANX1OnRo54KYDpbryxOl8cLXU/oj2MsZJ0uqcP8Hv+PwWW2rXy+ruBKvrssAAMy4pytu7dTOXKUSEVmVZCGpNRYvXowuXbogPDwcTk5OmDZtGhITEyGXX/00lEolPDw8jB5E9mDtnjwAwP3R7S0+ZD7Mzw1rn+mP8AB3FFfUYeyHafgj+1yLX6e2QYdpX+xFdb0OcZ3aYdrdnS1QLRGRdUgWknx9faFQKFBYWGi0vbCwEAEBASaP8fPzw7p161BVVYXTp0/j6NGjcHNzQ6dO7BBKN5ezZTXYkVUCALjfQrfaLqf2UGH1U3HoF+qDirpGTFi+CykZV2+lNeWN9UdwJF+Ldq5OWPRQFBQWnFqAiMjSJAtJTk5OiImJQWpqqmGbXq9Hamoq4uLirnmsSqVCcHAwGhsb8c0332DkyJGWLpfIqtbty4MQQL+OPtD4uFjtfT2dHfF/k/phcIQa9Y16TPl8Dz774/R1HfvTwXx8emHfBWOjoPZQ/cURRES2TdLbbUlJSVi2bBlWrlyJI0eOYMqUKaiqqkJiYiIAYMKECZg1a5Zh/507d2Lt2rXIzs7Gtm3bMGTIEOj1erz44otSnQKR2Qkh8E36GQCW67B9LSpHBT54JAbj+oVACODVdRlY9MsxXGsgbG5pNV785gAA4Ok7wnBHVz9rlUtEZDGSzpM0duxYFBcXY86cOSgoKEBUVBRSUlIMnblzcnKM+hvV1tbi1VdfRXZ2Ntzc3DBs2DB8+umn8PLykugMiMzvwJlyZBVXQekgx7DIQElqUMhlePO+nvBzV+Ld1ONY9MtxFFfUYd7InlfcQqtv1GPal3tRUduIPiFeeG5wV0lqJiIyNy5wS2Rj5nyXgf9LO417ewfh3XHST5L66R+nMee7DAgBDOkRgEUPRUHlqDA8/+aGI/jf1mx4qBywYcZAtPe23u1BImo72tQ8SUR0pfpGPb7ffxYAMDqmvcTVNHn01g5Y8nAfOCnkSDlUgInLd6G8pgEA8OvRQvxvazYAYP6Y3gxIRHRTYUgisiGbM4tQVt0Af3clbuvsK3U5BsMiA7Hi8VvgrnTAzpOlGPthGg6cKcNzX+0HADzWPxQJPUyPSiUislcMSUQ2pLnD9n3RwTY3fL5/mC9WPXUrfN2UOFpQgXvf34Hz1Q3oGeyBWcPCpS6PiMjsGJKIbERpVT02ZxYBAO7vYxu32i7XI8gTa6f0R2i7pttqbkoHvD+uD5QOir84kojI/kg6uo2ILvph/1k06AR6BHmgW4C71OVcVUg7F3w9pT/+tzUbgyLUCPV1lbokIiKLYEgishFr9zTPjWSbrUiX8nVT4pVh3aUug4jIoni7jcgGnCiqwP4z5XCQy3BvVJDU5RARERiSiGzCNxcWs72zmx983ZQSV0NERABDEpHkdHqBby+EJFvtsE1E1BYxJBFJLC3rHAq0tfBQOeCe7v5Sl0NERBcwJBFJ7JsLHbZH9A7iUHoiIhvCkEQkocq6RqRkFACwnWVIiIioCUMSkYR+OpiPmgYdOvq6IlrjJXU5RER0CYYkIgmtvdBhe3SfYMhktrUMCRFRW8eQRCSRM+erkZZ9DgAwKjpY4mqIiOhyDElEElm3t6kV6dZOPmjv7SJxNUREdDmGJCIJCCEME0jawzIkRERtEUMSkQT25pbhZEkVnB0VGBoZKHU5RERkAkMSkQSaF7Md0jMAbkquM01EZIsYkoisrK5Rhx/25wMA7u/DDttERLaKIYnIyn49UoTymgYEeKjQP8xX6nKIiOgqGJKIrKx5GZJR0cFQyDk3EhGRrWJIIrKikso6bMksBtA0gSQREdkuhiQiK/p+31k06gV6tfdEF7W71OUQEdE1MCQRWdHavU232jg3EhGR7WNIIrKSzIIKZORp4SCXYUTvIKnLISKiv8CQRGQlzXMj3RXuDx9XJ4mrISKiv8KQRGQFjTo9vt3LZUiIiOwJQxKRFezIOoeiijp4uTjirnA/qcshIqLrwJBEZAXfpDfdaru3dxCUDgqJqyEiouvBkERkYRW1Ddh4qAAAcD9vtRER2Q2GJCIL23AwH3WNeoT5uaJ3e0+pyyEiouvEkERkYd/saeqwfX+f9pDJuAwJEZG9YEgisqDc0mrsOlkKmQy4L5rLkBAR2RPJQ9KSJUsQGhoKlUqF2NhY7Nq165r7L1q0CN26dYOzszM0Gg2effZZ1NbWWqlaopZZe6EVqX9YOwR5OUtcDRERtYSkIWn16tVISkrC3LlzsWfPHvTu3RsJCQkoKioyuf8XX3yBl19+GXPnzsWRI0fw8ccfY/Xq1XjllVesXDnRXxNCcBkSIiI7JmlIWrBgASZPnozExERERERg6dKlcHFxwfLly03u//vvv2PAgAF4+OGHERoaisGDB2PcuHF/2fpEJIX00+dx+lw1XJwUSOgRIHU5RETUQpKFpPr6eqSnpyM+Pv5iMXI54uPjkZaWZvKY/v37Iz093RCKsrOzsWHDBgwbNuyq71NXVwetVmv0ILKG5g7bQ3sGwlXpIHE1RETUUpL9y11SUgKdTge1Wm20Xa1W4+jRoyaPefjhh1FSUoLbbrsNQgg0Njbi6aefvubttuTkZPzzn/80a+1086qqa4RcJoPSQQ65vPUj0WobdPjxwFkAwOg+7LBNRGSP7Op/b7ds2YI333wT//3vfxEbG4sTJ05gxowZeP311/Haa6+ZPGbWrFlISkoy/K7VaqHRaKxVMtm4Bp0e6afPY0tmMbZkFuFoQYXhOZWjHM6OCjg7KqC68HB2uvi7s5MCKge5yW2552tQUduIIE8Vbu3UTsIzJCKi1pIsJPn6+kKhUKCwsNBoe2FhIQICTPffeO211/Doo4/iiSeeAABERkaiqqoKTz75JGbPng25/Mq7h0qlEkql0vwnQHYrv7zGEIp2nDiHyrpGk/vVNuhR26DHeTS0+r3u6xN8Qy1SREQkHclCkpOTE2JiYpCamopRo0YBAPR6PVJTUzFt2jSTx1RXV18RhBSKpnWwhBAWrZfsV32jHrtPl+K3zGJsySxGZmGF0fPtXJ1we1c/3NnNDwM6+8LZUYGaBh1qLzxq6vWoadA1PeovbGv+uVGH2nrdJc/rDc87Oynw+ICOEp01ERHdKElvtyUlJWHixIno27cv+vXrh0WLFqGqqgqJiYkAgAkTJiA4OBjJyckAgBEjRmDBggWIjo423G577bXXMGLECENYIgKAs2WXthaVoKpeZ3hOLgOiNF64s5s/7uzmh55Bnle09rCjNRERSfpNMHbsWBQXF2POnDkoKChAVFQUUlJSDJ25c3JyjFqOXn31VchkMrz66qvIy8uDn58fRowYgTfeeEOqUyAbUdeow+5T57ElswhbMotxvKjS6Hlft+bWIn8M7OwLb1cniSolIiJ7IRNt7D6VVquFp6cnysvL4eHhIXU5dIO0tQ14bV0GNh0uRPVlrUXRId6480Iw6hHkwb5BRER2TIrvb95TILslhMDL3xzAhoMFAABfNyXuuNC3aGAXX3i5sLWIiIhajyGJ7NYXu3Kw4WABHOQyfPzYLRjY2ZetRUREZDYMSWSXjhZoMe+HwwCAF4d0wx1d/SSuiIiIbjaSrt1G1BrV9Y2Y9sVe1DXqcWc3PzxxWyepSyIiopsQQxLZnX9+fxgniirh767Ef8b05i02IiKyCIYksivf7cvD6t25kMmARQ9FoZ0bZ1MnIiLLYEgiu3GqpAqzv80AAPz9rs7oH+YrcUVERHQzY0giu1DXqMPfv9yLyrpG9Av1wfR7ukhdEhER3eQYksguvJOSiYN55fByccTicVFwUPCvLhERWRa/acjmpR4pxMfbTwIA/v1AbwR6OktcERERtQUMSWTT8str8Pya/QCAxAGhiI9QS1wRERG1FQxJZLN0eoEZq/bhfHUDegZ74OWh4VKXREREbQhDEtmsd1OPY9fJUrg6KfDeuD5QOiikLomIiNoQhiSySWlZ5/Der8cBAG/cF4mOvq4SV0RERG0NQxLZnNKqesxcvRd6AYyJaY9R0cFSl0RERG0QQxLZFCEEnl+zH4XaOoT5ueKfI3tIXRIREbVRDElkUz7efhK/Hi2Ck4Mc7z/cBy5ODlKXREREbRRDEtmMA2fK8HbKUQDAa3+LQPdAD4krIiKitowhiWxCRW0Dpn2xFw06gaE9A/BIbIjUJRERURvHkESSE0LglW8zkFNajWAvZ7w1uhdkMpnUZRERURvHkESS+2p3Ln7YfxYKuQzvPRwNT2dHqUsiIiJiSCJpHS+swNzvDwEAnh/cDX1CvCWuiIiIqAlDEkmmtkGHaV/sRW2DHgO7+OKp2ztJXRIREZEBQxJJZt6Ph5FZWAE/dyUWPBgFuZz9kIiIyHYwJJEk1h/Ixxc7cyCTAQsfjIKfu1LqkoiIiIwwJJHVpZ8+j5e+OQAAeObOMNzWxVfiioiIiK7E6YzJqnadLEXiJ7tQVa9D/7B2eDa+q9QlERERmcSQRFaTlnUOj6/4EzUNOtzW2RfLJvSFg4KNmUREZJsYksgqth8vwRP/9ydqG/S4o6sfPnw0BipHhdRlERERXRVDElnclswiPPlpOuob9bgn3B9LxvdhQCIiIpvHkEQW9cvhQjzz+R7U6/QYHKHG+w/3gZMDb7EREZHtY0gii0nJKMDfv9yDBp3AsMgALH4oGo7sg0RERHaCIYksYv2BfMxYtReNeoERvYOw8MHe7KRNRER2hd9aZHbf7cvD9AsB6f7oYAYkIiKySzbxzbVkyRKEhoZCpVIhNjYWu3btuuq+d955J2Qy2RWP4cOHW7Fiuppv0s/g2dX7oNMLjIlpj/ljGJCIiMg+Sf7ttXr1aiQlJWHu3LnYs2cPevfujYSEBBQVFZncf+3atcjPzzc8MjIyoFAoMGbMGCtXTpf76s9cPP/1fugFMK5fCN4e3QsKrsdGRER2SvKQtGDBAkyePBmJiYmIiIjA0qVL4eLiguXLl5vc38fHBwEBAYbHpk2b4OLiwpAksc93nsaL3xyAEMCjt3bAG6N6csFaIiKya5KGpPr6eqSnpyM+Pt6wTS6XIz4+Hmlpadf1Gh9//DEeeughuLq6mny+rq4OWq3W6EHmtfL3U5j9bQYAIHFAKOaN7MGAREREdk/SkFRSUgKdTge1Wm20Xa1Wo6Cg4C+P37VrFzIyMvDEE09cdZ/k5GR4enoaHhqN5obrpos+2paNud8fAgA8dXsnzPlbBGQyBiQiIrJ/kt9uuxEff/wxIiMj0a9fv6vuM2vWLJSXlxseubm5Vqzw5rb0tyz8a/0RAMDUu8Lw8tBwBiQiIrppSDpPkq+vLxQKBQoLC422FxYWIiAg4JrHVlVVYdWqVZg3b94191MqlVAqlTdcKxl7/9fj+PfPxwAAM+7pgpnxXRiQiIjopiJpS5KTkxNiYmKQmppq2KbX65Gamoq4uLhrHrtmzRrU1dXhkUcesXSZdAkhBBZuOmYISM8P7opnB3VlQCIiopuO5DNuJyUlYeLEiejbty/69euHRYsWoaqqComJiQCACRMmIDg4GMnJyUbHffzxxxg1ahTatWsnRdltkhAC//45E0s2ZwEAXh4ajqfvCJO4KiIiIsuQPCSNHTsWxcXFmDNnDgoKChAVFYWUlBRDZ+6cnBzI5cYNXpmZmdi+fTt+/vlnKUpus+ZvzMR/tzQFpFeHd8cTAztJXBEREZHlyIQQQuoirEmr1cLT0xPl5eXw8PCQuhy78ekfp/HauqZh/v8YEYHHBnSUuCIiImpLpPj+tuvRbWQdmzOLMPe7poD03KCuDEhERNQmMCTRNR0+q8W0z/dAL4AHYtpj2t2dpS6JiIjIKhiS6KoKymvx+Io/UVWvQ1yndnjzvkiOYiMiojaDIYlMqqprxKSVf6JAW4swP1csfSQGTg7860JERG0Hv/XoCjq9wPQv9+LQWS3auTrhk8f6wdPFUeqyiIiIrIohia7w+o+HkXq0CEoHOZZN7IuQdi5Sl0RERGR1DElk5JMdJ7Hi91MAgIVjo9AnxFvagoiIiCTCkEQGmw4XYt6PhwE0zaY9LDJQ4oqIiIikw5BEAICDZ8ox/cu9EAIY1y8ET93O2bSJiKhtY0gi5JXV4PGVf6KmQYeBXXwxb2QPDvUnIqI2jyGpjauobcCkFX+iuKIO3dTuWDK+DxwV/GtBRETEb8M2rEGnx9Qv9uJoQQX83JVYnngLPFQc6k9ERAQADq09sKysDLt27UJRURH0er3RcxMmTLjhwsiyhBCY+/0hbD1WDGdHBZZPvAXBXs5Sl0VERGQzWhWSfvjhB4wfPx6VlZXw8PAw6r8ik8kYkuzAsm3Z+GJnDmQyYPFDUYhs7yl1SURERDalVbfbnnvuOTz++OOorKxEWVkZzp8/b3iUlpaau0Yys58O5uPNDUcBAK8Oj8DgHgESV0RERGR7WhWS8vLyMH36dLi4cCZme7M35zxmrt4HAJgY1wGPDwiVtB4iIiJb1aqQlJCQgN27d5u7FrKw3NJqTP6/3ahr1OPucH+89rcIDvUnIiK6ilb1SRo+fDheeOEFHD58GJGRkXB0NB4Rde+995qlODKf8poGJK74EyWV9YgI9MB746LhwKH+REREVyUTQoiWHiSXX/3LVSaTQafT3VBRlqTVauHp6Yny8nJ4eHhIXY5V1Dfq8dgnu/B71jkEeKiwbuoABHiqpC6LiIjouknx/d2qlqTLh/yT7RJCYPa3B/F71jm4Oimw/LFbGJCIiIiuQ4vvtzQ0NMDBwQEZGRmWqIfM7NM/TmNN+hnIZcD7D/dBRFDbaD0jIiK6US0OSY6OjggJCbHpW2rURAiB5dtPAgBeHhqOu8L9Ja6IiIjIfrSq5+7s2bPxyiuvcE4kG3fgTDlOnauGs6MC42M7SF0OERGRXWlVn6T3338fJ06cQFBQEDp06ABXV1ej5/fs2WOW4ujGfLfvLABgUIQarspWr0BDRETUJrXqm3PUqFFmLoPMTacX+OFAU0gaGRUkcTVERET2p1Uhae7cueaug8wsLesciivq4OXiiIFd/KQuh4iIyO5wNsGb1Hf78gAAwyMD4eTAj5mIiKilWtWSJJfLr7mcBUe+Sau2QYeUjAIAwMioYImrISIisk+tCknffvut0e8NDQ3Yu3cvVq5ciX/+859mKYxab/PRIlTUNSLIU4W+HbylLoeIiMgutSokjRw58optDzzwAHr06IHVq1dj0qRJN1wYtV7zqLZ7o4Ihl3MBWyIiotYwa2eVW2+9FampqeZ8SWqh8poG/Hq0CABHtREREd0Is4WkmpoavPvuuwgOZh8YKW3MKEC9To+uajeEB7hLXQ4REZHdatXtNm9vb6OO20IIVFRUwMXFBZ999pnZiqOW+25/06i2kVHB1+xcT0RERNfWqpC0cOFCoy9guVwOPz8/xMbGwtubHYWlUqitxe9Z5wAA9/bmrTYiIqIb0aqQdPfdd0Oj0ZhsqcjJyUFISMgNF0Yt98P+sxACiOngDY2Pi9TlEBER2bVW9Unq2LEjiouLr9h+7tw5dOzYsUWvtWTJEoSGhkKlUiE2Nha7du265v5lZWWYOnUqAgMDoVQq0bVrV2zYsKFF73mz+n5/06i2UeywTUREdMNa1ZIkhDC5vbKyEiqV6rpfZ/Xq1UhKSsLSpUsRGxuLRYsWISEhAZmZmfD3979i//r6egwaNAj+/v74+uuvERwcjNOnT8PLy6s1p3FTyS6uxIEz5VDIZRgWGSh1OURERHavRSEpKSkJACCTyTBnzhy4uFy8paPT6bBz505ERUVd9+stWLAAkydPRmJiIgBg6dKlWL9+PZYvX46XX375iv2XL1+O0tJS/P7773B0dAQAhIaGXvM96urqUFdXZ/hdq9Ved332pHlupIFdfNHOTSlxNURERPavRbfb9u7di71790IIgYMHDxp+37t3L44ePYrevXtjxYoV1/Va9fX1SE9PR3x8/MVi5HLEx8cjLS3N5DHff/894uLiMHXqVKjVavTs2RNvvvnmNZdBSU5Ohqenp+Gh0Whacsp2QQhxya02TsFARERkDi1qSdq8eTMAIDExEYsXL4aHh0er37ikpAQ6nQ5qtdpou1qtxtGjR00ek52djV9//RXjx4/Hhg0bcOLECTzzzDNoaGjA3LlzTR4za9YsQwsY0NSSdLMFpQNnynGypAoqRzkGRaj/+gAiIiL6S63qk/TJJ58AAE6cOIGsrCzcfvvtcHZ2hhDConPz6PV6+Pv743//+x8UCgViYmKQl5eH+fPnXzUkKZVKKJU39+2n5lttgyIC4Kps1UdKREREl2nV6LbS0lLcc8896Nq1K4YNG4b8/HwAwKRJk/Dcc89d12v4+vpCoVCgsLDQaHthYSECAgJMHhMYGIiuXbtCoVAYtnXv3h0FBQWor69vzanYPZ1e4IcDHNVGRERkbq0KSTNnzoSjoyNycnKMOm+PHTsWKSkp1/UaTk5OiImJMVrrTa/XIzU1FXFxcSaPGTBgAE6cOAG9Xm/YduzYMQQGBsLJyak1p2L30rLOobiiDl4ujhjYxU/qcoiIiG4arQpJP//8M95++220b9/eaHuXLl1w+vTp636dpKQkLFu2DCtXrsSRI0cwZcoUVFVVGUa7TZgwAbNmzTLsP2XKFJSWlmLGjBk4duwY1q9fjzfffBNTp05tzWncFL7b17QMybDIQDg5mHW9YiIiojatVR1YqqqqjFqQmpWWlrao/8/YsWNRXFyMOXPmoKCgAFFRUUhJSTF05s7JyYFcfvGLX6PRYOPGjXj22WfRq1cvBAcHY8aMGXjppZdacxp2r7ZBh5SMAgAc1UZERGRuMnG1mSGvYdiwYYiJicHrr78Od3d3HDhwAB06dMBDDz0EvV6Pr7/+2hK1moVWq4WnpyfKy8tvaHSeLfjpYD6mfL4HQZ4qbH/pbsjlXNCWiIhuTlJ8f7eqJWn+/Pm4++67sXv3btTX1+PFF1/EoUOHUFpaih07dpi7RrqK5lFtI6KCGJCIiIjMrMUhqaGhAdOnT8cPP/yATZs2wd3dHZWVlbj//vsNa6qR5ZXXNODXzCIAvNVGRERkCS0OSY6Ojjhw4AC8vb0xe/ZsS9RE12FjRgHqG/XoqnZDeIC71OUQERHddFo1HOqRRx7Bxx9/bO5aqAW+2980qm1kVLBFJ/AkIiJqq1rVJ6mxsRHLly/HL7/8gpiYGLi6uho9v2DBArMUR6YVaWvxe9Y5AMC9vTmBJBERkSW0KiRlZGSgT58+AJomc7wUWzUs74cD+RACiOngDY3PlVMxEBER0Y1rVUhqXuiWpNE8geRILkNCRERkMZyi2c5kF1fiwJlyKOQyDI/kSEIiIiJLYUiyM9/vb5obaWAXX7Rzu/7ZzYmIiKhlGJLsiBDCMIEkb7URERFZFkOSHTmYV46TJVVQOcoxOCJA6nKIiIhuagxJdqS5FWlQRABcla3qc09ERETXiSHJTuj0Aj9c6I80knMjERERWRxDkp34I/sciirq4OXiiNu7+kldDhER0U2PIclONM+NNCwyEE4O/NiIiIgsjd+2dqC2QYefDhYA4K02IiIia2FIsgNbMotQUdeIIE8Vbgn1kbocIiKiNoEhyQ40j2obERUEuZxr4xEREVkDQ5KN09Y2IPVoEQBgZO9giashIiJqOxiSbFxKRgHqG/XoqnZD90B3qcshIiJqMxiSbNz3hmVIgiGT8VYbERGRtTAk2bAibS1+zyoBANzLUW1ERERWxZBkw344kA+9AGI6eEPj4yJ1OURERG0KQ5IN+/7CBJIjo9iKREREZG0MSTbqZEkV9p8ph0Iuw7DIQKnLISIianMYkmxU8zIkA7v4wtdNKXE1REREbQ9Dkg0SQlwyqo232oiIiKTAkGSDDuaVI7ukCipHOQZFBEhdDhERUZvEkGSDtp9oGvZ/Vzd/uCkdJK6GiIiobWJIskFZRVUAgIhAD4krISIiarsYkmxQdkklAKCTn5vElRAREbVdDEk2RgiB7OKmlqROfq4SV0NERNR2MSTZmHNV9SivaYBMBnT0ZUgiIiKSCkOSjWluRQr2cobKUSFxNURERG2XTYSkJUuWIDQ0FCqVCrGxsdi1a9dV912xYgVkMpnRQ6VSWbFay8ouZn8kIiIiWyB5SFq9ejWSkpIwd+5c7NmzB71790ZCQgKKioqueoyHhwfy8/MNj9OnT1uxYsvKLrnQH4m32oiIiCQleUhasGABJk+ejMTERERERGDp0qVwcXHB8uXLr3qMTCZDQECA4aFWq6+6b11dHbRardHDljW3JIX5syWJiIhISpKGpPr6eqSnpyM+Pt6wTS6XIz4+HmlpaVc9rrKyEh06dIBGo8HIkSNx6NChq+6bnJwMT09Pw0Oj0Zj1HMwt60KfpDC2JBEREUlK0pBUUlICnU53RUuQWq1GQUGByWO6deuG5cuX47vvvsNnn30GvV6P/v3748yZMyb3nzVrFsrLyw2P3Nxcs5+HudQ36pFTWg2AfZKIiIikZndrXsTFxSEuLs7we//+/dG9e3d8+OGHeP3116/YX6lUQqlUWrPEVssprYZOL+DqpIDawz5qJiIiullJ2pLk6+sLhUKBwsJCo+2FhYUICLi+hV0dHR0RHR2NEydOWKJEq7p0ZJtMJpO4GiIiorZN0pDk5OSEmJgYpKamGrbp9XqkpqYatRZdi06nw8GDBxEYGGipMq0mizNtExER2QzJb7clJSVh4sSJ6Nu3L/r164dFixahqqoKiYmJAIAJEyYgODgYycnJAIB58+bh1ltvRefOnVFWVob58+fj9OnTeOKJJ6Q8DbMwtCT5sj8SERGR1CQPSWPHjkVxcTHmzJmDgoICREVFISUlxdCZOycnB3L5xQav8+fPY/LkySgoKIC3tzdiYmLw+++/IyIiQqpTMBvDHElsSSIiIpKcTAghpC7CmrRaLTw9PVFeXg4PDw+pyzESPe9nnK9uwPrpt6FHkKfU5RAREdkMKb6/JZ9MkpqUVtXjfHUDAN5uIyIisgUMSTaiuT9SsJcznJ24sC0REZHUGJJsRDZHthEREdkUhiQbkVXSPLKNIYmIiMgWMCTZiKyiC2u2cWFbIiIim8CQZCOySzhHEhERkS1hSLIBDTo9cs41L2zL221ERES2gCHJBuSWVqNRL+DsqECAh0rqcoiIiAgMSTbh0pFtcjkXtiUiIrIFDEk2IKt5zTY/9kciIiKyFQxJNsDQksTh/0RERDaDIckGGEa2sdM2ERGRzWBIsgHNLUlhvN1GRERkMxiSJFZWXY9zVfUAgI683UZERGQzGJIklnWhFSnQUwVXpYPE1RAREVEzhiSJZRezPxIREZEtYkiSWHZJ88g29kciIiKyJQxJEssqampJCmNLEhERkU1hSJKYoSWJI9uIiIhsCkOShBp1epw+d3FJEiIiIrIdDEkSOnO+Bg06AZWjHEGezlKXQ0RERJdgSJJQ80zbHX3duLAtERGRjWFIklBWEW+1ERER2SqGJAk1tySFcaZtIiIim8OQJKHm2bY5so2IiMj2MCRJKLuYt9uIiIhsFUOSRMprGlBSWQeALUlERES2iCFJIs1rtqk9lHDjwrZEREQ2hyFJIoZbbVyzjYiIyCYxJEmkeWQb+yMRERHZJoYkiTTPkRTG/khEREQ2iSFJImxJIiIism0MSRLQ6QVOnasGwJYkIiIiW8WQJIG88zWob9TDyUGOIC8ubEtERGSLbCIkLVmyBKGhoVCpVIiNjcWuXbuu67hVq1ZBJpNh1KhRli3QzLIuDP/v5OsKBRe2JSIiskmSh6TVq1cjKSkJc+fOxZ49e9C7d28kJCSgqKjomsedOnUKzz//PAYOHGilSs3HEJLYH4mIiMhmSR6SFixYgMmTJyMxMRERERFYunQpXFxcsHz58qseo9PpMH78ePzzn/9Ep06drFiteWSXcI4kIiIiWydpSKqvr0d6ejri4+MN2+RyOeLj45GWlnbV4+bNmwd/f39MmjTpL9+jrq4OWq3W6CG1bLYkERER2TxJQ1JJSQl0Oh3UarXRdrVajYKCApPHbN++HR9//DGWLVt2Xe+RnJwMT09Pw0Oj0dxw3Tfq4sK2bEkiIiKyVZLfbmuJiooKPProo1i2bBl8fX2v65hZs2ahvLzc8MjNzbVwlddWUduAoormhW3ZkkRERGSrJF1Z1dfXFwqFAoWFhUbbCwsLERAQcMX+WVlZOHXqFEaMGGHYptfrAQAODg7IzMxEWFiY0TFKpRJKpdIC1bdOcyuSn7sSHipHiashIiKiq5G0JcnJyQkxMTFITU01bNPr9UhNTUVcXNwV+4eHh+PgwYPYt2+f4XHvvffirrvuwr59+2ziVtpfMcy07ctWJCIiIlsmaUsSACQlJWHixIno27cv+vXrh0WLFqGqqgqJiYkAgAkTJiA4OBjJyclQqVTo2bOn0fFeXl4AcMV2W8X+SERERPZB8pA0duxYFBcXY86cOSgoKEBUVBRSUlIMnblzcnIgl9tV16lrap4jKYz9kYiIiGyaTAghpC7CmrRaLTw9PVFeXg4PDw+rv/+QRVtxtKACnzx2C+4K97f6+xMREdkjKb6/b54mGjug1wucbJ5Iki1JRERENo0hyYryympQ16iHk0KO9t4uUpdDRERE18CQZEXN/ZFCfV24sC0REZGNY0iyIsPINq7ZRkREZPMYkqzIMEcS+yMRERHZPIYkK+IcSURERPaDIcmKmvsksSWJiIjI9jEkWUllXSMKtU0L24axTxIREZHNY0iykpMXbrX5ujnB04UL2xIREdk6hiQrubiwLVuRiIiI7AFDkpVkFXOmbSIiInvCkGQlFxe2ZUsSERGRPWBIspJstiQRERHZFYYkK2ha2LZ5+D9bkoiIiOwBQ5IV5GtrUdugh6NCBo23s9TlEBER0XVgSLKCrKKmVqQO7VzhoOAlJyIisgf8xraC7OaZtn3ZH4mIiMheMCRZQXYJ12wjIiKyNwxJVsCRbURERPaHIckKLs6RxJBERERkLxiSLKy6vhH55bUAuCQJERGRPWFIsrDmW20+rk7wdnWSuBoiIiK6XgxJFmbotM2RbURERHaFIcnCDMP/2R+JiIjIrjAkWVjWhdttXNiWiIjIvjAkWdjFliSGJCIiInvCkGRBQgicLOEcSURERPaIIcmCCrS1qK7XwUEuQ4iPi9TlEBERUQswJFlQVlFTK1JIOxc4cmFbIiIiu8JvbgvKLmle2Jb9kYiIiOwNQ5IFZRtGtrE/EhERkb1hSLKgLM6RREREZLcYkiyouSWJw/+JiIjsD0OShdTU65BXVgOAE0kSERHZI5sISUuWLEFoaChUKhViY2Oxa9euq+67du1a9O3bF15eXnB1dUVUVBQ+/fRTK1Z7fZrnR/JycYQPF7YlIiKyO5KHpNWrVyMpKQlz587Fnj170Lt3byQkJKCoqMjk/j4+Ppg9ezbS0tJw4MABJCYmIjExERs3brRy5dd2cWQb+yMRERHZI8lD0oIFCzB58mQkJiYiIiICS5cuhYuLC5YvX25y/zvvvBP33XcfunfvjrCwMMyYMQO9evXC9u3brVz5tTXPkcT+SERERPZJ0pBUX1+P9PR0xMfHG7bJ5XLEx8cjLS3tL48XQiA1NRWZmZm4/fbbTe5TV1cHrVZr9LCG5pYk9kciIiKyT5KGpJKSEuh0OqjVaqPtarUaBQUFVz2uvLwcbm5ucHJywvDhw/Hee+9h0KBBJvdNTk6Gp6en4aHRaMx6DldzcWQbb7cRERHZI8lvt7WGu7s79u3bhz///BNvvPEGkpKSsGXLFpP7zpo1C+Xl5YZHbm6uxesTQiC7uLkliSGJiIjIHjlI+ea+vr5QKBQoLCw02l5YWIiAgICrHieXy9G5c2cAQFRUFI4cOYLk5GTceeedV+yrVCqhVCrNWvdfKaqoQ1W9Dgq5DCE+DElERET2SNKWJCcnJ8TExCA1NdWwTa/XIzU1FXFxcdf9Onq9HnV1dZYosVWyippakUJ8XODkYJeNdURERG2epC1JAJCUlISJEyeib9++6NevHxYtWoSqqiokJiYCACZMmIDg4GAkJycDaOpj1LdvX4SFhaGurg4bNmzAp59+ig8++EDK0zCSdWGOJA7/JyIisl+Sh6SxY8eiuLgYc+bMQUFBAaKiopCSkmLozJ2TkwO5/GJrTFVVFZ555hmcOXMGzs7OCA8Px2effYaxY8dKdQpXyOaabURERHZPJoQQUhdhTVqtFp6enigvL4eHh4dF3mPi8l347Vgxku+PxLh+IRZ5DyIiorbEGt/fl2OHGQvIKuYcSURERPaOIcnMahsuLmzL221ERET2iyHJzE6dq4IQgIfKAe24sC0REZHdYkgys4szbbtBJpNJXA0RERG1FkOSmTXPkcRbbURERPaNIcnMsi/MkcRO20RERPaNIcnMuGYbERHRzYEhyYyaFra92CeJiIiI7BdDkhkVV9Shoq4RchnQoZ2L1OUQERHRDWBIMqOsC61IGh8XKB0UEldDREREN4IhyYyySy6MbOPCtkRERHaPIcmM2B+JiIjo5sGQZEbNI9s4RxIREZH9Y0gyo+Y+SZwjiYiIyP4xJJlJXaMOZ85XA2BLEhER0c2AIclMTp+rhl4A7koH+LkppS6HiIiIbpCD1AXcLM5V1sPLxREdfFy4sC0REdFNgCHJTOLC2mHfnMGoqddJXQoRERGZAW+3mZmzEyeRJCIiuhkwJBERERGZwJBEREREZAJDEhEREZEJDElEREREJjAkEREREZnAkERERERkAkMSERERkQkMSUREREQmMCQRERERmcCQRERERGQCQxIRERGRCQxJRERERCYwJBERERGZ4CB1AdYmhAAAaLVaiSshIiKi69X8vd38PW4NbS4kVVRUAAA0Go3ElRAREVFLVVRUwNPT0yrvJRPWjGQ2QK/X4+zZs3B3d4dMJjPra2u1Wmg0GuTm5sLDw8Osr01Xx+suDV53afC6S4PXXRqXXnd3d3dUVFQgKCgIcrl1egu1uZYkuVyO9u3bW/Q9PDw8+B+RBHjdpcHrLg1ed2nwukuj+bpbqwWpGTtuExEREZnAkERERERkAkOSGSmVSsydOxdKpVLqUtoUXndp8LpLg9ddGrzu0pD6ure5jttERERE14MtSUREREQmMCQRERERmcCQRERERGQCQxIRERGRCQxJZrJkyRKEhoZCpVIhNjYWu3btkroku5GcnIxbbrkF7u7u8Pf3x6hRo5CZmWm0T21tLaZOnYp27drBzc0No0ePRmFhodE+OTk5GD58OFxcXODv748XXngBjY2NRvts2bIFffr0gVKpROfOnbFixQpLn57deOuttyCTyTBz5kzDNl53y8jLy8MjjzyCdu3awdnZGZGRkdi9e7fheSEE5syZg8DAQDg7OyM+Ph7Hjx83eo3S0lKMHz8eHh4e8PLywqRJk1BZWWm0z4EDBzBw4ECoVCpoNBq88847Vjk/W6TT6fDaa6+hY8eOcHZ2RlhYGF5//XWjdcB43W/c1q1bMWLECAQFBUEmk2HdunVGz1vzGq9Zswbh4eFQqVSIjIzEhg0bWn5Cgm7YqlWrhJOTk1i+fLk4dOiQmDx5svDy8hKFhYVSl2YXEhISxCeffCIyMjLEvn37xLBhw0RISIiorKw07PP0008LjUYjUlNTxe7du8Wtt94q+vfvb3i+sbFR9OzZU8THx4u9e/eKDRs2CF9fXzFr1izDPtnZ2cLFxUUkJSWJw4cPi/fee08oFAqRkpJi1fO1Rbt27RKhoaGiV69eYsaMGYbtvO7mV1paKjp06CAee+wxsXPnTpGdnS02btwoTpw4YdjnrbfeEp6enmLdunVi//794t577xUdO3YUNTU1hn2GDBkievfuLf744w+xbds20blzZzFu3DjD8+Xl5UKtVovx48eLjIwM8eWXXwpnZ2fx4YcfWvV8bcUbb7wh2rVrJ3788Udx8uRJsWbNGuHm5iYWL15s2IfX/cZt2LBBzJ49W6xdu1YAEN9++63R89a6xjt27BAKhUK888474vDhw+LVV18Vjo6O4uDBgy06H4YkM+jXr5+YOnWq4XedTieCgoJEcnKyhFXZr6KiIgFA/Pbbb0IIIcrKyoSjo6NYs2aNYZ8jR44IACItLU0I0fQfplwuFwUFBYZ9PvjgA+Hh4SHq6uqEEEK8+OKLokePHkbvNXbsWJGQkGDpU7JpFRUVokuXLmLTpk3ijjvuMIQkXnfLeOmll8Rtt9121ef1er0ICAgQ8+fPN2wrKysTSqVSfPnll0IIIQ4fPiwAiD///NOwz08//SRkMpnIy8sTQgjx3//+V3h7exs+h+b37tatm7lPyS4MHz5cPP7440bb7r//fjF+/HghBK+7JVwekqx5jR988EExfPhwo3piY2PFU0891aJz4O22G1RfX4/09HTEx8cbtsnlcsTHxyMtLU3CyuxXeXk5AMDHxwcAkJ6ejoaGBqNrHB4ejpCQEMM1TktLQ2RkJNRqtWGfhIQEaLVaHDp0yLDPpa/RvE9b/5ymTp2K4cOHX3FteN0t4/vvv0ffvn0xZswY+Pv7Izo6GsuWLTM8f/LkSRQUFBhdM09PT8TGxhpddy8vL/Tt29ewT3x8PORyOXbu3GnY5/bbb4eTk5Nhn4SEBGRmZuL8+fOWPk2b079/f6SmpuLYsWMAgP3792P79u0YOnQoAF53a7DmNTbXvzsMSTeopKQEOp3O6EsCANRqNQoKCiSqyn7p9XrMnDkTAwYMQM+ePQEABQUFcHJygpeXl9G+l17jgoICk59B83PX2ker1aKmpsYSp2PzVq1ahT179iA5OfmK53jdLSM7OxsffPABunTpgo0bN2LKlCmYPn06Vq5cCeDidbvWvykFBQXw9/c3et7BwQE+Pj4t+mzakpdffhkPPfQQwsPD4ejoiOjoaMycORPjx48HwOtuDda8xlfbp6WfgUOL9iaysKlTpyIjIwPbt2+XupSbXm5uLmbMmIFNmzZBpVJJXU6bodfr0bdvX7z55psAgOjoaGRkZGDp0qWYOHGixNXdvL766it8/vnn+OKLL9CjRw/s27cPM2fORFBQEK87XRVbkm6Qr68vFArFFSN+CgsLERAQIFFV9mnatGn48ccfsXnzZrRv396wPSAgAPX19SgrKzPa/9JrHBAQYPIzaH7uWvt4eHjA2dnZ3Kdj89LT01FUVIQ+ffrAwcEBDg4O+O233/Duu+/CwcEBarWa190CAgMDERERYbSte/fuyMnJAXDxul3r35SAgAAUFRUZPd/Y2IjS0tIWfTZtyQsvvGBoTYqMjMSjjz6KZ5991tCKyutueda8xlfbp6WfAUPSDXJyckJMTAxSU1MN2/R6PVJTUxEXFydhZfZDCIFp06bh22+/xa+//oqOHTsaPR8TEwNHR0eja5yZmYmcnBzDNY6Li8PBgweN/uPatGkTPDw8DF9IcXFxRq/RvE9b/ZzuueceHDx4EPv27TM8+vbti/Hjxxt+5nU3vwEDBlwxxcWxY8fQoUMHAEDHjh0REBBgdM20Wi127txpdN3LysqQnp5u2OfXX3+FXq9HbGysYZ+tW7eioaHBsM+mTZvQrVs3eHt7W+z8bFV1dTXkcuOvPIVCAb1eD4DX3RqseY3N9u9Oi7p5k0mrVq0SSqVSrFixQhw+fFg8+eSTwsvLy2jED13dlClThKenp9iyZYvIz883PKqrqw37PP300yIkJET8+uuvYvfu3SIuLk7ExcUZnm8eij548GCxb98+kZKSIvz8/EwORX/hhRfEkSNHxJIlS9r0UHRTLh3dJgSvuyXs2rVLODg4iDfeeEMcP35cfP7558LFxUV89tlnhn3eeust4eXlJb777jtx4MABMXLkSJPDpKOjo8XOnTvF9u3bRZcuXYyGSZeVlQm1Wi0effRRkZGRIVatWiVcXFzazFD0y02cOFEEBwcbpgBYu3at8PX1FS+++KJhH173G1dRUSH27t0r9u7dKwCIBQsWiL1794rTp08LIax3jXfs2CEcHBzEv//9b3HkyBExd+5cTgEgpffee0+EhIQIJycn0a9fP/HHH39IXZLdAGDy8cknnxj2qampEc8884zw9vYWLi4u4r777hP5+flGr3Pq1CkxdOhQ4ezsLHx9fcVzzz0nGhoajPbZvHmziIqKEk5OTqJTp05G70FXhiRed8v44YcfRM+ePYVSqRTh4eHif//7n9Hzer1evPbaa0KtVgulUinuuecekZmZabTPuXPnxLhx44Sbm5vw8PAQiYmJoqKiwmif/fv3i9tuu00olUoRHBws3nrrLYufm63SarVixowZIiQkRKhUKtGpUycxe/Zso2HkvO43bvPmzSb/PZ84caIQwrrX+KuvvhJdu3YVTk5OokePHmL9+vUtPh+ZEJdMN0pEREREANgniYiIiMgkhiQiIiIiExiSiIiIiExgSCIiIiIygSGJiIiIyASGJCIiIiITGJKIiIiITGBIIiIiIjKBIYmILG7Lli2QyWRXLJZLRGTLGJKIyKzuvPNOzJw502hb//79kZ+fD09PT2mKuorQ0FAsWrRI6jKIyEY5SF0AEd38nJycEBAQIHUZREQtwpYkIjKbxx57DL/99hsWL14MmUwGmUyGU6dOXXG7bcWKFfDy8sKPP/6Ibt26wcXFBQ888ACqq6uxcuVKhIaGwtvbG9OnT4dOpzO8fl1dHZ5//nkEBwfD1dUVsbGx2LJly1XrEULgH//4B0JCQqBUKhEUFITp06cDaGrxOn36NJ599llDrc22b9+OgQMHwtnZGRqNBtOnT0dVVZXh+dDQULz++usYN24cXF1dERwcjCVLlpj3YhKR5BiSiMhsFi9ejLi4OEyePBn5+fnIz8+HRqMxuW91dTXeffddrFq1CikpKdiyZQvuu+8+bNiwARs2bMCnn36KDz/8EF9//bXhmGnTpiEtLQ2rVq3CgQMHMGbMGAwZMgTHjx83+R7ffPMNFi5ciA8//BDHjx/HunXrEBkZCQBYu3Yt2rdvj3nz5hlqBYCsrCwMGTIEo0ePxoEDB7B69Wps374d06ZNM3rt+fPno3fv3ti7dy9efvllzJgxA5s2bTLHZSQiWyGIiMzojjvuEDNmzDDatnnzZgFAnD9/XgghxCeffCIAiBMnThj2eeqpp4SLi4uoqKgwbEtISBBPPfWUEEKI06dPC4VCIfLy8oxe+5577hGzZs0yWct//vMf0bVrV1FfX2/y+Q4dOoiFCxcabZs0aZJ48sknjbZt27ZNyOVyUVNTYzhuyJAhRvuMHTtWDB061OT7EJF9YksSEUnCxcUFYWFhht/VajVCQ0Ph5uZmtK2oqAgAcPDgQeh0OnTt2hVubm6Gx2+//YasrCyT7zFmzBjU1NSgU6dOmDx5Mr799ls0NjZes679+/djxYoVRu+RkJAAvV6PkydPGvaLi4szOi4uLg5Hjhxp8XUgItvFjttEJAlHR0ej32Uymclter0eAFBZWQmFQoH09HQoFAqj/S4NVpfSaDTIzMzEL7/8gk2bNuGZZ57B/Pnz8dtvv13xXs0qKyvx1FNPGfouXSokJOS6z4+I7B9DEhGZlZOTk1Fna3OJjo6GTqdDUVERBg4ceN3HOTs7Y8SIERgxYgSmTp2K8PBwHDx4EH369DFZa58+fXD48GF07tz5mq/7xx9/XPF79+7dr/+EiMjmMSQRkVmFhoZi586dOHXqFNzc3ODj42OW1+3atSvGjx+PCRMm4D//+Q+io6NRXFyM1NRU9OrVC8OHD7/imBUrVkCn0yE2NhYuLi747LPP4OzsjA4dOhhq3bp1Kx566CEolUr4+vripZdewq233opp06bhiSeegKurKw4fPoxNmzbh/fffN7z2jh078M4772DUqFHYtGkT1qxZg/Xr15vlXInINrBPEhGZ1fPPPw+FQoGIiAj4+fkhJyfHbK/9ySefYMKECXjuuefQrVs3jBo1Cn/++edVb4N5eXlh2bJlGDBgAHr16oVffvkFP/zwA9q1awcAmDdvHk6dOoWwsDD4+fkBAHr16oXffvsNx44dw8CBAxEdHY05c+YgKCjI6LWfe+457N69G9HR0fjXv/6FBQsWICEhwWznSkTSkwkhhNRFEBHZk9DQUMycOfOKmcWJ6ObCliQiIiIiExiSiIiIiEzg7TYiIiIiE9iSRERERGQCQxIRERGRCQxJRERERCYwJBERERGZwJBEREREZAJDEhEREZEJDElEREREJjAkEREREZnw//ayjZOd9DyNAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Create a Neural SquareCB pearl agent with 1-hot action representation\n", + "\n", + "action_representation_module = OneHotActionTensorRepresentationModule(\n", + " max_number_actions= env._action_space.n,\n", + ")\n", + "\n", + "agent = PearlAgent(\n", + " policy_learner=NeuralBandit(\n", + " feature_dim = env.observation_dim + env._action_space.n,\n", + " hidden_dims=[64, 16],\n", + " training_rounds=50,\n", + " action_representation_module=action_representation_module,\n", + " exploration_module= SquareCBExploration(gamma = env.observation_dim * env._action_space.n * number_of_steps)\n", + " ),\n", + " replay_buffer=FIFOOffPolicyReplayBuffer(100_000),\n", + " device_id=-1,\n", + ")\n", + "\n", + "\n", + "info = online_learning(\n", + " agent=agent,\n", + " env=env,\n", + " number_of_steps=number_of_steps,\n", + " print_every_x_steps=100,\n", + " record_period=record_period,\n", + " learn_after_episode=True,\n", + ")\n", + "torch.save(info[\"return\"], \"SquareCB-return.pt\")\n", + "plt.plot(record_period * np.arange(len(info[\"return\"])), info[\"return\"], label=\"SquareCB\")\n", + "plt.xlabel(\"time step\")\n", + "plt.ylabel(\"return\")\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cjLo_VzyLNGR" + }, + "source": [ + "## Contextual Bandits learners: LinUCB\n", + "\n", + "Next, we describe how to use the neural version of the LinUCB algorithm with the Pearl library, which uses UCB type of exploration with neural architectures. The LinUCB and its neural version, are generalizations of the seminal Upper Confidence Bound (UCB) algorithm. Both execute a policy of the following form:\n", + "$$\n", + "\\pi(a,x) \\in \\arg\\max_a \\widehat{r}(x,a) + \\mathrm{score}(x,a),\n", + "$$\n", + "namely, both uses a function that estimates the expected reward with an additional bonus term, that quantifies the potential of choosing an action given a certain context. A common way to estimate the score function, in the linear case, when the features are $\\phi(x,a)$ is via:\n", + "$$\n", + "\\mathrm{score}(x,a) = \\alpha ||\\phi(x,a) ||_{A^{-1}}\\\\\n", + "\\text{where } A= \\lambda I + \\sum_{n\\leq t} \\phi(x_n,a_n)\\phi^T(x_n,a_n).\n", + "$$\n", + "\n", + "To implement the LinUCB algorithm in Pearl, use the NeuralLinearBandit policy learner module. This module supports (i) learning a reward model, and, (ii) calculates a score function by estimating the uncertainty using the last layer features. Further, set the exploration module to be UCBExploration and set the alpha hyper-parameters to enable the agent with the UCB-like update rule.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "background_save": true, + "base_uri": "https://localhost:8080/" + }, + "id": "cDauzO74nS4c", + "outputId": "90febc78-d0be-4793-e6de-6dc4d4dcd0df" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "episode 100, step 100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.028449539095163345\n", + "episode 200, step 200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.07872921228408813\n", + "episode 300, step 300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9911090135574341\n", + "episode 400, step 400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.059778787195682526\n", + "episode 500, step 500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.036440279334783554\n", + "episode 600, step 600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.14758507907390594\n", + "episode 700, step 700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.7987019419670105\n", + "episode 800, step 800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.11621314287185669\n", + "episode 900, step 900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.19707509875297546\n", + "episode 1000, step 1000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.19711564481258392\n", + "episode 1100, step 1100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.12591037154197693\n", + "episode 1200, step 1200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9391128420829773\n", + "episode 1300, step 1300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8919231295585632\n", + "episode 1400, step 1400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9510725736618042\n", + "episode 1500, step 1500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9114731550216675\n", + "episode 1600, step 1600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0798403024673462\n", + "episode 1700, step 1700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.881016194820404\n", + "episode 1800, step 1800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.196246936917305\n", + "episode 1900, step 1900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.054653525352478\n", + "episode 2000, step 2000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.021109413355588913\n", + "episode 2100, step 2100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.1222798153758049\n", + "episode 2200, step 2200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9989399313926697\n", + "episode 2300, step 2300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.957256555557251\n", + "episode 2400, step 2400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0547510385513306\n", + "episode 2500, step 2500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0274019241333008\n", + "episode 2600, step 2600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.10208769142627716\n", + "episode 2700, step 2700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.014515458606183529\n", + "episode 2800, step 2800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.2670891284942627\n", + "episode 2900, step 2900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8728530406951904\n", + "episode 3000, step 3000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0672060251235962\n", + "episode 3100, step 3100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0092743635177612\n", + "episode 3200, step 3200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9323505759239197\n", + "episode 3300, step 3300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9738248586654663\n", + "episode 3400, step 3400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.04729994386434555\n", + "episode 3500, step 3500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9858903288841248\n", + "episode 3600, step 3600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.168216347694397\n", + "episode 3700, step 3700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0214523077011108\n", + "episode 3800, step 3800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1597975492477417\n", + "episode 3900, step 3900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.05957194045186043\n", + "episode 4000, step 4000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.97993004322052\n", + "episode 4100, step 4100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9776469469070435\n", + "episode 4200, step 4200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.18466797471046448\n", + "episode 4300, step 4300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9100396037101746\n", + "episode 4400, step 4400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.12267585843801498\n", + "episode 4500, step 4500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.036028504371643\n", + "episode 4600, step 4600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1284470558166504\n", + "episode 4700, step 4700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9523490071296692\n", + "episode 4800, step 4800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.13477925956249237\n", + "episode 4900, step 4900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1282380819320679\n", + "episode 5000, step 5000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.0970977246761322\n", + "episode 5100, step 5100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8987484574317932\n", + "episode 5200, step 5200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1034988164901733\n", + "episode 5300, step 5300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.163879156112671\n", + "episode 5400, step 5400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.04256010055542\n", + "episode 5500, step 5500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9595437049865723\n", + "episode 5600, step 5600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.021222231909632683\n", + "episode 5700, step 5700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.960745096206665\n", + "episode 5800, step 5800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.02389819361269474\n", + "episode 5900, step 5900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0328565835952759\n", + "episode 6000, step 6000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9791380167007446\n", + "episode 6100, step 6100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1301804780960083\n", + "episode 6200, step 6200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0440269708633423\n", + "episode 6300, step 6300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.07074788957834244\n", + "episode 6400, step 6400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9250450134277344\n", + "episode 6500, step 6500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1489771604537964\n", + "episode 6600, step 6600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.07468389719724655\n", + "episode 6700, step 6700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.954470157623291\n", + "episode 6800, step 6800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8496315479278564\n", + "episode 6900, step 6900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0665429830551147\n", + "episode 7000, step 7000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.14749585092067719\n", + "episode 7100, step 7100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.2052836418151855\n", + "episode 7200, step 7200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1009894609451294\n", + "episode 7300, step 7300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.172480821609497\n", + "episode 7400, step 7400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0841423273086548\n", + "episode 7500, step 7500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.972356379032135\n", + "episode 7600, step 7600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8190946578979492\n", + "episode 7700, step 7700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8057371377944946\n", + "episode 7800, step 7800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9624033570289612\n", + "episode 7900, step 7900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1302701234817505\n", + "episode 8000, step 8000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0299205780029297\n", + "episode 8100, step 8100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.09449701756238937\n", + "episode 8200, step 8200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.011588551104068756\n", + "episode 8300, step 8300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.0351133830845356\n", + "episode 8400, step 8400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.2069748640060425\n", + "episode 8500, step 8500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9828856587409973\n", + "episode 8600, step 8600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0410696268081665\n", + "episode 8700, step 8700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.101954460144043\n", + "episode 8800, step 8800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0768948793411255\n", + "episode 8900, step 8900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8896203637123108\n", + "episode 9000, step 9000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9952375888824463\n", + "episode 9100, step 9100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0309820175170898\n", + "episode 9200, step 9200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1290706396102905\n", + "episode 9300, step 9300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.24415305256843567\n", + "episode 9400, step 9400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1267493963241577\n", + "episode 9500, step 9500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0297152996063232\n", + "episode 9600, step 9600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8899804949760437\n", + "episode 9700, step 9700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.7753347158432007\n", + "episode 9800, step 9800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0508570671081543\n", + "episode 9900, step 9900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.02403193525969982\n", + "episode 10000, step 10000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8975008726119995\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABR7klEQVR4nO3deVzUdf4H8NfMwMxwDnKfgoiK9wGCeLdhlHbZZWVqbNmlrcVuh1vZVlvY5c+2LLvsNl3LrC3TlDQlURLvAxSRU7lEGO6Bmc/vD3SUBHWQme8cr+fjMY+V73yP93xxnVefz+f7+ciEEAJERERENk4udQFERERE3YGhhoiIiOwCQw0RERHZBYYaIiIisgsMNURERGQXGGqIiIjILjDUEBERkV1gqCEiIiK74CR1AZfDYDDgxIkT8PDwgEwmk7ocIiIiugxCCNTW1iI4OBhyufnbUWwi1Jw4cQJhYWFSl0FERERdUFRUhNDQULNfxyZCjYeHB4C2m+Lp6SlxNURERHQ5tFotwsLCjN/j5mYToeZsl5OnpydDDRERkY2x1NARDhQmIiIiu8BQQ0RERHaBoYaIiIjsgk2Mqbkcer0eLS0tUpfh0BQKBZycnPjYPRERScIuQk1dXR2Ki4shhJC6FIfn6uqKoKAgKJVKqUshIiIHY/OhRq/Xo7i4GK6urvDz82MrgUSEENDpdKioqMDx48fRp08fi0y0REREdFaXQs2SJUvw+uuvo7S0FEOHDsXbb7+NuLi4DvdtaWlBamoqPvvsM5SUlKBfv3549dVXce21115R4eefXwgBPz8/uLi4dMs5qWtcXFzg7OyMgoIC6HQ6qNVqqUsiIiIHYvJ/Sq9cuRIpKSl4/vnnsWvXLgwdOhRJSUkoLy/vcP9nn30W77//Pt5++20cOnQIDz30EKZOnYrdu3dfcfHnYwuNdWDrDBERScXkb6BFixZh9uzZSE5OxoABA7B06VK4urpi2bJlHe7/xRdf4J///CcmT56MyMhIPPzww5g8eTLefPPNKy6eiIiI6CyTQo1Op0NWVhYSExPPnUAuR2JiIjIyMjo8prm5+YJuCBcXF6Snp3d6nebmZmi12nYvIiIioosxKdRUVlZCr9cjICCg3faAgACUlpZ2eExSUhIWLVqEo0ePwmAwYMOGDVi9ejVOnjzZ6XVSU1Oh0WiML0dbzFImk2HNmjVSl0FERGRTzD4A4q233kKfPn0QHR0NpVKJuXPnIjk5+aJjL+bPn4+amhrjq6ioyNxlWty9996Lm2++ucP3Tp48ieuuu+6yz/Xpp5/Cy8urw/c6CkjffvstJk6cCI1GA3d3dwwZMgQvvvgiqqqqjOeTyWTGl7u7O2JiYrB69erLromIiMjSTAo1vr6+UCgUKCsra7e9rKwMgYGBHR7j5+eHNWvWoL6+HgUFBcjOzoa7uzsiIyM7vY5KpTIuXumIi1gGBgZCpVKZ5dzPPPMMpk2bhpEjR+Lnn3/GgQMH8Oabb2Lv3r344osvjPt5enri5MmTOHnyJHbv3o2kpCTccccdyMnJMUtdREQkjWXpx/HyT4dQ19wqdSlXzKRQo1QqERMTg7S0NOM2g8GAtLQ0JCQkXPRYtVqNkJAQtLa24ttvv8VNN93UtYovQQiBBl2rJK/umvzv/NaV/Px8yGQyrF69GldddRVcXV0xdOjQTscwXUxmZiZeeeUVvPnmm3j99dcxevRoREREYNKkSfj2228xa9asdjUEBgYiMDAQffr0wb///W/I5XLs27evWz4jERFJ72RNI974JQcfbj2O9Qc6HkZiS0yepyYlJQWzZs1CbGws4uLisHjxYtTX1yM5ORkAMHPmTISEhCA1NRUAsGPHDpSUlGDYsGEoKSnBv/71LxgMBjz55JPd+0nOaGzRY8CC9WY596UcejEJrkrzzGf4zDPP4I033kCfPn3wzDPP4K677kJubi6cnC7/el999RXc3d3xyCOPdPh+Z11Yer0en3/+OQBgxIgRJtdORETW6d8/HkaDTo+Y8B6YOjxE6nKumMnfwNOmTUNFRQUWLFiA0tJSDBs2DOvWrTMOHi4sLGw3XqapqQnPPvss8vLy4O7ujsmTJ+OLL77o9AuUOvaPf/wDU6ZMAQC88MILGDhwIHJzcxEdHX3Z5zh69CgiIyPh7Ox8yX1ramrg7u4OAGhsbISzszM++OAD9O7du2sfgIiIrMqWIxX4af9JyGXASzcNglxu+/O9dalZYe7cuZg7d26H723evLndzxMmTMChQ4e6cpkucXFW4NCLSRa73p+vbS5Dhgwx/jkoKAgAUF5eblKoMaV7zMPDA7t27QIANDQ0YOPGjXjooYfg4+ODG2644bLPQ0RE1qepRY8F3x8AANw7uhcGBNvH2FWbX/vpz2Qymdm6gKR0fuvK2dmTDQYDgLZBvfX19TAYDO1ayaqrqwEAGo0GANC3b1+kp6ejpaXlkq01crkcUVFRxp+HDBmCX375Ba+++ipDDRGRjftgSx7yTzXA30OFxyf1kbqcbsM57e1Av3790Nraij179rTbfralpW/fvgCAu+++G3V1dXj33Xc7PM/ZENQZhUKBxsbGK66XiIikU3iqAUs25QIAnr1+ADzUlx6SYCvsr0nDhtTU1FwQRHx8fEw+z8CBA3HNNdfgr3/9K958801ERkYiJycHjz32GKZNm4aQkLbBX/Hx8XjyySfx97//HSUlJZg6dSqCg4ORm5uLpUuXYuzYsZg3bx6Atq6qsxMqNjY2YsOGDVi/fj0WLFhwZR+aiIgkI4TA8z8cQHOrAaN7++CGIUFSl9StGGoktHnzZgwfPrzdtvvuu69L51q5ciWef/55PPjggzhx4gRCQ0MxdepUPPfcc+32e/XVVxETE4MlS5Zg6dKlMBgM6N27N2677bZ2j3RrtVrj2B2VSoXw8HC8+OKLeOqpp7pUHxERSe+XQ2XYlFMBZ4UML940yO4Wg5aJ7ppcxYy0Wi00Gg1qamoumIivqakJx48fR69evS5YY4osj78PIiLr1KBrxaRFW1BS3Yg5V/XGE0mX/6BJV13s+9scOKaGiIjIAbz9ay5KqhsR4uWCuVfZz+Dg8zHUEBER2bnc8lp8tDUPAPCvGwfCRWm+KUikxFBDRERkx4QQeG7NQbToBa6O9sekAQFSl2Q2DDVERER27Ie9J5CRdwoqJzn+deNAqcsxK7sJNTYw3tkh8PdARGQ9tE0t+PdPhwEAc6+KQpi3q8QVmZfNhxqFoq1fUKfTSVwJAW1LKgC4rPWliIjIvBb9cgQVtc3o5euGByZESl2O2dn8PDVOTk5wdXVFRUUFnJ2d2y0TQJYjhEBDQwPKy8vh5eVlDJtERCSNgydq8HlGPgDgxZsGQuVk//8u23yokclkCAoKwvHjx1FQUCB1OQ7Py8sLgYGBUpdBROTQDAaBZ9ccgEEAU4YEYVwfP6lLsgibDzUAoFQq0adPH3ZBSczZ2ZktNEREVmBVVhF2F1bDTanAc1MGSF2OxdhFqAHaVpXmDLZEROToTtfrsPDnbADA45P6IlDjON+NHIBCRERkR15bn43TDS2IDvTArNERUpdjUQw1REREdmJX4Wl8nVkEAHjp5kFwVjjW17xjfVoiIiI71ao34Lk1BwAAt8WEYmSEt8QVWR5DDRERkR34cnsBDp7QQuPijPnXmX8FbmvEUENERGTjyrVNePOXIwCAJ5L6wcddJXFF0mCoISIisnGvrD2M2uZWDA3V4K64nlKXIxmGGiIiIhu27Vgl1uw5AZmsbXCwQi6TuiTJMNQQERHZKF2rAQu+PwgAuCc+HENCvaQtSGIMNURERDbq4/TjyC2vg4+bEv+4pp/U5UiOoYaIiMgGlVQ34j9pRwEA/5zcHxpXZ4krkh5DDRERkQ168X8H0diiR1yEN24ZESJ1OVbBbtZ+IiIicgRCCKzKKsb6g2VQyGV46eZBkMkcd3Dw+RhqiIiIbMQf+VVY+HM2sgpOAwD+OiYC/QI9JK7KejDUEBERWbkjZbV4bV02Nh4uBwConeW4b2wvzLu6r8SVWReGGiIiIit1oroR/7fhCL7dVQyDABRyGe6IDcNjiX0Q4KmWujyrw1BDRERkZaobdHhv8zF8si0fulYDAOC6QYH4R1I/9PZzl7g668VQQ0REZCWaWvT45Pd8vLc5F9qmVgBAfC9vPHVdNEb07CFxddaPoYaIiEhirXoDvskqxuKNR1GqbQIARAd64KnrojGxrx+fbrpMDDVEREQSEULgl0NleG1dNo5V1AMAQrxc8Pdr+uKmYSEOvY5TVzDUEBERSSDzeBUW/nwYuwqrAQA9XJ0x56oo3DMqHGpnhbTF2SiGGiIiIgvKKW17PDst+9zj2fePjcQDEyLhqeZSB1eCoYaIiMgCTtfr8PLaw/h2VzHEmcez7xwZhnlX94E/H8/uFgw1REREZpZdqsXsz3eiqKoRADB5cCD+cU0/RPLx7G7FUENERGRG6w6UIuW/e9Cg0yPM2wVv3Tmcj2ebSZdW6V6yZAkiIiKgVqsRHx+PzMzMi+6/ePFi9OvXDy4uLggLC8Pjjz+OpqamLhVMRERkCwwGgbc2HsVDX2ahQafH6N4++GHOWAYaMzK5pWblypVISUnB0qVLER8fj8WLFyMpKQk5OTnw9/e/YP/ly5fj6aefxrJlyzB69GgcOXIE9957L2QyGRYtWtQtH4KIiMia1De34h+r9uLnA6UAgHtHR+DZKf3hpOhSWwJdJpkQQphyQHx8PEaOHIl33nkHAGAwGBAWFoZHH30UTz/99AX7z507F4cPH0ZaWppx29///nfs2LED6enpl3VNrVYLjUaDmpoaeHp6mlIuERGRRRVVNWD25zuRXVoLZ4UML988GHeMDJO6LElY+vvbpMio0+mQlZWFxMTEcyeQy5GYmIiMjIwOjxk9ejSysrKMXVR5eXlYu3YtJk+e3Ol1mpubodVq272IiIisXcaxU7jxnXRkl9bC112FFQ+McthAIwWTup8qKyuh1+sREBDQbntAQACys7M7PObuu+9GZWUlxo4dCyEEWltb8dBDD+Gf//xnp9dJTU3FCy+8YEppREREkvpiewFe+OEgWg0Cg0M0eH9GDIK9XKQuy6GYvXNv8+bNeOWVV/Duu+9i165dWL16NX766Se89NJLnR4zf/581NTUGF9FRUXmLpOIiKhLdK0G/PO7/XhuzQG0GgRuHBqMVQ8lMNBIwKSWGl9fXygUCpSVlbXbXlZWhsDAwA6Pee655zBjxgzcf//9AIDBgwejvr4eDzzwAJ555hnI5RfmKpVKBZVKZUppREREFldZ14xHvtyFzPwqyGTAk0nReGhCJBeglIhJLTVKpRIxMTHtBv0aDAakpaUhISGhw2MaGhouCC4KRduaFiaOUSYiIrIaB0/U4KZ3fkdmfhU8VE74eFYsHp7Ym4FGQiY/0p2SkoJZs2YhNjYWcXFxWLx4Merr65GcnAwAmDlzJkJCQpCamgoAuOGGG7Bo0SIMHz4c8fHxyM3NxXPPPYcbbrjBGG6IiIhsyY/7TuAfq/aiqcWAXr5u+HBmLKL8OTuw1EwONdOmTUNFRQUWLFiA0tJSDBs2DOvWrTMOHi4sLGzXMvPss89CJpPh2WefRUlJCfz8/HDDDTfg5Zdf7r5PQUREZAEGg8CiDUfwzqZcAMD4vn54+87h0LhyIUprYPI8NVLgPDVERPZP12pAXXMrvN2UUpfSodqmFjy+ci82Hm4bV/rA+Eg8dW00FHJ2N3XG0t/fXPuJiIiswsNfZmHzkQq8eNNATI8Pl7qcdgpO1eP+z3biaHkdlE5yLLxlMG4ZESp1WfQnDDVERCS5AyU1SMsuBwA8890BVNbq8Lero6xi0G360UrMWb4LNY0tCPBU4f0ZsRgW5iV1WdQBLkJBRESS+2hrHgAg5MzcLv+38Qie+/4A9AbpRkgIIfDR1jzMXLYDNY0tGBbmhR/mjmWgsWIMNUREJKkT1Y34376TAID3Z8TgpZsGQiYDvtxeiLnLd6GpRW/xmhp1ejy+cg/+/dNhGARw64hQrHhgFAI81RavhS4fu5+IiEhSn27Lh94gMCrSG4NCNBgUooGPuwqPrdiDnw+U4nRDJj6YGQtPtWWeMCqqasCDX2Th0EktFHIZnpvSH7NGR1hFVxhdHFtqiIhIMrVNLfh6RyEAYPa4SOP2yYOD8OlfR8Jd5YTteVWY9v52lGubzF7PttxK3PhOOg6d1MLHTYmv7o/HvWN6MdDYCIYaIiKSzMo/ilDb3IpIPzdc1c+/3Xuje/tixQOj4OuuwuGTWtzy3jYcr6w3Sx1nx8/c8/EOnG5oweAQDf736FiMivQxy/XIPBhqiIhIEq16Az75PR9AWyuNvIP5XgaFaLD64dEI93FF8elG3PbeNuwrru7WOv48fuaWESFckNJGMdQQEZEkfj5QipLqRvi4KTF1eEin+/X0ccU3D43GoBBPnKrX4c4PtmPr0YpuqaGoqgG3vrcNa/acgEIuw/M3DMCbtw+F2pnL+NgihhoiIrK4s909ADAjIfySIcLPQ4UVDyRgTJQPGnR6/PXTP/D9npIrqqGj8TPJHD9j0xhqiIjI4jKPV2FvcQ1UTnLMGHV5swe7q5yw7N6RuH5IEFr0AvNW7MGy9OMmX7uj8TM/cPyMXeAj3UREZHEfbm0LI7eMCIWPu+qyj1M5KfCfO4fD112FT7fl48UfD6GirhlPJvW7rBaWRp0e81fvw5o9J85cPwSvTB3M7iY7wVBDREQWlVdRh7TstkUh7xvby+Tj5WfGvvh7qvDauhy8t/kYKmqbkXrLYDgrOu+AKD7dNv/MwRNt8888O6U/7uX8M3aFoYaIiCzq4/TjEAK4OtofUf7uXTqHTCbDIxOj4Oumwvzv9uObrGJU1euw5O4RcFFe2OqyLbdt/abTDS3wcVNiyfQR7G6yQxxTQ0REFlNVr8M3WcUAgNnjIy+x96XdMTIM798TA5WTHL9ml2P6R9txul5nfP/s+JkZyzI5fsYBMNQQEZHFfLm9AM2tBgwO0SC+l3e3nDNxQAC+uj8eGhdn7Cqsxu3vZ+BEdWO7+Wf0BmGcfyaE88/YLXY/ERGRRTS16PF5Rj4A4P5x3fvodGyEN1Y9lIBZyzKRW16HW97dBm83pXH9Jo6fcQxsqSEiIotYs7sElXU6BGvUmDw4qNvP3zfAA98+PBpR/u4o1Tbh0EktvN2U+PI+zj/jKNhSQ0REZmcwCHx0Zk6Z5DG9LvqU0pUI9nLBqgcT8I9Ve6HTG7Dw1iHsbnIgDDVERGR2vx2pQG55HdxVTpgWF2bWa/VwU+Lje0ea9Rpkndj9REREZvfhmSUR7ooLg6faWeJqyF4x1BARkVkdPFGDbcdOQSGX4d4xpk+2R3S5GGqIiMisPjqzJMKUwUEc30JmxVBDRERmc7KmEf/b27bO0uxxVz7ZHtHFMNQQEZHZfPp7PloNAvG9vDE4VCN1OWTnGGqIiMgs6ppbsTyzEABbacgyGGqIiMgsVv5RhNqmVkT6ueEv0f5Sl0MOgKGGiIi6XavegGVnJtu7f2wk5HLO5kvmx1BDRETdbt3BUpRUN8LHTYlbRoRIXQ45CIYaIiLqVkIIfHjmMe57RoVD7ayQuCJyFAw1RETUrXYWnMbeomooneSYkRAudTnkQBhqiIioW32wpW1JhFtHhMDXXSVxNeRIGGqIiKjbHK+sx8bDZQCA+8byMW6yLIYaIiLqNh+n50EI4C/R/ojyd5e6HHIwDDVERNQtTtfr8E1WMQBOtkfSYKghIqJu8eX2AjS1GDAoxBOjIr2lLoccEEMNERFdsaYWPT7LKADQ1kojk3GyPbI8hhoiIrpiP+w5gcq6ZgRp1Jg8OEjqcshBOUldABGRNTl8UovPMwpQcKoeqbcMRriPm9QlWb22yfbaHuNOHhMBZwX/e5mk0aW/eUuWLEFERATUajXi4+ORmZnZ6b4TJ06ETCa74DVlypQuF01E1J1a9Ab8tO8k7ng/A9e9tRVfZxZi27FTePCLLDTq9FKXZ/U2H6nA0fI6uKuccGdcT6nLIQdmckvNypUrkZKSgqVLlyI+Ph6LFy9GUlIScnJy4O9/4Sqsq1evhk6nM/586tQpDB06FLfffvuVVU5EdIXKa5uwIrMIX+0oQJm2GQCgkMuQNDAAmcdPI7u0Fv/8bj8W3TGUY0Qu4qMzrTR3jgyDp9pZ4mrIkZkcahYtWoTZs2cjOTkZALB06VL89NNPWLZsGZ5++ukL9vf2bj8CfsWKFXB1dWWoISJJCCGwq7Aan2fkY+3+k2jRCwCAr7sSd8f1xN3x4QjUqLE97xSmf7QD3+0uwYieXpiRECFt4Vbq4Ika/J57Cgq5DMlje0ldDjk4k0KNTqdDVlYW5s+fb9wml8uRmJiIjIyMyzrHxx9/jDvvvBNubp33Uzc3N6O5udn4s1arNaVMIqILNLXo8cPeE/g8Ix8HSs79mzKipxdmjY7AtYMCoXI6t/DiqEgfPH1tNF5eexgv/ngIA0M0GNGzhxSlWyUhBH7afxLPf38QADB5cBBCvFwkroocnUmhprKyEnq9HgEBAe22BwQEIDs7+5LHZ2Zm4sCBA/j4448vul9qaipeeOEFU0ojIupQUVUDvtxRgJV/FKG6oQUAoHSS46ahwZiZEIHBoZpOj71/XC/sLjqNtftL8ciXu/Dj38ZyLSMA5domPLvmAH451LYcQt8Adzx1bT+JqyKy8NNPH3/8MQYPHoy4uLiL7jd//nykpKQYf9ZqtQgLCzN3eURkJwwGgd+PVeKzbQVIyy6DaOthQoiXC2YkhOOO2DB4uykveR6ZTIbXbhuKnNJaHKuox6PLd+OL++Lg5KBP9wghsCqrGP/+8RC0Ta1wkssw56oozLkqCkonx7wnZF1MCjW+vr5QKBQoKytrt72srAyBgYEXPba+vh4rVqzAiy++eMnrqFQqqFT8ryEiMk1tUwu+zSrG59sLkFdRb9w+ro8vZiZE4C/R/lDITRvw665ywvszYnDjO78jI+8U3vjlCJ6+Lrq7S7d6RVUN+Od3+7H1aCUAYEioBq/dNgTRgZ4SV0Z0jkmhRqlUIiYmBmlpabj55psBAAaDAWlpaZg7d+5Fj121ahWam5txzz33dLlYIqLO/J5biQc+34n6M49gu6uccFtMKO4ZFX7FCytG+XvgtduGYO7y3Vj62zEMC/PCtYMu/h9y9sJgEPhiewFeXZeNBp0eKic5Uib1xX1jezlsixVZL5O7n1JSUjBr1izExsYiLi4OixcvRn19vfFpqJkzZyIkJASpqantjvv4449x8803w8fHp3sqJyI6o1VvwLNrDqBep0dvPzfcOzoCU0eEwl3VfT3s1w8Jxu7Canycfhz/WLUXfQPcEeln36tQH6uow9Pf7sMf+acBAHER3lh462C7/9xku0z+f/y0adNQUVGBBQsWoLS0FMOGDcO6deuMg4cLCwshl7dP7zk5OUhPT8cvv/zSPVUTEZ3nm6xiHK+sh4+bEt/PHdutYeZ8T18Xjf3FNcjMr8JDX2ZhzZwxcFXa38TsrXoDPtx6HP+38Qh0rQa4KRV4+rpoTI8Ph9zE7jsiS5IJcXYInfXSarXQaDSoqamBpyf7b4nonKYWPa56YzNO1jThuesH4D4zz5VSrm3ClLfTUVHbjBuHBuOtO4fZ1cR8h09q8eQ3+7C/pAYAML6vH16ZOgihPVwlroxskaW/v9khSkQ27cvtBThZ04RgjRrT480/Rb+/pxrvTh8BJ7kMP+w9gU+35Zv9mpbQ3KrHol9ycMPb6dhfUgNPtRPeuH0oPkseyUBDNoOhhohsVl1zK97dfAwAMC+xD9TOiksc0T1GRnjjn5P7AwBe/ukwduZXWeS65rK78DSu/086/vNrLloNAkkDA7AxZQJuiwm1q1Yosn8MNURksz7eehxV9TpE+rrh1hGhFr128pgIXD8kCK0GgUe+2oXy2iaLXr87NOr0+PePh3Dre9twtLwOvu5KLLl7BJbeEwN/T7XU5RGZjKGGiGxSVb0OH55ZSDHlmr4Wf7xYJpPh1VuHoI+/O8prmzF3+W606A0WreFKZBw7hWvf2oKP0o/DIICpw0Ow4fEJmDIkiK0zZLMYaojIJi397RjqmlsxMNgTkwcFSVKDm8oJS2fEwF3lhMzjVXht3aWXi5GSEALpRysxc1km7vpwOwpONSBIo8Yn947E/00bhh6XMcsykTWzv2cRicjuldY04bMzA3T/kdRP0seMe/u5443bh+ChL3fhw63HMbxnD0weLE3I6kyL3oC1+0/igy15OHiibTFPuQy4M64n5l8XDQ+1s8QVEnUPhhoisjlvpR1Fc6sBcRHemNjXT+pycO2gIDw4PhLvb8nDE2cm5ovy95C6LNQ1t2JFZiE++T0fJdWNAAAXZwXuiA3FfWMj0dOHTzWRfWGoISKbkl9Zj//uLAIAPHFtP6sZ//FEUj/sLa7G9rwqPPhFllknAbyUcm0TPtmWj6+2F0Db1AoA8HVXYlZCBO4ZFc5uJrJbDDVEZFMWbTgCvUHgqn5+GBnhLXU5Rk4KOd6+awSuf3srjlXU46lv9uGdu4dbNHQdLavFh1vzsGb3CejODFqO9HXD/eMiccuIEIs98k4kFYYaIrIZh05o8cPeEwCAv1/TT+JqLuTnocK702Nw5wcZ+Gn/SQxP98L94yLNek0hBHYcr8IHW/Lwa3a5cXtseA88MD4Sif0DuLQBOQyGGiKyGW/+kgMAuH5IEAaFaCSupmMx4T3w3PUDsOD7g0j9ORuDQzSIj+z+hXxb9QasP1iGD7Ycw97itiUNZDLgmgEBeGB8b8SE9+j2axJZO4YaIrIJWQVVSMsuh0IuQ8qkvlKXc1EzRoVjV8FprNlzAnOW78ZPfxuLgG6azK5B14pVO4vxUXoeiqraBv+qnOS4LSYU943txRW0yaEx1BCR1RNC4LV1ba00t8eEWv0Xt0wmwyu3DEZ2aS2yS2tx63vbEO7jCrlMduYFKOQyyC74swwKGSCXtf2skLf/c6teYN3BUlQ3tAAAerg6Y0ZCBGYmhMPXXSXxpyaSHkMNEVm9LUcrseN4FZROcvzt6j5Sl3NZXJVOWHpPDG54Jx3FpxtRfLqx287d09sVs8f1wm0xYXBRcvAv0VkMNURk1YQQeH1920y9M0aFI9jLReKKLl+ErxvS/j4B2/OqIISAQQjoDYBBCIgL/ixgEG0/t73O/NnQ/s8DgjWYNCAACg7+JboAQw0RWbWfD5TiQIkWbkoFHpnYW+pyTObvocaNQ4OlLoPIIXDtJyKyWq16g/GJp/vGRcKH40aI6CIYaojIaq3eXYJjFfXwcnXG7HG9pC6HiKwcQw0RWaXmVj3e2ngUAPDIxN5cdJGILomhhois0vIdhSipbkSgpxozEyKkLoeIbABDDRFZnfrmVrzzay4A4G9X9+GaRUR0WRhqiMjqfPL7cZyq1yHCxxW3x4ZKXQ4R2QiGGiKyKtUNOry/JQ8A8PikvnBW8J8pIro8/NeCiKzK0t/yUNvUiuhAD9wwhPO7ENHlY6ghIqtRrm3Cp9uOAwCeSOoHOWfNJSITMNQQkdV4+9dcNLUYMKKnF/4S7S91OURkYxhqiMgqFJ5qwNeZhQCAJ6+NhkzGVhoiMg1DDRFZhcUbj6DVIDCujy9GRfpIXQ4R2SCGGiKSXE5pLb7bUwIAeDIpWuJqiMhWMdQQkeTe+CUHQgDXDQrE4FCN1OUQkY1iqCEiSe0uPI0Nh8oglwF/v6av1OUQkQ1jqCEiSb2+PgcAcOuIUET5e0hcDRHZMoYaIpLMr9ll2HbsFJQKOeYl9pG6HCKycQw1RCQJbVML/rn6AABg1uhwhPZwlbgiIrJ1DDVEJImXfzyMUm0TInxckTKpn9TlEJEdYKghIovbcqQCK3cWQSYDXrttKFyUCqlLIiI7wFBDRBZV29SCp7/dBwCYlRCBuF7eEldERPaCoYaILOqVtdk4UdOEnt6uePJadjsRUfdhqCEii0k/Wmlc3+nVW4fAVekkcUVEZE+6FGqWLFmCiIgIqNVqxMfHIzMz86L7V1dXY86cOQgKCoJKpULfvn2xdu3aLhVMRLaprrkVT53pdpoxKhwJvbm+ExF1L5P/M2nlypVISUnB0qVLER8fj8WLFyMpKQk5OTnw9/e/YH+dTodJkybB398f33zzDUJCQlBQUAAvL6/uqJ+IbMSrP2ejpLoRoT1c8PR1XN+JiLqfTAghTDkgPj4eI0eOxDvvvAMAMBgMCAsLw6OPPoqnn376gv2XLl2K119/HdnZ2XB2du5SkVqtFhqNBjU1NfD09OzSOYhIOtuOVeLuD3cAAL66Px5jonwlroiILMHS398mdT/pdDpkZWUhMTHx3AnkciQmJiIjI6PDY3744QckJCRgzpw5CAgIwKBBg/DKK69Ar9d3ep3m5mZotdp2LyKyTQ26c91Od8f3ZKAhIrMxKdRUVlZCr9cjICCg3faAgACUlpZ2eExeXh6++eYb6PV6rF27Fs899xzefPNN/Pvf/+70OqmpqdBoNMZXWFiYKWUSkRV5bV0OiqoaEaxRYz67nYjIjMz+9JPBYIC/vz8++OADxMTEYNq0aXjmmWewdOnSTo+ZP38+ampqjK+ioiJzl0lEZrAj7xQ+3ZYPAFh46xB4qLvWBU1EdDlMGijs6+sLhUKBsrKydtvLysoQGBjY4TFBQUFwdnaGQnFuxtD+/fujtLQUOp0OSqXygmNUKhVUKpUppRGRlWnU6fHkmW6nabFhGN/XT+KKiMjemRRqlEolYmJikJaWhptvvhlAW0tMWloa5s6d2+ExY8aMwfLly2EwGCCXtzUMHTlyBEFBQR0GGiJbZjAIbDt2ChsPl0FvEFA6ydteirb/VZ15nduu6HCf83/2UDvZZAvHG7/koOBUA4I0ajxzfX+pyyEiB2DyI90pKSmYNWsWYmNjERcXh8WLF6O+vh7JyckAgJkzZyIkJASpqakAgIcffhjvvPMO5s2bh0cffRRHjx7FK6+8gr/97W/d+0nIIX2/pwQnqptwzcAA9PZzl6yO4tMNWLWzGN9kFaOkurFbzy2TARP6+uGuuJ64OtofTgrrnzNzZ34Vlv1+HADwyi2D4WmDoYyIbI/JoWbatGmoqKjAggULUFpaimHDhmHdunXGwcOFhYXGFhkACAsLw/r16/H4449jyJAhCAkJwbx58/DUU09136cgh3T4pBbzVuwBALy6LhvRgR64blAQpgwJRJS/h9mv39Six/qDpVi1sxi/H6vE2ckRPNROuH5IMPzclWjWG6BrPe+lb//n5ou8d/6fN+dUYHNOBQI8VZgWG4ZpcT0R4uVi9s/YFU0tejz5zT4IAdwWE4qr+l04fxURkTmYPE+NFDhPDXXk8ZV78N3uEvh7qFBVr0Or4dxf5T7+7pg8OAhThgShb0D3BpwDJTX4784irNldAm1Tq3H7mCgf3BEbhqSBgVA7d9+q08cr67Hij0J8s7MYp+p1AAC5DJjYzx93x/XExH5+VtV688raw/hgSx78PVTY8PgEaFzZSkPkqCz9/c1QQzappLoR41/bBL1B4Ie5YxDu7YZfDpXi5wOl2Hq0Ai36c3+to/zdMXlQICYPCUK/AA/IZDKTr1fdoMOa3SX4785iHDp5bt6kYI0at8WG4faYUIR5u3bLZ+tMc6sevxwsw/IdhcjIO2XcHqRR447YMNwZF4YgjbStN7sKT+O297bBIICPZsYicUDApQ8iIrvFUNMBhhr6s5d+PISP048jIdIHXz8wqt17NY0tSDtchrX7T2LLkUro9Abje5G+bpg8OAiTBwehf9DFA47BIJCeW4n/7izCLwfLjOdRKuS4ZmAA7ogNw5goXyjkpoekK5VXUYcVfxThm6xiVJ3XevOXaH/cHd8TE/r6W7yuphY9pvxnK45V1GPq8BD837RhFr0+EVkfhpoOMNTQ+WoaWpCwMA0NOj0+TR6JiRcZs6FtasGvh8vx0/6T+O1IBXSt5wJOhI+rMeAMDPY0BpyiqgasyirGt38a9Ns/yBPTYkNx8/AQeLlax5N7za16rDtQiq8zC7E9r8q4PVijxrSRPTFtZBgCNWqL1LLw52ws/e0YfN1V2Jgy3mruERFJh6GmAww10mvQtWJXQTVG9/aBXIKWifMt2ZSL19fnIDrQAz/PG3fZ3Um1TS34Nbsca/efxOacCjSfF3B6erti0oAAZJdq8Xvuua4dT7UTbh4egjtiwzAoRNPtn6U7Hauow9c7CvHNrmJUN7QAABRymbH1ZnwfP7O13uwtqsbUd3+HQQDvz4hB0sCO560iIsfCUNMBhhrpPbtmP77cXoh5V/fB45P6SlZHU4seY1/dhMq6Ziy6YyhuGRHapfPUN7caA86mnHI0tRjavT82yhd3jAzDNQMCunXQryU0tbS13izPLETm8XOtNyFeLrg9NhRJAwMRHdi1sUUdaW7V4/r/pONoeR1uHBqM/9w1vFvOS0S2j6GmAww10tIbBEa+vBFV9Tq4OCvw2xMT4e9pmS6NP1uRWYinV+9HsEaN3568Cs7d8NRPg64Vm7Ir8NuRcgR7ueDWEeYf9GspueW1WL6jCN/uKkZNY4txe2gPFyT2D8CkAQGI6+V9RffxjfU5eGdTLnzdlfjl8QnwdmO3ExG1YajpAEONtHbmV+G2pedWYZ8e3xMvTx1s8ToMBoHE//sNeRX1eHZKf9w/LtLiNdiqphY9fj5wEj/tK0V6bkW7likPtROu6uePxAEBmNDXDxqXy38Ee39xDW5+93foDQLvTR+B6wYHmaN8IrJRlv7+NnnyPXI8Gw63rfXVx98dR8vbnrr569heFp/Bd+PhMuRV1MND7YQ743pa9Nq2Tu2swNThoZg6PBSNOj3Scyux8VAZ0rLLUFmnww97T+CHvSfgJJchPtIbif0DkNg/4KItVrpWA574Zi/0BoEpQ4IYaIhIcgw1dEkbD7WFmr9d3QdrdpcgLbscr6/LwdIZMRat4/0teQCAe0aFw13Fv7pd5aJUYNKAtq4nvUFgT1E1Nh4uw8ZDZThaXoffc0/h99xTeOF/hxAd6IFJA9oCzuAQTbtB4u9sykV2aS283ZR48caBEn4iIqI2/Gagi8qrqMOxino4K2SY0M8PfQM8sCmnHOsOlmJX4WmM6NnDInXszK9CVsFpKBVyJI+OsMg1HYFCLkNMeA/EhPfAU9dGI7+yHhsPl2HDoTL8kV+F7NJaZJfW4u1fcxHgqcLV/QMwqX8ANK7OeHdTLgDgxZsGwsddJfEnISJiqKFL2Him62lUpA881c7wDHTGrSNCsSqrGAvXZmPlg6O67SmaiznbSjN1eIhkg5QdQYSvG+4fF4n7x0XidL0Om4+UY8OhMvyWU4EybTOW7yjE8h2Fxv2vHRiIKex2IiIrwVBDF7XxUDkAILH/uenuH5/UFz/sPYHM/Cr8ml2Oq/ubdyr83PI6bDjTBTZ7PAcHW0oPN6VxHE5zqx7b86qw8VAZNh4uw8maJvi4KfHSzYMsEmqJiC4HQw11qqpeh50FbfOcXN3/3Ky9wV4uuHdMBN7/LQ+vrsvGxH7mnZL/o61trTSJ/QMQ5W/ZwcnURuWkwIS+fpjQ1w8v3jQQR8rq0MPNGX4e7HYiIuthPUv7ktXZlF0Og2hbHiC0R/unYB6ZEAWNizOOlNXh26xis9VQrm3C6l0lAICHJrCVxhrIZDL0C/SAvwe7AYnIujDUUKfOjqeZ1P/CtZU0rs6Ye1UUAGDRhiNoatGbpYZPt+VDpzcgJrwHYiO8zXINIiKyDww11KGmFj1+O1IBAEgc0PGYmRkJ4QjxckGptgmf/J7f7TXUNbfii+0FAIAHOJaGiIgugaGGOpSRdwoNOj0CPFUYFNzxQo5qZwVSzqwD9e7mXFQ36Lq1hhWZhahtakWkrxsmmXkwMhER2T6GGurQ2Qn3EvsHXHRV7puHhyA60AO1Ta1Ycmbeku7QojdgWfpxAG1PPEm9MjgREVk/hhq6gBDCOJ6ms66nsxRyGZ66LhoA8Nm2AhSfbuiWGn7cdwInaprg667C1OEh3XJOIiKybww1dIEDJVqUaZvhqlQgIdLnkvtP7OuHhEgf6PQGLNpw5IqvL4TA+7+1PcadPCYCamfFFZ+TiIjsH0MNXeDsApbj+/hdVqCQyWR4+kxrzXe7S3DohPaKrr/laCWyS2vhqlTgnvjwKzoXERE5DoYauoBxPM0lup7ONzTMC1OGBEEI4LX12Vd0/fd/OwYAuHNkT2hcna/oXERE5DgYaqid4tMNOHRSC7kM+Ev0hfPTXMwT1/SDk1yGzTkV2HasskvX319cg23HTkEhl+G+cb26dA4iInJMDDXUTtrhtrWeYsO94e2mNOnYCF83TI/vCQBY+HM2DAZh8vXf39LWSnPj0GCEeLmYfDwRETkuhhpq59xTT6a10pz16NV94KZUYF9xDdYeOGnSsYWnGrB2f9sxs8dxsj0iIjINQw0ZaZtasD3vFID2q3KbwtddhQfG9wYAvL4+B7pWw2Uf+3F6HgwCGN/XDwOCPbt0fSIiclwMNWS05UgFWvQCkX5uiPTr+mrY94/rBV93FQpONWDFH4WXdUxVvQ4rdxYBAB7kkghERNQFDDVkdPappytdksBN5YR5iX0AAG9tPIq65tZLHvNFRgGaWgwYFOKJ0b0vPTcOERHRnzHUEIC2ZQl+zW4bJGzKo9yduXNkGCJ93XCqXocPtuRddN9GnR6fZeQDAB4Y3xsyGZdEICIi0zHUEADgj/wqaJta4e2mxIiePa74fM4KOZ5I6gcA+GhrHsprmzrd95tdxaiq1yG0hwsmDwq84msTEZFjYqghAMDGQ22tNH+J9oeimxaPvHZQIIaFeaFBp8d/0o52uI/eIPDR1raWnPvH9oKTgn8liYioa/gNQhBCYMPhUgBdf+qpIzKZDPPPLJ/wdWYR8irqLthn/cFSFJxqgJerM+4YGdZt1yYiIsfDUEM4Wl6HoqpGKJ3kGNfHt1vPHR/pg6uj/aE3CLzxS06799oWrmybbG9mQgRclU7dem0iInIsDDWEDWeeehrT2wduqu4PFk9eGw25DFi7vxS7C08bt+84XoW9xTVQOckxK4ELVxIR0ZVhqKHzZhHuvq6n8/UL9MCtI0IBAKk/Z0OItuUTzrbS3B4bCh93lVmuTUREjoOhxsGV1zZhT1E1gO4dT/Nnj0/qC5WTHJnHq7Appxw5pbXYlFMBmQy4fywn2yMioivHUOPgfj1cDiGAoaEaBHiqzXadYC8X3DsmAgDw6s85WHqmlea6QYGI8HUz23WJiMhxMNQ4OGPXkxlbac56ZEIUNC7OyCmrxXe7SwDAuE4UERHRlWKocWCNOj22Hq0EYL7xNOfTuDpj7lVRxp/je3ljWJiX2a9LRESOoUuhZsmSJYiIiIBarUZ8fDwyMzM73ffTTz+FTCZr91KrzdfNQZcvPbcSza0GhHi5IDrQwyLXnJEQjhAvFwDAQxPZSkNERN3H5Od3V65ciZSUFCxduhTx8fFYvHgxkpKSkJOTA39//w6P8fT0RE7OuTlKuLaPdTAuYDkgwGK/E7WzAiseGIWCUw0Y281z4hARkWMzuaVm0aJFmD17NpKTkzFgwAAsXboUrq6uWLZsWafHyGQyBAYGGl8BAebv6qCL0xsE0rItN57mfGHergw0RETU7UwKNTqdDllZWUhMTDx3ArkciYmJyMjI6PS4uro6hIeHIywsDDfddBMOHjx40es0NzdDq9W2e1H32lNUjco6HTzUToiP9Ja6HCIioitmUqiprKyEXq+/oKUlICAApaWlHR7Tr18/LFu2DN9//z2+/PJLGAwGjB49GsXFxZ1eJzU1FRqNxvgKC+OaQN3t7FNPE/v5w5mLSBIRkR0w+7dZQkICZs6ciWHDhmHChAlYvXo1/Pz88P7773d6zPz581FTU2N8FRUVmbtMh3N2PE1i/47HQREREdkakwYK+/r6QqFQoKysrN32srIyBAYGXtY5nJ2dMXz4cOTm5na6j0qlgkrFafPNJb+yHkfL6+Akl2FiX4YaIiKyDya11CiVSsTExCAtLc24zWAwIC0tDQkJCZd1Dr1ej/379yMoKMi0SqnbnO16iuvlDY2rs8TVEBERdQ+TH+lOSUnBrFmzEBsbi7i4OCxevBj19fVITk4GAMycORMhISFITU0FALz44osYNWoUoqKiUF1djddffx0FBQW4//77u/eT0GXbcEiap56IiIjMyeRQM23aNFRUVGDBggUoLS3FsGHDsG7dOuPg4cLCQsjl5xqATp8+jdmzZ6O0tBQ9evRATEwMtm3bhgEDBnTfp6DLdrpeh50FpwG0zU9DRERkL2RCCCF1EZei1Wqh0WhQU1MDT09Pqcuxad/tLsbjK/ciOtAD6x4bL3U5RERkxyz9/c1neR3MxkPlANj1RERE9oehxoE0t+rx25EKAJZZwJKIiMiSGGocyI68KtQ1t8LPQ4UhIRqpyyEiIupWDDUOZMN5E+7J5VxUlIiI7AtDjYMQQhjnp+F4GiIiskcMNQ7i4AktTtY0wcVZgTFRXCGbiIjsD0ONgzjbSjOujy/UzgqJqyEiIup+DDUOwtj1xKeeiIjITjHUOICTNY04UKKFTAb8JZoLWBIRkX1iqHEAG8889TSiZw/4unP1cyIisk8MNQ5gw2HOIkxERPaPocbO1Ta1IONYJQBg0gB2PRERkf1iqLFzW49WokUv0MvXDb393KUuh4iIyGwYauzcxvNmEZbJOIswERHZL4YaO9aqN+DXHI6nISIix+AkdQFkHtUNOixLP47qhhZ4uTojJryH1CURERGZFUONnTlQUoPPM/Lx/Z4TaG41AACmDg+Bk4KNckREZN8YauxAc6seP+8vxecZ+dhVWG3c3j/IEzMTwnFbTKh0xREREVkIQ40NO1HdiOU7CrHij0JU1ukAAE5yGa4bHIRZCeGICe/BwcFEROQwGGpsjBAC246dwucZ+dhwqAwG0bY90FONu+N74s64MPh7qKUtkoiISAIMNTaitqkFq3eV4IvtBcgtrzNuHxXpjZkJEZg0IADOHDdDREQOjKHGyh0pq8XnGfn4blcJ6nV6AICbUoFbRoRiRkI4+gZ4SFwhERGRdWCosUItegM2HCrD5xn52J5XZdze288NMxMicMuIEHionSWskIiIyPow1FiZ0/U63LZ0G45V1AMA5DJg0oAAzEqIQEJvHw78JSIi6gRDjZX5OP04jlXUw8vVGffEh+Pu+J4I9nKRuiwiIiKrx1BjRWoaW/DZtnwAwKu3DkHSwEBpCyIiIrIhfFzGinyRkY/a5lb0DXDHJK7VREREZBKGGitR39yKj9OPAwDmXBUFuZxjZ4iIiEzBUGMllu8oxOmGFkT4uOL6IcFSl0NERGRzGGqsQFOLHh9szQMAPDIxCgq20hAREZmMocYKrNpZhIraZgRr1Lh5eIjU5RAREdkkhhqJtegNWPpbWyvNQxN7Q+nEXwkREVFX8BtUYt/tLkFJdSN83VW4IzZM6nKIiIhsFkONhPQGgfc2HwMAPDC+F9TOCokrIiIisl0MNRL6af9JHK9smz14eny41OUQERHZNIYaiRgMAkt+zQUA/HVML7ipOLkzERHRlWCokcjGw2XIKauFu8oJsxIipC6HiIjI5jHUSEAIgSWb2lppZiaEQ+PqLHFFREREto+hRgJbj1Zib3EN1M5y3De2l9TlEBER2YUuhZolS5YgIiICarUa8fHxyMzMvKzjVqxYAZlMhptvvrkrl7Ub75wZS3N3XDh83FUSV0NERGQfTA41K1euREpKCp5//nns2rULQ4cORVJSEsrLyy96XH5+Pv7xj39g3LhxXS7WHuzIO4XM/CooFXI8MD5S6nKIiIjshsmhZtGiRZg9ezaSk5MxYMAALF26FK6urli2bFmnx+j1ekyfPh0vvPACIiMd+4v8nTNjaW6LDUWgRi1xNURERPbDpFCj0+mQlZWFxMTEcyeQy5GYmIiMjIxOj3vxxRfh7++P++6777Ku09zcDK1W2+5lD/YWVWPr0Uoo5DI8PKG31OUQERHZFZNCTWVlJfR6PQICAtptDwgIQGlpaYfHpKen4+OPP8aHH3542ddJTU2FRqMxvsLC7GP5gLOtNDcNC0aYt6vE1RAREdkXsz79VFtbixkzZuDDDz+Er6/vZR83f/581NTUGF9FRUVmrNIysku12HCoDDIZ8MjEKKnLISIisjsmTWPr6+sLhUKBsrKydtvLysoQGBh4wf7Hjh1Dfn4+brjhBuM2g8HQdmEnJ+Tk5KB37wu7YVQqFVQq+3oqaMmmtjWeJg8KQpS/u8TVEBER2R+TWmqUSiViYmKQlpZm3GYwGJCWloaEhIQL9o+Ojsb+/fuxZ88e4+vGG2/EVVddhT179thNt9Kl5FXU4ad9JwAAc65iKw0REZE5mLzgUEpKCmbNmoXY2FjExcVh8eLFqK+vR3JyMgBg5syZCAkJQWpqKtRqNQYNGtTueC8vLwC4YLs9e2/zMRgEcHW0PwYEe0pdDhERkV0yOdRMmzYNFRUVWLBgAUpLSzFs2DCsW7fOOHi4sLAQcjknKj6r+HQDvttdAgCY8xe20hAREZmLTAghpC7iUrRaLTQaDWpqauDpaVstHc+tOYAvthdgTJQPvrp/lNTlEBERWYylv7/ZpGJG5domrNzZ9uTW3Kv6SFwNERGRfWOoMaMPt+ZB12pATHgPjIr0lrocIiIiu8ZQYyZV9Tp8ub0QADD3L1GQyWQSV0RERGTfGGrM5JPfj6OxRY9BIZ6Y2NdP6nKIiIjsHkONGWibWvDptnwAwNyr2EpDRERkCQw1ZvBFRgFqm1rRx98d1wy4cKZlIiIi6n4MNd2sQdeKj7bmAWibPVguZysNERGRJTDUdLPlOwpxuqEF4T6uuH5IkNTlEBEROQyGmm7U1KLHB1vaWmkentAbTgreXiIiIkvht243+iarGOW1zQjSqHHLiFCpyyEiInIoDDXdpEVvwHubjwEAHhwfCaUTby0REZEl8Zu3m3y/5wRKqhvh667EnXE9pS6HiIjI4TDUdAMhBN7bnAsAuH9cJNTOCokrIiIicjwMNd3gWEU9jlXUQ+kkx/R4ttIQERFJgaGmG2w9WgEAGBnRAx5qZ4mrISIickwMNd0g/WglAGBcH67xREREJBWGmivUojdge94pAMDYKF+JqyEiInJcDDVXaHdhNep1evi4KTEgyFPqcoiIiBwWQ80VOjueZnSUL9d5IiIikhBDzRXaahxPw64nIiIiKTHUXIGahhbsK64GwFBDREQkNYaaK5CRVwmDAHr7uSFI4yJ1OURERA6NoeYKbOGj3ERERFaDoeYKpHM8DRERkdVgqOmiwlMNKKxqgJNchvhIH6nLISIicngMNV20NbftUe4RPXvAXeUkcTVERETEUNNFW4+0dT2NZdcTERGRVWCo6QK9QWDbMY6nISIisiYMNV2wr7ga2qZWeKqdMCTUS+pyiIiICAw1XXL2qafRvX2h4NIIREREVoGhpgvOLo3A8TRERETWg6HGRHXNrdhVeBoAMJ6T7hEREVkNhhoT7cg7hVaDQE9vV/T0cZW6HCIiIjqDocZE7HoiIiKyTgw1Jtp6tG3SvXFRDDVERETWhKHGBCdrGnGsoh5yWduTT0RERGQ9GGpMcLbraUioFzSuzhJXQ0REROdjqDEBV+UmIiKyXgw1l8lgEEjPPTNImONpiIiIrE6XQs2SJUsQEREBtVqN+Ph4ZGZmdrrv6tWrERsbCy8vL7i5uWHYsGH44osvulywVA6d1KKqXgc3pQLDe/aQuhwiIiL6E5NDzcqVK5GSkoLnn38eu3btwtChQ5GUlITy8vIO9/f29sYzzzyDjIwM7Nu3D8nJyUhOTsb69euvuHhLOttKMyrSB0onNnARERFZG5O/nRctWoTZs2cjOTkZAwYMwNKlS+Hq6oply5Z1uP/EiRMxdepU9O/fH71798a8efMwZMgQpKenX3HxlpTO+WmIiIismkmhRqfTISsrC4mJiedOIJcjMTERGRkZlzxeCIG0tDTk5ORg/Pjxne7X3NwMrVbb7iWlphY9MvOrAHCQMBERkbUyKdRUVlZCr9cjICCg3faAgACUlpZ2elxNTQ3c3d2hVCoxZcoUvP3225g0aVKn+6empkKj0RhfYWFhppTZ7TKPV0HXakCQRo3efu6S1kJEREQds8jgEA8PD+zZswd//PEHXn75ZaSkpGDz5s2d7j9//nzU1NQYX0VFRZYos1PnP/Ukk8kkrYWIiIg65mTKzr6+vlAoFCgrK2u3vaysDIGBgZ0eJ5fLERUVBQAYNmwYDh8+jNTUVEycOLHD/VUqFVQqlSmlmRXXeyIiIrJ+JrXUKJVKxMTEIC0tzbjNYDAgLS0NCQkJl30eg8GA5uZmUy4tmYraZhw+2TamZwznpyEiIrJaJrXUAEBKSgpmzZqF2NhYxMXFYfHixaivr0dycjIAYObMmQgJCUFqaiqAtvExsbGx6N27N5qbm7F27Vp88cUXeO+997r3k5jJ72e6ngYGe8LX3Xpaj4iIiKg9k0PNtGnTUFFRgQULFqC0tBTDhg3DunXrjIOHCwsLIZefawCqr6/HI488guLiYri4uCA6Ohpffvklpk2b1n2fwozY9URERGQbZEIIIXURl6LVaqHRaFBTUwNPT0+LXVcIgVGpaSjTNuPL++IZbIiIiExg6e9vTo17EUfL61CmbYbKSY7YCC6NQEREZM0Yai7ibNdTXC9vqJ0VEldDREREF8NQcxHpRysAcBZhIiIiW8BQ04nmVj2257UtjTA2yk/iaoiIiOhSGGo6saugGo0tevi6KxEd6CF1OURERHQJDDWdSM9t63oaG+ULuZxLIxAREVk7hppOpBvnp2HXExERkS1gqOlAdYMO+0pqALS11BAREZH1Y6jpwO+5pyAE0MffHYEatdTlEBER0WVgqOnA2fE049j1REREZDMYav5ECGGcdI/z0xAREdkOhpo/KTjVgOLTjXBWyBAf6S11OURERHSZGGr+ZOuZWYRH9OwBV6XJi5gTERGRRBhq/uRs19P4vhxPQ0REZEsYas7Tqjcg49gpAHyUm4iIyNYw1Jxnb3ENaptboXFxxqAQjdTlEBERkQkYas5zdjzNmCgfKLg0AhERkU1hqDlPuvFRbo6nISIisjUMNWfUNrVgd1E1AI6nISIiskUMNWdsz6uC3iAQ4eOKMG9XqcshIiIiEzHUnHF2PM1YziJMRERkkxhqzuB4GiIiItvGUAOgpLoReZX1UMhlSOjtI3U5RERE1AUMNQDSz3Q9DQ3VwFPtLHE1RERE1BUMNQC2nOl6GsuuJyIiIpvl8KHGYBDYlntmvScOEiYiIrJZDh9qDp7Q4nRDC9xVThga5iV1OURERNRFDh9qtua2jacZFekDZ4XD3w4iIiKb5fDf4luPnH2Um11PREREtsyhQ02jTo+sgtMAGGqIiIhsnUOHmh3HT0GnNyDEywW9fN2kLoeIiIiugEOHmrOzCI+N8oVMJpO4GiIiIroSDh1qthrnp2HXExERka1zkroAqQghMOcvUdh6pAJjohhqiIiIbJ3DhhqZTIYbhwbjxqHBUpdCRERE3cChu5+IiIjIfjDUEBERkV1gqCEiIiK7wFBDREREdqFLoWbJkiWIiIiAWq1GfHw8MjMzO933ww8/xLhx49CjRw/06NEDiYmJF92fiIiIqCtMDjUrV65ESkoKnn/+eezatQtDhw5FUlISysvLO9x/8+bNuOuuu7Bp0yZkZGQgLCwM11xzDUpKSq64eCIiIqKzZEIIYcoB8fHxGDlyJN555x0AgMFgQFhYGB599FE8/fTTlzxer9ejR48eeOeddzBz5szLuqZWq4VGo0FNTQ08PT1NKZeIiIgkYunvb5NaanQ6HbKyspCYmHjuBHI5EhMTkZGRcVnnaGhoQEtLC7y9vU2rlIiIiOgiTJp8r7KyEnq9HgEBAe22BwQEIDs7+7LO8dRTTyE4OLhdMPqz5uZmNDc3G3/WarWmlElEREQOyKJPPy1cuBArVqzAd999B7Va3el+qamp0Gg0xldYWJgFqyQiIiJbZFKo8fX1hUKhQFlZWbvtZWVlCAwMvOixb7zxBhYuXIhffvkFQ4YMuei+8+fPR01NjfFVVFRkSplERETkgEwKNUqlEjExMUhLSzNuMxgMSEtLQ0JCQqfHvfbaa3jppZewbt06xMbGXvI6KpUKnp6e7V5EREREF2PygpYpKSmYNWsWYmNjERcXh8WLF6O+vh7JyckAgJkzZyIkJASpqakAgFdffRULFizA8uXLERERgdLSUgCAu7s73N3du/GjEBERkSMzOdRMmzYNFRUVWLBgAUpLSzFs2DCsW7fOOHi4sLAQcvm5BqD33nsPOp0Ot912W7vzPP/88/jXv/51Wdc8+9Q5BwwTERHZjrPf2ybOHtNlJs9TI4Xi4mIOFiYiIrJRRUVFCA0NNft1bCLUGAwGnDhxAh4eHpDJZN12Xq1Wi7CwMBQVFXHcjgXxvkuD910avO/S4H2Xxp/vuxACtbW1CA4ObteLYy4mdz9JQS6XmzXhcTCyNHjfpcH7Lg3ed2nwvkvj/Puu0Wgsdl2u0k1ERER2gaGGiIiI7IJDhxqVSoXnn38eKpVK6lIcCu+7NHjfpcH7Lg3ed2lIfd9tYqAwERER0aU4dEsNERER2Q+GGiIiIrILDDVERERkFxhqiIiIyC44dKhZsmQJIiIioFarER8fj8zMTKlLshmpqakYOXIkPDw84O/vj5tvvhk5OTnt9mlqasKcOXPg4+MDd3d33HrrrSgrK2u3T2FhIaZMmQJXV1f4+/vjiSeeQGtra7t9Nm/ejBEjRkClUiEqKgqffvqpuT+eTVi4cCFkMhkee+wx4zbec/MpKSnBPffcAx8fH7i4uGDw4MHYuXOn8X0hBBYsWICgoCC4uLggMTERR48ebXeOqqoqTJ8+HZ6envDy8sJ9992Hurq6dvvs27cP48aNg1qtRlhYGF577TWLfD5ro9fr8dxzz6FXr15wcXFB79698dJLL7VbQ4j3vHts2bIFN9xwA4KDgyGTybBmzZp271vyPq9atQrR0dFQq9UYPHgw1q5da9qHEQ5qxYoVQqlUimXLlomDBw+K2bNnCy8vL1FWViZ1aTYhKSlJfPLJJ+LAgQNiz549YvLkyaJnz56irq7OuM9DDz0kwsLCRFpamti5c6cYNWqUGD16tPH91tZWMWjQIJGYmCh2794t1q5dK3x9fcX8+fON++Tl5QlXV1eRkpIiDh06JN5++22hUCjEunXrLPp5rU1mZqaIiIgQQ4YMEfPmzTNu5z03j6qqKhEeHi7uvfdesWPHDpGXlyfWr18vcnNzjfssXLhQaDQasWbNGrF3715x4403il69eonGxkbjPtdee60YOnSo2L59u9i6dauIiooSd911l/H9mpoaERAQIKZPny4OHDggvv76a+Hi4iLef/99i35ea/Dyyy8LHx8f8eOPP4rjx4+LVatWCXd3d/HWW28Z9+E97x5r164VzzzzjFi9erUAIL777rt271vqPv/+++9CoVCI1157TRw6dEg8++yzwtnZWezfv/+yP4vDhpq4uDgxZ84c4896vV4EBweL1NRUCauyXeXl5QKA+O2334QQQlRXVwtnZ2exatUq4z6HDx8WAERGRoYQou3/SHK5XJSWlhr3ee+994Snp6dobm4WQgjx5JNPioEDB7a71rRp00RSUpK5P5LVqq2tFX369BEbNmwQEyZMMIYa3nPzeeqpp8TYsWM7fd9gMIjAwEDx+uuvG7dVV1cLlUolvv76ayGEEIcOHRIAxB9//GHc5+effxYymUyUlJQIIYR49913RY8ePYy/i7PX7tevX3d/JKs3ZcoU8de//rXdtltuuUVMnz5dCMF7bi5/DjWWvM933HGHmDJlSrt64uPjxYMPPnjZ9Ttk95NOp0NWVhYSExON2+RyORITE5GRkSFhZbarpqYGAODt7Q0AyMrKQktLS7t7HB0djZ49exrvcUZGBgYPHoyAgADjPklJSdBqtTh48KBxn/PPcXYfR/49zZkzB1OmTLngvvCem88PP/yA2NhY3H777fD398fw4cPx4YcfGt8/fvw4SktL2903jUaD+Pj4dvfey8sLsbGxxn0SExMhl8uxY8cO4z7jx4+HUqk07pOUlIScnBycPn3a3B/TqowePRppaWk4cuQIAGDv3r1IT0/HddddB4D33FIseZ+7498ehww1lZWV0Ov17f5hB4CAgACUlpZKVJXtMhgMeOyxxzBmzBgMGjQIAFBaWgqlUgkvL692+55/j0tLSzv8HZx972L7aLVaNDY2muPjWLUVK1Zg165dSE1NveA93nPzycvLw3vvvYc+ffpg/fr1ePjhh/G3v/0Nn332GYBz9+5i/6aUlpbC39+/3ftOTk7w9vY26ffjKJ5++mnceeediI6OhrOzM4YPH47HHnsM06dPB8B7bimWvM+d7WPK78EmVukm6zZnzhwcOHAA6enpUpdi14qKijBv3jxs2LABarVa6nIcisFgQGxsLF555RUAwPDhw3HgwAEsXboUs2bNkrg6+/Tf//4XX331FZYvX46BAwdiz549eOyxxxAcHMx7Tp1yyJYaX19fKBSKC54KKSsrQ2BgoERV2aa5c+fixx9/xKZNmxAaGmrcHhgYCJ1Oh+rq6nb7n3+PAwMDO/wdnH3vYvt4enrCxcWluz+OVcvKykJ5eTlGjBgBJycnODk54bfffsN//vMfODk5ISAggPfcTIKCgjBgwIB22/r374/CwkIA5+7dxf5NCQwMRHl5ebv3W1tbUVVVZdLvx1E88cQTxtaawYMHY8aMGXj88ceNrZS855Zhyfvc2T6m/B4cMtQolUrExMQgLS3NuM1gMCAtLQ0JCQkSVmY7hBCYO3cuvvvuO/z666/o1atXu/djYmLg7Ozc7h7n5OSgsLDQeI8TEhKwf//+dv9n2LBhAzw9PY1fIAkJCe3OcXYfR/w9XX311di/fz/27NljfMXGxmL69OnGP/Oem8eYMWMumLLgyJEjCA8PBwD06tULgYGB7e6bVqvFjh072t376upqZGVlGff59ddfYTAYEB8fb9xny5YtaGlpMe6zYcMG9OvXDz169DDb57NGDQ0NkMvbf0UpFAoYDAYAvOeWYsn73C3/9lz2kGI7s2LFCqFSqcSnn34qDh06JB544AHh5eXV7qkQ6tzDDz8sNBqN2Lx5szh58qTx1dDQYNznoYceEj179hS//vqr2Llzp0hISBAJCQnG988+XnzNNdeIPXv2iHXr1gk/P78OHy9+4oknxOHDh8WSJUsc/vHi853/9JMQvOfmkpmZKZycnMTLL78sjh49Kr766ivh6uoqvvzyS+M+CxcuFF5eXuL7778X+/btEzfddFOHj70OHz5c7NixQ6Snp4s+ffq0e+y1urpaBAQEiBkzZogDBw6IFStWCFdXV4d6vPisWbNmiZCQEOMj3atXrxa+vr7iySefNO7De949amtrxe7du8Xu3bsFALFo0SKxe/duUVBQIISw3H3+/fffhZOTk3jjjTfE4cOHxfPPP89Huk3x9ttvi549ewqlUini4uLE9u3bpS7JZgDo8PXJJ58Y92lsbBSPPPKI6NGjh3B1dRVTp04VJ0+ebHee/Px8cd111wkXFxfh6+sr/v73v4uWlpZ2+2zatEkMGzZMKJVKERkZ2e4aju7PoYb33Hz+97//iUGDBgmVSiWio6PFBx980O59g8EgnnvuOREQECBUKpW4+uqrRU5OTrt9Tp06Je666y7h7u4uPD09RXJysqitrW23z969e8XYsWOFSqUSISEhYuHChWb/bNZIq9WKefPmiZ49ewq1Wi0iIyPFM8880+6RYN7z7rFp06YO/z2fNWuWEMKy9/m///2v6Nu3r1AqlWLgwIHip59+MumzyIQ4b3pGIiIiIhvlkGNqiIiIyP4w1BAREZFdYKghIiIiu8BQQ0RERHaBoYaIiIjsAkMNERER2QWGGiIiIrILDDVERERkFxhqiIiIyC4w1BAREZFdYKghIiIiu8BQQ0RERHbh/wHDzDBv3G5nvAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "# Create a Neural LinUCB pearl agent with 1-hot action representation\n", + "\n", + "action_representation_module = OneHotActionTensorRepresentationModule(\n", + " max_number_actions= env._action_space.n,\n", + ")\n", + "\n", + "agent = PearlAgent(\n", + " policy_learner=NeuralLinearBandit(\n", + " feature_dim = env.observation_dim + env._action_space.n,\n", + " hidden_dims=[64, 16],\n", + " training_rounds=50,\n", + " action_representation_module=action_representation_module,\n", + " exploration_module= UCBExploration(alpha=1.0)\n", + " ),\n", + " replay_buffer=FIFOOffPolicyReplayBuffer(100_000),\n", + " device_id=-1,\n", + ")\n", + "\n", + "\n", + "info = online_learning(\n", + " agent=agent,\n", + " env=env,\n", + " number_of_steps=number_of_steps,\n", + " print_every_x_steps=100,\n", + " record_period=record_period,\n", + " learn_after_episode=True,\n", + ")\n", + "torch.save(info[\"return\"], \"LinUCB-return.pt\")\n", + "plt.plot(record_period * np.arange(len(info[\"return\"])), info[\"return\"], label=\"LinUCB\")\n", + "plt.xlabel(\"time step\")\n", + "plt.ylabel(\"return\")\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QNUmNO77LNGR" + }, + "source": [ + "## Contextual Bandits learners: LinTS\n", + "\n", + "Lastly, we describe how to use the neural version of the LinTS algorithm with the Pearl library, namely, the algorithm which uses Thompson sampling exploration with neural architectures. The LinTS sampling is closely related to the LinUCB algorithm, with a key modification that often improves its convergence in practice: sample the score function from a probability, instead of fixing it determinstically. Practically, this often reduces over-exploring arms, since the score may be smaller than in the LinUCB algorithm.\n", + "\n", + "To implement the LinTS algorithm in Pearl, use the NeuralLinearBandit policy learner module. Further, set the exploration module to be ThompsonSamplingExplorationLinear. This enables the agent to sample the score based on its estimated uncertainty, rather to fix it as in LinUCB algorithm.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_7Cpzoi3nVAw", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "87e0a0b0-28fa-4c8a-cbac-ee9c20e1664c" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.10/dist-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n", + " and should_run_async(code)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "episode 100, step 100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -6.782996206311509e-05\n", + "episode 200, step 200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0620898008346558\n", + "episode 300, step 300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.128913164138794\n", + "episode 400, step 400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0612578392028809\n", + "episode 500, step 500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.01449716091156\n", + "episode 600, step 600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.14748625457286835\n", + "episode 700, step 700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.7561846375465393\n", + "episode 800, step 800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.0688522681593895\n", + "episode 900, step 900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.19547955691814423\n", + "episode 1000, step 1000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9439463019371033\n", + "episode 1100, step 1100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0238324403762817\n", + "episode 1200, step 1200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0274128913879395\n", + "episode 1300, step 1300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.004909634590149\n", + "episode 1400, step 1400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8957030177116394\n", + "episode 1500, step 1500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9626027345657349\n", + "episode 1600, step 1600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.020164569839835167\n", + "episode 1700, step 1700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8749321103096008\n", + "episode 1800, step 1800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0139015913009644\n", + "episode 1900, step 1900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.114064335823059\n", + "episode 2000, step 2000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.137966275215149\n", + "episode 2100, step 2100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0669292211532593\n", + "episode 2200, step 2200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9254041314125061\n", + "episode 2300, step 2300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.069034457206726\n", + "episode 2400, step 2400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.7684177756309509\n", + "episode 2500, step 2500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1040222644805908\n", + "episode 2600, step 2600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1308488845825195\n", + "episode 2700, step 2700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1927236318588257\n", + "episode 2800, step 2800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9832726716995239\n", + "episode 2900, step 2900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0864346027374268\n", + "episode 3000, step 3000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.18157958984375\n", + "episode 3100, step 3100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.035843849182129\n", + "episode 3200, step 3200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9285320043563843\n", + "episode 3300, step 3300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0561041831970215\n", + "episode 3400, step 3400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8969429135322571\n", + "episode 3500, step 3500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.234483003616333\n", + "episode 3600, step 3600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8475865125656128\n", + "episode 3700, step 3700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.008420705795288\n", + "episode 3800, step 3800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0597569942474365\n", + "episode 3900, step 3900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.110633134841919\n", + "episode 4000, step 4000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.978070080280304\n", + "episode 4100, step 4100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8320597410202026\n", + "episode 4200, step 4200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0280927419662476\n", + "episode 4300, step 4300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8820557594299316\n", + "episode 4400, step 4400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.039269208908081\n", + "episode 4500, step 4500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9891727566719055\n", + "episode 4600, step 4600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9681330323219299\n", + "episode 4700, step 4700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0180885791778564\n", + "episode 4800, step 4800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0547395944595337\n", + "episode 4900, step 4900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9265732169151306\n", + "episode 5000, step 5000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9476595520973206\n", + "episode 5100, step 5100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0067214965820312\n", + "episode 5200, step 5200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.321449875831604\n", + "episode 5300, step 5300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9141985774040222\n", + "episode 5400, step 5400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0031826496124268\n", + "episode 5500, step 5500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0458906888961792\n", + "episode 5600, step 5600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0051615238189697\n", + "episode 5700, step 5700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1755057573318481\n", + "episode 5800, step 5800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8318357467651367\n", + "episode 5900, step 5900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.12373097985982895\n", + "episode 6000, step 6000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9620298743247986\n", + "episode 6100, step 6100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8539725542068481\n", + "episode 6200, step 6200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0763007402420044\n", + "episode 6300, step 6300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1518890857696533\n", + "episode 6400, step 6400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9503635168075562\n", + "episode 6500, step 6500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.123961091041565\n", + "episode 6600, step 6600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0370116233825684\n", + "episode 6700, step 6700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0301270484924316\n", + "episode 6800, step 6800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.11941659450531\n", + "episode 6900, step 6900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: -0.004745124839246273\n", + "episode 7000, step 7000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8427696228027344\n", + "episode 7100, step 7100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9572257995605469\n", + "episode 7200, step 7200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.2062397003173828\n", + "episode 7300, step 7300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9617626667022705\n", + "episode 7400, step 7400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.2136000394821167\n", + "episode 7500, step 7500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9028815031051636\n", + "episode 7600, step 7600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9531269669532776\n", + "episode 7700, step 7700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0129711627960205\n", + "episode 7800, step 7800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0808316469192505\n", + "episode 7900, step 7900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1866376399993896\n", + "episode 8000, step 8000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9024198055267334\n", + "episode 8100, step 8100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.05154550075531\n", + "episode 8200, step 8200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8863436579704285\n", + "episode 8300, step 8300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8365844488143921\n", + "episode 8400, step 8400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.03874698281288147\n", + "episode 8500, step 8500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.961517333984375\n", + "episode 8600, step 8600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.2384059429168701\n", + "episode 8700, step 8700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.1409051418304443\n", + "episode 8800, step 8800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9610186219215393\n", + "episode 8900, step 8900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9460026621818542\n", + "episode 9000, step 9000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8658124208450317\n", + "episode 9100, step 9100, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0738046169281006\n", + "episode 9200, step 9200, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9109795689582825\n", + "episode 9300, step 9300, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9272820353507996\n", + "episode 9400, step 9400, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0422909259796143\n", + "episode 9500, step 9500, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9289524555206299\n", + "episode 9600, step 9600, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8740895986557007\n", + "episode 9700, step 9700, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.9682502746582031\n", + "episode 9800, step 9800, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0605299472808838\n", + "episode 9900, step 9900, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 1.0936775207519531\n", + "episode 10000, step 10000, agent=PearlAgent with NeuralLinearBandit, FIFOOffPolicyReplayBuffer, env=Contextual bandits with CB datasets\n", + "return: 0.8741152286529541\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAGwCAYAAAC99fF4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJwUlEQVR4nO3deXhTVf4G8Dd7mrZJ95Vu7DuUrRRkRK1TlUFQx0EHBVFx+cGwuTIojrjgoyOCygyMI6DjjOCCuIAgVgHZZSn7VqAtlKYLXdI9aXJ+f5QGIqnSNslN2vfzPHlob+5NvvcWmpdzzj1HJoQQICIiIiIHcqkLICIiIvJGDElERERETjAkERERETnBkERERETkBEMSERERkRMMSUREREROMCQREREROaGUugBPs9lsuHDhAgIDAyGTyaQuh4iIiK6BEAIVFRWIiYmBXO6ZNp52F5IuXLiAuLg4qcsgIiKiFjh37hw6dOjgkfdqdyEpMDAQQMNF1uv1EldDRERE18JkMiEuLs7+Oe4J7S4kNXax6fV6hiQiIiIf48mhMhy4TUREROQEQxIRERGREwxJRERERE60uzFJ18pqtcJisUhdRpuhUqmgUCikLoOIiOiaMST9ghACRqMRZWVlUpfS5gQFBSEqKorzUxERkU9gSPqFxoAUEREBnU7HD3QXEEKguroahYWFAIDo6GiJKyIiIvptDElXsFqt9oAUGhoqdTltip+fHwCgsLAQERER7HojIiKvx4HbV2gcg6TT6SSupG1qvK4c60VERL5A0pC0ZcsWjB49GjExMZDJZFizZs1vHrNp0yYMGDAAGo0GnTt3xooVK1xeF7vY3IPXlYiIfImkIamqqgr9+vXD4sWLr2n/s2fPYtSoUbjhhhuQmZmJGTNm4OGHH8aGDRvcXCkRERG1N5KOSbr11ltx6623XvP+S5YsQVJSEt58800AQI8ePbB161a89dZbSE9Pd1eZRERE1A751JikHTt2IC0tzWFbeno6duzY0eQxdXV1MJlMDo/25lq7MomIiOgynwpJRqMRkZGRDtsiIyNhMplQU1Pj9Jj58+fDYDDYH3FxcZ4o1eMeeOABjB071ulz+fn519xi98ADD0AmkzX5SExMBNDQ9fnnP/8ZMTEx0Gq16NChA8aMGYPjx4+76IyIiMjXCCFQYKpFdnGV1KW4RJufAmD27NmYNWuW/XuTydRmg1JToqKirnnfRYsW4bXXXrN/Hx0djeXLl+OWW24BACgUClgsFtx8883o1q0bVq9ejejoaJw/fx7ffvstJ+EkImoHai1WnC2uwpmiKpwuqsSZokqcufR9ZV09RnQJw38eSpG6zFbzqZAUFRWFgoICh20FBQXQ6/X2eXh+SaPRQKPRtPg9hRCosVhbfHxr+KkULrkjTCaT4YsvvsDYsWORnZ2NpKQkfP7553jnnXewa9cudOnSBUuWLEFqaqq9xe1KjTNlN8rMzMTp06eRkZGBhIQEAEBCQgKGDx/e6lqJiMg7NLQK1dlD0OmiKpwprsLpwkpcKK+BEM6Pk8sAi9Xm2WLdxKdCUmpqKtatW+ewbePGjUhNTXXbe9ZYrOg5V5q7547OS4dO7Z4f0Zw5c/D3v/8dXbp0wZw5c3DvvfciKysLSuVvv194eDjkcjk+++wzzJgxgxNDEpFLZBdXYVHGKVwoq0GniAB0iQhA54gAdIkIRKRew2lEXMxcb4Op1oLymoZHXmnN5Zah4kqcLapClbnpRgKDnwodw/3RKTwAHcP90TEsAJ3C/REfqoNG2TY+FyQNSZWVlcjKyrJ/f/bsWWRmZiIkJATx8fGYPXs28vLy8OGHHwIAHnvsMbz77rt4+umn8eCDD+KHH37AJ598grVr10p1Cj7rySefxKhRowAAL774Inr16oWsrCx07979N4+NjY3F22+/jaeffhovvvgiBg0ahBtuuAHjx49Hx44d3V06EbUxFbUWvPtjFpZtPQuLtaF5YtfZEod9AjVKdLKHpsvhKTbYDwp5+wxPQghUm62Xg061BabaenvoMTX+WXvF1zWXn7+WXhKFXIaEEF1DCApvCEEdwwPQMcwfIf7qNh9cJQ1Je/bswQ033GD/vnHs0MSJE7FixQrk5+cjNzfX/nxSUhLWrl2LmTNnYtGiRejQoQP+/e9/u/X2fz+VAkfnSTO9gJ/KfUm8b9++9q8b11IrLCy8ppAEAFOmTMGECROwadMm7Ny5E59++ileffVVfPXVV7j55pvdUjMRNZ/NJmCx2WCxCtRbbTBbL3+tVMgRG+R8qIKnavts33m8vv4EiivrAADXdw3HH/pGI/tiFbIKK3GqsBI5F6tRUVePzHNlyDxX5vAaGqUcHcOvDE4NfyaE+kOt9Kl7k36TqdaCg+fKceB8GfbnNlyLxuvWGoFaJQx+KkTqtQ4hqGN4AOJDdG3uOjaHpCFp5MiREE11agJOZ9MeOXIk9u/f78aqHMlkMrd1eUlJpVLZv278n4DN1rw+5MDAQIwePRqjR4/Gyy+/jPT0dLz88ssMSV6svNqC2norIgLZdeELrDbR0AVSXIkzRVU4W1yFnJJq1JjrYbEKWKw2WKw21FvFpfBjc9husQpYbU3/jgWA/nFBmDQ8Ebf1iYZK4bkPw705JfjbV0dxKK8cANAxzB/P/6EnbugecdW+5nqbPTQ1BqeswkqcLqpEXb0Nx/JNOJbvOL2LUi5DQqgO/eKCMKxTGIZ3DkW0QbpA2FwWqw0njBX2YJh5rgyniyqdjgNSymUw+Klg8FMh8NKfBj8V9JfCj8FPBb3D9stfB2iV7bYl7lq0vU9/koRMJkP37t2xfft2qUuhJmQXV2H0u1tRUVsPrUqO+BAd4kP8kRCqQ0KoDvEhOiSE+qNDsJ9HPyzbOyEEiivNOFtchbPFl+8QOltchdyL1TC7YQCsWimHSi5Dbb0NmefKMH1lJuavO477UxPw5yHxCPZXu/w9G+WX1+C1b4/jy8wLABq60abd1AUThyU22WKhVsrRNTIQXSMDHbZbbQLnS6txqqASWUWV9j+zCipQZbbidFEVThdVYfW+PAANQSy1UyiGdw5DasdQt55ncwghcL60BgfOlyHzUgvR4QvlqLVc/bPvEOyH/nFB9ke3qEAEaJT8T4+bMCS1IeXl5cjMzHTYFhoa6vL3yczMxAsvvID7778fPXv2hFqtxubNm7Fs2TI888wzLn8/aj2bTeDpzw6iorYeAFBrseFkQSVOFlReta9cBsQE+V0KT/5ICGkMUQ2Byl/DXxstUVVX33DLdHEVzhY1BKLG7xt/Ls6olXIkhfojKcwfSeH+SAr1R6BWCaVCDpVCBrVCbv9apZBfevzia6W8YT+5DAq5zP6BWlRRh//tysV/dubAaKrFGxtO4O2MU7hzQCwmDU+6KpS0Rq3Fin9tOYN/bjqNGosVMhlwz+A4PPH7bggLaNkdyAq5rOHvaKg/0nB5Dj0hBIymWpwwVmD32RJsO30Rh86XNQTQ4ir8d1cuZDKgR5QewzuHYlinMAxJCvHY3+3GbrPMc6WXWonKnXabBWqVDoGob4cghAe2/G5taj7+tmtDNm3ahOTkZIdtDz30kMvfp0OHDkhMTMSLL76I7Oxs+ySTL774ImbOnOny96PWW7E9G7uzS+CvVuCbaSMgA5BTUo3ci1XIuVh96etq5JRUodZiw/nSGpwvrcG2rItXvVZYgBrxITokhvpjaKdQ3NI7Cnqt6uo3JZwwVuDLzDysPZSPnIvVTe4nkwGxQX72sSBJlx4dw/0RY/CD3E3dIeGBGkxP64LHRnbE2oP5WLbtLA7nmfDx7nP4ePc5XNc5DA9el4iRXSNaXIMQAusOGfHqumPIK2uY9HdIYgjmju6J3rGG3zi6ZWQyGaINfog2+GFkt4buu/IaS0NgyirG9tPFOFlQiaP5JhzNN+G9n85CKZehf1wQhnUKxbDOYUiOD2rRHVoWqw3G8loYTbW4UFaD/PJa5Df+WV6L/PIaFFearzpOKZehR7T+ciiKD0JSqL/bfvZ0bWTi1wYFtUEmkwkGgwHl5eXQ6/UOz9XW1uLs2bNISkqCVquVqMK2i9dXGmeKKnHb2z+h1mLDy2N7476hCU3uK4RAYUVdQ3C6WIXckuorQlQVSqstVx2jVspxU/cIjOkfg5HdIqB14w0HvuB8aTW+OnABX2VewHFjhcNzof5qewBKunTLdMdwf8SH6LziugkhsCenFMu2nsWGI0Y0DmdKCvPHA8MS8ceBHZrV2nLkQjle/Poodl+6Uy3GoMVfR/XAqD7RkncPFVXUYfvpYmzPuojtZ4pxrsRx1QatSo7BiSEY1ikMwzqFoneswf7vI7+8MfzU4kJ5DfLLapFvaghDRZV1Tc4fdKW4ED/0jwu2h6JeMXqv+DvgzX7t89tdGJKuwA9x9+L19TyrTeBPS3dgb04pruschv88NKRVH06mWktDi9PFapwoqMC3h/JxqvByl12gRon03lEY0z8GqR1DoWwnY5suVtZh3aF8fJl5AXtySu3bVQoZRnZrCJDXdQ5DkM47xsBci3Ml1fjPzhx8vDvX3h0YqFVi3KA4TByWiLgQXZPHXqysw9+/O4mVP+dCiIbA8fj1nfHI7zrCT+2dQeBcSTW2ny7GtqyL2H764lXdX34qBerqrfiNcfAAALVCjiiDFtEGLWKC/BBl0CLGoG1o3QrSokOQDgYdW1+biyHJAxiSpMPr63n//ukMXl57DAEaJdbPGIEOwU1/sLWEEALH8ivw5YE8fJ15ARfKa+3PhQVo8Ie+0RjTPwb944Ikbzlwtaq6emw8WoA1mXn46VSx/S4ymQwYmhSKMf1jcGvvaJ//MKyqq8fn+85jxbZsnLm0HpdcBvy+ZxQmDU/EkKQQ+8/WXG/DhzuysSjjlD1Yje4Xg2dv7S7pVAPNJYTAqcJKbMtqCE27zlxERV3D+SjlMkTqtYgJuhR6LoWh6CA/xBgaAlGov5rdZG7AkOQBDEnS4fX1rKzCSox6+yfU1dsw/84+uHdIvFvfz2Zr6KppHINTdkXXXHyIDmP6x2BM/xh0jnDdYGBPM9fbsOVkEb48cAEbjxod7j7qE2vAmP4x+EPfGEQZ2t7fb5tNYPPJIizbdhY/nSq2b+8Vo8ek4UkI1qnwyrpjOFPUEKR6x+rxwuheGJwYIlXJLlNvbZiCIFCrQliAhrfMS4QhyQOuJSQlJiY2uRYctVxNTY197TiGJPey2gT+uGQ79ueWYUSXMHz4YOu62ZrLXG/D1qwifJl5ARuPFqD6iqUNekbrMaZ/DEb3i0GMD7Qu2GwCu7NL8GXmBaw7lI/ymsvhLzFUhzH9Y3F7/xh0Cg+QsErPOllQgeXbsrF633nU1Tveph4WoMbT6d1x18AODBPkUgxJHvBrF9lqteLkyZOIiIhwy63z7d3FixdRWFiIrl27cr03N1u6+TTmf3scgRolNsz8naRhpNpcj++PFeLL/XnYfLII9VcM6hiSFIIx/WNwW+9or5mzBmi4Q+ng+XJsOGLE1wcuIP+KbsSIQA1G92toFesTa2hz3YjNUVplxsc/5+LD7Tm4WFWHScOT8JcbOyOQdzuSGzAkecBvXeT8/HyUlZUhIiICOp2uXf8CdBUhBKqrq1FYWIigoCD7MijkHqcKKjDqna0w19vw+l198afBcVKXZFdaZca6ww0DnHdfsTaXUi5Dr1gDBiUEY3BiMAYmhHh0PhiL1YZDeeXYeeYidp4pwZ7sEofWr0CtErf1bhhfldIxlC0kv1BvtaHGYmU4IrdiSPKA37rIQggYjUaUlZV5vrg2LigoCFFRUQyeblRvteGuf27HgfPlGNktHMsfGOy11/tCWQ2+PnABX2ZewNFfLCkBNHRlDUwIwaDEhuDUKTzAZefyW6EIAIJ0KgzvHIbb+8VgZLfwNrOqOZGvYkjygGu9yFarFRbL1XPCUMuoVCp2sXnAPzZl4fX1JxCoVWLjzOt9ZgDx+dJq7MkuxZ6cEuzJLsWJgoqr5poJ0qkwKCHYHpz6xBqueV6Zaw1FKUkhGNoxFEM7hqJbZCDvUCLyIgxJHiDFRSbyhBPGCox+ZyvMVhvevLsf7hrYQeqSWqy8xoL9uaX24JR5ruyqdazUCjn6dDBgUGIwBiWEYGBCMEIujWtiKCJqexiSPIAhidoii9WGO/+xHYfyynFT9wj8e+Igr+1mawmL1YYjF0zYk11yKTiVOl3rqlO4PyL1WmSeK2MoImpjpPj85tptRG3A0s2ncSivHAY/FV69s0+bCkgAoFLI7cs3PDyiYexgzsVq7Mkpxd6cEvycXYqswkr7qu8AQxERtR5DEpGPO5ZvwqKMUwCAF2/vhUi9b4xDag2ZTIbEMH8khvnjj5e6FUurzNibU4rCijokxwcxFBFRqzEkEfkwi9WGJz89AItV4OaekRjTP0bqkiQT7K9GWs9Iqcsgojakfaw+SdRG/ePH0zhywYQgnQqv3NG7zXWzERFJiSGJyEcduVCOd35o6GabN6Y3IgLbfjcbEZEnMSQR+SBzvQ1PfHIA9TaBW3pFYXRfzmJORORqDElEPujdH7Nw3FiBEH81XmY3GxGRWzAkEfmYw3nlWPxjFgDgpTG9ERbguTXOiIjaE4YkIh9SV2/FE58cgNUmMKpPNEaxm42IyG0Ykoh8yNsZp3CioAKh/mrMG9NL6nKIiNo0hiQiH3HgXBmWbD4DAHh5bG+EspuNiMitGJKIfECtxYonP23oZru9Xwxu7cNuNiIid2NIIvIBC78/hVOFlQgL0ODF29nNRkTkCVyWhMiL1Vqs2JtTin9tOQ0AePWO3gj2V0tcFRFR+8CQROQhdfVWlFZZUFJlRlm1GSXVZpRWmVFSZUFptRklVWaUVl96XNqvxmK1H39Hcix+3ytKwjMgImpfGJKIXMxUa8Fne85jy6kilFRdCj9VZlSZrb99sBMqhQyDEkLwwuieLq6UiIh+DUMSkYucLqrEh9uz8dne800GIoVchmCdCsE6NYL91Qhp/NO/YVuI/+XtIf5qBOlUCNAoOaM2EZEEGJKIWsFmE9h8sgjLt2djy8ki+/bOEQG4Z3AcksL8EeyvbghAOjUCtUrI5Qw8RES+gCGJqAUau9Q+3JGN7IvVAACZDLipeyQeGJaI4Z1D2fpDROTjGJKImsFZl1qgVolxg+IwITUR8aE6iSskIiJXYUgi+g2/1qX2wLBE3JEcC38N/ykREbU1/M1O1IRf61KbNDwRwzqxS42IqC1jSCL6BXapERERwJBEBAAQQmDH6YtYuuUMNl/RpdYlIgAT2aVGRNQu8bc+tWtCCPx4ohDv/pCFfbllANilRkREDRiSqF2y2QTWHzHi3R+ycDTfBADQKOW4Z3AcHrquI7vUiIiIIYnal3qrDV8duIDFP2bhdFEVAECnVuD+oQl4aEQSIgK1EldIRETegiGJ2oW6eis+35uHJZtPI7ek4U41vVaJB4YnYdKwRAT7qyWukIiIvA1DErVpNWYrPt6di39tOQOjqRYAEOqvxkMjknD/0AQEalUSV0hERN6KIYnapIpaC/6zMwfv/3QWF6vMAIBIvQaP/q4T7h0SDz+1QuIKiYjI2zEkUZtSWmXG8m1nsWJ7Nky19QCAuBA/PH59Z9w1MBYaJcMRERFdG4YkahMKK2rx75/O4qOdOai+NAFkp3B/TLmhM27vFwOlQi5xhURE5GsYksin1VtteO3b4/hwZw7M9TYAQI9oPf5yY2ek94qCQs45joiIqGUYksinrfz5HP699SwAIDk+CH+5sTNu6BbBCSCJiKjVGJLIZ1ltAv/+6QwA4Kn0bvi/kZ0YjoiIyGUkH6ixePFiJCYmQqvVIiUlBbt3725yX4vFgnnz5qFTp07QarXo168f1q9f78FqyZtsPGpE9sVqGPxUeGBYIgMSERG5lKQhadWqVZg1axZeeOEF7Nu3D/369UN6ejoKCwud7v/cc89h6dKleOedd3D06FE89thjuOOOO7B//34PV05SE0JgyeaGVqT7hyZw8VkiInI5mRBCSPXmKSkpGDx4MN59910AgM1mQ1xcHP7yl7/g2WefvWr/mJgYzJkzB1OmTLFvu+uuu+Dn54ePPvromt7TZDLBYDCgvLwcer3eNSdCHrf7bAn+tHQH1Eo5tj1zI8IDNVKXREREbiTF57dkLUlmsxl79+5FWlra5WLkcqSlpWHHjh1Oj6mrq4NW67i2lp+fH7Zu3drk+9TV1cFkMjk8yPf9a8tpAMBdA2IZkIiIyC0kC0nFxcWwWq2IjIx02B4ZGQmj0ej0mPT0dCxYsACnTp2CzWbDxo0bsXr1auTn5zf5PvPnz4fBYLA/4uLiXHoe5HlZhRX4/lghZDLg4REdpS6HiIjaKMkHbjfHokWL0KVLF3Tv3h1qtRpTp07FpEmTIJc3fRqzZ89GeXm5/XHu3DkPVkzu8K8tDWORbu4RiU7hARJXQ0REbZVkISksLAwKhQIFBQUO2wsKChAVFeX0mPDwcKxZswZVVVXIycnB8ePHERAQgI4dm25N0Gg00Ov1Dg/yXYWmWqzZfwEA8Oj1bEUiIiL3kSwkqdVqDBw4EBkZGfZtNpsNGRkZSE1N/dVjtVotYmNjUV9fj88//xxjxoxxd7nkJZZvz4bZasOghGAMTAiRuhwiImrDJL1vetasWZg4cSIGDRqEIUOGYOHChaiqqsKkSZMAABMmTEBsbCzmz58PANi1axfy8vLQv39/5OXl4W9/+xtsNhuefvppKU+DPKSyrh4f7cwBADzyO7YiERGRe0kaksaNG4eioiLMnTsXRqMR/fv3x/r16+2DuXNzcx3GG9XW1uK5557DmTNnEBAQgNtuuw3/+c9/EBQUJNEZkCet3J2Litp6dAz3R1qPyN8+gIiIqBUknSdJCpwnyTdZrDZc//qPuFBei9fu7IN7hsRLXRIREXlQu5oniag5vjl4ARfKaxEWoMHY5FipyyEionaAIYm8nhACSy8tQTJpeCK0KoXEFRERUXvAkEReb8upYhw3VkCnVuC+lASpyyEionaCIYm8XuMSJPcMjodBp5K4GiIiai8YksirHc4rx7asi1DIZXjwukSpyyEionaEIYm82tJLS5D8oW80OgTrJK6GiIjaE4Yk8lrnSqqx7lDD4sWcPJKIiDyNIYm81vtbz8JqExjRJQy9YgxSl0NERO0MQxJ5pdIqM1b9fA4AW5GIiEgaDEnklT7amYMaixU9o/W4rnOY1OUQEVE7xJBEXqfWYsUHO7IBAI9e3xEymUzagoiIqF1iSCKvs3pfHoorzYgN8sNtfaKlLoeIiNophiTyKlabwHs/Ndz2/+B1SVAp+FeUiIikwU8g8iobjxbgbHEVDH4q3DM4TupyiIioHWNIIq/SuATJfUPj4a9RSlwNERG1ZwxJ5DX2ZJdgX24Z1Ao5Jg5LlLocIiJq5xiSyGss2dwwFunOAbGICNRKXA0REbV3DEnkFbIKK/H9sQLIZMBkTh5JRERegCGJvMK/L93RltYjEp3CAySuhoiIiCGJvEBhRS1W78sDADzKViQiIvISDEkkuRXbsmG22jAwIRiDEkOkLoeIiAgAQxJJrLKuHh/tzAHAhWyJiMi7MCSRpFb9fA6m2np0DPPHzT0ipS6HiIjIjiGJJGOx2rBs61kAwMMjOkIu50K2RETkPRiSSDJrD+Yjr6wGYQFq3DkgVupyiIiIHDAkkSSEEFi6peG2/weGJUKrUkhcERERkSOGJJLE1qxiHMs3QadW4L6hCVKXQ0REdBWGJJLE0ktLkIwbHIcgnVriaoiIiK7GkEQe92VmHrZmFUMhl+HB4UlSl0NEROQUQxJ5VHZxFf66+hAAYMrITogL0UlcERERkXMMSeQxdfVWTP14H6rMVgxJDMG0m7pIXRIREVGTGJLIY1779jgO55kQrFNh0b39oVTwrx8REXkvfkqRR3x3xIjl27IBAG/+qR+iDX7SFkRERPQbGJLI7fLKavDUZwcBAA9fl4Qbu3P5ESIi8n4MSeRWFqsN0z7ej/IaC/p1MODpW7pLXRIREdE1YUgit3pr40nszSlFoEaJd+4dALWSf+WIiMg38BOL3OanU0X45+bTAIDX7uqL+FDe7k9ERL6DIYncorCiFjNXZUII4M8p8RjVN1rqkoiIiJqFIYlczmoTmLkqE8WVZnSPCsTcP/SUuiQiIqJmY0gil/vnpixsy7oIP5UC7/45GVqVQuqSiIiImo0hiVzq5+wSLNh4EgAwb0wvdI4IlLgiIiKilmFIIpcprTJj2sf7YRPAHcmx+OPADlKXRERE1GIMSeQSQgg8+ekB5JfXomOYP14a2xsymUzqsoiIiFqMIYlcYtm2bGQcL4RaKcc7f05GgEYpdUlEREStwpBErXbwfBle+/YYAOC5UT3QK8YgcUVEREStx5BErWKqtWDq//bDYhVI7xWJ+4cmSF0SERGRSzAktQNCCLe97l9XH0JuSTVig/zw+l39OA6JiIjaDA4caeNWbDuLV9YdQ3JcMH7fKxK/7xnlsuVBVv58Dt8czIdSLsM7f06GQadyyesSERF5A4akNsxqE1iy+QwsVoHd2SXYnV2Cl9ceQ/eoQPy+VxR+3zMSvWL0LWr9OW404W9fHQEAPJneDQPig11dPhERkaQYktqwnWcuwmiqhV6rxMybu2Lj0QLsOluC48YKHDdW4O2MU4gN8rO3MA1ODIZS8ds9sNXmekz9337U1dtwfddwPDKiowfOhoiIyLMkH5O0ePFiJCYmQqvVIiUlBbt37/7V/RcuXIhu3brBz88PcXFxmDlzJmpraz1UrW9ZvS8PAPCHfjGYNDwJ/5s8FHufS8Obd/dDeq9IaFVy5JXVYPm2bNz73k4MfuV7PPnpAXx3xIgas7XJ1/3bV0eQVViJiEANFvypH+RyjkMiIqK2R9KWpFWrVmHWrFlYsmQJUlJSsHDhQqSnp+PEiROIiIi4av///e9/ePbZZ7Fs2TIMGzYMJ0+exAMPPACZTIYFCxZIcAbeq9pcj/WH8wEAdybH2rcH6dS4a2AH3DWwA2rMVvx0qgjfHS1AxrEClFZb8Nne8/hs73loVXL8rks40ntF4cbuEQj2VwMA1uzPwyd7zkMmAxbe0x+hARpJzo+IiMjdZMJdtz5dg5SUFAwePBjvvvsuAMBmsyEuLg5/+ctf8Oyzz161/9SpU3Hs2DFkZGTYtz3xxBPYtWsXtm7dek3vaTKZYDAYUF5eDr1e75oT8UJr9udhxqpMxIfosPmpkb857qjeasOenFJsOGLEd0cKkFdWY39OIZdhSGIIRnQNw+IfslBltmL6TV0w8+au7j4NIiIiANJ8fkvW3WY2m7F3716kpaVdLkYuR1paGnbs2OH0mGHDhmHv3r32LrkzZ85g3bp1uO2225p8n7q6OphMJodHe7B6f0NX29jk2GsamK1UyDG0YyheGN0LW5+5AWunXYdpN3VB96hAWG0CO85cxOvrT6DKbEVKUgim3dTF3adAREQkKcm624qLi2G1WhEZGemwPTIyEsePH3d6zJ///GcUFxfjuuuugxAC9fX1eOyxx/DXv/61yfeZP38+XnzxRZfW7u0KTbXYeqoIgGNX27WSyWToFWNArxgDZt3cFbkXq/HdUSO+O1qAunobFt2TDAXHIRERURsn+cDt5ti0aRNeffVV/OMf/8C+ffuwevVqrF27Fi+99FKTx8yePRvl5eX2x7lz5zxYsTS+OnABNgEMiA9CYph/q18vPlSHh0d0xCePpuLLKcMRZdC6oEoiIiLvJllLUlhYGBQKBQoKChy2FxQUICoqyukxzz//PO6//348/PDDAIA+ffqgqqoKjzzyCObMmQO5/OrMp9FooNG0r8HFn1+6q+2OAR0kroSIiMh3SdaSpFarMXDgQIdB2DabDRkZGUhNTXV6THV19VVBSKFQAHDf0hu+5rjRhGP5JqgUMvyhT7TU5RAREfksSacAmDVrFiZOnIhBgwZhyJAhWLhwIaqqqjBp0iQAwIQJExAbG4v58+cDAEaPHo0FCxYgOTkZKSkpyMrKwvPPP4/Ro0fbw1J798WlVqQrb9snIiKi5pM0JI0bNw5FRUWYO3cujEYj+vfvj/Xr19sHc+fm5jq0HD333HOQyWR47rnnkJeXh/DwcIwePRqvvPKKVKfgVaw2gTWZl7raktnVRkRE1BqSzpMkhbY8T9JPp4pw//u7YfBTYfecm6BRsnWNiIjahnY1TxK5XmNX2x/6RjMgERERtRJDUhtRba7H+iNGAMCdvKuNiIio1RiS2ogNR4yoNluRGKrDgPggqcshIiLyeQxJbcTqfc1bhoSIiIh+HUNSG1BgqsW2rGIAwB0tWIaEiIiIrsaQ1AZ8mZkHmwAGJQQjIbT1y5AQERERQ1KbsNq+DAlbkYiIiFyFIcnHHb1gwnFjBdQKOf7QJ0bqcoiIiNoMhiQf98X+8wAaliEx6FQSV0NERNR2MCT5MKtN4MvMCwCAO9nVRkRE5FIMST5sW1YxCivqEKxTYWS3CKnLISIialMYknzYF/sblyGJgVrJHyUREZEr8ZPVR1XV1WP94YZlSHhXGxERkesxJPmo9YeNqLFYkRTmj+S4IKnLISIianMYknxUY1fbHVyGhIiIyC0YknyQsbwW205zGRIiIiJ3YkjyQWsy8yAEMDgxGHEhOqnLISIiapMYknyMEAJfXFqG5M4BHSSuhoiIqO1iSPIxR/NNOFFQAbVSjtv6REtdDhERUZvFkORjGluR0npEwODHZUiIiIjchSHJh9RbbVhzaRmSO5LZ1UZERORODEk+ZGtWMYor6xDir8b1XcOlLoeIiKhNY0jyIY1zI43uG81lSIiIiNyMn7Q+orKuHhuONC5Dwq42IiIid2NI8hHrDxtRa7GhY5g/+nUwSF0OERFRm6ds6YFlZWXYvXs3CgsLYbPZHJ6bMGFCqwsjR6v3nQfAZUiIiIg8pUUh6euvv8b48eNRWVkJvV7v8KEtk8kYklwsv7wGO85cBACM5TIkREREHtGi7rYnnngCDz74ICorK1FWVobS0lL7o6SkxNU1tntr9l+AEMCQpBAuQ0JEROQhLQpJeXl5mDZtGnQ6fmC7mxDC3tV2J1uRiIiIPKZFISk9PR179uxxdS3kxJELJpwqrIRaKcetXIaEiIjIY1o0JmnUqFF46qmncPToUfTp0wcqlePyGLfffrtLiiNg9aVlSG7uGcllSIiIiDyoRSFp8uTJAIB58+Zd9ZxMJoPVam1dVQSgYRmSrw40LEPCrjYiIiLPalFI+uUt/+QeP11ahiTUX43fcRkSIiIij2r2mCSLxQKlUonDhw+7ox66QmNX2+h+MVApOO8nERGRJzX7k1elUiE+Pp5dam5WUWvBd5eWIblzALvaiIiIPK1FzRNz5szBX//6V86J5EbfHjairt6GTuH+6BPLZUiIiIg8rUVjkt59911kZWUhJiYGCQkJ8Pf3d3h+3759LimuPfviUlfbnQM6cBkSIiIiCbQoJI0dO9bFZdCV8souL0Mypn+MxNUQERG1Ty0KSS+88IKr66Ar/HCsAEDDMiQdgjmrORERkRR4y5QXOldaAwAci0RERCShFrUkyeXyXx0nwzvfWie/vBYAEG3QSlwJERFR+9WikPTFF184fG+xWLB//3588MEHePHFF11SWHtmLG9oSYpiSCIiIpJMi0LSmDFjrtr2xz/+Eb169cKqVavw0EMPtbqw9owtSURERNJz6ZikoUOHIiMjw5Uv2e7YbAIFpoaQFGXwk7gaIiKi9stlIammpgZvv/02YmM5O3RrXKwyw2IVkMmAiECN1OUQERG1Wy3qbgsODnYYuC2EQEVFBXQ6HT766COXFdceGS91tYUHaLheGxERkYRaFJLeeusth5Akl8sRHh6OlJQUBAcHu6y49ij/0qBtjkciIiKSVotC0o033oi4uDin0wDk5uYiPj6+1YW1V0b7eCSGJCIiIim1qD8nKSkJRUVFV22/ePEikpKSWl1Ue3b5zjYO2iYiIpJSi0KSEMLp9srKSmi1bAFpjcYxSWxJIiIiklazuttmzZoFAJDJZJg7dy50usvrilmtVuzatQv9+/dvdhGLFy/GG2+8AaPRiH79+uGdd97BkCFDnO47cuRIbN68+artt912G9auXdvs9/Y2HJNERETkHZoVkvbv3w+goSXp0KFDUKvV9ufUajX69euHJ598slkFrFq1CrNmzcKSJUuQkpKChQsXIj09HSdOnEBERMRV+69evRpms9n+/cWLF9GvXz/cfffdzXpfb2VvSdIzJBEREUmpWSHpxx9/BABMmjQJixYtgl6vb3UBCxYswOTJkzFp0iQAwJIlS7B27VosW7YMzz777FX7h4SEOHy/cuVK6HS6JkNSXV0d6urq7N+bTKZW1+wuQgj7mKSYII5JIiIiklKLxiQtX74cer0eWVlZ2LBhA2pqGrqImhqr1BSz2Yy9e/ciLS3tckFyOdLS0rBjx45reo33338f99xzD/z9/Z0+P3/+fBgMBvsjLi6uWTV6Umm1BXX1NgBAhJ4TSRIREUmpRSGppKQEN910E7p27YrbbrsN+fn5AICHHnoITzzxxDW/TnFxMaxWKyIjIx22R0ZGwmg0/ubxu3fvxuHDh/Hwww83uc/s2bNRXl5uf5w7d+6a6/O0xvFIYQFqaJQKiashIiJq31oUkmbMmAGVSoXc3FyHwdvjxo3D+vXrXVbcb3n//ffRp0+fJgd5A4BGo4Fer3d4eCve2UZEROQ9WjSZ5HfffYcNGzagQ4cODtu7dOmCnJyca36dsLAwKBQKFBQUOGwvKChAVFTUrx5bVVWFlStXYt68eddeuJfLtw/a5ngkIiIiqbWoJamqqsqhBalRSUkJNJprH0ujVqsxcOBAZGRk2LfZbDZkZGQgNTX1V4/99NNPUVdXh/vuu+/aC/dyRvtEkmxJIiIiklqLQtKIESPw4Ycf2r+XyWSw2Wx4/fXXccMNNzTrtWbNmoX33nsPH3zwAY4dO4bHH38cVVVV9rvdJkyYgNmzZ1913Pvvv4+xY8ciNDS0JafglfLZ3UZEROQ1WtTd9sYbb+DGG2/Enj17YDab8fTTT+PIkSMoKSnBtm3bmvVa48aNQ1FREebOnQuj0Yj+/ftj/fr19sHcubm5kMsds9yJEyewdetWfPfddy0p32sZTZxIkoiIyFvIRDPv27dYLLjlllswf/58bNy4EQcOHEBlZSUGDBiAKVOmIDo62l21uoTJZILBYEB5ebnXDeK+8c1NOFNUhf9NTsGwTmFSl0NEROQ1pPj8bnZLkkqlwsGDBxEcHIw5c+a4o6Z2SQhxxZgkDtwmIiKSWovGJN133314//33XV1Lu2aqrUe12QqAS5IQERF5gxaNSaqvr8eyZcvw/fffY+DAgVfNdr1gwQKXFNeeNLYiBelU8FNzIkkiIiKptSgkHT58GAMGDAAAnDx50uE5mUzW+qraocbZttmKRERE5B1aFJIaF7ol1+EcSURERN6lRWOSyPUa50iKDuKgbSIiIm/AkOQl7C1J7G4jIiLyCgxJXuJC45gkdrcRERF5BYYkL8E5koiIiLwLQ5KXMHLdNiIiIq/CkOQFKmotqKirB8CQRERE5C0YkrxAgamhFSlQq0SApkWzMhAREZGLMSR5gXzOkUREROR1GJK8QL59PBIHbRMREXkLhiQvwDmSiIiIvA9DkhfI551tREREXochyQsYL00kyTFJRERE3oMhyQuwJYmIiMj7MCR5AaOJs20TERF5G4YkidWYrSirtgBgSxIREZE3YUiSWGMrkr9aAb2WE0kSERF5C4YkieVfGrQdZdBCJpNJXA0RERE1YkiSmH2OJI5HIiIi8ioMSRLjnW1ERETeiSFJYvmcI4mIiMgrMSRJzMiWJCIiIq/EkCSxfPuYJIYkIiIib8KQJDF7S5KeA7eJiIi8CUOShGotVlysMgNgSxIREZG3YUiSUKGpDgCgUcoRpFNJXA0RERFdiSFJQlfe2caJJImIiLwLQ5KEGpck4Z1tRERE3ochSUL5nG2biIjIazEkSYhzJBEREXkvhiQJNY5JimFIIiIi8joMSRK63JLE7jYiIiJvw5AkIc62TURE5L0YkiRisdpQVNkwTxLHJBEREXkfhiSJFJhqIQSgVsgRolNLXQ4RERH9AkOSRBrHI0UaNJDLOZEkERGRt2FIkoh9PBIXtiUiIvJKDEkS4RxJRERE3o0hSSK8s42IiMi7MSRJxGhqmEiSLUlERETeiSFJImxJIiIi8m4MSRLhbNtERETejSFJAvVWGworGiaSZEsSERGRd2JIkkBxpRlWm4BCLkNYgEbqcoiIiMgJhiQJ5Jc3DNqODNRAwYkkiYiIvJLkIWnx4sVITEyEVqtFSkoKdu/e/av7l5WVYcqUKYiOjoZGo0HXrl2xbt06D1XrGo3jkaKDOB6JiIjIWymlfPNVq1Zh1qxZWLJkCVJSUrBw4UKkp6fjxIkTiIiIuGp/s9mMm2++GREREfjss88QGxuLnJwcBAUFeb74VsjnRJJEREReT9KQtGDBAkyePBmTJk0CACxZsgRr167FsmXL8Oyzz161/7Jly1BSUoLt27dDpVIBABITEz1ZsksYTY1LkjAkEREReSvJutvMZjP27t2LtLS0y8XI5UhLS8OOHTucHvPVV18hNTUVU6ZMQWRkJHr37o1XX30VVqu1yfepq6uDyWRyeEiNLUlERETeT7KQVFxcDKvVisjISIftkZGRMBqNTo85c+YMPvvsM1itVqxbtw7PP/883nzzTbz88stNvs/8+fNhMBjsj7i4OJeeR0vklzUM3I7mHElEREReS/KB281hs9kQERGBf/3rXxg4cCDGjRuHOXPmYMmSJU0eM3v2bJSXl9sf586d82DFzrEliYiIyPtJNiYpLCwMCoUCBQUFDtsLCgoQFRXl9Jjo6GioVCooFAr7th49esBoNMJsNkOtVl91jEajgUbjPXMR2WwCBSYuSUJEROTtJGtJUqvVGDhwIDIyMuzbbDYbMjIykJqa6vSY4cOHIysrCzabzb7t5MmTiI6OdhqQvFFxVR3qbQJyGRAe6D3hjYiIiBxJ2t02a9YsvPfee/jggw9w7NgxPP7446iqqrLf7TZhwgTMnj3bvv/jjz+OkpISTJ8+HSdPnsTatWvx6quvYsqUKVKdQrM1zpEUHqiBSuFTvZ1ERETtiqRTAIwbNw5FRUWYO3cujEYj+vfvj/Xr19sHc+fm5kIuvxwk4uLisGHDBsycORN9+/ZFbGwspk+fjmeeeUaqU2i2fC5sS0RE5BNkQgghdRGeZDKZYDAYUF5eDr1e7/H3/2B7Nl746ghu6RWFJfcP9Pj7ExER+SIpPr/Z3+NhvLONiIjINzAkeZixvHGOJIYkIiIib8aQ5GFsSSIiIvINDEke1rhuW0wQB24TERF5M4YkDxJCXG5J4uK2REREXo0hyYNKqy0w1zdMhBnJkEREROTVGJI8KP/SoO2wAA3USl56IiIib8ZPag9qnG2bd7YRERF5P4YkD7rAO9uIiIh8BkOSB3GOJCIiIt/BkORBnCOJiIjIdzAkeRDHJBEREfkOhiQPMtrnSOJEkkRERN6OIclDrpxIki1JRERE3o8hyUNMNfWosVgBcEwSERGRL2BI8pB8U8OdbcE6FbQqhcTVEBER0W9hSPKQy3e2cTwSERGRL2BI8hDe2UZERORbGJI8hIO2iYiIfAtDkodwtm0iIiLfwpDkIRyTRERE5FsYkjyEY5KIiIh8C0OShxi5bhsREZFPYUjygIpaCyrq6gEAUXqGJCIiIl/AkOQBja1Ieq0S/hqlxNUQERHRtWBI8oDLt/9z0DYREZGvYEjyAI5HIiIi8j0MSR7AiSSJiIh8D0OSBxgvLW7LliQiIiLfwZDkAWxJIiIi8j0MSR5g5GzbREREPochyQPYkkREROR7GJLcrNpcj/IaCwCGJCIiIl/CkORmjV1tARolArUqiashIiKia8WQ5GacI4mIiMg3MSS5GccjERER+SaGJDczmi61JHFhWyIiIp/CkORm+eUNE0myJYmIiMi3MCS5WX4Z50giIiLyRQxJbsYxSURERL6JIcnN7GOSGJKIiIh8CkOSG9VarCipMgNgSxIREZGvYUhyo4JLrUhalRwGP04kSURE5EsYktzo8ngkP8hkMomrISIiouZgSHIj+2zbnCOJiIjI5zAkuRHvbCMiIvJdDEluZLw0kSTvbCMiIvI9DEluZG9JCuJEkkRERL6GIcmNGudIiuaYJCIiIp/DkORGjS1J7G4jIiLyPV4RkhYvXozExERotVqkpKRg9+7dTe67YsUKyGQyh4dW630hxFxvQ3FlHQAO3CYiIvJFkoekVatWYdasWXjhhRewb98+9OvXD+np6SgsLGzyGL1ej/z8fPsjJyfHgxVfm8KKWggBqBVyhPirpS6HiIiImknykLRgwQJMnjwZkyZNQs+ePbFkyRLodDosW7asyWNkMhmioqLsj8jIyCb3raurg8lkcnh4gvGKrjZOJElEROR7JA1JZrMZe/fuRVpamn2bXC5HWloaduzY0eRxlZWVSEhIQFxcHMaMGYMjR440ue/8+fNhMBjsj7i4OJeeQ1MucDwSERGRT5M0JBUXF8NqtV7VEhQZGQmj0ej0mG7dumHZsmX48ssv8dFHH8Fms2HYsGE4f/680/1nz56N8vJy++PcuXMuPw9nGudI4ngkIiIi36SUuoDmSk1NRWpqqv37YcOGoUePHli6dCleeumlq/bXaDTQaDSeLBEA72wjIiLydZK2JIWFhUGhUKCgoMBhe0FBAaKioq7pNVQqFZKTk5GVleWOEluscUwS50giIiLyTZKGJLVajYEDByIjI8O+zWazISMjw6G16NdYrVYcOnQI0dHR7iqzRS63JHG2bSIiIl8keXfbrFmzMHHiRAwaNAhDhgzBwoULUVVVhUmTJgEAJkyYgNjYWMyfPx8AMG/ePAwdOhSdO3dGWVkZ3njjDeTk5ODhhx+W8jSuYuTitkRERD5N8pA0btw4FBUVYe7cuTAajejfvz/Wr19vH8ydm5sLufxyg1dpaSkmT54Mo9GI4OBgDBw4ENu3b0fPnj2lOoWr1FttKKxgSCIiIvJlMiGEkLoITzKZTDAYDCgvL4der3fLe+SX1yB1/g9QymU48fKtUMg5TxIREVFreOLz+5ckn0yyLWocjxSp1zIgERER+SiGJDfgeCQiIiLfx5DkBpwjiYiIyPcxJLkBZ9smIiLyfQxJbsA5koiIiHwfQ5IbcEwSERGR72NIcgOOSSIiIvJ9DEkuZrMJFJjYkkREROTrGJJcrLiyDvU2AbkMCA/QSF0OERERtRBDkos1drVFBGqhVPDyEhER+Sp+irsYxyMRERG1DQxJLsY5koiIiNoGhiQXyzexJYmIiKgtYEhyMc6RRERE1DYwJLkYZ9smIiJqGxiSXKyxJSmGLUlEREQ+jSHJhYQQ9pDEMUlERES+jSHJhUqqzDBbbZDJGuZJIiIiIt/FkORCjeORwgI0UCt5aYmIiHwZP8ldiHe2ERERtR0MSS5knyNJz5BERETk6xiSXIizbRMREbUdDEkuxDmSiIiI2g6GJBfKL+OYJCIioraCIcmFjFy3jYiIqM1gSHIRIQTyOSaJiIiozWBIcpHyGgtqLTYAQCTvbiMiIvJ5DEku0jhoO8RfDa1KIXE1RERE1FoMSS5iqrFAr1VyjiQiIqI2Qil1AW1FSsdQHPxbOsz1NqlLISIiIhdgS5KLcc02IiKitoGf6EREREROMCQREREROcGQREREROQEQxIRERGREwxJRERERE4wJBERERE5wZBERERE5ARDEhEREZETDElERERETjAkERERETnBkERERETkBEMSERERkRMMSUREREROKKUuwNOEEAAAk8kkcSVERER0rRo/txs/xz2h3YWkiooKAEBcXJzElRAREVFzVVRUwGAweOS9ZMKTkcwL2Gw2XLhwAYGBgZDJZC59bZPJhLi4OJw7dw56vd6lr01N43WXBq+7NHjdpcHrLo0rr3tgYCAqKioQExMDudwzo4XaXUuSXC5Hhw4d3Poeer2e/4gkwOsuDV53afC6S4PXXRqN191TLUiNOHCbiIiIyAmGJCIiIiInGJJcSKPR4IUXXoBGo5G6lHaF110avO7S4HWXBq+7NKS+7u1u4DYRERHRtWBLEhEREZETDElERERETjAkERERETnBkERERETkBEOSiyxevBiJiYnQarVISUnB7t27pS7JZ8yfPx+DBw9GYGAgIiIiMHbsWJw4ccJhn9raWkyZMgWhoaEICAjAXXfdhYKCAod9cnNzMWrUKOh0OkREROCpp55CfX29wz6bNm3CgAEDoNFo0LlzZ6xYscLdp+czXnvtNchkMsyYMcO+jdfdPfLy8nDfffchNDQUfn5+6NOnD/bs2WN/XgiBuXPnIjo6Gn5+fkhLS8OpU6ccXqOkpATjx4+HXq9HUFAQHnroIVRWVjrsc/DgQYwYMQJarRZxcXF4/fXXPXJ+3shqteL5559HUlIS/Pz80KlTJ7z00ksO64Dxurfeli1bMHr0aMTExEAmk2HNmjUOz3vyGn/66afo3r07tFot+vTpg3Xr1jX/hAS12sqVK4VarRbLli0TR44cEZMnTxZBQUGioKBA6tJ8Qnp6uli+fLk4fPiwyMzMFLfddpuIj48XlZWV9n0ee+wxERcXJzIyMsSePXvE0KFDxbBhw+zP19fXi969e4u0tDSxf/9+sW7dOhEWFiZmz55t3+fMmTNCp9OJWbNmiaNHj4p33nlHKBQKsX79eo+erzfavXu3SExMFH379hXTp0+3b+d1d72SkhKRkJAgHnjgAbFr1y5x5swZsWHDBpGVlWXf57XXXhMGg0GsWbNGHDhwQNx+++0iKSlJ1NTU2Pe55ZZbRL9+/cTOnTvFTz/9JDp37izuvfde+/Pl5eUiMjJSjB8/Xhw+fFh8/PHHws/PTyxdutSj5+stXnnlFREaGiq++eYbcfbsWfHpp5+KgIAAsWjRIvs+vO6tt27dOjFnzhyxevVqAUB88cUXDs976hpv27ZNKBQK8frrr4ujR4+K5557TqhUKnHo0KFmnQ9DkgsMGTJETJkyxf691WoVMTExYv78+RJW5bsKCwsFALF582YhhBBlZWVCpVKJTz/91L7PsWPHBACxY8cOIUTDP0y5XC6MRqN9n3/+859Cr9eLuro6IYQQTz/9tOjVq5fDe40bN06kp6e7+5S8WkVFhejSpYvYuHGjuP766+0hidfdPZ555hlx3XXXNfm8zWYTUVFR4o033rBvKysrExqNRnz88cdCCCGOHj0qAIiff/7Zvs+3334rZDKZyMvLE0II8Y9//EMEBwfbfw6N792tWzdXn5JPGDVqlHjwwQcdtt15551i/PjxQghed3f4ZUjy5DX+05/+JEaNGuVQT0pKinj00UebdQ7sbmsls9mMvXv3Ii0tzb5NLpcjLS0NO3bskLAy31VeXg4ACAkJAQDs3bsXFovF4Rp3794d8fHx9mu8Y8cO9OnTB5GRkfZ90tPTYTKZcOTIEfs+V75G4z7t/ec0ZcoUjBo16qprw+vuHl999RUGDRqEu+++GxEREUhOTsZ7771nf/7s2bMwGo0O18xgMCAlJcXhugcFBWHQoEH2fdLS0iCXy7Fr1y77Pr/73e+gVqvt+6Snp+PEiRMoLS1192l6nWHDhiEjIwMnT54EABw4cABbt27FrbfeCoDX3RM8eY1d9XuHIamViouLYbVaHT4kACAyMhJGo1GiqnyXzWbDjBkzMHz4cPTu3RsAYDQaoVarERQU5LDvldfYaDQ6/Rk0Pvdr+5hMJtTU1LjjdLzeypUrsW/fPsyfP/+q53jd3ePMmTP45z//iS5dumDDhg14/PHHMW3aNHzwwQcALl+3X/udYjQaERER4fC8UqlESEhIs3427cmzzz6Le+65B927d4dKpUJycjJmzJiB8ePHA+B19wRPXuOm9mnuz0DZrL2J3GzKlCk4fPgwtm7dKnUpbd65c+cwffp0bNy4EVqtVupy2g2bzYZBgwbh1VdfBQAkJyfj8OHDWLJkCSZOnChxdW3XJ598gv/+97/43//+h169eiEzMxMzZsxATEwMrzs1iS1JrRQWFgaFQnHVHT8FBQWIioqSqCrfNHXqVHzzzTf48ccf0aFDB/v2qKgomM1mlJWVOex/5TWOiopy+jNofO7X9tHr9fDz83P16Xi9vXv3orCwEAMGDIBSqYRSqcTmzZvx9ttvQ6lUIjIyktfdDaKjo9GzZ0+HbT169EBubi6Ay9ft136nREVFobCw0OH5+vp6lJSUNOtn05489dRT9takPn364P7778fMmTPtrai87u7nyWvc1D7N/RkwJLWSWq3GwIEDkZGRYd9ms9mQkZGB1NRUCSvzHUIITJ06FV988QV++OEHJCUlOTw/cOBAqFQqh2t84sQJ5Obm2q9xamoqDh065PCPa+PGjdDr9fYPpNTUVIfXaNynvf6cbrrpJhw6dAiZmZn2x6BBgzB+/Hj717zurjd8+PCrprg4efIkEhISAABJSUmIiopyuGYmkwm7du1yuO5lZWXYu3evfZ8ffvgBNpsNKSkp9n22bNkCi8Vi32fjxo3o1q0bgoOD3XZ+3qq6uhpyueNHnkKhgM1mA8Dr7gmevMYu+73TrGHe5NTKlSuFRqMRK1asEEePHhWPPPKICAoKcrjjh5r2+OOPC4PBIDZt2iTy8/Ptj+rqavs+jz32mIiPjxc//PCD2LNnj0hNTRWpqan25xtvRf/9738vMjMzxfr160V4eLjTW9GfeuopcezYMbF48eJ2fSu6M1fe3SYEr7s77N69WyiVSvHKK6+IU6dOif/+979Cp9OJjz76yL7Pa6+9JoKCgsSXX34pDh48KMaMGeP0Nunk5GSxa9cusXXrVtGlSxeH26TLyspEZGSkuP/++8Xhw4fFypUrhU6naze3ov/SxIkTRWxsrH0KgNWrV4uwsDDx9NNP2/fhdW+9iooKsX//frF//34BQCxYsEDs379f5OTkCCE8d423bdsmlEql+Pvf/y6OHTsmXnjhBU4BIKV33nlHxMfHC7VaLYYMGSJ27twpdUk+A4DTx/Lly+371NTUiP/7v/8TwcHBQqfTiTvuuEPk5+c7vE52dra49dZbhZ+fnwgLCxNPPPGEsFgsDvv8+OOPon///kKtVouOHTs6vAddHZJ43d3j66+/Fr179xYajUZ0795d/Otf/3J43mazieeff15ERkYKjUYjbrrpJnHixAmHfS5evCjuvfdeERAQIPR6vZg0aZKoqKhw2OfAgQPiuuuuExqNRsTGxorXXnvN7efmrUwmk5g+fbqIj48XWq1WdOzYUcyZM8fhNnJe99b78ccfnf4+nzhxohDCs9f4k08+EV27dhVqtVr06tVLrF27ttnnIxPiiulGiYiIiAgAxyQREREROcWQREREROQEQxIRERGREwxJRERERE4wJBERERE5wZBERERE5ARDEhEREZETDElERERETjAkEZHbbdq0CTKZ7KrFcomIvBlDEhG51MiRIzFjxgyHbcOGDUN+fj4MBoM0RTUhMTERCxculLoMIvJSSqkLIKK2T61WIyoqSuoyiIiahS1JROQyDzzwADZv3oxFixZBJpNBJpMhOzv7qu62FStWICgoCN988w26desGnU6HP/7xj6iursYHH3yAxMREBAcHY9q0abBarfbXr6urw5NPPonY2Fj4+/sjJSUFmzZtarIeIQT+9re/IT4+HhqNBjExMZg2bRqAhhavnJwczJw5015ro61bt2LEiBHw8/NDXFwcpk2bhqqqKvvziYmJeOmll3DvvffC398fsbGxWLx4sWsvJhFJjiGJiFxm0aJFSE1NxeTJk5Gfn4/8/HzExcU53be6uhpvv/02Vq5cifXr12PTpk244447sG7dOqxbtw7/+c9/sHTpUnz22Wf2Y6ZOnYodO3Zg5cqVOHjwIO6++27ccsstOHXqlNP3+Pzzz/HWW29h6dKlOHXqFNasWYM+ffoAAFavXo0OHTpg3rx59loB4PTp07jllltw11134eDBg1i1ahW2bt2KqVOnOrz2G2+8gX79+mH//v149tlnMX36dGzcuNEVl5GIvIUgInKh66+/XkyfPt1h248//igAiNLSUiGEEMuXLxcARFZWln2fRx99VOh0OlFRUWHflp6eLh599FEhhBA5OTlCoVCIvLw8h9e+6aabxOzZs53W8uabb4quXbsKs9ns9PmEhATx1ltvOWx76KGHxCOPPOKw7aeffhJyuVzU1NTYj7vlllsc9hk3bpy49dZbnb4PEfkmtiQRkSR0Oh06depk/z4yMhKJiYkICAhw2FZYWAgAOHToEKxWK7p27YqAgAD7Y/PmzTh9+rTT97j77rtRU1ODjh07YvLkyfjiiy9QX1//q3UdOHAAK1ascHiP9PR02Gw2nD171r5famqqw3Gpqak4duxYs68DEXkvDtwmIkmoVCqH72UymdNtNpsNAFBZWQmFQoG9e/dCoVA47HdlsLpSXFwcTpw4ge+//x4bN27E//3f/+GNN97A5s2br3qvRpWVlXj00UftY5euFB8ff83nR0S+jyGJiFxKrVY7DLZ2leTkZFitVhQWFmLEiBHXfJyfnx9Gjx6N0aNHY8qUKejevTsOHTqEAQMGOK11wIABOHr0KDp37vyrr7tz586rvu/Ro8e1nxAReT2GJCJyqcTEROzatQvZ2dkICAhASEiIS163a9euGD9+PCZMmIA333wTycnJKCoqQkZGBvr27YtRo0ZddcyKFStgtVqRkpICnU6Hjz76CH5+fkhISLDXumXLFtxzzz3QaDQICwvDM888g6FDh2Lq1Kl4+OGH4e/vj6NHj2Ljxo1499137a+9bds2vP766xg7diw2btyITz/9FGvXrnXJuRKRd+CYJCJyqSeffBIKhQI9e/ZEeHg4cnNzXfbay5cvx4QJE/DEE0+gW7duGDt2LH7++ecmu8GCgoLw3nvvYfjw4ejbty++//57fP311wgNDQUAzJs3D9nZ2ejUqRPCw8MBAH379sXmzZtx8uRJjBgxAsnJyZg7dy5iYmIcXvuJJ57Anj17kJycjJdffhkLFixAenq6y86ViKQnE0IIqYsgIvIliYmJmDFjxlUzixNR28KWJCIiIiInGJKIiIiInGB3GxEREZETbEkiIiIicoIhiYiIiMgJhiQiIiIiJxiSiIiIiJxgSCIiIiJygiGJiIiIyAmGJCIiIiInGJKIiIiInPh/gQJOq7Wg+ssAAAAASUVORK5CYII=\n" + }, + "metadata": {} + } + ], + "source": [ + "# Create a Neural LinTS pearl agent with 1-hot action representation\n", + "\n", + "action_representation_module = OneHotActionTensorRepresentationModule(\n", + " max_number_actions= env._action_space.n,\n", + ")\n", + "\n", + "agent = PearlAgent(\n", + " policy_learner=NeuralLinearBandit(\n", + " feature_dim = env.observation_dim + env._action_space.n,\n", + " hidden_dims=[64, 16],\n", + " training_rounds=50,\n", + " action_representation_module=action_representation_module,\n", + " exploration_module= ThompsonSamplingExplorationLinear()\n", + " ),\n", + " replay_buffer=FIFOOffPolicyReplayBuffer(100_000),\n", + " device_id=-1,\n", + ")\n", + "\n", + "\n", + "info = online_learning(\n", + " agent=agent,\n", + " env=env,\n", + " number_of_steps=number_of_steps,\n", + " print_every_x_steps=100,\n", + " record_period=record_period,\n", + " learn_after_episode=True,\n", + ")\n", + "torch.save(info[\"return\"], \"LinTS-return.pt\")\n", + "plt.plot(record_period * np.arange(len(info[\"return\"])), info[\"return\"], label=\"LinTS\")\n", + "plt.xlabel(\"time step\")\n", + "plt.ylabel(\"return\")\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pm18iQ_2LNGS" + }, + "source": [ + "## Summary\n", + "In this example, we showed how to use popular contextual bandits algorithms in Pearl. The figures that should be obtained upon running this code can be found in:\n", + "pearl/tutorials/cb_algorithms/cb_algorithms.png.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ANo74OTbLNGS" + }, + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "custom": { + "cells": [], + "metadata": { + "custom": { + "cells": [], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "fileHeader": "", + "fileUid": "4316417e-7688-45f2-a94f-24148bfc425e", + "isAdHoc": false, + "kernelspec": { + "display_name": "pearl (local)", + "language": "python", + "name": "pearl_local" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 + }, + "fileHeader": "", + "fileUid": "1158a851-91bb-437e-a391-aba92448f600", + "indentAmount": 2, + "isAdHoc": false, + "language_info": { + "name": "plaintext" + } + }, + "nbformat": 4, + "nbformat_minor": 2 + }, + "indentAmount": 2, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file