rl_cart_pole_dqn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reinforcement Learning: CartPole DQN Solution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"./screenshots/dqn.png\" alt=\"drawing\" width=\"1500\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import random\n",
    "import math\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Memory Buffer for Experience Replay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RingBuffer():\n",
    "    def __init__(self, size):\n",
    "        self.size = size\n",
    "        self.buffer_ar = []\n",
    "        for i in range(0, size):\n",
    "            self.buffer_ar.append(0)\n",
    "        self.counter = 0\n",
    "        self.full = False\n",
    "    \n",
    "    # add new experience to memory buffer\n",
    "    def add(self, value):\n",
    "        self.buffer_ar[self.counter] = value\n",
    "        self.counter += 1\n",
    "        if self.counter > self.size-1:\n",
    "            self.counter = 0\n",
    "            self.full = True\n",
    "            \n",
    "    # get n random samples from memory buffer\n",
    "    def get_random(self, last_n, sample_size):\n",
    "        if not self.full:\n",
    "            last_n = self.counter\n",
    "        if self.counter - last_n < 0:\n",
    "            end = np.linspace(self.counter-last_n, -1, -1*(self.counter-last_n))\n",
    "            start = np.linspace(0, self.counter-1, self.counter)\n",
    "            window = np.concatenate((end, start))\n",
    "        else:\n",
    "            window = np.linspace(self.counter-last_n, self.counter-1, last_n)\n",
    "            #print(window)\n",
    "        random_index = np.random.choice(a=window, size=sample_size, replace=False)\n",
    "        random_choices = []\n",
    "        for i in random_index:\n",
    "            random_choices.append(self.buffer_ar[int(i)])\n",
    "        return random_choices\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return str(self.buffer_ar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to decay epsilon; Exploration vs Exploitation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epsilon_by_frame(frame_idx):\n",
    "    #return 0\n",
    "    epsilon_start = 1.0\n",
    "    epsilon_final = 0.01\n",
    "    epsilon_decay = 500\n",
    "    return epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot our rewards and loss recurrently during agent training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20, 5))\n",
    "    plt.subplot(131)\n",
    "    plt.title(\"frame %s. reward: %s\" % (frame, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.ylim(0, 200)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct DQN network with Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DQN, self).__init__()\n",
    "        self.fc1 = nn.Linear(4, 128)\n",
    "        self.fc2 = nn.Linear(128, 128)\n",
    "        self.fc3 = nn.Linear(128, 2)\n",
    "        \n",
    "    def forward(self, states):\n",
    "        #print(states)\n",
    "        out = F.relu(self.fc1(states))\n",
    "        out = F.relu(self.fc2(out))\n",
    "        out = self.fc3(out)\n",
    "        #print(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Initialize our DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dqn = DQN()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Specify our network's optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.Adam(dqn.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv4AAAE/CAYAAAA+Occ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmYJGd1Jvr3xJJZe+9qra2WoBFCLJLcCDDDDma10cjGBjMgY3wFNlyv4wHb4w1jg+2xYbjYeOQLF/kaCxgwxmODDCMQQjYgSwK0C+1Sq/eluvbMjIgzf8T3RUZERm5VUZWdVe/vefrpqsisrMhSq/vEifc7n6gqiIiIiIhofXMGfQJERERERLT6WPgTEREREW0ALPyJiIiIiDYAFv5ERERERBsAC38iIiIiog2AhT8RERER0QbAwn+dEJELROS7IjIrIr846POh1SUiN4jIzw36PIiI1hMReUREXj7o8yBaLSz814//AuAGVZ1U1Y8M+mTyRORqEblPRCIR+ZncYz8jIqGIzKV+vdg8dpqIXCsi+0XkpIj8q4g8J/f1Py0ij4rIvIj8g4hsTT22VUS+YB57VER+ei3e76lERK4UkVtFZEZE9onIn4iIl3q848+o08+34HtdbL7Xgvn94tV8b0RERNQ7Fv7rx7kA7mr3oIi4a3guRb4P4BcA3Nbm8W+p6kTq1w3m+ASAfwfwQwC2ArgGwD+LyAQAiMhFAP4HgLcA2AlgAcBfpl73LwDUzWNvBvAx8zV9kdia//+SLtBXYAzALwPYDuA5AF4G4D+nHm/7M+rh55s+1wqALwL4WwBbEP+3+qI5TkRERAPGwn8dEJGvAXgJgI+abvlTROSTIvIxEfmSiMwDeImIvNbEgWZE5HER+b3Ua+wWERWRt5nHTojIO0Xk2SJyu4hMi8hHc9/3Z0XkHvPcfxGRc9udo6r+hapeD2Cpn/emqg+p6p+r6gFVDVX1agAVABeYp7wZwP9S1RtVdQ7AbwO4QkQmRWQcwI8D+G1VnVPVmwD8I+IitisTp/lDEflXxAXv+SKySUQ+LiIHROQJEXm/vagyXfEfMh//J/PzfJr5/OdE5B/Mx5eJyLfMz/SAiHw0XRybr3uXiNwP4H5z7BUicq+56/FRANLHz/BjqvpNVa2r6hMAPgXg+eZ1u/2M2v58C77ViwF4AD6sqjVz50kAvLTXcyUiOhWISFVEPmzuNu83H1fNY9tF5J/M3+HHReSbtjEkIu8x/zbMSnyX+2WDfSdEWSz81wFVfSmAbwJ4t+mW/8A89NMA/hDAJICbAMwDeCuAzQBeC+DnReTy3Ms9B8AeAD8F4MMAfgvAywFcBOAnReRFAGC+7jcBXAFgh/n+167gbVwiIkdF5Aci8tvtOt0mOlIB8IA5dBHiuwkAAFV9EHH3+inmV5j6ecA8t5+O/1sAXIX4Z/go4i52AODJAC4B8CMAbNb+G4iLXwB4IYCHALwo9fk3zMchgF9B3IF/HuIO/C/kvu/liP9bPE1EtgP4PID/ar7mQZjCHQBEZJf5B2hXj+/phWjeHer2M+r08827CMDtqqqpY7ejv583EdGp4LcAPBfAxQCeBeAyxH8HA8CvAdiH+N++nYj/LVQRuQDAuwE8W1UnAbwSwCNre9pEnbHwX9++qKr/qqqRqi6p6g2qeof5/HbEhfqLcl/zB+a5X0F8oXCtqh42neJvIi52AeAdAD6gqveoagDgjwBc3Knr38GNAJ4O4DTE3ec3Afj1/JNEZArA/w/g91X1pDk8AeBk7qknERfqnR7r1SdV9S7zHrcCeDWAX1bVeVU9DOBDAN5onvsNNH+eLwDwgdTnLzKPQ1VvVdVvq2qgqo8gjtLk/zt8QFWPq+oigNcAuFtVP6eqDcQXZAftE1X1MVXdrKqPdXszIvI2AHsB/DdzqNvPqJ+fYRk/byKiU8GbAbzP/Pt3BMDvo3kntAHgDADnqmrD3FFVxE2dKuKGja+qj5hmCdEpg4X/+vZ4+hMReY6IfF1EjojISQDvRNxBTjuU+nix4PMJ8/G5AP676TRPAziOONZxVr8naeI8D5sLkjsAvA/AT+TOfRTA/wLwbVX9QOqhOQBTuZecAjDb5bFepX+G5wLwARxIve//gfiCBYgL+xeIyOkAXACfAfB8EdkNYBOA75n38hRzm/igiMwgvmjK/3dIf98z05+bf2AeR5/MXZoPAni1qh41h7v9jPr5GZbx8yYiOhWcifgur/WoOQYAf4r4rvNXROQhEXkvAKjqA4jXU/0egMMi8mkRORNEpxAW/uub5j7/O8T57XNUdROAv0IfWfGcxwG8w3Sa7a9RVf23FZyvpenzMrnKfwDwBOI7DWl3Ib4Na597PuKOyw/ML09E9qSe/yx0WATd5lysxwHUAGxPvecpVb0ISP7SXwDwiwBuVNVZxJ35qwDcpKqReZ2PAbgXwB5VnUJ8mzj/3yH9fQ8AOCf1HiX9eS9E5FUA/hrAj5qLK6vbz6jTzzfvLgDPNOdnPRP9/byJiE4F+xE3e6xd5hhUdVZVf01VzwfwowB+1Wb5VfXvVPU/mK9VAH+8tqdN1BkL/41lEsBxVV0SkcsQrwFYrr8C8BvSnP6ySUTe0O7JIlIRkRHEBa4vIiOpxVCvFpGd5uOnIl5A+kXzuQ/gc4jvNrw1VTxbnwLwoyLyArNQ9X0A/t78xTwP4O8BvE9ExkXk+QBejzgu1DdVPQDgKwD+TESmRMQRkSfZdQ/GNxBnPG2e/4bc50D832EGwJx5vz/f5Vv/M4CLROQKs/bhFwGc3ut5i8hLEf+cflxVb869p24/o7Y/34JvdQPiW92/aBbGvdsc/1qv50pEdIq4FsB/FZEdZp3V7yCeWAYReZ2IPNk0OWYQ/70XSryfzktNs2oJ8b9b4YDOn6gQC/+N5RcQF3iziP8S++xyX0hVv4C4k/FpE1e5E3H+vZ2vIP5L8IcBXG0+fqF57GUAbpd4+tCXEBeif2Qe+2EAr0O8iHZamnP+X2DO4y7EkaVPATiMuKhOL5T9BQCj5rFrAfy8+RqYYnauz7f+VsSLi+8GcALxRckZqce/Yc7hxjafA/EozZ9GHIH5a8SRoLZMLOcNiGM6xxAvvv5X+7hZ3DvXYXHvbyOOGn0p9fP7curxtj+jbj9fEfmyiPymeW4d8aLktwKYBvCzAC43x4mIhsn7AdyCeEDBHYhHUb/fPLYHwP9GHG/8FoC/NCOoq4j/nj6K+G7vaYjv6BKdMiQ7gIOIiIiIiNYjdvyJiIiIiDYAFv5ERERERBsAC38iIiIiog2AhT8RERER0QbAwp+IiIiIaAPwBn0CALB9+3bdvXv3oE+DiOiUdOuttx5V1R2DPo9B4r8TRETF+vk34pQo/Hfv3o1bbrll0KdBRHRKEpFHB30Og8Z/J4iIivXzbwSjPkREREREGwALfyIiIiKiDYCFPxERERHRBsDCn4iIiIhoA2DhT0RERES0AbDwJyIiIiLaAFj4ExERERFtAF0LfxE5R0S+LiL3iMhdIvJL5vhWEfmqiNxvft9ijouIfEREHhCR20Xk0tV+E0RERERE1FkvHf8AwK+p6oUAngvgXSLyNADvBXC9qu4BcL35HABeDWCP+XUVgI+VftZERERERNSXrjv3quoBAAfMx7Micg+AswC8HsCLzdOuAXADgPeY43+jqgrg2yKyWUTOMK9DBABQVXzz/qN4wZ7tEJGWx/dPL2KuFuApOyc7vs6J+ToeOTaPS3ZtKeW8wkjxrQeP4T/s2V7K6wHAzQ8fxzPO2oTRitvy2MnFBh48ModLU+c/Xwtw94EZPHv31tLOwXrs2AL+9cGjAICK6+C1zzwDI37reT10ZA43P3wcWvoZbFzP3r0VTz5tYtCnQUS07nzrwWO49NzNqHqt/55RVtfCP01EdgO4BMB3AOy0xbyqHhCR08zTzgLweOrL9pljmcJfRK5CfEcAu3btWsap0zC77bFpvPUTN+Oz73geLjuvtcD903+5D/ccmMF1v/zCjq/zR1+6B9fddRB3/N4rSzmvG+8/grf9f/+Or/7KC7Gny0VHL04uNPDGq7+FP7j86Xjzc85tefxT33kUH/7q/bj7fa+E58Y34P7+u0/gd794J777Oz+CTaP+is8h7Y+vuxf/fEfzf0Xfc/Bjzzoz+fzx4wv4yPX34/O37UPEqr9UH7ziGRuy8BeRRwDMAggBBKq6V0S2AvgMgN0AHgHwk6p6YlDnSETD6+79M3jTX38bVz7vXPz+658+6NM55fVc+IvIBIDPA/hlVZ0p6tLapxYcaykhVPVqAFcDwN69e1libDDH5+sAgNmlRuHjJxbqmF0KOr5GI4zwlbsPYaEelnZe0wvxec3VOn/vXs3VA0QKTC8Uv8+ZxQD1MEItiJLCf2axgUjjn03Zhf+x+RouPmczPnDFM/Dq//7N5P0C8Z2GH/3oTVioh3jb88/Df3ruuRgtuBtAyzM12lefZb15iaoeTX1uo6IfFJH3ms/fM5hTI6JhdsL8O/aDQ3MDPpPh0NO/RCLiIy76P6Wqf28OH7IRHhE5A8Bhc3wfgHNSX342gP1lnTCtD3O1uBCuB1Hh4/O1ALWgc0H/7YeO4eRi/DpRpHCcthejPbMXEUFJ7e5aI369xTYXJ/b914II49Xs15R5QWPNLgU4fWoE5+8YTz63jszWML3QwJ/8+DPxk88+p91LEJWhXVSUiKgvK/+Xf2PpZaqPAPg4gHtU9c9TD/0jgCvNx1cC+GLq+FvNdJ/nAjjJfD/lzdXiorbWtvAP2z5mffnOg8nHjajzc3u1YM6r0eV798q+h/l68R0Ee3GTvshJvqakuw5ps0sBJkc8VD0XFdfJFP724y3jldK/L21oCuArInKriXgCuagogNPafjURUQ+Uq9J60kvH//kA3gLgDhH5njn2mwA+COCzIvJ2AI8BeIN57EsAXgPgAQALAN5W6hnTujBnisx2Hf+FetCx8A8jxVfuOpR8HoSKqvnTHEWK+XqAyZH+YzK2y94oq+Nv3kO3jv9So/lel1a1499Ifi6TI14mamU/nhzZ0JEUKt/zVXW/WQf2VRG5t9cv5FowIuqKLf++9DLV5ya0/7G+rOD5CuBdKzwvWuds1KddnGeuFqIeRFDVwqk/tz56Akfnarh012bc9tg0grBZqH/63x/HB798D25670sx1Wfxv9CIL0iCsKSOf5civpZEfVa/46+qSccfACZGvMxahhlzMcbCn8qkqvvN74dF5AsALkP7qGj+a7kWjIh6ovwboifcuZcGwnb823X1F0w0phEW/5983Z0HUfEcvOJppwMA6qlC/eDMEmaWAnz93sJaoiPbmW/3ffu1ZN5fu8K/qONf6/I1yz6XRoQg0lzHPx31iS/G+r1YImpHRMZFZNJ+DOBHANyJ9lFRIqK+iOlNs+7vDQt/GohOGf8o0qToLbojoKr4l7sO4oV7tmPzWFykBqmMv+3Wf/mOgy1f2828zfiX3vEv7t7bCxb7PKAZ9Wm3LmC58lGeiWo+6sOOP5VuJ4CbROT7AG4G8M+qeh3iqOgrROR+AK8wnxMR9a39kEkqwn/haSCaUZ/WAnshVQQXrQGYXmjgielFvO35u+GbEZjpqI8t2m/4wWEs1sPCjbPaWbRRn5IWC3fr3jcX9xZ0/GvldvzzUZ7JER+PH19IHreF/0SVfy1QOVT1IQDPKjh+DAVRUSKiZWPLvyfs+NNA2Gx5UWG/kMqdF10Y2GNjFQ++G1/qp6M+Nqaz1IjwjR/0F/dZKDnq0/vi3rXr+Nsoz2S1NeozVnGT/QSIiIhOdbbhz6k+veG/8DQQzYx/a0GcXnBadGFgj1U8B55T3PHfPOZjy5iP6+7sL+5ju+xBaYW/ifo02o3zjDK/pz8uO+Ofj/JM5hb3phf+EhERDYMOG8pSARb+NBC24CyM+qQK3npB1r4exo9XPCfp+DcyHf8II56LVzxtJ66/53DXjcAy37thFxWXlfHvHNsp6vjb8y17qk+z8PeT3+dqAdSMQpitNZY1ApWIiGjQONWnNyz8aSA6RX3SBW+t0T7qU3GdJOOfLtSDUOF7glc//QzM1gL824PHej6vZtRnrTL+rR1/O+Gn/I5/bnHviIcwUiyaiw52/ImIaNjYhj/r/t6w8KeB6DTOM51tt939NHuxUPUceKbjH6Q23KqHEXzHwQ8/eRsqnoNv91H42yx+UNoGXvHrLTZCRAWvWS+M+sRfM7dqHX8v87s9PrO0vE3PiIiIBoVBn/6w8Kc1F++sGxe39YIYznwqFlPU8U9n/Nt2/F0HVc/FVC7H3o2929DosGtwPzKd/IL3aov87OJe2/Evf3GvCDBeaY7ztMft7+z4ExHRMFJmfXrCwp/WXLqjX9jxT0d9CjP+6cLfZvyzi3t9Lz4+4rtJlKUX9rmNkjr+6YJ+viDnX7i41071WYVxnhNVD44T/2zsdB/b8Z9dCjDFwp+IiIYIoz794b/ytObmumT459OLeztN9UmNnQxSFwj1MEqm/Yz6bqb47qQRRskFRFByxh8oHumZFP6Zxb2r1fEPMrvyTuSiPnHHn1EfIiIaJgz79IOFP625uaV0hr//Of7pqI+9s5fu+AehJhcFoxW37Qz9lu+bel5pGf/UhU1+pKeqtmT8VTX5uOyOfz7KYz+eqwVohBGWGhEmuXkXERENISZ9esOoD625WVPYu44Uz/Gvd5njXxj1yY7ztIt++4n6LHT5vsuRfn/5KT3pi56iHXxXo+OfLfxt1KfRsvCXiIhoGDDq0x8W/rTmbIZ/63ilMOqTnnlfVICnx3naXWaDKFX4R5os+h31XSwWfI8i2Y5/+VGf/Cz/esEIz/R7my97nGduTn9zcW+QGvXJqA8REQ0PBn36w8Kf1pyN+mwbrxRGfebrAcYrLoDinX3T4zwLF/cGUXJ8rOJiqccCOh0JKm/n3ua55Dv4RSM8bdZ/63gF9SAqbT8BoLXjny382fEnIqIhxqxPT1j405qb7dLxn68F2DJeAdBlcW+7cZ5RlOn457P1lqpixnS6gWzHv1FW4d8IsXksfi/5yFGnjr99/2Vu4pUv/F1HMF5xMbsUJD8HdvyJiGiYiLDn3w8W/rTmko7/RLV4cW89xFZT+BYu7k1l/D0zmjLIjPPUJAI0UnGxWC/umt94/1Hsff//xtG5GoDsmNEyd+7dMhYX0/kivqjjbycQbUsK/3Jy/qpaOLVncsTHXI0ZfyIiGm7s9/eGhT+tOZvx3zZeyYyxtOZqATaN+nCk+zhP32vt+DfCZrym0zjPA9OLqAcR9k8vAshFfUrK+C+lOv7zuY3E0u+t1qbjX9Zkn1oQjyrNF/YTI14m6jPFjj8REQ0R2+9n0qc3LPxpzc3VAlQ9B2MVt7Cjv1ALMV7xUPGcwjsC9SCCI4DnOvAdW/hnN/CqZBb3hoU7+tnXnl6IYy62Iz9Z9UqL+tRTHf/8WNH0+oWlZKqPyfgndwnK6fi3i/JMmp2Nm4t72fEnIqLhwaRPf1j405qbrcVZ86rnIogUYW5m/nw9wFjVRcV1Cu8I1MMIFdPpt2M7g0zHX5PjoxUXYaSFhbztuE8v2sLfdL1H/VKjPhNVH74rWGiT8Z8c8ZKOv836bx2vAiiv49/s6Oc6/lUPM6mO/wQLfyIiGkLKsE9PWPjTmpuvBZioeknxno/zzNcCjFc8VH23bcffdvRtxr8RZTv+dnHviB9PByqa5d/s+NcBpDr+I16JU31CjPhOfOehTcZ/asRv7fhPlJvxb5fhnxrxzRz/BkZ9N/m5ERERDQMxYR9GfXrDf+Vpzc0tBZgY8VBtV/jXQ4xXPdPxL57jX/Higl5E4LtSkPFvRn0AFOb8G0H8t0Q+6jM1UmLHvxGh6rkYq3htM/5FHX+7uLesWf7t5vRPjniYMx1/xnyIiGjYMOrTHxb+tOZmk45+/McvnXVvhBHqQYTxiouq76DWpuNvLxoAwHOcTNQnCLW5uLcSPy/fbQeAehgfs4X/Yj3AqO+i4jkIovLm+Fd9B2NVtyXqY9/31KifdP/tsS1tFgQvV7uO/0TVS8Z5svAnIqJhxY5/b7oW/iLyCRE5LCJ3po59RkS+Z349IiLfM8d3i8hi6rG/Ws2Tp+E0Z7rLNq5TtLvtmOn4F071SWX8AZiOf/x/vKoiiBSek+34F0Z9gmzUZ74eYqziwsvdQViuKFLUwyhZyNwx6tOwG3jZjH/ZhX+7jr+PxUaIE/Otoz6JiIiGBev+3vTS4vskgI8C+Bt7QFV/yn4sIn8G4GTq+Q+q6sVlnSCtP/P1OONf9e3uvFHmMQCYqLqoek7xHP8gTC4aAMB3naRQtxcA9sJgtBL/ES/aCMs+1y7uXayHGKu68BynlKk+dg1B1XMx5nttd+6dGvVQCyKoanIBsGW8ePb/crXt+JvPD84s4ewto6V8LyIiorXCqE9/unb8VfVGAMeLHpN4u7SfBHBtyedF65jN+Dc7/s3i1na4xyrx1J96UNyp973m/+meK8liXHsBYBf9dsr414L84t4AY76HiieZ6NBy2e591XMwWnFbivh6quNvz8ee03glXgMxX9o4zwAiwEQlW/jbC4H904uc4U9EREOraGw3tVppxv8FAA6p6v2pY+eJyHdF5Bsi8oIVvj6tQ7O1AOPVZsa/nun4x8WxnfrTbufelo5/ZDv+UXIMSEV9Cjv+rXP8Rytxx7+MjL+d1FP1HYxXWwv/Zse/tfCveg7Gq14SfVqp2aUGJioeHCfbGpmsesn3ZsafiIiGjYAt/36s9F/6NyHb7T8AYJeqHhORHwLwDyJykarO5L9QRK4CcBUA7Nq1a4WnQcOiHsSLdyerzak+2Yy/7fjHUZ8TC23GeXr5qI/t+Me/+0nUxyzu7ZTxX2wW/jbjX7S2oF/Njr+LUd9rufhodvxt8R1iqRHCcwSeG68LKKvj325qTzrXz8KfiIhofVt2x19EPABXAPiMPaaqNVU9Zj6+FcCDAJ5S9PWqerWq7lXVvTt27FjuadCQsVGeiTaF/5x5fNx0/AsX96bGeQJxrCcIcx1/09nuOMc/FfWJIjWFfxxBCqISCn/b8TeLe1sz/s3xoUB8oVBLTSwar2Q7/tEK7kLMLhUv3k0X+1zcS0REtL6tJOrzcgD3quo+e0BEdoiIaz4+H8AeAA+t7BRpPbGF/cSIj6op3tPFvY3DjHeI+tSCgqiP6fTbrH9Pc/zNRUKkcfxosR4kHf8yNvCy5z7iuxirui0z+etBBN8VjFTsIue4428vVuKviX9e3/jBETzj9/4lWY/Qr3Yd/4lM4c+OPxER0XrWyzjPawF8C8AFIrJPRN5uHnojWhf1vhDA7SLyfQCfA/BOVS1cGEwbk50uk+34pxb3mkJ33ER92o3zrLaM84ySx4B4wS8AjFbaZ/zTuwKfXGg0oz6OU7hjcL8yHX/fQz2IEKa69vYCZsS8l6Wijr8579sePYH5eoh9JxaXdS7toz7s+BMREW0UXVt8qvqmNsd/puDY5wF8fuWnRevVXCrqUynYuTeZ6pN0/IsjOumMv5eK5tjf7R2BEa971AcAphfrqcW9JXX8U1N9xswFyEI9SArsehCh6rupsaah2fDLdPwrLo7O1QAAjx6bBwAcn19ux7+B87aPtxyfrDLjT0REtFFw515aU0nGf8RLoj6ZOf52Ay/fRcV122f83eINvBpB/LtnHnccQdVzigv/MN4hGABOLDSwUI93FPZLy/ibwt93C+881Mx+BLbjX2tEWGqEzY5/tdnxf/T4AoCVFP7FHf8R30lGn7LwJyIiWt9Y+NOami3o+Nca2Tn+YxU3Ltj94shN6869qQ28IjvOsznea7Rg11wgvoDYMVkFAByaWUKk8XM9s2ZgpTOB01Gf8Wpc+Kdz/nHH30k6/EsFHX+7IPjRY8sv/FUVM20W94pIkvPnHH8iIqL1jYU/ram51A6ytrOdLu7nzWQdII7rNEJtmWbTEvVJRXMaQTbqA8R3D9rN8T9tcgRAvIEVEBfbdiLQSmf5p2fyj/p2B+Eg83jFdZprHRoRao0wuQMwUfUwXwsxs9RICv7lFP61IEIj1LYdfXucHX8iIqL1jYU/ram5Wjwzfzy9uLeRnuoTYMJ0x5MNvnJd/+I5/nacZzbqAwAjFbdtxt92/A9MLwGAmeoTf+1Kc/52klDVd5OM/2JBx38k1fFfynT8PSw2QjxydD75mmPLKPxnluKfebvCfsLk/Lm4l4iIaH1j4U9raq4WQiTuwnuuA0fyGf8g0/EHso9HkSKItGWcp+3OF0Z9fLfNOE/FaMXFZNXD/pO24+8lX9tYYc4/3fFvLu4NM48XdfybGf/4a+49MAsgvrNxok3hf3y+jpNmB+K82dRdliLs+BMREW0MLPxpTc0tBZioeHBMnKbqudmoTy3ERNUzj7WO+7TPzU71aY7ztFEfP3VhMOoXd/xr5s7BpjE/G/UpqeOfnerTGvWpBxGqnpt0/GtBPM5zJNXxB4C7D8QbX194xlTbqM8vfOpWvOvvbit8LIlXVYs7+lMjHkZ8J/MzIyIiovWH/9JTqe7eP4PXfuSbydjOvLlaI7NpVMVzMot7F+oBxmzUp2CDL1v4VzMZfycp0m3nP1P4t13cG0/V2TzmY7+J+oyaDbyA5gZfy2UvWEZSUZ+F/FQfr9nxX2qEhR3/u/fPYPtEBedsHcWx+Vrh99p3YhH/9uDRZPxnmv2e9ueaNzXic2EvERHRBsDCn0p1z4EZ3LV/BgdPLhU+PlcLko4+EBfw6Y7/XC0eqQk0u/rpqI+9CEh3/CtequOf28ALiAvvxUZrEd8IFRXPwZaxSnJHYKziwXeczGstVy2I4Egc0WkX9ammCv9mx99JzgWIf6a7to5h63gFJ9rEeaYXGogU+Ordh1oeW2wEmdfLu+pF5+MDVzxjme+SiIiIhgULfyqV3Zk2bDMRZ64WYjxd+PtObnFvmHS6k6k/RYW/m+345xf3VnJRn6KMfz2M4LuCTaPNbvd4xYXvmak+K436mCiPiBTO8beLlD03nqW/1AjNHH83ORcgHoG6e9s4to5XcWKh3vKzrQdRcoflujsPtpxH0vGvFHf8n3r6FF524c4Q5aMTAAAgAElEQVQVvVciIiI69bHwp1KFaiM3xd3yhVqQFPZAXKCnO/pz6cW9nQr/XMY/GedZ0PEf9d1Mth6IL0zCSFFxXWweaxb+8c69JXX8G2Eymci+p/ncOE9b5Fc9J+n42wuesdQF0q5tY9g65kMVOLmY7fpPL8a5/81jPv7twaMtj9vCf9QvLvyJiIhoY2DhT6Xq1vFvRJrJ31c9Nyn8VTXT8S+M+hQs7q24TjKBJwgLFvcWZPwbqdfZMlZJjmem+pTS8Y/Pw7U7COen+pjHR8zFSRBpsrh3PNWh371tHFsn4tGjx3M5/2kT/7nikrPRCBVfuzcb91ns0vEnIiKijYGFP5XKFvztNr8Kwgie0+zGVzwnWQRbCyKEkSZRoMLFvUVRH1eSIr0eti7uHfFdLOUy/rVk+k826pOZ6lPCOE/7HuxrL2SiPs2FvFXPSTr17Tv+8QXKsbnsZB874vPFF+zA6VMj+PId2bhPM+rDcZ00GCLiish3ReSfzOfnich3ROR+EfmMiFS6vQYRURHFypp0Gw0LfypVt45/GGkSpQHM4l5ThM+bnHrr4t5slzz9GBBn/MNIoapJJz89x3+sEo8MDVLRnUZqOtBmU1A7En9uN/Baacd/KTWhJz4Pr3BxLxBv8jWzGL//oo7/uWZxLwCcWMgV/qbjv3W8glc9/XR84wdHMtGmxXoAESSLhokG4JcA3JP6/I8BfEhV9wA4AeDtAzkrIlo3RKT7k4iFP5Urshn/NkVzI4zgporyqt+M+tiieDw3x79bxt9+3Ai1OOqT7Izb+jq+62CLyfiPVTyICHynt3Ge7S5urJrZmdeKO/5xQa6qqIdR546/uQCarHrYOl7BtgnT8c/N8p82FwJbxit40QU7UAsi3LHvZPL4Qj3EqO/yL0UaCBE5G8BrAfy/5nMB8FIAnzNPuQbA5YM5OyKijYWFP5UqiDov7o07/qmoT2px71zS8c9m/NPjPovn+DcLdRv1SX+PkYKJOumMv13cayfveD1s4PX48QVc+DvX4XuPT7d9Ti0I20Z9GqFCtfkeq77bLPzNxULFc+C7gl3bxiAiyXkez0d9TMd/y5ifxIHS+ygsNELm+2mQPgzgvwCw/yNvAzCtqvYP6T4AZw3ixIiINhoW/lSqrhn/fNTHd1A3UR67SHWTKXBtjj897rOZ8W8WsulCPTAjOtPd7aTj3whbX8dzsGk0LpZtcZws7u2Q8b/10ROoBxHu2n+y7XNqjSgTr0kvMm5ewMTfc8RzMLPUMB8339t41cO528aS505WPRxfaO34VzwHo76b3C1JF/6L9TC5qCFaSyLyOgCHVfXW9OGCpxb+hSEiV4nILSJyy5EjR1blHImINhKu9qNSRTbj36ZbHoTZjn811fG32XWbZbed71rYJeqTKtQbYZS5sACahf9io3WtgO82O/42WuP30PG/9+AsALTdqMx+j6nMHgEeDs3Gz7e7FVcyGf9sxx8A3v2SJ+NpZ04ln2+dqOD4fL7jX8eWMR8ikmyOll5LsFAPMObzf3UaiOcD+DEReQ2AEQBTiO8AbBYRz3T9zwawv+iLVfVqAFcDwN69e7mCj4hohdjxp1IFPXT8sxn/5uJeW9DauErVdPVr6U59mC2YAaQW40ZohJpZ2AsAo5X48bZRn1Fb+NuoT/eM/30HZwB0K/yzi3tHKy4WavmOfzPjb39k6Y7/z73gfPzwk7Ynn28ZKyr8G8lI0jEzCnU+HfVhx58GRFV/Q1XPVtXdAN4I4Guq+mYAXwfwE+ZpVwL44oBOkYhoQ2HhT6Wyi3vbT/WJksWzQHaOvx1Laafs2M53vUvH395BCMJ4qk96YS/QnJKzUC+I+rjxFJ/Jqtcs/HvYwOs+2/Gf6dzxz071aWb8bXwpPcffqnaYvrNtvLXwn16oJ3ct7ESkfNSHGX86xbwHwK+KyAOIM/8fH/D5EBFtCLz/T6Xqtrg3CBVuKoqTnuN/fKGOiaqXFMM2499tjn9zqk9UWPgXZfztqE77tZvG/KQ4rnSJ+pxcbGC/6fR37Pg38nP8vWTn3nzGP32BkP6avC3jFdx9YCZz7MRCA3tOmwAQbxQ26rstHf/07sREg6CqNwC4wXz8EIDLBnk+REQbEQt/KpXN+LcrmoNIkygN0Jzjr6o4MV/HlvFmgeo4As+RzM697eb4A3acp8L38lGf1ox/EhkyRf5vvPpC7JiMd8btFvX5waG423/utrHuUZ9U937TqI+5WoAgjAo6/ukNx7p3/FU1WcAcd/yb+x+NV13Mp+5uLDZCjHLzLiIiog2PUR8qVW8beGUL/0jjC4ITC40k359+PNPxLxrn6abHeUbw2y3uLYj62LsDr33mGbjsvK3Z12vzHu41HfcXP2UHZmtBJlaTttTIRn22T1ahGt/ZsBcezYx/KurToeO/dbyCWhAlkSFVNRn/1CLiqpfr+AcY8xn1ISIi2uhY+FOpui3ubURRdo5/apOuEwt1bBnPFv7pKJB9HpDdoCuJ5kSm498m6pPt+GejPmn2wiFo0/G/9+AsJkc8XLxrM4DiuI+qtszx327e29HZekvHPxP16dDxtz8fm/OfWQoQRpos7gXinD8X9xIREVEeC38qVXNxb2vRHEXxplXpjL8tjGtBhOPz9YKOv9uS8XcdgZu6eEh3/BthlIkSAc2oT+Ecf7eg8Pc6Z/zvOziLp54+idOnRgEAhwoW+AaRIlK0dPwB4Nh8LRlRWi1a3Nuh478tV/jbXXs3Zzr+LuZrqagPF/cSEREReij8ReQTInJYRO5MHfs9EXlCRL5nfr0m9dhviMgDInKfiLxytU6cTk2dNvCyx9KFue1414IQJ+azWXX7eL7wzxfr6Sk8jai14z9SEPVJj/PMs3ck6gUdf1XFfYdmccHpkzhj0wgA4EBBx9+uRUgX9LZoPzpX69zxLzgna2uu8G/u2pvO+KcWEQcRgkhZ+BMREVFPHf9PAnhVwfEPqerF5teXAEBEnoZ4VvNF5mv+UkRYcWwgnTL+dtJPPuMPALNLAebrIbaOZ6fPxFGfbMY/X6xXvNQ4zyBqmePvuw58V7JRn4JFwunn29fL239yCbNLAS44fQqnm8L/4MnFlufZvQfSsZ2k4z9Xb925N9Px76fwj39PL4oer3rJugN7scPFvURERNS18FfVGwEc7/H1Xg/g06paU9WHATwAjmzbUGzB3ygomm3H383N8QeaOfl8xr9lcW/QWvinO/5B1DrOE4gL66I5/vmLBHt+jhSPJLUbdz319EmM+C42j/mFs/ztxUq6iJ+seqi4Do7M1ZoXBrmOf9Vzkmk9RfKFfzPqk874NzcKW2jEFwDs+BMRUVn+8//8Pn7rC3cM+jRoGVaS8X+3iNxuokBbzLGzADyees4+c4w2iGbHv7VoDs3FgF8wg98Wz/mMf0vHvyjqk2T8FfVQk51800Z9N5vx7xD1iV/TKbx4udds3PWUnZMAgNOnRgoX9zYL/2bBLSLYPlHB0dl66869fmvkp8hE1YPvCo7Zjv98m6iP6fjbix0W/kREVJbP3boPn/rOY4M+DVqG5Rb+HwPwJAAXAzgA4M/M8aJWZeEKSRG5SkRuEZFbjhw5sszToFNNqO0z/g1zMeAWRH0Oten4V9xsx78WRi3FcXOqT4RGEKFS0MUfrbiFUZ/86E/Ld6Rwjv99B2dx5qYRbBqNozWnbxpp0/HPdvStbRPVeHFvfo5/QeSniIhg63gFx+drAOKOvwiS8wHii4P5egBVbUZ9OM6TiIjWMdXigRyUtazCX1UPqWqoqhGAv0YzzrMPwDmpp54NYH+b17haVfeq6t4dO3Ys5zToFNQp42+PFY3zPGCK5y35qT6+m0zAAdpEfdzuUZ9R321Z3Ou7AscpjtV4rlM4zvPA9BLO2TqWfN62428K+/xozu0TFRydq7Xu3Os7hc8vcu62cdx3aA5AvLh306ifuZgar3qINB5f2uz4M+NPRETrjxT2nKmdZRX+InJG6tP/CMBO/PlHAG8UkaqInAdgD4CbV3aKNEw6TvUJ7VSf1gk2zY5/bnGv6yR5eKBdxr8Z9Wm0ifqM+K0d/6JRnpbvOoUbeE0v1jMXJ6dvGsHRuXpyB8F2+peSDH+2075toopjc61z/JOOf4dRntZlu7fizidOYr4WxHsf5C6Wxk2sZ74WYsFM9+EcfyIiIuplnOe1AL4F4AIR2ScibwfwJyJyh4jcDuAlAH4FAFT1LgCfBXA3gOsAvEtVwzYvTetQ56k+rR1/WxjbkZitHX8nM1azqGC3xXMQatLJzyvK+Psd8vS+K2gErR3/k4uNTKzm9Kl4ss+hmSU8emwez/r9r+Br9x4qXNwLANtt4R+E8FL7EfTT8b/svK0II8Vtj53A9EIjM8MfiDv+ADBfC5K7HMz4ExERUdf7/6r6poLDH+/w/D8E8IcrOSkaXnYDr6J8fFiQ8bdF+6GZJUyOeC0xnWou418vyPg3O/7xBl5Fuf3RiovDs43k80bYuePvuVJ41yJfaNuRnodmlvC1ew9jqRHh+nsO48UXnBaff66Dv32ignoY4ehcLXPnIon89NDxv/TcLXAdwc0PH8eJhTp2mosPyxb+c7WAi3uJiIgowZ17qVRBDx3/dEfeFvHH5uvJqMq0wqk+nTL+ocL32izuTWX8awWvk+Y7TsvFy1IjRC2IsKmg8H9iehGfu3UfAODmh483F/e2ZPzjWf77p5cyFzAjvpP5vZOJqoennzmF7zx8vLjjb/L8C/UQCw07x5+FPxER0UbHwp9K1UvG33VaM/4AWnbttY9327m3OdVHUQ+jZK5/Whz16fw6ab7rtGzgdXIxvmOQjvqcMTUKAPj0zY/j8GwNl+zajPsPz+HAdBxdymf2t03E73H/9OKyO/5AHPf53uPTODpXa834V23GP8Bi3c7x5+JeIiKijY6FP5XKRn3CDht4FWX8AWBrrnMN2I5/Npvf2vE3UZ8gjvoUdfJHfTdZ6Aqg7fPSr5nv+E8vxIX/5tFmoT016mHEd/Cth45h+0QF73nVUwEANz1wNH5/7Tr+Jxcz773aR8cfAC47bxvqQYRaEGFL7uc2URD14ThPIiIiYuFPpbJd8uKOf1xIe+moT6rQzc/wB+ILg+4795rCP1IEoWYuLKyiOf5FYz+T1yyY6tPcJbdZaIsIztgUd/1//NKzcemuLah6Dr7z8DFz/vk5/vF7XGpEK+r4P3v3luTj/J2SsaqN+sSLe6uek1lXQURERBsTC38qVdLxL9i5114MZBb3porv/K69QNzxj7R50VALWhf3ikg8hSeMEERaWNCPmKhPFNnFx9qx419xpWWOf1HUBwB2TsVd/DfsPQcVz8Glu7YksaJ8Ib91rAIxb7/qtUaeuu3ca20eq+Cpp8e7B+ejPhMV2/GP5/hzYS8REREBLPypZLa4L5qB39zAq/nHznEkWexb1PG3xbld4FsPwsJsvuc4yeLddlGf7Ot0merjtGb8p9sU/i+54DRccclZePJpEwCAZ5+3teX8k9d1naRQrxQU/t127k179u74++SjPmOpjH9c+DPfT0RERCz8qWS2o16U8W8URH2AZlc837mOH4v/iNq4T1HG376mLfyLoj626z1vcv61LnP8PVcy+wcAwEmb8c8V2u940ZPw5z91cfL5c0zh77tSGLHZbuI+6e6+iGDnVBU7JqttzynvhU/ZARHgrC2jmeO+66DiOfHi3kbAiT5EREQEoIc5/kT9CLV9xj8sWNwLmM53Ddg6Xry4F8h16gsK9orrJKMri6I+yYLXpQDbJ6po9DLVJ2qN+riOJK/VziW7NsNzpG1ef9t4FcAcKrnH/+n/fgEmR3r/X/LlF56GG3/9JThn61jLYxNVD/P1gFEfIiIiSrDjT6Wy8ZheM/5As/Nd3PGPi9Z6ECEII0QKVNzWQjbu+Mfd/KKde21BPbsUPye+c9B+wavvSkHUp45Noz5EOi+UHat4eMbZm9rm9bebrn7+8R2T1b6iPiJSWPQD8UjPeZPx50QfIiIiAtjxp5JFHTr+tpDOd+RtB7/dBl4AUA/DJHpTGPVxnGR0ZVHHf3IkvpswuxTHdbrv3Ou0RH2mFxrYPNp6V6LIW557Lm7fd7LwsW3jrRn/so1XPMzV4qk+NlpEREREGxsLfypV2HHn3riQbtfxL9rAyxbnS40oyfkXRn28ZuHvFRb+8R/1Gdvx77pzb2vH/+RiI7NrbydXXHo2rrj07MLHdrTp+JdpvOphoR5goR5grFJ8V4CIiIg2FkZ9qFSddu61j/m5nXVtnCe/aBZozvmvh50Lf89pLu4tivpM5Tr+vczxLxrnmZ/osxxFi3vLNl71MFcLsVgPubiXiIiIALDwp5Ili3vDgoy/6aC7buvi3qkRr7AQr5pjtUaULPCtFo3zdB0sNOJuflGEpzjj33lxb+sGXr1HfTqJF/f2vlnXckxU3XicZ4OLe4mIiCjGwp9KZdf0Fkd9iqf6VD2nMN8P5Dr+HTL+lfQ4z6KpPvnCv+tUH0nGj1rTC/XCOFK/7OLe1cz4j1U8LJg5/uz4ExEREcCMP5XM5viLoz5mjn+u8D9/x3hhzAdoTvCpB12iPm56cW9r1Md3HYz6bnZxb6c5/rkNvMJIMVsLMFVKx3/1oz4TVQ8zSwHqQYQxn/+bExEREQt/KlnYoePfCFt37gWA91/+jLav15zjHzYL/8KdewWLHeb4A3HcZ3YpSI0F7b3jP7vUgCpKifrsmKzCd6Wvmf39Gq+6mKvFdzcY9SEiIiKAhT+VLOzY8S/O+Hdiu+K1Rpeoj+fALC/oXPjXGsnrdNq5N97Aq/keptvs2rscI76Lz77jeXjSaRMrfq12xirN/7UZ9SEiIiKAGX8qWedxnsUZ/05Om6pCBHj8xAIaXab6JB+3ubCYHPExuxSgEcTn0XmOvyCMFJE555OLceFfxlQfALhk15Zk0tBqSO8uzI4/ERERASz8aRlmlxotC18tW+8XPW4n/fRT+I9VPJy/fRx3PjGDWqcNvFJFfLuCfnIkzr3XQhMJ6tLxB4CGuYMxvVhex38tjLPwJyIiohwW/tS31/0/N+HqGx8qfMwu7u3U8c9v4NXN08/ahLv3n+yY8U8fa9fxnxrxzUVLfB5FY0Etu0DYLvCdXqgDADaNDscuuBPVZrE/WmGij4iIiFj40zIcmF7CoZmlwsfsOM92GX/PEYj0V/hfdOYU9p9cwmHzPYum4aSL/W6LeztNB0pezyxAtncuZkqO+qy2dMafHX8iIiICWPhTn1QV9TBKuuZ5dgOvwqk+UdR3tx8ALjpzEwDgu49NAygu7NOTgjpFfWaXGknh32nnXtvxbyQd/+Eq/NNRn1GfhT8RERGx8Kc+2Yk4RTvzqmpS8Bc9HobaV77fuujMKQDAbY+dANBuqk9vi3uXGhEW6kHb12m+RvxYkMr4j1fcVd10q0xc3EtERER5w1HF0CmjFrQf15k+1C7jX7Srbjebxyo4a/MoHjm2AKDdVJ/msU5RHwA4Pl83z2t/EWJfw2b8Ty42hqbbD2SL/TFm/ImIiAg9FP4i8gkROSwid6aO/amI3Csit4vIF0Rkszm+W0QWReR75tdfrebJ09qzMZl6UUc/Vew3Cgv/aFkdfwB4+llTycfFU31SGX+nXeEfF+7H5uptXyd5DfN69n1OLzSwaWw4FvYC2Y4/5/gTERER0FvH/5MAXpU79lUAT1fVZwL4AYDfSD32oKpebH69s5zTpFOFLfwLozypOf1FHf8w0mVl/IFmzh/oPtXH99pFfeJi+Jjp+BctErbsHYRmx79eyq69a4XjPImIaCPZd2Jx0KcwFLoW/qp6I4DjuWNfUdXAfPptAGevwrnRKSiJ+hQs7rULe6uegzBSqGafE4TacUFtJ5mOf9Hi3lTH32vb8TeF/1wNQOfFvV6yuDd+vycXG0Mzwx+I72b4rphfTPQREdH6pIhrjbla0OWZBJST8f9ZAF9OfX6eiHxXRL4hIi9o90UicpWI3CIitxw5cqSE06C1YDv+RVGe0FwM2AhNvusflNDx912BU/Aa2Yx/+zn+QLPj3ynqU0kW9zan+gxTxh+Iu/6c6EODJCIjInKziHxfRO4Skd83x88Tke+IyP0i8hkRGZ4cHRHREFtR4S8ivwUgAPApc+gAgF2qegmAXwXwdyIyVfS1qnq1qu5V1b07duxYyWnQGkoK/6Ag6pN0/ONiM78AOIiWN9UHAE6brGL7RLXtqE5bxHfaJyAf9emn4z+92MCmIer4A8B4xctEfogGoAbgpar6LAAXA3iViDwXwB8D+JCq7gFwAsDbB3iOREQbxrILfxG5EsDrALxZTaZDVWuqesx8fCuABwE8pYwTpVNDPQwBNMdcptkOf9uOfxi1HbXZjYjgojOn2nbp7QVFp2K+ubg3jvq0u4iIX6+5gddSI0Q9iLB5SHbttcarLhf20kBpbM586ptfCuClAD5njl8D4PIBnB4R0YazrHagiLwKwHsAvEhVF1LHdwA4rqqhiJwPYA+Ah0o5Uzol1Bqm41+U8c8V/vl1AHHUZ/k3md76vHNx+77NhY/ZMaGdRnTmx3l2WtxrXycIdeg277LGq15yx4JoUETEBXArgCcD+AvEDaHp1DqxfQDOGtDpERFtKF0LfxG5FsCLAWwXkX0AfhfxFJ8qgK+aWMW3zQSfFwJ4n4gEAEIA71TV44UvTEOpZjfwKur4pxb3Fj0nXEHUBwBeduFOvOzCnYWPVdzuHX/fdTDiO8k4z27PBeKO//Ri/PxhWtwLABfsnMRSIxz0adAGp6ohgIvN2OcvALiw6GlFXysiVwG4CgB27dq1audIRLRRdC38VfVNBYc/3ua5nwfw+ZWeFJ266h2m+kRRtvDPR30aK4j6dNPs+He+ozA54uPIrIn6dNy512b8FSfm447/MI3zBIAP/vgzB30KRAlVnRaRGwA8F8BmEfFM1/9sAPvbfM3VAK4GgL179xZeHBDRxiZYnbpiveKcP+pLrcMGXkE+6pMr/Ffa8e/Evm63Cwsb9wG6beDVvGvxxHQ8G/iMzaMrPU2iDUVEdqQ2eBwF8HIA9wD4OoCfME+7EsAXB3OGREQbC0d+UF86dfzDKDvVp8xxnt3YIr7Tgl2gucAXQMeLkKTwDxWPH1+ACHDm5pESzpRoQzkDwDUm5+8A+Kyq/pOI3A3g0yLyfgDfRZu7yEREVC4W/tSXXnbubdfxD8Jo1cZL2ik83Tr+U6bjX/GctmM/49eLH6uHER4/sYDTp0aSCxoi6o2q3g7gkoLjDwG4bO3PiIhoY2PUh/pSD+LFooUbeLVk/FsX965Wx9/rYXEv0Iz6dLszkO747zu+iHO2jJVwlkRERESDw8Kf+lLr0PGPNNvxz4/8XMkGXt3Y8Ztet8K/Gkd9OuX749cx4zyjuON/9lbm+4mIiGi4sfCnviQ79xZk/INuGf9Qk0hO2WyHvtLj4t5eO/7ztRAHZ5bY8SciIqKhx8Kf+mKn+RRtDJWP+rRk/KMI7mqN83R6H+cJAL7X+TzsHYTHjs9DFThnKwt/IiIiGm4s/KkvyeLegox/pN0z/gOP+vTY8bcXEg8dmQcAnLOFUR8iIiIabiz8qS824x9GCtXWKA+QmuoT5jfwOnWiPt3uDNgLiYeOmsKfHX8iIiIaciz8qS+28Adac/6tHf813MDLdvy7XFjYqE+1y+JeEYHnCI7M1uC7gp1TnOFPREREw42FP/Wlnin8W6M8QGqqT9EGXquU8bcdfL9LQZ+e49+NvZg4a/Poqo0hJSIiIlorLPypLzUzxx9ojfK07tybvTAIogj+qmX8TeHf5fWTxb1doj7xa8XPYcyHiIiI1gMW/tSXTMe/YPEukJrqk78wCBXuKmX8bYSo5w28euj427sHZ3OUJxEREa0DLPypL/VUvCdf2Ae5qE/LHP9Ik/hM2ZpRn3IW9wLNi4lzuHkXERERrQMs/KkvnTL+zcW9cdSnaI7/qo/z7HFxb08df3NxwM27iIiIaD1g4U99qfWxuLew479qU32czPdup+I5qHoOqr10/F3b8WfhT0RERMOPhT/1Jd3xz3f0W6b6pC4MokihilXL+Dc7/t0vLLZPVDFhIj+dX9N2/Bn1ISIiouHXvfohSqkHERwBIm3f8S+a428XAq92xr/bzr0A8ImfeTa2jle6Ps9zBGMVt6fnEhEREZ3qWPhTX2pBiPGKh9la0Dq1J7eBV/qOgL0IWL2Mv4Nff+UFePmFO7s+94LTJ3t+zXO2jEGEM/yJiIho+LHwp77UgwhjVReztaBDx9/NfA40d/ntpSO/XO96yZNLfb2Lz9mM0Ypb6msSERERDQoLf+pLPYwwNeoDqCXFvJXP+K9lx381/MHlTx/0KRARERGVhot7qS+1IMJ4Jb5eDKJ24zxtxj+9EDj+2B2iwp+IiIhoPWHhT32pBRHGq2ZOf34DrzBb+KfvCNjH/FVa3EtEREREnfVU+IvIJ0TksIjcmTq2VUS+KiL3m9+3mOMiIh8RkQdE5HYRuXS1Tp7Wlqqinur4t9vAq2iOv/14tcZ5EhEREVFnvVZhnwTwqtyx9wK4XlX3ALjefA4Arwawx/y6CsDHVn6adCqwHfyxqpf53LKZfjtaM53xD4Yw409ERES0nvRU+KvqjQCO5w6/HsA15uNrAFyeOv43Gvs2gM0ickYZJ0uDVQtCAMCEjfpExVN9XEfgu5LN+IerO8efiIiIiDpbSe5ip6oeAADz+2nm+FkAHk89b585RkPO7trbjPpkO/5RqvB3HWHHn4iIiOgUshqB66LKTlueJHKViNwiIrccOXJkFU6DylY3XXsb9QlyGX9b3Lsi8BwHYciMPxEREdGpYiVV2CEb4TG/HzbH9wE4J/W8swHsz3+xql6tqntVde+OHTtWcBq0VmzH30Z9ihb3igBOQce/wagPERER0UCtpPD/R1/LqDoAACAASURBVABXmo+vBPDF1PG3muk+zwVw0kaCaLjVbNSnzeLeMFK4Ehf2niOZNQDDuIEXERER0XrS0869InItgBcD2C4i+wD8LoAPAvisiLwdwGMA3mCe/iUArwHwAIAFAG8r+ZxpQPIZ/6LFvXaDLteRzDjPIJX/JyIiIqK111Phr6pvavPQywqeqwDetZKTolOT7fiPVWzUp6Djbwp733UyG3w1N/Bixp+IiIhoEFiFUc/sOM/xZHFvrvDXZtSnteMfJceJiIiIaO2x8Kee2ajPiO/CkdbFvWGkcN10xr91qg8z/kRERESDwcKfemYL/6rnwHMdNIoy/m06/jYW5HGcJxEREZVE2E/sC6sw6pmd41/xHPiOtEZ9IoWTWtybviOQdPw5zpOIiIhoIFj4U89qjWzHP7+BVxhpEuXxXGb8iYiIiE4lLPypZ5mOv+ugEbUu7nWSqI+TyfgnU30Y9SEiIiIaCFZh1DOb8a+4DnxX0AgKOv4myuPnMv72Y5dRHyIiIqKBYOFPPUsW9/ouPDc7tQdoXdyb3uAr4FQfIiIiooFi4U89s3P8K64D33FaxnlG2lzc2y7jz8KfiIiIaDBY+FPPbMffdyXu+Oem+gRhc3Gv6ziZnX0DjvMkIiIiGihWYdSzWhih4jkQEfiuk4nyAKbjL80NvJjxJyIiotWk2v051MTCn3pWa0SoevEfGc91UM93/CNNxnW6uZ17G4z6EG04InKOiHxdRO4RkbtE5JfM8a0i8lURud/8vmXQ50pEtBGw8Kee1cNm4R9v4FWwc68p7H1XEKbuCIQhF/cSbUABgF9T1QsBPBfAu0TkaQDeC+B6Vd0D4HrzORERrTIW/tSzehCh4tqOf2vGP9J0x9/JdfxN1IeFP9GGoaoHVPU28/EsgHsAnAXg9QCuMU+7BsDlgzlDIqKNhYU/9awexBl/AGYDr2zHPwibhX9rxj+C5whEWPgTbUQishvAJQC+A2Cnqh4A4osDAKcN7syIiDYOFv7Us1oQouq5AEzhXzDOMzPHPz3VJxUDIqKNRUQmAHwewC+r6kwfX3eViNwiIrccOXJk9U6QiGiDYOFPPUt3/D2nNeqTzvh7+Q28UqM+iWjjEBEfcdH/KVX9e3P4kIicYR4/A8Dhoq9V1atVda+q7t2xY8fanDAR0TrGwp96Vg9zUZ8Oi3vdgnGenss/bkQbicTZvo8DuEdV/zz10D8CuNJ8fCWAL671uRERbUTeoE+AhketkVvcG+U6/pqe6pNd3BuYjD8RbSjPB/AWAHeIyPfMsd8E8EEAnxWRtwN4DMAbBnR+REQbCgt/6lk9jDAxEv+R8V2nIOqDZAMv15FkhCeQXfhLRBuDqt4EoN3/+C9by3MhIiJGfagP6XGeviuot0R9ml19L7eBVxApfEZ9iIiIiAaGlRj1LLu41+m4gZebW9wbcqoPERER0UCx8Kee1YIoGedZtIFX61Sf1AZeITP+RERERIPEwn+NXH/PIfz839466NNYkVqq418p2MArzO3cqwpEpviPp/qw8CciIiIalGUv7hWRCwB8JnXofAC/A2AzgP8LgN1t5TdV9UvLPsN14uaHj+PLdx5EFCmcIe1814MQVS811SfX8Y9Si3ttkR9Eiorp/rsOrzOJiIiIBmXZhb+q3gfgYgAQERfAEwC+AOBtAD6kqv+tlDNcJ2pB3B2vhxFGHHfAZ7M8tXzGP1KoKkRskZ9d3AsgmeUfMOpDRERENFBltWBfBuBBVX20pNdbd2pBaH6Pujzz1KSqqIdR0vH3TUe/EaY36UJyN8NGfuwC34BRHyIiIqKBKqvwfyOAa1Ofv1tEbheRT4jIlpK+x1CrNeIC2F4ADJu4u4/UBl6OOd68kIlUYSd25jv+YaTs+BMREREN0IoLfxGpAPgxAP/THPoYgCchjgEdAPBnbb7uKhG5RURuOXLkSNFT1pUk6jOkHX973pWk4x//3shs0hXBMzl+N/d4EGryGBERERGtvTIqsVcDuE1VDwGAqh5S1VBVIwB/DeCyoi9S1atVda+q7t2xY0cJp3FqG/aoT62l8DdRnjDd8U8t7s1n/KOIUR8iIiKiASqj8H8TUjEfETkj9dh/BHBnCd9j6NnC2UZ+ho3t+Cdz/J3Wjn96ZGc+488NvIiIiIgGa9lTfQBARMYAvALAO1KH/0RELgagAB7JPbZh2YK/Hg534V9JjfME4o25rDDSpONv7wjYjn+DUR8iIiKigVpR4a+qCwC25Y69ZUVntE4lUZ/GcC7uteef3sALQGZ33jC1uNfO7A+4uJeIiGhdqqfGfNNw4H+tNZKe4z+MaknUJ9vxtxl/VTVxHjvnvzXj7zLjT0REtG785Q0PDPoUqE8s/NfI0Gf8w1zUxxT49rht/LuSzfjbKFAQKXx2/ImIiNaNE/P1QZ8C9YmF/xqxEZ9hneqTLO5181N9mlEeAG3n+Adh824AEREREa09VmJrpBn1GdaMf35xb3YDr2bhb+b4J1N9UuM82fEnIiIiGhgW/mtk6KM+uXGefjLVx3T8Ndvxtxt8ZXbuZcafiIiIaGBY+K8ROxVnWBf3ttu5Nx/1cXIZf/t4wKk+RERE64p2f8qqE5YWfWHhvwbCSJPO+NB2/MPsOE8vt3jXFv72ODP+RERERKcWVmJroJ5a0Gs7/8PGXrDkO/75wt92+pOpPpGd6hMl8SAiIiIiWnss/NdAutivD+tUn7DNHH9T8EeaXdxrx32GqSiQy6gPERHRutEY0vjyRsbCfw3UMh3/4fyfpF3GPz2nH0Bq597mhYFqHHVixp+IiGj9uPbmxwd9CtQnFv5rIJ3rH9bCf64WAABG7FQfJ7u4N8ot7rWxnjDSZHMvOwKUiIiIiNYeK7E1kI76DGvh/9ixBZy5aSQ1x791Z9708WbHP0pm/TPqQ0RERDQ4LPzXQG0dLO598Og8ztsxnnyeFP5R8TjPJOMfaXJXgFEfIiIiosFh4b8Ghn1xr6ri4SNzOG97s/CvJHP84/fTXNxrOv5uc45/824A/7gRERERDQorsTUw7Bn/4/N1zCwFOG/7RHLMy23gle/qe6nFvfkZ/0RERES09lj4rwFb7IucuoX/Yj3E8z5wPa6782DLYw8fnQcAnJ/q+Hu5Of2245/fuTeMouSuADP+RERERIPDwn8N2KjPRNVD/RTN+D9weA4HTi7hvoOzLY89ZAr/dNQnGecZZDP+Nvuf7vjbqA838CIiIiIaHBb+a8B2+adG/NI6/mGkuO7OA1DTaV+pB47EBf/sUqPlsYePzsN3BWdvGU2OuY5ABMnEniC/uNdtLu5t7urLP25EREREg8JKbA3YjP/kiFfa4t5v3n8E7/zb23DHEydLeb37D80BAGaKCv8j89i1daxlca7vOmiE+Z17Wzv+duQnM/5EREREg8PCfw3YqM/UaHkd/+mFuEA/udhaqC/HA4fjwn92KWh57OGj85mFvZbvSJLft4t7k6k+TnMDr3wMiIiIiIjWHgv/NZCN+pST8Z81O+nO18p5vQeOFHf8o0jx8LF5nJ+a4W95rpN085OOv13cK80NvgJO9SEiIiIaOBb+a6BZ+JcX9ZkznfmFemuHvl/1IMKjxxYAtHb8959cRD2IMgt7Ld+Vlg28bKffcQSOZDfwYsafiIiIaHBYia2BWiOECDBe9UqL+thFuPO1lRf+jxybRxgpfFcwk4sOPVww0cfyXSeJ+uQLfyDevTee6mMy/oz6EBEREQ0MC/81UAsiVD0HVc/JbOa1EnM26lNfedTH5vsvOnNTS8e/aIa/5bmSdPMLC39Xshl/Rn2IiIiIBmbFhb+IPCIid4jI90TkFnNsq4h8VUTuN79vWfmpDq+48HdR9R3Uw5KjPl06/icXGvixj96E7z8+3fY5tvC/+JzNmFlqZEaEPnRkHuMVFzsmqy1f5ztOM+qjrYW/68QXBo2w9TEiIiIiWltldfxfoqoXq+pe8/l7AVyvqnsAXG8+37CaHX/XZN5XXvzP9tjxf+DIHG7fdxJ/+i/3tX/O4TmctXkUp01V0Qg1E0d6+Og8ztsxDpHWot1zBY2gU9RHEEZR8pjv8gYTERER0aCsViX2egDXmI+vAXD5Kn2foVALQlR9BxUv/nGX0fXvdXHvycU6AOCmB47iu4+dKHzO/YfnsGfnBKZGfADI5PwfPjqP8wtGeQI2w58r/CXd8Y/vCNjnsONPREREZSppH9MNo4zCXwF8RURuFZGrzLGdqnoAAMzvp+W/SESuEpFbROSWI0eOlHAap64k6mMK/zJy/nM9jvO08/59V/AXX3+g5fEwUjx0ZA5P3jGByREPADBjLiqCMMK+Ews4d9tY4Wv7XvsNvADT8Q8Vt++LNxmbMq9PRBuHiHxCRA6LyJ2pY4yDEhENQBmF//P1/7R359FtVfeix79bo0d5SGzHSQyZAyGFAAECpWEqY1varlJKHxd4dIBXCpfbYbWw4PXS4d5e2l5oaZmHS999FNrHUCgQKEPSEIYkJEBIyOTESZzB8zzJGvb745wjS7YkW4Mtyf591vKyfHQk7XOOfPTTPr/921qfBFwMfFcptXIsD9JaP6S1Xq61Xl5RUZGGZmQvr28o1QdIS2UfK/Afrce/3Qz8rz59Dq9vb2Lb4ciZfg+19+P1B1lQWYQn3+zxNysGtff5CGqojJLfD+YEXsHoE3hZt3c3dfOH1bVcvHQGCyqLE91MIUTuexy4aNgySQcVQogMSDnw11ofNn83Ac8BpwKNSqlqAPN3U6qvk8u8/gBuR1iqTxoC/6FynvF7/Dv7BrEpuOncBRS7Hdy3Zk/E/bXN3QBG4G/2yFuVfVp7vQCUF0YP/B12Ferxtwb32sJSfZx2xeYDHeQ5bPz0i8cltH1CiMlBa70WaBu2WNJBhRAiA1IK/JVShUqpYus2cAGwFXgBuMZc7Rrg+VReJ9eNSPVJw+y93WPM8e/o91GS76S0wMXnT5jJW7uaI6r27G40KvoYgX9kjn9bjzE+YFqRK+pzO8Nm7g2V7LRH9vgD3P65JVQW5yW2gUKIyWzUdFCYWimhQggxEVLt8a8C1imlPgI2AC9prV8B/gM4Xym1Gzjf/HvK8vqDEYN7U031GfQHQ88xWlWf9j4fpQVG4H7MjGK6Bvw0d3tD9+9u6mF6kZvSAhfFZuBvfalo6TUD/8LYgf+IOv5hPf5VnjzOWlTBV5fPTngbhRBiKqWECiHEREhptKXWei9wQpTlrcB5qTz3ZOL1BXA73GE9/qkF/uGz9Y5Wx7+jb5ASM3d/QaVRnae2qYdKj9EDv7Ohm2NmGLn3nnxrcK/V4298QZhWFCPVx6ZCPf7RBvc+fu2p2BRRS4EKIaa0RqVUtdb6iKSDCiHExJHC6hNg0B/E5bCHDe5NLdXHGthb5HaM2uPf2e+jrGBY4N9spPcEgppdjd0sNgP/fKcdu02Fxg+09hrjA0rNLw7DOe02/GZPf7TBvS6HDYfU7hdCjCTpoEIIkQESlU0AawKvdA3utVJxKj3uMVT1GQyl+lQWuyl2O0Iz9e5v7cXrD4YCf6UUnjxH2ODeQcoLXdhi1N932FVoMjKrxz/WukKIqUkp9STwLrBYKXVQKfVNJB1UCCEyQgqrTwCrqk+6Un2sHv+q4jz2NveaVxSif4fr6POFUn2UUsyvLAoN6N3ZYFT0sVJ9AIrznKHBva09XqbFqOgDxgRevmE5/g4J/IUQYbTWX49xl6SDCiHEBJMe/wlg1PG3k+dMT+BvpeJUeYygPFavvz8QpHvAT1nB0ODchZVFoVSfHQ3dKAULK8MD/7Ae/x6jxz8Wl2Mox99K+bFJPr8QQgghRFaSwH8ChKr62I0c/1RTfUI9/iV5EX8P12n23JcWDOXoL6gsornbS2efj50N3cyZVki+yx6635PnHBrc2zsYs5QnGD3+VsAfDI7M8RdCCCGEENlDAv9xFgxqBgPmzL3O9NTxt3rkq8za+H0xBvh2xAj8wZi4a2djN4urImfTDe/xb+nxxizlCdYEXmYdfz2ynKcQQgghhMgeEviPs0EzMHY77Ljs6RncG+rxN0ty9sbo8e/oswL/oeDdCvy3HupiX2tvaGCvxZNv5PgP+oN0DfhjlvKEkRN4KSWDe4UQQgghspUE/uPM67MC//Ae/xQD/wE/NjU0o27MHv8+YwKu8HKcs8sKcDlsrNp6BK0jB/bCUI9/u/nYeDn+TruKmMBLevuFEEIIMZEk9EiMBP7jzErrMXL8zcDfl3qPf5HbQZHbKMo0eo//UOBvtynmVxSxvq4NYGSPf56Tbu/Q7L7Tx5Djr7U2An/p7RdCCCGmFG2m+orcIIH/OLN6990OOw67DbtNMRhILce/a8BHcZ6TAnNQ7ug5/pHB+4LKIrSGPKeNo6cVRtxXnGd8mdjf2gdAeZxynk67Eej7gxL4CyGEEFPRqq0NmW6CSIAE/uMs1ONv1tl3O2yp9/gP+CnOc1Bo9fjHKOfZ0WfMvFvsjpyuYUGFkee/sLJ4RLDuMdOC9rX2AsSv6mNewfAHNAEtgb8QQggx1TR0DmS6CSGvbD2S6SZkvZwP/K3BpdlqICzHH8DlsKVlAq8i91Dg3+eNleNvTN41fMCtNcB3eJoPgMfs8a9rMQP/eFV9zOcdDASlx18IIYSYgoJZlOrzv/7v5kw3IevldOB/12u7WPqvr2Z1flko1cdppOW4Hba0VPUpynOQbz6n1eN/qKOfM+98k9omY4Kujn5fxORdlkVVRuA/fGAvGDn+APtaenHYVOjvaKzZgn1W4C8jbIQQQogp5Rcvbc90E0QCcjrwL8l34vUHQxNVjYf39rbyk+e3hia1StTIVB97ynX8ewaMHn+7TZHvtIdy/Lcd6uRgez/v7mkBjFSfkoKRgfuCyiJ+89UTuPyUmhH3FecNpfqUFbrilue0Kv609HgJSqqPEEIIIURWc4y+Svaq8hgDTxu7vCMGsKbqk8Nd/HLVdt7abQTRZ8yfxkVLqxN+nqHBvUOpPoMppid1e/2hQbiFbnuoqk+TWYlnR0M3YKT6RKvKo5TispNnR31uT77xvC09g1GvCISrLskH4EjHAP6ABP5CCCGEENksp3v8rQmsGrvSO7Bk84F2vvrAO2w73MUNZ88HoLV3MKnnGqrjP5Tqk47BvVYpzwKXI9Tjb5Xg3GkF/v2DUVN94ikOS+2ZHmfyLoCZpcb+P9zZT0BrbJLqI4QQQqSsfzDA717fnfXjGEXuye3Av9gIPBvSGPhvPdTJNY9tYHqxm1U3f4abP7sQgPZkA/+wOv5gBv4p5Pj7AkH6fYFQgF7gGtnjv7OxG601Hb2+qKk+8VhXEiD+5F0AlcV52G2KIx0DBIMah10CfyGEECJVv39zN3e/vos/b6zPdFPEJJPTgX+lmerTlKbAv6FzgKseXY8nz8mfvr2CKk8eboedIrcj+R7/aKk+KQT+VpBv9fgXusN7/I390D3gp76tn26vP+Eef6fdFho0HK+UJxiTgc3w5HG4ox+/DO4VQggh0sL6XE+1GIgQw+V04J/ntFNa4KSxy5uW51tf10p7n497rzyJWaX5oeVlhc4UevyHp/qkNri3e8AM/POsVB97qKpPU7c3VLN/fV0rEDlr71hZvf7xSnlaqkvyONzZL4N7hRBCiDTL3pqFIlfldOAPRrpPunL869uM2WoXV0UOai0vdKeQ4x+Z6pNqHf8es8ffCvALXY7QVYDmbi+nz58GwPq6NsCofJQoaxKvaaPk+ANUl+ZzpFMG9wohhBDpYl1Az+Zy5eE27mvLdBPEGOV84F/pcdPYnZ4e//q2fqYXucl32SOWlxc4aUtTqk+qdfytwD/U4++20+sNEAxqmru9zK8sYmZJHu/tNXr8E031gaEe/9Fy/AFmluRxpHOAQFAG9wohhBDpoMitz9MjWTR7r4gv5wP/Kk9e2nL869v7qCnPH7G8vNCdcqqPyx5exz+FwH9gWI6/y0HfoJ/2vkH8QU1lsZvFM4o52N4PJJfqY03aFa0U6HDVJXkM+oM093hlcK8QQgiRRl3jOE9ROv1p/f5MN0GM0SQI/N00dXsJBlO/HFbf3kdNWcGI5dOKXLT2DiZ1yc3rD+B22FBmb3iqqT7WRGKhqj5uO72DgVBFn8riPBbP8ITWL81Ppcd/bKk+YKRJSY+/EEIIkTprYtJ73qwd0/pN3QP87aPD49mkuN7bK6k+uSLpwF8pVaOUWq2U2q6U2qaUutlcfodS6pBS6kPz55L0NXekKk8egaBOOgff4g8EOdwxELXHv6zAhdcfDI2yj+VQR/+ILwdeXzCU5gNWOc+xDe5t6x2kf9hrhnL884Z6/Af9QY50Gj38FcXuiIm3SgtTyfEfS6qPsb/a+3yS4y+EEEKkgT+YWAfhNY9t5KYnPwh9YRAillR6/P3AD7TWxwIrgO8qpZaY992ttV5m/ryccivjqCxOzyReVp561B5/M9c9Xp7/4Y5+zvrVah5+a2/Ecq8/iNs5NGZgrHX8B3wBLv7dWv795e0Ry4en+hSY4xH2tRgDk61UHzDKbVqDgBNR7cmjtMA5psdWm5N4Wa8nhBBCiMQ8vekgc255KdS5l+inqdX5l47sh1z32ieNmW5CVks68NdaH9FabzZvdwPbgVnpathYVVm1/LtTC/zr243AuaZ8ZOBfPobA/729rfiDmgf/sTeil95K9bFYg3tHSxt6asMBGru87Grsjlje4/Wj1FDAX2gG5/taewFjsPP8iiIcNkVJvjOUYpSIb31mHi/edOaYHjut0IXL3D6p4y+EEEIk7v41RkpPgxnAJ/rZnSPFfybEY+vqMt2ErJaWHH+l1BzgRGC9uehGpdQWpdRjSqmydLxGLFUeq8c/tco+B9uMf7ZoPf5lVuDfFzvw31DXhtOuaO0d5MkNB0LLvf5hqT5m7/9gnGm4vf4AD641rhwc6uiPuK97wE+R2xE6KVhfAOpaeilyOyhwOXA5bMyrKExqYC9AvsvO7Cj7IRqlFDNLjGMgg3uFEEKI5FkBfLKfptL/Bu+aVQ1FdCkH/kqpIuAZ4F+01l3A/cB8YBlwBPjPGI+7Tin1vlLq/ebm5qRfv6LY6PFPNdWnvr0Pm4pMXbGEUn164gf+Zy2q4LS55Ty4dk8oj9/I8R9K9bGq+8Qr6fns5kMc6RzghJpSGswUJEuP1x+RglPoMm7vb+2jsnhoMO5XTprNhcfNiLvN6VJt5vnL4F4hhBAicVZnnnTci/GWUuCvlHJiBP1PaK2fBdBaN2qtA1rrIPAwcGq0x2qtH9JaL9daL6+oqEi6DU67jelFrtQD/7Y+qkvycdpH7pKyUVJ9mru97G3p5ZQ55dx07kIau7w8vekgYKb6OMN7/G3m8uiBvz8Q5P41ezhhdgmXL5+NP6gjtq1nwB+q6ANDqT4H2/uYHhb4X3/WfH580TFj2vZUWV+WJMdfCCGESFxtUw8gE2GJ8ZdKVR8FPAps11rfFba8Omy1LwNbk2/e2FQW56Wc6lPf3h+1og+AJ8+B065ipvpY/6inzi3n0wumsaymlIfW7kVrPTLVxxE/8P/blsMcaOvjxnMXMssslRme7tPt9YUm7wIodBtXE4KaiB7/iWRV9pHAXwghhEje7kbjC0DiOf5yrSDcnzceGH2lKSqVHv9PA1cB5w4r3fkrpdTHSqktwDnA99LR0HiqPO6oPf4Pr93L+jHmetW3Ra/hD8Y/YFmBK2aqz4a6NvKddpbOKkEpxeXLa9jf2see5h4z8A9L9XHETvUJBjV/eLOWY2YUc94xlcwuMwP/9qHAv8fM8bcUuIZuWxWOJlqox19SfYQQQoikWQF8oh+nVtifyRl/0zWZajr8+JmPM92ErJVKVZ91WmultT4+vHSn1voqrfWnzOWXaq2PpLPB0VR5Rvb493r9/HLVdv7r7X2jPn7AZ0yAFa2ij6W80BVzroD1dW2cfHRZKE3oMwunA7B2Vwte3/CqPsaXgGi1/F/Z1sCe5l5uPHcBNptiZpQe/0MdA6FxDTDU4w9GRZ9MkB5/IYQQInXJVuPsNkt9f3iwI42tScxbu1sy9tpi7HJ+5l4wAv/WXi++sEo5Ww52EtSw9XBnzMdZg2YPmj3qsVJ9wAj826Ok+nT2+djR0MWpc8tDy2rKC5g3vZC1u5sZHFbH3xrc6/VF9vhrbfT2z6so5OKlRrZUgctBeaEr1L7WHi8tPd6ICboie/wzE/hLjr8QQiRv9Y4m7nhhW6abIbKANvvuk/00PWCW9s6ExhTLqouJMWkCf62hpWeo1//DeuNb78H2fjr7Imey23a4k2/9cSPH/esrfFTfMVTDP04Jy/JCV9TBve/vb0NrIgJ/gJWLKnhvbytdA/5h5TzNVJ9h5Tzf3NHEJ0e6+O7ZCyIC6Fml+Rw2e/x3Nhg1/RdHBP5DXyoqMhX4S4+/EEIk7drHN/L4O/sy3QyRBULlPHOwnucjb0n9/FwwSQJ/q6RneODfHrq9LazX/99f3s7n7lnH+ro2ClwOfvzMFuqajW/Io6X6RAv819e14bLbWFZTGrH8MwunM+AL0tLjjZ7qM6zH/97VtdSU53PpspkRy2eV5odSfXZECfyddlto3ECmcvw9eQ4KXXYp5ymEEGJcHWzvG/PYvVxkzdybi+JNcpoJ4Z3BYsgkCfytSbyGLjN9WN8RyrXfdrgLgO4BH4+uq+PipTNY96NzufMrx7OjoZs/rK7F5bBRURS7x7y80EVnvy8inQjgje2NLJ9TRl5YOg/AinnTcJoTWkUd3BsYyvFv6Bxg84EOrjzt6BHlRGeV5XOovR+tNTsbuikvdI1oZ6HZ65+pVB+lFDd/duGILy1CCCEmxl/er5/UAbFl5a9W87WH3st0M8bN8x8eBpIfpOuLM0fQRJhzy0tc9ej60VecAG/uaMp0E7LSpAj8rUGt1ojyI539NHZ5OfeYSmaW5IXy/N/ZsUP2+AAAE+xJREFU00ogqLnmjDmUFDg5f0kVnzu+mrbeQWaX5WOLk6pSbtbyD8/zr23qZk9zLxctHTlRVqHbwfKjjfSfiDr+jpE5/mt3GxOYnbVo5HwGs0rz6fcFaO/zsaOxm8VVxSPKfBW4jHKjyc7Umw7XrZwftf1CCCHGpmvAN/pKMfzo6S2TOiC2JDv4NdeEf8x/7cF3oxYEiebBtXuSej2tNW/XtsQsC+r1B0Z0fMby1u4W5tzyElc/tiGptqTLz//2SUZfP1tNisB/WqEbl90WSoX58ICR37+sppQlM0vYesgI/NfuaqbQZeeko8pCj73jC8dRku9k3vTCuK8RCvx7h07Mqz5uAIg5Q+5nFhlXHEar4792VzOVxe6IQbuWWWZJz/q2PnY1dEek+VgK3XYqitwJ1/0VQgiRPfq8Ywvu4vmZBDsZM+ALsC4NlW0OdfRHBP7r69r46RiPa0dfcl8en9pYz5WPrOdvW6IXYlx8+ytcePfahJ5z7a7mjM4v0O310xFj/qWpbFIE/nab4vMnVPPM5oO09nj5sL4Dl93Gkpkels7ysLell75BP2/tbuH0+dNC6TZgDIh97oYz+PmXlsZ9DSvwb+0dyhl7ZVsDJx1VGko1Gm7lQqMHPF4d/0BQs662hc8srIgauFuTeL23t5V+XyDql4MCl4OKGG0QQgiRG6LN75Kox96u4+G1e9PQmuyWjRNW/eT5rfzTo+vZ1did0vM8s+kgw+v6/Gn9Aebc8tKoj401Oeho9rcaRU7q2/pirrO3JfGKQb6AJpjByzTLfvZaSlfSJqNJEfgD3HD2Arz+II+uq+OD+g6OnenB7bBz3MwStIZXtjZwoK2PlVHSUeZVFIUq08RiBf7W4JUDrX1sO9wVKr0ZzZJqD98/f1FEKtDwOv5bD3XS0edjpXl1YDhrEq83zFy1aD3+3zl7PjeesyBu+4UQYirq7POxO8VAbKKEj/1Kxb+9vJ05t7zEOb9Zw9u1LTyxfj+d/RMb/Nzzxm5+/8bucXv+sfaAp+qdPS08/vbYqtXsNQuFpDqo9K7XdsUszvPe3lYG/cFQOfJ0sb5IpbtIx6LbV3HFQ++xv7U3Y1/Wjr/j78y55SXm3PISG+raqGvppa6ll/beQQJBzaGOfi64+x/si/HFpqFzAP8Y05xygWP0VXLDgsoiLvlUNf/n3f0EgpqvnVIDwNJZHgAe+IeR92b1widqKNXHCPxf3RY/zQfAZlP883kLI5a5hqX6rN3VjFJw5oLogX9JvpMCl51N+40qRYuqRgb+8doghBBT2Tf+uJFN+9u5+byFzC7L5/wlVZTkG+Ohgjr5MsQtPV5K85047PH7z3q9fgpcdrSGZzYf5LbntvKvly5BoWjqHuD0edNC6372LiOV4vbPHcs7e1opyXey/UgX3zhzLvMrivAHgvT7AuQ77ZwW9rhY6lp6ufIRY6Dlbc9tjbvuOYsrWFZTxvVnzWMwEGTT/nbOWVw56mvEctdruwDjs/l7f/mQuy5fxrnHVHLM/36FX3xpKf+04uiknxvg8Xf2ccelx6X0HGPxPx429p8/qDlj/nTmVRSOKOZhaeo2Av6fvvAJr35v5bi054qwcRy///qJfOGEkUU1HltXxzfOnJvQ8wZDgX9q7Ytmw742zvr1mtDf9115EisXVZDvtDPgC+By2HDabQSDGqWMTAibUthsCl/A+JLjtNvSUjL88gffjXnf2b9ZE/M+i92mKCtwhb7cOWyKh69eTo/Xz8pFFRS7HXHHi2aDSRP4A9x4zgJeMvPTrPKaMzx5TCt0sauxh5ryfI6eFrtkZzxlBVaqjxH4r9p6hCXVHo5K8PmG5/i/tbuFpTNLmBajopBSilml+exu6uGo8gIK3ZPqkAkhxLiaO72QTfvb+d049j4nKjwI/y0j2/WLl7ZH/P2jp7eMe5tW72xm9c5m7n59V8KPPW6mh35fgJvPW8iqjxtoCpvI6TtPbAbghic283czGL77tV2hwH/T/jaau71cFOPquS8QpNfrZ9XWBi75VOQ6VurLDWfP5/qz5oe+0Fnaegfx5Dnw+oMpf3aGH5O6X14SNTX3gJkms7OxG38gGPdLYSCo2X6ki6WzSqLe39A5+mRYNz35ATc9+QHrfnxOxPKfvfgJP3vRuCJyzelHU17o5rxjK9l6qJNbnv2YX3xpKZcvr4lIe7YuIPxy1Q6+eebcmG3/y/v1XL68ZtS2xXOD+Z7IRYGgjrii4w9qrn18Y8rPe8qcMs5fUsV1K+en/FyjmVRR5LHVHj57bBWvb28MBf5KKZbM9PDW7hZWxsijHwun3YYnz0F77yAf1Xew+UAHPzh/UcLP43bYUMro6T9rUQWbD7Rz3cp5cR8zq8wI/KOl+QghhIjt15cdz9WnH82lf3g7002ZtKyS2Tc/9WHc9S4wB4e29g6OKV99uFuf/Tjq8vvW7OG+NYlVs7n9c8fS3jfIvav38Px3P80Jw+biiWfurS8DcP6SKi47eTb9gwHyXZFXARbctoqbz1vI92LECf/5953ct2YPN56zgB9euHjE/YmUojzzztUx7/vju/sBIr7Q3f7Xrdz+16Evn8fPLomowb/gtlUxn+9HT2+ZkC+iyfr1ZcfzpRNnAYTKo2ut2Xa4i+NmejjcOcCanU1UFLlx2m1sPdTJqXPLeWdPK4OBIPcn+D5Kl4372tm4r51phW6+cvLscX0tlQ0DZJYvX67ff//9tDxXfVsfr33SyLWfnhMK8u98ZQf3r9nDg1ednFJazDm/WUOR28GBtj6K8xw8e8MZSU2a9eA/9nD367sYMEt6PnXdClbEuWx723Mf88T6A/zzuQv4/gUjTxBCiMlNKbVJa7080+1IN6XURcDvADvwiNb6P2Ktm87PieE6+3109fsY8AW485UdvL49tfrfV552FHab4txjKjl+dikFLrvZ6ZO+FIBBfxCbItQzq7VO2/NbcUFXv5/eQT+HO/o50jnAA//YEwryJ7M7vrCEXU09zK8o4ucvpm8swfGzS9hysHP0FaeAknxnaNzJkmoPXn+AE2aX8uwHh5J+ztp/u3jU1LuJpLVmwBdEKWMC1sfW1fHOnta4Y0A++dmFFLgS75NP5DNiUvX4gzH77vDctguPm8Gm/e0x8+jHqqzAyeYDHczw5PHkt1ckPVPu9WfN58snzuLe1bXsbemNKC8ajVXSc/EMT1KvJ4QQ2UYpZQfuBc4HDgIblVIvaK0nvB5lSb4zlCbyyDWnTPTLJyU8TQNI65cK67lKCpyUFDiZaVaXi5ZPPhp/IMiRzgFe+OgwVZ48/vu9/XxU35G2to6HO8Zp4PBkDPrPWVzBQ1cvHzH5aLLu+tqytDxPNlBKha4ELasp5Z6vn5jhFhkmXeAfzbKaUv5y/ekpP89R5QUcaOvnT98+jZry5MYKWCo9efz0i/FLiFqOrfbgsCmOnx09F1AIIXLQqUCt1novgFLqKeCLgBSin0Qcdhs15QV816w8d9k4pzGAMRGay26LOgi3fzCA3ab46oPv8lF9B3ddfgJ2m+LJDQd4b2/buLctGY9fewotPYPMKs3npKNL+eRwF1++753Q/d86cy6PrDMqD12/ch4rF1Vw35pa3q5t5ajyAmZ48tiwr41Zpfn84IJFPLnhADsauuke8CfcljU/PJua8gI6+32hoicit0y6VJ/x1D8YYDAQHDGAaLxprWntHWR6jAHAQojJbTKm+iilLgMu0lp/y/z7KuA0rfWNYetcB1wHcNRRR528f//+jLRViK4BHxvr2lgxb1qoSlP3gB9PvkMmzxQZl8hnRPYkQ+WAfJd9woN+MC4XSdAvhJhkokVLET1RWuuHtNbLtdbLKyqSK8UsRDp48pycd2wVhW4j0LfZFCUFTgn6Rc6RwF8IIUQmHATC6wLOBg5nqC1CCDElSOAvhBAiEzYCC5VSc5VSLuAK4IUMt0kIISa1KTG4VwghRHbRWvuVUjcCr2KU83xMa70tw80SQohJTQJ/IYQQGaG1fhl4OdPtEEKIqUJSfYQQQgghhJgCJPAXQgghhBBiCpDAXwghhBBCiClAAn8hhBBCCCGmAAn8hRBCCCGEmAIk8BdCCCGEEGIKUFrr0dca70Yo1QzsT/Lh04GWNDYnkybLtkyW7QDZlmw0WbYDxr4tR2utK8a7Mdlskn9OZHP7srltIO1LRTa3DbK7fdnWtjF/RmRF4J8KpdT7WuvlmW5HOkyWbZks2wGyLdlosmwHTK5tyWbZvp+zuX3Z3DaQ9qUim9sG2d2+bG7baCTVRwghhBBCiClAAn8hhBBCCCGmgMkQ+D+U6Qak0WTZlsmyHSDbko0my3bA5NqWbJbt+zmb25fNbQNpXyqyuW2Q3e3L5rbFlfM5/kIIIYQQQojRTYYefyGEEEIIIcQocjrwV0pdpJTaqZSqVUrdkun2jJVSqkYptVoptV0ptU0pdbO5vFwp9ZpSarf5uyzTbR0rpZRdKfWBUupF8++5Sqn15rb8WSnlynQbR6OUKlVKPa2U2mEem9Nz9Zgopb5nvre2KqWeVErl5coxUUo9ppRqUkptDVsW9Tgowz3mOWCLUuqkzLU8Uozt+LX5/tqilHpOKVUadt+t5nbsVEpdmJlWTz6Z+JxI9Bwf732slLrGXH+3UuqaNLZxTOdspZTb/LvWvH9O2HOMy3s2kXNxhvbdmM+vE7H/0nXOjLW/lFInK6U+Nh9zj1JKpdi2hM+Dsf6PY+33VNoXdt8PlVJaKTXd/HtC99240Vrn5A9gB/YA8wAX8BGwJNPtGmPbq4GTzNvFwC5gCfAr4BZz+S3AnZluawLb9H3gT8CL5t9/Aa4wbz8AfCfTbRzDNvwR+JZ52wWU5uIxAWYBdUB+2LH4n7lyTICVwEnA1rBlUY8DcAmwClDACmB9pts/ynZcADjM23eGbccS8xzmBuaa5zZ7prch138y9TmR6Dk+1vsYKAf2mr/LzNtlaWrjmM7ZwA3AA+btK4A/m7fH7T2byLl4ovddoufXidh/Mc41adtfwAbgdPMxq4CLU2xbQudB4vwfx9rvqbTPXF4DvIoxd8j0TOy78frJ6Iun1HBjR74a9vetwK2ZbleS2/I8cD6wE6g2l1UDOzPdtjG2fzbwBnAu8KL5Bm8J+8eOOFbZ+AN4ME7matjynDsmGB9M9eZJyGEekwtz6ZgAc4Z9UEQ9DsCDwNejrZcNP8O3Y9h9XwaeMG9HnL/MD5zTM93+XP/Jls+J0c7xsd7HwNeBB8OWR6yXQnvGfM4Ofy+a55MWc/1xec8mei7OwL5L6Pw6Ufsv1XNmrP1l3rcjbHnEesm0bdh9o54HifF/HO99m2r7gKeBE4B9DAX+E77vxuMnl1N9rH8+y0FzWU4xL/udCKwHqrTWRwDM35WZa1lCfgv8CAiaf08DOrTWfvPvXDg284Bm4L+Ucfn7EaVUITl4TLTWh4DfAAeAI0AnsIncOybhYh2HXD4PfAOjBwhyezuyWcb36xjP8bHaOV7tT+ScHWqDeX+nuf54tS3Rc/GE7rskzq8Tvf8s6dpfs8zb49XOsZwHYy0fl1hDKXUpcEhr/dGwu7Jt3yUllwP/aHlSesJbkQKlVBHwDPAvWuuuTLcnGUqpzwNNWutN4YujrJrtx8aBcbnvfq31iUAvxuXRnGPmcn4R41LpTKAQuDjKqtl+TMYiF99rKKVuA/zAE9aiKKtl/XbkgIzu1wTO8bHamfb2J3HOnrC2mRI9F09o+5I4v070/htNou0Zt3YmcB6cyP+PAuA24CfR7k6wHVl5Xs/lwP8gRg6WZTZwOENtSZhSyonxgfCE1vpZc3GjUqravL8aaMpU+xLwaeBSpdQ+4CmMS8e/BUqVUg5znVw4NgeBg1rr9ebfT2N8+OTiMfksUKe1btZa+4BngTPIvWMSLtZxyLnzgDnw6/PAldq8/ksObkeOyNh+TfAcH6ud49H+RM/ZoTaY95cAbePUNuv1EjkXT+S+g8TPrxO9/yzp2l8HzdtpbWeC58FYy1tI/+fafIwvdR+Z/yOzgc1KqRlJtG9c9l2qcjnw3wgsNEd0uzAGzbyQ4TaNiTmq+1Fgu9b6rrC7XgCuMW9fg5EXmtW01rdqrWdrredgHIM3tdZXAquBy8zVsn5btNYNQL1SarG56DzgE3LwmGBcgl6hlCow32vWtuTUMRkm1nF4AbjarLawAui0Lm9nI6XURcCPgUu11n1hd70AXKGMCiBzgYUYg8JEajLyOZHEOT7W+/hV4AKlVJnZ03yBuSxpSZyzw9t8mbm+Zpzes0mciyds35kSPb9O6P4Lk5b9Zd7XrZRaYW7v1aT42ZHEeTDq/7G5H9P6uaa1/lhrXam1nmP+jxzEGKjfQBbsu7TI9CCDVH4wRljvwhjtfVum25NAu8/EuNyzBfjQ/LkEI1/tDWC3+bs8021NcLvOZqhCxDyMf9ha4P8B7ky3bwztXwa8bx6Xv2KMzs/JYwL8FNgBbAX+G6NKQk4cE+BJjNxZH8ZJ95uxjgPGpdR7zXPAx8DyTLd/lO2oxcgFtf7vHwhb/zZzO3aSBZUfJstPJj4nEj3Hx3sfY+RA15o/16a5naOes4E88+9a8/55YY8fl/dsIufiTOy7RM6vE7H/0nXOjLW/gOXmtu4B/sCwgddJtC3h8yAx/o9j7fdU2jfs/n0MDe6d0H03Xj8yc68QQgghhBBTQC6n+gghhBBCCCHGSAJ/IYQQQgghpgAJ/IUQQgghhJgCJPAXQgghhBBiCpDAXwghhBBCiClAAn8hhBBCCCGmAAn8hRBCCCGEmAIk8BdCCCGEEGIK+P/9Gp5/vFBroAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "memory_buffer = RingBuffer(1000)\n",
    "learn_counter = 0\n",
    "gamma = 1.0\n",
    "frame = 0\n",
    "all_rewards = []\n",
    "all_losses = []\n",
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "for episode in range(120):\n",
    "    observation = env.reset()\n",
    "    env.render()\n",
    "    lifespan = []\n",
    "    counter = 0\n",
    "    \n",
    "    while True:\n",
    "        frame += 1\n",
    "        counter += 1\n",
    "        env.render()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Epsilon greedy approach to decide whether to follow our policy or take random action\n",
    "        epsilon = epsilon_by_frame(frame)\n",
    "        if random.random() > epsilon:\n",
    "            observation_tensor = torch.tensor([observation], dtype=torch.float)\n",
    "            output = dqn(observation_tensor)\n",
    "            action = torch.argmax(output, dim=1).item()\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "            \n",
    "        # Take and observe action in real world\n",
    "        next_observation, reward, done, info = env.step(action)\n",
    "        memory_buffer.add([observation, action, reward, next_observation, done])\n",
    "        \n",
    "        learn_counter += 1\n",
    "        if learn_counter >= 128:\n",
    "            samples = memory_buffer.get_random(1000, 128)\n",
    "            s_0 = []\n",
    "            actions = []\n",
    "            rewards = []\n",
    "            s_1 = []\n",
    "            done_ar = []\n",
    "            for sample in samples:\n",
    "                s_0.append(sample[0])\n",
    "                actions.append(sample[1])\n",
    "                rewards.append(sample[2])\n",
    "                s_1.append(sample[3])\n",
    "                done_ar.append(sample[4])\n",
    "            \n",
    "            s_0_tensor = torch.tensor(s_0, dtype=torch.float)\n",
    "            s_0_output = dqn(s_0_tensor)\n",
    "            s_0_output = s_0_output[range(0, 128), actions]\n",
    "            \n",
    "            s_1_tensor = torch.tensor(s_1, dtype=torch.float)\n",
    "            s_1_output = dqn(s_1_tensor)\n",
    "            s_1_output = torch.max(s_1_output, dim=1).values\n",
    "            s_1_output = gamma * s_1_output\n",
    "            \n",
    "            rewards = torch.tensor(rewards, dtype=torch.float)\n",
    "            \n",
    "            done_ar = torch.tensor(done_ar, dtype=torch.float)\n",
    "            \n",
    "            error = rewards + (s_1_output * (1-done_ar)) - s_0_output\n",
    "            loss = torch.pow(error, 2)\n",
    "            loss = torch.mean(loss)\n",
    "            all_losses.append(loss.item())\n",
    "            \n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        \n",
    "        observation = next_observation.copy()\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "            \n",
    "        if frame % 200 == 0:\n",
    "            plot(frame, all_rewards, all_losses)\n",
    "            \n",
    "    lifespan.append(counter)\n",
    "    all_rewards.append(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}