DQN Cartpole v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "659ec131",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import random\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "from keras import models\n",
    "from keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f26a64b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# env = gym.make(\"CartPole-v0\")\n",
    "\n",
    "# def randomgames():\n",
    "    \n",
    "#     for i in range(1):\n",
    "#         state = env.reset()\n",
    "        \n",
    "#         for t in range(500):\n",
    "            \n",
    "#             env.render()\n",
    "#             action = env.action_space.sample()\n",
    "            \n",
    "#             new_state, reward, done, info = env.step(action)\n",
    "            \n",
    "            \n",
    "#             print(t, new_state, reward, done, info)\n",
    "            \n",
    "#             if done:\n",
    "#                 break\n",
    "#     env.close()\n",
    "                \n",
    "# randomgames()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b83ff628",
   "metadata": {},
   "outputs": [],
   "source": [
    "def NNmodel(input_shape, action_space):\n",
    "    \n",
    "    X_input = layers.Input(input_shape)\n",
    "    X = layers.Dense(512, input_shape = input_shape, activation = \"relu\", kernel_initializer = 'he_uniform')(X_input)\n",
    "    X = layers.Dense(256, activation = \"relu\", kernel_initializer = 'he_uniform')(X)\n",
    "    X = layers.Dense(64, activation = \"relu\", kernel_initializer = 'he_uniform')(X)\n",
    "    X_output = layers.Dense(action_space, activation = \"linear\", kernel_initializer = 'he_uniform')(X)\n",
    "    \n",
    "    model = models.Model(inputs = X_input, outputs = X_output, name='CartPole_DQN_model')   #Change\n",
    "    \n",
    "    model.compile(loss = \"mse\", optimizer = RMSprop(lr = 0.00025, rho = 0.95, epsilon = 0.01), metrics = [\"accuracy\"])\n",
    "    \n",
    "    model.summary()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "810349ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "    \n",
    "    def __init__(self):\n",
    "        \n",
    "        self.env = gym.make('CartPole-v1')\n",
    "        # by default, CartPole-v1 has max episode steps = 500\n",
    "        self.state_size = self.env.observation_space.shape[0]\n",
    "        self.action_size = self.env.action_space.n\n",
    "        self.EPISODES = 1000\n",
    "        self.testEPISODES = 100\n",
    "        self.memory = deque(maxlen=2000)\n",
    "        \n",
    "        self.gamma = 0.95    # discount rate\n",
    "        self.epsilon = 1.0  # exploration rate\n",
    "        self.epsilon_min = 0.001\n",
    "        self.epsilon_decay = 0.999\n",
    "        self.batch_size = 64\n",
    "        self.train_start = 1000\n",
    "        \n",
    "        # For stats\n",
    "        self.ep_rewards = []\n",
    "        self.aggr_ep_rewards = {'ep': [], 'avg': [], 'max': [], 'min': []}\n",
    "        self.STATS_EVERY = 50\n",
    "\n",
    "        \n",
    "\n",
    "        # create main model\n",
    "        #self.model = NNmodel(input_shape = self.state_size, action_space = self.action_size)    #Why doesn't this work?\n",
    "        self.model = NNmodel(input_shape=(self.state_size,), action_space = self.action_size)\n",
    "        \n",
    "    \n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "        if len(self.memory) > self.train_start:                \n",
    "            if self.epsilon > self.epsilon_min:\n",
    "                self.epsilon *= self.epsilon_decay\n",
    "\n",
    "    \n",
    "    def act(self, state):\n",
    "        if np.random.random() <= self.epsilon:\n",
    "            return random.randrange(self.action_size)\n",
    "        else:\n",
    "            return np.argmax(self.model.predict(state))\n",
    "        \n",
    "\n",
    "    def replay(self):\n",
    "        if len(self.memory) < self.train_start:\n",
    "            return\n",
    "        # Randomly sample minibatch from the memory\n",
    "        minibatch = random.sample(self.memory, min(len(self.memory), self.batch_size))\n",
    "\n",
    "        state = np.zeros((self.batch_size, self.state_size))\n",
    "        next_state = np.zeros((self.batch_size, self.state_size))\n",
    "        action, reward, done = [], [], []\n",
    "\n",
    "        # do this before prediction\n",
    "        # for speedup, this could be done on the tensor level\n",
    "        # but easier to understand using a loop\n",
    "        for i in range(self.batch_size):\n",
    "            state[i] = minibatch[i][0]\n",
    "            action.append(minibatch[i][1])\n",
    "            reward.append(minibatch[i][2])\n",
    "            next_state[i] = minibatch[i][3]\n",
    "            done.append(minibatch[i][4])\n",
    "\n",
    "        # do batch prediction to save speed\n",
    "        target = self.model.predict(state)\n",
    "        target_next = self.model.predict(next_state)\n",
    "\n",
    "        for i in range(self.batch_size):\n",
    "            # correction on the Q value for the action used\n",
    "            if done[i]:\n",
    "                target[i][action[i]] = reward[i]\n",
    "            else:\n",
    "                # Standard - DQN\n",
    "                # DQN chooses the max Q value among next actions\n",
    "                # selection and evaluation of action is on the target Q Network\n",
    "                # Q_max = max_a' Q_target(s', a')\n",
    "                target[i][action[i]] = reward[i] + self.gamma * (np.amax(target_next[i]))\n",
    "\n",
    "        # Train the Neural Network with batches\n",
    "        self.model.fit(state, target, batch_size=self.batch_size, verbose=0)\n",
    "        \n",
    "\n",
    "        \n",
    "\n",
    "    def load(self, name):\n",
    "        self.model = models.load_model(name)\n",
    "\n",
    "    def save(self, name):\n",
    "        self.model.save(name)\n",
    "            \n",
    "    def run(self):        \n",
    "        for e in range(self.EPISODES):\n",
    "            state = self.env.reset()\n",
    "            state = np.reshape(state, [1, self.state_size])\n",
    "            done = False\n",
    "            i = 0\n",
    "            \n",
    "            episode_reward = 0\n",
    "            \n",
    "            while not done:\n",
    "                #self.env.render()\n",
    "                action = self.act(state)\n",
    "                next_state, reward, done, _ = self.env.step(action)\n",
    "                next_state = np.reshape(next_state, [1, self.state_size])\n",
    "                \n",
    "                episode_reward += reward     #Without penalty for failure\n",
    "                \n",
    "                if not done or i == self.env._max_episode_steps-1:     #Hmmm messing with the environment rewards is allowed?\n",
    "                    reward = reward\n",
    "                else:\n",
    "                    reward = -100\n",
    "                    \n",
    "                #episode_reward += reward    #With penalty for failure\n",
    "                    \n",
    "                self.remember(state, action, reward, next_state, done)\n",
    "                state = next_state\n",
    "                i += 1\n",
    "                if done:                   \n",
    "                    print(\"episode: {}/{}, score: {}, e: {:.2}\".format(e, self.EPISODES, i, self.epsilon))\n",
    "                    if i == 500:\n",
    "                        print(\"Saving trained model as cartpole-dqn.h5\")\n",
    "                        self.save(\"cartpole-dqn.h5\")\n",
    "                        return\n",
    "                self.replay()\n",
    "                \n",
    "                \n",
    "            self.ep_rewards.append(episode_reward)\n",
    "            \n",
    "            if not e % self.STATS_EVERY:\n",
    "                average_reward = sum(self.ep_rewards[-self.STATS_EVERY:])/self.STATS_EVERY\n",
    "                self.aggr_ep_rewards['ep'].append(e)\n",
    "                self.aggr_ep_rewards['avg'].append(average_reward)\n",
    "                self.aggr_ep_rewards['max'].append(max(self.ep_rewards[-self.STATS_EVERY:]))\n",
    "                self.aggr_ep_rewards['min'].append(min(self.ep_rewards[-self.STATS_EVERY:]))\n",
    "                print(f'Episode: {e:>5d}, average reward: {average_reward:>4.1f}, current epsilon: {self.epsilon:>1.2f}')  \n",
    "                \n",
    "  \n",
    "    def test(self):\n",
    "        self.load(\"cartpole-dqn.h5\")\n",
    "        for e in range(self.testEPISODES):\n",
    "            state = self.env.reset()\n",
    "            state = np.reshape(state, [1, self.state_size])\n",
    "            done = False\n",
    "            i = 0\n",
    "            while not done:\n",
    "                #self.env.render()\n",
    "                action = np.argmax(self.model.predict(state))\n",
    "                next_state, reward, done, _ = self.env.step(action)\n",
    "                state = np.reshape(next_state, [1, self.state_size])\n",
    "                i += 1\n",
    "                if done:\n",
    "                    print(\"episode: {}/{}, score: {}\".format(e+1, self.testEPISODES, i))\n",
    "                    break\n",
    "                    \n",
    "    def performance(self):\n",
    "        plt.plot(self.aggr_ep_rewards['ep'], self.aggr_ep_rewards['avg'], label=\"average rewards\")\n",
    "        plt.plot(self.aggr_ep_rewards['ep'], self.aggr_ep_rewards['max'], label=\"max rewards\")\n",
    "        plt.plot(self.aggr_ep_rewards['ep'], self.aggr_ep_rewards['min'], label=\"min rewards\")\n",
    "        plt.legend(loc=4)\n",
    "        plt.show()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "446e983e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"CartPole_DQN_model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 4)]               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 512)               2560      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 64)                16448     \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 2)                 130       \n",
      "=================================================================\n",
      "Total params: 150,466\n",
      "Trainable params: 150,466\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "episode: 0/1000, score: 39, e: 1.0\n",
      "Episode:     0, average reward:  0.8, current epsilon: 1.00\n",
      "episode: 1/1000, score: 33, e: 1.0\n",
      "episode: 2/1000, score: 12, e: 1.0\n",
      "episode: 3/1000, score: 24, e: 1.0\n",
      "episode: 4/1000, score: 17, e: 1.0\n",
      "episode: 5/1000, score: 13, e: 1.0\n",
      "episode: 6/1000, score: 50, e: 1.0\n",
      "episode: 7/1000, score: 19, e: 1.0\n",
      "episode: 8/1000, score: 13, e: 1.0\n",
      "episode: 9/1000, score: 14, e: 1.0\n",
      "episode: 10/1000, score: 12, e: 1.0\n",
      "episode: 11/1000, score: 17, e: 1.0\n",
      "episode: 12/1000, score: 30, e: 1.0\n",
      "episode: 13/1000, score: 10, e: 1.0\n",
      "episode: 14/1000, score: 32, e: 1.0\n",
      "episode: 15/1000, score: 56, e: 1.0\n",
      "episode: 16/1000, score: 17, e: 1.0\n",
      "episode: 17/1000, score: 21, e: 1.0\n",
      "episode: 18/1000, score: 13, e: 1.0\n",
      "episode: 19/1000, score: 14, e: 1.0\n",
      "episode: 20/1000, score: 27, e: 1.0\n",
      "episode: 21/1000, score: 18, e: 1.0\n",
      "episode: 22/1000, score: 13, e: 1.0\n",
      "episode: 23/1000, score: 105, e: 1.0\n",
      "episode: 24/1000, score: 15, e: 1.0\n",
      "episode: 25/1000, score: 12, e: 1.0\n",
      "episode: 26/1000, score: 39, e: 1.0\n",
      "episode: 27/1000, score: 12, e: 1.0\n",
      "episode: 28/1000, score: 13, e: 1.0\n",
      "episode: 29/1000, score: 11, e: 1.0\n",
      "episode: 30/1000, score: 18, e: 1.0\n",
      "episode: 31/1000, score: 29, e: 1.0\n",
      "episode: 32/1000, score: 28, e: 1.0\n",
      "episode: 33/1000, score: 35, e: 1.0\n",
      "episode: 34/1000, score: 30, e: 1.0\n",
      "episode: 35/1000, score: 15, e: 1.0\n",
      "episode: 36/1000, score: 26, e: 1.0\n",
      "episode: 37/1000, score: 16, e: 1.0\n",
      "episode: 38/1000, score: 15, e: 1.0\n",
      "episode: 39/1000, score: 12, e: 1.0\n",
      "episode: 40/1000, score: 17, e: 1.0\n",
      "episode: 41/1000, score: 20, e: 1.0\n",
      "episode: 42/1000, score: 35, e: 0.98\n",
      "episode: 43/1000, score: 26, e: 0.96\n",
      "episode: 44/1000, score: 51, e: 0.91\n",
      "episode: 45/1000, score: 12, e: 0.9\n",
      "episode: 46/1000, score: 70, e: 0.84\n",
      "episode: 47/1000, score: 18, e: 0.82\n",
      "episode: 48/1000, score: 20, e: 0.81\n",
      "episode: 49/1000, score: 32, e: 0.78\n",
      "episode: 50/1000, score: 62, e: 0.73\n",
      "Episode:    50, average reward: 25.4, current epsilon: 0.73\n",
      "episode: 51/1000, score: 40, e: 0.71\n",
      "episode: 52/1000, score: 14, e: 0.7\n",
      "episode: 53/1000, score: 89, e: 0.64\n",
      "episode: 54/1000, score: 71, e: 0.59\n",
      "episode: 55/1000, score: 87, e: 0.54\n",
      "episode: 56/1000, score: 85, e: 0.5\n",
      "episode: 57/1000, score: 171, e: 0.42\n",
      "episode: 58/1000, score: 354, e: 0.3\n",
      "episode: 59/1000, score: 405, e: 0.2\n",
      "episode: 60/1000, score: 266, e: 0.15\n",
      "episode: 61/1000, score: 299, e: 0.11\n",
      "episode: 62/1000, score: 199, e: 0.092\n",
      "episode: 63/1000, score: 431, e: 0.06\n",
      "episode: 64/1000, score: 230, e: 0.047\n",
      "episode: 65/1000, score: 11, e: 0.047\n",
      "episode: 66/1000, score: 10, e: 0.046\n",
      "episode: 67/1000, score: 10, e: 0.046\n",
      "episode: 68/1000, score: 8, e: 0.046\n",
      "episode: 69/1000, score: 8, e: 0.045\n",
      "episode: 70/1000, score: 10, e: 0.045\n",
      "episode: 71/1000, score: 9, e: 0.044\n",
      "episode: 72/1000, score: 10, e: 0.044\n",
      "episode: 73/1000, score: 134, e: 0.038\n",
      "episode: 74/1000, score: 195, e: 0.032\n",
      "episode: 75/1000, score: 314, e: 0.023\n",
      "episode: 76/1000, score: 229, e: 0.018\n",
      "episode: 77/1000, score: 369, e: 0.013\n",
      "episode: 78/1000, score: 244, e: 0.0099\n",
      "episode: 79/1000, score: 261, e: 0.0076\n",
      "episode: 80/1000, score: 45, e: 0.0073\n",
      "episode: 81/1000, score: 10, e: 0.0072\n",
      "episode: 82/1000, score: 9, e: 0.0072\n",
      "episode: 83/1000, score: 9, e: 0.0071\n",
      "episode: 84/1000, score: 102, e: 0.0064\n",
      "episode: 85/1000, score: 158, e: 0.0055\n",
      "episode: 86/1000, score: 251, e: 0.0043\n",
      "episode: 87/1000, score: 258, e: 0.0033\n",
      "episode: 88/1000, score: 323, e: 0.0024\n",
      "episode: 89/1000, score: 148, e: 0.0021\n",
      "episode: 90/1000, score: 179, e: 0.0017\n",
      "episode: 91/1000, score: 9, e: 0.0017\n",
      "episode: 92/1000, score: 10, e: 0.0017\n",
      "episode: 93/1000, score: 10, e: 0.0017\n",
      "episode: 94/1000, score: 10, e: 0.0017\n",
      "episode: 95/1000, score: 174, e: 0.0014\n",
      "episode: 96/1000, score: 210, e: 0.0011\n",
      "episode: 97/1000, score: 393, e: 0.001\n",
      "episode: 98/1000, score: 220, e: 0.001\n",
      "episode: 99/1000, score: 262, e: 0.001\n",
      "episode: 100/1000, score: 224, e: 0.001\n",
      "Episode:   100, average reward: 151.5, current epsilon: 0.00\n",
      "episode: 101/1000, score: 320, e: 0.001\n",
      "episode: 102/1000, score: 331, e: 0.001\n",
      "episode: 103/1000, score: 207, e: 0.001\n",
      "episode: 104/1000, score: 10, e: 0.001\n",
      "episode: 105/1000, score: 9, e: 0.001\n",
      "episode: 106/1000, score: 9, e: 0.001\n",
      "episode: 107/1000, score: 10, e: 0.001\n",
      "episode: 108/1000, score: 9, e: 0.001\n",
      "episode: 109/1000, score: 11, e: 0.001\n",
      "episode: 110/1000, score: 74, e: 0.001\n",
      "episode: 111/1000, score: 130, e: 0.001\n",
      "episode: 112/1000, score: 147, e: 0.001\n",
      "episode: 113/1000, score: 293, e: 0.001\n",
      "episode: 114/1000, score: 10, e: 0.001\n",
      "episode: 115/1000, score: 10, e: 0.001\n",
      "episode: 116/1000, score: 8, e: 0.001\n",
      "episode: 117/1000, score: 9, e: 0.001\n",
      "episode: 118/1000, score: 10, e: 0.001\n",
      "episode: 119/1000, score: 9, e: 0.001\n",
      "episode: 120/1000, score: 10, e: 0.001\n",
      "episode: 121/1000, score: 9, e: 0.001\n",
      "episode: 122/1000, score: 14, e: 0.001\n",
      "episode: 123/1000, score: 205, e: 0.001\n",
      "episode: 124/1000, score: 243, e: 0.001\n",
      "episode: 125/1000, score: 311, e: 0.001\n",
      "episode: 126/1000, score: 432, e: 0.001\n",
      "episode: 127/1000, score: 500, e: 0.001\n",
      "Saving trained model as cartpole-dqn.h5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA12klEQVR4nO3deXhU1f3H8feZySSTjazsRAKI7FuIyF4WFRBlURAUFNSqdamA9WfRFquttNSita6VVkELikEEQbAiCKJWhUQUgYCAIIQlhGwkJJNkZs7vjzvZEwiQyU0m39fz8MzMvXdmvjfA556ce+65SmuNEEII32IxuwAhhBC1T8JdCCF8kIS7EEL4IAl3IYTwQRLuQgjhg/zMLgAgOjpax8bGml2GEEI0KElJSae11k2rWlcvwj02NpbExESzyxBCiAZFKfVzdeukW0YIIXyQhLsQQvggCXchhPBBEu5CCOGDJNyFEMIHSbgLIYQPknAXQggfJOEuhBBm2f46/LTFKx8t4S6EEGb4+lVY9zAkvemVj5dwF0KIuva/l+C/c6Hz9TDxNa98hYS7EELUpS+ehw2/g67jYfIS8PP3ytfUi7llhBCiUdi6ED79E3S/CSYuAqv3IljCXQgh6sKWv8KWP0OPm2HCq14NdpBwF0II79IaNv8Ztj4DvW6B8S+Dxer1r5VwF0IIb9Ha6Ib5/FnoMx1ueKFOgh0k3IUQwju0hk+egP+9AHEz4PrnwVJ3Y1gk3IUQorZpDR//Dr5+GeLvgusW1mmwg4S7EELULq3ho9/Ctteg370w5q+gVJ2XIeEuhBC1xe2G9Y9A4uvQ/wEYNd+UYIcLuIhJKWVVSu1QSn3oeR2plPpEKbXf8xhRZtvHlFIHlFL7lFKjvFG4EELUK243rJtjBPvAh0wNdriwK1RnAcllXs8FNmmtOwKbPK9RSnUFpgLdgNHAK0qpujk9LIQQZnC7Ye2vIWkJDH4YrvmjqcEONQx3pVQbYCzw7zKLxwPFM968CUwos3y51rpAa30IOAD0q5VqhRCivnG74IMHYMdSGPoojHzC9GCHmrfcnwceBdxlljXXWp8A8Dw28yxvDRwts12KZ1k5Sql7lFKJSqnEtLS0C61bCCHM53LCql/B92/DsMdhxO/qRbBDDcJdKXU9cEprnVTDz6xqz3SlBVov0lrHa63jmzZtWsOPFkKIesLlhFX3wA8JMOL3MOy3ZldUTk1GywwCximlrgPsQBOl1FIgVSnVUmt9QinVEjjl2T4FiCnz/jbA8dosWgghTOUqgpW/hD2r4eonYfAcsyuq5Lwtd631Y1rrNlrrWIwTpZ9qracDa4AZns1mAB94nq8BpiqlApRS7YCOwLZar1wIIczgLIQVM41gv/bpehnscGnj3BcACUqpu4AjwGQArfVupVQCsAdwAg9orV2XXKkQQpjNWWAE+771MHoB9L/P7IqqpbSu1B1e5+Lj43ViYqLZZQghRPWKHJBwG+zfYEwn0O9usytCKZWktY6vap1coSqEEOdTlA/vTocDG+H6v0P8nWZXdF4S7kIIcS6FebD8VvhpC4x7EeJuN7uiGpFwF0KI6hSehXemwqHPjZts9JlmdkU1JuEuhBBVKciFt6fAkf/BxNeg1xSzK7ogEu5CCFFRQQ4smwxHv4Eb/wU9Jpld0QWTcBdCiLIcZ2DpTXAsCW56HbrfaHZFF0XCXQghiuVnGcF+4juYvBi6jje7oosm4S6EEAD5mfCfiXByF9z8FnQea3ZFl0TCXQgh8jLgrfGQthemLIVOo82u6JJJuAshGrezp+GtCXD6R5iyDK641uyKaoWEuxCi8cpNg7fGQcZPcMs7cPlIsyuqNRLuQojGKSfVCPbMn+HWd6H9MLMrqlUS7kKIxufMCXjzBjhzDKatgHZDzK6o1km4CyEalzPHYcn1kHMSpq+EtgPNrsgrJNyFEI1HdooR7GdPw23vw2X9za7IayTchRCNQ9YRI9jzM+G2VRBzpdkVeZWEuxDC92UehiU3QEE23L4aWvc1uyKvk3AXQvi2jJ+MYC/Mhds/gFZ9zK6oTki4CyF8V/pBoyvG6YAZa6FlT7MrqjMS7kII33R6vxHs7iIj2Ft0N7uiOiXhLoTwPaf2GuPY0TDjQ2je1eyK6pyEuxDCt6TuMYLdYjWCvWknsysyhcXsAoQQotac/AHevB6sNpi5rtEGO0i4CyF8xYnvjRa7n90I9uiOZldkKgl3IUTDd+xbI9j9Q4xgj+pgdkWmkz53IUTDlpJk3EEpMMzoY49oa3ZF9YK03IUQDdfRbfCfCRAUYbTYJdhLSLgLIRqmn78yWuzB0Uawh19mdkX1ioS7EKLhOfwlLL0JQlsYwR7WxuyK6h0JdyFEw3JoKyybBGGtjWBv0srsiuolCXchRMNxcDMsu9nogpm5zmi5iypJuAshGoYDG+GdqRDZ3gj2kGZmV1SvSbgLIeq/HzfAO7cYFybNWGucRBXnJOEuhKjf9n0Ey2+FZl3g9jUQHGV2RQ2ChLsQov5KXgvv3gYtehg32giKNLuiBkPCXQhRP+1eDStmQqvexq3xAiPMraeBkXAXQtQ/u1bCe3ca9zqd/j7Yw8yuqME5b7grpexKqW1Kqe+VUruVUk95lkcqpT5RSu33PEaUec9jSqkDSql9SqlR3twBIYSP2bkCVv4SYq6C6SvB3sTsihqkmrTcC4ARWuteQG9gtFKqPzAX2KS17ghs8rxGKdUVmAp0A0YDryilrF6oXQjha757B1bdA20HwfT3ICDU7IoarPOGuzbkel7aPH80MB5407P8TWCC5/l4YLnWukBrfQg4APSrzaKFED5ox1JYfR/EDoFbE8A/2OyKGrQa9bkrpaxKqe+AU8AnWutvgOZa6xMAnsfiKwpaA0fLvD3Fs6ziZ96jlEpUSiWmpaVdwi4IIRq8pCXwwQPQYTjc+i74B5ldUYNXo3DXWru01r2BNkA/pdS5biOuqvqIKj5zkdY6Xmsd37Rp0xoVK4TwQdtfh7Wz4PJrYOo7YAs0uyKfcEGjZbTWWcAWjL70VKVUSwDP4ynPZilATJm3tQGOX2qhQggf9M0iWPcwXDEapi4Dm93sinxGTUbLNFVKhXueBwJXA3uBNcAMz2YzgA88z9cAU5VSAUqpdkBHYFst1y2EaOi+egU++j/oNBZu/g/4BZhdkU+pyW32WgJveka8WIAErfWHSqmvgASl1F3AEWAygNZ6t1IqAdgDOIEHtNYu75QvhGiQvnwBPpkHXcbBpDfAajO7Ip+jtK7UHV7n4uPjdWJiotllCCHqwufPwaanoNtEuPFfEuyXQCmVpLWOr2qd3CBbCFF3PvsbbH4auk+Cia+BVSLIW+QnK4TwPq1hywL4bAH0nAoTXgGLXNvoTRLuQgjv0ho2z4etf4Pe02DcixLsdUDCXQjhPVob/etf/B3ibofr/wEWma+wLki4CyG8Q2tjRMz/XoS+d8DY5yTY65CEuxCi9mkNHz8OX78CV94N1/0NVFUXrwtvkXAXQtQureGjR2HbIrjqPhj9Fwl2E0i4CyFqj9sN6x+BxNdhwINw7dMS7CaRcBdC1A63Gz6cBd++BYNmw9VPSrCbSMJdCHHp3C5Y82v4bhkMeQRG/F6C3WQS7kKIS+N2wer7Yedy+MVcGDZXgr0ekHAXQlw8lxNW/wp+WAHDfwe/eNTsioSHhLsQ4uK4iuD9u2H3Khj5BAz5jdkViTIk3IUQF85VBO/dCclr4Jo/wqBZZlckKpBwF0JcGGchvHcH7P0QRv0ZBjxgdkWiChLuQoiacxZAwgz48SMY8wxcda/ZFYlqSLgLIWqmyAEJt8H+DTD2Wbjyl2ZXJM5Bwl0IcX5F+bB8GhzcBDf8A/rONLsicR4S7kKIcyvMg+W3wE+fwbiXIO42sysSNSDhLoSoXuFZeHsKHP4CJrwKvW8xuyJRQxLuQoiqFeQYwX7kK7hxEfS82eyKxAWQcBdCVOY4A8smQ8p2uOnf0P0msysSF0jCXQhRniMblt4Ex3fApDeg2wSzKxIXQcJdCFEqPwv+MxFO7oTJS6DLDWZXJC6ShLsQwpCXYQR76m64+T/Q+TqzKxKXQMJdCGEE+1vjIG0fTF0GV4wyuyJxiSTchWjszp6Gt8bD6f0w9R3oeLXZFYlaIOEuRGOWewreHAeZh+DWd6HDcLMrErVEwl2IxionFd68AbKPwq0J0P4XZlckapGEuxCN0ZkTRrCfOQ7TVkDsYLMrErVMwl2Ixib7mBHsuakwfSW0HWB2RcILJNyFaEyyjsKb18PZdLhtFcT0M7si4SUS7kI0Fpk/G8Genw23r4Y28WZXJLxIwl2IxiDjkNEVU3DGCPbWcWZXJLxMwl0IX5d+0Aj2ojy4fQ206m12RaIOSLgL4ctOHzC6YpwFMGMttOhhdkWijki4C+Gr0n40gt3tgpkfQvNuZlck6pDlfBsopWKUUpuVUslKqd1KqVme5ZFKqU+UUvs9jxFl3vOYUuqAUmqfUkomqRCirp1KhiVjQWuYuU6CvRE6b7gDTuA3WusuQH/gAaVUV2AusElr3RHY5HmNZ91UoBswGnhFKWX1RvFCiCqk7oYl14OyGMHerLPZFQkTnDfctdYntNbfep7nAMlAa2A88KZnszeBCZ7n44HlWusCrfUh4AAgg2mFqAsnfzCC3eoPd6yHpleYXZEwSU1a7iWUUrFAH+AboLnW+gQYBwCgmWez1sDRMm9L8Syr+Fn3KKUSlVKJaWlpF1G6EKKc498Zo2JsQXDHOojqYHZFwkQ1DnelVAiwEpittT5zrk2rWKYrLdB6kdY6Xmsd37Rp05qWIYSoyrFvjfnY/UONYI9sb3ZFwmQ1CnellA0j2Jdprd/3LE5VSrX0rG8JnPIsTwFiyry9DXC8dsoVQlSSkghvTQB7uDEqJiLW5IJEfVCT0TIKeB1I1lo/V2bVGmCG5/kM4IMyy6cqpQKUUu2AjsC22itZCFHiyDdGsAdFGidPI9qaXZGoJ2oyzn0QcBvwg1LqO8+yx4EFQIJS6i7gCDAZQGu9WymVAOzBGGnzgNbaVduFC9Ho/fw/WDYZQpobFyiFVTq1JRqx84a71voLqu5HBxhZzXvmA/MvoS4hxLkc+hzevhmatDaCvUlLsysS9cwFjZYRQtQDP20xWuzhlxldMRLsogoS7kI0JAc2wdtTILIdzPgQQpubXZGopyTchWgo9m+Ed26BqI5GsIfIEGJRPQl3IRqCHz+G5bdA004wYw0ER5ldkajnJNyFqO/2rofl04zJv2asMYY9CnEeEu5C1GfJayHhNmjZE25bDYER532LECDhLkT9tXsVJMyAVnHGzawDw82uSDQgcrMOIeqjH96D9++BmH4wbQUEhJpdkWhgpOUuRH3z/bvw/t1wWX+Y9p4Eu7goEu5C1CffvQ2r7oW2gzwt9hCzKxINlIS7EPXFt2/B6vuh/TC4NQH8g82uSDRgEu5C1AeJi2HNr+HykXDLO+AfZHZFooGTcBfCbNv+BR/Oho6jYMoysAWaXZHwARLuQpjp63/C+keg03Uw5T9gs5tdkfAREu5CmOWrl+G/v4XO18PkN8EvwOyKhA+RcBfCDF/+Az5+HLqOh8lLwM/f7IqECXILnJzIzvfKZ8tFTELUtc+fhU1/hO43wcRFYJX/ho2J1ppthzJISExh/Q8nuLprc168pU+tf4/8qxKiLn32DGyeDz1uhgmvSrA3Iiey81mZlMKKpBR+Ts8jJMCP8b1bcfOVMV75PvmXJURd0Bq2/AU++yv0ugXGvwwWq9lVCS9zFLnYmJxKQmIKn+9PQ2vo3z6Sh0Z0ZEyPFgT5ey+CJdyF8Dat4dM/Gd0xfabDDS9IsPswrTW7j59hReJRVn93nOz8IlqF2fn18MuZ1DeGy6Lq5hoGCXchvElr2PgH4wRq35kw9u9gkXEMvijjbCGrdxxjRVIKySfO4O9nYVS3Ftwc34aBHaKxWlSd1iPhLoS3aA0bfg9fvQRX/hLG/E2C3cc4XW4+33+ahMSjbExOpcil6dkmjD+N78a4Xq0JC7KZVpuEuxDeoDX8dy5880+46lcwegGoum25Ce/5KS2XFUkprExK4VROAZHB/tw+IJbJ8W3o3KKJ2eUBEu5C1D63Gz76P9j+b+j/AIyaL8HuA3ILnKzbeZwViSkk/pyJ1aIYdkVTJsfHMKJzM/z96tdvZRLuQtQmtxvWPQxJi2HgQ3DNHyXYG7CKY9Lzi1x0aBrM3DGdubFPa5o1qb/TRUi4C1Fb3G5Y+xDs+A8MfhhGPiHB3kAdz8rn/W/Lj0mf0KcVk/rGEHdZOKoB/L1KuAtRG9wuY8re75bB0Edh+OMS7A2Mo8jFJ3tSWZFUfkz6rJEdGd3du2PSvaFhVStEfeR2wer7YOe7MOxxGPZbsysSNVQ8Jj0h8SgfmDgm3Rsk3IW4FC6ncVu8Xe/BiN/D0P8zuyJRA8Vj0hMSj7L3ZA7+fhZGd2vBZJPGpHuDhLsQF8tVBCt/CXtWw9VPwuA5ZlckzsHpcrN1fxorElPKj0mf0J1xPVuZOibdGyTchbgYzkJYeSckr4Vrn4aBvza7IlGNg2m5rEhM4f1vjTHpUfVwTLo3SLgLcaGcBbBiJuxbb1yc1P8+sysSFRSPSU9ITCHJMyZ9eKemTOpbP8eke4OEuxAXosgBCbfD/o/huoXQ726zKxIe1Y1Jf2xMZybW8zHp3iDhLkRNFTng3WlwYCNc/3eIv9PsigS+MSbdGyTchaiJonx45xb4aQuMexHibje7okatqjHpA9pHNdgx6d4gPwEhzqcwD96ZAoc+N26y0Wea2RU1SlWNSW8dHsivR3RkUlybBj0m3Rsk3IU4l4JceGcq/PwlTHwNek0xu6JGp7ox6TfHxzCwQxQWHxiT7g3nDXel1BvA9cAprXV3z7JI4F0gFjgM3Ky1zvSsewy4C3ABD2mtP/ZK5UJ4S04qnPgOjn8He9dC6m648V/QY5LZlTUaVY1J7+XDY9K9oSYt9yXAS8BbZZbNBTZprRcopeZ6Xv9WKdUVmAp0A1oBG5VSV2itXbVbthC1JOekEeLFYX7iO8g54VmpILojTF4CXcebVWGjUtWY9BkDYpkcH0OnFqFml9egnDfctdZblVKxFRaPB4Z5nr8JbAF+61m+XGtdABxSSh0A+gFf1VK9Qly8MyfKh/jx7yD3pGelgugroN1QaNkbWvWGFj0gQALF22RMundcbJ97c631CQCt9QmlVDPP8tbA12W2S/Esq0QpdQ9wD8Bll112kWUIUQWtjdZ3xRZ5bqqxXlmMIG8/zAjxlr09QR5iUsGNj9aabw5lsKKqMelxrWkW2rjGpHtDbZ9QrerMhq5qQ631ImARQHx8fJXbCHFeWsOZ45Vb5GdPGeuVBaI7QYcR5Vvk/sFmVdyoHc/KZ2VSCu99W35M+uT4GPrENN4x6d5wseGeqpRq6Wm1twQ8/5NIAWLKbNcGOH4pBQpRQms4c6xyi/xsmrFeWaBpZ7j86jIt8u4S5CYrHpOekHiULw6cLjcmfUz3lgT6W80u0SddbLivAWYACzyPH5RZ/rZS6jmME6odgW2XWqRohLSG7JTKLfK808Z6ZTWCvOO1pS3y5t3Bv/GOdS4qKiIlJQWHw2F2KQAUOt3kFTrJK3Rh13BnN38e7BNDUIAVP4sFyOHwwRyzy2wQ7HY7bdq0wWar+SihmgyFfAfj5Gm0UioF+ANGqCcope4CjgCTAbTWu5VSCcAewAk8ICNlxHlpDdlHK7fI89KN9coKzbrAFaNLW+TNuzXqIK9KSkoKoaGhxMbGmta94XS5ycwrIjOvEFXkIkQpWgfaiAiyERLgJ90uF0FrTXp6OikpKbRr167G76vJaJlbqlk1sprt5wPza1yBaFy0hqwj5UP8xPcVgrwrdBrjaZH3MYLcFmhezQ2Ew+EwJdi11uQ4nGTmFXLG4URrTZC/ldbhgYQF2vCzymiXS6GUIioqirS0tAt6n1yhKrxHa8j6uUKL/HvIzzDWW/yMFnmn6zwt8uIgl5ESF6sug91R5CIzr5CsvCKKXG78LBaigv2JDPbHbpN+9Np0MX+vEu6idmgNmYcrt8jzM431Fj+jRd7l+tI+8mYS5A2Ny63Jzi8k42wReYVOFIpQux+twgMJtfthkW6XekPCXVw4rSHzUOUWuSPLWG+xGS3yLuPK95H7BZhVsbgEWmvOFrrIPFtIdn4Rbq0J8LPSMsxOeJA/Nul2uShbtmxh4cKFfPjhh175fAl3cW5aQ8ZPlVvkjmxjvcUGzbsal+dLkPuUQqebzLxCMvMKKXS6sSpFeJCNiCB/gvytddYF5HK5sFq9282jtUZrjcXivQNVXexHWRLuopTb7WmR7ygT5juhwBPkVn+ja6XbxDJdK10lyOuhp9buZs/xMxf1Xqdb43S5cbmNawutFoWf1UKP1mE8Oa7bOd87YcIEjh49isPhYNasWdxzzz28+uqrHDp0iGeeeQaAJUuWkJSUxIsvvsjSpUt54YUXKCws5KqrruKVV17BarUSEhLCww8/zMcff8yzzz7Lp59+ytq1a8nPz2fgwIG89tprKKXYvn07d911F8HBwQwePJiPPvqIXbt24XK5mDt3Llu2bKGgoIAHHniAe++9t1ythw8fZsyYMQwfPpyvvvqK1atXk5CQQEJCAgUFBUycOJGnnnqKZ555BrvdzkMPPcScOXP4/vvv+fTTT9m0aROLFy9m6dKl3HfffWzfvp38/HwmTZrEU089BUBsbCx33nknGzZs4MEHHyQ8PJzZs2cTHR1NXFxcSS2fffYZs2bNAoz+9a1btxIaemlTX0i4N1Zud5kW+Q6jNV4xyJt3g+43lrbIm3UFP38Tixbe4tYap0vjdLvR2ggYm58Fm0WVtNBr0lB/4403iIyMJD8/nyuvvJKbbrqJSZMmMWDAgJJwf/fdd/nd735HcnIy7777Ll9++SU2m43777+fZcuWcfvtt3P27Fm6d+/OH//4RwC6du3KE088AcBtt93Ghx9+yA033MAdd9zBokWLGDhwIHPnzi2p4/XXXycsLIzt27dTUFDAoEGDuPbaaysNJdy3bx+LFy/mlVdeYcOGDezfv59t27ahtWbcuHFs3bqVoUOH8uyzz/LQQw+RmJhIQUEBRUVFfPHFFwwZMgSA+fPnExkZicvlYuTIkezcuZOePXsCxhj1L774AofDQceOHfn000+5/PLLmTKldProhQsX8vLLLzNo0CByc3Ox2y/9XJSEe2PgdkPGwfJ95Cd3QoGnZWcNMIK8x02lLfKmXSTIG7A/3HDuFjaUH5PuKHJhUYomlzgm/YUXXmDVqlUAHD16lP3799O/f3/at2/P119/TceOHdm3bx+DBg3i5ZdfJikpiSuvvBKA/Px8mjUzpqmyWq3cdNNNJZ+7efNmnnnmGfLy8sjIyKBbt24MGTKEnJwcBg4cCMCtt95a0n+9YcMGdu7cyXvvvQdAdnY2+/fvrxTubdu2pX///iXv2bBhA3369AEgNzeX/fv3c/vtt5OUlEROTg4BAQHExcWRmJjI559/zgsvvABAQkICixYtwul0cuLECfbs2VMS7sUhvnfvXtq1a0fHjh0BmD59OosWLQJg0KBBPPzww0ybNo0bb7yRNm3aXPDPviIJd19TNsiLu1dO7IRCz5WA1gDjkvwek8u0yLuAVebHbgzOOSY9yOa5cvTibNmyhY0bN/LVV18RFBTEsGHDSq6WnTJlCgkJCXTu3JmJEyeilEJrzYwZM/jLX/5S6bPsdntJ/7TD4eD+++8nMTGRmJgYnnzySRwOB1pXPyWV1poXX3yRUaNGnbPm4ODSqSm01jz22GOVum/A6F5ZvHgxAwcOpGfPnmzevJmDBw/SpUsXDh06xMKFC9m+fTsRERHMnDmz3FXCZb+jugPm3LlzGTt2LOvXr6d///5s3LiRzp07n7P285Fwb8jcbkg/UOFkZ5kg97Mbl+T3mlKmRd5ZgrwRqosx6dnZ2URERBAUFMTevXv5+uvSCWJvvPFG5s+fT9u2bfnrX/8KwMiRIxk/fjxz5syhWbNmZGRkkJOTQ9u2bcvX7gnK6OhocnNzee+995g0aRIRERGEhoby9ddf079/f5YvX17ynlGjRvHqq68yYsQIbDYbP/74I61bty4XtBWNGjWKefPmMW3aNEJCQjh27Bg2m41mzZoxdOhQFi5cyBtvvEGPHj14+OGH6du3L0opzpw5Q3BwMGFhYaSmpvLRRx8xbNiwSp/fuXNnDh06xMGDB+nQoQPvvPNOybqDBw/So0cPevTowVdffcXevXsl3BsNt8sI8opdK4W5xno/uzHbYa+ppS3ypp0kyBsxl9tNdn5RnY1JHz16NP/85z/p2bMnnTp1KunuAIiIiKBr167s2bOHfv36AUY/+tNPP821116L2+3GZrPx8ssvVwr38PBw7r77bnr06EFsbGxJNw4Yfet33303wcHBDBs2jLCwMAB++ctfcvjwYeLi4tBa07RpU1avXn3O+q+99lqSk5MZMGAAACEhISxdupRmzZoxZMgQ5s+fz4ABAwgODsZut5f0t/fq1Ys+ffrQrVs32rdvz6BBg6r8fLvdzqJFixg7dizR0dEMHjyYXbt2AfD888+zefNmrFYrXbt2ZcyYMRfwk6+aOtevNnUlPj5eJyYmml1G/eF2wen9lVvkRWeN9X6BRpAXh3ir3sa0tlY5VjdmycnJdO7cucox6ZHBNp8ck56bm0tIiDEP/4IFCzhx4gT/+Mc/TK7KO5KTk+nSpUu5ZUqpJK11fFXbSxqYze2C0z9WaJH/UBrktiAjyPtMLw3z6CskyEU5x7PyOeMoYl9qTrkx6ZFB/gTW4Zj0urZu3Tr+8pe/4HQ6adu2LUuWLDG7pHpDEqIuuZxGkJdtkZ/8AYryjPW2IGjRE+JuK9MivwIsMk+HqMxR5GLDnlRWeOZJX3RDS5paLTRvYifMbsNi8c1AL2vKlCnlhhSKUhLu3uJywul9lVvkznxjvS0YWvaEuBllWuQdJcjFOWmt2XXsDAmJR/ngu2OccThpHR7IQyM60iKsiPZN5VaBwiDhXhtcTkjbW6FFvqs0yP1DjBZ5/B2lLfKoyyXIRY2l5xaw+rvjrEg8yt6TOQT4WRjdvQU3x8cwoH0UFosiOTnZ7DJFPSLhfqFcRUaQl22Rp+4Cp2dcq38ItOwF8XeWtsijOkiQiwvmdLn57Mc0ViSmsGlvKkUuTa+YcJ6e0J0berUiLFBGQonqSbifi6sITiVXbpG7Coz1/qFG10r8XcZNJVr1hsgO4MXJh4TvO3AqlxVJR3n/22Ok5RQQFezPzIGxTI6P4YrmlzbfiGg8JNyLOQshLblCi3x3hSDvBf3uLu1akSAXtSTHUcS6nSdISDzKt0eysFoUwzs14+b4Ngzv3MznhjA2VE8++SQhISE88sgjZpdyXo0z3J2FcGpP+RZ56m5wFRrrA5qUBnmrPkaYR7aXIBe1yu3WfHMogxVJR/noh5PkF7m4vFkIj1/XmQl9WtMsVG5kci5OpxM/P+9FWF1MA+xNvh/uzgIjyMu2yE/tKRPkYUbXylX3lt6zM6KdBLnwmmNZ+axMSuG9pBSOZOQRGuDHhD6tuTm+Db1jwmtnTPpHc43RWbWpRQ8Ys6Da1YcPH2b06NEMHjyYr7/+ml69enHHHXfwhz/8gVOnTrFs2TL69evHtm3bmD17Nvn5+QQGBrJ48WI6derEc889x65du3jjjTf44YcfuOWWW9i2bRtBQaU3Ql+yZAnr1q3D4XBw9uxZ1q5dy69//Wt++OEHnE4nTz75JOPHj+e6665jwYIF9OzZkz59+jBx4kSeeOIJ5s2bR9u2bZk6dSrjx48nMzOToqIinn76acaPH1/lNMBLly7lrbfeIiYmhqZNm9K3b1/AmCTtn//8J35+fnTt2rXc9Af1gW+Fu7PAaIGXa5HvAXeRsd4eZrTIr/pV6clOCXJRByqOSdcaBnaIYs41HRndrSWB/r5xwv3AgQOsWLGCRYsWceWVV/L222/zxRdfsGbNGv785z+zevVqOnfuzNatW/Hz82Pjxo08/vjjrFy5ktmzZzNs2DBWrVrF/Pnzee2118oFe7GvvvqKnTt3EhkZyeOPP86IESN44403yMrKol+/flx99dUMHTqUzz//nNjYWPz8/Pjyyy8B+OKLL5g+fTp2u51Vq1bRpEkTTp8+Tf/+/Rk3bhxQfhrgpKQkli9fzo4dO3A6ncTFxZWE+4IFCzh06BABAQFkZWXV2c+4php0uGel72dD0ktEnUklMuNnok7/RGRRAcFao+xhRngPuL+0jzyiXc0mpRaiFpxrTPqkvm2IiawcXLXmHC1sb2rXrh09evQAoFu3bowcORKlFD169ODw4cOAMcHYjBkz2L9/P0opioqMxpfFYmHJkiX07NmTe++9t9o5Wq655hoiIyMBY5reNWvWsHDhQsCYZOzIkSMMGTKEF154gXbt2jF27Fg++eQT8vLyOHz4MJ06daKoqIjHH3+crVu3YrFYOHbsGKmpqUD5aYA///xzJk6cWHKQKT4AAPTs2ZNp06YxYcIEJkyYULs/yFrQoMP9cOoO/nTiU+NFANDamAs6wOJPZGAUkfYgoizpRGbvIKrgCJEnIokKjCLSHkmk3XgeHhCOn6VB/xhEPVPVmPQx3VswucyYdF8VEFB6Vy6LxVLy2mKx4HQ6AZg3bx7Dhw9n1apVHD58uNwMivv37yckJITjx49X+x0Vp+lduXIlnTp1KrdNYWEhiYmJtG/fnmuuuYbTp0/zr3/9q6TVvWzZMtLS0khKSsJmsxEbG1sy+2TFmSOr6yZbt24dW7duZc2aNfzpT39i9+7dXj0HcKHqTyUXoXvHG9gY3oEM/0DSCzLIcGSQnp9OhqP0eVpeGnvT95LhyMCpnZU+Q6GIsEeUBr49ishAz6O98sEg0C/QhD0V9V11Y9LnT+zO9T1lTHpZ2dnZtG7dGqDcXDDZ2dnMmjWLrVu38uCDD5ZM7Xsuo0aN4sUXX+TFF19EKcWOHTvo06cP/v7+xMTEkJCQwLx580hLS+ORRx4pGeWSnZ1Ns2bNsNlsbN68mZ9//rnKzx86dCgzZ85k7ty5OJ1O1q5dy7333ovb7ebo0aMMHz6cwYMH8/bbb5Obm0t4eHit/IxqQ4MOdz9bIM1b9aV5DbbVWnOm8AzpjnQy8jOMxyoOBrvTd5PuSOds8cRdFQT6BZYcAEoOBp7gr3hACAsIw6KkP9+XVRyTHh0iY9LP59FHH2XGjBk899xzjBgxomT5nDlzuP/++7niiit4/fXXGT58OEOHDi25O1NV5s2bx+zZs+nZsydaa2JjY0vuxjRkyBA2bdpEUFAQQ4YMISUlpWSa3mnTpnHDDTcQHx9P7969q507PS4ujilTptC7d2/atm1b8n6Xy8X06dPJzs5Ga82cOXPqVbCDTPlbLYfTQaYjs9xBIN2RXv5g4DlQZBZk4tbuSp9hVVYi7BEloV/VbwRR9qiS5/5Wua1dQ1Bfx6RXNSWs8B0y5W8tsfvZaRnSkpYhLc+7rcvtIrsw+5y/EWQ4MjiSc4T0/HQcLkeVnxNqC63yN4KquodCbaE+O41rfVTVmPSOMiZd1GMS7rXAarGWBO/lXH7e7fOK8iodBCr+RnAo+xCJqYlkFWRV+Rk2i63kO4t/Iyj7W0DZrqMIewQ2i/T5XgitNWcLXaSecbBu54lyY9InxrVmct9aHJMuhBdIuJsgyBZEkC2ImNCY827rdDvJKsiquluo+MDgSOdg1kHS89MpKh7TX0FYQFhp91AVvxEUHxwiAyMJ8gvyqdByutxk5ReRlVdIZl4RmWeNe4lmel4by8s+LyI7r4hCV2lX28AOUTx8zRWM6tbCZ8akC98m4V7P+Vn8iA6MJjow+rzbaq3JLcqt8jeCsr8p/Jj5I+kn0skpvpF2BXarvcquoKpGE4UHhGOtoxkvtdbkFZbe6LlcIJ8t8iwvH9KZeYXkOCqPkipmsyrCg/yJCDJuQ9cuOpi4IP+SZRHB/gxoH+XdMelCeIGEuw9RShHqH0qofyhtm7Q97/aFrsIqfwvIyC/tHkrNS2VP+h4yHZlVDiW1KAvhAeHVniSueDLZ7mf0TbvcusogzqoysEsfy7amKwoJ8CMi2EaEJ5xjo4M9z23lHkueB/sT7MO3oBONm4R7I+Zv9adFcAtaBLc477Zu7SanMKe0e6jMQeDU2dOknj1NWl46P2Ue5UxhFgXuvGo+KABcIbiKgnG7QtDOEHTxo+e5xR1CE/8Iwu1hRAYFcFlUEL1iwkpCu7iVXdyyDg+yER7oj7+fDDsVopiEuyjhcmvO5Ffsi66iT/qsi8y8ILLybGTmRVDgbFf5w1QhynqWoMA8goMcBAbmEeCfj9V2FgJycKkcCjmDw32cPOcZNOVb5IVAhvJD2yMgMAqLPRKrPRKbPYqAwEgC7VEU+kfi9IvCbYkEFQnIUNKGYM2aNezZs4e5c+eaXUqNbNmyhYULF5aMn28oJNx9lKPI6JvOPFt1t0fF7o7MvEKy84uo7rIHq0URHmgr6dpoExFEj9alLeeIci1q43lYkI0Av/P3x7vcLrIKssp1CxWfIyjbZXQ4+zDpjnQKiufYryDUP7TK6wiqOoEcYguR7hiTjBs3rtwcLReiLqbhdblcWK0N/6S5hHs953ZrzjiKyodzFX3RFUd+OIqq75sO8reW64NuHR5YPpyDy4d0eJA/oQF+XpsTxWqxGlf4BkbRkY7n3FZrTb4zv3z3kCOj0jUGB7MOst2xvdqhpP4W/0rXFFS8yKz4ub/VH6UUFiwopVAo47WylDxXlH9ttr9u+yt7M/bW6md2juzMb/v9ttr1NZ3yd8mSJSQmJvLSSy8xc+ZMmjRpQmJiIidPnuSZZ56pNOVAVdPwJiQkkJCQQEFBARMnTuSpp57imWeewW6389BDDzFnzhy+//57Pv30UzZt2sTixYtZunQp9913H9u3byc/P59Jkybx1FNPARAbG8udd97Jhg0bePDBBwkPD2f27NlER0cTFxdXUstnn33GrFmzAOMc19atWwkNrZ9XIku41yFHkatMEFcI57Plw7lsa9pdTWvaoiC8XEjb6daqSaUWdHFgFwd6TVrT9ZVSqnQoaZPzDyUtcheR5ciqPO1EmWGlp/NP82Pmj2Q4MqodSnqhzhX8Ja/Pta74dYWDStn3lVunFLNbz8Y/y+iayirIwuGs+mK5KqmKLysfoHIKcziac7TK9QrFydyTHDhwgJfefIknn3uS635xHf9+898k/DeBDes28MQfn2Dxu4vJKsgiryiPk2dPGgfqo+ms/Hgl+/ftZ8aUGQy5bki5zz+df5p9+/bx3KvP8dTCp9i8cTM/JP/A+s/Wo7Vm+qTprP1kLb2u6sUr/3iF6XdP5+ttX1NQUEBqTiqfbPmEvv37kunI5NF5jxIRGYHL5WL8mPFce8O1dO/R3bjC3A/Wb1qPw+Egrlscaz9eS4fLOzBz2kycbic5hTkseGYBf3v+bwwYOIDc3Fzcfu5yU5WUrbvkuaq8ruxri7Jgs9b+dSgS7hfB7dbkOJyVQrrq/unSbpH8Ile1nxlos5YL4pbhgcYJw7LD8iqM+Ai1e6817StsFhtNg5rSNKjpebfVWpNTlFO+WyjfCHy3dqMxugQ0umavq1qnNW5K12ldfv251p3zc9FYLdaSi9V+1etXlfat3Gt05ee68rqK6wucBdW+N8+ZR5u2bYjpGMNZ51naX9GevoP7klOUQ5sr2vDzzz+T5TCCvcBVQKYjk0JXIUNGDSGzMJPodtGcSj3F6bzT5b4/Iz+DVjGtuKz7ZZzKO8X6j9ezaeMmhvQz5nnJO5vHjt07GDdlHN9++y0HUg+AH3Tt1pWNX2zks62f8difH+N47nHefftd3nvrPZwuJ6dTT/O/Hf8jPDYcl3YxYMwAjuYcZe8Pe2kR0wL/5v4czTnKiAkjWPHWCo6cOULnuM785je/YexNY7n6+qtp0er8gxHOp0lAkxpd83KhvBbuSqnRwD8AK/BvrbU5E0yfR4GzTGu6mv7piv3U2flFuKppTisF4YGlQdwyzE6Xlk3Kjeyoalie3dZwW9O+QilFE/8mNPFvQmxYrNnlXLDk5GQua3KZad/vf8afkMAQOkUa0++GB4bTLqodnSM7Yz9jx6qtdInqwjch33DcfpwuUV0ICwijfXR7ukZ1BYzWbLfobkDpASkoJ4jw0HC6RBnzqkQGRPL7x37PPffeU/LdxQeZy9tfzpcffMmIoSPo0aMH+5P2c/LISUZfNZpDhw+x7J/L+N/X/yMiIoK77ryLcEs4l4dfjp/Fjy4tuxAdHk1uaC6BtkDah7cHYE/QHoJsQbQLa8f8J+Zzy8Rb+Pi/H3P7dbez9qO1dOrUqfoDYvHrcxxcvXX1uFfCXSllBV4GrgFSgO1KqTVa6z3e+D7wzProcFYO50qBXf4xr7D61rTdZinXcu7SoknlcK7QP93EbpPWtBC1oPjcRfFj8Qyro0ePZt68edx2222EhIRw7NgxbDYbzZo14xdDf8Hfn/07b7zxBj169ODRRx6lb9+++Pv54zjrICQ4hKaRTUlLS2PDfzcwcvhIAvwCUCjsfnbsfnZ6de/Fz4d/5vjPx+nQoQPvr3gfq7ISZAvi4MGD9IvrR7+4fiRtS+LIwSPE9Yirdh/M5K2Wez/ggNb6JwCl1HJgPFCr4b7n+BkeWr7DuJz8PK3pJnZbSbdHs9AAOjUPLTdWuqr+aWlNC1H/XHvttSQnJzNgwAAAQkJCWLp0Kc2aNWPIkCHMnz+fAQMGEBwcjN1uL5mmt1evXvTp04du3brRvn37au/0ZLfbWbRoEWPHjiU6OprBgweza9cuAJ5//nk2b96M1Wqla9eujBkzpm52+iJ4ZcpfpdQkYLTW+pee17cBV2mtHyyzzT3APQCXXXZZ3+omyz+Xoxl5/Hl9ctV90mVa1GGBNqzSmhY+Tqb89W31ZcrfqpK03FFEa70IWATGfO4X8yUxkUG8Or3vxbxVCCF8mreuBEgByp7+bQNUf1NEIYQQtcpb4b4d6KiUaqeU8gemAmu89F1CCI/6cGc1Ufsu5u/VK+GutXYCDwIfA8lAgtZ6tze+SwhhsNvtpKenS8D7GK016enp2O0Xdrcvr41z11qvB9Z76/OFEOW1adOGlJQU0tLSzC5F1DK73U6bNm0u6D1yhaoQPsJms9GuXRUzdIpGSSbAFkIIHyThLoQQPkjCXQghfJBXrlC94CKUSgMu/BLVUtHA6fNu5Tsa2/6C7HNjIft8Ydpqrauc8rRehPulUkolVncJri9qbPsLss+Nhexz7ZFuGSGE8EES7kII4YN8JdwXmV1AHWts+wuyz42F7HMt8Yk+dyGEEOX5SstdCCFEGRLuQgjhgxp0uCulRiul9imlDiil5ppdjzcopWKUUpuVUslKqd1KqVme5ZFKqU+UUvs9jxFm11qblFJWpdQOpdSHntc+vb8ASqlwpdR7Sqm9nr/vAb6830qpOZ5/07uUUu8opey+tr9KqTeUUqeUUrvKLKt2H5VSj3nybJ9SatSlfHeDDfcyN+EeA3QFblFKdTW3Kq9wAr/RWncB+gMPePZzLrBJa90R2OR57UtmYUwXXczX9xfgH8B/tdadgV4Y+++T+62Uag08BMRrrbsDVoz7Pvja/i4BRldYVuU+ev5fTwW6ed7ziifnLkqDDXfK3IRba10IFN+E26dorU9orb/1PM/B+A/fGmNf3/Rs9iYwwZQCvUAp1QYYC/y7zGKf3V8ApVQTYCjwOoDWulBrnYVv77cfEKiU8gOCMO7W5lP7q7XeCmRUWFzdPo4HlmutC7TWh4ADGDl3URpyuLcGjpZ5neJZ5rOUUrFAH+AboLnW+gQYBwCgmYml1bbngUcBd5llvry/AO2BNGCxpzvq30qpYHx0v7XWx4CFwBHgBJCttd6Aj+5vBdXtY61mWkMO9/PehNuXKKVCgJXAbK31GbPr8Ral1PXAKa11ktm11DE/IA54VWvdBzhLw++SqJann3k80A5oBQQrpaabW5XpajXTGnK4N5qbcCulbBjBvkxr/b5ncapSqqVnfUvglFn11bJBwDil1GGMrrYRSqml+O7+FksBUrTW33hev4cR9r6631cDh7TWaVrrIuB9YCC+u79lVbePtZppDTncG8VNuJVSCqMfNllr/VyZVWuAGZ7nM4AP6ro2b9BaP6a1bqO1jsX4O/1Uaz0dH93fYlrrk8BRpVQnz6KRwB58d7+PAP2VUkGef+MjMc4n+er+llXdPq4BpiqlApRS7YCOwLaL/hatdYP9A1wH/AgcBH5ndj1e2sfBGL+a7QS+8/y5DojCONO+3/MYaXatXtj3YcCHnueNYX97A4mev+vVQIQv7zfwFLAX2AX8Bwjwtf0F3sE4p1CE0TK/61z7CPzOk2f7gDGX8t0y/YAQQvightwtI4QQohoS7kII4YMk3IUQwgdJuAshhA+ScBdCCB8k4S6EED5Iwl0IIXzQ/wM1RLHUNwRsSQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "agent = DQNAgent()\n",
    "agent.run()\n",
    "agent.performance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e11297ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 0/100, score: 500\n",
      "episode: 1/100, score: 500\n",
      "episode: 2/100, score: 500\n",
      "episode: 3/100, score: 500\n",
      "episode: 4/100, score: 500\n",
      "episode: 5/100, score: 500\n",
      "episode: 6/100, score: 500\n",
      "episode: 7/100, score: 500\n",
      "episode: 8/100, score: 500\n",
      "episode: 9/100, score: 500\n",
      "episode: 10/100, score: 500\n",
      "episode: 11/100, score: 500\n",
      "episode: 12/100, score: 500\n",
      "episode: 13/100, score: 500\n",
      "episode: 14/100, score: 500\n",
      "episode: 15/100, score: 500\n",
      "episode: 16/100, score: 500\n",
      "episode: 17/100, score: 500\n",
      "episode: 18/100, score: 500\n",
      "episode: 19/100, score: 500\n",
      "episode: 20/100, score: 500\n",
      "episode: 21/100, score: 500\n",
      "episode: 22/100, score: 500\n",
      "episode: 23/100, score: 500\n",
      "episode: 24/100, score: 500\n",
      "episode: 25/100, score: 500\n",
      "episode: 26/100, score: 500\n",
      "episode: 27/100, score: 500\n",
      "episode: 28/100, score: 500\n",
      "episode: 29/100, score: 500\n",
      "episode: 30/100, score: 500\n",
      "episode: 31/100, score: 500\n",
      "episode: 32/100, score: 500\n",
      "episode: 33/100, score: 500\n",
      "episode: 34/100, score: 500\n",
      "episode: 35/100, score: 500\n",
      "episode: 36/100, score: 500\n",
      "episode: 37/100, score: 500\n",
      "episode: 38/100, score: 500\n",
      "episode: 39/100, score: 500\n",
      "episode: 40/100, score: 500\n",
      "episode: 41/100, score: 500\n",
      "episode: 42/100, score: 500\n",
      "episode: 43/100, score: 500\n",
      "episode: 44/100, score: 500\n",
      "episode: 45/100, score: 500\n",
      "episode: 46/100, score: 500\n",
      "episode: 47/100, score: 500\n",
      "episode: 48/100, score: 500\n",
      "episode: 49/100, score: 500\n",
      "episode: 50/100, score: 500\n",
      "episode: 51/100, score: 500\n",
      "episode: 52/100, score: 500\n",
      "episode: 53/100, score: 500\n",
      "episode: 54/100, score: 500\n",
      "episode: 55/100, score: 500\n",
      "episode: 56/100, score: 500\n",
      "episode: 57/100, score: 500\n",
      "episode: 58/100, score: 500\n",
      "episode: 59/100, score: 500\n",
      "episode: 60/100, score: 500\n",
      "episode: 61/100, score: 500\n",
      "episode: 62/100, score: 500\n",
      "episode: 63/100, score: 500\n",
      "episode: 64/100, score: 500\n",
      "episode: 65/100, score: 500\n",
      "episode: 66/100, score: 500\n",
      "episode: 67/100, score: 500\n",
      "episode: 68/100, score: 500\n",
      "episode: 69/100, score: 500\n",
      "episode: 70/100, score: 500\n",
      "episode: 71/100, score: 500\n",
      "episode: 72/100, score: 500\n",
      "episode: 73/100, score: 500\n",
      "episode: 74/100, score: 500\n",
      "episode: 75/100, score: 500\n",
      "episode: 76/100, score: 500\n",
      "episode: 77/100, score: 500\n",
      "episode: 78/100, score: 500\n",
      "episode: 79/100, score: 500\n",
      "episode: 80/100, score: 500\n",
      "episode: 81/100, score: 500\n",
      "episode: 82/100, score: 500\n",
      "episode: 83/100, score: 500\n",
      "episode: 84/100, score: 500\n",
      "episode: 85/100, score: 500\n",
      "episode: 86/100, score: 500\n",
      "episode: 87/100, score: 500\n",
      "episode: 88/100, score: 500\n",
      "episode: 89/100, score: 500\n",
      "episode: 90/100, score: 500\n",
      "episode: 91/100, score: 500\n",
      "episode: 92/100, score: 500\n",
      "episode: 93/100, score: 500\n",
      "episode: 94/100, score: 500\n",
      "episode: 95/100, score: 500\n",
      "episode: 96/100, score: 500\n",
      "episode: 97/100, score: 500\n",
      "episode: 98/100, score: 500\n",
      "episode: 99/100, score: 500\n"
     ]
    }
   ],
   "source": [
    "agent.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24907d71",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}