rl_pendulum_ddpg.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Open AI Gym Pendulum-v0 solution using DDPG (Deep Deterministic Policy Gradient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"./screenshots/ddpg.png\" alt=\"drawing\" width=\"1500\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import random\n",
    "import math\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Memory Buffer class for experience replay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MemoryBuffer():\n",
    "    def __init__(self, max_size):\n",
    "        self.buffer = deque(maxlen=max_size)\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        experience = (state, np.array([action]), np.array([reward]), next_state, done)\n",
    "        self.buffer.append(experience)\n",
    "        \n",
    "    def sample(self, batch_size):\n",
    "        state_batch = []\n",
    "        action_batch = []\n",
    "        reward_batch = []\n",
    "        next_state_batch = []\n",
    "        done_batch = []\n",
    "        \n",
    "        batch = random.sample(self.buffer, batch_size)\n",
    "        \n",
    "        for experience in batch:\n",
    "            state, action, reward, next_state, done = experience\n",
    "            state_batch.append(state)\n",
    "            action_batch.append(action)\n",
    "            reward_batch.append(reward)\n",
    "            next_state_batch.append(next_state)\n",
    "            done_batch.append(done)\n",
    "        \n",
    "        return state_batch, action_batch, reward_batch, next_state_batch, done_batch\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actor and Critic Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Actor(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(Actor, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)\n",
    "        self.fc2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.fc3 = nn.Linear(hidden_size, output_size)\n",
    "    \n",
    "    def forward(self, state):\n",
    "        output = F.relu(self.fc1(state))\n",
    "        output = F.relu(self.fc2(output))\n",
    "        output = torch.tanh(self.fc3(output))\n",
    "        return output * 2\n",
    "\n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(Critic, self).__init__()\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)\n",
    "        self.fc2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.fc3 = nn.Linear(hidden_size, output_size)\n",
    "    \n",
    "    def forward(self, state, action):\n",
    "        output = torch.cat([state, action], 1)\n",
    "        output = F.relu(self.fc1(output))\n",
    "        output = F.relu(self.fc2(output))\n",
    "        output = self.fc3(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "    def __init__(self, env, hidden_size=256, actor_lr=1e-4, critic_lr=1e-3, gamma=0.99, tau=1e-2, max_memory_size=50000):\n",
    "        self.num_states = env.observation_space.shape[0]\n",
    "        self.num_actions = env.action_space.shape[0]\n",
    "        self.gamma = gamma\n",
    "        self.tau = tau\n",
    "        \n",
    "        self.actor = Actor(self.num_states, hidden_size, self.num_actions)\n",
    "        self.actor_old = Actor(self.num_states, hidden_size, self.num_actions)\n",
    "        self.critic = Critic(self.num_states + self.num_actions, hidden_size, self.num_actions)\n",
    "        self.critic_old = Critic(self.num_states + self.num_actions, hidden_size, self.num_actions)\n",
    "        \n",
    "        for old_param, param in zip(self.actor_old.parameters(), self.actor.parameters()):\n",
    "            old_param.data.copy_(param.data)\n",
    "            \n",
    "        for old_param, param in zip(self.critic_old.parameters(), self.critic.parameters()):\n",
    "            old_param.data.copy_(param.data)\n",
    "            \n",
    "        self.memory_buffer = MemoryBuffer(max_memory_size)\n",
    "        self.critic_criterion = nn.MSELoss()\n",
    "        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)\n",
    "        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr)\n",
    "        \n",
    "    def get_action(self, state):\n",
    "        state = torch.from_numpy(state).type(torch.float).unsqueeze(0)\n",
    "        action = self.actor.forward(state)\n",
    "        action = action.detach().numpy().item()\n",
    "        return action\n",
    "    \n",
    "    def update(self, batch_size):\n",
    "        states, actions, rewards, next_states, done = self.memory_buffer.sample(batch_size)\n",
    "        states = torch.tensor(states, dtype=torch.float)\n",
    "        actions = torch.tensor(actions, dtype=torch.float)\n",
    "        rewards = torch.tensor(rewards, dtype=torch.float)\n",
    "        next_states = torch.tensor(next_states, dtype=torch.float)\n",
    "        done = torch.tensor(done, dtype=torch.float)\n",
    "        \n",
    "        qvals = self.critic.forward(states, actions)\n",
    "        next_actions = self.actor_old.forward(next_states)\n",
    "        next_qvals = self.critic_old.forward(next_states, next_actions.detach())\n",
    "        qprime = rewards + self.gamma * next_qvals\n",
    "        critic_loss = self.critic_criterion(qvals, qprime)\n",
    "        \n",
    "        policy_loss = -self.critic.forward(states, self.actor.forward(states)).mean()\n",
    "        \n",
    "        self.actor_optimizer.zero_grad()\n",
    "        policy_loss.backward()\n",
    "        self.actor_optimizer.step()\n",
    "        \n",
    "        self.critic_optimizer.zero_grad()\n",
    "        critic_loss.backward()\n",
    "        self.critic_optimizer.step()\n",
    "        \n",
    "        for old_param, param in zip(self.actor_old.parameters(), self.actor.parameters()):\n",
    "            old_param.data.copy_(param.data * self.tau + old_param.data * (1.0 - self.tau))\n",
    "            \n",
    "        for old_param, param in zip(self.critic_old.parameters(), self.critic.parameters()):\n",
    "            old_param.data.copy_(param.data * self.tau + old_param.data * (1.0 - self.tau))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epsilon_by_episode(episode):\n",
    "    epsilon_start = 0.1\n",
    "    epsilon_final = 0.00001\n",
    "    epsilon_decay = 9\n",
    "    #return epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)\n",
    "    return epsilon_start * math.exp(-1. * episode / epsilon_decay)\n",
    "\n",
    "def plot(frame, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20, 5))\n",
    "    plt.subplot(131)\n",
    "    plt.title(\"frame %s. reward: %s\" % (frame, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAE/CAYAAACkbK8cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYZGV58P/vXVW9r9PLrD09+w4MyLC4sAhEcUWJKEgEIwniGxPfX5JfAlETjZro+yaaGI2GKBHFNSCCAURQFFBAQYbZd2aYnunp7pneqrurq6u67vePc6q6uruqt6pTVd19f66rr+k659Sp51T1nLue+9lEVTHGGDO/+fJdAGOMMflnwcAYY4wFA2OMMRYMjDHGYMHAGGMMFgyMMcZgwWBOEZENIvKiiARF5M/yXR6TGRFZKSIqIoF8l8XMfRYM5pa/An6hqlWq+sV8F2YsEXmbiOwSkT4R+bWIbE7aVyIiXxCRkyLSJSL/LiJFY55/vYjsFZF+ETksIpck7btSRPaJyICIPCEiK3J5bYVIRM4SkUdF5LSIpBxQNMl7Wu5+DqdFpEdEnpzgte4RkVYR6RWRAyLyR0n7bnQ/8/jPgBvkznf3PzJm/5CI7Ex6/qdEZKeIREXkE2Ne92/GPDckIjERacjgrZufVNV+5sgP8DjwRxPs9+exbOuAXuB1QAC4AzgEBNz9fwc8BdQBjcCzwCeTnv97wDHgYpwvMcuAZe6+BqAHuA4oBf4v8OwMyxnI0/sz7nWBlYDOtEzABuAW4Brnv/q4/WnfU3f/PcD33M/DD5w/wWttAUrc3zcCp9IdD7wfOAxImv2/AP426fHNwJuAB4BPTHLNnwB+nq+/89n8k/cC2E+WPkj4OTAMDAJ9wHrgG8BXgIeBfuAq4C3Ai+6N+Xjyf66km88fuvu6gNuAC4AdQDfwpTGv+wFgr3vso8CKNOX7MPBQ0mMfEAKudB8/D1yXtP+9wPGkx78Gbklz7luBXyc9rnDPvXEK71v8mm8BXgGedLdf7L5mN/AScLm7/fXAzqTnPw78Junx08A73N9vd296QWAP8M6k494P/Ar4AtAJfNq94f4TcBo4AvwJGQSDpNdaS+pgMNF7usH9G6mewettAFqBd6fZ/wTwdxN8HsPAqhT77mGCYACI+37fnMv/e3Plx9JEc4SqXoHzzfrDqlqpqgfcXe8FPgNU4dyo+oGbgFqcwPAhEXnHmNNdhPNN/j3AvwAfxQkkW4B3i8hlAO7z/ga4Fufb41PAd9MUUdyfsY/PmmB/k4jUiIgf2AY0isghEWkRkS+JSJl77BacG3b8vejHuSlsSVOWVC4DNgFvFJFlwEM4N+g64C+B+0SkEXgGWCsiDW4u/yy3nFVuec533wfcMlwC1ACfBO4RkSVJr3kRzk1/Ic5n9MfAW4Hz3Ot9V3IBReR2EfmfaVxTWlN4Ty/CqTV80k0T7RSR35/knP8uIgPAPpxg8HCKY1YAlwLfTHOam4CnVPXlGVzWJcAi4L4ZPHfes2Aw9z2gqr9S1ZiqDqrqL1R1p/t4B87N+7Ixz/mUe+xPcYLHd1W1XVVP4NzoznOP+yDwj6q6V1WjwD8A56bJ1z8GXCYil4tIMU4QKQbK3f2PAB8RkUYRWQzEG8DLcf6DF+HcHC8BznXL8DH3mEqcNFGyHpwAOFWfUNV+VQ0BfwA8rKoPu+/TYzg1lzer6qD7+6U4N9MdOEH2tTi1iYOqegZAVf9bVU+65/g+cBC4MOk1T6rqv6lq1H3ddwP/oqrHVbUT+MfkAqrqZ1X1rdO4polM9p424QS6HmApTs3ubhHZlO6Eqvq/cN7zS4AfAuEUh012s78Jp0Y7EzcD96pq3wyfP69ZMJj7jic/EJGL3AbWDhHpwUkDjW1sa0v6PZTicaX7+wrgX0WkW0S6cdIdgpN7HkVV9+H8Z/0SzrfGBpzUSYt7yGdw0lfbcdIXPwIiQLv7mgD/pqqtqnoa+DzwZnd7H1A95iWrcdIzU5X8Pq0Arotfl3ttrwPi3+p/CVyOExB+iZPjvsz9+WX8JCJyk4hsTzrHWYx+r0d9Njg33eRtx6Za+DGNtI9M4SmTvachnPf/06o6pKq/xEnvvGGik6rqsKo+jRNMPpTikJuAu9Ncw+uAxcC9Uyj/2OeW4bQZpTy3mZwFg7lvbC+S7wAPAstVtQb4KqPTM9NxHPigqtYm/ZSp6q9TFkT1XlU9S1XrcRqMVwC/dfeFVPXDqrpMVVcDZ4AX3JtLF07QSDfF7m5ga/yBiFQAa9ztU5V87uPAt8ZcV4WqftbdPzYY/JIxwcCtHf0nzjfqelWtBXYx+r0eez2twPKkx81TLrzqt930YKWqvmkKx0/2nu6Y6munEcD5DBJE5LU4AS/dzf5m4Icz/GZ/Lc6XkV/M4LkGCwbzURXQqaqDInIhTpvCTH0VuENEtgC4+f3r0h0sIueLiN/Nvf8H8GO3xoCILBORpeK4GPg4TsCI+y/gT0VkoYgsAP43EM+f3w+cJSK/LyKlwN8CO+LnnoF7gLeJyBvd8pa66a0md/+vcRpJL8RpPN6NE9guAuLdLytwbrQd7vX9ISPtI+n8APgzEWlyr/H2GZYf9zXFfT+K3celIlKSdMhE7+mTOA3qd4hIwL2RX47TSWDs6ywUp4tqpft+vRG4AadTQ7KbgftUdVyNLemb/TdS7Ctyr8MHBNzr8Kc49zdV1ebkn6l8t2DbT/Z+cL4V/VHS42/gVPOTj3kXTvohiPMf/0vAPe6+lYzpvYLz7fHypMf3AB9Levw+YCcjvZPumqB8T7uv24kTDCqS9l0KHAUGgP3AjWOeWwT8O07vnlPAF4HSpP1X4TRchtz3YWXSvq8CX01TpnHX7G6/COdbfifODf0hoDlp/zPAE0mP7wX2jjnHZ9znx1Mwv4x/Pji9iZ4ec3wAp3fRGeBlxvQmwmlneWQafw/xa0v+OTqN93SLe539jO8NlSgLTueBX7rn6XX/Hv54TFlK3f1XpinrDTh/l+O6m+L8HY+9jvcn7V8GRIG1+f4/OJt/xH0zjTHGzGOWJjLGGGPBwBhjjAUDY4wxWDAwxhiDBQNjjDE4XdlmtYaGBl25cmW+i2GMMQXnhRdeOK2qjVM5dtYHg5UrV/L888/nuxjGGFNwRGTKU5pYmsgYY4wFA2OMMRYMjDHGYMHAGGMMFgyMMcZgwcAYYwwWDIwxxmDBwBhjDBYMjDHGMAdGIBtjzGx0pKOP517uTDyuKSvi7GU1NC0oQ2Smy5LPnAUDY8ys0z0wRHswDIAqnO4Lc/RMP8fODBAcjCSOqyotYlVDBasaKlhUXUr8FltXWUx1adG0XzcWU+761cv0hpzXCPh9vPeiZhoqSyZ55nh/c/9Onj3SOW57fUUxqxsrRgWE7/zRRQT83iZyLBgYY2YVVeXqf3mKU72D4/aVBHzUlI3c5LtDEYaisXHHLaou4dk7rpz2N/A9rb18+qG9AIg4gajI7+NDl6+Z9jXsPtHLu85v4i/fsAGA9uAgL7X0sON4N8e7BkYdn4uaggUDY8ys0tIV4lTvIO+7eAUXr64HYEGFUwNYVFWKzzdy4xyOKSe7Q7x8up8z/U5N4jcvd/Ld3xznRHeIpgXl03rtNjcA/ehPXsu5y2t57Wd/zt7W3hldQzAc5VXNC1hcUwrA4ppSzmmqhYtXTPt82WDBwBgzq+w+2QPA75/fxLnLayc81u8TlteVs7xu5KbfXFfOd39znH2twWkHg3hqqrHKSQttXFzFvlPTDwa7TzrP2by0etrP9Yr1JjLGzCq7T/bi9wkbF1fN6PkbFjs34JncxNt7nWDQUFkMwKYl1Rzu6GcwMjyt8+xt7cUnsGHRzK7BCxYMjDGzyq4TPaxtrKS0yD+j51eWBGiuK2dva3Daz+3oG6S2vIiSgPPaG5dUMRxTDrX3Tes8e1p7WdVQQVnxzK7BCxYMjDGzyu6TvWxZlll6ZePiKvbOsGawsGqk59CmJU45pttusOdkL5uX1kz79b1kwcAYM2u09w7SHgyzJcMb6cYl1Rw93U9oaHrpnY6+cKK9AGBlfQWlRb5p1TJ6BiKc6A6xeUnhtBeABQNjzCwSb3g9K8OG102Lq4gpHGyfXqrIqRmUJh77fcKGRdNrRN7TWniNx2DBwBgzi8R7EmV6I51JekdV6egbnSYC2Li4mr2tvajqlM6TCAZWMzDGmJnZdaKXlfXlVM1g9HCy5rpyyor800rv9IaiDEVjo9JEAJuWVNE1EEl0O53M3tZeGipLxp0n3ywYGGNmjd2tPWxZlnnDq88nbJjmGIGOPmfA2dib+Eb3G/6eKdYynMbjwqoVgAUDY8ws0TMQ4XhniC1ZupFuWlLFvlPBKad34mMMxtUM4uMWplDLGIrGONgeLLgUEVgwMMbMErtbnfaCs7LUJXPTkmq6ByIp5zhKJZ4GSm5ABqgpL2JpTemU2h8OtfcRGVarGRhjzEztPuHcbLNVM9g4jW/0AB3B1DUDcALLVFJOI43HhTPyOM6CgTFmVth1soclNaXUz2C66FQ2uNNZTHXwWXtwkJKAj+rS8VO6TXVaij0neykt8rGqoXL6BfaYZxPVicj/Bd4GDAGHgT9U1W4RWQnsBfa7hz6rqre5zzkf+AZQBjwMfESnmtAzxsw5e1t7OdM3BMCLr3RnPNgsWU1ZEctqy6ZVM1hYXZJyOunkaSnOchu4Q0PDPLD9BPc8d4yT3U4qqm8wyqYlVfh9uV+8ZjJezlr6GHCHqkZF5HPAHcBfu/sOq+q5KZ7zFeBW4FmcYHA18IiHZTTGFKjTfWHe8sWniCV9HXzPBcuz+hqbllSx62QPJ7tDAFSWBtIuetMeDNOYplYSH7fwuZ/sY3ldOUPRGI/vbaN7IMKmJdW8+ezFiLu0zhu3LM7qNWSLZ8FAVX+a9PBZ4F0THS8iS4BqVX3GffxN4B1YMDBmzugIhonGnMVmyosC1JSnHy9wpKOfmMLH37qZc5pq8Ilwdha6lSbbvLSGx/e285rP/hxwFsd55o4rqasoTln2NY2p0zsr6ys4f8UC9rYG2dsaRARevbqe979mJReuqsvLMpbTlav1DD4AfD/p8SoReRHoBT6mqk8By4CWpGNa3G3GmDngkZ2tfOjbv0s89vuEX/zl5aPWGkh29Ew/AFdtWsiK+gpPynTLa1exfEEZMVVOdIX44s8P8eIrXVy5adG4Y9uD4cRiOmP5fcJ9H3qNJ2XMlYyCgYg8DqSq83xUVR9wj/koEAW+7e5rBZpV9YzbRvAjEdkCpAqdKdsLRORWnHQSzc3NmVyCMSZHXjjWRUnAxyffvoX2YJjPP3aAXSd60gaDY2f6CfiEZbVlnpWppryI67Y5qaeBoShfeuIQLx3vHhcMwtFhekKRcVNRzCUZBQNVvWqi/SJyM/BW4Mp4Q7CqhoGw+/sLInIYWI9TE2hKenoTcDLN694J3Amwbds2a2A2ZhY40N7H2oWVXH9hMwNDUT7/2AEOd6RfB+DomQGaFpR5vhB8XHlxgPWLqnippWfcvom6lc4Vnr3LInI1ToPx21V1IGl7o4j43d9XA+uAI6raCgRF5GJxEmw3AQ94VT5jTG4dOBVMrOxVXhxgaU0phzv60x7/ypkBz9JD6WxtquWllu5xo5ITA86qLRjMxJeAKuAxEdkuIl91t18K7BCRl4B7gdtUtdPd9yHga8AhnO6o1nhszBzQE3JG+q5LWuZxzcLKtDUDVeXomX5W1k9vjeJMbV1eS/dAhFc6B0ZtT9QMKktTPW1O8LI30do02+8D7kuz73ngLK/KZIzJj4NtTl/+DYtHeuOsaazkv58/jqqO623TNRAhOBjNec3gnCant9JLLT2jXttqBsYYkwUH2pwawLqFSTWDxgr6h4Zp6x0/9XO8J9GKHNcMNiyuoiTg46Xj3aO2dwTDiEB9ii6nc4UFA2OM5w60Bako9o/qGRTvs58qVXQsEQxyWzMo8vs4a1kNO1rGBoNB6iuKc9aYnQ9z98qMMQXjQFuQtYuq8CVNw7BmoRMMjqQIBkdPDyACy+u861aazjlNNew80UN0OJbY1hEM01g1d9sLwIKBMSYHDrQF2bBo9OjdhVUlVJYEUvYoOnamn6U1ZZQE/LkqYsK5y2sZjMQSqS1wp6KYw91KwYKBMcZjZ/rCnO4bYv2i0dM2iwirGytSpomOnhlgZUNu2wvitjbVAoxKFXUEx699PNdYMDDGeCr+DXtsMACn3eBwe+o2g1y3F8StqC+npqyIl9xgEIupmyayYGCMMTN2wO1WmjoYVHCyZ5D+cDSxrWcgQtdAJOdjDOJEhHOaath+3BmJ3DUwRDSmVjMwxphMHGgLUl0aYFGKPvrxHkUvnx5pNzjWmZ+eRMm2NtVyoC3I159+ma8//TIwt6eiAAsGxhiPHWgLsn5RVcppnOM9ipLbDY6ecUb/rsxjMHjdugZiqnzqf/bw7784TMAnKWs2c0muprA2xsxDqsqBtj7ecs6SlPtX1JfjE0b1KDrm1hKa08xmmgsXr65n1yfeSHTYmaOoOOCjrDj3PZtyyYKBMcYz7cEwPaEI6xemXhSmJOBneV35qJrBsc4BFleX5v3mW1Eyv26P8+tqjTE5EXPXqtx/ym08Xpw+xTK2R5HTkyh/tYL5yoKBMSarvvXMUT7+wO5R2ybKt69prOBXh04zHFP8PuHomQFev6HR41KasSwYGGOy6smDp1lYVcKNF60AnCklGtIsJA9OzSAcjfHt545RURygIxjOa0+i+cqCgTEmqw60BblgZR0fuWrdlI4/2502+m+TahObl1Z7UjaTngUDY0zWDAxFeaVzgGvPa5r8YNeWpTU8e8eVDAw5A89KivyerntsUrNgYIzJmoNtfag66wJMx+KauT0j6Gxgg86MMVkT7z003WBg8s+CgTEma/a3BSkt8uV1wJiZGQsGxpisOdAWZN3CKvy+8VNPmMJmwcAYkzX7TgXn/Bw+c5UFA2NMVnT2D9ERDLPR2gtmJQsGxpismMrUE6ZwWTAwxmRFfBEbqxnMThYMjDFZsb8tSE1Z0ZxfEWyusmBgjMmK/aeCbEiziI0pfBYMjDEZU1UOnAraYLNZzLNgICKfEJETIrLd/Xlz0r47ROSQiOwXkTcmbb/a3XZIRG73qmzGmOw62TNIMBy1xuNZzOu5ib6gqv+UvEFENgPXA1uApcDjIrLe3f1l4PeAFuC3IvKgqu7xuIzGGOA/fnk4sTC9CNxwYTPnNNVO6bkH4tNQ2BiDWSsfE9VdA3xPVcPAyyJyCLjQ3XdIVY8AiMj33GMtGBjjsa7+If7xkX1UlQYoL/Zzpm+IwUiML7zn3Amfs/tkLwCP720DLBjMZl4Hgw+LyE3A88BfqGoXsAx4NumYFncbwPEx2y/yuHzGFIzjnQP8x5OHOdA2sgTkFRsXcttlazx/7YPuspNfvOE8Xr9hIX/wtec4crp/wuf81X07eGxPW+Jxc105NeVFnpbTeCejYCAijwOLU+z6KPAV4FOAuv/+M/ABIFVXAyV1+4Wmed1bgVsBmpubp11uY/KtrXeQ450DAESGlftfbOGHvzuBT4Rzm2vxCRw7M8DXnno5R8HASfOscxeuX9VQwY+2n0BV0/YO2nOyl0vXN/KnV6wFYPkCm5xuNssoGKjqVVM5TkT+E/gf92ELsDxpdxNw0v093faxr3sncCfAtm3bUgYMYwrV9uPd3HDns4Qiw4ltxQEff3DxCm67bE1ibv+vPXWETz+0lzN9YeonWDYyla7+IT7/2AHCUec1yor8/MUbN1Bdmvqb+8G2PiqKRxaVWd1YQXAwyum+IRpTjBsYGIpyojvEey5YzgUr66ZVNlOYPEsTicgSVW11H74T2OX+/iDwHRH5PE4D8jrgNzg1hnUisgo4gdPI/F6vymdMPrx8up8PfOO3NFQV8/fXnEXAnd1zw+IqFlaNXuAlPuHbgbY+Xj3NYPCT3af41rPHWFRdgiq0B8NsXV7Lta9KvQLZwfYgaxdWJmoBqxoqEuVNFQyOdDgppHhNwsx+XrYZ/B8RORcn1XMU+CCAqu4WkR/gNAxHgT9R1WEAEfkw8CjgB+5S1d2pTmzMbNQeHOSmu54D4JsfuChxw00n3mf/QFuQV6+pn9Zr7Wjpobo0wLN3XElM4exPPMqOlp70waCtj0vWNSYer2l0bvJHOvq4cNX4b/7xtNJaCwZzhmfBQFXfN8G+zwCfSbH9YeBhr8pkTC5Fh2Pcds8LnOweBJxv5/3hKN+79eJJAwHAwqoSasqKEnP+TMfOE92c01SLiOAXOGtZDduPd6c8tmcgQnswzLpFIzf2pbVlFAd8ia6mYx1q78PvE1bUT34dZnawEcjGeKS1Z5DH97bj8zk31/NX1HLX+y9g6/Kp9d0XEdYvqpx2MBiMDLP/VJBzmmoS285dXsuek70MRWPjjo9/y1+fFAz8PmFlfTmHO9IHgxX15RQH7BYyV+RjnIEx80J70KkR/P9v3Mhl6xsnOTq19Yuq+PFLJyfs1TPWvlNBIsM6KhhsbaplaDjGvlO94waSxbuVrls4eozA6obKRKAY61B7n7UXzDEW1o3xSFtvGCCjWTw3LK6idzCaONdU7Gxx0kFnJ930ty53AsNLKVJFB9v6KCsa6UkUt6qxglc6B4gOj65NDEVjHD0zYO0Fc4wFA2M80t7r1AwWVZdOcmR68R5F+6eRKtrR0kN9RTFLa0Zed1ltGQ2VxWw/3jPu+HhPIt+YdYtXN1QQGVaOd4VGbT92pp/hmFowmGMsGBjjkbZgmCK/sCCDUbmJ7qWnph4Mdp7o4eymmlFpJRFha1MtL7WkrhmkSvmsbox3L+0btf2Qm1Za22hTT8wlFgyM8Uh7b5jGypKM5vevqyimobJkyo3IoaFhDrQFOWdZzbh9W5fXcrijj+BgJLGtdzDCqd5B1i5KEQwa4t1LRzcix4PBmoXWk2gusWBgjEfag4MszCBFFLdh8dR7FO1p7XHGFaSYbfScphpUnZpD3MG21I3HAAsqillQXjRujqKD7X0sqy2jvNj6n8wlFgyM8Uh7bzgrS0CuX1TFgbY+YrHJZ17Z0eLc6JN7EsVtdQPES0ntBodSdCtNtqqhgiMd49NE1l4w91gwMMYjTs0g82CwYVEVocgwLWMaclPZ2dLDwqqSlI3WCyqKWVFfPqpH0cG2PkoCPprSTDK3urFyVJooFlOOnLZupXORBQNjPBCODtM1EGFRVeZponXT6FG040RPylpB3NhG5IPtfaxprMTvS92usaqhgvZgmL5wFIAT3SEGIzGrGcxBFgyM8UBH0B1jkIWaQTyFM1m7QV84yuGOPs5eln6E89bltbT2DCa6vR5sC6ZNEQGsifcocmsHNifR3GUtQMZ4IDHgLAsNyFWlRSyrLRsXDDqCYV441snhjn5UlbbeMKqp2wviznUHn138jz/DJ0I0pomaRyqr4xPWne7j7KaakW6lFgzmHAsGxnigw52KIhsNyODUDn62t513feXXAJzuC3P0zMC442rLizivOX3N4LzlC/jomzfRHRoCIODzcd221DOZgrN6mQj89mgnaxor+d2xbhoqS6gtL87wikyhsWBgjAfa42miLLQZANx40QqGkqaF2LC4ihsubGbbyjo2L6km4Hdy/n6RcSOJk/l8wh9funrKr1ta5GdVQwX3PPsK9zz7CgCXrGuY4VWYQmbBwBgPtPUO4vcJ9RXZ+QZ91eZFXLV5UVbONV133XxBYjI7gLNTDGgzs58FA2M8EB99PNG39NliZUMFK6ew/oKZ3aw3kTEeaAuGWZSFnkTG5IoFA2M80N47SGOW2guMyQULBsZ4oD0YzsoYA2NyxYKBMVk2FI3R2T+UldHHxuSKBQNjsux0X/ZGHxuTKxYMjMmytsQKZxYMzOxhwcCYLMv2gDNjcsGCgTFZFp8ELltTURiTCxYMjMmy9mAYn0B9pQUDM3tYMDAmy9p6B2moLEm7RoAxhcimozDTpqp84fGDo1bMeve25bzlnCV5LFXhaA+GU640Zkwh86xmICLfF5Ht7s9REdnubl8pIqGkfV9Nes75IrJTRA6JyBdFxL5aFaAfPH+cL/7sIKd6BukORfjdsS6+9ezRfBerYGRr7WNjcsmzmoGqvif+u4j8M9CTtPuwqp6b4mlfAW4FngUeBq4GHvGqjGb6jp3p55M/3sNr1tRzzy0X4fMJf/797Tz3cme+izYtA0NRjneG6AlFEtsWVZewoj7zCdnag4NsXW4ze5rZxfM0kfvt/t3AFZMctwSoVtVn3MffBN6BBYOCMRxT/vwHL+H3Cf903dbEjJzL68q5f/sJhqIxigOF1QzVOxjh7x7Yze6TzncRVegaiCQGho11yboG3v+albx+w8IZzTgaGY5xpn/IupWaWScXbQaXAG2qejBp2yoReRHoBT6mqk8By4CWpGNa3G0mj3ad6KHdXbXr6YNneOFYF/96/bksrS1LHNNcV46qs1j6qgKa6vhgW5APfusFXukc4PUbFxJwb+7VpUU015ezvK6cuvJi4snI7ce7+dYzx7jl7uepKglQ5Aa2hspi/uGdZ7NtZV3K1/n1odN8/rEDDAwNMxxTVG30sZl9MgoGIvI4sDjFro+q6gPu7zcA303a1wo0q+oZETkf+JGIbAFSfQ3TNK97K046iebm5pkW30yis3+Ia778K4ZjIx/D27cu5e1bl446rrm+HHBSSPkMBr892skxdynI7oEhvvDYAcqK/Xznjy/mwlWpb+TJXru2gVsvXc1Pdp3it0c7Ufeyf3mgg/fc+Sx/8Yb13HbpmkSNIRwd5p9/eoA7nzxCc1056921hNcurOSy9Y3eXKQxHskoGKjqVRPtF5EAcC1wftJzwkDY/f0FETkMrMepCSQvxtoEnEzzuncCdwJs27YtZcAwmXuppZvhmPLZa89m05Jq/D5h85Jqxrbrr6hzgsHxzvFr8uZKLKbc+LXnGIqOLA25dXktX/2DV7GkpmyCZ45W5Pfxtq1LeVtSwOsdjHDHD3fyf36yn5/tbWeFG/x2n+hlf1uQGy9q5mNv2UxZsT97F2RMjnmdJroK2KeqifSPiDQCnao6LCKrgXWtNKL2AAAgAElEQVTAEVXtFJGgiFwMPAfcBPybx+UzE9hxvAcReMs5S6gqLUp7XGNVCSUBH6/kMRj0hCIMRWP82RVruW7bcgCW1ZZlZaWx6tIivnTDebxmTT1ff+rlxNxDFcUBvnbTtrwtR2lMNnkdDK5ndIoI4FLg70UkCgwDt6lqvCvKh4BvAGU4DcfWeJxHO090s7qhYsJAACAiNNeV5zUYdA0MAc4Sjcvdmko2iQg3XrSCGy9akfVzG1MIPA0Gqvr+FNvuA+5Lc/zzwFlelslMjaryUksPl6xtmNLxTjAIeVyq9LoGnC6iC7K0AL0x801h9QM0BeNU7yAdwTBnN02tv/zyunKOdw6gmp8mnG63ZrCg3IKBMTNhwcCktKPF6Zd/TlPtlI5vriunLxyls3/Iy2KllagZlE+c0jLGpGbBwKS0o6WbgE/YsrR6SsfHe9jkq90gXjOotZqBMTNiwcCktKOlh/WLqigtmlp3yea6/AaDzv4h/D6hutTmXjRmJiwYmHFUlR0tPZwzxfYCgKYF+R1r0DUQYUF50bgxEMaYqbFgYMZ5pXOAnlBkyu0FAGXFfhZWleQ1TWQpImNmzoKBGeelROPx9GbezOdYg66BIWs8NiYDFgzMODuOd1Mc8LFhcdW0ntdcX84rZ/JVM4hYzcCYDFgwMOPsaOlh85JqivzT+/NoriuntXeQcHTYo5KlZzUDYzJjXS8MqsrP97UTHIwCsOtkD9ed3zTJs8ZLTGXdFWJ1Y2W2i5mWqtLVH7HRx8ZkwIKBYU9rL7fc/fyobRetrp/2eZK7l+YyGAwMDTM0HLPRx8ZkwIKBoSPorPr1pfeex5alNRT5hWW1U5/2Oa45T1NZdyWmorA0kTEzZcHAJNYB3ri4KqPFaRqrSigt8iUWmMmVbncqCmtANmbmLBgYet1gUF2W2Tfr+FTWD+1s5XiXExAuWdfIH1zs7bTPXTZJnTEZs95EJlEzqMkwGAD8/quaqCkr4tiZAX59+Ax3Pf1yxuecTHySuroKSxMZM1NWMzD0hCKUFfkpCWS+bOMHL1vDBy9bA8Bf/OAlnjl8OuNzTqar3yapMyZTVjMw9IQiWakVjFVa5GMwaU1ir8TTRLUeXIMx84UFA+NhMPATjng/AK17IEJVaYDANAfJGWNG2P8eQ/fA7K8ZWOOxMZmxYGDoCUUy7kmUSknAz3BMiQx7GxC6Bmz0sTGZsmBg6PWwzQBg0ONUUbfNS2RMxiwYGE/bDAAGI97WDDr7LU1kTKYsGMxzkeEY/UPD3gSDQDwYeF0ziFBrNQNjMmLBYJ6Ljz724mZa4qaJwh42Ig9FY/SFo1YzMCZDFgzmuWyOPh5rJE3kXc2gO+RORWENyMZkxILBPOdlMCgJxGsGHgYDdyoKa0A2JjMWDOa57ixNUpdKLhqQO/ttkjpjssGCwTzXO9vTRPGpKKxmYExGMg4GInKdiOwWkZiIbBuz7w4ROSQi+0XkjUnbr3a3HRKR25O2rxKR50TkoIh8X0Ts657HvG0ziI8z8K5m0JVIE9mfijGZyEbNYBdwLfBk8kYR2QxcD2wBrgb+XUT8IuIHvgy8CdgM3OAeC/A54Auqug7oAm7JQvnMBHoGPAwGbtdSL9sMbC0DY7Ij42CgqntVdX+KXdcA31PVsKq+DBwCLnR/DqnqEVUdAr4HXCMiAlwB3Os+/27gHZmWz0wsPn11cSD7GcNctBl0D0QoLfJRVpz59NvGzGdethksA44nPW5xt6XbXg90q2p0zPZxRORWEXleRJ7v6OjIesHnk56QdwO24r2JvGwz6LLRx8ZkxZQWtxGRx4HFKXZ9VFUfSPe0FNuU1AFIJzh+/EbVO4E7AbZt25bymPlg36lefra3PfF405Iqrti4aFrn8GoqCkiqGXicJrJFbYzJ3JSCgapeNYNztwDLkx43ASfd31NtPw3UikjArR0kH29S+IeH9/HkgZGa0YLyIl782zdM6xxezVgKyTUDbxuQbYyBMZnzMk30IHC9iJSIyCpgHfAb4LfAOrfnUDFOI/ODqqrAE8C73OffDKSrdcx7qsqOlm7eva2JA59+Ex9+/Vq6QxFiselVlLysGfh8QnHA5+kCN10DQzb62JgsyEbX0neKSAvwauAhEXkUQFV3Az8A9gA/Af5EVYfdb/0fBh4F9gI/cI8F+Gvgz0XkEE4bwtczLd9c1dIVonsgwtbltRQHfNSWF6EKwXB08icn8TIYAJQGfJ7OTdRtNQNjsmJKaaKJqOr9wP1p9n0G+EyK7Q8DD6fYfgSnt5GZxM4TPQCcvawGGBlBPN21CTwPBkX+rDcg//ZoJ139QyjxtQysZmBMpjIOBiY/drT0UOQXNiyuAkbGCfSEIqMaZCYSGY4x4NH01XElRb6sBoNjZ/q57qvPjNrWtKAsa+c3Zr6yYDBL7TzRzcbF1ZS4A7uqS92awWBkyufwcvRxXGnAn9UG5P2nggD86/XnsqaxkoBfWL+wKmvnN2a+smAwC6kqO1t6eMs5SxPbapLSRFOVk2BQ5M9q19Ijp/sBeP3GhYkAaIzJnE1UNwu90jlA72CUc5pqEtuqy5y43huaegNyIhh42ABbmuU00eH2PhqrSiwQGJNlFgxmoR0toxuPYXSbwVTlrGaQxTTRkdP9rG6oyNr5jDEOCwaz0M4TPRT7faxfNJIrrywJ4JPpBQMvp6+OKwn4s9q19EhHH6sbK7N2PmOMw4LBLLSzpYdNS6pGTS4nIlSXFU2rAbnbwxlL40qKsjforLN/iK6BCGsarWZgTLZZMJhlYjFl14kezk5qL4irKSsqvDRRIHvjDI509AGwxmoGxmSdBYNZ5uiZfoLh6Kj2grjq0qJp9yYqL/ZT5Pfuz6C0yMdgltJERzqcnkSrrWZgTNZZMJhlRkYe147bN5OagZe1AsjuCOTDHX0U+300LSjPyvmMMSNsnEEBG4wMc7ovzJm+ocQN9Yl97ZQEfKxbND5VUl0W4FTv4JTPn5tg4HQtVVWc9Ytm7nBHPysbyvH7MjuPMWY8CwYF5B1f/hV7TvYCoCiR4dQzkF64si5laqcgawYBPzGFaEwp8md2Ez9yus9GGxvjEQsGBUJV2X68m20rFnDBqjrA6S7aUFlMfUUJ5UnLOq5blPqGWF06vWDQG4rQXOdtyqWkaGS1s0zaJiLDMV45M8DVW1KtsWSMyZQFgwIR74t/xaaF/K/L187oHNVlRQxFYwxGhhOrjE0kV20G4CxwU1U68/O80jlANKbWk8gYj1gDcoEIDTltAmVTuImnM935iboHcpMmgszXQbaeRMZ4y2oGYzyxr51HdrUmHm9bWce7t011UuiZC0UyDwaJNQ0GIyysnvhr+FA0Riji7fTVMJImCmc4Wd1hd4yBjT42xhsWDMb4z6eO8PzRLuori+kbjPLjl1p5+9alU0q7ZCL+zbmsOPOawVTaDXIxSR2MThNl4khHHw2VJZ4HL2PmK0sTjTEYGebCVXU8c8eVfPGG8whFhnn2yBnPXzdeM8gk6FSXTn3m0lyMPoaR68m0ZnCko99SRMZ4yGoGY4SjMeoqnBj56jX1lAR8/GJ/B5dvWOjp6w5mIU00Wc1gKBrjyQMdDA3HOHrGycFXe95mEO9NlFnN4HBHH1efZT2JjPGKBYMxwtFYYvWw0iI/r1lTz8/3tfN3b9uc8aCpiYSGYonXnKnqSYLBI7ta+cj3to/a1lTr7ZKRJUWTNyAPRoaJqTOmIjKs9AxE6BwYojcUQXEa17sGIqxusPYCY7xiwWCMcHSYkqTZQK/YuJAn9u/mcEc/axd6dzPKSgNy6cS9iY509CMCD/3pJfh9QkWJ3/OpHUqLJq4ZPLKzlQ99+3dTOtfmpdVZK5cxZjQLBmOEI7FEDxhwllfkgd08sa89N8GgeObNOMUBH2VF/rQ1g5auEIurS3N6U52sa+n+NmdN49vftBEB/D6hpqyIuopiqsuK8Lm1sbIiP5uW2OhjY7xiwWCM5DQRQNOCctYvquSJ/e388aWrPXvdwaHMG5DBaTdIt6ZBS9cAy3M8yVuiN1GaBuTugQhVJQFuu2xNLotljBnDehONMTZNBE7t4DcvdxKcxsIx05WNNBE4k9VNVDNoWuBtG8FY8TRROE2aqDcU8bx7qzFmchYMkqiqWzMYEww2LCQaU54+eNqz1w5lYZwBuDWDFF1LI8MxWnvyEQwmrhnkYkoMY8zkLBgkGRqOoTrSAybu/BULqCoN8NM9bZzpC3OmL5y1Ofrj4ucrDWQeDFLVDE71DBJTcr4WQLF/4gbkbgsGxhQEazNIEp8sbmzNoMjv47L1jdz/4gnuf/EEAA2VxTz911dkbWRyKOKkp3wZztVfXVrE3lBw3PbjnQMANNXltmbg8wnFgfTrIPeEIqxPsTaDMSa3MqoZiMh1IrJbRGIisi1p+++JyAsistP994qkfb8Qkf0ist39WehuLxGR74vIIRF5TkRWZlK2mYjntccGA4C/efMmPnXNFv7+mi3ceulqTvcN8atD2UsbDQ4NZ5wiAmesQaoG5JauEEDOG5DBGXiWriZlaSJjCkOmNYNdwLXAf4zZfhp4m6qeFJGzgEeBZUn7b1TV58c85xagS1XXisj1wOeA92RYvmmJT5lQkiJVs7S2jPe9eiXgjOT97nOv8NPdbVy5aVFWXjsUGc44RQROMAgORhmO6agVwVq6BvAJLK7JYB7pGXKWvhyfJlJ1Bph5PQraGDO5jGoGqrpXVfen2P6iqp50H+4GSkWkZJLTXQPc7f5+L3CleDnkN4VEmqho4relOODj9RsX8vjeNoZjqVcjm65QJJaVmkH8W3bf4OhG5ONdIZbUlGW0wMxMlRb5U85NNBiJMTQco7asOOdlMsaMlos7w+8DL6pqOGnbf7kpoo8n3fCXAccBVDUK9AD1OShfwkiaaPKb8hu2LOJM/xAvHOvKymuHhqa2IM1k4pPVjW1EbukayHlPojhnHeTxNYNcTZZnjJncpMFARB4XkV0pfq6ZwnO34KR7Ppi0+UZVPRu4xP15X/zwFKdI+bVbRG4VkedF5PmOjo7JijFliTTRJDUDgMs3LKQ44OPR3aey8tqDkWHKpvC6k6lJWtMgmTPGIPftBeAE11RdS7tDQ4AFA2MKwaRtBqp61UxOLCJNwP3ATap6OOl8J9x/gyLyHeBC4JtAC7AcaBGRAFADdKYp053AnQDbtm3LTp6G9L2JUqksCfC6tQ38dM8pPvaWTeMmsTvU3sdnHtpDf9jtMlrs57PXns3SNBPDhSLZa0CG0TWDcHSYU72Dea4ZjA8GPQNOGWtt0JkxeedJmkhEaoGHgDtU9VdJ2wMi0uD+XgS8FacRGuBB4Gb393cBP1fVrN3op2IkGEztpvyGzYs43hlib+vorpxtvYPcfNdvePF4N36fMDTsTB394ivdac/l1Ayy12aQHAxauwdRheV1+akZpGtAtjSRMYUj066l7xSRFuDVwEMi8qi768PAWuDjY7qQlgCPisgOYDtwAvhP9zlfB+pF5BDw58DtmZRtJuJ94adSMwC4avMiRBiVKgoORnj/f/2W7oEh7rnlIr5768V86b3nAdAXTj+dRWiKi9hPJtU6yPFupfmqGZQE/ClrBt0WDIwpGBl1LVXV+3FSQWO3fxr4dJqnnZ/mXIPAdZmUJ1OD0fiaAlMLBg2VJWxbsYCHdraydXkNAF9/+mUOtgW56/0XcNYyZ1tVidvDJzzBnP5D2akZpEoTHe9yB5zlMU00FB1fM+jN0dKbxpjJ2QjkJCM1g6nflN9y9hI+8eM9fOAbI8Mm/um6rVy6vjHxuKLEOd/Y7p7JstVmUFHsx++TUQ3ILV0DBHzC4urcjzGAeJooRZtBKIJPoLLY/gyNyTf7X5hkOg3Ice979Uq2raxLjDdYUF5Mc/3o3HzA76O0yEf/0CTBIAs1AxGhunT0zKUtXSGW1JYSyMMYA3Dez8EUNYNud8BZplNwGGMyZ8EgyXQbkMFZjCWeDppIZUmAYJqaQSymDEZi4ybIm6mxM5ce7xygqTY/jccwcc3A2guMKQw2a2mS6YwzmK7KkgD94dTBIB6EslEzAKfdYGzNYHmOJ6hLFu9aOrZzWE8oQq0FA2MKggWDJBNNVJepigmCwcjCNtl53eRprAcjw7QHw3kbcAbOtNwxdRa7T9YdsnmJjCkUFgyShKMxigO+cQPIsqGyJEBwsmCQhQZkcKaxjjcgn+jOb7dSGFngZuz8RL2WJjKmYFgwSJJqyctsmShNlFjYJotponi3zZExBvlsM0i9wE1PKGKjj40pENaAnMRZ8jI7N+SxKksD9HWkqRkMZWf947ja8iJO9w2x+W9/QtRNzeSzzSD+niY3IquqNSAbU0AsGCQJR8avf5wtE7UZDGY5TXT9BcuJqRJzu7suqSnL2xgDGGmQT04T9YWdNRcsGBhTGCwYJAlHhz3pSQRQNUHX0pEG5OwEgxX1Fdzxpk1ZOVc2xNNfyWmieAO3rWVgTGGwNoMkgxHv0kQVJQHC0RiR4fGDr+Jpomy1GRSakWAwUjPodmcstd5ExhQGCwZJvG5ABlKmikJZbkAuNKWBeJpoJBD22iR1xhQUCwZJnAZkb4NBX4pgkO02g0KTqmaQSBNZbyJjCoIFgyThaMyzb+eVpemDQbZ7ExWakhRdS236amMKiwWDJOGId2miignSRINZno6i0JSm6FpqC9sYU1gsGCQZimZvsrix4mmiVD2K4jUDrwJRviXSRNHRwaDIL5TP0dSYMbPN3Lz7zFAu2gz6UyxwMxgZprTIN2enck41Arl7wBlw5sXUH8aY6bNgkMTT3kSJNoPxS19may2DQpVqbqJem6TOmIJiwSBJ2MNxBvHVvFItfRnK0pKXhSoeYMcOOrPpq40pHBYMkoSjMc9GIE+09GUoMkzpHM6diwjFAV9iWVGA7tCQNR4bU0AsGLiGY8rQsHdtBhMtfTk4x9NE4Aw8G9ubyIKBMYXDgoFraAZLXk5XuqUvQ5HhOTv6OM5Z+jIpTTQQobbc5iUyplBYMHAllrz0sHtnujUN5nqbAbjBwH2Ph2NK72DUGpCNKSAWDFzxeXO8ajMAZ+BZ6ukovBv5XChKi3yJZUWDgzbgzJhCY8HAFb9RlXqcJko3N9FcnZcoLrlmMDJ9tQUDYwqFBQNXIk3kYc0gbZooMkyZh69bCEqSGpDj01dbzcCYwjG370DTEM5FA3Jp6prBXB90BqMbkBPzEtmMpcYUjIyCgYhcJyK7RSQmItuStq8UkZCIbHd/vpq073wR2Skih0Tki+LORyAidSLymIgcdP9dkEnZpisXDcjplr4MDc3tcQbgBNl4zcDSRMYUnkzvfLuAa4EnU+w7rKrnuj+3JW3/CnArsM79udrdfjvwM1VdB/zMfZwz8TYDL4NBqqUvYzElHI3Ng5qBL1H7sumrjSk8Gd35VHWvqu6f6vEisgSoVtVnVFWBbwLvcHdfA9zt/n530vacGOlN5N1NOdXSl/FG1bkfDPx0BMN88se7eXD7CcCWvDSmkHjZZrBKRF4UkV+KyCXutmVAS9IxLe42gEWq2grg/rvQw7KNE09heD3OAEavaZBY2GaOp4nOa67FJ3DvCy3sOxVk24oFc747rTGzSWCyA0TkcWBxil0fVdUH0jytFWhW1TMicj7wIxHZAqSar1inXNqRMt2Kk2qiubl5uk9PaaQB2ftg0BeOJkbfxhe28bJLayG48aIV3HjRinwXwxiTxqTBQFWvmu5JVTUMhN3fXxCRw8B6nJpAU9KhTcBJ9/c2EVmiqq1uOql9gvPfCdwJsG3btmkHk1RGupZ625sIRi99Ga8ZzPUGZGNMYfPka7CINIqI3/19NU5D8RE3/RMUkYvdXkQ3AfHaxYPAze7vNydtz4lc1AxSLX0ZT0/N9TYDY0xhy7Rr6TtFpAV4NfCQiDzq7roU2CEiLwH3Arepaqe770PA14BDwGHgEXf7Z4HfE5GDwO+5j3MmMQLZy5pBiqUvQxYMjDEFYNI00URU9X7g/hTb7wPuS/Oc54GzUmw/A1yZSXkykauJ6mD00pcjDcg2/s8Ykz92B3KFozF8AgEP1yFOtfRlvGZgPWuMMflkwcAVjjpLXnq5QHuqpS+tzcAYUwgsGLjCkWFPJ6mD1EtfzpdxBsaYwmbBwOXUDLx9O1ItfWkNyMaYQmDBwBVPE3lt7NKXgznoxWSMMZOxYOAajAx7XjOA8WsahHIwDYYxxkzG7kCucDTmeZsBjF/6ctBdy8DLhmtjjJmMBQNXODqcszTR2OkorPHYGJNvFgxc4UiM0hzUDCpLAqN7E82DVc6MMYXPgoErZw3IpYFxvYlyEYSMMWYidhdyOWmiHLUZJPcmsjSRMaYAWDBw5WKcAThLX/aFLU1kjCksFgxc4Uhu0kRjl74cjAzbGANjTN5ZMHCFo95PRwHjl74MRWJWMzDG5J0FA1eu0kTJS1+C1QyMMYXBggGgqu4I5Nz0JoKRYBAasjYDY0z+WTAAojElprmZEmLs0pehiPUmMsbknwUDktY/zmGbQXyyupCliYwxBSCjZS/ninBisrjcTEcB8OvDZwgNDTMUtQZkY0z+WTBgpGaQi5HAjVUl+H3CnU8eSWxbWF3i+esaY8xELBiQlCbKQc2grqKYp//69XQPOOsgB3zCmsZKz1/XGGMmYsEAZ4wB5G5NgSU1ZSypKcvJaxljzFRYAzLO6GPITQOyMcYUIrv7kds0kTHGFCILBuQ+TWSMMYXG7n6MLEpvNQNjzHxlwYCkmoG1GRhj5qmM7n4icp2I7BaRmIhsS9p+o4hsT/qJici57r5fiMj+pH0L3e0lIvJ9ETkkIs+JyMpMyjYdiQZkSxMZY+apTO9+u4BrgSeTN6rqt1X1XFU9F3gfcFRVtycdcmN8v6q2u9tuAbpUdS3wBeBzGZZtyqwB2Rgz32UUDFR1r6run+SwG4DvTuF01wB3u7/fC1wpIpJJ+aYqniaytYiNMfNVLu5+72F8MPgvN0X08aQb/jLgOICqRoEeoD4H5bOagTFm3pt0BLKIPA4sTrHro6r6wCTPvQgYUNVdSZtvVNUTIlIF3IeTRvomkKoWoGnOeytwK0Bzc/NklzCpeJtBsbUZGGPmqUmDgapelcH5r2dMrUBVT7j/BkXkO8CFOMGgBVgOtIhIAKgBOtOU6U7gToBt27alDBjTEY4OU+QX/L6cZKWMMabgePZVWER8wHXA95K2BUSkwf29CHgrTiM0wIPAze7v7wJ+rqoZ3+inwlny0lJExpj5K6OJ6kTkncC/AY3AQyKyXVXf6O6+FGhR1SNJTykBHnUDgR94HPhPd9/XgW+JyCGcGsH1mZRtOsLRYetWaoyZ1zIKBqp6P3B/mn2/AC4es60fOD/N8YM4NYmcC0diFgyMMfOa3QGBwWiMElttzBgzj1kwwFn20moGxpj5zO6AxBuQ7a0wxsxfdgfEbUC2NJExZh6bt8tePnWwIzF1dXswzLJaW4bSGDN/zdtgcPt9OznRHUo83tpUm8fSGGNMfs3bYHDX+y8gMhxLPF7TWJnH0hhjTH7N22CwYXFVvotgjDEFwxqQjTHGWDAwxhhjwcAYYwwWDIwxxmDBwBhjDBYMjDHGYMHAGGMMFgyMMcZgwcAYYwwWDIwxxgCSozXnPSMiHcCxGT69ATidxeLMBvPxmmF+Xvd8vGaYn9ed7ppXqGrjVE4w64NBJkTkeVXdlu9y5NJ8vGaYn9c9H68Z5ud1Z+OaLU1kjDHGgoExxhgLBnfmuwB5MB+vGebndc/Ha4b5ed0ZX/O8bjMwxhjjmO81A2OMMczTYCAiV4vIfhE5JCK357s8XhGR5SLyhIjsFZHdIvIRd3udiDwmIgfdfxfku6zZJiJ+EXlRRP7HfbxKRJ5zr/n7IlKc7zJmm4jUisi9IrLP/cxfPdc/axH5/9y/7V0i8l0RKZ2Ln7WI3CUi7SKyK2lbys9WHF907287RORVU3mNeRcMRMQPfBl4E7AZuEFENue3VJ6JAn+hqpuAi4E/ca/1duBnqroO+Jn7eK75CLA36fHngC+419wF3JKXUnnrX4GfqOpGYCvO9c/Zz1pElgF/BmxT1bMAP3A9c/Oz/gZw9Zht6T7bNwHr3J9bga9M5QXmXTAALgQOqeoRVR0Cvgdck+cyeUJVW1X1d+7vQZybwzKc673bPexu4B35KaE3RKQJeAvwNfexAFcA97qHzMVrrgYuBb4OoKpDqtrNHP+scdZxLxORAFAOtDIHP2tVfRLoHLM53Wd7DfBNdTwL1IrIksleYz4Gg2XA8aTHLe62OU1EVgLnAc8Bi1S1FZyAASzMX8k88S/AXwEx93E90K2qUffxXPzMVwMdwH+56bGviUgFc/izVtUTwD8Br+AEgR7gBeb+Zx2X7rOd0T1uPgYDSbFtTnepEpFK4D7gf6tqb77L4yUReSvQrqovJG9Ocehc+8wDwKuAr6jqeUA/cygllIqbI78GWAUsBSpwUiRjzbXPejIz+nufj8GgBVie9LgJOJmnsnhORIpwAsG3VfWH7ua2eLXR/bc9X+XzwGuBt4vIUZwU4BU4NYVaN5UAc/MzbwFaVPU59/G9OMFhLn/WVwEvq2qHqkaAHwKvYe5/1nHpPtsZ3ePmYzD4LbDO7XFQjNPg9GCey+QJN1f+dWCvqn4+adeDwM3u7zcDD+S6bF5R1TtUtUlVV+J8tj9X1RuBJ4B3uYfNqWsGUNVTwHER2eBuuhLYwxz+rHHSQxeLSLn7tx6/5jn9WSdJ99k+CNzk9iq6GOiJp5MmpKrz7gd4M3AAOAx8NN/l8fA6X4dTPdwBbHd/3oyTQ/8ZcND9ty7fZfXo+i8H/sf9fTXwG+AQ8N9ASb7L58H1ngs8737ePwIWzPXPGvgksA/YBXwLKJmLnzXwXZx2kQjON/9b0n22OGmiL7v3t504va0mfQcvMRoAAAA7SURBVA0bgWyMMWZepomMMcaMYcHAGGOMBQNjjDEWDIwxxmDBwBhjDBYMjDHGYMHAGGMMFgyMMcYA/w8PP8ZeozZHAAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make('Pendulum-v0')\n",
    "\n",
    "agent = Agent(env)\n",
    "batch_size = 128\n",
    "rewards = []\n",
    "avg_rewards = []\n",
    "frame = 0\n",
    "\n",
    "for episode in range(50):\n",
    "    state = env.reset()\n",
    "    episode_reward = 0\n",
    "    \n",
    "    for step in range(500):\n",
    "        env.render()\n",
    "        frame += 1\n",
    "        \n",
    "        action = agent.get_action(state)\n",
    "        epsilon = epsilon_by_episode(episode)\n",
    "        random_noise = torch.randn(1) * epsilon # needs decay later\n",
    "        action += random_noise\n",
    "        new_state, reward, done, _ = env.step(action)\n",
    "        agent.memory_buffer.push(state, action, reward, new_state, done)\n",
    "        \n",
    "        if len(agent.memory_buffer) > batch_size:\n",
    "            agent.update(batch_size)\n",
    "    \n",
    "        state = new_state\n",
    "        episode_reward += reward\n",
    "        \n",
    "        if done:\n",
    "            #print(\"episode: {}, reward: {}\".format(episode, np.round(episode_reward, decimals=2)))\n",
    "            break\n",
    "            \n",
    "        if frame%40 == 0:\n",
    "            plot(frame, rewards)\n",
    "    \n",
    "    rewards.append(episode_reward)\n",
    "    avg_rewards.append(np.mean(rewards[:-10]))\n",
    "            \n",
    "    rewards.append(episode_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}