第20章-多智能体强化学习入门.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 10107,
     "status": "ok",
     "timestamp": 1650012696153,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "-_L_dhppItIk",
    "outputId": "6c1eecf0-fd72-4d13-ad05-192463636129"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'ma-gym'...\n",
      "remote: Enumerating objects: 1072, done.\u001b[K\n",
      "remote: Counting objects: 100% (141/141), done.\u001b[K\n",
      "remote: Compressing objects: 100% (131/131), done.\u001b[K\n",
      "remote: Total 1072 (delta 61), reused 31 (delta 6), pack-reused 931\u001b[K\n",
      "Receiving objects: 100% (1072/1072), 3.74 MiB | 4.47 MiB/s, done.\n",
      "Resolving deltas: 100% (524/524), done.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import rl_utils\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "! git clone https://github.com/boyu-ai/ma-gym.git\n",
    "import sys\n",
    "sys.path.append(\"./ma-gym\")\n",
    "from ma_gym.envs.combat.combat import Combat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HdZSfYc7ItIn"
   },
   "outputs": [],
   "source": [
    "class PolicyNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim):\n",
    "        super(PolicyNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = torch.nn.Linear(hidden_dim, action_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(x))))\n",
    "        return F.softmax(self.fc3(x), dim=1)\n",
    "\n",
    "\n",
    "class ValueNet(torch.nn.Module):\n",
    "    def __init__(self, state_dim, hidden_dim):\n",
    "        super(ValueNet, self).__init__()\n",
    "        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)\n",
    "        self.fc2 = torch.nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.fc3 = torch.nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(x))))\n",
    "        return self.fc3(x)\n",
    "\n",
    "\n",
    "class PPO:\n",
    "    ''' PPO算法,采用截断方式 '''\n",
    "    def __init__(self, state_dim, hidden_dim, action_dim, actor_lr, critic_lr,\n",
    "                 lmbda, eps, gamma, device):\n",
    "        self.actor = PolicyNet(state_dim, hidden_dim, action_dim).to(device)\n",
    "        self.critic = ValueNet(state_dim, hidden_dim).to(device)\n",
    "        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),\n",
    "                                                lr=actor_lr)\n",
    "        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),\n",
    "                                                 lr=critic_lr)\n",
    "        self.gamma = gamma\n",
    "        self.lmbda = lmbda\n",
    "        self.eps = eps  # PPO中截断范围的参数\n",
    "        self.device = device\n",
    "\n",
    "    def take_action(self, state):\n",
    "        state = torch.tensor([state], dtype=torch.float).to(self.device)\n",
    "        probs = self.actor(state)\n",
    "        action_dist = torch.distributions.Categorical(probs)\n",
    "        action = action_dist.sample()\n",
    "        return action.item()\n",
    "\n",
    "    def update(self, transition_dict):\n",
    "        states = torch.tensor(transition_dict['states'],\n",
    "                              dtype=torch.float).to(self.device)\n",
    "        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(\n",
    "            self.device)\n",
    "        rewards = torch.tensor(transition_dict['rewards'],\n",
    "                               dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        next_states = torch.tensor(transition_dict['next_states'],\n",
    "                                   dtype=torch.float).to(self.device)\n",
    "        dones = torch.tensor(transition_dict['dones'],\n",
    "                             dtype=torch.float).view(-1, 1).to(self.device)\n",
    "        td_target = rewards + self.gamma * self.critic(next_states) * (1 -\n",
    "                                                                       dones)\n",
    "        td_delta = td_target - self.critic(states)\n",
    "        advantage = rl_utils.compute_advantage(self.gamma, self.lmbda,\n",
    "                                               td_delta.cpu()).to(self.device)\n",
    "        old_log_probs = torch.log(self.actor(states).gather(1,\n",
    "                                                            actions)).detach()\n",
    "\n",
    "        log_probs = torch.log(self.actor(states).gather(1, actions))\n",
    "        ratio = torch.exp(log_probs - old_log_probs)\n",
    "        surr1 = ratio * advantage\n",
    "        surr2 = torch.clamp(ratio, 1 - self.eps,\n",
    "                            1 + self.eps) * advantage  # 截断\n",
    "        actor_loss = torch.mean(-torch.min(surr1, surr2))  # PPO损失函数\n",
    "        critic_loss = torch.mean(\n",
    "            F.mse_loss(self.critic(states), td_target.detach()))\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        critic_loss.backward()\n",
    "        self.actor_optimizer.step()\n",
    "        self.critic_optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2805926,
     "status": "ok",
     "timestamp": 1649963248923,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "t8FsMOFPItIp",
    "outputId": "2f453795-508c-45ff-91e1-fb8b81eb5e9c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n",
      "Iteration 0: 100%|██████████| 10000/10000 [07:17<00:00, 22.85it/s, episode=10000, return=0.310]\n",
      "Iteration 1: 100%|██████████| 10000/10000 [05:43<00:00, 29.08it/s, episode=20000, return=0.370]\n",
      "Iteration 2: 100%|██████████| 10000/10000 [05:30<00:00, 30.26it/s, episode=30000, return=0.560]\n",
      "Iteration 3: 100%|██████████| 10000/10000 [04:54<00:00, 33.96it/s, episode=40000, return=0.670]\n",
      "Iteration 4: 100%|██████████| 10000/10000 [04:20<00:00, 38.46it/s, episode=50000, return=0.670]\n",
      "Iteration 5: 100%|██████████| 10000/10000 [03:52<00:00, 43.09it/s, episode=60000, return=0.620]\n",
      "Iteration 6: 100%|██████████| 10000/10000 [03:55<00:00, 42.53it/s, episode=70000, return=0.610]\n",
      "Iteration 7: 100%|██████████| 10000/10000 [03:40<00:00, 45.26it/s, episode=80000, return=0.640]\n",
      "Iteration 8: 100%|██████████| 10000/10000 [03:48<00:00, 43.81it/s, episode=90000, return=0.650]\n",
      "Iteration 9: 100%|██████████| 10000/10000 [03:42<00:00, 44.91it/s, episode=100000, return=0.770]\n"
     ]
    }
   ],
   "source": [
    "actor_lr = 3e-4\n",
    "critic_lr = 1e-3\n",
    "num_episodes = 100000\n",
    "hidden_dim = 64\n",
    "gamma = 0.99\n",
    "lmbda = 0.97\n",
    "eps = 0.2\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\n",
    "    \"cpu\")\n",
    "\n",
    "team_size = 2\n",
    "grid_size = (15, 15)\n",
    "#创建Combat环境，格子世界的大小为15x15，己方智能体和敌方智能体数量都为2\n",
    "env = Combat(grid_shape=grid_size, n_agents=team_size, n_opponents=team_size)\n",
    "\n",
    "state_dim = env.observation_space[0].shape[0]\n",
    "action_dim = env.action_space[0].n\n",
    "#两个智能体共享同一个策略\n",
    "agent = PPO(state_dim, hidden_dim, action_dim, actor_lr, critic_lr, lmbda, eps,\n",
    "            gamma, device)\n",
    "\n",
    "win_list = []\n",
    "for i in range(10):\n",
    "    with tqdm(total=int(num_episodes / 10), desc='Iteration %d' % i) as pbar:\n",
    "        for i_episode in range(int(num_episodes / 10)):\n",
    "            transition_dict_1 = {\n",
    "                'states': [],\n",
    "                'actions': [],\n",
    "                'next_states': [],\n",
    "                'rewards': [],\n",
    "                'dones': []\n",
    "            }\n",
    "            transition_dict_2 = {\n",
    "                'states': [],\n",
    "                'actions': [],\n",
    "                'next_states': [],\n",
    "                'rewards': [],\n",
    "                'dones': []\n",
    "            }\n",
    "            s = env.reset()\n",
    "            terminal = False\n",
    "            while not terminal:\n",
    "                a_1 = agent.take_action(s[0])\n",
    "                a_2 = agent.take_action(s[1])\n",
    "                next_s, r, done, info = env.step([a_1, a_2])\n",
    "                transition_dict_1['states'].append(s[0])\n",
    "                transition_dict_1['actions'].append(a_1)\n",
    "                transition_dict_1['next_states'].append(next_s[0])\n",
    "                transition_dict_1['rewards'].append(\n",
    "                    r[0] + 100 if info['win'] else r[0] - 0.1)\n",
    "                transition_dict_1['dones'].append(False)\n",
    "                transition_dict_2['states'].append(s[1])\n",
    "                transition_dict_2['actions'].append(a_2)\n",
    "                transition_dict_2['next_states'].append(next_s[1])\n",
    "                transition_dict_2['rewards'].append(\n",
    "                    r[1] + 100 if info['win'] else r[1] - 0.1)\n",
    "                transition_dict_2['dones'].append(False)\n",
    "                s = next_s\n",
    "                terminal = all(done)\n",
    "            win_list.append(1 if info[\"win\"] else 0)\n",
    "            agent.update(transition_dict_1)\n",
    "            agent.update(transition_dict_2)\n",
    "            if (i_episode + 1) % 100 == 0:\n",
    "                pbar.set_postfix({\n",
    "                    'episode':\n",
    "                    '%d' % (num_episodes / 10 * i + i_episode + 1),\n",
    "                    'return':\n",
    "                    '%.3f' % np.mean(win_list[-100:])\n",
    "                })\n",
    "            pbar.update(1)\n",
    "\n",
    "# /usr/local/lib/python3.7/dist-packages/gym/logger.py:30: UserWarning:[33mWARN:\n",
    "# Box bound precision lowered by casting to float32[0m\n",
    "#   warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n",
    "\n",
    "# Iteration 0: 100%|██████████| 10000/10000 [05:22<00:00, 31.02it/s, episode=10000,\n",
    "# return=0.220]\n",
    "# Iteration 1: 100%|██████████| 10000/10000 [04:03<00:00, 41.07it/s, episode=20000,\n",
    "# return=0.400]\n",
    "# Iteration 2: 100%|██████████| 10000/10000 [03:37<00:00, 45.96it/s, episode=30000,\n",
    "# return=0.670]\n",
    "# Iteration 3: 100%|██████████| 10000/10000 [03:13<00:00, 51.55it/s, episode=40000,\n",
    "# return=0.590]\n",
    "# Iteration 4: 100%|██████████| 10000/10000 [02:58<00:00, 56.07it/s, episode=50000,\n",
    "# return=0.750]\n",
    "# Iteration 5: 100%|██████████| 10000/10000 [02:58<00:00, 56.09it/s, episode=60000,\n",
    "# return=0.660]\n",
    "# Iteration 6: 100%|██████████| 10000/10000 [02:57<00:00, 56.42it/s, episode=70000,\n",
    "# return=0.660]\n",
    "# Iteration 7: 100%|██████████| 10000/10000 [03:04<00:00, 54.20it/s, episode=80000,\n",
    "# return=0.720]\n",
    "# Iteration 8: 100%|██████████| 10000/10000 [02:59<00:00, 55.84it/s, episode=90000,\n",
    "# return=0.530]\n",
    "# Iteration 9: 100%|██████████| 10000/10000 [03:03<00:00, 54.55it/s, episode=100000,\n",
    "# return=0.710]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "executionInfo": {
     "elapsed": 20,
     "status": "ok",
     "timestamp": 1649963248923,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "OT2mwoZdItIq",
    "outputId": "6ea70d1d-bb28-456e-ffca-fe4f8106e0b8"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd5hU1dnAf+/MNsrSQZTi0gQBxYLYo0YsWGM0sURNjEaTaBK/mIKJMUZjYkyxfXzGEjWx9wqKiqDGQlMEARGU3pHetsyc749b9s6de2fu7O7szu6+v+fZZ++c286du3ve85bzvmKMQVEURVEAYk3dAUVRFKVwUKGgKIqiuKhQUBRFUVxUKCiKoiguKhQURVEUFxUKiqIoiosKBUVp5ojIFBG5rKn7obQMVCgoBYWILBGR0fb290QkISLbRWSriMwSkdPsfceKSNLet01EFojIJZ7rlIrIn0VkmYjsEpGFIvJLEZFGeo4SEbnBvu8O+7keEJGKxrh/VETkIRH5Y1P3QykcVCgohc4Hxpj2QCfgX8BTItLZ3rfK3tcB+DVwn4gMtfc9DRwPnAKUAxcBlwN3NFK/nwHOAC4AOgIjgJl2nxSlYFGhoDQLjDFJ4AGgDTDAt88YY14ANgFDReR44ETgbGPMp8aYGmPMh8CFwJUiMjDoHiKyr22K2Swic0XkDM++h0RknIiMtzWTqSIyIOQ6o4ETgDONMdPt+28xxowzxvzLPmYvEXlJRDaKyCIR+YHn/BtE5GkRecS+1xwR2UdErhWRdSKyXERO9N12gIhMszWqF0Wki+d6T4vIGhHZIiLviMgwu/1y4DvAr2yN6+Uo70Jp2ahQUJoFIlIEXAZsBxb69sVE5CwsbWIO1oA81Riz3HucMWYqsIKA2bqIFAMvA68DPYCfAI+KyGDPYecBfwA6A4uAm0O6OxqY5r+/jyfsvuwFnAP8SUS+7tl/OvCwfa+PgYlY/6+9gBuBe3zXuxj4PrAnUAPc6dn3KjDIfq6PgEcBjDH32tu3GmPaG2NOz9BfpZWgQkEpdA4Tkc3AGuB84CxjzBZ73172vg3A74GLjDELgG7A6pDrrbb3p90HaA/cYoypMsa8Bbxi39PheWPMNGNMDdZgekDIPbpmuD8i0gc4Evi1MWa3MWYWcD/WwO7wrjFmon2vp4Hudt+qsQRKhYh08hz/sK0V7QB+B3xbROIAxpgHjDHbjDGVwA3ACBHpGNY/pXVT1NQdUJQsfGiMOSpk3ypjTO+A9g1YM+Mg9rT3+9kLWG6bqRyWYs3MHdZ4tndiCZEgvgL2Cdnn3GujMWab714jPZ/XerZ3ARuMMQnPZ+z7b7a3vVrJUqAY6CYiG7A0mm9hCRbn+boBW1AUH6opKC2RN4FD7Rm5i4gcCvQB3go4ZxXQR0S8/xN9gZV1vP8oEQkSWM69uohIeQPcy8H7rH2BaizhdwFwJpZJqyNQYR/jRGFpmmQlBRUKSovDGPMmMAl4VkSGiUhcRA4DHgHuNsYsDDhtKtbs/1ciUiwix2LZ9Z+o4/3fAJ4XkYNFpEhEykXkhyLyfdvX8D7wZxEpE5H9gUvt/tWVC0VkqIi0xfI5PGNrFuVAJZb20hb4k++8tUD/etxXaWGoUFBaKmcDk4HXsJzTj2CFtP4k6GBjTBWWEBiDNcP+P+BiY8xndbz/OcAE4EksM82nWOahN+3952PN2lcBzwO/t4VJXXkYeAjLxFUG/NRu/w+WOWklMA/40Hfev7AitjaLyAv1uL/SQhAtsqMoiqI4qKagKIqiuKhQUBRFUVxUKCiKoiguKhQURVEUl2a3eK1bt26moqKiqbuhKIrSrJg5c+YGY0z3bMc1O6FQUVHBjBkzmrobiqIozQoRWRrlODUfKYqiKC4qFBRFURQXFQqKoiiKiwoFRVEUxUWFgqIoiuKiQkFRFEVxUaGgKIqiuKhQUBRFySPGGJ6esZz/fLCEmkQy6/FNTbNbvKYoitKc+OCLr/jlM7MBqEkYvn9UvybuUWZUU1AURckj2ypr3O1NO6uasCfRUKGgKIqSR2Ii2Q8qIFQoKIqi5JHmJRJUKCiKkoFtu6v53oPTWLV5V97vVZNI8qNHZvLpyi15v1euPPLhUirGjuc/HyzJemxlTYIf/GcGC9duA6CZKQoqFBRFCeeV2auZsmA9d7y5MO/3+nLDDl79dA3/8+SsvN8rV6574VMArn9xbtZjZy3bzBvz1vKb5+cAaj5KQUROFpEFIrJIRMYG7O8rIpNF5GMRmS0ip+SzP4qi5IYznBlMg17342WbWL5xZ+C+ut5p3dbdTFu8se6damCSBl77dHVW+9G6bbuZ+uVXjdOpCOQtJFVE4sA44ARgBTBdRF4yxszzHHYd8JQx5m4RGQpMACry1SdFUXIjX5Pcs/7vfUriMT6/eUztvezfxtRNLJz+v/9l7dZKltxyagP0sO44vZ+5dBMzl27ivEP6ZDz+rHHvs3Lzribvt0M+NYVRwCJjzJfGmCrgCeBM3zEG6GBvdwRW5bE/iqLUEf84nUyaOg/eu6oSAFT5FnLVVwCt3VoJ1F2o5ItsYagrbX9NofQ7n0KhF7Dc83mF3eblBuBCEVmBpSX8JI/9URQlR8Sev/uHqzF3vMtRf5kceM6pd77LfjdMDL3myixOa/+9KsaO54cPz8zWVZeaZOoVVmzaScXY8bw4a2Xka9QH/9heFI82zNYkDbe8+hkVY8eTtJ9h9orNVIwdz/tfbGjobobS1I7m84GHjDG9gVOAh0UkrU8icrmIzBCRGevXr2/0TipKq8WevfsHugVrt4UO7nNXbWXb7prAfQC7qy1NoSRtsHRuln7Oa3PXROktAJU1qRrI3FVbAXj5k6YxRBTHoqlA1Ykk97/7pbWdtJ7h/S8sX8PbCxpv3MunUFgJeI1pve02L5cCTwEYYz4AyoBu/gsZY+41xow0xozs3j1r3WlFURqARNLwOzvqxnE0z1iykUG/nVCv61Y7ZiPfWOmYT+piRHlm5gp3u8onFBL2rDsecXDOxuTP1uV0fFRNoaomSczu4/KNO/nN83OaJFdSPoXCdGCQiPQTkRLgPOAl3zHLgOMBRGRfLKGgqoCiFABfrt9eO+u2R+pz/vkB1Yn62b6dQdo/RifqYVP/xdOfuNt+oeCYk6IOztm45KHpOR1fFFEYVSWSxG3HyjVPz+axqct4b1HjRyXlTSgYY2qAq4CJwHysKKO5InKjiJxhH3YN8AMR+QR4HPieKRRvi6I0El+u386KTcHhmf59xhjeW7SB5Rt38uX67ZGuv3zjTpZ+tSOlbeHabazdujulbdXmXXzhuWZpUbz2vpB2jVyZt2orG7ZXukLFH7/vCIv6DgGVNYmUz85se9OOKhas2Zbz9RLJ+vXno2WbIh03d+VWV5vxC7bGXBad1yypxpgJWA5kb9v1nu15wJH57IOiFDpf//vbAIEhif59E+as4crHPnL3RwljPPrWyWnHnnDbO2ltR9zyVkqbf+Z+zF+nZL2Xl2TSuOYQgFPufJdu7Uu47dwDgHShkExVSupMmKbw/hdfcdLt7+Qc+nmfbeePin9Nx+drownvSx6aTocya0h2TGwNvT4kCk3taFYUJQdWbg7WKPKBd4Zcl9n7rupEWtuG7VXU2JqCf/JbH/ORF7+jubqedvmoGllD4GgKQX3esqu6UfqgQkFp9kxesI6KseNDTTDNhQG/ye7AlQh2hK27q6kYO56KseP52RMf59yPirHjmTR/LUnPIP3CrODInYv+NTX0Ojur0oUCeEJG/T4F13xU23bbG5+nnZ/tfZ9213/Z74aJVIwdT/9rx7vrIurCXyd+xlMzVqS1r99WScXY8UyYszqlvbImwQX3hX8nAIs37KBi7HhueGkuFWPHp+xzhYJPsG3ZWc2IP7zOv99fUoenyA0VCkqz56np1nKY2SsKL5FaLkSxXUdZ4LV6c62v4MWQwTwbT05fHqk/7y4Mj5/3L05zcGz8aeYjN/qo9r53TErPuRTlfTshsUlDvRzj4yZ/Edg+d5V17yemL09pX7Epe+LAD+2UFg8FDPDOd1KdttbCum7PjmVZr19fVCgozR5nMGnKtGPJpOEvr33G2q27ueftL5i/eiu7qhLc+PI8dlaFx+z7eXXOal77NDwmP0pytbpEXm7dnW6aeH3u2twv5CHNWWrzo0ctn4i3n1t2VXPjy1YGnOUbd/HszBU8OX1Z4PkmWNEIJZmDWerlT1YxaX7253bWWrzz+Xq+2l7pttc3m6wrFHwC1dGK+nRuW6/rR0HLcSrNHneQaEKpMGPpJu6e8gWfrtzCuws38NeJC/ifE/bhgfcW06ltMT89flCk6zgDZpgzNMozSh2+iH+8nm6mue3N9LZcyGbL9/bz1tc+Y44nZfY1nhBTP+4kwD49m0aTS/TQTx63zG3ZnNFe09hHyzZzwtA9ANi8s352f+fZ/Oajr3ZYqTK6l5fW6/pRUE1BafbU/ss3nVSosUNn5tmrZ2uSxh0UvQuQFq3bzvbKaJrDum2709qiPWHwIBi0EGrm0k3MX72VHRH7lAtfrNvOJnswSwYMzF5NIcz/EIT/fYdpJA5+oRAkJIwxzF6x2f0c1F8v67fVagde4ZdNK1kWkhnW3zfH5OVcznGelxXnf8hWoaA0ewpBU3BwZnRhjP7H23zvgWmRrjXq5klpbVG0gDAb+t8CtIGz736fMXe8G8kWnis/evQjjv+HFVLrz0cEqc8S5n8Iwv++/esS/PiFQJAGc/+7iznjf99zP2cLQ/3zq5+5216hlM1S9eKsVRmPcb4nJ82Fc6xzD+/akXyhQkFpMTSpTAj4R/cnk3PCOmcsjbaYKYgogi/MXOI4R4P4akdl6L76sNEWkkF98moKuaVzSPUh7a7OoimY7EJhxtLUOgzzV2+N3BuvUKjvQjfne3C67O27CBTH8/9XrkJBaQHkZ4HP9soaKsaOtwql1KEH/gE8aIboD2nMRjRNIXiQLM6Q5sEf219X/jxhfmB70BoEQbj6iY/53oPT3LULUXAudfnDM5k0f23WSm13T0mNIBr5xzddR7HDjsrUzyKS1SzlUJVI8sS0ZfS/dnykNRFORbYg/BqVV1iWFcXr5C/KFRUKSrPH+T9q6LKHSzZYaR3unLSoXtcJmvU5BMXhZyLKEwaZaiDzLLMyy2w7Kve8E2x2SQQM+iLW+ocpC9bnZj7ybN/7zpd8kGPVssqaZFqNgx2+CDGByOsbqmqS3PTKPJKGyP6iMPxJ+7zvsrQR/AmgQkFpARhfNEoUqmqS3P7m52kzRodnZq7gE9vxWGQPpu8uXM+UBakZMj9bs5WnZywP1AL83Xnuo/RFUFH7bIzhtjc+5663stdKzqYpBDlR1/jyIAWFqNYHxxHvxfvomdY7eNldneAtT5bSz9fmnssI4N3PU+/nd7TPWLqJp2emrkEIoyqRdAV+mEDOhpNGvKQodUh20n4DlBY1znCtIalKs8f5N8xFKPzngyXc/uZC4iL8JCBc1Jt105m9XfQvy0HsDVc8+fZ3AXj40lEp53v74izG+vWz4WaDbGzaWR24kCuIMLu2M/BESSfx4ZcNW+s46J7xOtjHH5+WunZhUx1DQH/17Gy+7SmTud1X/2HZxp38cXywKcxPdU3SzdsU1eTkp01JnKpdyYwmvrLi/DuZQTUFpZmyZVc1G7ZXsm7bbnf1qjcFRFVN0i0Mv2VntevwTCYNSzbscM/xrxyFdKenP/Xx9soa1m1NDxf1IqQKhrAMo1Edk2HZPYNSPWwMiYByhFtdZtd9u2ReNLV8486M9vSg50wmoWeH3FboNqSBcPGG2neyox6pMKoSSVcTyhRyevDenUP3tSuxBvxM32FjaQoqFJRmyaF/epORf3yTUTdPYqYTzeMZMa59bg5H3zqZHZU1jLjxdQ666Q0A7nxrIcf+bYo7MJYEzFa3+maNfjvvWePeY9SfUsNFM02+l361MzTDaFRrw/n3fRjYftRfJqeFZP7siWDHqwis3rKLU+/8b7SbeijKMKvfUVnD0bdO5tfPzg6tHRDkSE4ak/Pq6/ZlxbmdkIHj/jaFtz+3yrfUZ51GVSLpvkdvsR8/h/XvErqvjS0UwsyZkG5ayhcqFJRmSVAYond8cWz//kydU+yyhk4tgSB1fasvG2VRLPWYhevSs2b6hzwRcSNFVm8J1yrqG8IIZCx96SdqGmcvD186KqMT35ndPvfRSg7r35Wy4hjfPCi1HHvQoq5E0uQcTdO+NLMJ5ecn7JPT9b6w32VdfQEQ3WR0/qi+ofvalVqW/Ezhtf6/w3yhQkFpNmzbXc3BN73hJhTzs35bJfvdMDElxtw/GDl5iJwqXIFCwedk/e+iDdwVYM/3ztD9aQkSSasYjrPtp2LseD5etilUKFzyYLQFbgC/f2kul/07ezWwoniMtRkEVBg9yjObeLwDalVNkv17deKIAd1Cj3FIGpPzgsNsA2OPHNNAFDfA7DuqUIiJUF4a7MZtW5LdXxC1glt9UaGgNBvmrNzCVzuqQsM4J81fx7bdNTz43mK3zW+2cOLRnbj8IJU8KGb/7wH33LqrdoYeFFLpFF0PS30wbvKiUKEwOYdC7eNnr+bN+dnrBvfq1Caj1hJGPJZeT8ExqXUvL015hs27qiiKC2cdaGkKHdtY5p6g50wkTc5hxNmc5LnWYS5ugIE2U2oL7+PFREKd621Lssf8xFQoKEoq0xZbETFh//gfLrYG4ZpE7QzUm6DsxVkrXXPSJ8utcNOg2P2oC6kmzq3NZpop1ULY9RJJUy+zRa4kk4bVW3JPZxETYcP2VOf15V/rz5jhPenctjhlwP987XaSxhCPCd85tC+VNQn+/voCHptqRQ3ts0d791hLKETvhzEmReA7nHVgL44b3B3I7PsIIlO0T1TCHPsAZx/U23UQxwS3BrOfKJpCI8kEDUlVmg+3v2mZcMKEgiMAvLP2v7xWm6PmZ0/Moo0vrC9oUIhq57/uhU/d7UyLv8KulzTQv3s7NmzPT4qJtH4Yw/I6FiIKqvoVEyFp0p/PCWctjsfYXZ3krrdqF/95NbNc5eGUz9cHhsq2KYmz0Q4kytXu3hDmo4kZUox7o9BEJHS2H0UoNHSYcBiqKSjNjmwmB2tmbh3jX2HqH8CK4jGqE0k3myfUrSxkpjQRC0JCQNdu3Z2XMMORIaGPSZM9S2cQYd+GiGU6CdN2gt6Td9BOJE3gd71Hh2C/wM7KYG2sbXHcdXbnGstfEpc6lRqNSkzEDZWOSbhfIIr5qLFQoaA0O7LZjb2rZ0t8moB/Za0xhmufm8OBN73hrk9IBKy+zUZdFi19tmYb7y7c0OBmgTAbdzJpUqqyRSVo0DTGGvCMCf++gkw53kExkTQEndqzY5vA6wWtigZrlu0IhVyFbFEsVq/KbNkQqTX7xERCJzTtskRVNSYqFJRmRzZNodrjU/Dni/FPapPG8PInq9zzILpPwUum+PJsHDWoO98/sl+dz/fj7f5Pvj7Q3a5OJENn9ROv/hqH9guOow+bSMfEnu2HyMOg99TGYyZJGBNoWtsrpORkWC6iNiVF7rvLVVMwhAub+vAze5W8SG0Sw5hI6ISmtCie4m9pSlQoKAXFB198xdl3v091IsmcFVs46//eSxsMsvkGvatCp2SJ4vn5U5+4foUTbnubddt251S+0aE+WUZL4jGG9Cyv8/kOjqbjzW00oncndzssWR3A4J7l7NUpeIYetJbAmgGLbT4Kfvag9+T16YSZjyq6tQu83tjngtOE1EdTuP3Nzzn3nuCFgX7alcTpFfId+enV2TqurDheu35Gws138ZgwuGeHjNfUkFSlVfLLZz5h5tJNrNmym5vGz+PjZZv5eHlq/YF4LDzeG+zoo4j3M6bWHLVi0y6e/2hlnSKCcsny6ae0OJZzuGFQKG2Va/6q7f8eHcr4zSlDMl5r3AUHpZ3n8MuTBjOge/AgHYs55qPg7yse4PT1O1SDzv3J1wfyo2MHZOyzl93VCVco5Lrqd+6qrSllQDPx4lVHRRb+Zx3Yi6uOG8g1Jw6uXVWZ4c+qOC6UZem7rmhWWiXeiaMTgjp9capQmPzZejq3Kwm9RnWO5gBvZI2hbquMK+thPiqNx1L8Csfa4ZWZ+PXJ6QN9dY3Vb6+mUxQXLv9a5gH2pGFWfeGgWfuVxw0MXXUcsx3NoUIh0HyUKsz9i/7Acrr++uQhkQvKfLWjyjUfRR04/b6mbBzYtxMDe7TPWuXNoTge4xcnDaZ9aZFrRjMZpEI8FnNNnd3aB/9th4WzNjQqFJSCxDvQ+AvI76pOZMxVU51I1qs0Z52EQn3MR0WxFFtzlNj5oJxNlQlrwPIKhSgDqzNoZatLHHRekFBwZvlBj3HEgK6UFMU4pMKKkMqkYUVdQ3DqfnvWagoRz8lFE/Fety51J3550mAgNcLIn2CwKCZuqc12IVpwY61oUaGg5B1jTIpfYFdVgmTSuCkngvDnLPKTqRbyqs27s9bKDUOoWx6cbP318/UhPdzt0qJYymzcP5CfPmKvtPODBkwnAso7SEeJ23dunaswFHudgneB4P69O7paTJD56KRhPfn8j2M4fl9LO8mUFTSqDX1En07udbwRT09dcbi7/dAlh7jbS245lTMOSP9OM+FoIJmE2KzrTwhsv/CwvVlyy6kpWsy+e6b6kOIxcf0hFV2DzXX5DJ31okJByTv3vfsl+17/Guu3VbKzqoZ9r3+Nix+YxtDrJ/LWZ8ELf3b6nMtRyhw6bNxRxbptdV8QVhdNIdfoI69DtKQo1XzkH/CDnKdFGYSCt/tRVvg6AilXB3tMrHxTl/1nRtq1IFhTcAbGuHvP8Ovnstr4gD6WQ907y/Z+b34TWK7pNaJoILkk9/OHMBfFxM25FZamXDUFpcXw4iwr5HPNlt3urPK/drK4t0Oig/xaRC5Cob54hcJh/buwX6+OWc/ZlaNZwS8UvM/nn92XBZRhDDILOYO6d3APGsyuHRPseM4kDKf99nje+eVxnDB0D7ctaGD1Cjf/fm9wgNdcdup+e/LBtV9Pu5YjFO48/8DQfk39zfEA3HbuAYz/6VF0KCvmw2uP57+/Pi5j+cpc7fNRfBW5xAr4zY3xmLBuqzWRCYsCayRFQYWC0niIpK8BqApZE+BUOXP43Qtz89YvLwa405MRdVd1MmNxFPe4DKawIBz7MUBJPM52z2pdv7+krCg99j4o3t1xtqaYjwKEQljIZ6blGT3Ky+jbta07I4fgQdArCPzmn8GesFtv/yu6tXWL7XjPaV9mCZEDPGG1fvawz2tbUsSwvSzh3bNjGb07t035jv1dzdXnFCQUysuK6Ny2tr5DrmnAvRTFxTVBOuGsfjI5qhsSFQpKo1KVqJtZ6NmA+sb5wmt6Wr5xZyRTQ64+Be8gU1wkbLNNB53aFvOaJ9EeBBdsD5rpOsIgmSIU0o8Lcz7nupI7aBD0ygG/4ErJGOrZF7drT1xzwj68eNWRbvv9F4/kh8cMoE+XaGsD/ARpWGF9y4ajcT37oyO44mv93Xav/ymXS/71WyN48Hu1fo6yojg3f2M/fnB0P0ZV5LaIsKFRoaDkjfXbKpm2eGPKH7O/iMj42atdB9qX67ezcnPuWTxzJcrM38FErA4WlpcnDO9gHRNxawRfdlT6yuaSeDRNwRmgvKGlQcIjzFbvCJXHLjs0U9ddgoSlV1D4116k+BsCjvvJ8YPc2T5YGs3YMUPqPAMvDdCw3HsG5mUKv48jxA/euzNXHGNFLgmpWpnkUCy0V6c2HDekB0cM6ApYWlHfrm357alDQxfgqU9BafZ8Y9x7fPueD1La/DPqXdUJtxbAVY993Cj9yjRL9NdnNkQzNezI0XzkHZjjIm400teH7MGVx6WGS3q1CkdoBD2DM9P3ugYCndQhEUlOZbCBdrqFiw7bO+2YY/ax1lCM3rdHoLD0NvkHWe8nr1yKYt8fvW+PrMf4CXr280f1AYJn9f4Mul5O2782Wsl7rldTiPJ3cmDfTimOZCdhY3uPv8Wv3Tnhu40lFQonNZ/S4gia9QdF6ayxc/zPX7M1bV8+yDQI+aOenMRv2VifY7ST1+kqAiMrurDkllMBGLpXB3550hDOvvt9Zi7dlGLuue60oVx32lAmL0gvquP4a5JJw3cO7cvNZ+0XeO+SouDnOfOAXpx5gFUcx+mLn+G9Orr7glJGe7+rTN9bLINGEcT937VMLRVjx2c91sEvFLzPFHTPspI423z+nC7tSvjod6mhpl7NN5GjUHj+x0emfHaEQnmZRyj4hPbDlx7KkN+9pj4FpWl5d+F6KsaOZ96q+g/U3j/loBq0v3txLv9+f0mjrdjMFKb57/eXpHxOGkOfkBDB1OPC9wWViPQOAmGaixNFFOTkDPquzr33Q577aAUJu8hNGLkmjQsj6HV5xzN/H7yCwLsvV/t+VLxO9s5tU1cJBybrC/hehu2Vno/IqcEwvFfHFKGQa5grQP9u7dP65//7dDSu4RGi4BoC1RSUQJyqYjOWbmRowD9GLhhPqGSYQ/bxacus2VsDVCL713dHUhyPcfEDwXWOMxeh993fwAWj+rJ6yy7GTf4i671vPXt/huxZzhn/+57bNuFnR7NxRxUn3vaO21ZeVhu1EtYfZ0Ya5AMIG0ifmbkiY5nLO847gH2zJF6rD2EDP6QKEe++fNSUcHjy8sNYu62S/XqnDqiZzEf3XzyS7uWlVNYk0xaZgWXqeeqKwxmyZzn73/C655q5C4Xbzh3B/NXb6Nq+duLgN7sVxWM8+6PDGdi9/kkTo6BCQWlwXp+7JsV05AxuiaQJXeT12ZrgQjR14ZB+XejgGXT9ZJqZ+lesJo0hFrNs/lGEwrcP6ZPW1q19Kd3al3LhYX155EOrLGV7j6YQZj5xxFOQUMg0AHmT/PlxzEMNQVBaDMkgFLx4+5/JIVxfDu3fNfj+AX1zNLJu5aWM6BMeBgswKiDNeF30nfKy4rRrBTnWD947OCIpH6hQUBqcyx+emfI54dEUcl3526ltcUoahZF7d2bG0k0ZzsjuuDxucHc6ty2JFOZaO+7V38TxP6P3Yd6qrezZqQ1HDezmtoeNnSaT+ShMkJjg2n57V+0AACAASURBVMc3nD40ZTbaEPijic85uDdXjx5U28cMq4i9/c8UOurn6tGD2LKrmgffW5JbZ314+/LUFYcz6bO1nHdIX+55+wuG11EzFoF7LzqYxRt21KtvAD88ZgD/fDv7JCQfqFBQAnFm9w1h7XXsrjWJ3IXC94/sxz/eqE2Id+lR/Vi+aSdrt4Y7drPZqNuWFPH3b4/IKBSG9+rApyu3us49/yV7dihjzdbUKmbZ7tu1fSnP2Y5Gry06TIi5PoUAH0ioUMCqUeCfCX+vAYv4+Pvn8LdvjUj5nB6SWrudaj6KrilcPXofgHoLBe93PqpfF3e2fsvZ+9f5miLCicN61qtfDmPHDGkyoZBXR7OInCwiC0RkkYiMDTnm2yIyT0Tmishj+eyPUgcawPmb8MTP76rKbYGU33QSt3P4gxUZEkQ2226UYu1jhu8J1ApH/zW/HhAimYu/NCUdRIZZPwSHkGbSFJJJk1Vb+kaOCeGC75XZ/+P0oL9dj8Gb2C+eYj5q/HiXRoppqBfd2pfQrYG1uyjk7W2ISBwYB4wBhgLni8hQ3zGDgGuBI40xw4Cr89UfJTcaMvitdqUt7I6Yj97Bv/q2KC5u38aePITnfnxE2jnOgDk2JMdP0Mzbz7dHWr4B515eofDcj4/guMHpQiGXxUspi7yyOJqjRh+BXQ8iS/QRWLmC6ku2KKb12y1t7pC9u7Do5jHudwp+81Hj1yeui1O4sZn2m9FMs3M7NSb5FNGjgEXGmC+NMVXAE8CZvmN+AIwzxmwCMMakB18rzR5HKFz4r6nszFAHIQh/JEY8FnNnqEVxCQwjdE4J+7ePVK/AHoide3nHkJJ4LGtJ0FwIu5azyKljm3SneWhGbGMJk2yrgOuTp8ehTUnmwdwRXMN6dUjLweTVjjIlrssX+QqDbUhiMcm5Il9DkE+fQi9guefzCsC/fn4fABF5D4gDNxhjXvNfSEQuBy4H6Nu3b146q+QPb9qFTR6ncRT8pp4ij/moKB5Lm0WfP6qPO+CFaTtBSeLS7mtrE0Hmo3hM3FoBJfEYJw7bg1dmrw6UQn/8xvCs9wqbtf71W/vzzYN6MbBHekH3sEHN+a4bY82Hv7Smn3MO7k2bkjin759uqmpq81EzkAlNRlMvXisCBgHHAucD94lIWiyYMeZeY8xIY8zI7t2zlypUGo4XP17JnyfMj1zgIyhM0bvad9PO8OI4QRTHAnwK9nZRTNJSQztpGCA8gViUamSOHd9xpnq7ERNxNZgD+nbigkPDJypHeqKMwggTCuVlxZw4rGfgYruwc5y0Cw2pyYThL63ppyge48wDegXOdr3fZz5DUsNoCE2ppZLPP52VgDdou7fd5mUF8JIxptoYsxj4HEtIKE2MM6DOWLqJe975MnIW0Gw5gFZsyi3hnX9GXBwXV0DFY5JmCvL+s3ujY757eG0enygFU4rjwin79eTf3x8FpA7CIrX9shLmWdtBw0yUGWlWx3iArShM4BlXiGW/8S3f3I/v1yMq6QyP4/j+i0fmdK5XU8imcQTxm1OG8PMT9sn5PCU7+TQfTQcGiUg/LGFwHnCB75gXsDSEB0WkG5Y56cs89qnZM3/1Vgb2aJ/VLr5u624Mtfnm60umhcZLNuygS/sSOpQVu7lcwsg1hts/HsZjsRRNwT/rDzKb/OjYAfz65CH8+4OlQDSfgojwf985uLYf3mghj1BIZlgoZh0bvu/oQd14d+GGrLP6oAE+LA+O05co5qPzRtXPFNuxTXFojqRseL+zDgE+k2xc/rXcaiwr0cmbpmCMqQGuAiYC84GnjDFzReRGETnDPmwi8JWIzAMmA780xnyVrz41d5Zv3MmYO97lj6/My3rsqD9N4tA/TWqwe2cq1Xjs36bwjXFWWgd/gZgg6mPP9foU4jFJ8zl4hciBfS1LpH/FqFcoePMSZVrFKimagrjP4E2tHTQOR7FSRDVlnLr/nu522OtwhEGhR9d4BV15SKH6xmDM8IZZV9CSyOvbMMZMACb42q73bBvg5/aPkgXHHj9zWeYVvQ1D6qhjsiwv+HK9pQFsj1BXYI8OZazesjvrcUFY6xTs6KNYLM0U5B0MjxjQjU+uP5GObVNnok6W0Ll/OIlltqAFePqKw9nnulcD75tiPsJb1zh6NlA/zjWizOo//cNJlHkEYJiQdi7VFFErueCNKmuqvvq/U8VCv5FmhDPANFYFJi9JY3hz3lr2+e2rGU1EUTSFru2DF50F4Y/9L47HUjWFDEIBSBMIzjXAKvLujZHPVIfXO26J1PbK61MIPi+DUHCOifBf2L60KCVqKsw56zqaC1smFIQm4/9OFQv9RpoRzv9RAyQSzYpf8CSMYdyURVQlksxfHZ5O259QLoiu7aKv0nSeuVenNlw7Zgj9u7XzJIqTNHt+lPhz70CQqdqWl9Q6AR4B7bln0OK1TJevz7g4sEd7/v6tEVx4WKpfIJGsdcIXMoXev6hMuuaYpu5Cg6NCoRnhDDre8NBX56xm+pKNKcc9OX2Zu72jsoY7Jy1MqygG8MLHK/l05Rb389MzlrMgJFtp0hh3MN+QoaBMUEiqn7os3R/eqwNXHDOAWCw1+shPlBmo1zmdqbaCF+9lBXHvk/RGH/mc0fbB4de0f9dV8zv74N4pKbgBZq/YYvelsAfdliIUBnRPX0PS3FGh0Ixw/s+9g8iPHv2Ib/0zteTlr5+d427/deIC/vHG59biKh9XPzmL0+76r/v5l8/M5qTb30k7zrln+1LLZJFpAVoiglDo0i73aBPvLNypeeCYe44cWJseOcpYU+IrhRkFr7Dp2bEs5V045h/vlZxBL4pPoT7mwPMP6cteHdMjzAp90C0E85ESjAqFZkStySL6KLJ1tzWAV3s0hRWbdlJVU/t5+cadrN2a2fGbSBoWrttubydJJg3LvtqZdly26wB0Dklklwmv3d0xUTnpHx697DCG9LQKkEQZDL1+iKhOTue771FemuJ7MCZYsDhCLLO/wb5GpB4E07drW96/Nj0/TmNVsasrhS60WjOaOrsZURefgtcpC7BlVzVH/WVyii366Fsnh57ncP+7i5lrl+asThjGTV7E39/4PM2m+rsX52btU6c24UJh8B7lLFhba8Jy1PPDAoqleHMCuam+I5mPctcUxDeAe81H4pqPJO34zGNfujmwoSj06KNCF1qtGdUUmhHeiJcw/Pscc44ziC3faM3uZyzJLax1yue1uQprkkmmLrb8GLmuUAZoUxL+Z/fwZaNSPg/v1ZH3x36diw7bO+3YdgFpFqJpCl6ncUSh4P/sMR9l8m1kElJ+QdOQFLhMiBRxpTQNqik0QzJNLP02/drcPcJ3H5jmrnXo0aEstATmhu2VPDljeUqbsw4B4E8TPnMXfVV7zFAVY8dH6n9QfQCHoIF+r05tAo8NzKmT42KxXAcnfzrrjm2KA7OyRtEUogj5ulLo5plC719rRoVCM8IZOjKtLk6YEKEg8Pbn6932TJkpl21M9xX4WWdHINUkcyuaA5nTTEQZLCb89OjQdBm5OjD993vsB4cGT919lx3QvT1/OGMYY/brSWV1+ncQpR9BgQN15bEfHMoF903N6f5NiQqFwkWFQjPCGeCDxpBk0irB+Oa81JIUbty6b5CozrCewDv7z0ZVIvcRLVOW0iiD2dC9OjDUV0e3tmxmboON//gjBmTLaupJsndEBQArN9smtJQFbtbvTP4fN8Q4SkezcMSAbhzevysffGlliSn0QVd9CoWLWvaaEc6kPGhm6WgIVz72Ueo5IQ7YjEIhh4E+EaIp/OaU4KpnUH9NIQi/Qz2IX540mMF7lKe0NcSMOmiAu/HMYXRsU0y7DBlAG1JT8F4PCt+nUOhCqzWjmkIzwpkNuxqDZzQJMyk5i8lWbU51CFfXhI9E1TmYhDbuCF6zcOYBvWhbUsR1L3yati/TgrF8jhVXHjeQK48bmNKW++AU4McIkHFnHdibsw7snflKrqO5YaRC6qrrwh50Cz06qjWjmkIzwhn3nd9e00TYOO4Iixt9mVUzDfy5mI9uCsnYGhOhd+dgB3GmegaNvRI36tjklP30LpSrvYaT5iI3DrCzsvYKcaTniverK/SZuJqPChfVFJoRfg3BawIK0xTCLEGZzEc1DZBcKR4Tjh3cgwcvOYRLHpyesi+fSchyHWuiCqHysmLeuuYYegUIuroOcJcd1Z9jB/dgH59Jq66kRlUV9qBb6EKrNaNCoRnh1xC8g7c/6sghbNaf0XwUIaldNtyi7Xt2SNsXpRxmrjRG4tj+IXlunLE4Vy0nFpMGEwiQqqkUvPmowPvXmlHzUTOiNvrI+p3wqAEmSWDSu7CspZU14XUPcnE0h+HY2YNmrMXxGK9dfTR/OGNY4Ll3nX9gve/fmDRFKvNsFLp5RjWFwkWFQjMi6fMpeP0CCWOY5suWCvDV9uCMpksC8hY5zAi4Tq5kKgtZHI8xpGcHvntERWDq6tM9tX+bE009DpuU7QKUVB5UJhQuWYWCiLQVkd+JyH3250Eiclr+u6ak40QfWZ9qEqnRR97FSw6ZBv8wnpi+PPtBWXDMA0GaglcQhIWnXnz43nTLoRiPQ1OMNe3LLCvsNQVUSL4hTID5pNBTe7dmovgUHgRmAofbn1cCTwOv5KtTSjC1LgSDMYbtlbXhoFU5RAw1Bq6m4BSgkfQUEWD5F3bZjzH5F8e67TeeOZwbzxwe+X75SBURleJ4rM4F7PNFQ5gAldZJFPPRAGPMrUA1gDFmJ00zIWv1OGsOkgYen7ac0f+orX1wzF/TM502JY7ZKB4QrunVDrwCoj528BF2eKe/6ExrpdA1BQf/YkKl6YmiKVSJSBts24WIDADCS28pecOb++jlT1al7Cu0maFjNgpa2OVdvJZa26Du9/vTWfvxvSMq6BlQcKY1UlNgfw9BTLz6ay3ifU39zfGB+a+aK1GEwg3Aa0AfEXkUOBK4JJ+dUtIZP3s1D763GIDNO6vdHDeFjuNbKCmKsdv+x/EuXkupbVAP72NZcZz9e3eq8/ktAa8JrSHWmuSbwT1bhpawR4fmL9i8ZJ2bGWNeB74JfA94HBhpjCksW0Ur4MrHPmLG0txqIDQmp+zXM7C9OB7j6tGDeP7HR7ptXkdzQ5mPlFROH7FnU3dBaaZEiT6aZIz5yhgz3hjzijFmg4hMaozOKU3LkBxmcqfsFz4IXT16H/b1LGKLh0QfFfoq3ObC708fSmlReCI+RclEqFAQkTIR6QJ0E5HOItLF/qkAejVWB5WmI5ewwVxWqHqve4mdfhpUU2godLWwUh8yaQpXYIWiDrF/Oz8vAv+b/64pTU3QxL1NcTwwLXZdJ/nfPqRP7TVUU2gQ9GtU6kOoUDDG3GGM6Qf8whjT3xjTz/4ZYYxRodDA/Pb5OZHLWTYWQTNOkeAsp1Fmp13bZV6MpqkPGgZdGKbUh6zRR8aYu0RkODAUKPO0/yefHWttPDp1WaPdqzguKSGsb/78aylrHhzCxuiSAHu1Vyi8dNWRafsBXr36aFZv3h3aLzUf1Q8n+EjNR0p9iOJo/j1wl/1zHHArcEae+9Vq8YYVGmO4efw8Fq3b3qD36OBb4DWwRzndy0vTjgubcQbVd/auMQgLDe1RXuYuMvPiZE3VsaxhUIVLqQ9R1imcA4wAPjbGXCIiewCP5LdbrZeqRNKNHFm5eRf3vbuYVz9dU+/rdmpbzOadVj6JMrtgjAhcfbyVrycoS0SQOUdIDSN1iInwhzOG0aFN7tnYn/nhETz/8cpAYaPkjmoKSn2I8h+8yxiTFJEaEekArAP6ZDtJqRvVCUOp/VbWb6u023JbLfnPCw/mh4/MdD+PquhCu9I4kxesB6C02Bp8j92nOz8bPSj0OmEzzkBNQcQtZJ8rI/p0CtQglNxwMqOqTFDqQ5Sp2QwR6QTchxV99BHwQV571YrxJrY76//eByCR4+pUZ9B38KePcBaPpV41/R5nHpAeeXzhYXtTWpzuU2jOTuLiuFDRtW1Td6PBUE1BqQ8ZNQWxjMp/NsZsBv4pIq8BHYwxsxuld62QoGynuaYsKPM5gmMiKf6BIltKeE1GQeaj7xzal3MP6cOg374KwKKbxxCPCdOX1K6sdsxSZcXN1/Tz2U1jmroLDUp9ckgpSkahYIwxIjIB2M/+vKQxOtWaCTIVOb6AqPg1Bf+A7ySk89Z1DhI7IpJSOtOprdy2JF1TaM4raJuzluNFo4+UhiDKnOIjETkk7z1RAKisZ12Eiq5t0zSFpDEpUU1Bg6Cz/9Kj+mW9RxuPUHBqQKuTuHDQdQpKfYjyn3wo8IGIfCEis0Vkjoio+ShP1DcP/tf26Z5mPkjTFGLpmoLD8fv2yHoPr6bgrHdozppCS6OFKD5KExEl+uikvPdCcalLBbWY1FZlSxqTZj5IGpOyWO07h+7N9CWbUoRF0mN6+PGxA1LSAf/zwoNSwmLbFtf+2Th1opuzT6GloOYjpSGIsqJ5aWN0pLXjlKusiqgpXHBoXx6zV0G/cOWRzFq+metfnEvSpM8UE8awvbIGgKeuONzVRlKFgvWhOC786uTU3EYnD9+Tk4fXZkH1mo+ca6imUDioSFDqQ16ndyJysogsEJFFIjI2w3Fni4gRkZH57E8h4/wjV0fUFLz5h2Iirp8gmTRpNuWkwRUKHdoU0beLFX7pNRU5pT7jEUJXnMVrZ4zYy23zO7eVxsd57epTUOpD7stPIyIicWAccAKwApguIi8ZY+b5jisHfgZMzVdfmgPOpD0RsQC9NypIBIrtwbw6YdJyCBlj2L7bEgrlZcX06tSGWdefQMc2tekunPsWRTRIz77hRNoWx3nJLgsalCRPaVxqzUdN2w+leZPP/+RRwCJjzJfGmCrgCeDMgONuAv4ChGdKawU4/9AX/Wsaj09b5s7cw/CXsSwuskaC6kQy0KfQxc5QWl5mzQM6tS1JmVE6t/PWT85Eh7JiiuIx93qa9rpwUJ+CUh+iJMT7pogsFJEtIrJVRLaJyNYI1+4FLPd8XoGvOI+IHAT0McZkzBktIpeLyAwRmbF+/foIt27eXPvcnKwL1rz5h2IirpCoSSbT0hwkk/DgJYdw1/kHpiXDqz0mN03BYcJPj+aeiw7O6Rwlv+jiNaU+RDEf3QqcboyZ35A3FpEY8A+s2s8ZMcbcC9wLMHLkyMKvSJ4D05dsDFycli21RUoZS6n9XFVj0oWCMezRoYzTPT4AP46jOYpPwUufLm3p06XlpIhoCahPQakPUYTC2joKhJWkJs7rbbc5lAPDgSn2H3FP4CUROcMYM6MO92uWfOufwWmksvkWvDZ878rj6kQywNGcXY665iM1AzVbvnvE3qzdtpsB3do3dVeUZkwUoTBDRJ4EXgAqnUZjzHNZzpsODBKRfljC4DzgAs/5W4BuzmcRmYJV5a3VCIRMZFuvkNF85Ds2l9RJLSXlQ2vEHzqsKHUhilDoAOwETvS0GSCjUDDG1IjIVcBEIA48YIyZKyI3AjOMMS/Vsc+tgr+/viDjfu9isbhHKFTXpEuAKJqCg2oKitK6ibJ47ZK6XtwYMwGY4Gu7PuTYY+t6n5bIuws3pLWdO7IP89dsZfaKLZR7HMbi9SkkghzNOQgFDS1VlFZNqFAQkV8ZY24VkbsISKJpjPlpXnvWyjEBeUvjcWFgj/bMXrElJewwFhM3H1F5WRHiMyA54ahRUPORorRuMmkKjnNZbfxNQJDFR4CrjhtI13YlHN6/q9seExjSs5zrTxvK6SP2SjEXjerXhetPGxr5vmo+UpTWTSahMEBERgGPGmNqGqtDisWKTbvS2kSgf/f2/PbU1EHeKaLzfTvt9dqt1jrA7uWlPHXF4TndVzUFRWndZBIKvYHbgSEiMgd4D3gfeN8Ys7ExOqdEw7+CtXPbEtqVxLnu1H1zvpZqCorSugkVCsaYXwCISAkwEjgCuAS4V0Q2G2Oi2ySUQDZsr8x+kAe/r8DBP46XFMWYe+PJdeqTagqK0rqJEpLaBisstaP9swqYk89OtQamLd7It+8JXrgWhj+qqCgm1CTT6yfUB10Nqyitm0zRR/cCw4BtWBlM3wf+YYzZFHaOEp23P1+X8zn+4bqsOM72ypo0YaEoilJXMgWl9wVKgTVYK5JXAJsbo1MtiWdmruCapz5Ja1+9JfeksP5ZvBNq6q2qpiiKUh8y+RROFmsUGoblT7gGGC4iG4EPjDG/b6Q+Nmt+8bQlEP7+7REp7burE6HnHDWwG/9dlL54zc8jlx7Ky7NX0a199HUIYbzyk6OYtVxlvqK0djIuXzUWn2KtSn4VKwJpAFZRHKUehDmNAU7ZLzV/zWhPhTQvfbu25crjBjaIH2B4r45ceNje9b6OoijNm1ChICI/FZEnRGQZ8DZwGvAZ8E2gSyP1r8Vg/KvRfOP4IRWda3f59jmnqu9AUZR8k0lTqACeBg41xgwwxlxkjLnbGPOJMSZaIWHFpdKX9dQ/vj9y2aFZr5FJu1AURWkIMvkUft6YHWnp7K5OUFYcdz/7TT6ZwkrVjawoSmOhKTEbid3VmTWFeAah4OQyUvORoij5RoVCI+GPNvIP8FEK36tMUBQl36hQaCR2+YVCDufmUCNHURSlXqhQyDNOLqF0TSG6WHBkgpqPFEXJNyoU8kxx3BrJ/ZpCwlMNzZ+Z1AlPPefg3gAM7G4VYte8RIqi5JsoCfGUelASj7G7Okmlz9GcyFAic2CPcpbccioAf/vWCO6e8gWgPgVFUfKPCoU8sW7bbkbdPMn9nElT8NKtfWlam1uaU6WCoih5RoVCnvhy/Y6Uz36fQo1HKDhbz//4CHp1bhN6TV28pihKvlGhkCf8w7ejKRhjuHXiAuav3pp2zoF9O6e1Wec0dO8URVGCUaGQJ/xOYWfx2o6qhOsjiMpFh+/NvFVbueJr/Rusf4qiKEFo9FEj8frcNcxavjlFg2hfGk0mdygrZtx3DqJzu/qnyFYURcmECoU84V+gPHXxRr4x7r0U30KRHa6alkFVURSliVChkCfClhR4s6UWx/XrVxSlsNBRqZE54pa33O312yoBzYKqKErhoEKhAFDrkaIohYIKhTyR0DJEiqI0Q1Qo5IlMaSwc+ndv1wg9URRFiY4KhTwRRShcckRF/juiKIqSAyoU8kQigqOg1FOeU1EUpRBQoZAnkhE0hdIi/foVRSksdFTKA9WJJJc8ND3rcWWqKSiKUmCoUMgD23bXRDpONQVFUQoNHZXyQNS0FaVFqikoilJYqFDIA0GRR53aFqe1lRXr168oSmGho1IeqPYJhVP335M7zjsw7biimH79iqIUFnkdlUTkZBFZICKLRGRswP6fi8g8EZktIpNEZO989qexqK5JXc4cFyEekCGvXamajxRFKSzyJhREJA6MA8YAQ4HzRWSo77CPgZHGmP2BZ4Bb89WfxqQmmSoUimJC3JdL+8Yzh9E1oB6zoihKU5LPymujgEXGmC8BROQJ4ExgnnOAMWay5/gPgQvz2J8G49KHpnNwRWd+fOzAtH0/e+LjNJ9CPCZu7QSHPTu2CU2vrSiK0lTk03zUC1ju+bzCbgvjUuDVoB0icrmIzBCRGevXr2/ALtaNSZ+t49bXFgTue3HWKl6ZvTqlLR4TYgESIKhNURSlKSkIT6eIXAiMBP4atN8Yc68xZqQxZmT37t0bt3M5MPXLrwLb4zGhyF+KjfTqbIqiKE1NPs1HK4E+ns+97bYURGQ08FvgGGNMZR77k3fOvffDwPbzR/UNbFdNQVGUQiOfmsJ0YJCI9BOREuA84CXvASJyIHAPcIYxZl0e+9JkLLx5DMN7dUxzNBtj1KegKErBkTehYIypAa4CJgLzgaeMMXNF5EYROcM+7K9Ae+BpEZklIi+FXK7Z4tRh9puPDKopKIpSeOTTfIQxZgIwwdd2vWd7dD7vX0gEJb9TkaAoSqFREI7m1kDbknShoJqCoiiFhgqFRqJtSapSZgzqU1AUpeBQodBIBCW/E5UKiqIUGCoU8siBfTu52+kCoHbV8zUn7NNIPVIURclMXh3NLZGgWgm7qhJpaSwAnv/xkVmvt+SWUxukX4qiKA2BCoUc8ZdK+GT5Zs4c917TdEZRFKWBUfNRjviT3U1fsjHyuZOuOYaRe3cGLEezoihKoaFCIUeSntG8sibBH8fPj3zugO7t6dmxDNDII0VRChM1H+WIV1N4f1FwArxM/P70YXRuW8Lx++7RkN1SFEVpEFQo5EjCoynsrErkfH738lJu+sbwhuySoihKg6HmoxxJJr1CoaYJe6IoitLwqFDIEa/5aFd1qqZw7ODufPS7Exq7S4qiKA2GCoUc8ZqPdvnMRzFJL7upKIrSnFChkCPJpGfbF1YaE6Ekrl+poijNFx3BcsSrKSR9iw3isfS6CYqiKM0JFQo54nU0VyeSKfviMUmrsKYoitKcUKEQkQVrtrFxR1WKo3nt1tSS0iKimU8VRWnWqFCIyEm3v8OYO95JMR89Pm1ZyjGlHn/CmQfs1Wh9UxRFaSh08VoOrN1amWI+8tO9QykAn/9xjPoWFEVplqhQiIA3XXYiQya7wXuUA1BSpAqYoijNExUKEahOhK9NcLjt3BF844BejdUlRVGUvKBT2gDGz17NlAXr3M9PzVjubn//oemB55x1YG91MiuK0uxRTSGAKx/7CKitinbdC5+6+zbtrE47/venD22cjimKouQZ1RQysPSrHeyozJ707pIj+zVCbxRFUfKPagoZOOavUzjYrpSmKIrSGlBNIQszl25q6i4oiqI0GioUFEVRFBc1H9WDZ390RFpSPEVRlOaMCgUPL3y8kiF7lkc+Xv0NiqK0NFQoeLj6yVmRjrvpG8N5edaqPPdGURSl8Wm1QmHTjiricaFDWTEAyzfujHzuRYftzUWH7Z2vrimKojQZrdbRfOBNbzDq5jcBKy320bdObuIeKYqiND2tVdVGZAAACa9JREFUVigA7K62iuQs3rC9iXuiKIpSGLQ681EyaTjvvg/dz4vWbU9JeKcoitKaaXWawpqtu5m2eKP7+e4pX6SV1VQURWmttDqh4HcoT5izmiemL09p69zWcj53KCvilP16NlrfFEVRmprWJxQ27Ur5vKs6kaI5AJQWxeleXsqVxw2kslq1CEVRWg+tTyhECD0Vgem/Hc0Vxwxg/fZKwFq9rCiK0tLJq1AQkZNFZIGILBKRsQH7S0XkSXv/VBGpyGd/AJZvyi4UDupbu1J5eK+OAPTt0jZvfVIURSkU8hZ9JCJxYBxwArACmC4iLxlj5nkOuxTYZIwZKCLnAX8Bzs1Xn8DSFA7o04lfnDiY/3ywhNfnrU3Z/8KVR7q1lgGuP20olxxRQffyUmZcN5oajVRSFKUFk09NYRSwyBjzpTGmCngCONN3zJnAv+3tZ4DjJU81LZ+avpwT/vE2nyzfwoDu7TlqUDf6BMz+9+/VkTYlcfdzWXGcQbaQ6Na+lJ4dy/LRPUVRlIIgn+sUegHesJ4VwKFhxxhjakRkC9AV2OA9SEQuBy4H6Nu3b50606ltMYP2aM8+e5Rz/qg+AFw9ehBrtuwmFhNG79uD9dsqicW0zrKiKK2XZrF4zRhzL3AvwMiRI+tkvzlxWE9OHJYaXlpeVsy47xxU/w4qiqK0EPJpPloJ9PF87m23BR4jIkVAR+CrPPZJURRFyUA+hcJ0YJCI9BOREuA84CXfMS8B37W3zwHeMkar1iiKojQVeTMf2T6Cq4CJQBx4wBgzV0RuBGYYY14C/gU8LCKLgI1YgkNRFEVpIvLqUzDGTAAm+Nqu92zvBr6Vzz4oiqIo0Wl1K5oVRVGUcFQoKIqiKC4qFBRFURQXFQqKoiiKizS3CFARWQ8srePp3fCtlm4F6DO3DvSZWwf1eea9jTHdsx3U7IRCfRCRGcaYkU3dj8ZEn7l1oM/cOmiMZ1bzkaIoiuKiQkFRFEVxaW1C4d6m7kAToM/cOtBnbh3k/ZlblU9BURRFyUxr0xQURVGUDKhQUBRFUVxajVAQkZNFZIGILBKRsU3dn1wQkT4iMllE5onIXBH5md3eRUTeEJGF9u/OdruIyJ32s84WkYM81/quffxCEfmup/1gEZljn3Nnvsqi5oqIxEXkYxF5xf7cT0Sm2v180k7LjoiU2p8X2fsrPNe41m5fICInedoL7m9CRDqJyDMi8pmIzBeRw1v6exaR/7H/rj8VkcdFpKylvWcReUBE1onIp562vL/XsHtkxBjT4n+wUnd/AfQHSoBPgKFN3a8c+r8ncJC9XQ58DgwFbgXG2u1jgb/Y26cArwICHAZMtdu7AF/avzvb253tfdPsY8U+d0xTP7fdr58DjwGv2J+fAs6zt/8J/Mje/jHwT3v7POBJe3uo/b5LgX7230G8UP8msGqWX2ZvlwCdWvJ7xirJuxho43m/32tp7xn4GnAQ8KmnLe/vNeweGfva1P8EjfRCDgcmej5fC1zb1P2qx/O8CJwALAD2tNv2BBbY2/cA53uOX2DvPx+4x9N+j922J/CZpz3luCZ8zt7AJODrwCv2H/wGoMj/XrHqdhxubxfZx4n/XTvHFeLfBFblwcXYASD+99cS3zO1ddq72O/tFeCklviegQpShULe32vYPTL9tBbzkfOH57DCbmt22OrygcBUYA9jzGp71xpgD3s77Hkzta8IaG9qbgd+BSTtz12BzcaYGvuzt5/us9n7t9jH5/pdNCX9gPXAg7bJ7H4RaUcLfs/GmJXA34BlwGqs9zaTlv2eHRrjvYbdI5TWIhRaBCLSHngWuNoYs9W7z1hTgRYTXywipwHrjDEzm7ovjUgRlonhbmPMgcAOLJXfpQW+587AmVgCcS+gHXByk3aqCWiM9xr1Hq1FKKwE+ng+97bbmg0iUowlEB41xjxnN68VkT3t/XsC6+z2sOfN1N47oL0pORI4Q0SWAE9gmZDuADqJiFMx0NtP99ns/R2Br8j9u2hKVgArjDFT7c/PYAmJlvyeRwOLjTHrjTHVwHNY774lv2eHxnivYfcIpbUIhenAIDuioQTLQfVSE/cpMnYkwb+A+caYf3h2vQQ4EQjfxfI1OO0X21EMhwFbbBVyInCiiHS2Z2gnYtlbVwNbReQw+14Xe67VJBhjrjXG9DbGVGC9r7eMMd8BJgPn2If5n9n5Ls6xjzd2+3l21Eo/YBCWU67g/iaMMWuA5SIy2G46HphHC37PWGajw0Skrd0n55lb7Hv20BjvNewe4TSlk6mRnTynYEXtfAH8tqn7k2Pfj8JS+2YDs+yfU7BsqZOAhcCbQBf7eAHG2c86Bxjpudb3gUX2zyWe9pHAp/Y5/4vP2dnEz38stdFH/bH+2RcBTwOldnuZ/XmRvb+/5/zf2s+1AE+0TSH+TQAHADPsd/0CVpRJi37PwB+Az+x+PYwVQdSi3jPwOJbPpBpLI7y0Md5r2D0y/WiaC0VRFMWltZiPFEVRlAioUFAURVFcVCgoiqIoLioUFEVRFBcVCoqiKIqLCgWlVSIiCRGZ5fnJmD1TRH4oIhc3wH2XiEi3+l5HUfKFhqQqrRIR2W6Mad8E912CFXe+obHvrShRUE1BUTzYM/lb7dz000RkoN1+g4j8wt7+qVi1LWaLyBN2WxcRecFu+1BE9rfbu4rI62LVC7gfa2GSc68L7XvMEpF7xKodEReRh8SqLTBHRP6nCb4GpRWjQkFprbTxmY/O9ezbYozZD2tl6O0B544FDjTG7A/80G77A/Cx3fYb4D92+++B/xpjhgHPA30BRGRf4FzgSGPMAUAC+A7WiuZexpjhdh8ebMBnVpSsFGU/RFFaJLvswTiIxz2/bwvYPxt4VERewEpFAVYqkrMBjDFv2RpCB6ziKt+028eLyCb7+OOBg4HpdpGsNljJyl4G+ovIXcB44PW6P6Ki5I5qCoqSjgnZdjgVKzfNQViDel0mVwL82xhzgP0z2BhzgzFmEzACmIKlhdxfh2srSp1RoaAo6Zzr+f2Bd4eIxIA+xpjJwK+xUje3B97FMv8gIscCG4xV8+Id4AK7fQxWgjuwkpSdIyI97H1dRGRvOzIpZox5FrgOS/AoSqOh5iOltdJGRGZ5Pr9mjHHCUjuLyGygEqu0oZc48IiIdMSa7d9pjNksIjcAD9jn7aQ2XfEfgMdFZC7wPlaqaIwx80TkOuB1W9BUA1cCu7AqrzkTtmsb7pEVJTsakqooHjRkVGntqPlIURRFcVFNQVEURXFRTUFRFEVxUaGgKIqiuKhQUBRFUVxUKCiKoiguKhQURVEUl/8HWZEthZf0+MwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "win_array = np.array(win_list)\n",
    "#每100条轨迹取一次平均\n",
    "win_array = np.mean(win_array.reshape(-1, 100), axis=1)\n",
    "\n",
    "episodes_list = np.arange(win_array.shape[0]) * 100\n",
    "plt.plot(episodes_list, win_array)\n",
    "plt.xlabel('Episodes')\n",
    "plt.ylabel('Win rate')\n",
    "plt.title('IPPO on Combat')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第20章-多智能体强化学习入门.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}