From 047f696c30955a15580c131a8d3e2f68b4b15b5f Mon Sep 17 00:00:00 2001 From: ariel Date: Thu, 21 Mar 2024 10:16:39 +0100 Subject: [PATCH] Remove old demo --- .../demos/pong-multiplayer/train-ppo.ipynb | 466 ------------------ examples/demos/pong-multiplayer/utils.py | 61 --- 2 files changed, 527 deletions(-) delete mode 100644 examples/demos/pong-multiplayer/train-ppo.ipynb delete mode 100644 examples/demos/pong-multiplayer/utils.py diff --git a/examples/demos/pong-multiplayer/train-ppo.ipynb b/examples/demos/pong-multiplayer/train-ppo.ipynb deleted file mode 100644 index b5957c0..0000000 --- a/examples/demos/pong-multiplayer/train-ppo.ipynb +++ /dev/null @@ -1,466 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "outputs": [], - "source": [ - "import datetime\n", - "\n", - "import torch\n", - "\n", - "from cogment_lab.envs.gymnasium import GymEnvironment\n", - "from cogment_lab.envs.pettingzoo import ParallelEnvironment\n", - "from cogment_lab.process_manager import Cogment\n", - "from cogment_lab.utils.coltra_utils import convert_trial_data_to_coltra\n", - "from cogment_lab.utils.runners import process_cleanup\n", - "from cogment_lab.utils.trial_utils import concatenate\n", - "\n", - "from coltra import HomogeneousGroup\n", - "from coltra.buffers import Observation\n", - "from coltra.models import MLPModel\n", - "from coltra.policy_optimization import CrowdPPOptimizer\n", - "\n", - "from cogment_lab.actors.nn_actor import ColtraActor\n", - "\n", - "from tqdm import trange\n", - "import matplotlib.pyplot as plt\n" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:12.885044Z", - "start_time": "2024-02-16T22:24:11.469627Z" - } - }, - "id": "ae18bdf6be9b04d6", - "execution_count": 1 - }, - { - "cell_type": "code", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "logs/logs-2024-02-16T23:24:12.885158\n" - ] - } - ], - "source": [ - "logpath = f\"logs/logs-{datetime.datetime.now().isoformat()}\"\n", - "\n", - "cog = Cogment(log_dir=logpath)\n", - "\n", - "print(logpath) " - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:12.888832Z", - "start_time": "2024-02-16T22:24:12.886164Z" - } - }, - "id": "57591f82d1bf34bb", - "execution_count": 2 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "True" - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Launch an environment in a subprocess\n", - "\n", - "cenv = ParallelEnvironment(env_path=\"pettingzoo.butterfly.cooperative_pong_v5.parallel_env\",\n", - " render=True,\n", - " make_kwargs={\"bounce_randomness\": True, \"cake_paddle\": False})\n", - "\n", - "# cenv = ParallelEnvironment(env_path=\"utils.PongEnv\",\n", - "# render=False)\n", - "\n", - "await cog.run_env(env=cenv, \n", - " env_name=\"pong\",\n", - " port=9011, \n", - " log_file=\"env.log\")\n" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:14.990845Z", - "start_time": "2024-02-16T22:24:14.167795Z" - } - }, - "id": "841ec2e8b0e0433f", - "execution_count": 3 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "Discrete(3)" - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cenv.env.action_space(\"paddle_0\")" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:14.998795Z", - "start_time": "2024-02-16T22:24:14.991362Z" - } - }, - "id": "a4fb51af6f5c8e59", - "execution_count": 4 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "True" - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from coltra.envs.spaces import ObservationSpace\n", - "import numpy as np\n", - "from coltra import DAgent, CAgent, Observation\n", - "from cogment_lab.core import CogmentActor\n", - "from cogment_lab.actors.nn_actor import ColtraActor\n", - "from coltra.models import MLPModel, BaseModel, ImageMLPModel\n", - "\n", - "from utils import ColtraImageActor, FloatImageMLPModel\n", - "\n", - "# Create a model using coltra\n", - "\n", - "model = FloatImageMLPModel(\n", - " config={\n", - " \"hidden_sizes\": [64, 64],\n", - " }, \n", - " observation_space=ObservationSpace(image=cenv.env.observation_space(\"paddle_0\")), \n", - " action_space=cenv.env.action_space(\"paddle_0\")\n", - ")\n", - "\n", - "# Put the model in shared memory so that the actor can access it\n", - "model.share_memory()\n", - "actor = ColtraImageActor(model=model)\n", - "\n", - "\n", - "await cog.run_actor(\n", - " actor=actor,\n", - " actor_name=\"coltra\",\n", - " port=9021,\n", - " log_file=\"actor.log\"\n", - ")" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:20.273961Z", - "start_time": "2024-02-16T22:24:18.456459Z" - } - }, - "id": "4caf921e07e8b41", - "execution_count": 5 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "{'pong': ,\n 'coltra': }" - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cog.processes" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T17:26:39.515796Z", - "start_time": "2024-02-16T17:26:39.511987Z" - } - }, - "id": "ef047d1e0d9eae7", - "execution_count": 6 - }, - { - "cell_type": "code", - "outputs": [], - "source": [ - "ppo = CrowdPPOptimizer(HomogeneousGroup(actor.agent), config={\n", - " \"gae_lambda\": 0.95,\n", - " \"minibatch_size\": 128,\n", - "})" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T17:26:39.722372Z", - "start_time": "2024-02-16T17:26:39.517711Z" - } - }, - "id": "a8387d924a30031", - "execution_count": 7 - }, - { - "cell_type": "code", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "mean_reward: 57.7: 70%|██████▉ | 1391/2000 [4:37:09<2:01:20, 11.96s/it] \n" - ] - }, - { - "ename": "ValueError", - "evalue": "No samples provided", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mValueError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[8], line 15\u001B[0m\n\u001B[1;32m 5\u001B[0m episodes \u001B[38;5;241m=\u001B[39m []\n\u001B[1;32m 7\u001B[0m trial_id \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m cog\u001B[38;5;241m.\u001B[39mstart_trial(\n\u001B[1;32m 8\u001B[0m env_name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpong\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 9\u001B[0m session_config\u001B[38;5;241m=\u001B[39m{\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mrender\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;28;01mFalse\u001B[39;00m},\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 13\u001B[0m },\n\u001B[1;32m 14\u001B[0m )\n\u001B[0;32m---> 15\u001B[0m multi_data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m cog\u001B[38;5;241m.\u001B[39mget_trial_data(trial_id\u001B[38;5;241m=\u001B[39mtrial_id, env_name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpong\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 16\u001B[0m \u001B[38;5;66;03m# data = multi_data[\"gym\"]\u001B[39;00m\n\u001B[1;32m 18\u001B[0m all_data \u001B[38;5;241m=\u001B[39m concatenate([multi_data[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpaddle_0\u001B[39m\u001B[38;5;124m\"\u001B[39m], multi_data[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpaddle_1\u001B[39m\u001B[38;5;124m\"\u001B[39m]])\n", - "File \u001B[0;32m~/projects/air/cogment-lab/cogment_lab/process_manager.py:516\u001B[0m, in \u001B[0;36mCogment.get_trial_data\u001B[0;34m(self, trial_id, env_name, fields, use_tqdm, tqdm_kwargs)\u001B[0m\n\u001B[1;32m 513\u001B[0m env \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39menvs[env_name]\n\u001B[1;32m 514\u001B[0m agent_specs \u001B[38;5;241m=\u001B[39m env\u001B[38;5;241m.\u001B[39magent_specs\n\u001B[0;32m--> 516\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m format_data_multiagent(\n\u001B[1;32m 517\u001B[0m datastore\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdatastore,\n\u001B[1;32m 518\u001B[0m trial_id\u001B[38;5;241m=\u001B[39mtrial_id,\n\u001B[1;32m 519\u001B[0m actor_agent_specs\u001B[38;5;241m=\u001B[39magent_specs,\n\u001B[1;32m 520\u001B[0m fields\u001B[38;5;241m=\u001B[39mfields,\n\u001B[1;32m 521\u001B[0m use_tqdm\u001B[38;5;241m=\u001B[39muse_tqdm,\n\u001B[1;32m 522\u001B[0m tqdm_kwargs\u001B[38;5;241m=\u001B[39mtqdm_kwargs,\n\u001B[1;32m 523\u001B[0m )\n\u001B[1;32m 525\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m data\n", - "File \u001B[0;32m~/projects/air/cogment-lab/cogment_lab/utils/trial_utils.py:359\u001B[0m, in \u001B[0;36mformat_data_multiagent\u001B[0;34m(datastore, trial_id, actor_agent_specs, fields, use_tqdm, tqdm_kwargs)\u001B[0m\n\u001B[1;32m 356\u001B[0m actor_data \u001B[38;5;241m=\u001B[39m {}\n\u001B[1;32m 358\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m actor_id, samples \u001B[38;5;129;01min\u001B[39;00m actor_samples\u001B[38;5;241m.\u001B[39mitems():\n\u001B[0;32m--> 359\u001B[0m actor_data[actor_id] \u001B[38;5;241m=\u001B[39m \u001B[43mextract_data_from_samples\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 360\u001B[0m \u001B[43m \u001B[49m\u001B[43msamples\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mactor_agent_specs\u001B[49m\u001B[43m[\u001B[49m\u001B[43mactor_id\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfields\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mactor_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mactor_id\u001B[49m\n\u001B[1;32m 361\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 363\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m actor_id, reward_samples \u001B[38;5;129;01min\u001B[39;00m actor_reward_samples\u001B[38;5;241m.\u001B[39mitems():\n\u001B[1;32m 364\u001B[0m actor_data[actor_id]\u001B[38;5;241m.\u001B[39mrewards \u001B[38;5;241m=\u001B[39m extract_rewards_from_samples(reward_samples, actor_name\u001B[38;5;241m=\u001B[39mactor_id)\n", - "File \u001B[0;32m~/projects/air/cogment-lab/cogment_lab/utils/trial_utils.py:163\u001B[0m, in \u001B[0;36mextract_data_from_samples\u001B[0;34m(samples, agent_specs, fields, actor_name)\u001B[0m\n\u001B[1;32m 161\u001B[0m sample_count \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlen\u001B[39m(samples)\n\u001B[1;32m 162\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m sample_count \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[0;32m--> 163\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNo samples provided\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 165\u001B[0m cog_observation_space \u001B[38;5;241m=\u001B[39m agent_specs\u001B[38;5;241m.\u001B[39mget_observation_space()\n\u001B[1;32m 166\u001B[0m observation_space \u001B[38;5;241m=\u001B[39m cog_observation_space\u001B[38;5;241m.\u001B[39mgym_space\n", - "\u001B[0;31mValueError\u001B[0m: No samples provided" - ] - } - ], - "source": [ - "all_rewards = []\n", - "\n", - "for t in (pbar := trange(2_000)):\n", - " num_steps = 0\n", - " episodes = []\n", - "\n", - " trial_id = await cog.start_trial(\n", - " env_name=\"pong\",\n", - " session_config={\"render\": False},\n", - " actor_impls={\n", - " \"paddle_0\": \"coltra\",\n", - " \"paddle_1\": \"coltra\"\n", - " },\n", - " )\n", - " multi_data = await cog.get_trial_data(trial_id=trial_id, env_name=\"pong\")\n", - " # data = multi_data[\"gym\"]\n", - " \n", - " all_data = concatenate([multi_data[\"paddle_0\"], multi_data[\"paddle_1\"]])\n", - " all_data.observations = all_data.observations.reshape(all_data.observations.shape[0], -1).astype(np.float32) / 255.\n", - " all_data.last_observation = all_data.last_observation.reshape(all_data.last_observation.shape[0], -1).astype(np.float32) / 255.\n", - "\n", - " # Preprocess data\n", - " record = convert_trial_data_to_coltra(all_data, actor.agent)\n", - "\n", - " # Run a PPO step\n", - " metrics = ppo.train_on_data({\"crowd\": record}, shape=(1,) + record.reward.shape)\n", - " \n", - " mean_reward = metrics[\"crowd/mean_episode_reward\"]\n", - " all_rewards.append(mean_reward)\n", - " pbar.set_description(f\"mean_reward: {mean_reward:.3}\")\n", - " \n", - " if t % 100 == 0:\n", - " state_dict = model.state_dict()\n", - " torch.save(state_dict, f\"models/model_{t}.pth\")" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:03:49.789653Z", - "start_time": "2024-02-16T17:26:39.723981Z" - } - }, - "id": "426fd254873b8466", - "execution_count": 8 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "[]" - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFaklEQVR4nO3dfXwU5aEv8N9u3oEkQDAJkQRS5YgIpQiCEdtazSkqtb5w6sFLPVa9crRQRS0qt4Ue8SVoW6V4ENRr0d4jUr1XsVqF0oCoxxAggAhqwIIQxCS8mGwIkLd97h8xy85mNpmZfWbmmd3f9/NJm+zOy7MjM/Pb5218QggBIiIiIoX43S4AERERUSQGFCIiIlIOAwoREREphwGFiIiIlMOAQkRERMphQCEiIiLlMKAQERGRchhQiIiISDnJbhfAimAwiEOHDiEzMxM+n8/t4hAREZEBQgg0NTWhoKAAfn/PdSSeDCiHDh1CYWGh28UgIiIiC2pqajBkyJAel/FkQMnMzATQ+QGzsrJcLg0REREZEQgEUFhYGLqP98STAaWrWScrK4sBhYiIyGOMdM9gJ1kiIiJSDgMKERERKYcBhYiIiJTDgEJERETKYUAhIiIi5TCgEBERkXIYUIiIiEg5DChERESkHAYUIiIiUo7pgPLee+/hqquuQkFBAXw+H1atWqV5XwiB+fPnY/DgwcjIyEBpaSn27NmjWebYsWOYPn06srKy0L9/f9x66604fvx4TB+EiIiI4ofpgNLc3IwxY8ZgyZIluu8//vjjWLx4MZYtW4bKykr07dsXkydPxqlTp0LLTJ8+Hbt27cLatWvx1ltv4b333sOMGTOsfwoiIiKKKz4hhLC8ss+H119/Hddccw2AztqTgoIC3HvvvfjlL38JAGhsbEReXh5eeOEFTJs2DZ9++ilGjhyJzZs3Y/z48QCA1atX48orr8TBgwdRUFDQ634DgQCys7PR2NjIZ/EQERF5hJn7t9SHBe7btw+1tbUoLS0NvZadnY2JEyeioqIC06ZNQ0VFBfr37x8KJwBQWloKv9+PyspKXHvttd2229LSgpaWltDfgUBAZrHJg9Z9VoePD3b+O/D5gOG5/fCf6z/HsJy+GD9sAJL8Prz50SFs/uJrfHf4IJyRmYaMlCTsONiI//ndYgDAa1u/xNYDX2P80AEYNqgvMlKS8PdP69AvLRmpyX4cbmrBGZlpONHagXFDB+Do8Vbk9EsNlaG+qQVfN7finPzen8oZCyGAY83afUf65FAA1XVNyM1Mw6SzBwEAao6dRHswiOJBfW0tH5FX7D3cjPQUPwr6Z7hdFGyoPozUZD9Kzspxuyi6zuyfgVsmFcPv7/2hfnaRGlBqa2sBAHl5eZrX8/LyQu/V1tYiNzdXW4jkZAwcODC0TKSysjI8+OCDMotKHnfLC1t0X991KIC/fvyV5rX39xzR/H3Xyu2av9dXHwaqD+tu7x+HmwEAOw42Ri3Lh/842ltxHdNwog2767T9ud6N8tmIyH2f1Ta5XYSoxhYNwLihA1zbv9SAYpe5c+finnvuCf0dCARQWFjoYokoUSX7ffj3738LXzWcwmvbvgQATBg2EBcU23MS7z96Am/t6Axc+VnpmDruzG7LbNx7DFX7v9a8Nvm8PKzZVQcAuOScM3BeAZtCKbHtONgY+rJy5eh8V2sWl6z/R+j3woEZ+PGY3rs2OOnPm2tw5HgrTrS2u1oOqQElPz8fAFBXV4fBgweHXq+rq8N3vvOd0DL19fWa9drb23Hs2LHQ+pHS0tKQlpYms6jkYTF0m4pZarIfcyaPwJYvjoUCyqSzB+Gu0uG27O/d6vpQQDlzQAbmTB7RbZnfrvmsW0C5+jtnhgLK5eflY9qEIlvKR+QVL374RSigXDd2CEpH5vWyhn3e/rgW+4501s6OyM/SPa/dtO6zwzhyvBUuXmoBSJ4Hpbi4GPn5+SgvLw+9FggEUFlZiZKSEgBASUkJGhoaUFVVFVpm3bp1CAaDmDhxosziEMWVaC3BPp13wl/xudeETEQe5nI+MV+Dcvz4cXz++eehv/ft24ft27dj4MCBKCoqwuzZs/Hwww9j+PDhKC4uxrx581BQUBAa6XPuuefi8ssvx2233YZly5ahra0Ns2bNwrRp0wyN4CFyO9UD2pu+nQHAx3RBRA5T5apjOqBs2bIFP/jBD0J/d/UNuemmm/DCCy/gvvvuQ3NzM2bMmIGGhgZcfPHFWL16NdLT00PrvPTSS5g1axYuu+wy+P1+TJ06FYsXL5bwcYjs5eaJGy2r6L2uCVDKXG6I3OPUl4p40HV83GxOBywElEsuuaTHQvt8PixYsAALFiyIuszAgQOxYsUKs7smAuB+tWMnn85vdu4letDo9VVejImU4ovyu2rcvtbyWTxEXhG1E4pOHxTmEyKySJUaJgYU8hw3qx27+oQ41wfFvm0TJRJ2HLcgnkbxEJF9oo/i6fk1drQlUozi/WFU6bfGgEKe43a7KOBcAAi/UJjrJOtMHxkiij+hTrIuX20ZUIhMcHUUT9TusJwHhahXmtDOk8IIt6d0YEAhUhiHRhKR01S51DCgkOe4muq/OXNV6tfR6zwo6hSViGBs+gBXfXPRYA0KERkStQ9KL8sqeQEkcpgv6h+kKgYU8hy3O24BzvXxMDRRm14NioHOtUTkPhWuZ5G6Lhlul4wBhcgEFae6J6LecfJC89ye6p4BhTzH7XZRwMEmFAOb1u0PwyswkbJ8io8oUuXLEAMKURziRG1E3sAmnugYUIgscKqPh3Y/JvqgcKI2Ig0j5xKphQGFyCOiT3XPidqIvET1YcY+DjMmssbtkwZwrsMdwwURuYedZInIAHPP4gn7XcFvaEROU3UUj8p9UNzGgEKeo9oJ7dw8KL0vc/o1zoNCRNaEHhbIJh4iMsJcJ9mw320qDxFZ45UaTre/CjKgkOe4meq7QoJTFxiOxiGSgx3HjVMlNDGgEJng9syKejiKh8i7VGuyDuf25Y4BhTxHhdPZsXlQDDyVWPd1PhmNiKxS5JLBgEJkgrsTPBnfNzvJEmmp1O9Dc36qkgbCnJ5JlsOMiUxxs5mla99O3fTZVEMU39wOAT1hEw8RxUSvVoejeIjIKlW+DDGgkOe4GeqdbuIxEjR664LC544QqdXsqVJzk56uMrldt8OAQmSBtvOqMxcYczPJcngyEXkbAwp5jtvtos5SuzMdEcVGxT4op2eSZSdZIs/R9sJ3l24Tj4HhyUQJhf2yPIcBhUhhxuZB4URtRF6lYs2oKtcMBhTyHlenutf+f+TvTuzbyOuqd8IjInWFOslymDGRdzh9wmqfZsygQWQVJ1c2z+3+MQwo5DlunzRAZHBwYae9vswGdyJVqT70X5XiMaAQmaDKiavBidqIyAZs4iEyyc2TpmvfTs2DYmQ+E07URtQ77bmkzjmhQo2wqhhQiDyCQYOInNB1rWENCpFJ7k51H/pN5zUb9hfld80ynEmWyFO80vnd7bodBhSiOMR5UIi0eE4Yp8rhYUAhz3F7+mXAuU6oVudb4TwoRN6gch8Ut6+1DChEHmEmZqj05FYiFXBkm3GqXDMYUMhzVPi+oZ30ycZRPJqgochVg4hionoNZ1eJ3L7WMqAQxSF+WySimHEUD5E5CnRBcWyUjJSgwYRC5NjcRfFAlePDgELkFewkSxQXFLn/R3W6iYedZIlMcfukAdwZssigQUROcru2mgGFyCNMDTPmKB4iDVXPCRW+cEVS5fgwoJD3qHc+28bqhYKdZIkoVm5fahlQiCxwo4+HqXlQ2CGQSFma2hwlv0KoUSYGFPIcV5/F4/j+1KyWJvIa1ioa13Ws2AeFyIPcCA5mvmkx2BB5g4p9ULq4XTYGFPIct1O9k9gHhYicpso1gwGFyAI3AoC5UTzW1iNKBG6fE16Zp8jtL4MMKEQKs/o0YyJSV/iN3+1mFD2qXGsYUMhzVDuhnTuZTfRB0T7NUHpJiMg6VQJANF21Om5faaUHlI6ODsybNw/FxcXIyMjAWWedhYceeggiLDIKITB//nwMHjwYGRkZKC0txZ49e2QXhcjzrFf/spMsUTjtcHt3TwptSRQ+QV1u45EeUB577DEsXboU//mf/4lPP/0Ujz32GB5//HE89dRToWUef/xxLF68GMuWLUNlZSX69u2LyZMn49SpU7KLQ3HI7XZRwKV5UCw/i4eIyDhVvtQky97ghx9+iKuvvhpTpkwBAAwbNgwvv/wyNm3aBKCz9mTRokX49a9/jauvvhoA8Kc//Ql5eXlYtWoVpk2bJrtIRJ7FoEEU31Rrsg7ndsmk16BcdNFFKC8vx+7duwEAH330ET744ANcccUVAIB9+/ahtrYWpaWloXWys7MxceJEVFRUyC4OxSG3TxogorrYqXlQLI/iYbQh4sg241SZqE16DcoDDzyAQCCAESNGICkpCR0dHXjkkUcwffp0AEBtbS0AIC8vT7NeXl5e6L1ILS0taGlpCf0dCARkF5tISVbbqsNDCa/FRIrRnJ/qnaGqlEl6Dcorr7yCl156CStWrMDWrVvx4osv4ne/+x1efPFFy9ssKytDdnZ26KewsFBiiclrhIuxvuvGr053O338tkhEsXLzWgvYEFDmzJmDBx54ANOmTcPo0aNx44034u6770ZZWRkAID8/HwBQV1enWa+uri70XqS5c+eisbEx9FNTUyO72ESGuBuOXNs1kecp1Z8rfFSrEo3WEVw/QJ2kB5QTJ07A79duNikpCcFgEABQXFyM/Px8lJeXh94PBAKorKxESUmJ7jbT0tKQlZWl+aHE5Xa7KODc04KtXlS9MlMlEamn64rh9qVWeh+Uq666Co888giKiopw3nnnYdu2bXjiiSdwyy23AOi8mM+ePRsPP/wwhg8fjuLiYsybNw8FBQW45pprZBeHSCo3O5ya2TcfFkikMMX7oHRx+8ug9IDy1FNPYd68efj5z3+O+vp6FBQU4N///d8xf/780DL33XcfmpubMWPGDDQ0NODiiy/G6tWrkZ6eLrs4RFJ1NfFoAoCte7S2dYYSIi1taHf5BFG8icf14/MN6QElMzMTixYtwqJFi6Iu4/P5sGDBAixYsED27omIiEgCt6MTn8VDnuNmtaPT3yxkPCxQkS9DRK5SqpOs4k08qpSIAYXIAjeeMmxuHhRr6xERnZ6oLc6GGRPZTcU2W7tYflSgj51kiZSleB8UVTCgEFngxkRo1qe6l14UIs/hOWGcKoeHAYU8x+2hb07ilPVEcUjxPihd3L7WMqAQmeDr9otzFxhTNSjsg0JEFqkyzJgBhTzHzVDv9L6tPpWYE7URaSkV2hXvg3J6Jll2kiXyHNUDgFJDKonIk9jEQ2SSqw/sc3p/DBpE8Uf1PiiKFIkBhciErmjkSq0JR/EQxUChWk/Fm3i6uF0yBhQihWmf+WMxoajydYiIPKHrWsMmHiKT3DxnfBH/ryrV+8gQJTTFm3hUuWYwoBApzI0p9YniEc8f89xufmJAIc9xu9oRcGeeADN7ZOdaIoUp3gdFlWsGAwqRR1if6l6Vyw0RecHphwW6Ww4GFPIg979xqH7LZygh0lJqZJvifVBUwYBCpDClZr8kooSgyrWGAYU8x+1qR8Cdb2BWm3iIiKxwc1JMgAGFSGlWn2bsehU2kWJ8bFYxTJXrBwMKeY4CFSjusPiwQCIiM9hJlsjDGACIyDLFhxl3cbtkDCjkOW6neif5ovxuakUiUmsUj/LUOEAMKERWqN5JVo3rCxHp8Uh/GLe/DDKgkOe4WSXq9I3f6jBjdS95RKS6UB8UTnVP5B1d3yhYQ0HkLUo910rxPihuH54uDCjkOW5XOzrJ6lOJOZMsEcXK7WstAwqRCV33fTdu/6bmQbGtFEQUM8X7oKjy/YYBhTzH7VTvJKvV0qpcYIhUwcdGmOf2pZYBhcgC1ZtQeAEmIqtC1w9OdU9kjoqdyuyinbuBoYPIKqv9uRKRKseHAYXIAkXO36hUucAQkXe5/VWQAYU8J5H6oECloZFEJAeHGRvCgEJkgephQfXyETnOp/sr6ehqTnb7yyADCpHCNO3mvKwSxQfFhxl3cbt2hwGFyBTfN//r/EXF1DBjhS96RERGMKAQmeLsNwqfxWppNvEQaSn1NGPF+6B0YRMPxaXGE204erzFlm27fdIA7lzgXL+oElFCOP2wQHcxoEjy1o5DqPjHUbeL4Zjq2ib818b96Ah2/ycshMCYBX/DuIf/jhOt7brrbz3wNf5f1cFur59s7cCLH36BmmMnpJdZDmdTAjMJURxSvA+KKmVKdrsA8WDfkWbMWrENAPDFwikul8YZkxe9BwBISfLhXy8o0rwXnlkONZzE2bmZ3da/7ukPAQBFOX1wwbCBodd/97dqPP/BPix85zN8+tDluvtWuUrUDcWD+nZ7zR92AcxK52lOpJ3oUI0bsOrcrq1mDYoEXzWedLsIrtlxsDGm9fcdadb8/d+fHwEAnGzriGm78cJn4JvWRWcNwpzJ52heS/L7sPxnF2Dp9POR0y/N1jISUXxRpTmZAYU8x+1U75aeLhozf3B2t9d+MCIXV4webGOJiCieuV1bzYBCZIFT3zAU+SJD5HlKjeJRXOjwsImHvEzvRBc2V3EkaAUKHxZIFC8UH2asyqWGAYXIAqd6uVudB4WIKFZuRycGFAlUGZKlCrv/UdtdQ6MqVb7VEHmRUmFf9WHGilxsGFCILFDk/CUiso3bXwYZUMhzEqn+RPuwQCKKC6r3Qfnm/92urGZAoZjoVU+6/Y/aCW6EBVWqXYm8SBP2eS71TJHDw4BCnuNmAHL8uhbebq7IRYOIYqR4H5Qubn/XZEAh6VSsspSN38CIKF51hSa3a8MZUCTgvcpp8R+Auig18oDIw3guGafKPY0BhaRzO3U7wZXzV5WrBhElBLdrwxlQKCZu3DMTIQB14fNXichpqlxrGFCILHAjmLEChcg6pZ7Fo/gw4y5ufxm0JaB8+eWX+OlPf4qcnBxkZGRg9OjR2LJlS+h9IQTmz5+PwYMHIyMjA6WlpdizZ48dRaE4pO7pLJ/PI739iSh+uB7gviE9oHz99deYNGkSUlJS8M477+CTTz7B73//ewwYMCC0zOOPP47Fixdj2bJlqKysRN++fTF58mScOnVKdnGIbOHGKB5VLhpEnqTpJOvyyaT4Fw9VypQse4OPPfYYCgsLsXz58tBrxcXFod+FEFi0aBF+/etf4+qrrwYA/OlPf0JeXh5WrVqFadOmyS4S2Ujvn7Hb1YLxRI3LBBElorib6v4vf/kLxo8fj5/85CfIzc3F2LFj8dxzz4Xe37dvH2pra1FaWhp6LTs7GxMnTkRFRYXuNltaWhAIBDQ/qnL7P2giSNRDzLBCFCcU74OiSm2t9ICyd+9eLF26FMOHD8eaNWtwxx134M4778SLL74IAKitrQUA5OXladbLy8sLvReprKwM2dnZoZ/CwkLZxY5J+H/LRL15hlPxhPMqzdwNilw0iLxIO9W9iwXxELev5NIDSjAYxPnnn49HH30UY8eOxYwZM3Dbbbdh2bJllrc5d+5cNDY2hn5qamokljh2bv9HTDSJWkulSrswEcVI+T4ondy+1EoPKIMHD8bIkSM1r5177rk4cOAAACA/Px8AUFdXp1mmrq4u9F6ktLQ0ZGVlaX5UlZi3zsTh/KN41Lt4EVGcU6SKSXpAmTRpEqqrqzWv7d69G0OHDgXQ2WE2Pz8f5eXlofcDgQAqKytRUlIiuziO0DbxJFZE0RvNYvchcPMIu/pfV41rBpEnKXLP9RS3m+ulj+K5++67cdFFF+HRRx/F9ddfj02bNuHZZ5/Fs88+C6DzhjZ79mw8/PDDGD58OIqLizFv3jwUFBTgmmuukV0cIk/j80OIyGmqXGukB5QLLrgAr7/+OubOnYsFCxaguLgYixYtwvTp00PL3HfffWhubsaMGTPQ0NCAiy++GKtXr0Z6errs4jgusepP9Nl9DNyspHLzxOUTlInISW43CEgPKADwox/9CD/60Y+ivu/z+bBgwQIsWLDAjt27yu3/oGQv/ucl8iZOdW9c1/Fxu2R8Fg95joontBPcvqYSUWJQpXO+LTUoiSxRb57h4rmjsOOjeDgPClH8UXyY8TVjCzC2qD8GZ7vb7YIBhbwnfvNPjxhQiKzTPHiTJ1OPhub0xdCcvm4Xg008ssVx5YFhPATyaGa/VPCbFhFZoHgfFFUwoEjANO6sRD2d+c+MiBIJAwrFRO+myVokIlKNUnMKKd4HRRUMKJLx5mw/V+dBcfhawloTIkpUDCiSsT0xvrkbjphWiChxMKCQfLY/iydxQqAvyu9EZI5SE7WRIQwokrGJJ765eWHjRZWIEgkDigThN45Eyyd6HbzsruFwMwQ6vW826xDFIQ4zNoQBhcgj2NufyDrtKB6eS17AgCJZPE/zbpTdh8DNI+z4KB4X901ENuEwY0MYUIg8gpcxIkokDCiSsf6EZOLDAolkCX8Wj4vFIMMYUCRLtBYe3Zlkbd5n4jaj8apKRImDAYVIYRzFQ0SJigFFtkT9ch/G7hqORD3EzCpE1in1LB4yhAFFgvB/7Ik2pp0nunN4rIkokTCgkHS2R7TEyoAhbO4hss4X9Q9SFQOKZAnbf5OIiEgiBhTJmE/sl2jNaF34pY8oTnCqe0MYUEi68Fok1ijJwxYeIut8nL3VcxhQJEvcOTqck6iHmAGFKE4wLBnCgCJZot079SdqC6++jC+8mBAROYMBhTwnYWtQGI6ILOODN72HAUWC8Ptlot48NdgHxR68qBJRAmFAIc9J1MzDfEJEiYQBRQLNqJWEvX2S3ThRG5EcPJO8gQFFtgTLJ3o3TU2Tlw0HJFFHSvGiSkSJhAFFgkS9YRIREdmFAUUyRhX7J2pL1GPMFh4iOdhc6g0MKBIk8igenubO4TBjojjBqe4NYUAh6TQTtdlRg5Kg5zO/9BFRImFAkYCjeMgJzCdEcrh+LnGqe0MYUCRL1G/3znLvILMWg4jIGQwoEujVmhgd2aO3nBBC2ZFBRsrVW42S3jaCQYENuw/jWHNrTOWzm6v/WRiOiCiBMKDIEHbTeuTtT/HOx19hzIN/w/rq+h5X+2DPEYx58G/4646vNK/f8sJmTFn8ATZ/cQz/8ZddaDrVJqWYb2z/Eov+vtty+Pnf7+/F+Q+txe66ptBrz7y3F29+dCj092e1Acx/Y2fo76r9X+M//rILv1tTjVe21HTb5vrPOo/RK1tqcNMfN6G+qSX03r8s/RCf1QYAAPuONGPeqp04+PUJS2WPB6wKJpKDNaHekOx2AeLNX3d8FQocNy/fjC8WTom67E+frwQAzFyxFVO+fXq59dWHAQA/WVYBoLPG4cGrR8VctrtWbgcAXHz2IIwfNtD0+g//9VMAwLxVOzWv/+LlbbhqTAEA4PJF72vem//GLs3f148v1Pz9zs5aAMCaXbXd9rdl/9e4fNH7+GLhFFz/TAUON7Vg8xfHcNdlw02XXRa9C9vIwVmO7HvYoD6O7IeISAUMKBLYXev/j8PNUrd3NMZmlFi/fehV4PQ2L8Hhb2pWPqttQtDhZpZ7//mf8Pu1u7u9vmHOJagLtGB4Xqat+y+/9/toPNmGwdkZtu6HiEglbOJJQLH2o3C7e4zTI6UuOSdX9/WhOX0xodh8TZRZZ53RD+cXDbB9P0SJgs2l3sCAIoHbN2wvEUI/Xpi5XPB4ExHFPwaUhOTeHT5a84yZZqOgwwmFHeqIiJzHgCKB1yZni7mJJ4Z1O4LqDqEmogTh9pcOTnVvCAOKYuL95h299sP4FSPODxEREYEBRQqZN0wnbr6x7iKWLx/RAoqZZhR+4yCiWLjebMup7g1hQElArjfxxLZ7BIMxbiAGvJQQETmDAUUCmd/n471uIGonWRPbcPMYxft/HyIiVTCgKMaJPihuNpEEgyLKRG3GtxHv/XSIyF6sCfUGBhQJEu6GGcPHlTFE2OnDHR6eeGEjInIGA4pinLj3upmnOoTQrcEx01GMnWSJiOIfA4oEUvugeGAUT0z7lrDzRKuwIiK5env2F6mBAYUc1RGll6y5YcZERBTvbA8oCxcuhM/nw+zZs0OvnTp1CjNnzkROTg769euHqVOnoq6uzu6i2EfmPCgO3H7d7DPTIaGTrONT3bPnCRGR42wNKJs3b8YzzzyDb3/725rX7777brz55pt49dVXsWHDBhw6dAjXXXednUUhiWIJUWziISK3uf6Vg1PdG2JbQDl+/DimT5+O5557DgMGnH5UfGNjI55//nk88cQTuPTSSzFu3DgsX74cH374ITZu3GhXcWwl8x+YV2++RmtlOqLNJGuqkywREcU72wLKzJkzMWXKFJSWlmper6qqQltbm+b1ESNGoKioCBUVFbrbamlpQSAQ0PyQdXaEIKPbjNo8Y2qmNkYUIvIwTnVvSLIdG125ciW2bt2KzZs3d3uvtrYWqamp6N+/v+b1vLw81NbW6m6vrKwMDz74oB1FlcJr98tYa3z0Pq/RLUabqM2MaLPR2oUd/oniC89pb5Beg1JTU4O77roLL730EtLT06Vsc+7cuWhsbAz91NTUSNmuirwQdvSKaLTjqpSp7r1wkIiIKCbSA0pVVRXq6+tx/vnnIzk5GcnJydiwYQMWL16M5ORk5OXlobW1FQ0NDZr16urqkJ+fr7vNtLQ0ZGVlaX5UIvVpxo6M4nFvm9GHGbMPChERnSa9ieeyyy7Dxx9/rHnt5ptvxogRI3D//fejsLAQKSkpKC8vx9SpUwEA1dXVOHDgAEpKSmQXh3TE/DRjnQ0YDVbBKDPJmsEmHiKKBft9eIP0gJKZmYlRo0ZpXuvbty9ycnJCr99666245557MHDgQGRlZeEXv/gFSkpKcOGFF8oujiO8NpNsrPSKGGsnWTbxEFHC4DBjQ2zpJNubJ598En6/H1OnTkVLSwsmT56Mp59+2o2iKMeRZ/HEur5eJ1kTTTzMF0RE1BtHAsq7776r+Ts9PR1LlizBkiVLnNi97RLtG71uDYrhJh79101NdZ9Yh5uIJHO92ZbDjA3hs3gU40TYsWMfZpp49BY1c4q6OdU9HzJGROQMBhQJpPZBkbgt2/ahExAMDzPmKB4iIjKAAYVM02/iMSb6VPcm9s+EQkQU9xhQJJA6D4oHqlBi6SQrROxNTI438bBVh4jIcQwoqrHp3iskDmvTXT/GidqIiIjCMaBI4exNt7mlHQe/PhH6u+FEK+oDp3pcx+5aHqO1Gh1ROsn21MYTWeOSaKOmiEgu1op6AwOKYozUblxYVo6LH1uPfUeaAQDfWbAWEx4tR9OpNmP7sKOJx/C60fqgRL9iPL6mutf9ExFRfGFAkcDpG2bTqXYAwH9/fkTz+v6jJ/QWB6ANEDFP1Kb3mtEalGD349Xbukvf/Yfmb8enund2d0REBAYU5ZgJO36fz3AwsLtZxGho0GsKCgqTE7VxoDERxcD1ydE41b0hDCgSuDUPit9nrdOpqw8L1Cmv2fDEJh4iovjHgOJhPp/xmgvb7+mGa1C6Lytgch4UE8sSEUVyvZMsp7o3hAFFArkjZIxvzOfzGR49E75YzMOMY+gkqzdRmzDbxMN5UIiI4h4DimLMNfH43Gni0Sml8aAkuq0fFMLUtwg28RARxT8GFAnc6uTkg4lgILGMscwkGy1QsZMsETmFlaLewIAigVtT3fv9QDBofrt23N4NN/EEhc4wY3P7cn4yWl7OiIicxoCiGDO1A34TfVC0O4l1qvvuoj2l2MiuzX4GNvEQEcU/BhQJ3Lpf+ny+qE8HtlMsnVR1O8mCTTxE5Bwfe757AgOKakxN1KatuTDaYdaemWSNrRvUeRZPZ+AxlVCIiCjOMaBIIHPYq5kt+eDT9MfoqalE0wfFhoQS00RtJndvqVkrBvyyRUTkPAYUjwm/wft92iaTHgOKzFE8euUyMVFbt6cTB83Og2J8WSKiSK5/5+BU94YwoCimt5tveAjxRTTxGJ5VNtZOsrqTrRl9WKD1afJPL09ERPGOAUUxvd2sOzQBRTuKp6c+KKoMM9ar5RHC5FT3fJoxEXkZp7o3hAFFAidvmOH76hxmfPrvnob62v1Aw5g6ycJcE4/TfVCIKL6wX5k3MKAoprd7b0dkHxRLTTxWStbz+sabeLq/ZnaqeyIiin8MKBI42ckpvIkncqK2nuZEkTvSyPpInKDQn0lW5YcFEhGR8xhQFNPbrVeE10D4tM0dPY/iMb6P3sTyLB79YcY9r+yPCC9OT3UfPqkTq4aJvI8TtXkDA4oEcp/FY7yTbOTTjI1PNx/rKJ7urxntFxIU3QNJb6tGXkw4LI+IKP4xoHhMZBAI/7OnfGJ3q4jhpxlbGMUT+R5beIiI4h8DigROPs04spakw8JU93YwWquhO4cKRI9VrpFvMZ8QEcU/BhSPCa+BEEIYnkk2/K4e+yge/VoQIzo623hMrduticfpqe4d3RsREQEMKFI4ebsMryQR0N6s3ZzqPpYmnt76r7CJh4jiCqe6N4QBRTGmmniEdl4R408zlt9J1ngTT/eA09sw425NPDyfiYjiHgOKBHbPMRIuGJG8NX8b7CQbcxOP3jwoZpp4dPQ0UZs/IqHwacZE5Gmc6t4QBhSPCb/BC6GtUenxWTwSyxDbMOPuE7WZbuIxtCciIvIyBhQJ7L75h4usMQnv09HTTLKafVgqWc/rG55JVm+itl6beCI7yRrcGREReRYDimJ6u/dG3t81XVIMTnVvxw3e+MMCddaFyXlQWIdCRBT3GFBkcPB+qWniQWQTT/T1tFPdy+8ka/QgdAihM5NsL008LneSZRsxEZHzGFBMONbcivaeUoAEvd2stU08QvdZPMdb2nGqraOHfcRaLr2hwla206m+qaXHdSKbeAIn24ztzAbsMEtE5AwGFIP21DXh/IfW4l+WVXR77/VtX/a6fvmndbjx+UrUNp6CEAL3vLJd8/7t/6cKj779KS79/YZu637w+REMe+CvWP7f+/A/nqsMvS6grVFZs6sWB46ewEVl5fj+b9fj6+ZWAMCOgw2YHrZeuLc//gr/9sdNOHq855Bw3//dcXq/es00Atj8xbEet4FvyvvCh19oXpv27MYej2FjRCAp/6y+1/3Yhf1fiLwpOYnfLrwm2e0CeMVr39xAt9c0dHuvYu/RXte/9cUtAIBfr9qJmT84C69t1d6QV++q7XUbD775iebvyDlFTrZ2YMOewwicakfgVDs2fXEMk8/Lx3VPf4j2oLbmpcvPX9oKAHhs9Wd4/F/GRN33q1UHT6+v874QAtc/s7HXz9ARBJ7ZsLfb60e/CVNERHYYOTgLV47OR15WuttFIYMYUAzySwrfR5tbcLKH5hdztFHB7/OhI6wJqqt/SntE+4tec8yxZuPNJnrDgg038Xiwg2t4s07knCxE5A0+nw9PTx/ndjHIBAYUg6R2lJR0j47MCZHdT6PtJvZn8ei8ZvRDeS+faDCfEFHMONW9IeyDYpDMG5PMhw5rhxkbmzE29lE8UcYKG1k3pj0TEVGiYEAxKHIkSSxkJWYRsS0BY8OJYw1IMeQTx59ELBubeIgoZpzq3hAGFINk/hOSdY+OrDHpnEY+fNixPQXQ74NidKr7mHbtOl5KiIicwYBikMwvzrLu0d36nHRr4tHfky1T3SdKEw8TChGRIxhQDJJVDSeEvKfxdgaSyCae3rcd6+71yp8oTTxEROQMBhSDZA0zBiBvFA8iwoYQ3TrN6ok1IOlP1GbwQYUezCccZkxE5DwGFINk3Zd8PrnDysK3FdTpk6K/Tqz71HktjudBCcd4QkTkDAYUg2SO4glKepyPEELb5ySiV0rUYcYxj+LRa+KJ3xqUcKxAISJyBgOKQSp2kgV6mQcl2v5taeKxvq6XcEggEZEzGFAMknljktVRtFsn2Yi/7WriiWWqe1kdhN3CGhQiImcwoBikYg1K5DDjYESTT7QdxRqQ9MKI4U6yMe3ZHR7PVESkGk51b4j0gFJWVoYLLrgAmZmZyM3NxTXXXIPq6mrNMqdOncLMmTORk5ODfv36YerUqairq5NdFKlk5ZPIWg7Z2zIyk6wdN1zjw4zl75uIiOKP9ICyYcMGzJw5Exs3bsTatWvR1taGH/7wh2hubg4tc/fdd+PNN9/Eq6++ig0bNuDQoUO47rrrZBdFKpnDS+2aSTZyjpXow4zl7D+yLMaW815C4TBjIpKKU90bIv1pxqtXr9b8/cILLyA3NxdVVVX43ve+h8bGRjz//PNYsWIFLr30UgDA8uXLce6552Ljxo248MILZRdJCrnDjOXpqYknWhCxo0oxnpt4wjGfEBE5w/Y+KI2NjQCAgQMHAgCqqqrQ1taG0tLS0DIjRoxAUVERKioqdLfR0tKCQCCg+fEyaTPJovuoHfWbeLwXUcKLzIBCROQMWwNKMBjE7NmzMWnSJIwaNQoAUFtbi9TUVPTv31+zbF5eHmpra3W3U1ZWhuzs7NBPYWGhncXWFT4PSjDGNhJ5TTwioklH2+bjZBZIlGfxsDqWiMgZtgaUmTNnYufOnVi5cmVM25k7dy4aGxtDPzU1NZJKaFz4VPcxTxUfY1nCtxM5aEc7L4r+nuwY6hvPTzMOrzVhDQoRkTOk90HpMmvWLLz11lt47733MGTIkNDr+fn5aG1tRUNDg6YWpa6uDvn5+brbSktLQ1paml1FNSS8c2SHEDEdOGnNHBFtPEJom3WiT9QmZ/eRRTG0nAebeMIxnxAROUN6DYoQArNmzcLrr7+OdevWobi4WPP+uHHjkJKSgvLy8tBr1dXVOHDgAEpKSmQXR5rwb86x3GMjR97EInIelG5T3yvYSdbzWIVCROQI6TUoM2fOxIoVK/DGG28gMzMz1K8kOzsbGRkZyM7Oxq233op77rkHAwcORFZWFn7xi1+gpKRE2RE8gPabc0esfVAkBQQhtP1hjDfxSNm9JZ6fSdbtAhARJQjpAWXp0qUAgEsuuUTz+vLly/Gzn/0MAPDkk0/C7/dj6tSpaGlpweTJk/H000/LLopc4Z1kY7jJ+nxym1i0NSjdn26su44NGcHoMVE9n/T234cVKEREzpAeUIxU9aenp2PJkiVYsmSJ7N3bJvy+FOvTiGXVYEQOMw52JhTN+9HXlCteHhbY24MQmU+IiJzBZ/EY5JM4ikfaPCg6M8dG9kmJtp5sxocZK55QeuFjFQoRkSMYUAwKn/+iI9aAIqkKRe9mr+mTEnWqew4zjkYvf2iGGTtXFCKihMaAYpCmb0eMd1lpTTyiexMPZ5K1FytQiIicwYBikJFn3BjdTqw1MKFtIWLeE52HB0ZbTzqDG411BJTbOJMsEZEzGFAMCv/mH2vAkDdRm9AOK0bE1PdRVrOjicdo3xKP5xO28RAROYQBxaDw+2osTTw+n9xahJ5qTKIGEVuGGRtdztsJhfmEiMgZtk1172V1gVNY91k9BmenIz0lCa3tQXz4+dHQ+9c+/SHKrhuNfx6Zh9rGU6a3L3OY8Ybd9ZrthtfObN53DJ+PPK67nhF7Dx/H4aYWTPxWTq/L/m2X/oMeI3k+oDChEBE5ggFFx6SF69DeQ4o4crwFt/1pC75YOAUPvLbD1LYjZ3+NxdpP6rBx77HwrWvCx/rqw1hfvUGnDMb2f+nvO9ddd+/3e112ffVhQ9vsiHEOGbexDwoRkTPYxKOjp3AS6cDRE6a3L6uT7AefH9H8HTkvSjRm89Huuu61MFa5NYrnn/L6aZ5I3WXWD842tR3WoBAROYMBxWE+n9yJ2jR/67ymu56UvVvj1iieX00ZiQnFAzWvXTv2TPxy8jmmtsOAQkTkDAYUh8ls4okUOQ9K9DK4F1Fk1R7J4NdJG3r5QzvVPRMKEZETGFBcYFclghDGwoebGUGhfKLb5NMb1qAQETmDAcVhsocZhxNCGGzi6b6QUzdelUbxJBlMKAwlRETOY0BxgV1NLIb7oPTyxF47qTSTrJUH//FhgUREzmBAcZjMqe71tm2kF4qblRh29b+xIsnCv37GEyIiZzCguMCuuUBExNT30RgaimxbR15bNmuJXifZ3rAChYjIGQwoDvP5FGjiMbAtu/qKqD6KpzfMJ0REzmBAcYFd/TA6hxnLaeKxrxlK7YDSWx8T9kEhInIGA4oL7BxmbKR6xM2hyCp1kmUfFCIidTGguMCu5hNhcNt6S0RWDNhXy2PLZi3xW5gIhRUoRETOYECJkZX7rW1zgQhj5dGrQYl8ya4mHpVG8Vjpg8I6FCIiZzCgxMhsnwohbO6DIqmTrLBppJFKnWSTOIqHiEhZDCgxUqkGxWgTj5F8ZFsNikIBxdJU9/KLQUREOhhQYmT2huvzAUEb50Gx2sQTWTOg/VzyQoVCLTzsg0JEpDAGlBhZqRCwbQhv6H8srBuxXnhfEZmhwut9UPg0YyIiZzCgxMhK1rCticfgVPdmm4FkllepPiisQSEiUhYDSoysTDxmZy2CkeYjvSJ3G2YctpDMTr0K5RP9idp6WYcBhYjIGQwoMbLWSVZ6MUKM1FB0LRIernpq4lEpVMikV4HS20dlEw8RkTMYUGJk9uZt59OMAWO1M13NQD0VI2hTDYpKrDTxMJ8QETmDASVGVvpn2NrEY6J/SU9LhocSlfqNyKT3XJ1em3jsKQoREUVgQImR2Vu3z2fvXCCGso9OE0/3YcZhi8dpQEmyUoHCTihERI5gQIlBMGhs5tZIHTbNgwIYfRbPN0084a9F9kHRNPHIKJl6LM2DYkM5iIioOwaUGASNPj44jBD21kgY6S/StUhPYSb8PZVmf5XJ0jwoTChERI5gQIlBh8Fn3+itZxdDNSii906yHUEGFD3MJ0REzmBAiYEQ1qa6t3NUjJFN6y0Sea8O/1gqzf4qU5KFf/3sg0JE5AwGlBh0BI09+yZcZxOPLcUBYHCYsdD+f+TvQOQoHhklUw9rUIiI1MWAEoOgwSaeyD4ndtagGJuorXMZo31Q4nUUj5WAwoRCROQMBpQYBING+3yc/l2FYcYi4v+Bnp9mzInaTuNMskREzmBAiUGw8+l8vYpc5PP647aUBzBW26E31X0k7cMCYy2VmvQqUHqrVGEXFCIiZyS7XQDV/O/39xpeduxDa3tdZtgDf9X8ve1Ag9kimbLjYGOvy3z8ZSO+NfevmuC09pM6fGvu6bKGv/fY6s/kFVAh7INCRKQu1qBEePivn7pdBEcEdTrrBsXpH1W7naSn+DE8t5+pdbLSkzFh2ED8ryvP1bw+siALAHD5efmh1/7fHRdplsnNTMPg7AzceOFQAMAdl5xlpdiOWHjdaADA0unnu1wSIurJ/B+NDP0++5+Hu1gStfmEB3tABgIBZGdno7GxEVlZWVK3HVnjEc1ZZ/TFPw43AwAeuXYUbrigCB1C4KG3PsGfKvZHXW/9Ly/BgD4p6JOajKZTbeiXnoyW9iCEANo6gshMT8Y5v14dWv7j//ghOoICqcl+HG9pR0ZKUqjJpaW9A+kpSfD7fOiTkoQjzS0AgMy0FHQIgY4OgbZgEEEhkJGShJb2IPqlJSNwqk1TpoyUJJxs6+hWVr/PhxS/H2MW/A0AkNM3FR/OvRSNJ9uQnZGCtg6BvqlJaGpphw8Ibd/nA5pbOpCS5MOvV+3EG9sPAQB2P3wFGk62wgcf0lP8SEtOQkqSD8db2nGyrQN9U5PRdKodF5aVA+g8iaeePwQpyT5kpCShubUDqUl+JPt96BACbR3BUC2I3+fDisr9+I83PwntCwBSknxoDwqkfDOmuLU9GOpTk5GaBKCzqavtm6FKqcl+tLYHkezvXM/vA5LD1k1NVjvTe6GMRNR5rgJIuPPVzP2bTTwWDeybGgooWekp8Pt98MOH7IyUHtfLy0pDn9TOw57TLw0AkJacFHX5zPTT2+ta7zTtvnIz03vcd+Y3/5+e0n1/4fuJpl96MtKSk5Cb2bl+2jfFyfpm3cywZbs+U/jxSE3265YxMz0ltP/2sA4vKUk+ZPc5vX6/tNOf3w9fKHSc3n5S2O+n30sJe+iO3sXA5/MhNbn7MqkRnWi9cCHxQhmJiOeqETxCFiX7Tx+68NEgvfVrsDS0VRFWSm52xFL4sTTbOdfDh5aIiCIwoFiUnKQfSnobumplaKuXmQ0ZSb7wgGJyll5zuyIiIoUxoFgU3rwQnjl6yx9erkGxwmwXp/DDE6/Dm4mIqHcMKBaF92vQNPH0klASrAIFwaC55cOPZSzhhoiIvI0BxaJkTQ2K8T4oifawObPNNP6YmngS69gSEcUzBhSLUqLUmiQlWADpjdlmGj+beIiICAwoliVH6YPCfKJlvpnGeg0KERHFDwYUizR9UEyM4vEyK3EhlpBhetX4PfRERAmHAcWi8HlQfCb6oCSaWJppgiZX5pEnIoofDCgWJUcZxRPP+cSJidq065pbPtE6IBMRxTNXA8qSJUswbNgwpKenY+LEidi0aZObxTElfB6UJJuOYjzcb2PpRsI+KEREicu1gPLnP/8Z99xzD37zm99g69atGDNmDCZPnoz6+nq3imRKsqbWxJ4kEQ/NRbH1QWETDxFRonItoDzxxBO47bbbcPPNN2PkyJFYtmwZ+vTpgz/+8Y9uFcmU8FE8dg0tjof+ts428VjeFRERKcaVgNLa2oqqqiqUlpaeLojfj9LSUlRUVHRbvqWlBYFAQPPjthQTDwi0euOMhz4VMXWSZRMPEVHCciWgHDlyBB0dHcjLy9O8npeXh9ra2m7Ll5WVITs7O/RTWFjoVFGjSgl7VLa/l6OYbLEqJB5qUMw204TjRG1ERInLE6N45s6di8bGxtBPTU2N20XShI5ea1As9o6Ijz4o1tfls3iIiBJXshs7HTRoEJKSklBXV6d5va6uDvn5+d2WT0tLQ1pamlPFM0Q7iqfnO6OwNMVZfASUjhgSitl1+SweIqL44UoNSmpqKsaNG4fy8vLQa8FgEOXl5SgpKXGjSKaFz4NiV5CIg3zCTrJERGSJKzUoAHDPPffgpptuwvjx4zFhwgQsWrQIzc3NuPnmm90qkikpfv1n8cgUDzUonAeFiIiscC2g/Ou//isOHz6M+fPno7a2Ft/5znewevXqbh1nVRVtJlmZ4qGTrJPzoBARUfxwLaAAwKxZszBr1iw3i2CZ9mnGnKgtGiebeIiIKH54YhSPijTzoNhU1cF5UMyO4vH+8SIiok4MKGHMPD1XW4NiR2nio4mH86AQEZEVDChhOkzcTFPC+6AkSBOPlbzg6Dwo1ndFRESKYUAJY6ZJITlsFI99Dwu0ZbOOiq0PCidqIyJKVAwoYYJB48uGT29v2ygexRKKldLE1gfF3PKcqI2IKH4woIQx8409vFnHrtuiak08VsTWB4WdUIiIEhUDShgzfVCcqN1QrALFktjmQZFYECIi8hQGlDBbvjhmeFknajfioQbFTLNZt3XZB4WIKGExoITZf/SE4WVzM08/vDA9JSn0+9m5/bote+3YM02VY2DfVADA9/7pDFPr2WVQv87PaqU8Fw8fBADITDM/J+DYwv6mlh+a08f0PoiISE0+4cH5xAOBALKzs9HY2IisrCxp2/2opgF/+6QWdYEWNJxoxff+6QzsPdyMUWdm41tn9EXNsRM43tKOnL6puHzUYKyvrocPwCXn5Gq289rWgzje0o5/1B/HBcUDMfm8fLyypQYXnTUIxYP69lqOLxtOYt1n9fiX84cgIzWp1+XtdqjhJMo/rcPUcUPQJ9Vc0DjV1oFXqw7iB+ecgSEDjAWIvYePY+PeY7h+/BDNfDNGvPnRIRT0z8C4oQNMrUdERPYzc/9mQCEiIiJHmLl/s4mHiIiIlMOAQkRERMphQCEiIiLlMKAQERGRchhQiIiISDkMKERERKQcBhQiIiJSDgMKERERKYcBhYiIiJTDgEJERETKYUAhIiIi5TCgEBERkXIYUIiIiEg5yW4XwIquBzAHAgGXS0JERERGdd23u+7jPfFkQGlqagIAFBYWulwSIiIiMqupqQnZ2dk9LuMTRmKMYoLBIA4dOoTMzEz4fD6p2w4EAigsLERNTQ2ysrKkbtsrEv0YJPrnB3gMAB4DgMcA4DEA5B4DIQSamppQUFAAv7/nXiaerEHx+/0YMmSIrfvIyspK2H+MXRL9GCT65wd4DAAeA4DHAOAxAOQdg95qTrqwkywREREphwGFiIiIlMOAEiEtLQ2/+c1vkJaW5nZRXJPoxyDRPz/AYwDwGAA8BgCPAeDeMfBkJ1kiIiKKb6xBISIiIuUwoBAREZFyGFCIiIhIOQwoREREpBwGlDBLlizBsGHDkJ6ejokTJ2LTpk1uF0mKsrIyXHDBBcjMzERubi6uueYaVFdXa5Y5deoUZs6ciZycHPTr1w9Tp05FXV2dZpkDBw5gypQp6NOnD3JzczFnzhy0t7c7+VGkWbhwIXw+H2bPnh16LRGOwZdffomf/vSnyMnJQUZGBkaPHo0tW7aE3hdCYP78+Rg8eDAyMjJQWlqKPXv2aLZx7NgxTJ8+HVlZWejfvz9uvfVWHD9+3OmPYklHRwfmzZuH4uJiZGRk4KyzzsJDDz2keS5IvB2D9957D1dddRUKCgrg8/mwatUqzfuyPu+OHTvw3e9+F+np6SgsLMTjjz9u90czrKdj0NbWhvvvvx+jR49G3759UVBQgH/7t3/DoUOHNNuI52MQ6fbbb4fP58OiRYs0rzt+DAQJIYRYuXKlSE1NFX/84x/Frl27xG233Sb69+8v6urq3C5azCZPniyWL18udu7cKbZv3y6uvPJKUVRUJI4fPx5a5vbbbxeFhYWivLxcbNmyRVx44YXioosuCr3f3t4uRo0aJUpLS8W2bdvE22+/LQYNGiTmzp3rxkeKyaZNm8SwYcPEt7/9bXHXXXeFXo/3Y3Ds2DExdOhQ8bOf/UxUVlaKvXv3ijVr1ojPP/88tMzChQtFdna2WLVqlfjoo4/Ej3/8Y1FcXCxOnjwZWubyyy8XY8aMERs3bhTvv/++OPvss8UNN9zgxkcy7ZFHHhE5OTnirbfeEvv27ROvvvqq6Nevn/jDH/4QWibejsHbb78tfvWrX4nXXntNABCvv/665n0Zn7exsVHk5eWJ6dOni507d4qXX35ZZGRkiGeeecapj9mjno5BQ0ODKC0tFX/+85/FZ599JioqKsSECRPEuHHjNNuI52MQ7rXXXhNjxowRBQUF4sknn9S85/QxYED5xoQJE8TMmTNDf3d0dIiCggJRVlbmYqnsUV9fLwCIDRs2CCE6T9CUlBTx6quvhpb59NNPBQBRUVEhhOj8x+33+0VtbW1omaVLl4qsrCzR0tLi7AeIQVNTkxg+fLhYu3at+P73vx8KKIlwDO6//35x8cUXR30/GAyK/Px88dvf/jb0WkNDg0hLSxMvv/yyEEKITz75RAAQmzdvDi3zzjvvCJ/PJ7788kv7Ci/JlClTxC233KJ57brrrhPTp08XQsT/MYi8Mcn6vE8//bQYMGCA5jy4//77xTnnnGPzJzKvp5tzl02bNgkAYv/+/UKIxDkGBw8eFGeeeabYuXOnGDp0qCaguHEM2MQDoLW1FVVVVSgtLQ295vf7UVpaioqKChdLZo/GxkYAwMCBAwEAVVVVaGtr03z+ESNGoKioKPT5KyoqMHr0aOTl5YWWmTx5MgKBAHbt2uVg6WMzc+ZMTJkyRfNZgcQ4Bn/5y18wfvx4/OQnP0Fubi7Gjh2L5557LvT+vn37UFtbqzkG2dnZmDhxouYY9O/fH+PHjw8tU1paCr/fj8rKSuc+jEUXXXQRysvLsXv3bgDARx99hA8++ABXXHEFgMQ4BuFkfd6Kigp873vfQ2pqamiZyZMno7q6Gl9//bVDn0aexsZG+Hw+9O/fH0BiHINgMIgbb7wRc+bMwXnnndftfTeOAQMKgCNHjqCjo0Nz4wGAvLw81NbWulQqewSDQcyePRuTJk3CqFGjAAC1tbVITU0NnYxdwj9/bW2t7vHpes8LVq5cia1bt6KsrKzbe4lwDPbu3YulS5di+PDhWLNmDe644w7ceeedePHFFwGc/gw9nQe1tbXIzc3VvJ+cnIyBAwd64hg88MADmDZtGkaMGIGUlBSMHTsWs2fPxvTp0wEkxjEIJ+vzev3cCHfq1Cncf//9uOGGG0IPxkuEY/DYY48hOTkZd955p+77bhwDTz7NmKybOXMmdu7ciQ8++MDtojiqpqYGd911F9auXYv09HS3i+OKYDCI8ePH49FHHwUAjB07Fjt37sSyZctw0003uVw6Z7zyyit46aWXsGLFCpx33nnYvn07Zs+ejYKCgoQ5BhRdW1sbrr/+egghsHTpUreL45iqqir84Q9/wNatW+Hz+dwuTghrUAAMGjQISUlJ3UZs1NXVIT8/36VSyTdr1iy89dZbWL9+PYYMGRJ6PT8/H62trWhoaNAsH/758/PzdY9P13uqq6qqQn19Pc4//3wkJycjOTkZGzZswOLFi5GcnIy8vLy4PwaDBw/GyJEjNa+de+65OHDgAIDTn6Gn8yA/Px/19fWa99vb23Hs2DFPHIM5c+aEalFGjx6NG2+8EXfffXeoVi0RjkE4WZ/X6+cGcDqc7N+/H2vXrg3VngDxfwzef/991NfXo6ioKHR93L9/P+69914MGzYMgDvHgAEFQGpqKsaNG4fy8vLQa8FgEOXl5SgpKXGxZHIIITBr1iy8/vrrWLduHYqLizXvjxs3DikpKZrPX11djQMHDoQ+f0lJCT7++GPNP9Cukzjypqeiyy67DB9//DG2b98e+hk/fjymT58e+j3ej8GkSZO6DS/fvXs3hg4dCgAoLi5Gfn6+5hgEAgFUVlZqjkFDQwOqqqpCy6xbtw7BYBATJ0504FPE5sSJE/D7tZe9pKQkBINBAIlxDMLJ+rwlJSV477330NbWFlpm7dq1OOecczBgwACHPo11XeFkz549+Pvf/46cnBzN+/F+DG688Ubs2LFDc30sKCjAnDlzsGbNGgAuHQNLXWvj0MqVK0VaWpp44YUXxCeffCJmzJgh+vfvrxmx4VV33HGHyM7OFu+++6746quvQj8nTpwILXP77beLoqIisW7dOrFlyxZRUlIiSkpKQu93DbH94Q9/KLZv3y5Wr14tzjjjDM8MsdUTPopHiPg/Bps2bRLJycnikUceEXv27BEvvfSS6NOnj/iv//qv0DILFy4U/fv3F2+88YbYsWOHuPrqq3WHnI4dO1ZUVlaKDz74QAwfPlzZIbaRbrrpJnHmmWeGhhm/9tprYtCgQeK+++4LLRNvx6CpqUls27ZNbNu2TQAQTzzxhNi2bVtohIqMz9vQ0CDy8vLEjTfeKHbu3ClWrlwp+vTpo8wQ256OQWtrq/jxj38shgwZIrZv3665RoaPRonnY6AnchSPEM4fAwaUME899ZQoKioSqampYsKECWLjxo1uF0kKALo/y5cvDy1z8uRJ8fOf/1wMGDBA9OnTR1x77bXiq6++0mzniy++EFdccYXIyMgQgwYNEvfee69oa2tz+NPIExlQEuEYvPnmm2LUqFEiLS1NjBgxQjz77LOa94PBoJg3b57Iy8sTaWlp4rLLLhPV1dWaZY4ePSpuuOEG0a9fP5GVlSVuvvlm0dTU5OTHsCwQCIi77rpLFBUVifT0dPGtb31L/OpXv9LciOLtGKxfv173/L/pppuEEPI+70cffSQuvvhikZaWJs4880yxcOFCpz5ir3o6Bvv27Yt6jVy/fn1oG/F8DPToBRSnj4FPiLApFImIiIgUwD4oREREpBwGFCIiIlIOAwoREREphwGFiIiIlMOAQkRERMphQCEiIiLlMKAQERGRchhQiIiISDkMKERERKQcBhQiIiJSDgMKERERKYcBhYiIiJTz/wETPpnLpVjq4wAAAABJRU5ErkJggg==" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(all_rewards)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:03:57.005733Z", - "start_time": "2024-02-16T22:03:56.864684Z" - } - }, - "id": "c8afda46e3c79cf3", - "execution_count": 9 - }, - { - "cell_type": "code", - "outputs": [], - "source": [ - "import torch\n", - "\n", - "torch.save(model.state_dict(), \"model.pth\")" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T17:07:37.597554Z", - "start_time": "2024-02-16T17:07:37.428788Z" - } - }, - "id": "14da818f0fc53e58", - "execution_count": 11 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "" - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import torch\n", - "\n", - "state_dict = torch.load(\"models/model_1000.pth\")\n", - "model.load_state_dict(state_dict)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:59.463675Z", - "start_time": "2024-02-16T22:24:59.411132Z" - } - }, - "id": "ac65b7da064c23f", - "execution_count": 10 - }, - { - "cell_type": "code", - "outputs": [ - { - "data": { - "text/plain": "True" - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "PONG_ACTIONS = [\"no-op\", \"ArrowUp\", \"ArrowDown\"]\n", - "\n", - "actions = PONG_ACTIONS\n", - "await cog.run_web_ui(actions=actions, log_file=\"human.log\", fps=30)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:24:27.047709Z", - "start_time": "2024-02-16T22:24:26.501042Z" - } - }, - "id": "8c718662ac8b6719", - "execution_count": 7 - }, - { - "cell_type": "code", - "outputs": [ - { - "ename": "ValueError", - "evalue": "No samples provided", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mValueError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[34], line 10\u001B[0m\n\u001B[1;32m 1\u001B[0m trial_id \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m cog\u001B[38;5;241m.\u001B[39mstart_trial(\n\u001B[1;32m 2\u001B[0m env_name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpong\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 3\u001B[0m session_config\u001B[38;5;241m=\u001B[39m{\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mrender\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;28;01mTrue\u001B[39;00m},\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 7\u001B[0m },\n\u001B[1;32m 8\u001B[0m )\n\u001B[0;32m---> 10\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m cog\u001B[38;5;241m.\u001B[39mget_trial_data(trial_id)\n", - "File \u001B[0;32m~/projects/air/cogment-lab/cogment_lab/process_manager.py:516\u001B[0m, in \u001B[0;36mCogment.get_trial_data\u001B[0;34m(self, trial_id, env_name, fields, use_tqdm, tqdm_kwargs)\u001B[0m\n\u001B[1;32m 513\u001B[0m env \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39menvs[env_name]\n\u001B[1;32m 514\u001B[0m agent_specs \u001B[38;5;241m=\u001B[39m env\u001B[38;5;241m.\u001B[39magent_specs\n\u001B[0;32m--> 516\u001B[0m data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m format_data_multiagent(\n\u001B[1;32m 517\u001B[0m datastore\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdatastore,\n\u001B[1;32m 518\u001B[0m trial_id\u001B[38;5;241m=\u001B[39mtrial_id,\n\u001B[1;32m 519\u001B[0m actor_agent_specs\u001B[38;5;241m=\u001B[39magent_specs,\n\u001B[1;32m 520\u001B[0m fields\u001B[38;5;241m=\u001B[39mfields,\n\u001B[1;32m 521\u001B[0m use_tqdm\u001B[38;5;241m=\u001B[39muse_tqdm,\n\u001B[1;32m 522\u001B[0m tqdm_kwargs\u001B[38;5;241m=\u001B[39mtqdm_kwargs,\n\u001B[1;32m 523\u001B[0m )\n\u001B[1;32m 525\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m data\n", - "File \u001B[0;32m~/projects/air/cogment-lab/cogment_lab/utils/trial_utils.py:359\u001B[0m, in \u001B[0;36mformat_data_multiagent\u001B[0;34m(datastore, trial_id, actor_agent_specs, fields, use_tqdm, tqdm_kwargs)\u001B[0m\n\u001B[1;32m 356\u001B[0m actor_data \u001B[38;5;241m=\u001B[39m {}\n\u001B[1;32m 358\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m actor_id, samples \u001B[38;5;129;01min\u001B[39;00m actor_samples\u001B[38;5;241m.\u001B[39mitems():\n\u001B[0;32m--> 359\u001B[0m actor_data[actor_id] \u001B[38;5;241m=\u001B[39m \u001B[43mextract_data_from_samples\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 360\u001B[0m \u001B[43m \u001B[49m\u001B[43msamples\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mactor_agent_specs\u001B[49m\u001B[43m[\u001B[49m\u001B[43mactor_id\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfields\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mactor_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mactor_id\u001B[49m\n\u001B[1;32m 361\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 363\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m actor_id, reward_samples \u001B[38;5;129;01min\u001B[39;00m actor_reward_samples\u001B[38;5;241m.\u001B[39mitems():\n\u001B[1;32m 364\u001B[0m actor_data[actor_id]\u001B[38;5;241m.\u001B[39mrewards \u001B[38;5;241m=\u001B[39m extract_rewards_from_samples(reward_samples, actor_name\u001B[38;5;241m=\u001B[39mactor_id)\n", - "File \u001B[0;32m~/projects/air/cogment-lab/cogment_lab/utils/trial_utils.py:163\u001B[0m, in \u001B[0;36mextract_data_from_samples\u001B[0;34m(samples, agent_specs, fields, actor_name)\u001B[0m\n\u001B[1;32m 161\u001B[0m sample_count \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlen\u001B[39m(samples)\n\u001B[1;32m 162\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m sample_count \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[0;32m--> 163\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNo samples provided\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 165\u001B[0m cog_observation_space \u001B[38;5;241m=\u001B[39m agent_specs\u001B[38;5;241m.\u001B[39mget_observation_space()\n\u001B[1;32m 166\u001B[0m observation_space \u001B[38;5;241m=\u001B[39m cog_observation_space\u001B[38;5;241m.\u001B[39mgym_space\n", - "\u001B[0;31mValueError\u001B[0m: No samples provided" - ] - } - ], - "source": [ - "trial_id = await cog.start_trial(\n", - " env_name=\"pong\",\n", - " session_config={\"render\": True},\n", - " actor_impls={\n", - " \"paddle_0\": \"coltra\",\n", - " \"paddle_1\": \"web_ui\",\n", - " },\n", - ")\n", - "\n", - "data = await cog.get_trial_data(trial_id)\n" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2024-02-16T22:29:35.106072Z", - "start_time": "2024-02-16T22:29:34.903722Z" - } - }, - "id": "7867d5bd495d8c30", - "execution_count": 34 - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/demos/pong-multiplayer/utils.py b/examples/demos/pong-multiplayer/utils.py deleted file mode 100644 index 013133c..0000000 --- a/examples/demos/pong-multiplayer/utils.py +++ /dev/null @@ -1,61 +0,0 @@ -import numpy as np -import torch -from coltra import CAgent, DAgent, Observation -from coltra.envs.spaces import ObservationSpace -from coltra.models import BaseModel -from coltra.models.mlp_models import FlattenMLPModel -from gymnasium import Space -from gymnasium.spaces import Box -from pettingzoo.butterfly.cooperative_pong_v5 import parallel_env -from supersuit import dtype_v0, normalize_obs_v0 - -from cogment_lab.core import CogmentActor - - -class FloatImageMLPModel(FlattenMLPModel): - def __init__(self, config: dict, observation_space: ObservationSpace, action_space: Space): - assert "image" in observation_space.spaces, "ImageMLPModel requires an observation space with image" - - vector_size = observation_space.vector.shape[0] if "vector" in observation_space.spaces else 0 - image_size = np.prod(observation_space.spaces["image"].shape) - new_vector_size = vector_size + image_size - - new_observation_space = ObservationSpace({"vector": Box(-np.inf, np.inf, (new_vector_size,))}) - - super().__init__(config, new_observation_space, action_space) - - def _flatten(self, obs: Observation) -> Observation: - if not hasattr(obs, "image"): - return obs - image: torch.Tensor = obs.image - - if len(image.shape) == 3: # no batch - dim = 0 - else: # image.shape == 4, batch - dim = 1 - - vector = torch.flatten(image, start_dim=dim) - - if hasattr(obs, "vector"): - vector = torch.cat([obs.vector, vector], dim=dim) - - return Observation(vector=vector.to(torch.float32)) - - -class ColtraImageActor(CogmentActor): - def __init__(self, model: BaseModel): - super().__init__(model) - self.model = model - self.agent = DAgent(self.model) if self.model.discrete else CAgent(self.model) - - async def act(self, observation: np.ndarray, rendered_frame=None): - obs = Observation(image=(observation / 255.0).astype(np.float32)) - action, _, _ = self.agent.act(obs) - return action.discrete - - -def PongEnv(*args, **kwargs): - env = parallel_env(*args, **kwargs) - env = dtype_v0(env, np.float32) - env = normalize_obs_v0(env, env_max=255.0) - return env