training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "import chainer\n",
    "import chainer.functions as F\n",
    "import chainer.links as L\n",
    "import chainerrl\n",
    "from chainerrl import explorers\n",
    "import pygame\n",
    "import sys\n",
    "from time import sleep\n",
    "import random\n",
    "import math\n",
    "import numpy as np\n",
    "from gym import spaces\n",
    "\n",
    "# COLORS\n",
    "white = (255, 255, 255)\n",
    "black = (0, 0, 0)\n",
    "red = (255, 0, 0)\n",
    "green = (0, 255, 0)\n",
    "blue = (0, 0, 255)\n",
    "purple = (255, 0, 255)\n",
    "\n",
    "# SIZES\n",
    "AGENT_SIZE = 50\n",
    "BULLET_WIDTH = 20\n",
    "BULLET_HEIGHT = 10\n",
    "GRENDADE_SIZE = 10\n",
    "WEAPON_SIZE = 20\n",
    "\n",
    "class Entity:\n",
    "    def __init__(self, name, xy, angle, speed, height, width, game_dims=(1000,800)):\n",
    "        self.name = name\n",
    "        self.x, self.y = xy\n",
    "        self.height = height\n",
    "        self.width = width\n",
    "        self.speed = speed\n",
    "        self.dimx, self.dimy = game_dims\n",
    "\n",
    "        if name == 1:\n",
    "            self.angle = math.radians(angle)  # -1 to 1 \n",
    "            self.dx = self.speed * math.cos(self.angle)\n",
    "            self.dy = self.speed * math.sin(self.angle)\n",
    "            self.counter = 0\n",
    "        elif name == 2:\n",
    "            self.angle = angle\n",
    "            self.counter = 12\n",
    "        \n",
    "    def update(self, agent_xy):\n",
    "        if self.name == 1:\n",
    "            self.x += self.dx\n",
    "            self.y += self.dy\n",
    "        elif self.name == 2:\n",
    "            if self.counter >= 2:\n",
    "                self.x += self.speed\n",
    "                self.y -= (self.angle * abs(self.angle)) * 1\n",
    "                self.angle -= 1\n",
    "                self.counter -= 1\n",
    "            elif self.counter >= 0:\n",
    "                self.x -= 4\n",
    "                self.y -= 4\n",
    "                self.height += 8\n",
    "                self.width += 8\n",
    "                self.counter -= 1\n",
    "            else:\n",
    "                self.x = -100\n",
    "                self.y = -100\n",
    "\n",
    "        agent_x, agent_y = agent_xy\n",
    "\n",
    "        has_hit_x = self.x >= agent_x - self.width and self.x <= agent_x + AGENT_SIZE\n",
    "        has_hit_y = self.y >= agent_y - self.height and self.y <= agent_y + AGENT_SIZE\n",
    "\n",
    "        exit_boundary = self.x > self.dimx-50 or self.x < 000 or self.y > self.dimy-50 or self.y < 0\n",
    "\n",
    "        return has_hit_x and has_hit_y or exit_boundary\n",
    "    def __repr__(self):\n",
    "        return self.name + str((self.x,self.y))\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, xy=(400,100), game_dims=(1000,800), show=False):\n",
    "        self.jumps = 0\n",
    "        self.maxJumps = 2\n",
    "        self.xpos, self.ypos = xy\n",
    "        self.gravityPull = 0.5\n",
    "        self.gravityCurrent = 0\n",
    "        self.xCurrent = 0\n",
    "        self.touchingObst = 0\n",
    "        self.show = show\n",
    "        self.dimx, self.dimy = game_dims\n",
    "    def jump(self):\n",
    "        if self.jumps < self.maxJumps:\n",
    "            self.gravityCurrent = -10\n",
    "            self.jumps = self.jumps + 1\n",
    "    def left(self):\n",
    "        if self.touchingObst == 0:\n",
    "            self.xCurrent = -10\n",
    "    def right(self):\n",
    "        if self.touchingObst == 0:\n",
    "            self.xCurrent = 10\n",
    "    def update(self):\n",
    "        # CONTROL GRAVITY\n",
    "        self.gravityCurrent = self.gravityCurrent + self.gravityPull\n",
    "\n",
    "        # RATE OF DECREASE OF LEFT/RIGHT MOVEMENTS\n",
    "        if self.xCurrent > 0:\n",
    "            self.xCurrent = self.xCurrent - 0.5\n",
    "        if self.xCurrent < 0:\n",
    "            self.xCurrent = self.xCurrent + 0.5\n",
    "\n",
    "        # UPDATE XY COORDINATES\n",
    "        self.ypos = self.ypos + self.gravityCurrent\n",
    "        self.xpos = self.xpos + self.xCurrent\n",
    "\n",
    "        # BOUNDARIES\n",
    "        if self.xpos > self.dimx-50:\n",
    "            self.xpos = self.dimx-50\n",
    "        if self.xpos < 000:\n",
    "            self.xpos = 000\n",
    "        if self.ypos > self.dimy-50:\n",
    "            self.ypos = self.dimy-50+1\n",
    "            self.gravityCurrent = 0\n",
    "            self.jumps = 0\n",
    "            \n",
    "    def display(self, gameDisplay):\n",
    "        if self.show:\n",
    "            pygame.draw.rect(gameDisplay, red, (self.xpos, self.ypos, AGENT_SIZE, AGENT_SIZE))\n",
    "    def act(self, agent_action):\n",
    "        if agent_action == 0:\n",
    "            self.left()\n",
    "        elif agent_action == 1:\n",
    "            self.right()\n",
    "        elif agent_action == 2:\n",
    "            self.jump()\n",
    "        self.update()\n",
    "\n",
    "class Env:\n",
    "    def __init__(self, game_dims=(1000, 800),show=False, test=False):\n",
    "        self.agent = Agent((400,500), show=show, game_dims=game_dims)\n",
    "        self.dimx, self.dimy = game_dims\n",
    "        self.set_default_rewards()\n",
    "        self.observation_space = 5\n",
    "        self.show = show\n",
    "        pygame.init()\n",
    "        self.play = True\n",
    "        \n",
    "        # GAME DIMENSIONS\n",
    "        self.game_dims = game_dims\n",
    "        self.generator_action_space = spaces.Box(np.array([0,0,0,0]), np.array([2,self.dimx,self.dimy,360]), dtype=np.float32)\n",
    "        self.agent_action_space = spaces.Discrete(3)\n",
    "        min_obs = np.array([0]*5 + [0,0,5,0]*5)\n",
    "        max_obs = np.array([2,self.dimx,self.dimy,1,self.dimy*2] + [self.dimx,self.dimy,50,360]*5)\n",
    "        self.observation_space = spaces.Box(min_obs, max_obs, dtype=np.float32)\n",
    "        \n",
    "        # DELAY FOR WEAPON ENTITIES\n",
    "        self.delay = 0\n",
    "        self.entity_limit = 5\n",
    "        self.entity_free_keys = [0,1,2,3,4]\n",
    "        self.entity_dict = {}\n",
    "        \n",
    "        # LOAD PRETRAINED MODELS\n",
    "        if test:\n",
    "            self.chosen_one = self.load_chosen_one()\n",
    "        \n",
    "    def get_free_key(self):\n",
    "        if len(self.entity_free_keys) > 0:\n",
    "            return self.entity_free_keys.pop(0)\n",
    "        else:\n",
    "            return None\n",
    "    \n",
    "    # CONTROL MOVEMENTS\n",
    "    Mouse_x = -200\n",
    "    Mouse_y = -200\n",
    "    MMouse_x = -200\n",
    "    MMouse_y = -200\n",
    "    bullet_quantity = 0\n",
    "    def execute(self):\n",
    "        agent_action = None\n",
    "        weapon_action = (0,0,0)\n",
    "        events = pygame.event.get()\n",
    "        for event in events:\n",
    "            if event.type == pygame.KEYDOWN:\n",
    "                if event.key == pygame.K_SPACE:\n",
    "                    agent_action = 2\n",
    "                if event.key == pygame.K_LEFT and self.agent.touchingObst == 0:\n",
    "                    agent_action = 0\n",
    "                if event.key == pygame.K_RIGHT and self.agent.touchingObst == 0:\n",
    "                    agent_action = 1\n",
    "                if event.key == pygame.K_ESCAPE:\n",
    "                    agent_action = -1\n",
    "            if event.type == pygame.QUIT:\n",
    "                pygame.display.quit()\n",
    "                quit()\n",
    "                agent_action = -1\n",
    "            if event.type == pygame.MOUSEMOTION:\n",
    "                self.MMouse_x, self.MMouse_y = pygame.mouse.get_pos()\n",
    "            if event.type == pygame.MOUSEBUTTONDOWN:\n",
    "                self.Mouse_x, self.Mouse_y = pygame.mouse.get_pos()\n",
    "#                 print('mouse finally down')\n",
    "            elif event.type == pygame.MOUSEBUTTONUP:\n",
    "                MyClock = pygame.time.Clock()\n",
    "                MyClock.tick(60)\n",
    "#                 print('mouse finally up')\n",
    "#                 self.bullet_quantity += 1\n",
    "#                 print(self.bullet_quantity)\n",
    "            \n",
    "        action = (agent_action, weapon_action)\n",
    "\n",
    "        return action\n",
    "\n",
    "\n",
    "    def create_entity(self, weapon_action):\n",
    "#         print(\"wa:\", weapon_action)\n",
    "        wep_type, wep_x, wep_y, angle = weapon_action\n",
    "        wep_xy = wep_x, wep_y\n",
    "        \n",
    "        if self.delay != 0:\n",
    "            self.delay -= 1\n",
    "        wep_type = 1\n",
    "        if wep_type == 1 and self.delay == 0:\n",
    "            ent = Entity(wep_type, wep_xy, angle, 10, BULLET_HEIGHT, BULLET_WIDTH)\n",
    "            ent_key = self.get_free_key()\n",
    "            if ent_key != None:\n",
    "                self.entity_dict[ent_key] = ent\n",
    "            # self.entity_list.append(ent)\n",
    "            self.delay = 20\n",
    "        if wep_type == 2 and self.delay == 0:\n",
    "            ent = Entity(wep_type, wep_xy, random.randint(5,12), random.randint(5,50), GRENDADE_SIZE, GRENDADE_SIZE)\n",
    "            self.entity_list.append(ent)\n",
    "            self.delay = 20\n",
    "    def update_entities(self):\n",
    "        # UPDATE ENTITIES\n",
    "        collided = []\n",
    "        for key, ent in self.entity_dict.items():\n",
    "            collide = ent.update((self.agent.xpos, self.agent.ypos))\n",
    "            if not collide:\n",
    "                if self.show:\n",
    "                    pygame.draw.rect(self.gameDisplay, blue, (ent.x, ent.y, ent.width, ent.height))\n",
    "            else:\n",
    "                self.agent.agent_reward = -20\n",
    "                self.generator_reward = 20\n",
    "                collided.append(key)\n",
    "        for ent_key in collided:\n",
    "            self.entity_free_keys.append(ent_key)\n",
    "            del self.entity_dict[ent_key]\n",
    "    def display_game(self):\n",
    "        if self.show:\n",
    "            self.gameDisplay = pygame.display.set_mode(self.game_dims, 0, 32)\n",
    "            self.gameDisplay.fill(white)\n",
    "    def display_background(self):\n",
    "        # DISPLAY BACKGROUND\n",
    "        if self.show:\n",
    "            pygame.font.init()\n",
    "            myFont = pygame.font.SysFont('Futura PT Light', 60)\n",
    "            textsurface = myFont.render('The Chosen One', False, black)\n",
    "            self.gameDisplay.blit(textsurface, (200,200))\n",
    "            pygame.display.update()\n",
    "    def set_default_rewards(self):\n",
    "        self.agent.agent_reward = 1\n",
    "        self.generator_reward = -1\n",
    "    global bullet_entity\n",
    "    def step(self, action, train=False):\n",
    "        # SET DEFAULT REWARDS FOR AGENT AND GENERATOR\n",
    "        self.set_default_rewards()\n",
    "\n",
    "        # DISPLAY GAME\n",
    "        self.display_game()\n",
    "        \n",
    "        agent_action, weapon_action = action\n",
    "        \n",
    "        # MOVE THE AGENT\n",
    "        self.agent.act(agent_action)\n",
    "        if self.show:\n",
    "            self.agent.display(self.gameDisplay)\n",
    "        \n",
    "        if train:\n",
    "            bullet_entity = weapon_action\n",
    "            if weapon_action[0] == 1:\n",
    "                self.bullet_quantity += 1\n",
    "        else:\n",
    "            bullet_entity = self.create_gun(self.Mouse_x, self.Mouse_y, self.agent.xpos) #create gun on click\n",
    "        \n",
    "        if self.bullet_quantity > 0:\n",
    "            # CREATE WEAPON ENTITY\n",
    "            self.create_entity(bullet_entity)\n",
    "            self.bullet_quantity -= 0.2\n",
    "        \n",
    "        # UPDATE ENTITIES\n",
    "        self.update_entities()\n",
    "        \n",
    "        # DISPLAY BACKGROUND\n",
    "        self.display_background()\n",
    "\n",
    "        \"\"\"RETURNS:\n",
    "        reward - (agent_reward, generator_reward)\n",
    "        state - getGameState()\n",
    "        done - CURRENT: DEFAULT: False\n",
    "        done - TODO: whether game is completed, e.g. HP <= 0\n",
    "        \"\"\"\n",
    "        reward = (self.agent.agent_reward, self.generator_reward)\n",
    "        state = self.getGameState()\n",
    "        return (reward, state, False)\n",
    "    def getGameState(self):\n",
    "        a = self.agent\n",
    "        agent_values = np.array([\n",
    "            a.jumps,\n",
    "            a.xpos//1000,\n",
    "            a.ypos//1000,\n",
    "            a.touchingObst,\n",
    "            a.gravityCurrent,\n",
    "            # TODO: height, width, dy, dx, direction, bounding box\n",
    "        ])\n",
    "        entity_values = np.array([])\n",
    "        for i in range(5):\n",
    "            if i in self.entity_dict:\n",
    "                e = self.entity_dict[i]\n",
    "                vals = [e.x//1000, e.y//1000, e.speed, e.angle]\n",
    "            else:\n",
    "                vals = [0,0,0,0]\n",
    "            entity_values = np.append(entity_values, vals)\n",
    "        values = np.append(agent_values, entity_values)\n",
    "        return values\n",
    "        \n",
    "    def create_gun(self, x , y, nX):\n",
    "        def rot_center(image, angle):\n",
    "            \"\"\"rotate an image while keeping its center and size\"\"\"\n",
    "            orig_rect = image.get_rect()\n",
    "            rot_image = pygame.transform.rotate(image, angle)\n",
    "            rot_rect = orig_rect.copy()\n",
    "            rot_rect.center = rot_image.get_rect().center\n",
    "            rot_image = rot_image.subsurface(rot_rect).copy()\n",
    "            return rot_image\n",
    "\n",
    "        gun = pygame.image.load('assets/assaultrifle2.png')\n",
    "        gunRect = gun.get_size()\n",
    "        gunScaled = pygame.transform.scale(gun, (180 , 180))\n",
    "        imagePosX = x - (gunRect[0]/2) + 0\n",
    "        if(nX > imagePosX):\n",
    "            imagePosX += 181\n",
    "        imagePosY = y - (gunRect[1]/2) + 90\n",
    "        scaleBy = 1\n",
    "        dx = imagePosX - self.MMouse_x + 90\n",
    "        dy = imagePosY - self.MMouse_y + 90\n",
    "        rads = math.atan2(-dy,dx)\n",
    "        degs = math.degrees(rads)\n",
    "        #gunScaled = pygame.transform.rotozoom(gunScaled, degs , scaleBy)\n",
    "        rot_img = rot_center(gunScaled, degs)\n",
    "        if self.show:\n",
    "            self.gameDisplay.blit(rot_img, (imagePosX, imagePosY)) #create image and center on mouse click\n",
    "        #pygame.display.update()\n",
    "        return (1, (imagePosX + 90,imagePosY+90), -degs)\n",
    "    \n",
    "    def reset(self):\n",
    "        \"\"\"Resets the game. Returns (reward, state, done).\"\"\"\n",
    "        self.__init__(game_dims=self.game_dims, show=self.show)\n",
    "        return self.getGameState()\n",
    "    \n",
    "    def test_step(self):\n",
    "        state = self.reset()\n",
    "        run = True\n",
    "        while run:\n",
    "            sleep(0.001)\n",
    "\n",
    "            # Create Gun at random place and angles\n",
    "#             agent_action = random.randint(0,2)\n",
    "#             print(\"State:\", state)\n",
    "            agent_action = self.chosen_one.act(state)\n",
    "#             print(agent_action)\n",
    "            \n",
    "            action = self.execute()\n",
    "            if action == -1:\n",
    "                break\n",
    "            _, weapon_action = action\n",
    "            \n",
    "#             print(weapon_action)\n",
    "            wep_x, wep_y, angle = weapon_action\n",
    "            weapon_action = (1, wep_x, wep_y, angle)\n",
    "            action = (agent_action, weapon_action)\n",
    "            reward, state, done = self.step(action)\n",
    "    \n",
    "    def load_chosen_one(self):\n",
    "        # FOR CONTINUOUS ACTION SPACE\n",
    "        obs_space = self.observation_space\n",
    "        obs_size = obs_space.low.size\n",
    "\n",
    "        \"\"\"AGENT HYPERPAREMETERS\"\"\"\n",
    "        agent_action_space = self.agent_action_space\n",
    "        agent_action_size = agent_action_space.n\n",
    "\n",
    "        # Q FUNCTION AND ADAM OPTIMIZER\n",
    "        agent_q_func = chainerrl.q_functions.FCStateQFunctionWithDiscreteAction(\n",
    "            obs_size, agent_action_size,\n",
    "            n_hidden_layers=5, n_hidden_channels=100)\n",
    "\n",
    "        # Use Adam to optimize q_func. eps=1e-2 is for stability.\n",
    "        agent_optimizer = chainer.optimizers.Adam(eps=1e-2)\n",
    "        agent_optimizer.setup(agent_q_func)\n",
    "\n",
    "        # Set the discount factor that discounts future rewards.\n",
    "        agent_gamma = 0.95\n",
    "\n",
    "        # Use epsilon-greedy for exploration\n",
    "        agent_explorer = chainerrl.explorers.ConstantEpsilonGreedy(\n",
    "            epsilon=0.3, random_action_func=self.agent_action_space.sample)\n",
    "\n",
    "        # DQN uses Experience Replay.\n",
    "        # Specify a replay buffer and its capacity.\n",
    "        agent_replay_buffer = chainerrl.replay_buffer.ReplayBuffer(capacity=10 ** 6)\n",
    "\n",
    "        # Since observations from CartPole-v0 is numpy.float64 while\n",
    "        # Chainer only accepts numpy.float32 by default, specify\n",
    "        # a converter as a feature extractor function phi.\n",
    "        agent_phi = lambda x: x.astype(np.float32, copy=False)\n",
    "\n",
    "        # CHOSEN ONE AGENT\n",
    "        chosen_one = chainerrl.agents.DoubleDQN(\n",
    "            agent_q_func, agent_optimizer, agent_replay_buffer, agent_gamma, agent_explorer,\n",
    "            replay_start_size=500, update_interval=1,\n",
    "            target_update_interval=100, phi=agent_phi)\n",
    "\n",
    "        chosen_one.load(\"chosen_one_model\")\n",
    "        chosen_one.get_statistics()\n",
    "\n",
    "        return chosen_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observation space: Box(25,)\n",
      "Agent action space: Discrete(3)\n",
      "Generator action space: Box(4,)\n",
      "State: [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0.]\n"
     ]
    }
   ],
   "source": [
    "env = Env(game_dims=(700, 500), show=False, test=False)\n",
    "\n",
    "# SHOW ENVIRONMENT VALUES\n",
    "print('Observation space:', env.observation_space)\n",
    "print('Agent action space:', env.agent_action_space)\n",
    "print('Generator action space:', env.generator_action_space)\n",
    "\n",
    "state = env.reset()\n",
    "print(\"State:\", state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Create Gun at random place and angles\n",
    "# agent_action = random.randint(0,2)\n",
    "\n",
    "# wep_type = 1  # gun\n",
    "# wep_xy = (50, 700)\n",
    "# wep_x, wep_y = wep_xy\n",
    "# angle = 20\n",
    "\n",
    "# generator_action = (wep_type, wep_x, wep_y, angle)\n",
    "# action = (agent_action, generator_action)\n",
    "# reward, state, done = env.step(action)\n",
    "# print(\"Agent Reward:\", reward[0])\n",
    "# print(\"Generator Reward:\", reward[1])\n",
    "# print(\"State:\", state)\n",
    "# print(\"Done:\", done)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FOR CONTINUOUS ACTION SPACE\n",
    "obs_space = env.observation_space\n",
    "obs_size = obs_space.low.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"AGENT HYPERPAREMETERS\"\"\"\n",
    "agent_action_space = env.agent_action_space\n",
    "agent_action_size = agent_action_space.n\n",
    "\n",
    "# Q FUNCTION AND ADAM OPTIMIZER\n",
    "agent_q_func = chainerrl.q_functions.FCStateQFunctionWithDiscreteAction(\n",
    "    obs_size, agent_action_size,\n",
    "    n_hidden_layers=5, n_hidden_channels=100)\n",
    "\n",
    "# Use Adam to optimize q_func. eps=1e-2 is for stability.\n",
    "agent_optimizer = chainer.optimizers.Adam(eps=1e-2)\n",
    "agent_optimizer.setup(agent_q_func)\n",
    "\n",
    "# Set the discount factor that discounts future rewards.\n",
    "agent_gamma = 0.995\n",
    "\n",
    "# Use epsilon-greedy for exploration\n",
    "agent_explorer = chainerrl.explorers.ConstantEpsilonGreedy(\n",
    "    epsilon=0.3, random_action_func=env.agent_action_space.sample)\n",
    "\n",
    "# DQN uses Experience Replay.\n",
    "# Specify a replay buffer and its capacity.\n",
    "agent_replay_buffer = chainerrl.replay_buffer.ReplayBuffer(capacity=10 ** 6)\n",
    "\n",
    "# Since observations from CartPole-v0 is numpy.float64 while\n",
    "# Chainer only accepts numpy.float32 by default, specify\n",
    "# a converter as a feature extractor function phi.\n",
    "agent_phi = lambda x: x.astype(np.float32, copy=False)\n",
    "\n",
    "# CHOSEN ONE AGENT\n",
    "chosen_one = chainerrl.agents.DoubleDQN(\n",
    "    agent_q_func, agent_optimizer, agent_replay_buffer, agent_gamma, agent_explorer,\n",
    "    replay_start_size=500, update_interval=1,\n",
    "    target_update_interval=100, phi=agent_phi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"GENERATOR HYPERPARAMETERS\"\"\"\n",
    "# generator_action_space = env.generator_action_space\n",
    "# generator_action_size = generator_action_space.low.size\n",
    "\n",
    "# # Q FUNCTION FOR CONTINUOUS VARIABLES\n",
    "# generator_q_func = chainerrl.q_functions.FCQuadraticStateQFunction(\n",
    "#     obs_size, generator_action_size,\n",
    "#     n_hidden_layers=5,\n",
    "#     n_hidden_channels=100,\n",
    "#     action_space=generator_action_space,\n",
    "# )\n",
    "\n",
    "# # Use the Ornstein-Uhlenbeck process for exploration\n",
    "# generator_ou_sigma = (generator_action_space.high - generator_action_space.low) * 0.2\n",
    "# generator_explorer = explorers.AdditiveOU(sigma=generator_ou_sigma)\n",
    "\n",
    "# # Use Adam to optimize q_func. eps=1e-2 is for stability.\n",
    "# generator_optimizer = chainer.optimizers.Adam(eps=1e-2)\n",
    "# generator_optimizer.setup(generator_q_func)\n",
    "\n",
    "# # Set the discount factor that discounts future rewards.\n",
    "# generator_gamma = 0.95\n",
    "\n",
    "# # DQN uses Experience Replay.\n",
    "# # Specify a replay buffer and its capacity.\n",
    "# generator_replay_buffer = chainerrl.replay_buffer.ReplayBuffer(capacity=10 ** 6)\n",
    "\n",
    "# # Since observations from CartPole-v0 is numpy.float64 while\n",
    "# # Chainer only accepts numpy.float32 by default, specify\n",
    "# # a converter as a feature extractor function phi.\n",
    "# generator_phi = lambda x: x.astype(np.float32, copy=False)\n",
    "\n",
    "# # GENERATOR AGENT\n",
    "# generator = chainerrl.agents.DoubleDQN(\n",
    "#     generator_q_func, generator_optimizer, generator_replay_buffer, generator_gamma, generator_explorer,\n",
    "#     replay_start_size=500, update_interval=1,\n",
    "#     target_update_interval=100, phi=generator_phi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Gun at random place and angles\n",
    "def random_generator():\n",
    "    wep_type = 1  # gun\n",
    "    wep_xy = (300, 50)\n",
    "    wep_x, wep_y = wep_xy\n",
    "    angle = random.randint(0, 180)\n",
    "\n",
    "    generator_action = (wep_type, wep_x, wep_y, angle)\n",
    "    return generator_action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep: 1 R_agent: 59 agent stat: [('average_q', 0.049428596888345784), ('average_loss', 0), ('n_updates', 0)]\n",
      "ep: 2 R_agent: 59 agent stat: [('average_q', 1.3023650930436987), ('average_loss', 0.8762904407997203), ('n_updates', 500)]\n",
      "ep: 3 R_agent: 59 agent stat: [('average_q', 3.9905193086896036), ('average_loss', 0.8871740913308757), ('n_updates', 1000)]\n",
      "ep: 4 R_agent: 17 agent stat: [('average_q', 7.397228225204353), ('average_loss', 0.9678035094282564), ('n_updates', 1500)]\n",
      "ep: 5 R_agent: 17 agent stat: [('average_q', 11.025302104235157), ('average_loss', 1.0252214755778564), ('n_updates', 2000)]\n",
      "ep: 6 R_agent: 17 agent stat: [('average_q', 14.681930549387314), ('average_loss', 1.054911879230721), ('n_updates', 2500)]\n",
      "ep: 7 R_agent: 59 agent stat: [('average_q', 18.34007138583503), ('average_loss', 0.9479438549951417), ('n_updates', 3000)]\n",
      "ep: 8 R_agent: -4 agent stat: [('average_q', 22.82862105349215), ('average_loss', 1.0716724220337663), ('n_updates', 3500)]\n",
      "ep: 9 R_agent: 38 agent stat: [('average_q', 27.824252491913413), ('average_loss', 1.116277019169835), ('n_updates', 4000)]\n",
      "ep: 10 R_agent: 17 agent stat: [('average_q', 32.81828317596963), ('average_loss', 1.0043286889750018), ('n_updates', 4500)]\n",
      "ep: 11 R_agent: 17 agent stat: [('average_q', 37.093378772134244), ('average_loss', 1.1093807398584314), ('n_updates', 5000)]\n",
      "ep: 12 R_agent: 17 agent stat: [('average_q', 40.903400815435376), ('average_loss', 1.2203918350474439), ('n_updates', 5500)]\n",
      "ep: 13 R_agent: -4 agent stat: [('average_q', 44.75077570188146), ('average_loss', 1.0767076825571222), ('n_updates', 6000)]\n",
      "ep: 14 R_agent: 17 agent stat: [('average_q', 47.994437687664), ('average_loss', 1.086142973878003), ('n_updates', 6500)]\n",
      "ep: 15 R_agent: -4 agent stat: [('average_q', 51.064577176746226), ('average_loss', 1.1534897394249308), ('n_updates', 7000)]\n",
      "ep: 16 R_agent: 59 agent stat: [('average_q', 54.2599276932624), ('average_loss', 1.0638148441860904), ('n_updates', 7500)]\n",
      "ep: 17 R_agent: 38 agent stat: [('average_q', 57.18420011288037), ('average_loss', 1.0268255144447935), ('n_updates', 8000)]\n",
      "ep: 18 R_agent: 17 agent stat: [('average_q', 60.1564928394579), ('average_loss', 1.0120277247096987), ('n_updates', 8500)]\n",
      "ep: 19 R_agent: 38 agent stat: [('average_q', 62.86578866548256), ('average_loss', 1.0392989743918293), ('n_updates', 9000)]\n",
      "ep: 20 R_agent: 38 agent stat: [('average_q', 65.72492217176426), ('average_loss', 1.0751420261837388), ('n_updates', 9500)]\n",
      "ep: 21 R_agent: 17 agent stat: [('average_q', 68.5184340642095), ('average_loss', 1.0409604312994927), ('n_updates', 10000)]\n",
      "ep: 22 R_agent: 38 agent stat: [('average_q', 70.55553134519651), ('average_loss', 1.2178173304476108), ('n_updates', 10500)]\n",
      "ep: 23 R_agent: 59 agent stat: [('average_q', 74.08844007934273), ('average_loss', 1.1789266164921732), ('n_updates', 11000)]\n",
      "ep: 24 R_agent: 17 agent stat: [('average_q', 76.83147721558012), ('average_loss', 1.1693118511888927), ('n_updates', 11500)]\n"
     ]
    }
   ],
   "source": [
    "# MAIN TRAINING LOOP\n",
    "n_episodes = 1000\n",
    "max_episode_len = 500\n",
    "\n",
    "R_agent_history = []\n",
    "R_generator_history = []\n",
    "for i in range(1, n_episodes + 1):\n",
    "    obs = env.reset()\n",
    "    agent_reward = 0\n",
    "    generator_reward = 0\n",
    "    done = False\n",
    "    R_agent = 0  # return (sum of rewards)\n",
    "    R_generator = 0\n",
    "    t = 0  # time step\n",
    "    while not done and t < max_episode_len:\n",
    "        # Uncomment to watch the behaviour\n",
    "        # env.render()\n",
    "#         print(obs)\n",
    "        agent_action = chosen_one.act_and_train(obs, agent_reward)\n",
    "#         generator_action = generator.act_and_train(obs, generator_reward)\n",
    "        generator_action = random_generator()\n",
    "#         print(generator_action)\n",
    "        action = (agent_action, generator_action)\n",
    "        \n",
    "        (reward, obs, done) = env.step(action, train=True)\n",
    "        agent_reward, generator_reward = reward\n",
    "\n",
    "        R_agent += agent_reward\n",
    "        R_generator += generator_reward\n",
    "        R_agent_history.append(R_agent)\n",
    "        R_generator_history.append(R_generator)\n",
    "        t += 1\n",
    "    if i % 1 == 0:\n",
    "        print('ep:', i,\n",
    "              'R_agent:', R_agent,\n",
    "#               'R_generator:', R_generator,\n",
    "              'agent stat:', chosen_one.get_statistics())\n",
    "#               'generator statistics:', generator.get_statistics())\n",
    "    chosen_one.stop_episode_and_train(obs, agent_reward, done)\n",
    "#     generator.stop_episode_and_train(obs, generator_reward, done)\n",
    "print('Finished.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chosen_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chosen_one.save(\"chosen_one_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator.save(\"generator_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "the-chosen-one",
   "language": "python",
   "name": "the-chosen-one"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}