organism.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How many generations ?\n",
      "20\n",
      "Gen :  1\n",
      "Episode: 100\n",
      "Total time in this generation is : 9838, average time taken per episode of generation is : 98.38\n",
      "Total penalty : 885 and average penalty : 8.85\n",
      "The number of times the organism ate : 3\n",
      "Training finished.\n",
      "\n",
      "Gen :  2\n",
      "Episode: 100\n",
      "Total time in this generation is : 4953, average time taken per episode of generation is : 49.53\n",
      "Total penalty : 495 and average penalty : 4.95\n",
      "The number of times the organism ate : 1\n",
      "Training finished.\n",
      "\n",
      "Gen :  3\n",
      "Episode: 100\n",
      "Total time in this generation is : 4790, average time taken per episode of generation is : 47.9\n",
      "Total penalty : 590 and average penalty : 5.9\n",
      "The number of times the organism ate : 6\n",
      "Training finished.\n",
      "\n",
      "Gen :  4\n",
      "Episode: 100\n",
      "Total time in this generation is : 4878, average time taken per episode of generation is : 48.78\n",
      "Total penalty : 553 and average penalty : 5.53\n",
      "The number of times the organism ate : 5\n",
      "Training finished.\n",
      "\n",
      "Gen :  5\n",
      "Episode: 100\n",
      "Total time in this generation is : 4837, average time taken per episode of generation is : 48.37\n",
      "Total penalty : 534 and average penalty : 5.34\n",
      "The number of times the organism ate : 5\n",
      "Training finished.\n",
      "\n",
      "Gen :  6\n",
      "Episode: 100\n",
      "Total time in this generation is : 4795, average time taken per episode of generation is : 47.95\n",
      "Total penalty : 404 and average penalty : 4.04\n",
      "The number of times the organism ate : 6\n",
      "Training finished.\n",
      "\n",
      "Gen :  7\n",
      "Episode: 100\n",
      "Total time in this generation is : 4716, average time taken per episode of generation is : 47.16\n",
      "Total penalty : 271 and average penalty : 2.71\n",
      "The number of times the organism ate : 9\n",
      "Training finished.\n",
      "\n",
      "Gen :  8\n",
      "Episode: 100\n",
      "Total time in this generation is : 4819, average time taken per episode of generation is : 48.19\n",
      "Total penalty : 241 and average penalty : 2.41\n",
      "The number of times the organism ate : 8\n",
      "Training finished.\n",
      "\n",
      "Gen :  9\n",
      "Episode: 100\n",
      "Total time in this generation is : 4831, average time taken per episode of generation is : 48.31\n",
      "Total penalty : 135 and average penalty : 1.35\n",
      "The number of times the organism ate : 7\n",
      "Training finished.\n",
      "\n",
      "Gen :  10\n",
      "Episode: 100\n",
      "Total time in this generation is : 4741, average time taken per episode of generation is : 47.41\n",
      "Total penalty : 120 and average penalty : 1.2\n",
      "The number of times the organism ate : 8\n",
      "Training finished.\n",
      "\n",
      "Gen :  11\n",
      "Episode: 100\n",
      "Total time in this generation is : 4691, average time taken per episode of generation is : 46.91\n",
      "Total penalty : 101 and average penalty : 1.01\n",
      "The number of times the organism ate : 8\n",
      "Training finished.\n",
      "\n",
      "Gen :  12\n",
      "Episode: 100\n",
      "Total time in this generation is : 4668, average time taken per episode of generation is : 46.68\n",
      "Total penalty : 119 and average penalty : 1.19\n",
      "The number of times the organism ate : 10\n",
      "Training finished.\n",
      "\n",
      "Gen :  13\n",
      "Episode: 100\n",
      "Total time in this generation is : 4498, average time taken per episode of generation is : 44.98\n",
      "Total penalty : 86 and average penalty : 0.86\n",
      "The number of times the organism ate : 16\n",
      "Training finished.\n",
      "\n",
      "Gen :  14\n",
      "Episode: 100\n",
      "Total time in this generation is : 4750, average time taken per episode of generation is : 47.5\n",
      "Total penalty : 96 and average penalty : 0.96\n",
      "The number of times the organism ate : 10\n",
      "Training finished.\n",
      "\n",
      "Gen :  15\n",
      "Episode: 100\n",
      "Total time in this generation is : 4598, average time taken per episode of generation is : 45.98\n",
      "Total penalty : 96 and average penalty : 0.96\n",
      "The number of times the organism ate : 14\n",
      "Training finished.\n",
      "\n",
      "Gen :  16\n",
      "Episode: 100\n",
      "Total time in this generation is : 4641, average time taken per episode of generation is : 46.41\n",
      "Total penalty : 120 and average penalty : 1.2\n",
      "The number of times the organism ate : 9\n",
      "Training finished.\n",
      "\n",
      "Gen :  17\n",
      "Episode: 100\n",
      "Total time in this generation is : 4658, average time taken per episode of generation is : 46.58\n",
      "Total penalty : 92 and average penalty : 0.92\n",
      "The number of times the organism ate : 15\n",
      "Training finished.\n",
      "\n",
      "Gen :  18\n",
      "Episode: 100\n",
      "Total time in this generation is : 4704, average time taken per episode of generation is : 47.04\n",
      "Total penalty : 83 and average penalty : 0.83\n",
      "The number of times the organism ate : 11\n",
      "Training finished.\n",
      "\n",
      "Gen :  19\n",
      "Episode: 100\n",
      "Total time in this generation is : 4520, average time taken per episode of generation is : 45.2\n",
      "Total penalty : 108 and average penalty : 1.08\n",
      "The number of times the organism ate : 12\n",
      "Training finished.\n",
      "\n",
      "Gen :  20\n",
      "Episode: 100\n",
      "Total time in this generation is : 4769, average time taken per episode of generation is : 47.69\n",
      "Total penalty : 102 and average penalty : 1.02\n",
      "The number of times the organism ate : 8\n",
      "Training finished.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import gym_organism\n",
    "\n",
    "env = gym.make(\"Organism-v0\")\n",
    "\n",
    "env.reset() # reset environment to a new, random state\n",
    "\n",
    "q_table = np.zeros([env.observation_space.n, env.action_space.n])\n",
    "\n",
    "\n",
    "\"\"\"Training the agent\"\"\"\n",
    "print(\"How many generations ?\")\n",
    "gen = int(input())\n",
    "\n",
    "import random\n",
    "from time import sleep\n",
    "from IPython.display import clear_output\n",
    "\n",
    "# Hyperparameters\n",
    "alpha = 0.1\n",
    "gamma = 0.6\n",
    "epsilon = 0.1\n",
    "gen_done = False\n",
    "for j in range(gen):\n",
    "    \n",
    "    print(\"Gen : \", j+1)\n",
    "    time_taken = 0\n",
    "    total_penalty = 0\n",
    "    # For plotting metrics\n",
    "    all_epochs = []\n",
    "    all_penalties = []\n",
    "    frames = []\n",
    "    times_eaten = 0\n",
    "    if j == 0:\n",
    "        #If it is the first generation\n",
    "        for i in range(1, 101):\n",
    "            epochs, penalties, reward, = 0, 0, 0\n",
    "            done = False\n",
    "            state = env.reset()\n",
    "            while epochs<100 and not done:\n",
    "                if random.uniform(0, 1) < epsilon:\n",
    "                    action = env.action_space.sample() # Explore action space\n",
    "                else:\n",
    "                    action = np.argmax(q_table[state]) # Exploit learned values\n",
    "\n",
    "                next_state, reward, done, info = env.step(action)\n",
    "\n",
    "                old_value = q_table[state, action]\n",
    "                next_max = np.max(q_table[next_state])\n",
    "\n",
    "                new_value = (1 - alpha) * old_value + alpha * (reward + gamma * next_max)\n",
    "                q_table[state, action] = new_value\n",
    "\n",
    "                if reward == -10:\n",
    "                    penalties += 1\n",
    "\n",
    "                frames.append({\n",
    "                'frame': env.render(mode='ansi'),\n",
    "                'state': state,\n",
    "                'action': action,\n",
    "                'reward': reward\n",
    "                }\n",
    "            )\n",
    "\n",
    "    #             state = next_state\n",
    "\n",
    "    #             epochs += 1\n",
    "\n",
    "    #         if i % 100 == 0:\n",
    "    #             clear_output(wait=True)\n",
    "    #             print(f\"Episode: {i}\")\n",
    "    #         time_taken += epochs\n",
    "    #     print(f\"Total time in this generation is : {time_taken}\")\n",
    "    #     print(\"Training finished.\\n\")\n",
    "                state = next_state\n",
    "\n",
    "                epochs += 1\n",
    "    #         print(done)\n",
    "            if done:\n",
    "                times_eaten+=1\n",
    "                gen_done = True\n",
    "            if i % 100 == 0:\n",
    "                print(f\"Episode: {i}\")\n",
    "            time_taken += epochs\n",
    "            total_penalty += penalties\n",
    "    else:\n",
    "        if gen_done:\n",
    "            gen_done = False\n",
    "            for i in range(1, 101):\n",
    "                epochs, penalties, reward, = 0, 0, 0\n",
    "                done = False\n",
    "                state = env.reset()\n",
    "                while epochs<50 and not done:\n",
    "                    if random.uniform(0, 1) < epsilon:\n",
    "                        action = env.action_space.sample() # Explore action space\n",
    "                    else:\n",
    "                        action = np.argmax(q_table[state]) # Exploit learned values\n",
    "\n",
    "                    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "                    old_value = q_table[state, action]\n",
    "                    next_max = np.max(q_table[next_state])\n",
    "\n",
    "                    new_value = (1 - alpha) * old_value + alpha * (reward + gamma * next_max)\n",
    "                    q_table[state, action] = new_value\n",
    "\n",
    "                    if reward == -10:\n",
    "                        penalties += 1\n",
    "\n",
    "                    frames.append({\n",
    "                    'frame': env.render(mode='ansi'),\n",
    "                    'state': state,\n",
    "                    'action': action,\n",
    "                    'reward': reward\n",
    "                    }\n",
    "                )\n",
    "\n",
    "        #             state = next_state\n",
    "\n",
    "        #             epochs += 1\n",
    "\n",
    "        #         if i % 100 == 0:\n",
    "        #             clear_output(wait=True)\n",
    "        #             print(f\"Episode: {i}\")\n",
    "        #         time_taken += epochs\n",
    "        #     print(f\"Total time in this generation is : {time_taken}\")\n",
    "        #     print(\"Training finished.\\n\")\n",
    "                    state = next_state\n",
    "\n",
    "                    epochs += 1\n",
    "        #         print(done)\n",
    "                if done:\n",
    "                    times_eaten+=1\n",
    "                    gen_done = True\n",
    "                if i % 100 == 0:\n",
    "                    print(f\"Episode: {i}\")\n",
    "                time_taken += epochs\n",
    "                total_penalty += penalties\n",
    "#     final_frames.append(frames)\n",
    "    print(f\"Total time in this generation is : {time_taken}, average time taken per episode of generation is : {time_taken/i}\")\n",
    "    print(f\"Total penalty : {total_penalty} and average penalty : {total_penalty/100}\")\n",
    "    print(f\"The number of times the organism ate : {times_eaten}\")\n",
    "    print(\"Training finished.\\n\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------------------------------+\n",
      "| : | : :\u001b[35mx\u001b[0m: : : : : : : : : : : : : : | |\n",
      "| : | : : : : | : | : : : : : : : : : | |\n",
      "| : : : : : : : : : : : | : : : : | : : |\n",
      "| | : | : : | : :x: : : : : : : : : : : |\n",
      "|x| : | : : : : : | : : | : : | : : : : |\n",
      "| : : : : : | : : : : : : : : : : : : :x|\n",
      "| : : : | : | : : : : : : : : | : : : : |\n",
      "| : | : : | : : : | : : : : : | : : : | |\n",
      "| : : : : : : : : : : : : : : : : : : : |\n",
      "| : | : : : : : | :x: : | : : : : : : : |\n",
      "| : | : : : : : | : : : | : : : : : : : |\n",
      "| : : :\u001b[43m \u001b[0m: | : : : : : : : : : : : :x: : |\n",
      "| : | : : : : : | : : : : : : | : : : : |\n",
      "| : : : | : : : : : | : : | : | : : : :x|\n",
      "| : | : : : : : : : | : : : : : : : : : |\n",
      "| : : : :x| : : | : : : : : | : : | : : |\n",
      "| : : : : : : : | : : : : | : : : : : : |\n",
      "| : : | : : : : : : : | : : : : : | : | |\n",
      "| : : : : : : : | : : : : : : : : : : : |\n",
      "| : : | : : : : : : : | : : |x: : : : : |\n",
      "+---------------------------------------+\n",
      "  (West)\n",
      "\n",
      "Timestep: 45\n",
      "State: 2018\n",
      "Action: 3\n",
      "Reward: -1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-8d2d4beb42bb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m         \u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m.1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mprint_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-7-8d2d4beb42bb>\u001b[0m in \u001b[0;36mprint_frames\u001b[1;34m(frames)\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"Action: {frame['action']}\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"Reward: {frame['reward']}\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m         \u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m.1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mprint_frames\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mframes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def print_frames(frames):\n",
    "    for i, frame in enumerate(frames):\n",
    "        clear_output(wait=True)\n",
    "        print(frame['frame'])\n",
    "        print(f\"Timestep: {i + 1}\")\n",
    "        print(f\"State: {frame['state']}\")\n",
    "        print(f\"Action: {frame['action']}\")\n",
    "        print(f\"Reward: {frame['reward']}\")\n",
    "        sleep(.1)\n",
    "\n",
    "print_frames(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}