[Lunar Lander] Deep Q-Network (DQN).ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kuds/rl-lunar-lander/blob/main/%5BLunar%20Lander%5D%20Deep%20Q-Network%20(DQN).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nS90f-pm5nV9"
      },
      "source": [
        "# Deep Q-Network (DQN)\n",
        "---\n",
        "In this notebook, you will implement a DQN agent with OpenAI Gym's LunarLander-v2 environment.\n",
        "\n",
        "### 1. Import the Necessary Packages"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install swig"
      ],
      "metadata": {
        "id": "i8cD0fYr5sVm",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gymnasium gymnasium[box2d] stable_baselines3"
      ],
      "metadata": {
        "id": "jLnLyxAJ6saY",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "aZ1YZWDq5nV-"
      },
      "outputs": [],
      "source": [
        "import gymnasium\n",
        "from gymnasium.wrappers import RecordVideo\n",
        "\n",
        "import stable_baselines3\n",
        "from stable_baselines3 import DQN\n",
        "from stable_baselines3.common.evaluation import evaluate_policy\n",
        "from stable_baselines3.common.env_util import make_vec_env\n",
        "from stable_baselines3.common.monitor import Monitor\n",
        "from stable_baselines3.common.callbacks import EvalCallback\n",
        "from stable_baselines3.common.vec_env import VecVideoRecorder, DummyVecEnv\n",
        "\n",
        "from collections import namedtuple, deque\n",
        "import torch.optim as optim\n",
        "import os\n",
        "import platform\n",
        "import time\n",
        "import random\n",
        "import numpy\n",
        "import torch\n",
        "import torch.backends.quantized\n",
        "import scipy\n",
        "import IPython\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot\n",
        "from importlib.metadata import version\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Python Version: {platform.python_version()}\")\n",
        "print(f\"Torch Version: {version('torch')}\")\n",
        "print(f\"Is Cuda Available: {torch.cuda.is_available()}\")\n",
        "print(f\"Cuda Version: {torch.version.cuda}\")\n",
        "print(f\"Gymnasium Version: {version('gymnasium')}\")\n",
        "print(f\"Numpy Version: {version('numpy')}\")\n",
        "print(f\"Scipy Version: {version('scipy')}\")\n",
        "print(f\"Stable Baselines3 Version: {version('stable_baselines3')}\")\n",
        "print(f\"IPython Version: {version('ipython')}\")"
      ],
      "metadata": {
        "id": "CXsFLvy-458L",
        "outputId": "dfc1e494-0b9e-4088-d55d-ac93bcb91d5b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python Version: 3.10.12\n",
            "Torch Version: 2.4.0+cu121\n",
            "Is Cuda Available: True\n",
            "Cuda Version: 12.1\n",
            "Gymnasium Version: 0.29.1\n",
            "Numpy Version: 1.26.4\n",
            "Scipy Version: 1.13.1\n",
            "Stable Baselines3 Version: 2.3.2\n",
            "IPython Version: 7.34.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "backend = torch.backends.quantized.engine\n",
        "print(f\"Currently using backend: {backend}\")"
      ],
      "metadata": {
        "id": "PiRLHjAkTN6z",
        "outputId": "4c4ea7d1-be6c-4917-a72b-e6704ef83af6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Currently using backend: x86\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R8ps4GLm5nV-"
      },
      "source": [
        "### 2. Instantiate the Environment and Agent\n",
        "\n",
        "Initialize the environment in the code cell below."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0O8Hk6tG5nV-",
        "outputId": "ea188583-d202-4f3b-cb91-ac4dce54c664"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "State shape:  (8,)\n",
            "Number of actions:  4\n"
          ]
        }
      ],
      "source": [
        "env = gymnasium.make('LunarLander-v2')\n",
        "print('State shape: ', env.observation_space.shape)\n",
        "print('Number of actions: ', env.action_space.n)\n",
        "env.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "n97od6Ll5nV_"
      },
      "outputs": [],
      "source": [
        "env = gymnasium.make('LunarLander-v2')\n",
        "env.reset()\n",
        "for step in range(200):\n",
        "    env.render()\n",
        "    action = env.action_space.sample()\n",
        "    observation, reward, terminated, truncated, info = env.step(action)\n",
        "    if terminated or truncated:\n",
        "        env.reset()\n",
        "env.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tlYEpVzh5nV_"
      },
      "source": [
        "### 3. Train the Agent with DQN\n",
        "\n",
        "Run the code cell below to train the agent from scratch.  You are welcome to amend the supplied values of the parameters in the function, to try to see if you can get better performance!\n",
        "\n",
        "Alternatively, you can skip to the next step below (**4. Watch a Smart Agent!**), to load the saved model weights from a pre-trained agent."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class QNetwork(nn.Module):\n",
        "    \"\"\"Actor (Policy) Model.\"\"\"\n",
        "\n",
        "    def __init__(self, state_size, action_size, seed, fc1_units=128, fc2_units=128):\n",
        "        \"\"\"Initialize parameters and build model.\n",
        "        Params\n",
        "        ======\n",
        "            state_size (int): Dimension of each state\n",
        "            action_size (int): Dimension of each action\n",
        "            seed (int): Random seed\n",
        "            fc1_units (int): Number of nodes in first hidden layer\n",
        "            fc2_units (int): Number of nodes in second hidden layer\n",
        "        \"\"\"\n",
        "        super(QNetwork, self).__init__()\n",
        "        self.seed = torch.manual_seed(seed)\n",
        "        self.fc1 = nn.Linear(state_size, fc1_units)\n",
        "        self.fc2 = nn.Linear(fc1_units, fc2_units)\n",
        "        self.fc3 = nn.Linear(fc2_units, action_size)\n",
        "\n",
        "    def forward(self, state):\n",
        "        \"\"\"Build a network that maps state -> action values.\"\"\"\n",
        "        x = F.relu(self.fc1(state))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        return self.fc3(x)"
      ],
      "metadata": {
        "id": "8AQGk5RP81a-"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "BUFFER_SIZE = int(1e5)  # replay buffer size\n",
        "BATCH_SIZE = 64         # minibatch size\n",
        "GAMMA = 0.99            # discount factor\n",
        "TAU = 1e-3              # for soft update of target parameters\n",
        "LR = 5e-4               # learning rate\n",
        "UPDATE_EVERY = 4        # how often to update the network\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "class Agent():\n",
        "    \"\"\"Interacts with and learns from the environment.\"\"\"\n",
        "\n",
        "    def __init__(self, state_size, action_size, seed):\n",
        "        \"\"\"Initialize an Agent object.\n",
        "\n",
        "        Params\n",
        "        ======\n",
        "            state_size (int): dimension of each state\n",
        "            action_size (int): dimension of each action\n",
        "            seed (int): random seed\n",
        "        \"\"\"\n",
        "        self.state_size = state_size\n",
        "        self.action_size = action_size\n",
        "        self.seed = random.seed(seed)\n",
        "\n",
        "        # Q-Network\n",
        "        self.qnetwork_local = QNetwork(state_size, action_size, seed).to(device)\n",
        "        self.qnetwork_target = QNetwork(state_size, action_size, seed).to(device)\n",
        "        self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)\n",
        "\n",
        "        # Replay memory\n",
        "        self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)\n",
        "        # Initialize time step (for updating every UPDATE_EVERY steps)\n",
        "        self.t_step = 0\n",
        "\n",
        "    def step(self, state, action, reward, next_state, done):\n",
        "        # Save experience in replay memory\n",
        "        self.memory.add(state, action, reward, next_state, done)\n",
        "\n",
        "        # Learn every UPDATE_EVERY time steps.\n",
        "        self.t_step = (self.t_step + 1) % UPDATE_EVERY\n",
        "        if self.t_step == 0:\n",
        "            # If enough samples are available in memory, get random subset and learn\n",
        "            if len(self.memory) > BATCH_SIZE:\n",
        "                experiences = self.memory.sample()\n",
        "                self.learn(experiences, GAMMA)\n",
        "\n",
        "    def act(self, state, eps=0.):\n",
        "        \"\"\"Returns actions for given state as per current policy.\n",
        "\n",
        "        Params\n",
        "        ======\n",
        "            state (array_like): current state\n",
        "            eps (float): epsilon, for epsilon-greedy action selection\n",
        "        \"\"\"\n",
        "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
        "        self.qnetwork_local.eval()\n",
        "        with torch.no_grad():\n",
        "            action_values = self.qnetwork_local(state)\n",
        "        self.qnetwork_local.train()\n",
        "\n",
        "        # Epsilon-greedy action selection\n",
        "        if random.random() > eps:\n",
        "            return np.argmax(action_values.cpu().data.numpy())\n",
        "        else:\n",
        "            return random.choice(np.arange(self.action_size))\n",
        "\n",
        "    def learn(self, experiences, gamma):\n",
        "        \"\"\"Update value parameters using given batch of experience tuples.\n",
        "\n",
        "        Params\n",
        "        ======\n",
        "            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples\n",
        "            gamma (float): discount factor\n",
        "        \"\"\"\n",
        "        states, actions, rewards, next_states, dones = experiences\n",
        "\n",
        "        # Get max predicted Q values (for next states) from target model\n",
        "        Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)\n",
        "        # Compute Q targets for current states\n",
        "        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))\n",
        "\n",
        "        # Get expected Q values from local model\n",
        "        Q_expected = self.qnetwork_local(states).gather(1, actions)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = F.mse_loss(Q_expected, Q_targets)\n",
        "        # Minimize the loss\n",
        "        self.optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        # ------------------- update target network ------------------- #\n",
        "        self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)\n",
        "\n",
        "    def soft_update(self, local_model, target_model, tau):\n",
        "        \"\"\"Soft update model parameters.\n",
        "        θ_target = τ*θ_local + (1 - τ)*θ_target\n",
        "\n",
        "        Params\n",
        "        ======\n",
        "            local_model (PyTorch model): weights will be copied from\n",
        "            target_model (PyTorch model): weights will be copied to\n",
        "            tau (float): interpolation parameter\n",
        "        \"\"\"\n",
        "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
        "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)\n",
        "\n",
        "\n",
        "class ReplayBuffer:\n",
        "    \"\"\"Fixed-size buffer to store experience tuples.\"\"\"\n",
        "\n",
        "    def __init__(self, action_size, buffer_size, batch_size, seed):\n",
        "        \"\"\"Initialize a ReplayBuffer object.\n",
        "\n",
        "        Params\n",
        "        ======\n",
        "            action_size (int): dimension of each action\n",
        "            buffer_size (int): maximum size of buffer\n",
        "            batch_size (int): size of each training batch\n",
        "            seed (int): random seed\n",
        "        \"\"\"\n",
        "        self.action_size = action_size\n",
        "        self.memory = deque(maxlen=buffer_size)\n",
        "        self.batch_size = batch_size\n",
        "        self.experience = namedtuple(\"Experience\", field_names=[\"state\", \"action\", \"reward\", \"next_state\", \"done\"])\n",
        "        self.seed = random.seed(seed)\n",
        "\n",
        "    def add(self, state, action, reward, next_state, done):\n",
        "        \"\"\"Add a new experience to memory.\"\"\"\n",
        "        e = self.experience(state, action, reward, next_state, done)\n",
        "        self.memory.append(e)\n",
        "\n",
        "    def sample(self):\n",
        "        \"\"\"Randomly sample a batch of experiences from memory.\"\"\"\n",
        "        experiences = random.sample(self.memory, k=self.batch_size)\n",
        "\n",
        "        states = torch.from_numpy(np.vstack([e.state for e in experiences if e is not None])).float().to(device)\n",
        "        actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(device)\n",
        "        rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(device)\n",
        "        next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])).float().to(device)\n",
        "        dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(device)\n",
        "\n",
        "        return (states, actions, rewards, next_states, dones)\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"Return the current size of internal memory.\"\"\"\n",
        "        return len(self.memory)"
      ],
      "metadata": {
        "id": "ANgyqOQA8yVm"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "Vn1axrI35nV_"
      },
      "outputs": [],
      "source": [
        "def dqn(n_episodes=4000, max_t=1000, eps_start=1.0, eps_end=0.01, eps_decay=0.995):\n",
        "    \"\"\"Deep Q-Learning.\n",
        "\n",
        "    Params\n",
        "    ======\n",
        "        n_episodes (int): maximum number of training episodes\n",
        "        max_t (int): maximum number of timesteps per episode\n",
        "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
        "        eps_end (float): minimum value of epsilon\n",
        "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
        "    \"\"\"\n",
        "    scores = []                        # list containing scores from each episode\n",
        "    scores_window = deque(maxlen=100)  # last 100 scores\n",
        "    eps = eps_start                    # initialize epsilon\n",
        "    for i_episode in range(1, n_episodes+1):\n",
        "        state,info = env.reset()\n",
        "        score = 0\n",
        "        for t in range(max_t):\n",
        "            action = agent.act(state, eps)\n",
        "            next_state, reward, done, info,_ = env.step(action)\n",
        "            agent.step(state, action, reward, next_state, done)\n",
        "            state = next_state\n",
        "            score += reward\n",
        "            if done:\n",
        "                break\n",
        "        scores_window.append(score)       # save most recent score\n",
        "        scores.append(score)              # save most recent score\n",
        "        eps = max(eps_end, eps_decay*eps) # decrease epsilon\n",
        "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
        "        if i_episode % 100 == 0:\n",
        "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
        "        if np.mean(scores_window)>=225.0:\n",
        "            print('\\nEnvironment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_window)))\n",
        "            torch.save(agent.qnetwork_local.state_dict(), 'checkpoint_lunar.pth')\n",
        "            break\n",
        "    return scores"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Train DQN model for Lunar Lander\n",
        "env = gymnasium.make('LunarLander-v2')\n",
        "agent = Agent(state_size=8, action_size=4, seed=0)\n",
        "scores = dqn(n_episodes=1000)\n",
        "env.close()\n",
        "\n",
        "# Plot the scores\n",
        "fig = matplotlib.pyplot.figure()\n",
        "ax = fig.add_subplot(111)\n",
        "matplotlib.pyplot.plot(np.arange(len(scores)), scores)\n",
        "matplotlib.pyplot.ylabel('Score')\n",
        "matplotlib.pyplot.xlabel('Episode #')\n",
        "matplotlib.pyplot.show()"
      ],
      "metadata": {
        "id": "tFab51XI5uoO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1TUIujGd5nV_"
      },
      "source": [
        "### 4. Watch a Smart Agent!\n",
        "\n",
        "In the next code cell, you will load the trained weights from file to watch a smart agent!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ru4IUM7w5nV_"
      },
      "outputs": [],
      "source": [
        "env = gymnasium.make('LunarLander-v2',render_mode='rgb_array')\n",
        "\n",
        "# Wrap the environment to record video\n",
        "env = RecordVideo(env, video_folder='./video', episode_trigger=lambda e: True)\n",
        "\n",
        "agent.qnetwork_local.load_state_dict(torch.load('checkpoint_lunar.pth'))\n",
        "\n",
        "for i in range(3):\n",
        "    state,info = env.reset()\n",
        "    for j in range(500):\n",
        "        action = agent.act(state)\n",
        "        env.render()\n",
        "        state, reward, terminated, truncated, info= env.step(action)\n",
        "        if terminated or truncated:\n",
        "            break\n",
        "\n",
        "env.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bqWzNgQT5nWA"
      },
      "source": [
        "### 5. Explore\n",
        "\n",
        "In this exercise, you have implemented a DQN agent and demonstrated how to use it to solve an OpenAI Gym environment.  To continue your learning, you are encouraged to complete any (or all!) of the following tasks:\n",
        "- Amend the various hyperparameters and network architecture to see if you can get your agent to solve the environment faster.  Once you build intuition for the hyperparameters that work well with this environment, try solving a different OpenAI Gym task with discrete actions!\n",
        "- You may like to implement some improvements such as prioritized experience replay, Double DQN, or Dueling DQN!\n",
        "- Write a blog post explaining the intuition behind the DQN algorithm and demonstrating how to use it to solve an RL environment of your choosing.  "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Using Stable Baseline3 Library for DQN"
      ],
      "metadata": {
        "id": "OfYdu1T5zdzG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "env_str = \"LunarLander-v2\"\n",
        "log_dir = \"./logs/{}\".format(env_str)"
      ],
      "metadata": {
        "id": "ussIoxUV0tw1"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create Training Environment\n",
        "env = make_vec_env(env_str, n_envs=1)\n",
        "\n",
        "# Create Evaluation Environment\n",
        "env_val = make_vec_env(env_str, n_envs=1)\n",
        "\n",
        "eval_callback = EvalCallback(env_val,\n",
        "                             best_model_save_path=log_dir,\n",
        "                             log_path=log_dir,\n",
        "                             eval_freq=10000,\n",
        "                             render=False,\n",
        "                             deterministic=True,\n",
        "                             n_eval_episodes=20)\n",
        "\n",
        "# Initialize DQN\n",
        "model = DQN('MlpPolicy', env, verbose=0)\n",
        "\n",
        "# Train the model\n",
        "model.learn(total_timesteps=1000000, progress_bar=True, callback=eval_callback)\n",
        "\n",
        "# Save the model\n",
        "model.save(os.path.join(log_dir, \"dqn_lunar_lander\"))\n",
        "\n",
        "mean_reward, std_reward = evaluate_policy(model, env, n_eval_episodes=20)\n",
        "print(f\"Mean reward: {mean_reward:.2f} +/- {std_reward:.2f}\")\n",
        "\n",
        "env.close()\n",
        "env_val.close()"
      ],
      "metadata": {
        "id": "onWidMEFyNnI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the best model\n",
        "env = make_vec_env(env_str, n_envs=1, seed=0)\n",
        "best_model_path = os.path.join(log_dir, \"best_model.zip\")\n",
        "best_model = DQN.load(best_model_path, env=env)\n",
        "\n",
        "mean_reward, std_reward = evaluate_policy(best_model, env, n_eval_episodes=20)\n",
        "print(f\"Best Model - Mean reward: {mean_reward:.2f} +/- {std_reward:.2f}\")\n",
        "\n",
        "# Record video of the best model playing Lunar Lander\n",
        "env = VecVideoRecorder(env, \"./videos/\",\n",
        "                       video_length=5000,\n",
        "                       record_video_trigger=lambda x: x == 0,\n",
        "                       name_prefix=\"best_model_lunar_lander_dqn\")\n",
        "\n",
        "obs = env.reset()\n",
        "for _ in range(5000):\n",
        "    action, _states = best_model.predict(obs)\n",
        "    obs, rewards, dones, info = env.step(action)\n",
        "    env.render()\n",
        "    if dones:\n",
        "      break\n",
        "\n",
        "env.close()"
      ],
      "metadata": {
        "id": "0OJHVOO3zWze"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the evaluations.npz file\n",
        "data = numpy.load(os.path.join(log_dir, \"evaluations.npz\"))\n",
        "\n",
        "# Extract the relevant data\n",
        "timesteps = data['timesteps']\n",
        "results = data['results']\n",
        "\n",
        "# Calculate the mean and standard deviation of the results\n",
        "mean_results = numpy.mean(results, axis=1)\n",
        "std_results = numpy.std(results, axis=1)\n",
        "\n",
        "# Plot the results\n",
        "matplotlib.pyplot.figure()\n",
        "matplotlib.pyplot.plot(timesteps, mean_results)\n",
        "matplotlib.pyplot.fill_between(timesteps,\n",
        "                               mean_results - std_results,\n",
        "                               mean_results + std_results,\n",
        "                               alpha=0.3)\n",
        "\n",
        "matplotlib.pyplot.xlabel(\"Timesteps\")\n",
        "matplotlib.pyplot.ylabel(\"Mean Reward\")\n",
        "matplotlib.pyplot.title(f\"DQN Performance on {env_str}\")\n",
        "matplotlib.pyplot.show()"
      ],
      "metadata": {
        "id": "-utKWuNhCoVk"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.14"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}