notebook_1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "colab": {
      "name": "1. The Rail Environment",
      "provenance": [],
      "collapsed_sections": [
        "TKOaUC3dc9m_",
        "1sVC6olS1qGi",
        "FembDogv1qHK"
      ]
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wMy8Du_B1qGe",
        "colab_type": "text"
      },
      "source": [
        "# Part 1: The Rail Environment\n",
        "\n",
        "In this notebook, we will see how to create, interact with and render railway systems.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TKOaUC3dc9m_",
        "colab_type": "text"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ibjxleKO1qGk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Install Flatland\n",
        "%cd /content\n",
        "!git clone https://gitlab.aicrowd.com/flatland/flatland.git/ --branch 223_UpdateEditor_55_notebooks\n",
        "%cd flatland\n",
        "!pip install -e ."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1sVC6olS1qGi",
        "colab_type": "text"
      },
      "source": [
        "# The big picture\n",
        "\n",
        "Let's first go over the main use cases of `RailEnv`, the Flatland environment. Additional details will be provided in the second section."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VzYewu331qG7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flatland.envs.rail_env import RailEnv\n",
        "from flatland.envs.rail_generators import random_rail_generator\n",
        "\n",
        "# Build a random 15x15 rail networks from a list of probability per cell type\n",
        "transition_prob = [\n",
        "    1.0,  # Type 0 - empty cell\n",
        "    1.0,  # Type 1 - straight\n",
        "    1.0,  # Type 2 - simple switch\n",
        "    0.3,  # Type 3 - diamond crossing\n",
        "    0.5,  # Type 4 - single slip\n",
        "    0.5,  # Type 5 - double slip\n",
        "    0.2,  # Type 6 - symmetrical\n",
        "    0.0,  # Type 7 - dead end\n",
        "    0.2,  # Type 8 - turn left\n",
        "    0.2,  # Type 9 - turn right\n",
        "    1.0  # Type 10 - mirrored switch\n",
        "]\n",
        "\n",
        "rail_generator = random_rail_generator(cell_type_relative_proportion=transition_prob)\n",
        "\n",
        "random_env = RailEnv(\n",
        "    width=15,\n",
        "    height=15,\n",
        "    rail_generator=rail_generator,\n",
        "    number_of_agents=1\n",
        ")\n",
        "\n",
        "# Call reset() to initialize the environment\n",
        "observation, info = random_env.reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "630t2mOw1qG4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import PIL\n",
        "from flatland.utils.rendertools import RenderTool\n",
        "\n",
        "# Render the environment\n",
        "def render_env(env):\n",
        "    env_renderer = RenderTool(env, gl=\"PILSVG\")\n",
        "    env_renderer.render_env()\n",
        "\n",
        "    image = env_renderer.get_image()\n",
        "    pil_image = PIL.Image.fromarray(image)\n",
        "    display(pil_image)\n",
        "\n",
        "render_env(random_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NgRSpW_Z1qGv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# By default, the environment provides global observations\n",
        "for agent_handle in random_env.get_agent_handles():\n",
        "    print('Observations for agent {}:'.format(agent_handle))\n",
        "    agent_obs = observation[agent_handle]\n",
        "\n",
        "    print('- Transition map\\n{}\\n'.format(np.transpose(agent_obs[0], (2, 0, 1))))\n",
        "    print('- Agent position\\n{}\\n'.format(np.transpose(agent_obs[1], (2, 0, 1))))\n",
        "    print('- Agent target \\n{}\\n'.format(np.transpose(agent_obs[2], (2, 0, 1))))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-uyBYOzpLuUk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flatland.envs.rail_generators import sparse_rail_generator\n",
        "\n",
        "# More realistic networks can be built using sparse_rail_generator\n",
        "sparse_env = RailEnv(\n",
        "    width=30,\n",
        "    height=30,\n",
        "    rail_generator=sparse_rail_generator(\n",
        "        max_num_cities=3,  # Number of cities (= train stations)\n",
        "        grid_mode=False,  # Distribute the cities evenly in a grid\n",
        "        max_rails_between_cities=2,  # Max number of rails connecting to a city\n",
        "        max_rails_in_city=3  # Number of parallel tracks in cities\n",
        "    ),\n",
        "    number_of_agents=1\n",
        ")\n",
        "\n",
        "observation, info = sparse_env.reset()\n",
        "\n",
        "render_env(sparse_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0tCsn7W31qGs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "from flatland.envs.rail_generators import rail_from_manual_specifications_generator\n",
        "\n",
        "# Environments can also be created from fixed configurations\n",
        "# Here we define a 6x4 railway from a 2D array\n",
        "specs = [[(0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)],\n",
        "         [(0, 0), (0, 0), (0, 0), (0, 0), (7, 0), (0, 0)],\n",
        "         [(7, 270), (1, 90), (1, 90), (1, 90), (2, 90), (7, 90)],\n",
        "         [(0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)]]\n",
        "\n",
        "rail_shape = np.array(specs).shape\n",
        "\n",
        "fixed_env = RailEnv(\n",
        "    width=rail_shape[1],\n",
        "    height=rail_shape[0],\n",
        "    rail_generator=rail_from_manual_specifications_generator(specs),\n",
        "    number_of_agents=1\n",
        ")\n",
        "\n",
        "observation, info = fixed_env.reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kAqre2BE1qHB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# A real controller would be implemented here\n",
        "class RandomController:\n",
        "    def __init__(self, action_size):\n",
        "        self.action_size = action_size\n",
        "\n",
        "    def act(self, observations):\n",
        "        actions = dict()\n",
        "        for agent_handle, observation in enumerate(observations):\n",
        "            action = np.random.randint(self.action_size)\n",
        "            actions.update({agent_handle: action})\n",
        "        return actions"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nGemVk01qHE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from IPython.display import clear_output\n",
        "\n",
        "# The environment provides a gym-like interface.\n",
        "def run_episode(env):\n",
        "    controller = RandomController(env.action_space[0])\n",
        "    observations, info = env.reset()\n",
        "\n",
        "    score = 0\n",
        "    actions = dict()\n",
        "\n",
        "    for step in range(50):\n",
        "        clear_output(wait=True)\n",
        "\n",
        "        actions = controller.act(observations)\n",
        "        next_observations, all_rewards, dones, _ = env.step(actions)\n",
        "\n",
        "        for agent_handle in env.get_agent_handles():\n",
        "            score += all_rewards[agent_handle]\n",
        "\n",
        "        render_env(env)\n",
        "        print('Timestep {}, total score = {}'.format(step, score))\n",
        "\n",
        "        if dones['__all__']:\n",
        "            print('All done!')\n",
        "            return\n",
        "\n",
        "    print(\"Episode didn't finish after 50 timesteps.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKynbq-r1qHG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run an episode in the random environment\n",
        "run_episode(random_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2oPzOBTEPuLa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run an episode in the sparse environment\n",
        "run_episode(sparse_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6jm8oVYwL0Mm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run an episode in the fixed environment\n",
        "run_episode(fixed_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FembDogv1qHK",
        "colab_type": "text"
      },
      "source": [
        "# In more details\n",
        "\n",
        "Let's now dive into more details about some of the points mentioned above."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bUsYavBl1qHL",
        "colab_type": "text"
      },
      "source": [
        "The basic usage of the `RailEnv` environment consists in creating a `RailEnv` object endowed with\n",
        "-  a **rail generator**, that generates new rail networks on each reset,\n",
        "- an **observation generator**, that provides a suitable observation vector to the agents. \n",
        "\n",
        "As we have seen above, the environment provides very complete observations by default. You typically won't use this object as-is. One of the main objectives of the Flatland challenge is to **find suitable observations** to solve the task at hand. We will see how this works in the next notebook.\n",
        "\n",
        "For now, let's see more precisely how we can create rail networks and use them to train agents."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bb9Hni7m1qHM",
        "colab_type": "text"
      },
      "source": [
        "Generating rail networks\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n_to1JlG1qHN",
        "colab_type": "text"
      },
      "source": [
        "There are multiple ways to generate a rail network. The simpler one is to describe it explicitely cell by cell:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "si4DJco51qHO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "specs = [[(0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)],\n",
        "         [(0, 0), (0, 0), (0, 0), (0, 0), (7, 0), (0, 0)],\n",
        "         [(7, 270), (1, 90), (1, 90), (1, 90), (2, 90), (7, 90)],\n",
        "         [(0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5d49n9OO1qHT",
        "colab_type": "text"
      },
      "source": [
        "`specs` is a 2-dimensional array of tuples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiEPqQvG1qHU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rail_shape = np.array(specs).shape\n",
        "rail_shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_gVOohSa1qHX",
        "colab_type": "text"
      },
      "source": [
        "This `specs` array represent a 4x6 2D grid of tuples. In each tuple, the first element represent the **cell type**, and the second the **rotation** of the cell (0, 90, 180 or 270 degrees clockwise).\n",
        "\n",
        "The following image gives an overview of the eight basic cell types. These can be rotated in steps of 45° and mirrored along the North-South of East-West axis.\n",
        "\n",
        "![cell types](https://github.com/MasterScrat/getting-started/blob/master/assets/transitions.png?raw=1)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jzaAnE-C1qHY",
        "colab_type": "text"
      },
      "source": [
        "Here's the full list of cell types. This list can be found in `flatland.core.grid.rail_env_grid.RailEnvTransitions`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aCR19sb11qHa",
        "colab_type": "text"
      },
      "source": [
        "```\n",
        "transition_list = [int('0000000000000000', 2),  # empty cell - Case 0\n",
        "                   int('1000000000100000', 2),  # Case 1 - straight\n",
        "                   int('1001001000100000', 2),  # Case 2 - simple switch\n",
        "                   int('1000010000100001', 2),  # Case 3 - diamond drossing\n",
        "                   int('1001011000100001', 2),  # Case 4 - single slip\n",
        "                   int('1100110000110011', 2),  # Case 5 - double slip\n",
        "                   int('0101001000000010', 2),  # Case 6 - symmetrical\n",
        "                   int('0010000000000000', 2),  # Case 7 - dead end\n",
        "                   int('0100000000000010', 2),  # Case 1b (8)  - simple turn right\n",
        "                   int('0001001000000000', 2),  # Case 1c (9)  - simple turn left\n",
        "                   int('1100000000100010', 2)]  # Case 2b (10) - simple switch mirrored\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qhsG9Fxo1qHb",
        "colab_type": "text"
      },
      "source": [
        "The `RailEnv` constructor accepts a `rail_generator` parameter, which is used to generate the rail networks. When creating fixed networks, we use the `rail_from_manual_specifications_generator`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4esDQpSy1qHb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fixed_env = RailEnv(\n",
        "    width=rail_shape[1],\n",
        "    height=rail_shape[0],\n",
        "    rail_generator=rail_from_manual_specifications_generator(specs),\n",
        "    number_of_agents=1\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cJtnBBqK1qHe",
        "colab_type": "text"
      },
      "source": [
        "A call to `reset()` is necessary to fully initialize the environment. As usual with gym-like environment, this returns the initial observation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULpr1Kff1qHf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "observations, info = fixed_env.reset()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N7bATfPTxA_1",
        "colab_type": "text"
      },
      "source": [
        "This result in a fixed rail network:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r8hz5zhrxEaP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "render_env(fixed_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ggAouJ6jVPVE",
        "colab_type": "text"
      },
      "source": [
        "We can also create a randomly generated network using `random_rail_generator`. In this case, we can optionally specify the probability of each cell type, using the same list as described before. This results in a rail network that is different every time `reset()` is called."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eW_atvmeVPE_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "transition_prob = [\n",
        "    1.0,  # Type 0 - empty cell\n",
        "    1.0,  # Type 1 - straight\n",
        "    1.0,  # Type 2 - simple switch\n",
        "    0.3,  # Type 3 - diamond crossing\n",
        "    0.5,  # Type 4 - single slip\n",
        "    0.5,  # Type 5 - double slip\n",
        "    0.2,  # Type 6 - symmetrical\n",
        "    0.0,  # Type 7 - dead end\n",
        "    0.2,  # Type 8 - turn left\n",
        "    0.2,  # Type 9 - turn right\n",
        "    1.0  # Type 10 - mirrored switch\n",
        "]\n",
        "\n",
        "random_env = RailEnv(\n",
        "    width=15,\n",
        "    height=15,\n",
        "    rail_generator=random_rail_generator(cell_type_relative_proportion=transition_prob),\n",
        "    number_of_agents=1\n",
        ")\n",
        "\n",
        "observations, info = random_env.reset()\n",
        "\n",
        "render_env(random_env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HSHKm7tJ1qHi",
        "colab_type": "text"
      },
      "source": [
        "Running an agent\n",
        "--"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AVz8IkRG1qHj",
        "colab_type": "text"
      },
      "source": [
        "`RailEnv` is targeted at multi-agents experiments. For this purpose, it is derived from RLLib's `MultiAgentEnv` class. You can [read more details about it here](https://ray.readthedocs.io/en/latest/rllib-env.html).\n",
        "\n",
        "The environment is run by supplying the `step` function with a **dictionary** of actions, whose keys are agents’ handles and the corresponding values are the selected actions. This dictionary is passed to the environment which checks the validity of all actions and update the environment state.\n",
        "\n",
        "The environment returns an array of new observations, a reward dictionary for all the agents as well as a flags indicating which agents are done. This information can be used to update the policy of your agent and if `done[‘__all__’] == True` the episode terminates."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3NEDJVq1qHk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flatland.envs.rail_env import RailEnvActions\n",
        "\n",
        "controller = RandomController(random_env.action_space[0])\n",
        "observations, info = random_env.reset()\n",
        "actions = controller.act(observations)\n",
        "\n",
        "# Perform a single action per agent\n",
        "for (handle, action) in actions.items():\n",
        "    print('Agent {} will perform action {} ({})'.format(handle, action, RailEnvActions.to_char(action)))\n",
        "    next_obs, all_rewards, dones, info = random_env.step({handle: action})\n",
        "\n",
        "print('Rewards for each agent: {}'.format(all_rewards))\n",
        "print('Done for each agent: {}'.format(dones))\n",
        "print('Misc info: {}'.format(info))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XaYekObr1qHn",
        "colab_type": "text"
      },
      "source": [
        "In the example above, we use `env.get_agent_handles()` to enumerate through the handles, and `RailEnvActions.to_char` to get a symbol representing the agent's direction: **B**ackward, **F**orward, **L**eft, **R**ight or **S**top."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7M_sD0nKrL4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "RailEnvActions??"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}