analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of the trajectories.\n",
    "\n",
    "Formatting standards:\n",
    "\n",
    "- MC : $k_{\\varepsilon}$, $mean$, $median$, $sd$\n",
    "- SARSA and Q-Learning: $\\lambda$, $k_{\\alpha}$, $k_{\\varepsilon}$, $mean$, $median$, $sd$\n",
    "- Expected SARSA: $k_{\\alpha}$, $k_{\\varepsilon}$, $mean$, $median$, $sd$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected SARSA\n",
    "\n",
    "Will write to a file Summary/Exp_SARSA_summary with this format\n",
    "k_alpha, k_epsilon, mean, median standard deviation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def extract_parameters_EXP_SARSA(file_name):\n",
    "    \"\"\"\n",
    "    - WRITTEN WITH HELP FROM CHATGPT -\n",
    "    Extracts parameters from the expected SARSA title object.\n",
    "    We only have two parameters to extact, k_alpha and k_epsilon\n",
    "    \"\"\"\n",
    "    # Extracting the 'k' parameter\n",
    "    k_alpha_match = re.search(r\"(?<=k_alpha_)[\\d.e-]+\", file_name)\n",
    "    k_alpha = k_alpha_match.group() if k_alpha_match else None\n",
    "\n",
    "    # Extracting the 'alpha' parameter\n",
    "\n",
    "    # Extracting the 'epsilon' parameter\n",
    "    k_epsilon_match = re.search(r\"(?<=k_epsilon)[\\d.e-]+\", file_name)\n",
    "    k_epsilon = float(k_epsilon_match.group()[:-1]) if k_epsilon_match else None\n",
    "\n",
    "    return float(k_alpha), float(k_epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# - WRITTEN WITH HELP FROM CHATGPT -\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "# Specify the folder path EXP_SARSA\n",
    "folder_path = \"Trajectories/Exp_SARSA_trajectories\"\n",
    "\n",
    "# Target directory to store the summary.\n",
    "# It is stored with this form:\n",
    "# k_alpha, k_epsilon, mu, med, sd\n",
    "target_file = \"Summary/Exp_SARSA_summary\"  # File where we want to append the results\n",
    "\n",
    "# To delete the previous one if already existing.\n",
    "with open(target_file, \"w\") as t_file:\n",
    "    pass\n",
    "\n",
    "# Loop over all files in the folder\n",
    "for file_name in os.listdir(folder_path):\n",
    "    # Now we scan all the files\n",
    "    if file_name.endswith(\".txt\"):\n",
    "        # Construct the full file path\n",
    "        file_path = os.path.join(folder_path, file_name)\n",
    "        k_alpha, k_epsilon = extract_parameters_EXP_SARSA(file_name)\n",
    "        # Perform operations on the file\n",
    "        with open(file_path, \"r\") as file:\n",
    "            # Read the contents of the file\n",
    "            # print(f\"File: {file_name}\")\n",
    "            data = np.loadtxt(file_path)\n",
    "            data = data[-1000:]\n",
    "            mu = np.mean(data)\n",
    "            med = np.median(data)\n",
    "            sd = np.std(data)\n",
    "        with open(target_file, \"a\") as t_file:\n",
    "            t_file.write(\n",
    "                str(k_alpha)\n",
    "                + \" \"\n",
    "                + str(k_epsilon)\n",
    "                + \" \"\n",
    "                + str(mu)\n",
    "                + \" \"\n",
    "                + str(med)\n",
    "                + \" \"\n",
    "                + str(sd)\n",
    "                + \"\\n\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SARSA and Q-Leearning\n",
    "\n",
    "Will write to a file with the following standard:\n",
    "lambda\\_, k_alpha, k_epsilon, mean, median, std\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_parameters_SARSA_Q(file_name):\n",
    "    # - WRITTEN WITH HELP FROM CHATGPT -\n",
    "    # Extracting the 'k_alpha' parameter\n",
    "    k_alpha_match = re.search(r\"(?<=k_alpha_)[\\d.e-]+\", file_name)\n",
    "    k_alpha = k_alpha_match.group() if k_alpha_match else None\n",
    "\n",
    "    # Extracting lambda parameter\n",
    "    lambda_match = re.search(r\"(?<=lambda_)[\\d.e-]+\", file_name)\n",
    "    lambda_ = lambda_match.group() if lambda_match else None\n",
    "\n",
    "    # Extracting the 'k_epsilon' parameter\n",
    "    k_epsilon_match = re.search(r\"(?<=k_epsilon)[\\d.e-]+\", file_name)\n",
    "    k_epsilon = float(k_epsilon_match.group()[:-1]) if k_epsilon_match else None\n",
    "\n",
    "    return float(lambda_), float(k_alpha), float(k_epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# - WRITTEN WITH HELP FROM CHATGPT -\n",
    "# Specify the folder path SARSA!\n",
    "folder_path = \"Trajectories/SARSA_trajectories\"\n",
    "\n",
    "# Target directory to store the summary.\n",
    "# It is stored with this form:\n",
    "# k_alpha, k_epsilon, mu, med, sd\n",
    "target_file = \"Summary/SARSA_summary\"  # File where we want to append the results\n",
    "\n",
    "# To delete the previous one if already existing.\n",
    "with open(target_file, \"w\") as t_file:\n",
    "    pass\n",
    "\n",
    "# Loop over all files in the folder\n",
    "for file_name in os.listdir(folder_path):\n",
    "    # Now we scan all the files\n",
    "    if file_name.endswith(\".txt\"):\n",
    "        # Construct the full file path\n",
    "        file_path = os.path.join(folder_path, file_name)\n",
    "        lambda_, k_alpha, k_epsilon = extract_parameters_SARSA_Q(file_name)\n",
    "        # Perform operations on the file\n",
    "        with open(file_path, \"r\") as file:\n",
    "            # Read the contents of the file\n",
    "            # print(f\"File: {file_name}\")\n",
    "            data = np.loadtxt(file_path)\n",
    "            data = data[-1000:]\n",
    "            mu = np.mean(data)\n",
    "            med = np.median(data)\n",
    "            sd = np.std(data)\n",
    "        with open(target_file, \"a\") as t_file:\n",
    "            t_file.write(\n",
    "                str(lambda_)\n",
    "                + \" \"\n",
    "                + str(k_alpha)\n",
    "                + \" \"\n",
    "                + str(k_epsilon)\n",
    "                + \" \"\n",
    "                + str(mu)\n",
    "                + \" \"\n",
    "                + str(med)\n",
    "                + \" \"\n",
    "                + str(sd)\n",
    "                + \"\\n\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# - WRITTEN WITH HELP FROM CHATGPT -\n",
    "# Specify the folder path Q LEARNING\n",
    "folder_path = \"Trajectories/Q_trajectories\"\n",
    "\n",
    "# Target directory to store the summary.\n",
    "# It is stored with this form:\n",
    "# k_alpha, k_epsilon, mu, med, sd\n",
    "target_file = \"Summary/Q_summary\"  # File where we want to append the results\n",
    "\n",
    "# To delete the previous one if already existing.\n",
    "with open(target_file, \"w\") as t_file:\n",
    "    pass\n",
    "\n",
    "# Loop over all files in the folder\n",
    "for file_name in os.listdir(folder_path):\n",
    "    # Now we scan all the files\n",
    "    if file_name.endswith(\".txt\"):\n",
    "        # Construct the full file path\n",
    "        file_path = os.path.join(folder_path, file_name)\n",
    "        lambda_, k_alpha, k_epsilon = extract_parameters_SARSA_Q(file_name)\n",
    "        # Perform operations on the file\n",
    "        with open(file_path, \"r\") as file:\n",
    "            # Read the contents of the file\n",
    "            # print(f\"File: {file_name}\")\n",
    "            data = np.loadtxt(file_path)\n",
    "            data = data[-1000:]\n",
    "            mu = np.mean(data)\n",
    "            med = np.median(data)\n",
    "            sd = np.std(data)\n",
    "        with open(target_file, \"a\") as t_file:\n",
    "            t_file.write(\n",
    "                str(lambda_)\n",
    "                + \" \"\n",
    "                + str(k_alpha)\n",
    "                + \" \"\n",
    "                + str(k_epsilon)\n",
    "                + \" \"\n",
    "                + str(mu)\n",
    "                + \" \"\n",
    "                + str(med)\n",
    "                + \" \"\n",
    "                + str(sd)\n",
    "                + \"\\n\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MC\n",
    "\n",
    "Will write to a file with the following standard: k_epsilon mu, med, std\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_parameters_MC(file_name):\n",
    "    \"\"\"\n",
    "    # - WRITTEN WITH HELP FROM CHATGPT -\n",
    "    Extracts parameters from the expected SARSA.\n",
    "    We only have two parameters to extact, k_alpha and k_epsilon\n",
    "    - WRITTEN USING CHATGPT -\n",
    "    \"\"\"\n",
    "\n",
    "    # Extracting the 'epsilon' parameter\n",
    "    k_epsilon_match = re.search(r\"(?<=k_epsilon)[\\d.e-]+\", file_name)\n",
    "    k_epsilon = float(k_epsilon_match.group()[:-1]) if k_epsilon_match else None\n",
    "\n",
    "    # Print the extracted parameters\n",
    "    # print(f\"epsilon: {k_epsilon}\")\n",
    "    return float(k_epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# - WRITTEN WITH HELP FROM CHATGPT -\n",
    "# Specify the folder path EXP_SARSA\n",
    "folder_path = \"Trajectories/MC_trajectories\"\n",
    "\n",
    "# Target directory to store the summary.\n",
    "# It is stored with this form:\n",
    "# k_alpha, k_epsilon, mu, med, sd\n",
    "target_file = \"Summary/MC_summary\"  # File where we want to append the results\n",
    "\n",
    "# To delete the previous one if already existing.\n",
    "with open(target_file, \"w\") as t_file:\n",
    "    pass\n",
    "\n",
    "# Loop over all files in the folder\n",
    "for file_name in os.listdir(folder_path):\n",
    "    # Now we scan all the files\n",
    "    if file_name.endswith(\".txt\"):\n",
    "        # Construct the full file path\n",
    "        file_path = os.path.join(folder_path, file_name)\n",
    "        k_epsilon = extract_parameters_MC(file_name)\n",
    "        # Perform operations on the file\n",
    "        with open(file_path, \"r\") as file:\n",
    "            # Read the contents of the file\n",
    "            # print(f\"File: {file_name}\")\n",
    "            data = np.loadtxt(file_path)\n",
    "            data = data[-1000:]\n",
    "            mu = np.mean(data)\n",
    "            med = np.median(data)\n",
    "            sd = np.std(data)\n",
    "        with open(target_file, \"a\") as t_file:\n",
    "            t_file.write(\n",
    "                str(k_epsilon) + \" \" + str(mu) + \" \" + str(med) + \" \" + str(sd) + \"\\n\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of RESULTS\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SARSA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lambda_</th>\n",
       "      <th>k_alpha</th>\n",
       "      <th>k_epsilon</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>sd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>2633.698</td>\n",
       "      <td>1831.0</td>\n",
       "      <td>2632.137291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>2616.643</td>\n",
       "      <td>1570.0</td>\n",
       "      <td>2967.536161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00300</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>1427.381</td>\n",
       "      <td>943.0</td>\n",
       "      <td>1468.154914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>1159.787</td>\n",
       "      <td>689.0</td>\n",
       "      <td>1313.103380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>1057.360</td>\n",
       "      <td>671.0</td>\n",
       "      <td>1125.386756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>55.374</td>\n",
       "      <td>42.0</td>\n",
       "      <td>52.530164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>54.297</td>\n",
       "      <td>43.0</td>\n",
       "      <td>40.202671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>51.211</td>\n",
       "      <td>36.0</td>\n",
       "      <td>47.497710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>48.352</td>\n",
       "      <td>41.0</td>\n",
       "      <td>30.044968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>41.118</td>\n",
       "      <td>33.0</td>\n",
       "      <td>31.209775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    lambda_  k_alpha  k_epsilon      mean  median           sd\n",
       "30      0.2  0.00003    0.00005  2633.698  1831.0  2632.137291\n",
       "48      0.8  0.00003    0.00500  2616.643  1570.0  2967.536161\n",
       "27      0.8  0.00300    0.00005  1427.381   943.0  1468.154914\n",
       "44      0.8  0.00003    0.00050  1159.787   689.0  1313.103380\n",
       "49      0.5  0.00003    0.05000  1057.360   671.0  1125.386756\n",
       "..      ...      ...        ...       ...     ...          ...\n",
       "14      0.0  0.00000    0.00000    55.374    42.0    52.530164\n",
       "5       0.0  0.00030    0.05000    54.297    43.0    40.202671\n",
       "29      0.8  0.00000    0.00000    51.211    36.0    47.497710\n",
       "6       0.0  0.03000    0.00050    48.352    41.0    30.044968\n",
       "21      0.0  0.03000    0.00500    41.118    33.0    31.209775\n",
       "\n",
       "[68 rows x 6 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "file_path = \"Summary/SARSA_summary\"\n",
    "\n",
    "# Read the text file into a DataFrame\n",
    "df = pd.read_csv(file_path, delimiter=\" \", header=None)\n",
    "df.columns = [\"lambda_\", \"k_alpha\", \"k_epsilon\", \"mean\", \"median\", \"sd\"]\n",
    "df.sort_values(\"mean\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Q_Learning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lambda_</th>\n",
       "      <th>k_alpha</th>\n",
       "      <th>k_epsilon</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>sd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>4783.610</td>\n",
       "      <td>3186.0</td>\n",
       "      <td>4952.148230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>3624.031</td>\n",
       "      <td>2343.0</td>\n",
       "      <td>3996.748732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>3595.010</td>\n",
       "      <td>2482.5</td>\n",
       "      <td>3684.509191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>2989.408</td>\n",
       "      <td>2004.5</td>\n",
       "      <td>3071.134308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>2462.887</td>\n",
       "      <td>1660.0</td>\n",
       "      <td>2481.273004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>52.598</td>\n",
       "      <td>41.5</td>\n",
       "      <td>46.253177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>51.494</td>\n",
       "      <td>36.5</td>\n",
       "      <td>46.329688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>51.297</td>\n",
       "      <td>34.0</td>\n",
       "      <td>43.372581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>49.111</td>\n",
       "      <td>43.0</td>\n",
       "      <td>29.886062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>46.793</td>\n",
       "      <td>38.0</td>\n",
       "      <td>29.864095</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    lambda_  k_alpha  k_epsilon      mean  median           sd\n",
       "62      0.5  0.00003    0.00005  4783.610  3186.0  4952.148230\n",
       "61      0.8  0.00003    0.00005  3624.031  2343.0  3996.748732\n",
       "0       0.8  0.00030    0.05000  3595.010  2482.5  3684.509191\n",
       "63      0.0  0.00003    0.00005  2989.408  2004.5  3071.134308\n",
       "43      0.8  0.00030    0.00005  2462.887  1660.0  2481.273004\n",
       "..      ...      ...        ...       ...     ...          ...\n",
       "4       0.2  0.00000    0.00000    52.598    41.5    46.253177\n",
       "59      0.0  0.00000    0.00000    51.494    36.5    46.329688\n",
       "44      0.8  0.00000    0.00000    51.297    34.0    43.372581\n",
       "67      0.0  0.00030    0.00500    49.111    43.0    29.886062\n",
       "1       0.2  0.03000    0.05000    46.793    38.0    29.864095\n",
       "\n",
       "[68 rows x 6 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = \"Summary/Q_summary\"\n",
    "\n",
    "# Read the text file into a DataFrame\n",
    "df = pd.read_csv(file_path, delimiter=\" \", header=None)\n",
    "df.columns = [\"lambda_\", \"k_alpha\", \"k_epsilon\", \"mean\", \"median\", \"sd\"]\n",
    "df.sort_values(\"mean\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Expected SARSA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k_alpha</th>\n",
       "      <th>k_epsilon</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>sd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>127.305</td>\n",
       "      <td>93.0</td>\n",
       "      <td>111.714001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>122.280</td>\n",
       "      <td>93.0</td>\n",
       "      <td>97.512961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>105.501</td>\n",
       "      <td>73.0</td>\n",
       "      <td>101.061734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.00300</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>77.693</td>\n",
       "      <td>57.0</td>\n",
       "      <td>64.672009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>70.235</td>\n",
       "      <td>53.0</td>\n",
       "      <td>56.271234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.00300</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>69.928</td>\n",
       "      <td>53.0</td>\n",
       "      <td>56.139672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>68.440</td>\n",
       "      <td>50.0</td>\n",
       "      <td>55.853061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>68.149</td>\n",
       "      <td>53.0</td>\n",
       "      <td>56.845798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00050</td>\n",
       "      <td>62.359</td>\n",
       "      <td>46.0</td>\n",
       "      <td>52.975580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.00030</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>55.847</td>\n",
       "      <td>42.0</td>\n",
       "      <td>51.682391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>49.982</td>\n",
       "      <td>35.0</td>\n",
       "      <td>42.942213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.00300</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>46.173</td>\n",
       "      <td>36.0</td>\n",
       "      <td>31.712412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>39.014</td>\n",
       "      <td>33.0</td>\n",
       "      <td>29.767294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.00300</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>36.391</td>\n",
       "      <td>27.0</td>\n",
       "      <td>29.010035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.03000</td>\n",
       "      <td>0.00005</td>\n",
       "      <td>23.479</td>\n",
       "      <td>23.0</td>\n",
       "      <td>12.422945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>23.342</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10.888298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.00003</td>\n",
       "      <td>0.00500</td>\n",
       "      <td>21.181</td>\n",
       "      <td>13.0</td>\n",
       "      <td>13.868967</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k_alpha  k_epsilon     mean  median          sd\n",
       "11  0.00030    0.00005  127.305    93.0  111.714001\n",
       "1   0.03000    0.00500  122.280    93.0   97.512961\n",
       "16  0.00030    0.00500  105.501    73.0  101.061734\n",
       "14  0.00300    0.00500   77.693    57.0   64.672009\n",
       "2   0.00003    0.00005   70.235    53.0   56.271234\n",
       "15  0.00300    0.00050   69.928    53.0   56.139672\n",
       "13  0.03000    0.00050   68.440    50.0   55.853061\n",
       "4   0.00030    0.00050   68.149    53.0   56.845798\n",
       "10  0.00003    0.00050   62.359    46.0   52.975580\n",
       "12  0.00030    0.05000   55.847    42.0   51.682391\n",
       "0   0.00000    0.00000   49.982    35.0   42.942213\n",
       "3   0.00300    0.05000   46.173    36.0   31.712412\n",
       "5   0.03000    0.05000   39.014    33.0   29.767294\n",
       "9   0.00300    0.00005   36.391    27.0   29.010035\n",
       "7   0.03000    0.00005   23.479    23.0   12.422945\n",
       "6   0.00003    0.05000   23.342    23.0   10.888298\n",
       "8   0.00003    0.00500   21.181    13.0   13.868967"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = \"Summary/Exp_SARSA_summary\"  # FOR SARSA\n",
    "\n",
    "# Read the text file into a DataFrame\n",
    "df = pd.read_csv(file_path, delimiter=\" \", header=None)\n",
    "\n",
    "\n",
    "df.columns = [\"k_alpha\", \"k_epsilon\", \"mean\", \"median\", \"sd\"]\n",
    "df.sort_values(\"mean\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k_epsilon</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>sd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.00500</td>\n",
       "      <td>199.460</td>\n",
       "      <td>143.0</td>\n",
       "      <td>175.705630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.00050</td>\n",
       "      <td>138.655</td>\n",
       "      <td>100.5</td>\n",
       "      <td>121.950547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.05000</td>\n",
       "      <td>123.642</td>\n",
       "      <td>93.0</td>\n",
       "      <td>102.413084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.00005</td>\n",
       "      <td>101.278</td>\n",
       "      <td>75.5</td>\n",
       "      <td>87.256603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>48.743</td>\n",
       "      <td>36.5</td>\n",
       "      <td>38.674578</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   k_epsilon     mean  median          sd\n",
       "2    0.00500  199.460   143.0  175.705630\n",
       "3    0.00050  138.655   100.5  121.950547\n",
       "0    0.05000  123.642    93.0  102.413084\n",
       "4    0.00005  101.278    75.5   87.256603\n",
       "1    0.00000   48.743    36.5   38.674578"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_path = \"Summary/MC_summary\"  # FOR SARSA\n",
    "\n",
    "# Read the text file into a DataFrame\n",
    "df = pd.read_csv(file_path, delimiter=\" \", header=None)\n",
    "\n",
    "\n",
    "df.columns = [\"k_epsilon\", \"mean\", \"median\", \"sd\"]\n",
    "df.sort_values(\"mean\", ascending=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "flappy_bird",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}