Explained.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Workflow\n",
    "\n",
    "- __author__ = 'BlackDChase,MR-TLL'\n",
    "- __version__ = '1.0.0'\n",
    "- [Hyperparameter](#Hyperparameters)\n",
    "- [Training A3C](#Training-A3C)\n",
    "- [Testing](#Testing)\n",
    "- [Comparison](#Comparison)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd modules\n",
    "from lstm import LSTM\n",
    "from env import LSTMEnv as ENV\n",
    "#from TempEnv import TempEnv as ENV\n",
    "from Agent import GOD\n",
    "import log\n",
    "import sys\n",
    "import torch\n",
    "from torch import nn, multiprocessing, device, Tensor\n",
    "import matplotlib.pyplot as plt\n",
    "import sys,os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Hyperparameters\n",
    "\"\"\"\n",
    "stateSize = 13\n",
    "log.info(f\"stateSize = {stateSize}\")\n",
    "keywords={\n",
    "        \"n\":20,\n",
    "        \"e\":10,\n",
    "        \"t\":75,\n",
    "        \"a\":8,\n",
    "        \"s\":100,\n",
    "        \"d\":False,\n",
    "        \"alr\":1e-3,\n",
    "        \"clr\":7e-3,\n",
    "        \"p\":None,\n",
    "        \"f\":True,\n",
    "    }\n",
    "stateSize = 13\n",
    "log.info(f\"stateSize = {stateSize}\")\n",
    "\n",
    "\"\"\"\n",
    "Making the action space\n",
    "\"\"\"\n",
    "n=int(keywords['a']//2)\n",
    "if keywords['a']%2==0:\n",
    "    actionSpace = [i for i in range(-n*5,n*5+1,5)]\n",
    "    actionSpace.pop(len(actionSpace)//2)\n",
    "else:\n",
    "    actionSpace = [i/10 for i in range(-n*55,n*55+1,55)]\n",
    "keywords['a']=actionSpace\n",
    "p=None\n",
    "folder=\"../Saved_model/\" \n",
    "folder=!echo $folder$(ls $folder -Art | grep 'Olog' | tail -n 1)\n",
    "folder=folder[0]\n",
    "\n",
    "# Comment this if not fine tuning the previously trained model\n",
    "p=!echo \"$folder$(ls $folder | grep 'CritcModel.pt')\" | rev | cut -b 14- | rev\n",
    "p=p[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "actionSpace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training A3C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Master Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\"\"\" For Training\n",
    "god = GOD(\n",
    "    stateSize=int(stateSize),\n",
    "    actionSpace=keywords[\"a\"],\n",
    "    debug=keywords[\"d\"],\n",
    "    maxEpisode=int(keywords[\"e\"]),\n",
    "    nAgent=int(keywords[\"n\"]),\n",
    "    trajectoryLength=int(keywords[\"t\"]),\n",
    "    alr=keywords[\"alr\"],\n",
    "    clr=keywords[\"clr\"]\n",
    ")\n",
    "print(\"Master Agent Made\")\n",
    "log.info(\"GOD inititated\")\n",
    "log.info(f\"State Size = {int(stateSize)}\")\n",
    "log.info(f\"Action Space = {keywords['a']}\")\n",
    "log.info(f\"Debug = {keywords['d']}\")\n",
    "log.info(f\"Max Episode = {int(keywords['e'])}\")\n",
    "log.info(f\"Number of Agent = {int(keywords['n'])}\")\n",
    "log.info(f\"Trajectory Length = {int(keywords['t'])}\")\n",
    "log.info(f\"Actor Learning Rate = {keywords['alr']}\")\n",
    "log.info(f\"Critic Learning Rate = {keywords['clr']}\")\n",
    "#\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train and Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "god.train()\n",
    "god.saveModel(\"../Saved_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract Log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!./extractTrainLog.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding Latest Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords['p']=p[0]\n",
    "# To Load Newly saved Model\n",
    "folder=\"../Saved_model/\" \n",
    "folder=!echo $folder$(ls $folder -Art | grep 'Olog' | tail -n 1)\n",
    "folder=folder[0]\n",
    "\n",
    "# Comment this if not fine tuning the previously trained model\n",
    "p=!echo \"$folder$(ls $folder | grep 'CritcModel.pt')\" | rev | cut -b 14- | rev\n",
    "p=p[0]\n",
    "# To crewate a new LOG\n",
    "import log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\"\"\" For Testing\n",
    "god = GOD(\n",
    "    debug=keywords[\"d\"],\n",
    "    stateSize=int(stateSize),\n",
    "    actionSpace=keywords[\"a\"],\n",
    "    path=keywords[\"p\"],\n",
    ")\n",
    "print(\"Master Agent Made\")\n",
    "log.info(\"GOD inititated\")\n",
    "log.info(f\"State Size = {int(stateSize)}\")\n",
    "log.info(f\"Action Space = {keywords['a']}\")\n",
    "log.info(f\"Debug = {keywords['d']}\")\n",
    "log.info(f\"Path of Model = {keywords['p']}\")\n",
    "log.info(f\"Steps tested for = {keywords['s']}\")\n",
    "#\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Enviornment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\"\"\"# Using LSTM Model as ENV\n",
    "output_size = 13\n",
    "input_dim = output_size\n",
    "hidden_dim = 128\n",
    "layer_dim = 1\n",
    "model = LSTM(output_size, input_dim, hidden_dim, layer_dim,debug=keywords[\"d\"])\n",
    "model.loadM(\"ENV_MODEL/lstm_model.pt\")\n",
    "log.info(f\"LSTM Model = {model}\")\n",
    "#env=ENV(model,\"../Dataset/13_columns.csv\")\n",
    "env=ENV(\n",
    "    model=model,\n",
    "    dataset_path=\"../datasets/normalized_weird_13_columns_with_supply.csv\",\n",
    "    actionSpace=actionSpace,\n",
    "    debug=keywords[\"d\"],\n",
    ")\n",
    "env.reset()\n",
    "print(\"Environment inititated\")\n",
    "\"\"\"\n",
    "env = ENV(stateSize,actionSpace)\n",
    "#\"\"\"\n",
    "log.info(\"Environment inititated\")\n",
    "\n",
    "god.giveEnvironment(env)\n",
    "log.info(\"Environment parsed, Boss inititated\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "for 1000 timesteps\n",
    "a3cStates -> Predicted States\n",
    "normalStates -> Given by ENV(LSTM) without A3C agent\n",
    "Profit -> (Demand-Supply)*Price\n",
    "\"\"\"\n",
    "time=keywords['s']\n",
    "a3cStates = god.test(time=time)\n",
    "normalStates = god.getNormalStates(time=time)\n",
    "a3cProfit,normalProfit,diff = god.compare(a3cState=a3cStates,normalState=normalStates)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract Log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "./extractTestLog.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Profit of A3C, LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(dpi=100)\n",
    "ax.set_xlabel(f\"Time step\")\n",
    "\n",
    "color='r'\n",
    "ax.plot(normalProfit,color=color)\n",
    "ax.tick_params(axis='y',labelcolor=color)\n",
    "ax.set_ylabel('Profit w/o A3C',color=color)\n",
    "\n",
    "ax1 = ax.twinx()\n",
    "color='b'\n",
    "ax1.plot(a3cProfit,color=color)\n",
    "ax1.tick_params(axis='y',labelcolor=color)\n",
    "ax1.set_ylabel('A3C Profit',color=color)\n",
    "\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Advantageous Actor Critc's advantage over Environmnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(dpi=100)\n",
    "ax.set_xlabel(f\"Time step\")\n",
    "ax.plot(diff)\n",
    "ax.set_ylabel(f\"A3C Profit - Normal Profit\")\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}