CartPole-v0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Actor Critic Method\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from gym.wrappers.monitoring.video_recorder import VideoRecorder #For recording\n",
    "import sys\n",
    "sys.path.append('n:\\\\FFmpeg\\\\bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "n:\\python\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior.\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved video.\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "state = env.reset()\n",
    "video_recorder = None\n",
    "video_recorder = VideoRecorder(env, 'videos/start.mp4', enabled=True)\n",
    "for _ in range(1000):\n",
    "    env.render()\n",
    "    video_recorder.capture_frame()\n",
    "    env.step(env.action_space.sample()) # take a random action\n",
    "\n",
    "print(\"Saved video.\")\n",
    "video_recorder.close()\n",
    "video_recorder.enabled = False\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed = 42\n",
    "gamma = 0.99\n",
    "max_steps_per_episode = 10000\n",
    "env = gym.make('CartPole-v0')\n",
    "env.seed(seed)\n",
    "eps = np.finfo(np.float32).eps.item() #Machine limits for floating point types.\n",
    "num_inputs = 4\n",
    "num_actions = 2\n",
    "num_hidden1 = 128\n",
    "num_hidden2 = 256\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputs = layers.Input(shape=(num_inputs,))\n",
    "common = layers.Dense(num_hidden1,activation = 'relu')(inputs)\n",
    "action = layers.Dense(num_actions, activation = 'softmax')(common)\n",
    "critic = layers.Dense(1)(common)\n",
    "\n",
    "model = keras.Model(inputs = inputs, outputs = [action,critic])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer dense is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "WARNING:tensorflow:From n:\\python\\lib\\site-packages\\tensorflow_core\\python\\ops\\math_grad.py:1394: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "optimizer = keras.optimizers.Adam(learning_rate = learning_rate)\n",
    "huber_loss = keras.losses.Huber()\n",
    "action_probs_history = []\n",
    "critic_value_history = []\n",
    "rewards_history = []\n",
    "running_reward = 0\n",
    "episode_count = 0\n",
    "\n",
    "while True:\n",
    "    state = env.reset()\n",
    "    episode_reward = 0\n",
    "    with tf.GradientTape() as tape :\n",
    "        for timestep in range(1,max_steps_per_episode):\n",
    "#             env.render()\n",
    "            state = tf.convert_to_tensor(state)\n",
    "            state = tf.expand_dims(state,0)\n",
    "            \n",
    "            action_probs, critic_value = model(state)\n",
    "            critic_value_history.append(critic_value[0,0])\n",
    "            \n",
    "            action = np.random.choice(num_actions, p = np.squeeze(action_probs))\n",
    "            \n",
    "            action_probs_history.append(tf.math.log(action_probs[0,action]))\n",
    "            \n",
    "            state, reward, done, _ = env.step(action)\n",
    "            \n",
    "            rewards_history.append(reward)\n",
    "            episode_reward += reward\n",
    "            if done :\n",
    "                break\n",
    "#         print(\"rewards\",episode_reward)\n",
    "        running_reward = (0.05* episode_reward) + ((1-0.05)* running_reward)\n",
    "        returns = []\n",
    "        discounted_sum = 0\n",
    "        \n",
    "        for i in rewards_history[::-1]:\n",
    "            discounted_sum = i + (gamma* discounted_sum)\n",
    "            returns.insert(0,discounted_sum)\n",
    "            \n",
    "        #Normalize\n",
    "        \n",
    "        returns = np.array(returns)\n",
    "        returns = (returns - np.mean(returns))/(np.std(returns)+eps)\n",
    "        returns = returns.tolist()\n",
    "        \n",
    "        history = zip(action_probs_history, critic_value_history, returns)\n",
    "        \n",
    "        actor_losses = []\n",
    "        critic_losses = []\n",
    "        \n",
    "        for log_prob, value, ret in history:\n",
    "            dif = ret - value\n",
    "            actor_losses.append(-log_prob*dif)\n",
    "            critic_losses.append(huber_loss(\n",
    "            tf.expand_dims(value,0), tf.expand_dims(ret,0)))\n",
    "            \n",
    "        loss_value = sum(actor_losses) + sum(critic_losses)\n",
    "        grads = tape.gradient(loss_value, model.trainable_variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
    "        \n",
    "        action_probs_history.clear()\n",
    "        critic_value_history.clear()\n",
    "        rewards_history.clear()\n",
    "        \n",
    "    episode_count += 1\n",
    "    if episode_count %10 ==0:\n",
    "        template = \"running reward : {:.2f} at episode {}\"\n",
    "        print(template.format(running_reward, episode_count))\n",
    "        \n",
    "    if running_reward > 195:\n",
    "        print(\"Solved at episode {}!\".format(episode_count))\n",
    "        video_record = None\n",
    "        video_record = VideoRecorder(env, 'videos/end.mp4', enabled=True)\n",
    "        for _ in range(1000):\n",
    "            env.render()\n",
    "            video_record.capture_frame()\n",
    "            state = tf.convert_to_tensor(state)\n",
    "            state = tf.expand_dims(state,0)\n",
    "            action_probs, _ = model(state)\n",
    "            action = np.random.choice(num_actions, p = np.squeeze(action_probs))\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            \n",
    "        print(\"Saved video.\")\n",
    "        video_record.close()\n",
    "        video_record.enabled = False\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}