muzero_base.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 530
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 16139,
     "status": "ok",
     "timestamp": 1594397837983,
     "user": {
      "displayName": "Paul Norridge",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjNlDOv29xQkBsQIEn5-nR54xvwmx2Xg7RxWoRsfA=s64",
      "userId": "03209312088987020040"
     },
     "user_tz": -60
    },
    "id": "FRi6eZNBbtkx",
    "outputId": "4b6a9df3-4c49-47b8-e172-dc25ce90e9b2",
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 743,
     "status": "ok",
     "timestamp": 1594398844022,
     "user": {
      "displayName": "Paul Norridge",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjNlDOv29xQkBsQIEn5-nR54xvwmx2Xg7RxWoRsfA=s64",
      "userId": "03209312088987020040"
     },
     "user_tz": -60
    },
    "id": "NHyE48yYZ5Nn"
   },
   "outputs": [],
   "source": [
    "\n",
    "import tensorflow as tf\n",
    "import random\n",
    "import numpy as np\n",
    "from statistics import mean \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3158,
     "status": "ok",
     "timestamp": 1594398848058,
     "user": {
      "displayName": "Paul Norridge",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjNlDOv29xQkBsQIEn5-nR54xvwmx2Xg7RxWoRsfA=s64",
      "userId": "03209312088987020040"
     },
     "user_tz": -60
    },
    "id": "MWqmloprbVFa"
   },
   "outputs": [],
   "source": [
    "\n",
    "from mcts import Node, run_mcts\n",
    "from gamecomponents import Policy\n",
    "from game import Game\n",
    "from replaybuffer import ReplayBuffer\n",
    "from network import Network, SharedStorage\n",
    "from helpers import KnownBounds\n",
    "from muzero_core import play_game, train_network, MuZeroConfig, make_aigym_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P9oP7LtTm80-"
   },
   "outputs": [],
   "source": [
    "c = make_aigym_config('CartPole-v0')\n",
    "s = SharedStorage()\n",
    "r = ReplayBuffer()\n",
    "\n",
    "# tweaks to the configuration\n",
    "c.num_simulations = 60\n",
    "c.training_steps = 150\n",
    "c.batch_size = 32\n",
    "c.td_steps = 25\n",
    "c.discount = 1. \n",
    "c.lr_init = 0.005\n",
    "c.lr_decay_steps = 100e3\n",
    "c.max_moves = 2000\n",
    "c.num_unroll_steps = 4\n",
    "\n",
    "c.epsilon = 0.001\n",
    "\n",
    "# a little bit of help - starting point for value fn bounds \n",
    "c.known_bounds = KnownBounds(max = 20, min = -20)\n",
    "\n",
    "# Set up the network parameters\n",
    "Network.N = 4\n",
    "\n",
    "# tweak the MCTS parameter\n",
    "Node.root_exploration_fraction = 0.30\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate some game runs to get us started\n",
    "for _ in range(30):\n",
    "  n = s.latest_network()\n",
    "  g1 = play_game(c, n)\n",
    "  r.save_game(g1)\n",
    "  print(sum(g1.rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "KBIt87EqxVn2",
    "outputId": "5acf4bc4-c5d4-4f45-f692-d59e95dbcf89",
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "for iter in range(400):\n",
    "\n",
    "  train_network(c,s,r)\n",
    "\n",
    "  n = s.latest_network()\n",
    "\n",
    "  # play some games\n",
    "  # if the model has improved significantly then repeat for \n",
    "  # up to 25 times to add new information to the replay buffer \n",
    "  tot = 0\n",
    "  N = 0\n",
    "  better = True\n",
    "  while better:\n",
    "    g1 = play_game(c, n)\n",
    "    print(g1.length())\n",
    "    tot += (g1.length())\n",
    "    N += 1\n",
    "\n",
    "    better = tot/N > max(r.game_len)*0.5 and N < 25\n",
    "    r.save_game(g1)\n",
    "    \n",
    "  print((iter, tot/N))\n",
    "  print('----')\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "muzero_base.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}