playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config = {'use_only_solutions': True, 'word_len': 3, 'num_episodes_to_train': 10000000, 'greedy_player': False, 'reward_success': 15, 'reward_failure': -15, 'reward_yellow': 0.5, 'reward_green': 1.5, 'rounds_to_failure': 6, 'sync_every': 1000, 'learn_every': 3, 'lr': 0.00025, 'gamma': 0.9, 'n_embd': 64, 'n_head': 2, 'n_layer': 3, 'dropout': 0, 'save_every': 100000, 'burnin': 1000, 'exploration_rate_decay': 0.999991, 'exploration_rate_min': 0, 'batch_size': 32, 'memory_size': 1000, 'debug': False, 'log_every': 5000, 'save_dir': PosixPath('checkpoints/2023-03-19T18-13-04')}\n",
      "evaluating greedy agent\n",
      "state is !canWWWforWWWbut reward is 15 solution is but\n",
      "state is !webWGWsex reward is 15 solution is sex\n",
      "state is !hisWWWdayWWYyou reward is 15 solution is you\n",
      "state is !anyWWWwhoWYWhisGWWher reward is 15 solution is her\n",
      "state is !topWWWhadWGYday reward is 15 solution is day\n",
      "played 2000 games, won 100.0% of games, average game length for wins 3.273\n",
      "evaluating expectation agent\n",
      "state is !oneWGWand reward is 15 solution is and\n",
      "state is !one reward is 15 solution is one\n",
      "state is !oneWWYset reward is 15 solution is set\n",
      "state is !oneWWWdayWGWwasWGGhas reward is 15 solution is has\n",
      "state is !oneWWWdayWGGmayWGGway reward is 15 solution is way\n",
      "played 2000 games, won 100.0% of games, average game length for wins 2.891\n",
      "transformer model initialized, number parameters =  156456\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "from expectation_player import ExpectationPlayer\n",
    "from greedy_player import GreedyWordlePlayer\n",
    "from wordle_environment import WordleEnvironment\n",
    "import numpy as np\n",
    "import torch\n",
    "from pathlib import Path\n",
    "from rl_player import RLPlayer\n",
    "from game_room import evaluate_player, train_player\n",
    "use_only_solutions = True\n",
    "word_len = 3\n",
    "num_episodes_to_train = 10000000\n",
    "greedy_player = False\n",
    "reward_success = 15\n",
    "reward_failure = -15\n",
    "reward_yellow = 0.5\n",
    "reward_green = 1.5\n",
    "rounds_to_failure = 6\n",
    "sync_every = 1000\n",
    "learn_every = 3\n",
    "lr = 0.00025\n",
    "gamma = 0.9\n",
    "n_embd = 64\n",
    "n_head = 2\n",
    "n_layer = 3\n",
    "dropout = 0\n",
    "save_every = 100000\n",
    "burnin = 1000\n",
    "learn_every = 3\n",
    "exploration_rate_decay = 0.999991\n",
    "exploration_rate_min = 0\n",
    "batch_size = 32\n",
    "memory_size = 1000\n",
    "save_dir = Path(\"checkpoints\") / datetime.datetime.now().strftime(\"%Y-%m-%dT%H-%M-%S\")\n",
    "save_dir.mkdir(parents=True)\n",
    "debug = False\n",
    "log_every = 5000\n",
    "# create config variable from global variables (except for those starting with _)\n",
    "config = {k: v for k, v in globals().items() if not k.startswith('_') and isinstance(v, (int, float, bool, str))}\n",
    "config['save_dir'] = save_dir\n",
    "print(f\"config = {config}\")\n",
    "device = 'cpu'\n",
    "\n",
    "env = WordleEnvironment(config)\n",
    "\n",
    "# get greedy score for comparison\n",
    "greedy_agent = GreedyWordlePlayer(config)\n",
    "print(\"evaluating greedy agent\")\n",
    "__ = evaluate_player(greedy_agent, env, num_games_to_evaluate=2000)\n",
    "\n",
    "expectation_player = ExpectationPlayer(config)\n",
    "print(\"evaluating expectation agent\")\n",
    "__ = evaluate_player(expectation_player, env, num_games_to_evaluate=2000)\n",
    "\n",
    "# now train RL agent\n",
    "agent = GreedyWordlePlayer(config) if config['greedy_player'] else RLPlayer(config, device)\n",
    "train_player(agent, env, config, save_dir)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear.weight\n",
      "100\n",
      "linear.bias\n",
      "10\n",
      "embedding.weight\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n",
    "a = nn.Linear(4, 10)\n",
    "b = torch.rand(2, 4)\n",
    "m = torch.randn(10, 10)\n",
    "c = a(b)\n",
    "c = c.unsqueeze(0)\n",
    "(c @ m).shape"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "tensor([[0, 0, 0],\n        [1, 2, 3],\n        [4, 5, 6],\n        [3, 3, 3],\n        [3, 3, 3],\n        [3, 3, 3]])"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#create tensor from 0 to 100\n",
    "import torch\n",
    "a = torch.tensor([1,2,3])\n",
    "b = torch.tensor([4,5,6])\n",
    "l = [a,b]\n",
    "\n",
    "torch.stack([torch.tensor([0,0,0]), *l, *[torch.tensor([3,3,3])]*3])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "data": {
      "text/plain": "['adb',\n ('W', 'W', 'W'),\n ('W', 'W', 'Y'),\n ('W', 'W', 'G'),\n ('W', 'Y', 'W'),\n ('W', 'Y', 'Y'),\n ('W', 'Y', 'G'),\n ('W', 'G', 'W'),\n ('W', 'G', 'Y'),\n ('W', 'G', 'G'),\n ('Y', 'W', 'W'),\n ('Y', 'W', 'Y'),\n ('Y', 'W', 'G'),\n ('Y', 'Y', 'W'),\n ('Y', 'Y', 'Y'),\n ('Y', 'Y', 'G'),\n ('Y', 'G', 'W'),\n ('Y', 'G', 'Y'),\n ('Y', 'G', 'G'),\n ('G', 'W', 'W'),\n ('G', 'W', 'Y'),\n ('G', 'W', 'G'),\n ('G', 'Y', 'W'),\n ('G', 'Y', 'Y'),\n ('G', 'Y', 'G'),\n ('G', 'G', 'W'),\n ('G', 'G', 'Y'),\n ('G', 'G', 'G')]"
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "letters = 'WYG'\n",
    "combinations = itertools.product(letters, repeat=3)\n",
    "\n",
    "a = ['adb'] + [c for c in combinations]\n",
    "a"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'p' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[57], line 6\u001B[0m\n\u001B[1;32m      3\u001B[0m     p \u001B[38;5;241m=\u001B[39m [\u001B[38;5;241m1\u001B[39m,\u001B[38;5;241m2\u001B[39m,\u001B[38;5;241m3\u001B[39m]\n\u001B[1;32m      5\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m a \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m1\u001B[39m:\n\u001B[0;32m----> 6\u001B[0m     \u001B[38;5;28mprint\u001B[39m(\u001B[43mp\u001B[49m[\u001B[38;5;241m0\u001B[39m])\n",
      "\u001B[0;31mNameError\u001B[0m: name 'p' is not defined"
     ]
    }
   ],
   "source": [
    "a = 1\n",
    "if a == 2:\n",
    "    p = [1,2,3]\n",
    "\n",
    "if a == 1:\n",
    "    print(p[0])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}