Tic-Tac-Toe-Code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import timeit\n",
    "import numpy as np\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "!pip --version\n",
    "!python --version\n",
    "!pip list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Change the dimension of the board here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BM = 6 #6 for 6x6, 3 for 3x3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transform the Tic - Tac - Toe board to 6 transformations\n",
    "board_dim = BM\n",
    "matrix = [[(index, elem%board_dim) for elem in val] for index,val in enumerate(np.array(range(board_dim**2)).reshape(board_dim, board_dim))]\n",
    "\n",
    "def rotate(mat, N):\n",
    "    # Consider all squares one by one \n",
    "    for x in range(0, int(N/2)):      \n",
    "        # Consider elements in group    \n",
    "        # of 4 in current square \n",
    "        for y in range(x, N-x-1):   \n",
    "            # store current cell in temp variable \n",
    "            temp = mat[x][y] \n",
    "            # move values from right to top \n",
    "            mat[x][y] = mat[y][N-1-x] \n",
    "            # move values from bottom to right \n",
    "            mat[y][N-1-x] = mat[N-1-x][N-1-y] \n",
    "            # move values from left to bottom \n",
    "            mat[N-1-x][N-1-y] = mat[N-1-y][x] \n",
    "            # assign temp to left \n",
    "            mat[N-1-y][x] = temp \n",
    "    return mat\n",
    "  \n",
    "# Function to pr the matrix \n",
    "def displayMatrix(mat, N):  \n",
    "    for i in range(0, N): \n",
    "        for j in range(0, N): \n",
    "            print (mat[i][j], end = ' ') \n",
    "        print (\"\") \n",
    "\n",
    "def anticlock_wise(input_matrix, board_dim, number=1):\n",
    "    \n",
    "    diagonal_flip = copy.deepcopy(input_matrix)\n",
    "    anti_diagonal_flip = copy.deepcopy(input_matrix)\n",
    "    matrix = copy.deepcopy(input_matrix)\n",
    "    \n",
    "    input_matrix_list = np.array(matrix).reshape(1,board_dim**2, 2)[0]\n",
    "    transform_dictionary = {tuple(entry):[] for entry in input_matrix_list}\n",
    "    \n",
    "    #displayMatrix(matrix, board_dim)\n",
    "    #print('\\n')  \n",
    "    \n",
    "    for i in range(number):\n",
    "        matrix = rotate(matrix,board_dim)\n",
    "        new_matrix_list = np.array(matrix).reshape(1,board_dim**2, 2)[0]\n",
    "        count=0\n",
    "        for key in transform_dictionary:\n",
    "            transform_dictionary[key].append(tuple(new_matrix_list[count]))\n",
    "            count+=1\n",
    "        #displayMatrix(matrix, board_dim)\n",
    "        #print('\\n')\n",
    "        \n",
    "    \n",
    "    \n",
    "    #displayMatrix(anti_diagonal_flip, board_dim)\n",
    "    #print('\\n')  \n",
    "    \n",
    "    \n",
    "    for row in range(0, board_dim):\n",
    "        for col in range(row):\n",
    "            diagonal_flip[row][col],diagonal_flip[col][row] = diagonal_flip[col][row],diagonal_flip[row][col]\n",
    "    \n",
    "    #displayMatrix(diagonal_flip, board_dim)\n",
    "    #print('\\n')\n",
    "    \n",
    "    new_matrix_list = np.array(diagonal_flip).reshape(1,board_dim**2, 2)[0]\n",
    "    count=0\n",
    "    for key in transform_dictionary:\n",
    "        transform_dictionary[key].append(tuple(new_matrix_list[count]))\n",
    "        count+=1\n",
    "            \n",
    "    for row in range(0, board_dim):\n",
    "        for col in range(board_dim-1-row):\n",
    "            anti_diagonal_flip[row][col],anti_diagonal_flip[board_dim-1-col][board_dim-1-row] = anti_diagonal_flip[board_dim-1-col][board_dim-1-row],anti_diagonal_flip[row][col]\n",
    "            \n",
    "    #displayMatrix(anti_diagonal_flip, board_dim)\n",
    "    #print('\\n')\n",
    "    \n",
    "\n",
    "    new_matrix_list = np.array(anti_diagonal_flip).reshape(1,board_dim**2, 2)[0]\n",
    "    count=0\n",
    "    for key in transform_dictionary:\n",
    "        transform_dictionary[key].append(tuple(new_matrix_list[count]))\n",
    "        count+=1\n",
    "    \n",
    "    return transform_dictionary\n",
    "transform_dict = anticlock_wise(matrix, board_dim, number=3) #anticlockwise_rotation (3) + diagonal flip\n",
    "#transform_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import the previous learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"RL_6x6_Values.p\", \"rb\") as f:\n",
    "    Imported_values = pickle.load(f)\n",
    "    #Imported_values = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RL Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RL_Agent(object):\n",
    "    \n",
    "    #Value updation happens here\n",
    "    \n",
    "    def __init__(self, board_dim=BM, Imported_values=Imported_values): \n",
    "        \n",
    "        self.count = 0 #Beginning of the time\n",
    "        self.board_dim = board_dim\n",
    "        self.rl_board = np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim)\n",
    "        self.gamma = 0.9\n",
    "        self.values = Imported_values#dict()\n",
    "        self.start = None #Episode start\n",
    "        self.score = None #Episode score card\n",
    "        self.episode = []\n",
    "        self.slot = 0 #Beginning of episode\n",
    "\n",
    "        \n",
    "    #Choose a slot\n",
    "    #If the game is started by RL agent, a empty is chosen randomly\n",
    "    #Otherwise, the next move is chosen such that next state is having the highest value \n",
    "    def play(self, last_move, start=None):\n",
    "        if start:\n",
    "            self.start = start\n",
    "            cord_of_empty_x_y = np.where(self.rl_board == 'e')\n",
    "            cord = list(zip(cord_of_empty_x_y[0],cord_of_empty_x_y[1])) \n",
    "            random_slot = cord[np.random.choice(range(len(cord)),1)[0]] #1st position is randomly picked\n",
    "            self.rl_board[random_slot[0]][random_slot[1]] = 'x'\n",
    "            self.episode.append(random_slot)\n",
    "            self.slot+=1\n",
    "            return random_slot\n",
    "        else:\n",
    "            self.slot+=2\n",
    "            self.rl_board[last_move] = 'o'\n",
    "            self.episode.append(last_move)\n",
    "            next_slot = None\n",
    "            if self.slot in self.values:\n",
    "                board_string_format = ''.join([''.join(row) for row in self.rl_board])\n",
    "                possible_next_states=[key for key in self.values[self.slot].keys() if [(i,j) for i,j in zip(board_string_format,key) if i!=j] == [('e', 'x')]]\n",
    "                if possible_next_states:\n",
    "                    next_state_string_format = sorted({state:self.values[self.slot][state] for state in possible_next_states}, key = lambda x: self.values[self.slot][x][0], reverse=True)[0]\n",
    "                    if self.values[self.slot][next_state_string_format][0] > 0:\n",
    "                        next_action = [i for i,ch in enumerate(board_string_format) if next_state_string_format[i]!=ch][0]\n",
    "                        next_slot = (next_action//self.board_dim , next_action%self.board_dim)\n",
    "                    else:\n",
    "                        not_to_consider_states = {state for state in possible_next_states if self.values[self.slot][state][0]<0}\n",
    "                        not_to_consider_states_next_action = [[i for i,ch in enumerate(board_string_format) if state[i]!=ch][0] for state in not_to_consider_states]\n",
    "                        not_to_consider_states_next_slot = [(slot//self.board_dim, slot%self.board_dim) for slot in not_to_consider_states_next_action]\n",
    "                        cord_of_empty_x_y = np.where(self.rl_board == 'e')\n",
    "                        cord = list(zip(cord_of_empty_x_y[0],cord_of_empty_x_y[1]))\n",
    "                        cord = list(set(cord).difference(set(not_to_consider_states_next_slot)))\n",
    "                        if not cord:\n",
    "                            next_action = [i for i,ch in enumerate(board_string_format) if next_state_string_format[i]!=ch][0]\n",
    "                            next_slot = (next_action//self.board_dim , next_action%self.board_dim)\n",
    "                        else:\n",
    "                            next_slot = cord[np.random.choice(range(len(cord)),1)[0]]\n",
    "                        #print(\"BINCY CHECK\", 'possible_next_states', possible_next_states, 'self.values[self.slot]=', self.values[self.slot], 'not_to_consider_states_next_action = ', not_to_consider_states_next_action, 'not_to_consider_states_next_slot =', not_to_consider_states_next_slot, 'cord=', cord,  'next_slot=', next_slot)\n",
    "            if next_slot:\n",
    "                self.rl_board[next_slot[0]][next_slot[1]] = 'x'\n",
    "                self.episode.append(next_slot)\n",
    "                return next_slot\n",
    "            else:\n",
    "                cord_of_empty_x_y = np.where(self.rl_board == 'e')\n",
    "                cord = list(zip(cord_of_empty_x_y[0],cord_of_empty_x_y[1])) \n",
    "                random_slot = cord[np.random.choice(range(len(cord)),1)[0]] #1st position is randomly picked\n",
    "                self.rl_board[random_slot[0]][random_slot[1]] = 'x'\n",
    "                self.episode.append(random_slot)\n",
    "                return random_slot\n",
    "                                \n",
    "    #Get notified by the judge regarding the end of the game with the winner\n",
    "    def notify(self, opp_move, score):\n",
    "        if self.episode[-1]!=opp_move:\n",
    "            self.episode.append(opp_move)\n",
    "        self.score = score\n",
    "        self.update_values()\n",
    "        self.initialize_board()\n",
    "    \n",
    "    #Update the value function from the episode (x12 times learning)\n",
    "    def update_values(self):\n",
    "        \n",
    "        \n",
    "        #6x2 Transformations:\n",
    "        for learn in ['SELF','OPP']:\n",
    "            \n",
    "            if learn == 'SELF':\n",
    "                SCORE = self.score\n",
    "                if self.start == 'x':\n",
    "                    START = 'x'\n",
    "                else:\n",
    "                    START = 'o'\n",
    "                \n",
    "            else:\n",
    "                \n",
    "                if self.score == 'x':\n",
    "                    SCORE = 'o'\n",
    "                elif self.score == 'o':\n",
    "                    SCORE = 'x'\n",
    "                else:\n",
    "                    SCORE = 'd'\n",
    "                \n",
    "                if self.start == 'x':\n",
    "                    START = 'o'\n",
    "                else:\n",
    "                    START = 'x'\n",
    "                    \n",
    "                \n",
    "            \n",
    "            for i in range(6):\n",
    "                if not i:\n",
    "                    episode_transformed = self.episode\n",
    "                else:\n",
    "                    episode_transformed = list(map(lambda x:transform_dict[x][i-1], self.episode))\n",
    "                board_transformed = np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim)\n",
    "                if START == 'o':\n",
    "                    for (i,j) in episode_transformed[::2]:\n",
    "                        board_transformed[i][j] = 'o'\n",
    "                    for (i,j) in episode_transformed[1::2]:\n",
    "                        board_transformed[i][j] = 'x'\n",
    "                else:\n",
    "                    for (i,j) in episode_transformed[::2]:\n",
    "                        board_transformed[i][j] = 'x'\n",
    "                    for (i,j) in episode_transformed[1::2]:\n",
    "                        board_transformed[i][j] = 'o'\n",
    "\n",
    "                #if SCORE != 'd':\n",
    "                G = 0\n",
    "                slot = len(episode_transformed)\n",
    "                if SCORE == 'x':\n",
    "                    reward = 1\n",
    "                elif SCORE == 'd':\n",
    "                    reward = -1\n",
    "                else:\n",
    "                    reward = -10\n",
    "                for time in range(1,len(episode_transformed))[::-1]:\n",
    "                    if board_transformed[episode_transformed[time][0]][episode_transformed[time][1]] == 'x':\n",
    "                        G = ((self.gamma*G) + reward)\n",
    "                        reward = 0\n",
    "                        board_string_format = ''.join([''.join(row) for row in board_transformed])\n",
    "                        if not slot in self.values:\n",
    "                            self.values[slot] = {board_string_format:[G,1]}\n",
    "                        elif not board_string_format in self.values[slot]:\n",
    "                            self.values[slot][board_string_format] = [G,1]\n",
    "                        else:\n",
    "                            self.values[slot][board_string_format][1]+=1\n",
    "                            self.values[slot][board_string_format][0]+=(G - self.values[slot][board_string_format][0])/self.values[slot][board_string_format][1]   \n",
    "\n",
    "                    board_transformed[episode_transformed[time][0]][episode_transformed[time][1]] = 'e'\n",
    "                    slot-=1\n",
    "                #print(self.values)\n",
    "                #print('\\n')\n",
    "    \n",
    "    #To initialize the board and game after each episode\n",
    "    def initialize_board(self):\n",
    "        self.count +=1\n",
    "        self.rl_board = np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim)\n",
    "        self.start = None\n",
    "        self.score = None\n",
    "        self.episode = []\n",
    "        self.slot = 0 #Beginning of episode\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create an RL agent object and methods to play and notify\n",
    "rl_agent1 = RL_Agent(BM)\n",
    "rl_agent1_play = getattr(rl_agent1, 'play')\n",
    "rl_agent1_notify = getattr(rl_agent1, 'notify')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Player"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Random_Player(object):\n",
    "    def __init__(self, board_dim=BM):\n",
    "        self.board_dim = board_dim\n",
    "        self.rp_board = np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim)\n",
    "        self.start = None\n",
    "        \n",
    "\n",
    "    #Choose an empty slot with equal probability and update the last move\n",
    "    def play(self, last_move, start=None):\n",
    "        if start:\n",
    "            self.start = start\n",
    "        else:\n",
    "            self.rp_board[last_move] = 'x'\n",
    "        random_slot = self.random()\n",
    "        self.rp_board[random_slot[0]][random_slot[1]] = 'o'\n",
    "        return random_slot\n",
    "        \n",
    "    #Choose a random empty slot with equal probability\n",
    "    def random(self):\n",
    "        cord_of_empty_x_y = np.where(self.rp_board == 'e')\n",
    "        cord = list(zip(cord_of_empty_x_y[0],cord_of_empty_x_y[1])) \n",
    "        random_slot = cord[np.random.choice(range(len(cord)),1)[0]] #1st position is randomly picked\n",
    "        return random_slot\n",
    "    \n",
    "    #Dummy function; get notification from the judge\n",
    "    def notify(self, last_move=None, score=None):\n",
    "        self.initialize_board()\n",
    "    \n",
    "    #To initialize the board at the end of the game\n",
    "    def initialize_board(self):\n",
    "        self.rp_board = np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create an random player object and methods to play and notify\n",
    "random_player1 = Random_Player(BM)\n",
    "random_player1_play = getattr(random_player1, 'play')\n",
    "random_player1_notify = getattr(random_player1, 'notify')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Judge"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Judge's responsibility:\n",
    "- Decide the starting player\n",
    "- Regulate the game (whose turn when)\n",
    "- Evaluate the winner\n",
    "- Notify both the players about the end of the game and who is the winner\n",
    "- Keep the statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "class Judge(object):\n",
    "    def __init__(self, board_dim=BM):\n",
    "        self.board_dim = board_dim\n",
    "        self.judge_board = np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim)\n",
    "        self.winning_pos = np.concatenate([[list(range(r, r+(self.board_dim*(self.board_dim-1))+1, self.board_dim)) for r in range(self.board_dim)] ,[list(range(c,c+self.board_dim)) for c in range(0,self.board_dim**2, self.board_dim)], [np.array(range(self.board_dim**2)).reshape(self.board_dim, self.board_dim).diagonal()], [np.array(range(self.board_dim-1,self.board_dim**2,self.board_dim))-range(self.board_dim)]])\n",
    "        self.winning_pos_cord = np.array([[[wpe//self.board_dim, wpe%self.board_dim] for wpe in  wp] for wp in self.winning_pos])\n",
    "        self.stat = [0, {'o': 0,'x':0, 'd':0 }]\n",
    "        self.step = 0\n",
    "        \n",
    "    #For changing the board\n",
    "    def change_board(self, board): #board should be np.array\n",
    "        self.judge_board = board\n",
    "        \n",
    "    #To check if the game is done; Identifies the draw even if there are empty slots\n",
    "    def isdone(self):\n",
    "        #Isdone will be called only from 2*(board_dim)-1 moves\n",
    "        winning_positions_status = [[self.judge_board[cord[0]][cord[1]] for cord in wps] for wps in self.winning_pos_cord]\n",
    "        if ['x']*self.board_dim in winning_positions_status:\n",
    "            return 'x'\n",
    "        elif ['o']*self.board_dim in winning_positions_status:\n",
    "            return 'o'\n",
    "        elif [wp for wp in winning_positions_status if 'e' in wp]:\n",
    "            return None\n",
    "        else:\n",
    "            return 'd' #Judge doensn't wait for the entire game but check if there is any scope of winning for anyone\n",
    "    \n",
    "    #To regulate the game\n",
    "    #Alternative chances to each player\n",
    "    #Identifies the end of the game\n",
    "    #Notifies the players\n",
    "    #Keep the statistics\n",
    "    def game(self, start, number_of_games, method1_play, method2_play, method1_notify, method2_notify):\n",
    "        for num in range(number_of_games):\n",
    "            if start == 'o':\n",
    "                check_iter = 1\n",
    "            else:\n",
    "                check_iter = 2\n",
    "            score = self.isdone()\n",
    "            last_move = None\n",
    "\n",
    "            #self.print_board()\n",
    "            while not score:\n",
    "                if not self.step:\n",
    "                    if check_iter%2:\n",
    "                        last_move=method1_play(last_move, start) #Random player\n",
    "                        self.judge_board[last_move] = 'o'\n",
    "                    else:\n",
    "                        last_move=method2_play(last_move, start) #RL Agent\n",
    "                        self.judge_board[last_move] = 'x'\n",
    "                    # HERE\n",
    "                    #self.print_board()\n",
    "                else:\n",
    "                    if check_iter%2:\n",
    "                        last_move=method1_play(last_move) #Random player\n",
    "                        self.judge_board[last_move] = 'o'\n",
    "                    else:\n",
    "                        last_move=method2_play(last_move) #RL Agent\n",
    "                        self.judge_board[last_move] = 'x'\n",
    "                    # HERE\n",
    "                    #self.print_board()\n",
    "                self.step+=1\n",
    "                #last_move is cordinate\n",
    "                check_iter+=1\n",
    "                score = self.isdone()\n",
    "            self.step = 0 #Initialize\n",
    "            self.update_stats(score)\n",
    "            method2_notify(last_move, score) #RL Agent\n",
    "            method1_notify(last_move, score) #Random player\n",
    "            self.change_board(np.array(['e']*(self.board_dim**2)).reshape(self.board_dim, self.board_dim))\n",
    "    \n",
    "    #Update the statistics\n",
    "    def update_stats(self, score):\n",
    "        self.stat[0]+=1\n",
    "        self.stat[1][score]+=1\n",
    "    \n",
    "    #To print the board\n",
    "    def print_board(self):\n",
    "        for i in range(self.board_dim): \n",
    "            for j in range(self.board_dim): \n",
    "                print(self.judge_board[i][j], end = \" \") \n",
    "            print()\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create an judge object and methods to play and notify\n",
    "judge = Judge(BM)\n",
    "#Below for just one game\n",
    "#judge.game('o',1, random_player1_play, rl_agent1_play, random_player1_notify, rl_agent1_notify)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Plot % of win, loss and draw for N games\n",
    "## Alternate Turns for each player\n",
    "win, loss, draw, total = [],[],[],[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Multiples of 100 only\n",
    "GAME=10000\n",
    "x = range(0,GAME)\n",
    "\n",
    "for iter in range(1,GAME+1):\n",
    "    if iter%2:\n",
    "        judge.game('x',1, random_player1_play, rl_agent1_play, random_player1_notify, rl_agent1_notify)\n",
    "    else:\n",
    "        judge.game('o',1, random_player1_play, rl_agent1_play, random_player1_notify, rl_agent1_notify)\n",
    "    if not iter%100:\n",
    "        win.append(judge.stat[1]['x'])\n",
    "        loss.append(judge.stat[1]['o'])\n",
    "        draw.append(judge.stat[1]['d'])\n",
    "        total.append(judge.stat[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse = 1\n",
    "plt.plot(total[::sparse], win[::sparse], label='win', linewidth=3)\n",
    "plt.plot(total[::sparse], loss[::sparse], label='loss', linewidth=3)\n",
    "#plt.plot(total[::sparse], draw[::sparse], label='draw', linewidth=3)\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "judge.stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving values to pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"RL_6x6_Values.p\", \"wb\") as f:\n",
    "    pickle.dump(rl_agent1.values, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}