Keras_sample_weight_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_probability as tfp\n",
    "from tensorflow.keras import Input\n",
    "from tensorflow.keras.layers import Dense\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dim, action_dim, fcnet_hiddens = 24, 1, [32, 32, 32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_neural_net_stochastic(input_dim, action_dim, fcnet_hiddens):\n",
    "    \"\"\"Build a keras model to output a stochastic policy.\n",
    "    Parameters\n",
    "    ----------\n",
    "    input_dim : int\n",
    "        dimension of input layer\n",
    "    action_dim : int\n",
    "        action_space dimension\n",
    "    fcnet_hiddens : list\n",
    "        list containing size of each hidden layer (length of list is number of hidden layers)\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    Keras model (untrained)\n",
    "    \"\"\"\n",
    "    input_layer = Input(shape=(input_dim, ))\n",
    "    curr_layer = input_layer\n",
    "\n",
    "    for i in range(len(fcnet_hiddens)):\n",
    "        size = fcnet_hiddens[i]\n",
    "        dense = Dense(size, activation=\"tanh\")\n",
    "        curr_layer = dense(curr_layer)\n",
    "\n",
    "    out = Dense(2 * action_dim, activation=None)(curr_layer)\n",
    "    model = tf.keras.Model(inputs=input_layer, outputs=out, name=\"policy_network\")\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loss(stochastic, variance_regularizer):\n",
    "    \"\"\"Get appropriate loss function for training.\n",
    "    Parameters\n",
    "    ----------\n",
    "    stochastic : bool\n",
    "        determines if policy to be learned is deterministic or stochastic\n",
    "    variance_regularizer : float\n",
    "        regularization hyperparameter to penalize high variance policies\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    Keras loss function to use for imitation learning.\n",
    "    \"\"\"\n",
    "    if stochastic:\n",
    "        return negative_log_likelihood_loss(variance_regularizer)\n",
    "    else:\n",
    "        return tf.keras.losses.mean_squared_error\n",
    "\n",
    "def negative_log_likelihood_loss(variance_regularizer):\n",
    "    \"\"\"Negative log likelihood loss for learning stochastic policies.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    variance_regularizer : float\n",
    "        regularization hyperparameter to penalize high variance policies\n",
    "    Returns\n",
    "    -------\n",
    "    Negative log likelihood loss function with variance regularization.\n",
    "    \"\"\"\n",
    "\n",
    "    def nll_loss(y, network_output):\n",
    "        assert network_output.shape[1] % 2 == 0, \"Stochastic policies must output vectors of even length\"\n",
    "\n",
    "        action_dim = network_output.shape[1] // 2\n",
    "\n",
    "        # first half of network_output is mean, second half is log_std\n",
    "        means, log_stds = network_output[:, :action_dim], network_output[:, action_dim:]\n",
    "        stds = tf.math.exp(log_stds)\n",
    "        variances = tf.math.square(stds)\n",
    "\n",
    "        # Multivariate Gaussian distribution\n",
    "        dist = tfp.distributions.MultivariateNormalDiag(loc=means, scale_diag=variances)\n",
    "        loss = dist.log_prob(y)\n",
    "        loss = tf.negative(loss)\n",
    "        loss = tf.reduce_mean(loss) + (variance_regularizer * tf.norm(variances))\n",
    "        return loss\n",
    "\n",
    "    return nll_loss\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compile_network(model):\n",
    "    \"\"\"\n",
    "    Compiles Keras network with appropriate loss and optimizer\n",
    "    \"\"\"\n",
    "    loss = get_loss(True, 10)\n",
    "    model.compile(loss=loss, optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_neural_net_stochastic(input_dim, action_dim, fcnet_hiddens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compile_network(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_batch = np.random.rand(600, 1)\n",
    "action_batch = action_batch.reshape(action_batch.shape[0], action_dim)\n",
    "observation_batch = np.random.rand(600, 24)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Error 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.train_on_batch(observation_batch, action_batch, sample_weight=np.random.rand(600,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Error 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.train_on_batch(observation_batch, action_batch, sample_weight=np.random.rand(600,1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "flow3.7",
   "language": "python",
   "name": "flow3.7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}