12_tensorflow_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "excellent-daily",
   "metadata": {},
   "source": [
    "# Custom Models and Training with TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "upset-korean",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FIXME: meke autocompletion working again\n",
    "%config Completer.use_jedi = False\n",
    "\n",
    "import os\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "capable-mandate",
   "metadata": {},
   "source": [
    "## Tensors and Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "terminal-craft",
   "metadata": {},
   "source": [
    "### Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "collected-gateway",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# matrix\n",
    "tf.constant([[1., 2., 3.], [4., 5., 6.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "chinese-location",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# scalar\n",
    "tf.constant(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "demonstrated-understanding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = tf.constant([[1., 2., 3.], [4., 5., 6.]])\n",
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "stunning-flower",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "rotary-motion",
   "metadata": {},
   "source": [
    "### Indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "sensitive-stuart",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[2., 3.],\n",
       "       [5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[:, 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "metric-transfer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 1), dtype=float32, numpy=\n",
       "array([[2.],\n",
       "       [5.]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acute-symposium",
   "metadata": {},
   "source": [
    "### Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "talented-payday",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "agricultural-treaty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "alike-welding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[14., 32.],\n",
       "       [32., 77.]], dtype=float32)>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t @ tf.transpose(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "documented-anatomy",
   "metadata": {},
   "source": [
    "### Using keras.backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "sunrise-turning",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "\n",
    "# Common alias for Keras backend\n",
    "K = keras.backend\n",
    "\n",
    "K.square(K.transpose(t)) + 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aboriginal-weapon",
   "metadata": {},
   "source": [
    "### Tensors and NumPy\n",
    "**Notice** that TF uses single precision by default while NumPy uses double precision."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "disabled-marketplace",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "muslim-demonstration",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# equivalent: `np.array(t)`\n",
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "offensive-strand",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "medieval-masters",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "centered-cleaners",
   "metadata": {},
   "source": [
    "### Type Conversions\n",
    "TF expects homogeneous data types which are checked and an exeption is raised when they do not match."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "affecting-circus",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These raise an `InvalidArgumentError`\n",
    "# `tf.constant(2.) + tf.constant(40)`\n",
    "# `tf.constant(2.) + tf.constant(40, dtype=tf.float64)\n",
    "\n",
    "# These is ok\n",
    "t2 = tf.constant(40, dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sealed-gates",
   "metadata": {},
   "source": [
    "### Strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "injured-devil",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(b\"hello world\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "equal-december",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'caf\\xc3\\xa9'>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.constant(\"café\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "unauthorized-massachusetts",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in \"café\"])\n",
    "u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "understanding-lighting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=4>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.strings.unicode_encode(u, \"UTF-8\")\n",
    "tf.strings.length(b, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "green-pharmacology",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 233], dtype=int32)>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_decode(b, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "prime-czech",
   "metadata": {},
   "source": [
    "### String arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "strange-peeing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([4, 6, 5, 2], dtype=int32)>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = tf.constant([\"Café\", \"Coffee\", \"caffè\", \"咖啡\"])\n",
    "\n",
    "tf.strings.length(p, unit=\"UTF8_CHAR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "seventh-gamma",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857]]>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = tf.strings.unicode_decode(p, \"UTF8\")\n",
    "r"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "peripheral-triple",
   "metadata": {},
   "source": [
    "### Ragged tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "extreme-lindsay",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(6,), dtype=int32, numpy=array([ 67, 111, 102, 102, 101, 101], dtype=int32)>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "perfect-parish",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232]]>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r[1:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "hungry-winning",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233], [67, 111, 102, 102, 101, 101], [99, 97, 102, 102, 232], [21654, 21857], [65, 66], [], [67]]>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2 = tf.ragged.constant([[65, 66], [], [67]])\n",
    "tf.concat([r, r2], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "inappropriate-charge",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[67, 97, 102, 233, 68, 69, 70], [67, 111, 102, 102, 101, 101, 71], [99, 97, 102, 102, 232], [21654, 21857, 72, 73]]>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3 = tf.ragged.constant([[68, 69, 70], [71], [], [72, 73]])\n",
    "tf.concat([r, r3], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "liquid-nicaragua",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=string, numpy=array([b'DEF', b'G', b'', b'HI'], dtype=object)>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.strings.unicode_encode(r3, \"UTF-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "employed-nigeria",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4, 6), dtype=int32, numpy=\n",
       "array([[   67,    97,   102,   233,     0,     0],\n",
       "       [   67,   111,   102,   102,   101,   101],\n",
       "       [   99,    97,   102,   102,   232,     0],\n",
       "       [21654, 21857,     0,     0,     0,     0]], dtype=int32)>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.to_tensor()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "green-silicon",
   "metadata": {},
   "source": [
    "### Sparse tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "difficult-rough",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 1]\n",
      " [1 0]\n",
      " [2 3]], shape=(3, 2), dtype=int64), values=tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "s = tf.SparseTensor(\n",
    "    indices=[[0, 1], [1, 0], [2, 3]],\n",
    "    values=[1., 2., 3.],\n",
    "    dense_shape=[3, 4],\n",
    ")\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "little-arctic",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 1., 0., 0.],\n",
       "       [2., 0., 0., 0.],\n",
       "       [0., 0., 0., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "compatible-liabilities",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unsupported operand type(s) for +: 'SparseTensor' and 'float'\n"
     ]
    }
   ],
   "source": [
    "# This is ok\n",
    "s2 = s * 2.0\n",
    "\n",
    "# This is not ok\n",
    "try:\n",
    "    s3 = s + 1.\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "applied-workshop",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[ 30.,  40.],\n",
       "       [ 20.,  40.],\n",
       "       [210., 240.]], dtype=float32)>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4 = tf.constant([[10., 20.], [30., 40.], [50., 60.], [70., 80.]])\n",
    "\n",
    "tf.sparse.sparse_dense_matmul(s, s4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "effective-serbia",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparseTensor(indices=tf.Tensor(\n",
      "[[0 2]\n",
      " [0 1]], shape=(2, 2), dtype=int64), values=tf.Tensor([1. 2.], shape=(2,), dtype=float32), dense_shape=tf.Tensor([3 4], shape=(2,), dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "s5 = tf.SparseTensor(\n",
    "    indices=[[0, 2], [0, 1]],\n",
    "    values=[1., 2.],\n",
    "    dense_shape=[3, 4],\n",
    ")\n",
    "print(s5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "overall-panama",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indices[1] = [0,1] is out of order. Many sparse ops require sorted indices.\n",
      "    Use `tf.sparse.reorder` to create a correctly ordered copy.\n",
      "\n",
      " [Op:SparseToDense]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.sparse.to_dense(s5)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "entitled-education",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 4), dtype=float32, numpy=\n",
       "array([[0., 2., 1., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]], dtype=float32)>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s6 = tf.sparse.reorder(s5)\n",
    "\n",
    "tf.sparse.to_dense(s6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sticky-shower",
   "metadata": {},
   "source": [
    "### Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "athletic-sweet",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 6), dtype=int32, numpy=\n",
       "array([[ 2,  3,  4,  5,  6,  7],\n",
       "       [ 0,  7,  9, 10,  0,  0]], dtype=int32)>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set1 = tf.constant([[2, 3, 5, 7], [7, 9, 0, 0]])\n",
    "set2 = tf.constant([[4, 5, 6], [9, 10, 0]])\n",
    "\n",
    "tf.sparse.to_dense(tf.sets.union(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "clean-adventure",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=int32, numpy=\n",
       "array([[2, 3, 7],\n",
       "       [7, 0, 0]], dtype=int32)>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.difference(set1, set2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "northern-infrared",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=int32, numpy=\n",
       "array([[5, 0],\n",
       "       [0, 9]], dtype=int32)>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.sparse.to_dense(tf.sets.intersection(set1, set2))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "peaceful-gauge",
   "metadata": {},
   "source": [
    "### Variables\n",
    "All tensors shown above were immutable. To be able to mutate a tensor one can use a `Variable`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "bright-traveler",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "about-black",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.assign(2 * v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "invisible-opening",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[0, 1].assign(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "bronze-discretion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  0.],\n",
       "       [ 8., 10.,  1.]], dtype=float32)>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v[:, 2].assign([0., 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "affected-berkeley",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'ResourceVariable' object does not support item assignment\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    v[1] = [7., 8., 9.]\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "alone-apartment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[100.,  42.,   0.],\n",
       "       [  8.,  10., 200.]], dtype=float32)>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.scatter_nd_update(indices=[[0, 0], [1, 2]], updates=[100., 200.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "muslim-share",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[4., 5., 6.],\n",
       "       [1., 2., 3.]], dtype=float32)>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_delta = tf.IndexedSlices(\n",
    "    values=[[1., 2., 3.], [4., 5., 6.]],\n",
    "    indices=[1, 0],\n",
    ")\n",
    "\n",
    "v.scatter_update(sparse_delta)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "elder-nicholas",
   "metadata": {},
   "source": [
    "### Tensor Arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "above-harvest",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2,), dtype=float32, numpy=array([ 3., 10.], dtype=float32)>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = tf.TensorArray(dtype=tf.float32, size=3)\n",
    "array = array.write(0, tf.constant([1., 2.]))\n",
    "array = array.write(1, tf.constant([3., 10.]))\n",
    "array = array.write(2, tf.constant([5., 7.]))\n",
    "\n",
    "array.read(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "increased-breed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[1., 2.],\n",
       "       [0., 0.],\n",
       "       [5., 7.]], dtype=float32)>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.stack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "authorized-senate",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(2,), dtype=float32, numpy=array([2., 3.], dtype=float32)>,\n",
       " <tf.Tensor: shape=(2,), dtype=float32, numpy=array([4.6666665, 8.666667 ], dtype=float32)>)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean, variance = tf.nn.moments(array.stack(), axes=0)\n",
    "mean, variance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "enhanced-thomson",
   "metadata": {},
   "source": [
    "## Customizing Models and Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "selective-amino",
   "metadata": {},
   "source": [
    "### Custom Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "serial-contemporary",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Load and split the California Housing dataset\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(housing.data, housing.target.reshape(-1, 1), random_state=42)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train_full, random_state=42)\n",
    "\n",
    "# Scale the inputs\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "arctic-solution",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAD8CAYAAACiqQeGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABE30lEQVR4nO3deZxV8//A8de7mvYNlRYRUr5RU0pJ0UKkIlu+oeyyfhWyxFfE15afrcWSkESKRBJC0yIqIq1UlrRR2qealpn374/3LdM0zdxp7p1z75338/G4j+5y5pz36dx73/dzzufz/oiq4pxzzrnYUCToAJxzzjn3D0/MzjnnXAzxxOycc87FEE/MzjnnXAzxxOycc87FEE/MzjnnXAzxxOxcHBGR1iKiIlKpgLZ3lYikFsS2nHPGE7NzUSYiw0RkfDbPNwkl2VoBhOWci1GemJ1ziEjxoGNwzhlPzM7FiOxOU4tIrdBzTbIsfoqIzBGRNBGZLSKNs6zrVBGZIiLbRGSliLwoIuUzvT459Nz/ichaYHoe4rxBRJaKyM7Qv9dn8/riUGx/i8hnIlIs9Fp9EflSRDaLSKqI/CgibfLy/+RcovPE7Fx8+j/gHqAJ8CswXkRKgyU/YCIwDkgGLgQaAq9lWUc3QIDTgCvC2aiIXAAMAp4DTgSeB14QkXNDrzcBBgP9gLrAGcCnmVbxNrAaaBqK6SEgLbxddq5wKBZ0AM4VEu2z6USVnx/Gj6jqZwAicjWwArgMGArcBYxS1af3LCwiNwE/iEgVVV0Tevo3Vb0zj9vtDbypqoNCjxeHWuv3AB8BRwJbgXGqugVYBvyY6e+PAv5PVX8KPV6ax+07l/C8xexcwZiKtRAz3y7Lx/q+2XNHVVOBeUC90FONgW6hU8WpoR8Ee05VH5tpHbMPYrv/Yv/T3l9l2vbnWDL+TUTeEpErRaRcpmWfAYaKyCQRuV9Ejj+IGJxLaJ6YnSsY21R1aeYb1srNLCP0r2R6LukgtlUEazk3zHRLBo4D5mRabutBrPtAFCDUSj4JuAT4A+gD/CQi1UOvP4Ql8Q+AU4G5InJNBONwLu55YnYudqwN/Vst03MND7DsKXvuiEgZ7HrvotBT3wMnZP0hELptz2eMi4AWWZ5rCSzc80BVd6vqJFXtAzQAygCdMr2+RFUHqGpH4FXgunzG5FxC8WvMzsWOpcBy4CERuReoBfz3AMv+N9SbehXQF9iJdawCeBKYISIvAS8DW4DjgXNV9YZ8xvgU8K6IzMY6mLUHLsc6mCEinbDT5VOB9UAboBywSERKYZ3W3gV+Bw7HkvrMfMbkXELxxOxcjFDVXSLSFXgB6zA1B7gP2K84CXAv8DTW83kB0ElVt4bWM1dETgf+B0wBimI9t8dGIMYPROQ/WCew57DryTer6kehRTYC52M/FkoDvwDXqeq00FjpQ4Bh2FmBdaF9653fuJxLJKKqQcfgnHPOuRC/xuycc87FkLATs4gUFZEfDlDzt4SIjApVAZrptX+dc865g5OXFnNP/un1mdW1wAZVrQ08i3U+cc4551wehZWYReQIoCM2NjI7nYE3QvffA84QETnAss4555w7gHBbzM8Bd/NPAYSsamDDPFDV3cAm4LD8Buecc84VNrkOlwqNS1yjqrNFpHV+NiYiPYAeACVLlmx85JFH5md1MS0jI4MiRXL+3bNpUxIlS6ZTosSBfu/ErnD2L54l6v4tX74cVaWwf/biWTzuX3q6sHu3hPVdF4/7lxeLFy/+W1Ur57RMOOOYWwDniUgHoCRQXkRGqGq3TMusBGoCK0LTu1XAxijuQ1WHAEMA6tatqz///HN4exKHJk+eTOvWrXNdbudOEIGkgym8GKBw9y9eJer+tW7dmo0bNzJnzpygQ4maRD12e8Tb/q1YAampcHyYVdHjbf/ySkSW5bZMrj9LVLWPqh6hqrWArsCkLEkZbHq5K0P3Lw4t4wOkw3DllfDll0FH4Zxz0TF3Lnz8cdBRxJeDrvwlIg8D36nqOKze7ZsishQrw9c1QvElvNdfh5Ilg47COeeio0MHu7nw5Skxq+pkYHLoft9Mz6cBXSIZWGFRsiS88go0bQrJyUFH45xzkfP667BqFdx/f9CRxBevlR0DqleH0qWDjsI55yLr3/+G9euDjiL+eGKOAR07wsaNsHkzlC8fdDTOOZd/P/4IO3bY2UCXN4nbJz3OPPCAdwJzziWO1athWa79j112vMUcIwYMsGFTzjkX73btgvbtg44ifnmLOUaIwEsvwYcfBh2Jc87lz5NPwtNPBx1F/PIWcww59VQ45JCgo3DOufzp0we2bw86ivjlLeYY0qABqPp1Gedc/Bo/Hr7+GsqWDTqS+OUt5hgzbhxUqgRHHRV0JM45l3clS3rRpPzyxBxjbr016Aicc+7g/P03tGkDRYsGHUl881PZMeiVV+Cpp4KOwjnn8qZ/fxg+POgo4p+3mGPQuedCqVJBR+Gcc3nTvz9kxN8stjHHW8wxqGpVWLkSZswIOhLnnAvPs8/CV19BAk+lXGD8vzBGLV8Ov/0WdBTOOReeU06BWrWCjiIx+KnsGHX22fbvrl2QlBRsLM45l5NFi2y4Z5kyQUeSGLzFHMPefhvuuCPoKJxzLmdDh8KsWUFHkThybTGLSElgKlAitPx7qvpglmWuAp4CVoaeGqSqQyMbauFz/vlw8cVBR+Gccznz8puRFU6LeQfQVlWTgYZAexE5JZvlRqlqw9DNk3IElC4NixfDG28EHYlzzmWvWzeYMyfoKOJDuD3Wc03MalJDD5NCNz3oyEJWrCjFpk35XUviK13ah04552LXAw9AvXpBRxH7li6FRo3CW1ZUc8+xIlIUmA3UBgar6j1ZXr8KeBxYCywGblfV5dmspwfQwx41blyr1hQef3weVaumhRdtHElNTaVshIrFqsLffxencuWdEVlfJERy/2JRou5fr169SE9PZ+DAgUGHEjWJeuz2iKX9++qrSjRsuIGyZdMjts5Y2r9ImTevAv/974ls3pwEyGxVbZLjH6hq2DegIpACnJjl+cOAEqH7NwCTcltX8eKNFFSrVFGdMUMTTkpKSsTWNWuW6rnnRmx1ERHJ/YtFibp/rVq10uTk5KDDiKpEPXZ7xMr+ZWSo9uqlum5dZNcbK/sXKSNGqBYvrgqq55yjCnynueTHPPXKVtWNocTcPsvz61R1R+jhUKBxbus68shttGsHa9ZA69bw7rt5iaRwadLE52l2zsWWXbusqMihhwYdSWxShYcesmvwO3faPAjjxoX3t7kmZhGpLCIVQ/dLAe2An7IsUy3Tw/OARbluuIjy8cfQowekpcEll8Djj9vOuH2JwIoVcP31QUfinHOwezfUrw/r1wcdSWxKS4PLL4d+/awS2oABMHAgFAuzckg4i1UD3ghdZy4CjFbV8SLyMNYkHwfcJiLnAbuB9cBV4Ww8KQleegnq1oXeveG++6wX8ssvQ/Hi4e1AYVGtmv14UbVE7ZxzQSlWDL79FsqXDzqS2LN2LVxwAUyfbnNSv/MOdOyYt3WE0yt7rqo2UtUGqnqiqj4cer5vKCmjqn1U9QRVTVbVNqr6U85r/YeIFdF4/33rgTxsGJx1lv8Sy6pYMTvlP21a0JE45wqz9HS4/34oUSLoSGLPTz9ZadLp0+GII6x2eF6TMsRQ5a/zz7ekU706TJliO7dkSdBRxZa0NHjuOTuN5JxzQdixA2rU8LOaWX35peWtX3+Fxo2tElpy8sGtK2YSM8BJJ8HMmbYzS5bYTk6dGnRUsaNcOTuz4JOQO+eCsm4d3HyzX1LL7NVXoX172LTJGplTptjlx4MVU4kZ/mn+d+pkp7PPPNMn3s5sxw4bpJ6amvuyzjkXSatXW5lgn3PZZGTAPffAddfZmcy77oIxY/I/mUfMJWawC+YffAC9elmX/CuvtOoy/maw6zoffmj/R845V5CqVbN54n3OZdi2Dbp0gf797SzmkCF2PxL/NzH731u0qI2RGzzY7v/vf3DZZXadtbA78kj7f9kZO4XAnHMJ7rffrGXop7DtzEGrVnZpsUIF+PTTyA5njdnEvMfNN8P48XZ9ddQoaNvWipIUZiKwcSNea9w5V2CqVIFrrgk6iuDNnQvNmsF338HRR8M339gl10iK+cQMdlF9+nRrKX7zjf2nLFwYdFTB2jNcwU/vO+eibd06mDcPTj016EiCNWECtGgBy5dD8+Z2Wv9f/4r8duIiMYNVmZk5E5o2hd9/t/+Uzz8POqpgXXQR/PBD0FE45xLdL7/AJ58EHUWwBg+Gc8+1jrddu8KkSXYWIRriJjEDVK0KKSnWK3DzZjjnHLvgXlh9/LGNl3POuWjJyLAGUb9+QUcSjPR06NnTal1nZEDfvvD221CyZPS2GVeJGaw62KhR0KeP/YfdcIOV80yP3KxjcaN4cXj+eTvN75xz0TBkiCWjwmjLFujc2WpdJyXZ0N1+/aLfAS7MktqxpUgReOwxOO44mwTj6adtEuq33sr/+LF406SJXXt3zrlouO66wtnRdPlyO3X94482g9bYsXD66QWz7bhrMWd29dUwcSIccoiN7T39dFi1KuioClaLFjacbNmyoCNxziWajz+2ySoOOyzoSArW7NnWyfjHH6FOHevfVFBJGeI8MQO0aWM9tY89Fr7/3q6FzJkTdFQFa+xYL13qnIu8okULXwngDz6wJLx6tU0c9M03ULt2wcYQ94kZbNrIGTPgtNNg5Upo2dLGPhcWt9wC3bv7XNbOuchZvdpm+mvaNOhICoYq/N//wYUXWlWvq66Czz6z09gFLSESM0ClSjZ8qls32LrVLtg//3zhSVZjx8KddwYdhXMuUdx/vyWmwmDXLrjxRqt1rWp9mF57LbgZtHLt/CUiJYGpQInQ8u+p6oNZlikBDAcaA+uAf6vq7xGPNhclSlivuTp1rBdhr16weLEl6GJx2c0tfG3aWIk455yLhFdfDTqCgrFxo9W8/uILGwI1fLg9DlI4LeYdQFtVTQYaAu1F5JQsy1wLbFDV2sCzwJMRjTIPRGzCi7fftkT9wgvWs27z5qAiKhgVK9rA93ffDToS51y869HDiookel3s336zamZffGHFQiZPDj4pQxiJWc2eSQaTQresJ4g7A2+E7r8HnCES7CG99FKrzFK5shUYb9GicPRcXrky6Aicc/Hu8svhqKOCjiK69pR3XrQITjjBel43axZ0VEY0jIuwIlIUmA3UBgar6j1ZXp8PtFfVFaHHvwDNVPXvLMv1AHoAVK5cufHo0aMjshM5WbWqJPfdV59ly8pwyCE7+d//5lGv3paobzc1NZWyAc3NmJpajLJld0d5G8HtX0FI1P3r1asX6enpDBw4MOhQoiZRj90e0d6/WbMOpVGjDSQlBdNBpyCO36RJVXjiiePZtasITZqs58EHF1C2bMFUqWrTps1sVW2S40KqGvYNqAikACdmeX4+cESmx78AlXJaV506dbSgbNigeuaZqqBasqTqu+9Gf5spKSnR30g2li1TbdhQNSMjutsJav8KSqLuX6tWrTQ5OTnoMKIqUY/dHtHcv507VS+9VHXr1qhtIlfR3L+MDNVHHrFcAKo33qi6a1fUNpct4DvNJdfmqVe2qm4MJeb2WV5aCdQEEJFiQAWsE1hMqFjRZgW5/nqbz7lLF3j88cTssX3kkTBrVuJfG3LORZ6I9c8pXTroSCJvxw648krrgyQCzz5rfZBisWNwrolZRCqLSMXQ/VJAO+CnLIuNA64M3b8YmBT6ZRAzkpLg5ZfhqafsoNx3H1x7LezcGXRkkZeRYcPG0tKCjsQ5Fy9Wr7ZJcRJxKtl166BdO3jzTfvR8cEHNmonVhsw4bSYqwEpIjIX+Bb4XFXHi8jDInJeaJlXgcNEZClwB3BvdMLNHxGb8GLMGChVCl5/Hc4+G9avDzqyyCpRAi67zGqKO+dcOKpVsw6zifa9sXgxnHIKTJsG1avbv+edl/vfBSmcXtlzVbWRqjZQ1RNV9eHQ831VdVzofpqqdlHV2qraVFV/jXbg+XHBBXZwqlWz7vHNm9skGImkQwfrdbhjR9CROOdi3S+/WL2HRKuJPWWKJeWlS6FRI7vMd9JJQUeVuwT7bRS+xo2te3xysv2iatbMknUiefdd+P33oKNwzsW6pCSoWTPoKCJr2DA7fb1hg9WymDoVatQIOqrwFNrEDPZGnDYNOna009lnngkjRgQdVeQMGmRV0ArjXNXOufD89ZdNl3vhhUFHEhkZGVZO9OqrrdTm7bdbyeJ4GkFXqBMzQLlyNmVkz57WEax7dyvnGVtd1w7e7bfDO+8EHYVzLlZNnAgvvhh0FJGxfbsVl3rsMZsV64UX4Jln4m+GrBjsKF7wihaF556D446D226DRx6xaxKvvWa1U+PZAw/YfNXOOZed7t2DjiAy/vrLJi+aOdMaXO++a51741GhbzFndsstNl1kuXIwciS0bQtr1wYdVf4cdpidrv/gg6Ajcc7Fmt694eOPg44i/xYssH5CM2daLYevv47fpAyemPdzzjkwfbod3D21VBcuDDqq/ClbFipUCDoK51ysueMOm8Qhnk2caPuwbJnNHT1zJpx4YtBR5Y8n5mzUr28H9+ST9519JF41bgynnWZDIpxzDmDUKLuMF8+Xul56yYaGbt5sFR0nT4aqVYOOKv88MR9A1ap2kC+6CDZtgvbt4ZVXgo7q4M2cCQ8/HHQUzrlY8fPPsVv5Kjfp6dbav+kmu9+nj3VyLVUq6MgiwxNzDkqXhtGj4d577eD36AF33RWfJetatIA33sh9Oedc4tuwwUafVKkSdCR5l5pqQ7uefdbGX7/2mvXCTqSKZQm0K9FRpIhNePHqq1bs/P/+z1rRW7cGHVne/fWXDbj3cc3OFV47dlg1rNTUoCPJu5Ur4fTTYdw4OwU/caKNV040npjDdM019iaoWNF6OLdqBatWBR1V3lSpAk8/HX9j+pxzkVOiBMyfH18FNwB++ME6d/3wAxx7rHXObd066KiiwxNzHrRpAzNm2Jti9mzrsf3jj0FHFT4R69j24oteQ9u5wujXX23626SkoCPJm48+sg6sq1bZvzNmQN26QUcVPZ6Y86huXXtTtGwJK1bYtdt4GgcoYteXNm4MOhLnXEGrWtXmJI4Xqlb8qXNnu3zYrRt8/jlUqhR0ZNHlifkgVKpkw6e6dbM3y3nnwYAB8VPG8777rPfitm1BR+KcKyhLl8KiRdaoiAe7d1vRp9tvt+/Whx+G4cPtVHyi88R8kEqUsDdJv37WS7tnT/jPf+zNFA/uvBO+/DLoKJxzBeXXX+H774OOIjybN0OnTnbZrUQJePttKy8cr8O78irXWtkiUhMYDhwOKDBEVZ/Pskxr4EPgt9BT7++ZtzmRidiQg9q1rWfg4MFWxGPUqKAjy93LLyfW8ALn3IFt3QpnnRV0FOH5888StGhhHdQqVbJJhuK9OllehfPVvBu4U1XrAacAt4hIvWyWm6aqDUO3hE/KmV12GUyaZG+iTz+1685//hnb51uKFLHe5U88EXQkzrlo69kT3n8/6ChyN3Mm3HxzY+bPh+OPt8eFLSlDGC1mVV0NrA7d3yIii4AaQJxXkI6sFi3sTdSxo/3Su/nmxhxzjHXvj1VNm0LDhkFH4ZyLtniY1vG992ymq7S04pxxhj2uWDHoqIIhmoceSyJSC5gKnKiqmzM93xoYA6wAVgG9VXVBNn/fA+gBULly5cajR4/OR+ixKTW1GA8+eALff38IxYunc999P9GqVexOUbVpUzHmzatIy5Z/5+nvUlNTKRtvAyHzIFH3r1evXqSnpzNw4MCgQ4maRD12e+Rl/1Rh4MDaXHbZH1SqtDPKkR0cVXj77SMZOvQYAM466w/uuus3ihWLk960edSmTZvZqtokx4VUNawbUBaYDVyYzWvlgbKh+x2AJbmtr06dOpqodu5U7dhxpdpbTvWJJ1QzMoKOKnurVqnee2/e/y4lJSXiscSSRN2/Vq1aaXJyctBhRFWiHrs98rJ/GRmqY8eq7toVtXDyZccO1auvtu9JEdX+/VUnTUoJOKroAr7TXPJjWN1/RCQJaxG/par7XalQ1c2qmhq6PwFIEpEEH2l2YElJcOedi+nf3zqI3XsvXHcd7IzBH6zVqlnJ0U2bgo7EORdJqjbmt3NnKycca9avtzmTX3/dhm+OGWNzERSWntc5yTUxi4gArwKLVPWZAyxTNbQcItI0tN51kQw03ojYm2zMGHvTvfaazVC1YUPQke1vxw673rxlS9CROOciZe1aGDEiNusrLF0KzZv/M03j1KlwwQVBRxU7wmkxtwC6A21FZE7o1kFEbhSRG0PLXAzMF5EfgQFA11CTvdC74AJ701WtCikp9mZcujToqPZVogTMmwflygUdiXMuEtLT4dBDrdZCrA2LnDbNyhkvXgwNGsCsWdAk5yuuhU6uh0xVv1JVUdUG+s9wqAmq+pKqvhRaZpCqnqCqyap6iqp+Hf3Q40eTJvbma9DA5kA95RT46qugo9pX8eLWwp89O+hInHP59fHHVhM71owYAWeeaaexO3Sw78GaNYOOKvbE2G+pxFWzpr0JO3SAdevgjDPsTRpLunSB444LOgrnXH6dey48/3zuyxUUVXjwQRsOtXOnVUn88EM/S3cgnpgLULly9ma87TZ7c3bvbm/WWDnp37SpzXc6Y0bQkTjnDtZLL9m12/Llg47EpKXB5ZdbresiRWxegQEDYrNDWqzwxFzAihWzX7IDB9qb9OGH7U2blhZ0ZGbZMqup65yLT40aQa1aQUdh1q61s4MjR9r8zx99ZK1llzP/zRKQW2+FY46Bf//b3rTLllmJzMqVg42rfXv7d906OOywYGNxzuXNpElWwrJkyaAjsZmsOnaE336zS3njx1s/G5c7bzEHqEMHmD7d3rRff209FRctCjoq6zXeqVPsnGJ3zuVOFYYNi41hj19+aSNQfvsNGje2csWelMPniTlgDRrYm/bkk+1N3Lx58NMx1q5tQxp8oL9z8WPHDhseFfRZt6FD7czbpk02XHTKFCtk5MLniTkGVKtmnTUuusjezO3bwyuvBBtTkSL2ofrzz2DjcM7l7vffbSKdIM9yZWTAPffYMK3du+Huu20iijJlgospXnlijhGlS8Po0fbG3r0bevSwN3ZGRjDxFClipUQrFdrCqs7Fj1q17Md9UGe5tm2z4Zb9+1sH1yFD4MknY6+4Sbzw/7YYUqSIzY88dKi9uZ96Ci6+2CY5D0KzZnYa6uefg9m+cy53Y8fCq68GNyZ49Wpo1crme65Qweakj8XiJvHEE3MMuvZa+Owzm4t07Fh7069aFUwsq1ZZD23nXGw66aTgSlrOnWs/4L/7Do4+Gr75xoZHufzxxByj2ra1N/kxx1iZzGbN4McfCz6O7t2thKhfa3Yu9kycaNdwk5MLftsTJth17eXLbYjWzJnwr38VfByJyBNzDDv+eHuzt2gBK1ZAy5ZWA7egffYZ3H9/wW/XOZezadNg48aC3+6gQVb2MzUVLr3URpIE3Rs8kXhijnGVKsEXX8Bll9mH4LzzrGpYQYqFXuLOuX2tXw+PPGLDGwtKerqVFP7Pf6xjat++8NZbsVHQJJF4Yo4DJUvahBcPPWQfhj0fjN27C2b7IlYytGXL4DqiOef+sX49nHYa7NpVcNvcsgU6d7aGQfHi8Oab0K+f1zuIhlwTs4jUFJEUEVkoIgtEpGc2y4iIDBCRpSIyV0ROik64hZeITXgxYoR9KAYNstbz5s0Fs/3Spa3V7GMSnQveoYfCnDmQlFQw21u+/J9LaYcdZmfxunUrmG0XRuG0mHcDd6pqPeAU4BYRqZdlmXOA40K3HsCLEY3S7XX55VYPt1Il+OQT+7D88UfBbPtf/4I33oCffiqY7Tnn9vfVV5V4+OGCS8rffWczz82dC3Xq2Oxzp51WMNsurHJNzKq6WlW/D93fAiwCamRZrDMwXM0MoKKIeBG2KGnRwj4cxx8P8+bZh+bbbwtm22XLFsx2nHPZO+mkDfz73wWzrbFj4fTTbVRG69Y2UqQgr2kXVnm6xiwitYBGwMwsL9UAlmd6vIL9k7eLoGOPtYkv2raFv/76Z4B/tF10ERx5JPzxR+nob8w5t4+33oKNG5OoWze621G1AkcXXQTbt8PVV9vojEMPje52nQl72kcRKQuMAXqp6kFd2RSRHtipbipXrszkyZMPZjVxITU1tUD2r08foWTJOkyYUI2LLoIePX6ha9flUe2QMWvWocyadShHHjk5ehsJWEEdv4K2ceNG0tPTE3Lf9kjUYwcwe3Z16teP7v7t3i0899xxfPxxdQCuv/5XLr30D77+Omqb3EciH7+wqWquNyAJ+Ay44wCvvwxcmunxz0C1nNZZp04dTWQpKSkFtq2MDNX+/VXtd67qtdeq7tgR3W2mpKREfRtBKsjjV5BatWqlycnJQYcRVYl67ObNs3+juX8bNqiecYZ9j5QsqTp6dNQ2dUCJevz2AL7TXHJuOL2yBXgVWKSqzxxgsXHAFaHe2acAm1R1dX5/NLjwiMBdd8GYMVCqlNXNPecc2LAhetvcsaMIycmxMferc4nu779tmGQ0h0j++us/084efrhNitGlS/S25w4snGvMLYDuQFsRmRO6dRCRG0XkxtAyE4BfgaXAK8DN0QnX5eTCC23SiapVred28+bwyy/R2VaJEhl8/XVwhfOdKyzS022I0qRJNrlNNHz9tZXe/eknOOEEqzjYrFl0tuVyl+thVtWvgByvWIaa57dEKih38E4+2T5UnTpZj+1mzeCDD2xYVaQdcgg8+6x9kM86K/Lrd87BsGH2A/uxx6Kz/pEjrXPXjh1w9tkwapTNEuWC45W/EtCRR8JXX9np7HXrbLaXt96KzrZOPx0aNIjOup1zcNVV0Lt35NeraiU9L7vMkvJNN8H48Z6UY4En5gRVvjyMGwe33go7d1qVnocesg9jJDVubNe4x42L7Hqdc3D77fD775EfprRjB1x5pdW6FrEzX4MHR+9UucsbT8wJrFgxq2s7YAAUKWJ1bbt1s7rXkbR9u1UFcs5F1llnwRFHRHadf/8N7dpZresyZeDDD6FXL695HUs8MRcC//mPtWjLloW334Yzz4S1ayO3/lq14L//tV6dkW6RO1cYbd1qn9VzzoESJSK33p9/tk5e06ZB9er277nnRm79LjI8MRcSHTvadecjjoDp0//pgRkpqnDttbBsWeTW6VxhtXZt5EdUTJ78z0iNRo1g1iz718UeT8yFSHKyfRgbN7bW7Smn2JjFSBCx4Ry1atnwDufcwVm1yoY8PvBA5Nb5+ut2WnzDBpuVbupUqOFFk2OWJ+ZCplo1G+t8wQWwaRO0bw9Dh0Zm3SK2rgcfjMz6nCuMhgyBd96JzLoyMuC+++Caa2zu5jvusJr6PhlNbPM+eIVQmTLw3nvQpw/07w/XXw9LlsDjj1snsfy45BLv2encwUpPj9zoie3bref1u+9C0aI2h/uNN+b+dy543mIupIoUgSefhFdesUTav7+V39u2LX/rLV/efpnv+YXunAvP1q3QsKF9BvPbQ/qvv6BNG0vK5cvDhAmelOOJJ+ZC7rrr4NNPrajA++/b9JGr81nlvHx5O1VetGhkYnSuMChTxvp8lM7njKrz51vFv5kz4aijrLOnV+aLL56YHWecYROgH3MMfPedfah//PHg1ydiQzC++MJOkTvncvb++3ZtuUqV/K3ns8+gRQsbHbEnOZ94YmRidAXHE7MD4F//ghkz4NRTYflyq609YUL+1vnnn1YS1DmXs6ZNbZREfrz4og2L3LzZLkulpNgsUS7+eGJ2e1WubKfSLrsMUlOt1Tto0MGv74or7Atn3rzIxehconntNTuNfbA159PTrbf1zTfb/fvus17dpUpFNk5XcDwxu32ULAkjRtiQp4wMqxqWn3lgf//dxmN6RTDn9qdqp50Ptj9GaqpN9/rss5CUZOOVH300/6MrXLD88Ln9iNiQjREjoHhxq7fduTNs2ZL3dR1zjE07uX27J2fnMtuyxUpk9utnHSbzauVKm91t3DibgnXiRJuJysW/XBOziLwmImtEZP4BXm8tIptEZE7o1jfyYbogXH65ndo+7DC73tyyJfzxx8Gt64orrCSgc87Mnm3DFQ/GDz/YZaIffoData1/SOvWEQ3PBSicUhDDgEHA8ByWmaaqnSISkYspLVtaz86OHW0GqWbN4KOP8r6eN96w62jOOdi40RLpwSTT6dMP47HHbLzzaadZj+5KlSIcoAtUri1mVZ0KrC+AWFyMOvZYG07Vpo31tD79dJg6NW/fBGXK2K/77t2jFKRzcaRzZ1i4MG9/o2rXkh944ES2bbPP0uefe1JORKJhXPgTkVrAeFXdb0SciLQGxgArgFVAb1VdcID19AB6AFSuXLnx6NGjDzbumJeamkrZBCtIu2uX8Oyzdfjkk2oA9OjxC127Lg+7StHu3cLKlaU46qh8lhcrAIl4/AB69epFeno6AwcODDqUqIn1Y6dqn6XixcPvdJGeLgwYUJtx42zmiWuu+Y1u3ZYl5BzKsX788qtNmzazVbVJjgupaq43oBYw/wCvlQfKhu53AJaEs846depoIktJSQk6hKjIyFB94glV+3pRve461Z0787aOfv1UlyyJTnyRkqjHr1WrVpqcnBx0GFEVy8du6lTVK67I299s3Kh61ln2eStRQvWBBxZEJ7gYEcvHLxKA7zSX/Jjv6QZUdXOm+xNE5AURqaSqf+d33S72iMA990Ba2nyeeOJEhg61KSTfe896hoajcWMrAepcYXPqqVCzZvjL//47dOoECxZYnYEPP4QdO9YA9aIVoosB+R4uJSJVReyEiog0Da3T6z0luFat/mbKFKssNGmSfeGEO7F7x442Rvr996Mbo3Ox5I477DNSq1Z4y8+caZ0tFyywynwzZ0Lz5lEN0cWIcIZLjQS+AeqKyAoRuVZEbhSRPXOVXAzMF5EfgQFA11Bz3SW4pk1h1iyoXx9++slKCk6fHt7fpqXBokXRjc+5WHL22XDkkeEt++671mN7zRo480z4+ms4+uiohudiSK6nslX10lxeH4QNp3KF0JFHwldfQdeu8Mkn0LatVR+67LKc/+6oo+D++22yjLp1reKYc4not99g2jQby58bVXjiCSurCTZX+uDBVtXLFR4xO6X95s2bWbNmDbvidFLfChUqsCiBm4RZ9+/555M47bQq3HdfeS6/3GaV6ts393llX3rJ5m4++eQoB+xcQHbuDK/k5s6dcMMNMGyYfW6eespOfydiz2uXs5hMzJs3b+avv/6iRo0alCpVConDd+aWLVsoV65c0GFETeb9U1W2b9/OxRevpEYNuPrq8jz0kCXnoUNzbg2/+KL9++efULVq9ON2riB98YUV6albN+fl1q+3mtdTpth8zG+9BeefXyAhuhgUk7Wy16xZQ40aNShdunRcJuXCRkQoXbo0NWrU4NRT1zBuHJQta18uZ54Ja9fm/Pc//2ynwr1ngkskqjB6NPydy/iUJUusU9eUKVCtGkyd6km5sIvJxLxr1y5K+ZxlcadUqVLs2rWLjh3tuvMRR1hnsFNOsc5hB1K3rtXk3r3bems7F+82bbKzQEOG2OfgQKZNs8/H4sWQnGw9rxs3Lrg4XWyKycQMeEs5DmU+ZsnJ1mO7cWMb59y8uQ2rOpCiRaFXLxgzJvpxOhdtX3xhs7Ll5M034Ywz7DR2x46WpPMyxtklrphNzC7+Vatmp+fOP9+K9p99Nrz66oGXf/RRuPhiP6Xt4tv69XDRRfZ+zo6qdYy84grYtcvmO//wQ0jgLikujzwxu6gqU8ZawXfdZaeqr7sO7r03+1PWFSvChg02fnPnzoKO1Ln8277dJnnZvDn73tRpaTad6iOPQJEi1qp+/vnwem27wsMTs4u6IkWgf3+73lasGDz5JFxyiU1bl9Whh8LLL0Px4gUfp3P5sXu3jUD4/nsoX37/19eutXH+I0da58jx4+HWWws+Thf7PDFHWOvWrbk1n5+2SKwjNxs2bODwww/nlzDqaHbp0oWnn34639u8/norQlKhgrWiW7eG1av3X+7446329uDB+d6kcwXm4YdteGB2PyoXLbLymt98Y9eRp0+Hc84p+BhdfPDEXEg99thjdOjQgWOPPTbXZfv27cujjz7Kpk2b8r3dM8+0L6ejj4Zvv7Uvq3nz9l/u5JPtmrRz8UDVJne5NJs6iV98YZ0ff/vN3tczZ0KDBgUfo4sfnpgLkZ2hC7fbtm1j6NChXHvttWH9Xf369TnmmGMYMWJEROL4179gxgz7slq+HFq0sJZ0ZkcdBccea192uY0DdS5IS5ZAhw5WGCTrNMKvvALt29vwqYsugsmTrVOkcznxxBwFGRkZ9OvXj0qVKlGlShV69+5NRqi3U3anqa+66io6deq0z3O7d++mZ8+eHHLIIRxyyCHcdddde9cBVm2rf//+HHvssZQqVYr69evvlzhbt27NTTfdRO/evalcuTItWrQAYMKECYjI3scA/fv3R0T2u/Xt2xeA8847j5EjR0bs/6hKFRs+1bUrbNliU9tlPXUtAo0aQYkSEduscxFXuzYMGLBvZ6+MDLj7bujRA9LT7Qfm6NGWvJ3LTdwkZpFgbgfjrbfeomjRonz99dcMGjSI5557jlGjRuV5HRkZGXzzzTe8/PLLDBkyhOeee27v6//973959dVXGTx4MAsXLqRPnz7ccMMNfPzxx/usZ8SIEagq06ZNY/jw4QBMmzaNxo0b7zPu+KabbmL16tV7b3feeSdVq1blilDl/aZNmzJr1iy2b99+cP8p2ShZEt5+24aOZGRYR5iePe2LbI+uXa218fLLEduscxFz1VVWue644/55butWG/b31FPW2XHoUJuYokjcfNu6oMVkrex4V69ePf773/9Srlw56tSpwyuvvMKXX37JpdldgDqAatWqMWDAAESE448/nsWLF/PMM89wxx13sHXrVp555hkmTpzIaaedBsDRRx/NrFmzGDx4MB07dty7nqOPPnq/jlvLli2jevXq+zxXrly5vbWvn3zySUaOHMnkyZOpXbs2ANWrV2fXrl2sWrUqrOvS4RKBfv3si+3aa63l8csv1nN1z7jOEiWsx6tzsebmm63FvMeqVXDeeTB7tg3/GzPGemI7lxdx8xtONZjbwWiQpWdH9erVWbNmTZ7Wccopp+zTom3evDkrV65k8+bNLFy4kLS0NNq3b0/ZsmX33l588cX9elk3zqa+3/bt2yl5gJklHn/8cQYOHEhKSgp1M1Xe31MiNZIt5sy6dbNOMocdBh9/bIX/ly+31ypXhltusetzCxZEZfPO5cn48dYSbtrUWsVgU5g2a2ZJ+ZhjrJOjJ2V3MHJtMYvIa0AnYI2qnpjN6wI8D3QAtgFXqer3kQ40niRlmTxVRPZeHy5SpAiaJePndWrLPev66KOPODLLzOtZt12mTJn9/r5SpUps2LBhv+f/97//8dJLL+3TUt5j/fr1AFSuXDlPsebFaadZp7COHWHuXPvS++gjaNLEXv/zT58Cz8WG+vUh80mnjz+2yy6pqdaZcexY+0Hp3MEIp8U8DGifw+vnAMeFbj2AF/MfVuKqXLkyq7MM3v3xxx/3W27mzJn7JPAZM2ZQvXp1ypcvT7169ShRogTLli2jdu3a+9yOOuqoXGNo1KgRCxcu3Oe5hx9+mCFDhjBlypT9kjLA/PnzqVGjBocffni4u3pQate2lkabNpaITz/dvuTAvvhOP91OD2a+Du1cQVm3zuZIrlkTTjrJnhs40E5fp6bCZZfZmR9Pyi4/ck3MqjoVWJ/DIp2B4WpmABVFxAcEHEDbtm355JNPGDduHD///DN33HEHy/ecs81k1apV9OrVi59//pn33nuPp556ittvvx2w68G9e/emd+/evPbaayxdupQ5c+bw0ksvMWTIkFxjOPvss1m0aBHr1q0DrKU8YMAA3nnnHcqUKcOff/7Jn3/+SVpa2t6/mTZtGmcX0MDiQw+FTz+Fq6+2EocXXWQdaVQtIX/5pdXedq6glS5tp6uLFLF+D//5j9W6zsiAhx6CESNynn/cuXBEovNXDSBzZlkRem6/mk4i0gNrVVO5cmUmT56c7QorVKjAli1bIhBawUtPT2fnzp2kp6fv3Yddu3axe/dutmzZQpcuXfjuu++4+uqrAbj++uvp1KkT69at27t8eno6l1xyCdu3b6dZs2aICN27d+e6667bu8zdd99NhQoV6N+/PzfddBPlypWjQYMG9OzZc5/17Ny5c7//y1q1atG4cWOGDRvG9ddfz1NPPcXmzZv3GT4FMG7cOFq3bk1aWhpjx47l/fff32fd2R2jtLS0Ax7XvOreHYoVO5JXXjmGu++GKVNW0avXEi65RPn22yIsWFCexo03RmRbWaWmpkZsP2LJxo0bSU9PT8h92yNax27kyJqcccYaDj98BxMmFOXhh+sxc+ZhJCVlcNddP9Gq1RqmTIn4ZveTqO/NPRJ9/8KiqrnegFrA/AO8Nh5omenxl0CT3NZZp04dPZCFCxce8LV4sXnz5qBDyNEnn3yiderU0d27d+e67KBBg7Rdu3b7PHeg/YvGsXv3XdWSJa073hlnqG7YoLpkiep//hPxTe2VkpISvZUHqFWrVpqcnBx0GFEVjWOXkaH6+uuqmzapLlumWr++vR8PO0x12rSIby5Hifre3CPR9w/4TnPJj5Holb0SyDyL6BGh51wMa9++PbfccgsrVqzIddmkpCQG5ja5bBRdfLFNH3n44XYau3lzO5U4YICVOVyyJLDQXCHw2Wf2vrvqKli8+J8ysnXrWnnNli2DjtAlmkicyh4H3Coi7wDNgE2qms3UBC7W3HbbbWEt16NHjyhHkrumTe1LsFMnmD/fvhw//NCScpEi+xZ4cC6SSpe2aRnff9+G9W3fbp0Tx4yBQw4JOjqXiHJtMYvISOAboK6IrBCRa0XkRhG5MbTIBOBXYCnwCnBz1KJ1hdpRR9msPO3bW/3stm1tJp/u3eGrr2yuW+ciZflymyu5ZUt7f110kSXla66xzomelF205NpiVtUcy1WFzpnfErGInMtB+fI2trlnT3jhBRuesmQJ/PWX9eauVy/oCF2iKFbMpijt0cOKiYCV1rz7bh9P76Irbip/ObdHsWIwaBA895x9QT74IGzebNWWJkw4+IptzgHs2mVzK6va8KehQ20I1Hvv2WQUnpRdtHmtbBeXRKzVfMwxNgfuiBFWY7tmTWjVCrIpeOZc2NLT7VLJzz9bp8Nx46yfg3MFwROzi2vnnmvX/zp1sopha9bYl+natVBA9VBcAnnkEbsc8sIL1o+hfn27dBJGQT3nIsZPZbu417AhzJplJRJ/+QVat4ZhwwIOysWlv/+2fgt//22dDL/6ypOyK3iemF1CqF4dpk6Fzp1hyxa7HvjYY5CSEnRkLh688AJccYWNjd+506Zz/Ogj62zoXEHzxOwSRpkyNra0d2+rY3z//fDMM1bH2LkD2bHDkvCbb9qY+Oeft86FxfxCnwuIJ2aXUIoWtQkvXn7Z7o8fb9PwTZsWdGQuFo0YAccfb+OSy5SxojW33eY9r12wPDFH2AUXXMAhhxxC9+7dgw6lUOvRAz75xMahzpgBN91k00g6t8dPP8EDD8Dvv0ONGv90InQuaJ6YI6xnz54MHz48z3+3fPlyWrduTb169WjQoAHvvvtuFKIrXNq1g6+/hlq1YMECaxl98EHQUblYMGgQNGhgSfmkk6zzYMOGQUflnPHEHGGtW7emXLlyef67YsWK8dxzz7Fw4UImTpxIr1692Lp1axQiLFzq1bMa282bw6ZNVr7z00+DjsoFacgQ6NXLCol07mydBqtXDzoq5/7hiTlGVKtWjYahn+xVq1alUqVKrF+/PtigEkSVKjBpEnTtCqmpcM450K9f0FG5gpaRYb2tb7jBCoj07m2dBb0YjYs1nphj0OzZs0lPT6dmzZq5L+zCUrIkvPWWXVMEeOgh6+STnh5oWK6AbN9uxWhefNF6Xr/8snUSLFo06Mic258n5hizfv16rrjiCoYMGRJ0KAmnSBGrgTx8uA2FGTjQWs9btgQdmYumP/+0nvkTJkC5cnYpIwZmMnXugDwxF6D+/fsjIvvd+vbtC8COHTs4//zzuffeezn11FMDjjZxde9uE99XrAiff25zOy9fHnRULhrmzbNOXj/8YBW8ZsywToHOxTJPzBF25pln0qVLFyZOnMgRRxzBN998s/e1m266idWrV++93XnnnVStWpUrrrgCVeWqq66ibdu2PtSqAJx+Onz7LdSuDYsWwcknw+zZQUflIumzz+DUU61u+sknW89rnxbUxYOwErOItBeRn0VkqYjcm83rV4nIWhGZE7pdF/lQ48MXX3zB2rVr+euvv1ixYgXNmzff+1q5cuWoWrUqVatW5Y033mDkyJFMnjyZ2rVrM336dEaNGsUHH3xAw4YNadiwIfPmzQtwTxJf7drWY7tVK5vP+dRTYezYoKNykTBggF2mSE2Ff/8bpkyxToDOxYNci86JSFFgMNAOWAF8KyLjVHVhlkVHqeqtUYgx4Tz++OMMHjyYlJQU6tSpA0DLli3J8NqRBe7QQ2HiRLjmGuscduGF1imoceOgI3MHIz0dBg06ljFj7PG998Kjj1r/AufiRThv16bAUlX9VVV3Au8AnaMbVvYeeshuAHXqwOLFdvpxz5fonXfC00/b/erVYdUqmDzZZhsC6/Cxp09VuXLW6eejj6y3JtisMm+/bfcPpiRf5uvG5cuX3+9aMsD//vc/Bg8ezOTJk/cmZRes4sWtTvJjj9nju+6C/v3rsGtXsHG5vElNhfPOgzFjalKsGLzxBjz+uCdlF39EVXNeQORioL2qXhd63B1olrl1LCJXAY8Da4HFwO2qul93GhHpAfQAqFy5cuPRo0dnu80KFSpQu3btg9mfQK1YsYIePXqwdu1aihYtyj333MMFF1yw9/UnnniC4cOHM378eI455pgAI82/9PR0imYz1mTp0qVs2rQpgIgiY9Kkygx/tCzpGUWoclIZ+vVbSNmyu4MOK2J69epFeno6AwcODDqUiFq1qiR9+tQn44/1lCq5i1ue2ERycvy+D3OSmppK2bJlgw4jahJ9/9q0aTNbVZvkuJCq5ngDLgaGZnrcHRiUZZnDgBKh+zcAk3Jbb506dfRAFi5ceMDXYtmqVav0hx9+UFXVJUuWaPXq1TU1NVVVVR955BE97LDDdPr06bp69eq9t+3btwcY8cHbvHlzts/H67HLbMYM1YoVdyio1qmj+ssvQUcUOa1atdLk5OSgw4ioiRNVK1ZUBdXjjlN9880ZQYcUVSkpKUGHEFWJvn/Ad5pLfgznJM9KIHOliyNCz2VO7utUdUfo4VCgUF6hy1y96/DDD99bvUtVeeqpp1i3bh0tWrSgWrVqe2/Tp08PNmi3n2a/j2L8Ff2pW9culyQn2yxVLraowpNPwtlnw8aN0KEDzOkzikaLPw46NOfyJZzE/C1wnIgcLSLFga7AuMwLiEi1TA/PAxZFLsT49MMPP+yt3iUibNq0KdtfRmeccUbQobqsXnyRE6a8x8yZNttQaqr1Q7j/fq8UFitSU6FbN+vcpQp9+lh/kdJvvEiNceNyX4FzMSzXxKyqu4Fbgc+whDtaVReIyMMicl5osdtEZIGI/AjcBlwVrYDjwfr167nhhhu8elecq1DB5ud94gnrDPjYYzb+ec2aoCMr3ObMsaIhb78NpUtbvevHHvNOXi5xhPVWVtUJqlpHVY9V1UdDz/VV1XGh+31U9QRVTVbVNqr6UzSDjmV7qnfdfvvtXr0rARQpAvfcY5XCype3aSSTk+GLL4KOrPBRtfHJTZvCb7/ZOPRvv7Uhbs4lEv+NGUGaqXrXpZdeGnQ4LoLatLEKYS1bWu3ldu3gjjsgLS3oyAqH9evhoougZ0+brvGGG2DuXK/k5RKTJ+YIyly9q0WLFl69K8FUrw4pKTZDVZEi8Oyzlhjmzg06ssT20Udw3HFWla18eRg9Gl56CUqVCjoy56Ij18pfLnyZq3dt2bKFcuXKBRyROyjvvceC6dNpkc1LxYrZDFWdOsEFF9gp1SZNLFnfey8kJRV4tAlrwwa4/nr2VvFKTrbkfPTROfxRDsfOuXjhLWbnsqpUiV0VKuS4SNOmNpTqhhvs1GrfvpagZ80qoBgT3EcfwQknWFJOSoJnnrEqfzkmZQjr2DkX6zwxO5fVsGFU/fTTXBcrU8ZOqU6caFMKzp1rU0j27OlzPB+s33+H88+30pqrV0Pz5jB/Ptx+O2RTaG5/YR4752KZJ2bnssrjl3u7drBwoSVkEes5fNxxMGIE+Lwk4UlLgwcftBr4H35o14/794dp0+y5sHlidgnAE7NzEVC6NDz3HHz/vU2q8tdf0L27tfi+/jro6GKXql03rlfPrt3v2mXTNC5ZYpOJhNVKdi7BeGJ2LoIaNrQ5nl97DapWtWvOLVpA167wyy9BRxdbJk+2Hy4XXmid6OrVg0mT4J13oEaNoKNzLjgxm5g1l1mvXOzxY2aKFoWrr7ZW3913Q4kSMGqUnd6++mpYujToCIM1eza0b29jw2fOhMqV7WzDnDn2nHOFXUwm5qSkJLZv3x50GC6Ptm/fTpKPF9qrbFmbZGHJEmsxi8CwYXbNtGtX69VdWKha9bQ2baz3+mefWee5Bx+EX3+16/P+1nHOxGRirlKlCitXrmTbtm3eCosDqsq2bdtYuXIlVapUCTqc/JswgblPPBGx1dWsCSNHWiK++mp7btQoqFsXWre2Xt2J+jbfvdsKgtSrB2eeaaevixeHO++EZcvgoYfsB0zERPjYOReEmCwwUr58eQBWrVrFrl27Ao7m4KSlpVGyZMmgw4iarPuXlJTE4YcfvvfYxbXSpcmIwrE79li79vzAAzbpwrBhMGWK3WrWtM5OV1xhk2fEu19/haFD4cUXbUpGsP26+2646SY45JAobThKx865ghSTiRksOcfzl/zkyZNp1KhR0GFETULv3wsvUH3xYmvORsHRR8Mrr1hyfuUVGDQIli+H226zlmS7dla4pH17a13Giy1bbN7qQYP27Yl+7LHQuzdceWUBlNGM8rFzriDEbGJ2LjCjR1NlTzMviipXhvvus5byBx9Y6zIlBSZMsFvp0nDZZTYX9Bln2DXZWLNhg1Xpeu89i3nPfNXFisGll1pJzZYt7fp6gSigY+dcNIWVmEWkPfA8UBQYqqpPZHm9BDAcaAysA/6tqr9HNlTnElNSEnTpYrfly+169LBhNpvV0KF2K1bMkvM551hj8MQTgxnjm5YGM2bA55/D1KnWMs5cRKVZM+jWDS6/PIqnq51LcLkmZhEpCgwG2gErgG9FZJyqLsy02LXABlWtLSJdgSeBf0cjYOcSWc2adh327rutxOeHH1rnqfnzrSfzZ5/ZcqVLW3Ju1856OR9/vJ0yjmTP5p07raLZDz/YUKbp02HePHt+DxE4+WTr1Hb++VCtWuS271xhFU6LuSmwVFV/BRCRd4DOQObE3Bl4KHT/PWCQiIh6l2rnDlqDBnZ74AFYs8aS8tixliR/+82Kl2SeNKNoUTj8cKs8VrmyJcmqVaFKFZsusVgx64i1dWsxvvkGtm6168J7bitX2npXr7bW+rp1/5yazhrX6afDWWfBaadBxYoF9B/iXCEhueVOEbkYaK+q14UedweaqeqtmZaZH1pmRejxL6Fl/j7QekuXLq1NmzaNwC7Epo0bN1Ixgb+xEnr/5sxh9+7dFGvSJOhIDmjnTti82W6bNsGOHXbL3ZzQvw3D2k6pUlCypCXfsmWhXLkYH28cB8cuvxL6s0fi79+UKVNmq2qOb9AC7fwlIj2AHqGHO6ZMmTK/ILdfwCoBB/xhkgASf/+mTEnU/asE4e3b9u1227Ah2iFFVCIfOygMn73E3r+6uS0QTmJeCdTM9PiI0HPZLbNCRIoBFbBOYPtQ1SHAEAAR+S63Xw3xzPcvviXy/iXyvoHvX7wrDPuX2zLhVP76FjhORI4WkeJAV2BclmXGAVeG7l8MTPLry84551ze5dpiVtXdInIr8Bk2XOo1VV0gIg8D36nqOOBV4E0RWQqsx5K3c8455/IorGvMqjoBmJDlub6Z7qcBXfK47SF5XD7e+P7Ft0Tev0TeN/D9i3eFfv9y7ZXtnHPOuYITk7NLOeecc4VVTCRmEblTRFREKgUdSySJyCMiMldE5ojIRBGpHnRMkSQiT4nIT6F9HCsiFYOOKVJEpIuILBCRDBFJmB6iItJeRH4WkaUicm/Q8USSiLwmImtCdRUSjojUFJEUEVkYem/2DDqmSBGRkiIyS0R+DO1bv6BjigYRKSoiP4jI+JyWCzwxi0hN4Czgj6BjiYKnVLWBqjYExgN9c1k+3nwOnKiqDYDFQJ+A44mk+cCFwNSgA4mUTOV1zwHqAZeKSL1go4qoYUD7oIOIot3AnapaDzgFuCWBjt8OoK2qJmPVb9qLyCnBhhQVPYFFuS0UeGIGngXuBhLuYreqbs70sAwJto+qOlFVd4cezsDGuCcEVV2kqj8HHUeE7S2vq6o7gT3ldROCqk7FRoUkJFVdrarfh+5vwb7gawQbVWSoSQ09TArdEur7UkSOADoCQ3NbNtDELCKdgZWq+mOQcUSTiDwqIsuBy0m8FnNm1wCfBB2Ey1ENYHmmxytIkC/2wkZEagGNgJkBhxIxodO8c4A1wOeqmjD7FvIc1gjNyGW56JfkFJEvgKrZvHQ/cB92Gjtu5bR/qvqhqt4P3C8ifYBbgQcLNMB8ym3/Qsvcj51me6sgY8uvcPbNuVgjImWBMUCvLGfl4pqqpgMNQ31VxorIiaqaEP0FRKQTsEZVZ4tI69yWj3piVtUzs3teROoDRwM/is2ifgTwvYg0VdU/ox1XpBxo/7LxFjYWPK4Sc277JyJXAZ2AM+Kt2lsejl2iCKe8rothIpKEJeW3VPX9oOOJBlXdKCIpWH+BhEjMQAvgPBHpAJQEyovICFXtlt3CgZ3KVtV5qlpFVWupai3stNpJ8ZSUcyMix2V62Bn4KahYokFE2mOnZs5T1W1Bx+NyFU55XRejxFowrwKLVPWZoOOJJBGpvGdUh4iUAtqRQN+XqtpHVY8I5bquWNnqbJMyxEbnr0T2hIjMF5G52Cn7hBneEDIIKAd8HhoS9lLQAUWKiFwgIiuA5sDHIvJZ0DHlV6ij3p7yuouA0aq6INioIkdERgLfAHVFZIWIXBt0TBHWAugOtA193uaEWmCJoBqQEvqu/Ba7xpzjkKJE5pW/nHPOuRjiLWbnnHMuhnhids4552KIJ2bnnHMuhnhids4552KIJ2bnnHMuhnhids4552KIJ2bnnHMuhnhidq4QEZFJmYpTpInIJUHH5JzblxcYca4QEpGbgDbApaHJA5xzMSLqk1g452KLiFwBnANc5EnZudjjidm5QkREumBzg3dW1V1Bx+Oc258nZucKidCcsDcDnVQ1Leh4nHPZ82vMzhUSIrIOWA9sDT01UFVfDTAk51w2PDE755xzMcSHSznnnHMxxBOzc845F0M8MTvnnHMxxBOzc845F0M8MTvnnHMxxBOzc845F0M8MTvnnHMxxBOzc845F0P+H0wWqjbf6qUjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x252 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error = tf.abs(error) < 1\n",
    "    squared_loss = tf.square(error) / 2\n",
    "    linear_loss  = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "\n",
    "# Plot the Huber loss function\n",
    "plt.figure(figsize=(8, 3.5))\n",
    "\n",
    "z = np.linspace(-4, 4, 200)\n",
    "\n",
    "plt.plot(z, huber_fn(0, z), \"b-\", linewidth=2, label=\"huber($z$)\")\n",
    "plt.plot(z, z**2 / 2, \"b:\", linewidth=1, label=r\"$\\frac{1}{2}z^2$\")\n",
    "plt.plot([-1, -1], [0, huber_fn(0., -1.)], \"r--\")\n",
    "plt.plot([1, 1], [0, huber_fn(0., 1.)], \"r--\")\n",
    "\n",
    "plt.gca().axhline(y=0, color='k')\n",
    "plt.gca().axvline(x=0, color='k')\n",
    "plt.axis([-4, 4, 0, 4])\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"$z$\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Huber loss\", fontsize=14)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "knowing-trigger",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 3s 4ms/step - loss: 1.0443 - mae: 1.4660 - val_loss: 0.2862 - val_mae: 0.5866\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.2379 - mae: 0.5407 - val_loss: 0.2382 - val_mae: 0.5281\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc759fa6730>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "# Build a model with custom Huber loss function\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])\n",
    "model.compile(loss=huber_fn, optimizer=\"nadam\", metrics=[\"mae\"])\n",
    "\n",
    "# Train the model for few epochs\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "underlying-resource",
   "metadata": {},
   "source": [
    "### Saving and loading models that contain custom components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "prime-shannon",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.2054 - mae: 0.4982 - val_loss: 0.2209 - val_mae: 0.5050\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.1999 - mae: 0.4900 - val_loss: 0.2127 - val_mae: 0.4986\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc75845aeb0>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_path = os.path.join(\"data\", \"my_model_with_a_custom_loss.h5\")\n",
    "\n",
    "# Save the model - *name* of the custom function is saved with the model\n",
    "model.save(model_path)\n",
    "\n",
    "# To load the model one must bind names of custom components back to instances\n",
    "model = keras.models.load_model(model_path, custom_objects={\"huber_fn\": huber_fn})\n",
    "\n",
    "# Let's check that the restored model works\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "pressed-bathroom",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.2318 - val_loss: 0.2540\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.2309 - val_loss: 0.2372\n",
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.2147 - val_loss: 0.2133\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2119 - val_loss: 0.1992\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc758170580>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Factory function for the Huber loss\n",
    "def create_huber(threshold=1.0):\n",
    "    \n",
    "    # Parametrized Huber loss function\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    \n",
    "    return huber_fn\n",
    "\n",
    "# Build new model and train it a bit\n",
    "model.compile(loss=create_huber(threshold=2.0), optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "# Save new model - only the loss function is saved, not the whole closure (i.e. not the parameter)\n",
    "model_path = os.path.join(\"data\", \"my_model_with_a_custom_loss_2.h5\")\n",
    "model.save(model_path)\n",
    "\n",
    "# To load this model we must rebuild the loss function instance\n",
    "model = keras.models.load_model(model_path, custom_objects={\"huber_fn\": create_huber(2.0)})\n",
    "\n",
    "# Finally, let's check that it worked\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "grand-george",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.3123 - mae: 1.3345 - val_loss: 0.3378 - val_mae: 0.5485\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.2659 - mae: 0.5270 - val_loss: 0.2660 - val_mae: 0.5089\n",
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.2286 - mae: 0.4970 - val_loss: 0.2120 - val_mae: 0.4723\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.2216 - mae: 0.4904 - val_loss: 0.2045 - val_mae: 0.4725\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# An alternative is to subclass the `Loss` class\n",
    "#  - note that subclassing some components (like this one) has not yet been standardized\n",
    "class HuberLoss(keras.losses.Loss):\n",
    "    \n",
    "    # Pass custom parameters as instance arguments\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    \n",
    "    # Implementation of the loss function itself\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    \n",
    "    # This method is called to restore custom parameters\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}\n",
    "    \n",
    "# Build new model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])\n",
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])\n",
    "\n",
    "# Train it for a bit\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "# Save new model - only the loss function is saved, not the whole closure (i.e. not the parameter)\n",
    "model_path = os.path.join(\"data\", \"my_model_with_a_custom_loss_3.h5\")\n",
    "model.save(model_path)\n",
    "\n",
    "# To load this model we must rebuild the loss function instance\n",
    "model = keras.models.load_model(model_path, custom_objects={\"HuberLoss\": HuberLoss})\n",
    "\n",
    "# Finally, let's check that it worked\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "# Check if custom parameter survived\n",
    "model.loss.threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wanted-milan",
   "metadata": {},
   "source": [
    "### Custom activation functions, initializers, regularizers and constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "measured-paper",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 2.3829 - mae: 1.1635 - val_loss: 1.4154 - val_mae: 0.5607\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.6299 - mae: 0.5410 - val_loss: 1.4399 - val_mae: 0.5137\n"
     ]
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "def my_softplus(z):\n",
    "    # equivalent to `tf.nn.softplus(z)`\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "def my_positive_weights(weights):\n",
    "    # Equivalent to `tf.nn.relu(weights)`\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)\n",
    "\n",
    "# Build a model with these custom components\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(\n",
    "        30,\n",
    "        activation=\"selu\",\n",
    "        kernel_initializer=\"lecun_normal\",\n",
    "        input_shape=input_shape\n",
    "    ),\n",
    "    keras.layers.Dense(\n",
    "        1,\n",
    "        activation=my_softplus,\n",
    "        kernel_regularizer=my_l1_regularizer,\n",
    "        kernel_constraint=my_positive_weights,\n",
    "        kernel_initializer=my_glorot_initializer),\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])\n",
    "\n",
    "# Train the model a bit\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "# Save the model\n",
    "model_path = os.path.join(\"data\", \"my_model_with_many_custom_parts.h5\")\n",
    "model.save(model_path)\n",
    "\n",
    "# Load the model just like we did it with custom loss function\n",
    "model = keras.models.load_model(\n",
    "    model_path,\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "loved-egyptian",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 2.3829 - mae: 1.1635 - val_loss: 1.4154 - val_mae: 0.5607\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.6299 - mae: 0.5410 - val_loss: 1.4399 - val_mae: 0.5137\n"
     ]
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Subclassing these components works as well\n",
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    \n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    \n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    \n",
    "    def get_config(self):\n",
    "        # The Regularizer does not use any configs on its own so not need to call super\n",
    "        return {\"factor\": self.factor}\n",
    "    \n",
    "# Build new model\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(\n",
    "        30,\n",
    "        activation=\"selu\",\n",
    "        kernel_initializer=\"lecun_normal\",\n",
    "        input_shape=input_shape,\n",
    "    ),\n",
    "    keras.layers.Dense(\n",
    "        1,\n",
    "        activation=my_softplus,\n",
    "        kernel_regularizer=MyL1Regularizer(0.01),\n",
    "        kernel_constraint=my_positive_weights,\n",
    "        kernel_initializer=my_glorot_initializer),\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])\n",
    "\n",
    "# Train the model a bit\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "# Save the model\n",
    "model_path = os.path.join(\"data\", \"my_model_with_many_custom_parts.h5\")\n",
    "model.save(model_path)\n",
    "\n",
    "# Load the model just like we did it with custom loss function\n",
    "model = keras.models.load_model(\n",
    "    model_path,\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "supreme-harmony",
   "metadata": {},
   "source": [
    "### Custom metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "korean-label",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 3.5903 - huber_fn: 1.5558\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.8054 - huber_fn: 0.3095\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc750629b80>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Build new model with Huber loss as a metric (as an artificial example)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[create_huber(2.0)])\n",
    "\n",
    "# Train the model a bit\n",
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "distinguished-quilt",
   "metadata": {},
   "source": [
    "**Warning**: if you use the same function as the loss and a metric, you may be surprised to see different results. This is generally just due to floating point precision errors: even though the mathematical equations are equivalent, the operations are not run in the same order, which can lead to small differences. Moreover, when using sample weights, there's more than just precision errors:\n",
    "\n",
    "* the loss since the start of the epoch is the mean of all batch losses seen so far. Each batch loss is the sum of the weighted instance losses divided by the batch size (not the sum of weights, so the batch loss is not the weighted mean of the losses).\n",
    "* the metric since the start of the epoch is equal to the sum of weighted instance losses divided by sum of all weights seen so far. In other words, it is the weighted mean of all the instance losses. Not the same thing.\n",
    "\n",
    "If you do the math, you will find that `loss = metric * mean` of sample weights (plus some floating point precision error)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "local-desert",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.1245 - huber_fn: 0.2515\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1216 - huber_fn: 0.2473\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss=create_huber(2.0), optimizer=\"nadam\", metrics=[create_huber(2.0)])\n",
    "\n",
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "dominant-posting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.11749906837940216, 0.11906626312604573)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = history.history[\"loss\"][0]\n",
    "loss_from_metric = history.history[\"huber_fn\"][0] * sample_weight.mean()\n",
    "\n",
    "loss, loss_from_metric"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adverse-xerox",
   "metadata": {},
   "source": [
    "#### Streaming metrics\n",
    "Also known as **Stateful metrics**. These metrics are gradually updated batch after batch - i.e. over the whole epoch, in contrast to normal metrics which are computed just over batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "rental-momentum",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = keras.metrics.Precision()\n",
    "\n",
    "# Updated once (batch 1)\n",
    "precision([0, 1, 1, 1, 0, 1, 0, 1], [1, 1, 0, 1, 0, 1, 0, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "rotary-thesis",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Updated twice (batch 2)\n",
    "precision([0, 1, 0, 0, 1, 0, 1, 1], [1, 0, 1, 1, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "composed-harrison",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the current state\n",
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "sustainable-sugar",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Access the variables that are being tracked in the state\n",
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "instructional-residence",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Manually reset the state\n",
    "precision.reset_states()\n",
    "\n",
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "middle-initial",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Custom metric subclass for the Huber loss/metric\n",
    "class HuberMetric(keras.metrics.Metric):\n",
    "    \n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.threshold = threshold\n",
    "        # Initialize the state\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "    \n",
    "    def huber_fn(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    \n",
    "    # This method is responsible for the state update\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(metric))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "    \n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "    \n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}\n",
    "\n",
    "# Try this custom metric\n",
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "ordered-engineering",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))\n",
    "\n",
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "jewish-logistics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "sitting-diamond",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "incredible-financing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 987us/step - loss: 1.5182 - huber_metric: 1.5182\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2909 - huber_metric: 0.2909\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc794124a30>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Build a model with custom Huber metric\n",
    "threshold = 2.0\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])\n",
    "model.compile(loss=create_huber(threshold), optimizer=\"nadam\", metrics=[HuberMetric(threshold)])\n",
    "\n",
    "# Train the model for few epochs\n",
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "clear-washer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.7800 - HuberMetric: 1.5417\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1462 - HuberMetric: 0.3000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.44554394483566284, 0.4432064726421927)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Even simple implementation that takes advantage of the `Mean` class\n",
    "class HuberMetric(keras.metrics.Mean):\n",
    "    \n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "    \n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super().update_state(metric, sample_weight)\n",
    "    \n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}\n",
    "    \n",
    "\n",
    "# Build a model with custom Huber metric\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])\n",
    "model.compile(loss=keras.losses.Huber(threshold), optimizer=\"nadam\", metrics=[HuberMetric(threshold)])\n",
    "\n",
    "# Train the model with some sample weights\n",
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2, sample_weight=sample_weight)\n",
    "\n",
    "# Compare the loss and metric\n",
    "loss = history.history[\"loss\"][0]\n",
    "loss_from_metric = history.history[\"HuberMetric\"][0] * sample_weight.mean()\n",
    "loss, loss_from_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "positive-coach",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.2377 - HuberMetric: 0.2377\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2279 - HuberMetric: 0.2279\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try saving and restoring the model\n",
    "model_path = os.path.join(\"data\", \"my_model_with_a_custom_metric_v2.h5\")\n",
    "model.save(model_path)\n",
    "\n",
    "model = keras.models.load_model(model_path, custom_objects={\"HuberMetric\": HuberMetric})\n",
    "\n",
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)\n",
    "\n",
    "# -1 instead of 0 is because `tf.keras` adds an extra metric to the first position\n",
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "intellectual-march",
   "metadata": {},
   "source": [
    "### Custom layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "casual-amplifier",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float32, numpy=array([0.36787945, 1.        , 2.7182817 ], dtype=float32)>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exponential_layer = keras.layers.Lambda(lambda x: tf.exp(x))\n",
    "\n",
    "exponential_layer([-1., 0., 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "wrong-launch",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 0s 1ms/step - loss: nan - val_loss: nan\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: nan - val_loss: nan\n",
      "162/162 [==============================] - 0s 734us/step - loss: nan\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Try building, training and testing a model with custom exponential layer\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "    exponential_layer,\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "\n",
    "model.fit(X_train_scaled, y_train, epochs=5, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ancient-lewis",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 4.1268 - val_loss: 0.9472\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.7086 - val_loss: 0.6219\n",
      "162/162 [==============================] - 0s 904us/step - loss: 0.5474\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5473727583885193"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Custom Dense layer\n",
    "class MyDense(keras.layers.Layer):\n",
    "    \n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        # Number of neurons in this layer\n",
    "        self.units = units\n",
    "        # Get the Keras function by the parameter\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    # This method is called the first time Keras uses the model\n",
    "    def build(self, batch_input_shape):\n",
    "        # Register variables for the input weights\n",
    "        self.kernel = self.add_weight(\n",
    "            name=\"kernel\",\n",
    "            shape=[batch_input_shape[-1], self.units],\n",
    "            initializer=\"glorot_normal\",\n",
    "        )\n",
    "        # Regiter variables for the biases\n",
    "        self.bias = self.add_weight(\n",
    "            name=\"bias\",\n",
    "            shape=[self.units],\n",
    "            initializer=\"zeros\",\n",
    "        )\n",
    "        # This call must be at the end\n",
    "        super().build(batch_input_shape)\n",
    "\n",
    "    # Compute this layer's output\n",
    "    def call(self, X):\n",
    "        return self.activation(X @ self.kernel + self.bias)\n",
    "\n",
    "    # Get the shape of layer's output\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return tf.TensorShape(batch_input_shape.as_list()[:-1] + [self.units])\n",
    "\n",
    "    # Get bindings necessary to restore this layer\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {\n",
    "            **base_config,\n",
    "            \"units\": self.units,\n",
    "            \"activation\": keras.activations.serialize(self.activation),\n",
    "        }\n",
    "\n",
    "# Build, train and evaluate a model with custom dense layer\n",
    "model = keras.models.Sequential([\n",
    "    MyDense(30, activation=\"relu\", input_shape=input_shape),\n",
    "    MyDense(1)\n",
    "])\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "modular-desire",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try saving and restoring this model\n",
    "model_path = os.path.join(\"data\", \"my_model_with_a_custom_layer.h5\")\n",
    "model.save(model_path)\n",
    "\n",
    "model = keras.models.load_model(model_path, custom_objects={\"MyDense\": MyDense})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "imported-purchase",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Example of a layer with two inputs and two outputs\n",
    "class MyMultiLayer(keras.layers.Layer):\n",
    "    # X is a tuple of two (or more) inputs\n",
    "    def call(self, X):\n",
    "        X1, X2 = X\n",
    "        # Return a collection of outputs\n",
    "        return X1 + X2, X1 * X2\n",
    "\n",
    "    # Similarly here the argument and return value is a collection\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        batch_input_shape1, batch_input_shape2 = batch_input_shape\n",
    "        return [batch_input_shape1, batch_input_shape2]\n",
    "\n",
    "# Try this multi-layer out\n",
    "inputs1 = keras.layers.Input(shape=[2])\n",
    "inputs2 = keras.layers.Input(shape=[2])\n",
    "outputs1, outputs2 = MyMultiLayer()((inputs1, inputs2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "emotional-petite",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.5496 - val_loss: 0.5329\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.4430 - val_loss: 0.4786\n",
      "162/162 [==============================] - 0s 748us/step - loss: 0.3990\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.39900389313697815"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This example is actually an existing layer called `GaussianNoise`\n",
    "class AddGaussianNoise(keras.layers.Layer):\n",
    "    \n",
    "    def __init__(self, stddev, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.stddev = stddev\n",
    "\n",
    "    # Call can additionally receive a `training` flag\n",
    "    def call(self, X, training=None):\n",
    "        if training:\n",
    "            # Add some Gaussian noise\n",
    "            noise = tf.random.normal(tf.shape(X), stddev=self.stddev)\n",
    "            return X + noise\n",
    "        return X\n",
    "\n",
    "    # This is probably not necessary, Keras/TF can infer it\n",
    "    def compute_output_shape(self, batch_input_shape):\n",
    "        return batch_input_shape\n",
    "\n",
    "# Try this layer out\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continent-black",
   "metadata": {},
   "source": [
    "### Custom Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "another-commonwealth",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_scaled = X_test_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "aware-waterproof",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "363/363 [==============================] - 3s 2ms/step - loss: 22.7478\n",
      "Epoch 2/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 1.2735\n",
      "Epoch 3/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.9792\n",
      "Epoch 4/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.5908\n",
      "Epoch 5/5\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.5544\n",
      "162/162 [==============================] - 0s 1ms/step - loss: 0.6498\n"
     ]
    }
   ],
   "source": [
    "from operator import add\n",
    "\n",
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "\n",
    "# Residual block composed of:\n",
    "#  - n dense layers in sequence\n",
    "#  - input connected and added to the final output\n",
    "class ResidualBlock(keras.layers.Layer):\n",
    "    \n",
    "    def __init__(self, n_layers, n_neurons, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        # Keras can automatically detect that `hidden` contains trackable objects and \n",
    "        # add their weights to between layer's variables\n",
    "        self.hidden = [\n",
    "            keras.layers.Dense(n_neurons, activation=\"elu\", kernel_initializer=\"he_normal\")\n",
    "            for _ in range(n_layers)\n",
    "        ]\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        return inputs + Z\n",
    "\n",
    "# Model composed of a sequence:\n",
    "#  - single dense layer\n",
    "#  - a residual block with 3 repeats\n",
    "#  - followed by single residual block\n",
    "#  - and finally a dense layer\n",
    "class ResidualRegressor(keras.models.Model):\n",
    "    \n",
    "    def __init__(self, output_dim, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = keras.layers.Dense(\n",
    "            30,\n",
    "            activation=\"elu\",\n",
    "            kernel_initializer=\"he_normal\",\n",
    "        )\n",
    "        self.block1 = ResidualBlock(2, 30)\n",
    "        self.block2 = ResidualBlock(2, 30)\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        Z = self.hidden1(inputs)\n",
    "        for _ in range(1 + 3):\n",
    "            Z = self.block1(Z)\n",
    "        Z = self.block2(Z)\n",
    "        return self.out(Z)\n",
    "\n",
    "\n",
    "# Try this model out\n",
    "model = ResidualRegressor(1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=5)\n",
    "score = model.evaluate(X_test_scaled, y_test)\n",
    "y_pred = model.predict(X_new_scaled)  # test set predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exterior-scheme",
   "metadata": {},
   "source": [
    "### Losses and Metrics Based on Model Internals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "ceramic-firewall",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 2s 2ms/step - loss: 1.6313 - reconstruction_error: 1.0474\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 3ms/step - loss: 0.4536 - reconstruction_error: 0.4022\n"
     ]
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "class ReconstructingRegressor(keras.models.Model):\n",
    "    \n",
    "    def __init__(self, input_dim, output_dim, hidden_dim=30, n_hidden=5, recon_weight=0.05, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        # Weight of the recostruction loss\n",
    "        #  - we don't want it to overweight the main objective\n",
    "        self.recon_weight = recon_weight\n",
    "        \n",
    "        # Sequence of dense hidden layers\n",
    "        self.hidden = [\n",
    "            keras.layers.Dense(hidden_dim, activation=\"selu\", kernel_initializer=\"lecun_normal\")\n",
    "            for _ in range(n_hidden)\n",
    "        ]\n",
    "        \n",
    "        # Actual output layer\n",
    "        self.out = keras.layers.Dense(output_dim)\n",
    "        \n",
    "        # Auxiliary layer the output of which serves to compute the reconstruction loss\n",
    "        #  - it must be created here because it is potentially the first time we know the shape of inputs\n",
    "        self.reconstruct = keras.layers.Dense(input_dim)\n",
    "        \n",
    "        # Metric that tracks the mean reconstruction error\n",
    "        self.reconstruction_mean = keras.metrics.Mean(name=\"reconstruction_error\")\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        # Compute the outputs before the final layer\n",
    "        Z = inputs\n",
    "        for layer in self.hidden:\n",
    "            Z = layer(Z)\n",
    "        \n",
    "        reconstruction = self.reconstruct(Z)\n",
    "        \n",
    "        # Reconstuction loss is a mean squared difference between the output of the aux layer and inputs\n",
    "        #  - the idea is to preserve as much information as possible which serves as regularization\n",
    "        recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
    "        \n",
    "        # Register the reconstruction loss\n",
    "        self.add_loss(self.recon_weight * recon_loss)\n",
    "        \n",
    "        if training:\n",
    "            # Compute and register the mean of reconstruction loss as a metric\n",
    "            recon_mean = self.reconstruction_mean(recon_loss)\n",
    "            self.add_metric(recon_mean)\n",
    "        \n",
    "        return self.out(Z)\n",
    "    \n",
    "# Test the model\n",
    "model = ReconstructingRegressor(input_dim=X_train_scaled.shape[1], output_dim=1)\n",
    "model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
    "history = model.fit(X_train_scaled, y_train, epochs=2)\n",
    "y_pred = model.predict(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "amended-greensboro",
   "metadata": {},
   "source": [
    "### Computing Gradients with Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "medical-construction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(36.000003007075065, 10.000000003174137)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example function\n",
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + 2 * w1 * w2\n",
    "\n",
    "\n",
    "# Computing an approximate derivative at a point\n",
    "def approx_df(f, w1, w2, eps=1e-6):\n",
    "    fval = f(w1, w2)\n",
    "    dw1 = (f(w1 + eps, w2) - fval) / eps\n",
    "    dw2 = (f(w1, w2 + eps) - fval) / eps\n",
    "    return dw1, dw2\n",
    "\n",
    "# Compute derivatives of f at point (5, 3)\n",
    "approx_df(f, w1=5, w2=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "classical-sampling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute the same gradients with TensorFlow's autodiff\n",
    "w1, w2 = tf.Variable(5.), tf.Variable(3.)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "gradients = tape.gradient(z, [w1, w2])\n",
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "headed-husband",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A non-persistent GradientTape can only be used tocompute one set of gradients (or jacobians)\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "\n",
    "# The `gradient` method can be called just once\n",
    "try:\n",
    "    dz_dw2 = tape.gradient(z, w2)\n",
    "except RuntimeError as ex:\n",
    "    print(ex)\n",
    "    \n",
    "# To be able to call it multiple times, the tape must be made persistent\n",
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "dz_dw1 = tape.gradient(z, w1)\n",
    "dz_dw2 = tape.gradient(z, w2)\n",
    "\n",
    "# This call releases all resources held by the tape\n",
    "#  - typically this is left to the GC when the tape goes out of scope\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "varied-sperm",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, None]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tape tracks only variables\n",
    "c1, c2 = tf.constant(5.), tf.constant(3.)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(c1, c2)\n",
    "\n",
    "# So this example returns `None`\n",
    "gradients = tape.gradient(z, [c1, c2])\n",
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "hollow-performer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Auxiliary tensors can be tracked by calling `watch`\n",
    "with tf.GradientTape() as tape:\n",
    "    tape.watch(c1)\n",
    "    tape.watch(c2)\n",
    "    z = f(c1, c2)\n",
    "\n",
    "# Due to watching c1 and c2 this call returns a value\n",
    "gradients = tape.gradient(z, [c1, c2])\n",
    "gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "light-gambling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=136.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=30.0>]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.GradientTape() as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tape.gradient([z1, z2, z3], [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "wooden-winning",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    z1 = f(w1, w2 + 2.)\n",
    "    z2 = f(w1, w2 + 5.)\n",
    "    z3 = f(w1, w2 + 7.)\n",
    "\n",
    "tf.reduce_sum(tf.stack([tape.gradient(z, [w1, w2]) for z in (z1, z2, z3)]), axis=0)\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "objective-acquisition",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.GradientTape(persistent=True) as hessian_tape:\n",
    "    \n",
    "    with tf.GradientTape() as jacobian_tape:\n",
    "        z = f(w1, w2)\n",
    "    \n",
    "    jacobians = jacobian_tape.gradient(z, [w1, w2])\n",
    "\n",
    "    \n",
    "hessians = [\n",
    "    hessian_tape.gradient(jacobian, [w1, w2])\n",
    "    for jacobian in jacobians\n",
    "]\n",
    "\n",
    "del hessian_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "local-chassis",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=36.0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=10.0>]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jacobians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "actual-century",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[<tf.Tensor: shape=(), dtype=float32, numpy=6.0>,\n",
       "  <tf.Tensor: shape=(), dtype=float32, numpy=2.0>],\n",
       " [<tf.Tensor: shape=(), dtype=float32, numpy=2.0>, None]]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "caring-starter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=30.0>, None]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# `stop_gradient` can be used to stop gradient propagation\n",
    "#  - it will act just like `identity`\n",
    "def f(w1, w2):\n",
    "    return 3 * w1 ** 2 + tf.stop_gradient(2 * w1 * w2)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = f(w1, w2)\n",
    "\n",
    "tape.gradient(z, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "literary-punishment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=float32, numpy=nan>]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Some computations might be numerically unstable\n",
    "#  - for instance a Softmax activation for large inputs\n",
    "x = tf.Variable(100.)\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_softplus(x)\n",
    "\n",
    "tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "serial-middle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(1,), dtype=float32, numpy=array([1000.], dtype=float32)>,\n",
       " [<tf.Tensor: shape=(1,), dtype=float32, numpy=array([nan], dtype=float32)>])"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fortunately for many activations we know the analytical solution (derivative)\n",
    "#  - We can register custom activation that additionally returns a function evaluating the gradient\n",
    "@tf.custom_gradient\n",
    "def my_better_softplus(z):\n",
    "    exp = tf.exp(z)\n",
    "    \n",
    "    # The gradient function is evaluated as part of the chain rule (in autodiff)\n",
    "    #  - so it receives gradients and it should multiply its output by them\n",
    "    def my_softplus_gradients(grad):\n",
    "        return grad / (1 + 1 / exp)\n",
    "    \n",
    "    return tf.math.log(exp + 1), my_softplus_gradients\n",
    "\n",
    "# Actually in this case this re-implementation would be stable\n",
    "def my_better_softplus(z):\n",
    "    return tf.where(z > 30., z, tf.math.log(tf.exp(z) + 1.))\n",
    "\n",
    "# Let's test our custom implementation on very large input\n",
    "x = tf.Variable([1000.])\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    z = my_better_softplus(x)\n",
    "\n",
    "z, tape.gradient(z, [x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "associate-saying",
   "metadata": {},
   "source": [
    "### Custom Training Loops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "technical-eagle",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "11610/11610 [==============================] - mean: 1.3955 - mean_absolute_error: 0.5722\n",
      "Epoch 2/5\n",
      "11610/11610 [==============================] - mean: 0.6774 - mean_absolute_error: 0.5280\n",
      "Epoch 3/5\n",
      "11610/11610 [==============================] - mean: 0.6351 - mean_absolute_error: 0.5177\n",
      "Epoch 4/5\n",
      "11610/11610 [==============================] - mean: 0.6384 - mean_absolute_error: 0.5181\n",
      "Epoch 5/5\n",
      "11610/11610 [==============================] - mean: 0.6440 - mean_absolute_error: 0.5222\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "def random_batch(X, y, batch_size=32):\n",
    "    \"\"\"Sample a batch from given dataset with corresponding labels\"\"\"\n",
    "    idx = np.random.randint(len(X), size=batch_size)\n",
    "    return X[idx], y[idx]\n",
    "\n",
    "def progress_bar(iteration, total, size=30):\n",
    "    running = iteration < total\n",
    "    c = \">\" if running else \"=\"\n",
    "    p = (size - 1) * iteration // total\n",
    "    fmt = \"{{:-{}d}}/{{}} [{{}}]\".format(len(str(total)))\n",
    "    params = [iteration, total, \"=\" * p + c + \".\" * (size - p - 1)]\n",
    "    return fmt.format(*params)\n",
    "    \n",
    "\n",
    "def print_status_bar(iteration, total, loss, metrics=None, size=30):\n",
    "    metrics = \" - \".join(\"{}: {:.4f}\".format(m.name, m.result()) for m in [loss] + (metrics or []))\n",
    "    progress = progress_bar(iteration, total, size)\n",
    "    print(f\"\\r{progress} - {metrics}\", end=\"\" if iteration < total else \"\\n\")\n",
    "\n",
    "\n",
    "# Build simple sequential model\n",
    "l2_reg = keras.regularizers.l2(0.05)\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"elu\", kernel_initializer=\"he_normal\", kernel_regularizer=l2_reg),\n",
    "    keras.layers.Dense(1, kernel_regularizer=l2_reg)\n",
    "])\n",
    "\n",
    "# Define hyperparameters\n",
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.mean_squared_error\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.MeanAbsoluteError()]\n",
    "\n",
    "\n",
    "# Test out custom training loop with progress tracking\n",
    "#  - we omit the fact that some examples may not ever be sampled\n",
    "#  - related issue is that the batch size is not a multiple of the training set size\n",
    "#  - gradient clipping hyperparameters are also omitted\n",
    "\n",
    "# Repeat the training for `n_epochs`\n",
    "for epoch in range(1, n_epochs + 1):\n",
    "    \n",
    "    print(f\"Epoch {epoch}/{n_epochs}\")\n",
    "    \n",
    "    # Run each training epoch for `n_steps`\n",
    "    for step in range(1, n_steps + 1):\n",
    "        \n",
    "        # Sample a random batch of training examples\n",
    "        X_batch, y_batch = random_batch(X_train_scaled, y_train)\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            # Feed the training batch forward through the model\n",
    "            #  - here we omit `training` flag to handle special layers\n",
    "            y_pred = model(X_batch)\n",
    "            # Add the mean main loss (MSE in this example) and additional model losses\n",
    "            main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "            loss = tf.add_n([main_loss] + model.losses)\n",
    "        \n",
    "        # Evaluate gradients based on recorded forward pass and make an optimization step (Nadam)\n",
    "        gradients = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        \n",
    "        # Apply parameter transformations if such are defined in the model\n",
    "        for variable in model.variables:\n",
    "            if variable.constraint is not None:\n",
    "                variable.assign(variable.constraint(variable))\n",
    "        \n",
    "        # Account the loss from current step to the mean total loss\n",
    "        mean_loss(loss)\n",
    "        \n",
    "        # Update metrics\n",
    "        for metric in metrics:\n",
    "            metric(y_batch, y_pred)\n",
    "        \n",
    "        print_status_bar(step * batch_size, len(y_train), mean_loss, metrics)\n",
    "    \n",
    "    print_status_bar(len(y_train), len(y_train), mean_loss, metrics)\n",
    "    \n",
    "    # Reset metrics at the end of each epoch\n",
    "    for metric in [mean_loss] + metrics:\n",
    "        metric.reset_states()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vanilla-academy",
   "metadata": {},
   "source": [
    "## TensorFlow Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "beneficial-index",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def cube(x):\n",
    "    return x ** 3\n",
    "\n",
    "cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "intelligent-conditions",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "intelligent-masters",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.eager.def_function.Function at 0x7fc759010f10>"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube = tf.function(cube)\n",
    "tf_cube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "looking-howard",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=8>"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "independent-bookmark",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "neither-sauce",
   "metadata": {},
   "source": [
    "### TF Functions and Concrete Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "suited-operation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.func_graph.FuncGraph at 0x7fc758ba4790>"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function = tf_cube.get_concrete_function(tf.constant(2.0))\n",
    "concrete_function.graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "specialized-cooperation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "exotic-secretariat",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function is tf_cube.get_concrete_function(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eligible-inquiry",
   "metadata": {},
   "source": [
    "### Exploring Function Definitions and Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "outdoor-breakdown",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'pow/y' type=Const>,\n",
       " <tf.Operation 'pow' type=Pow>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ops = concrete_function.graph.get_operations()\n",
    "ops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "disciplinary-identification",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'x:0' shape=() dtype=float32>,\n",
       " <tf.Tensor 'pow/y:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op = ops[2]\n",
    "list(pow_op.inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "continued-edmonton",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'pow:0' shape=() dtype=float32>]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow_op.outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "blond-gnome",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Operation 'x' type=Placeholder>"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_operation_by_name('x')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "musical-antenna",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Identity:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.graph.get_tensor_by_name('Identity:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "sized-jacket",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"__inference_cube_594558\"\n",
       "input_arg {\n",
       "  name: \"x\"\n",
       "  type: DT_FLOAT\n",
       "}\n",
       "output_arg {\n",
       "  name: \"identity\"\n",
       "  type: DT_FLOAT\n",
       "}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concrete_function.function_def.signature"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "expired-joint",
   "metadata": {},
   "source": [
    "### How TF Functions Trace Python Functions to Extract Their Computation Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "turkish-municipality",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: Tensor(\"x:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "@tf.function\n",
    "def tf_cube(x):\n",
    "    print(\"print:\", x)\n",
    "    return x ** 3\n",
    "\n",
    "result = tf_cube(tf.constant(2.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "desirable-subject",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=8.0>"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TF AutoGraph processes each function just once per input type and then caches it\n",
    "#  - so the side-effect in `tf_cube` is not executed for the second time\n",
    "#  - note: python literals are considered new and trigger execution graph generation\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "fancy-chassis",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "print: 2\n",
      "print: 3\n",
      "print: Tensor(\"x:0\", shape=(1, 2), dtype=float32)\n",
      "print: Tensor(\"x:0\", shape=(2, 2), dtype=float32)\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function tf_cube at 0x7fc758c45670> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "print: Tensor(\"x:0\", shape=(3, 2), dtype=float32)\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function tf_cube at 0x7fc758c45670> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    }
   ],
   "source": [
    "result = tf_cube(2)\n",
    "result = tf_cube(3)\n",
    "\n",
    "# New shape: trace!\n",
    "result = tf_cube(tf.constant([[1., 2.]]))\n",
    "\n",
    "# New shape: trace!\n",
    "result = tf_cube(tf.constant([[3., 4.], [5., 6.]]))\n",
    "\n",
    "# New shape: trace!\n",
    "result = tf_cube(tf.constant([[7., 8.], [9., 10.], [11., 12.]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "becoming-salad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing Tensor(\"images:0\", shape=(None, 28, 28), dtype=float32)\n",
      "Python inputs incompatible with input_signature:\n",
      "  inputs: (\n",
      "    tf.Tensor(\n",
      "[[[0.7413678  0.62854624]\n",
      "  [0.01738465 0.3431449 ]]\n",
      "\n",
      " [[0.51063764 0.3777541 ]\n",
      "  [0.07321596 0.02137029]]], shape=(2, 2, 2), dtype=float32))\n",
      "  input_signature: (\n",
      "    TensorSpec(shape=(None, 28, 28), dtype=tf.float32, name=None))\n"
     ]
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "\n",
    "@tf.function(input_signature=[tf.TensorSpec([None, 28, 28], tf.float32)])\n",
    "def shrink(images):\n",
    "    print(\"Tracing\", images)\n",
    "    # Drop half the rows and columns\n",
    "    return images[:, ::2, ::2]\n",
    "\n",
    "img_batch_1 = tf.random.uniform(shape=[100, 28, 28])\n",
    "img_batch_2 = tf.random.uniform(shape=[50, 28, 28])\n",
    "img_batch_3 = tf.random.uniform(shape=[2, 2, 2])\n",
    "\n",
    "# Traces the function.\n",
    "preprocessed_images = shrink(img_batch_1)\n",
    "\n",
    "# Reuses the same concrete function.\n",
    "preprocessed_images = shrink(img_batch_2)\n",
    "\n",
    "# Rejects unexpected types or shapes\n",
    "try:\n",
    "    preprocessed_images = shrink(img_batch_3)\n",
    "except ValueError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "crude-developer",
   "metadata": {},
   "source": [
    "### Using Autograph To Capture Control Flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "pursuant-galaxy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A static for loop using `range`\n",
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in range(10):\n",
    "        x += 1\n",
    "    return x\n",
    "\n",
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "established-notion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'add/y' type=Const>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'add_1/y' type=Const>,\n",
       " <tf.Operation 'add_1' type=AddV2>,\n",
       " <tf.Operation 'add_2/y' type=Const>,\n",
       " <tf.Operation 'add_2' type=AddV2>,\n",
       " <tf.Operation 'add_3/y' type=Const>,\n",
       " <tf.Operation 'add_3' type=AddV2>,\n",
       " <tf.Operation 'add_4/y' type=Const>,\n",
       " <tf.Operation 'add_4' type=AddV2>,\n",
       " <tf.Operation 'add_5/y' type=Const>,\n",
       " <tf.Operation 'add_5' type=AddV2>,\n",
       " <tf.Operation 'add_6/y' type=Const>,\n",
       " <tf.Operation 'add_6' type=AddV2>,\n",
       " <tf.Operation 'add_7/y' type=Const>,\n",
       " <tf.Operation 'add_7' type=AddV2>,\n",
       " <tf.Operation 'add_8/y' type=Const>,\n",
       " <tf.Operation 'add_8' type=AddV2>,\n",
       " <tf.Operation 'add_9/y' type=Const>,\n",
       " <tf.Operation 'add_9' type=AddV2>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "looking-violin",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=15>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A dynamic loop using `tf.while_loop`\n",
    "@tf.function\n",
    "def add_10(x):\n",
    "    condition = lambda i, x: tf.less(i, 10)\n",
    "    body = lambda i, x: (tf.add(i, 1), tf.add(x, 1))\n",
    "    _, final_x = tf.while_loop(condition, body, [tf.constant(0), x])\n",
    "    return final_x\n",
    "\n",
    "add_10(tf.constant(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "excellent-chile",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'Const' type=Const>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "add_10.get_concrete_function(tf.constant(5)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "weird-messenger",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'x' type=Placeholder>,\n",
       " <tf.Operation 'range/start' type=Const>,\n",
       " <tf.Operation 'range/limit' type=Const>,\n",
       " <tf.Operation 'range/delta' type=Const>,\n",
       " <tf.Operation 'range' type=Range>,\n",
       " <tf.Operation 'sub' type=Sub>,\n",
       " <tf.Operation 'floordiv' type=FloorDiv>,\n",
       " <tf.Operation 'mod' type=FloorMod>,\n",
       " <tf.Operation 'zeros_like' type=Const>,\n",
       " <tf.Operation 'NotEqual' type=NotEqual>,\n",
       " <tf.Operation 'Cast' type=Cast>,\n",
       " <tf.Operation 'add' type=AddV2>,\n",
       " <tf.Operation 'zeros_like_1' type=Const>,\n",
       " <tf.Operation 'Maximum' type=Maximum>,\n",
       " <tf.Operation 'while/maximum_iterations' type=Const>,\n",
       " <tf.Operation 'while/loop_counter' type=Const>,\n",
       " <tf.Operation 'while' type=StatelessWhile>,\n",
       " <tf.Operation 'Identity' type=Identity>]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A dynamic for loop using `tf.range` (captured by autograph)\n",
    "@tf.function\n",
    "def add_10(x):\n",
    "    for i in tf.range(10):\n",
    "        x = x + 1\n",
    "    return x\n",
    "\n",
    "add_10.get_concrete_function(tf.constant(0)).graph.get_operations()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "latter-concord",
   "metadata": {},
   "source": [
    "### Handling Variables and Other Resources in TF Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "complex-registrar",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(counter, c=1):\n",
    "    return counter.assign_add(c)\n",
    "\n",
    "increment(counter)\n",
    "increment(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "bibliographic-driving",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"counter\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function(counter).function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "great-submission",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = tf.Variable(0)\n",
    "\n",
    "@tf.function\n",
    "def increment(c=1):\n",
    "    return counter.assign_add(c)\n",
    "\n",
    "increment()\n",
    "increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "working-recorder",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name: \"assignaddvariableop_resource\"\n",
       "type: DT_RESOURCE"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function_def = increment.get_concrete_function().function_def\n",
    "function_def.signature.input_arg[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "choice-calvin",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=2>"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Counter:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.counter = tf.Variable(0)\n",
    "\n",
    "    @tf.function\n",
    "    def increment(self, c=1):\n",
    "        return self.counter.assign_add(c)\n",
    "\n",
    "c = Counter()\n",
    "c.increment()\n",
    "c.increment()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "ultimate-parliament",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "def tf__add(x):\n",
       "    with ag__.FunctionScope('add_10', 'fscope', ag__.ConversionOptions(recursive=True, user_requested=True, optional_features=(), internal_convert_user_code=True)) as fscope:\n",
       "        do_return = False\n",
       "        retval_ = ag__.UndefinedReturnValue()\n",
       "\n",
       "        def get_state():\n",
       "            return (x,)\n",
       "\n",
       "        def set_state(vars_):\n",
       "            nonlocal x\n",
       "            (x,) = vars_\n",
       "\n",
       "        def loop_body(itr):\n",
       "            nonlocal x\n",
       "            i = itr\n",
       "            x = (ag__.ld(x) + 1)\n",
       "        i = ag__.Undefined('i')\n",
       "        ag__.for_stmt(ag__.converted_call(ag__.ld(tf).range, (10,), None, fscope), None, loop_body, get_state, set_state, ('x',), {'iterate_names': 'i'})\n",
       "        try:\n",
       "            do_return = True\n",
       "            retval_ = ag__.ld(x)\n",
       "        except:\n",
       "            do_return = False\n",
       "            raise\n",
       "        return fscope.ret(retval_, do_return)\n",
       "\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def display_tf_code(func):\n",
    "    from IPython.display import display, Markdown\n",
    "\n",
    "    if hasattr(func, \"python_function\"):\n",
    "        func = func.python_function\n",
    "    \n",
    "    code = tf.autograph.to_code(func)\n",
    "    display(Markdown(f\"```python\\n{code}\\n```\"))\n",
    "\n",
    "# Display the code of the example function generated by autograph\n",
    "display_tf_code(add_10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fixed-leave",
   "metadata": {},
   "source": [
    "### Using TF Functions with tf.keras (or Not)\n",
    "\n",
    "By default, `tf.keras` will automatically convert your custom code into TF Functions, no need to use `tf.function()`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "cutting-criminal",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "346/363 [===========================>..] - ETA: 0s - loss: 2.8501 - my_mae: 1.2689Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "363/363 [==============================] - 2s 2ms/step - loss: 2.7755 - my_mae: 1.2455 - val_loss: 0.5569 - val_my_mae: 0.4819\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.4697 - my_mae: 0.4911 - val_loss: 0.4664 - val_my_mae: 0.4576\n",
      "162/162 [==============================] - 0s 675us/step - loss: 0.4164 - my_mae: 0.4639\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.41635245084762573, 0.4639027416706085]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Custom loss function\n",
    "def my_mse(y_true, y_pred):\n",
    "    print(\"Tracing loss my_mse()\")\n",
    "    return tf.reduce_mean(tf.square(y_pred - y_true))\n",
    "\n",
    "\n",
    "# Custom metric function\n",
    "def my_mae(y_true, y_pred):\n",
    "    print(\"Tracing metric my_mae()\")\n",
    "    return tf.reduce_mean(tf.abs(y_pred - y_true))\n",
    "\n",
    "\n",
    "# Custom layer\n",
    "class MyDense(keras.layers.Layer):\n",
    "    def __init__(self, units, activation=None, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.activation = keras.activations.get(activation)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        self.kernel = self.add_weight(\n",
    "            name='kernel', \n",
    "            shape=(input_shape[1], self.units),\n",
    "            initializer='uniform',\n",
    "            trainable=True,\n",
    "        )\n",
    "        self.biases = self.add_weight(\n",
    "            name='bias', \n",
    "            shape=(self.units,),\n",
    "            initializer='zeros',\n",
    "            trainable=True,\n",
    "        )\n",
    "        super().build(input_shape)\n",
    "\n",
    "    def call(self, X):\n",
    "        print(\"Tracing MyDense.call()\")\n",
    "        return self.activation(X @ self.kernel + self.biases)\n",
    "\n",
    "# Custom model\n",
    "class MyModel(keras.models.Model):\n",
    "    def __init__(self, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.hidden1 = MyDense(30, activation=\"relu\")\n",
    "        self.hidden2 = MyDense(30, activation=\"relu\")\n",
    "        self.output_ = MyDense(1)\n",
    "\n",
    "    def call(self, input):\n",
    "        print(\"Tracing MyModel.call()\")\n",
    "        hidden1 = self.hidden1(input)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = keras.layers.concatenate([input, hidden2])\n",
    "        output = self.output_(concat)\n",
    "        return output\n",
    "\n",
    "# Build custom model\n",
    "model = MyModel()\n",
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])\n",
    "\n",
    "# Train and evaluate the model\n",
    "model.fit(X_train_scaled, y_train, epochs=2, validation_data=(X_valid_scaled, y_valid))\n",
    "model.evaluate(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "spanish-hazard",
   "metadata": {},
   "source": [
    "One can turn this off by creating the model with `dynamic=True`. Alternatively, one can `compile` a model with `run_eagerly=True` to achieve the same effect."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "medium-austria",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n",
      "Tracing MyModel.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing MyDense.call()\n",
      "Tracing loss my_mse()\n",
      "Tracing metric my_mae()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.507260322570801, 2.0566811561584473]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Build the same model but prevent TF from automatically building and optimizing function graphs\n",
    "model = MyModel(dynamic=True)\n",
    "model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])\n",
    "\n",
    "# Train and evaluate the model\n",
    "model.fit(X_train_scaled[:64], y_train[:64], epochs=1, validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
    "model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continental-balance",
   "metadata": {},
   "source": [
    "Alternatively, one can compile a model with `run_eagerly=True` to achieve the same effect."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sized-bennett",
   "metadata": {},
   "source": [
    "## Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "verified-customs",
   "metadata": {},
   "source": [
    "### Implement a custom layer that performs *Layer Normalization*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "quiet-treasurer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=5.496049e-08>"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class LayerNormalization(keras.layers.Layer):\n",
    "    \n",
    "    def __init__(self, eps=0.001, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.eps = eps\n",
    "\n",
    "    def build(self, batch_input_shape):\n",
    "        input_shape = batch_input_shape[-1:]\n",
    "        self.alpha = self.add_weight(\"alpha\", shape=input_shape, initializer=\"ones\")\n",
    "        self.beta = self.add_weight(\"beta\", shape=input_shape, initializer=\"zeros\")\n",
    "        super().build(batch_input_shape)\n",
    "    \n",
    "    def call(self, X):\n",
    "        mean, variance = tf.nn.moments(X, axes=-1, keepdims=True)\n",
    "        # Note: Adding epsilon to variance is preferable to adding is to std\n",
    "        #  - training would bomb because the defivative of `sqrt(z)` is undefined for `z=0`\n",
    "        std = tf.sqrt(variance + self.eps)\n",
    "        return self.alpha * (X - mean) / std + self.beta\n",
    "    \n",
    "    def get_config(self):\n",
    "        return {**super().get_config(), \"eps\": self.eps}\n",
    "\n",
    "# Let's check that the custom layer produces similar output to the Keras layer\n",
    "\n",
    "X = X_train.astype(np.float32)\n",
    "\n",
    "custom_layer_norm = LayerNormalization()\n",
    "keras_layer_norm = keras.layers.LayerNormalization()\n",
    "\n",
    "def mean_impl_error(X, keras_layer, custom_layer):\n",
    "    impl_error = keras.losses.mean_absolute_error(keras_layer(X), custom_layer(X))\n",
    "    return tf.reduce_mean(impl_error)\n",
    "\n",
    "mean_impl_error(X, custom_layer_norm, keras_layer_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "stylish-hunter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=2.666699e-08>"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To be sure, let's test it again with alpha and beta completely random\n",
    "random_alpha = np.random.rand(X.shape[-1])\n",
    "random_beta = np.random.rand(X.shape[-1])\n",
    "\n",
    "custom_layer_norm.set_weights([random_alpha, random_beta])\n",
    "keras_layer_norm.set_weights([random_alpha, random_beta])\n",
    "\n",
    "mean_impl_error(X, custom_layer_norm, keras_layer_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "large-trick",
   "metadata": {},
   "source": [
    "### Train a model using a custom training loop to tackle the Fashion MNIST dataset\n",
    "1. Display the epoch, iteration, mean training loss, and mean accuracy over each epoch (updated at each iteration), as well as the validation loss and accuracy at the end of each epoch.\n",
    "1. Try using a different optimizer with a different learning rate for the upper layers and the lower layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "domestic-number",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca11a5a2b9c54a0cb394d41e706a2e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5cf7fb8c984490ab9be1e49e60bdf1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8f6a39e69604779bf50d327987be831",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d289a52e72f841398ccfe0ae3e2024c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53a3ad83241347db97b8479399a41ba7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3196efb04173413cb3a2101916f5dbed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import OrderedDict\n",
    "\n",
    "from tqdm.notebook import trange\n",
    "\n",
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Load, split and scale the Fashion MNIST dataset\n",
    "(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()\n",
    "\n",
    "max_value = 255.\n",
    "val_split = 5000\n",
    "input_shape = X_train[0].shape\n",
    "output_dim = len(np.unique(y_test))  # no. classes\n",
    "\n",
    "X_train_full = X_train_full.astype(np.float32) / max_value\n",
    "X_valid, X_train = X_train_full[:val_split], X_train_full[val_split:]\n",
    "y_valid, y_train = y_train_full[:val_split], y_train_full[val_split:]\n",
    "X_test = X_test.astype(np.float32) / max_value\n",
    "\n",
    "# Define hypeparameters\n",
    "hidden_dim = 100\n",
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "optimizer = keras.optimizers.Nadam(lr=0.01)\n",
    "loss_fn = keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.SparseCategoricalAccuracy()]\n",
    "\n",
    "\n",
    "# Build a simple sequential model with single ReLU hidden layer and Softmax output\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=input_shape),\n",
    "    keras.layers.Dense(hidden_dim, activation=\"relu\"),\n",
    "    keras.layers.Dense(output_dim, activation=\"softmax\"),\n",
    "])\n",
    "\n",
    "# Custom training loop\n",
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        \n",
    "        with trange(1, n_steps + 1, desc=f\"Epoch {epoch}/{n_epochs}\") as steps:\n",
    "            for step in steps:\n",
    "                \n",
    "                # Sample a batch\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                \n",
    "                # Feed the batch forward through the model and compute the batch losses\n",
    "                with tf.GradientTape() as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                    \n",
    "                # Evaluate gradients and make an optimization step\n",
    "                gradients = tape.gradient(loss, model.trainable_variables)\n",
    "                optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                \n",
    "                # Apply parameter trainsformations if these are defined\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))\n",
    "                        \n",
    "                status = OrderedDict()\n",
    "                \n",
    "                # Evaluate and record the total mean loss\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                \n",
    "                # Update metrics\n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                    \n",
    "                steps.set_postfix(status)\n",
    "                \n",
    "            # Compute validation prediction and validation accuracy\n",
    "            y_pred = model(X_valid)\n",
    "            acc = keras.metrics.sparse_categorical_accuracy(tf.constant(y_valid, dtype=np.float32), y_pred)\n",
    "            \n",
    "            # Record the mean validation loss and accuracy\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(acc)\n",
    "            \n",
    "            steps.set_postfix(status)\n",
    "        \n",
    "        # Reset metrics at the end of each epoch\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "organized-terry",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82b94ae287164c01beef073d86922ac6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "All epochs:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5aa2d09bdaf142f2ab8b35c4765fca9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b11363e5b8f4e4380897bda3d86ad4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1342fa3217ef4168b5c1f6f5d0fe3469",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a879aab6a7b14ea1aac8016138ca734e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cb39b6a22d44df4a3d461d4001047d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 5/5:   0%|          | 0/1718 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Reset TF session and RNG state\n",
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# Build a new model with the same structure but split into lower and upper layers\n",
    "lower_layers = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[28, 28]),\n",
    "    keras.layers.Dense(100, activation=\"relu\"),\n",
    "])\n",
    "\n",
    "upper_layers = keras.models.Sequential([\n",
    "    keras.layers.Dense(10, activation=\"softmax\"),\n",
    "])\n",
    "both_layers = [lower_layers, upper_layers]\n",
    "\n",
    "model = keras.models.Sequential(both_layers)\n",
    "\n",
    "# Define hyperparameters\n",
    "n_epochs = 5\n",
    "batch_size = 32\n",
    "n_steps = len(X_train) // batch_size\n",
    "loss_fn = keras.losses.sparse_categorical_crossentropy\n",
    "mean_loss = keras.metrics.Mean()\n",
    "metrics = [keras.metrics.SparseCategoricalAccuracy()]\n",
    "\n",
    "# Define separate optimizers for the lower and upper layer\n",
    "lower_optimizer = keras.optimizers.SGD(lr=1e-4)\n",
    "upper_optimizer = keras.optimizers.Nadam(lr=1e-3)\n",
    "optimizers = [lower_optimizer, upper_optimizer]\n",
    "\n",
    "# Custom training loop using two optimizers for lower and upper layers\n",
    "with trange(1, n_epochs + 1, desc=\"All epochs\") as epochs:\n",
    "    for epoch in epochs:\n",
    "        \n",
    "        with trange(1, n_steps + 1, desc=\"Epoch {}/{}\".format(epoch, n_epochs)) as steps:\n",
    "            for step in steps:\n",
    "                \n",
    "                # Sample a batch\n",
    "                X_batch, y_batch = random_batch(X_train, y_train)\n",
    "                \n",
    "                # Feed the batch forward through the model and compute the batch losses\n",
    "                with tf.GradientTape(persistent=True) as tape:\n",
    "                    y_pred = model(X_batch)\n",
    "                    main_loss = tf.reduce_mean(loss_fn(y_batch, y_pred))\n",
    "                    loss = tf.add_n([main_loss] + model.losses)\n",
    "                \n",
    "                # Evaluate gradients and make an optimization step in both optimizers\n",
    "                for layers, optimizer in zip(both_layers, optimizers):\n",
    "                    gradients = tape.gradient(loss, layers.trainable_variables)\n",
    "                    optimizer.apply_gradients(zip(gradients, layers.trainable_variables))\n",
    "                \n",
    "                # Manually release resources held by the autodiff\n",
    "                del tape\n",
    "                \n",
    "                # Apply parameter trainsformations if these are defined\n",
    "                for variable in model.variables:\n",
    "                    if variable.constraint is not None:\n",
    "                        variable.assign(variable.constraint(variable))                    \n",
    "                \n",
    "                status = OrderedDict()\n",
    "                \n",
    "                # Evaluate and record the total mean loss\n",
    "                mean_loss(loss)\n",
    "                status[\"loss\"] = mean_loss.result().numpy()\n",
    "                \n",
    "                for metric in metrics:\n",
    "                    metric(y_batch, y_pred)\n",
    "                    status[metric.name] = metric.result().numpy()\n",
    "                \n",
    "                steps.set_postfix(status)\n",
    "            \n",
    "            # Compute validation prediction and validation accuracy\n",
    "            y_pred = model(X_valid)\n",
    "            acc = keras.metrics.sparse_categorical_accuracy(tf.constant(y_valid, dtype=np.float32), y_pred)\n",
    "            \n",
    "            # Record the mean validation loss and accuracy\n",
    "            status[\"val_loss\"] = np.mean(loss_fn(y_valid, y_pred))\n",
    "            status[\"val_accuracy\"] = np.mean(acc)\n",
    "            \n",
    "            steps.set_postfix(status)\n",
    "        \n",
    "        # Reset metrics at the end of each epoch\n",
    "        for metric in [mean_loss] + metrics:\n",
    "            metric.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "flexible-basket",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}