gradient_descent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook copied from https://github.com/ChakriCherukuri/mlviz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Gradient Descent</h2>\n",
    "* Given a the multi-variable function $\\large {F(x)}$ differentiable in a neighborhood of a point $\\large a$\n",
    "* $\\large F(x)$ decreases fastest if one goes from $\\large a$ in the direction of the negative gradient of $\\large F$ at $\\large a$, $\\large -\\nabla{F(a)}$\n",
    "\n",
    "<h3>Gradient Descent Algorithm:</h3>\n",
    "* Choose a starting point, $\\large x_0$\n",
    "* Choose the sequence $\\large x_0, x_1, x_2, ...$ such that\n",
    "$ \\large x_{n+1} = x_n - \\eta \\nabla(F(x_n) $\n",
    "\n",
    "So convergence of the gradient descent depends on the starting point $\\large x_0$ and the learning rate $\\large \\eta$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import sleep\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from ipywidgets import *\n",
    "import bqplot.pyplot as plt\n",
    "from bqplot import Toolbar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = lambda x: np.exp(-x) * np.sin(5 * x)\n",
    "df = lambda x: -np.exp(-x) * np.sin(5 * x) + 5 * np.cos(5 *x) * np.exp(-x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(0.5, 2.5, 500)\n",
    "y = f(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_sol_path(x, y):\n",
    "    with sol_path.hold_sync():\n",
    "        sol_path.x = x\n",
    "        sol_path.y = y\n",
    "    \n",
    "    with sol_points.hold_sync():\n",
    "        sol_points.x = x\n",
    "        sol_points.y = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(x0, f, df, eta=.1, tol=1e-6, num_iters=10):\n",
    "    x = [x0]\n",
    "    i = 0\n",
    "    \n",
    "    while i < num_iters:\n",
    "        x_prev = x[-1]\n",
    "        grad = df(x_prev)\n",
    "        x_curr = x_prev - eta * grad\n",
    "        x.append(x_curr)\n",
    "        sol_lbl.value = sol_lbl_tmpl.format(x_curr)\n",
    "        sleep(.5)\n",
    "        \n",
    "        update_sol_path(x, [f(i) for i in x])\n",
    "        \n",
    "        if np.abs(x_curr - x_prev) < tol:\n",
    "            break\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "txt_layout = Layout(width='150px')\n",
    "x0_box = FloatText(description='x0', layout=txt_layout, value=2.4)\n",
    "eta_box = FloatText(description='Learning Rate', \n",
    "                    style={'description_width':'initial'}, \n",
    "                    layout=txt_layout, value=.1)\n",
    "\n",
    "go_btn = Button(description='GO', button_style='success', layout=Layout(width='50px'))\n",
    "reset_btn = Button(description='Reset', button_style='success', layout=Layout(width='100px'))\n",
    "\n",
    "sol_lbl_tmpl = 'x = {:.4f}'\n",
    "sol_lbl = Label()\n",
    "sol_lbl.layout.width = '300px'\n",
    "\n",
    "# plot of curve and solution\n",
    "fig_layout = Layout(width='720px', height='500px')\n",
    "fig = plt.figure(layout=fig_layout, title='Gradient Descent', display_toolbar=True)\n",
    "fig.pyplot = Toolbar(figure=fig)\n",
    "\n",
    "curve = plt.plot(x, y, colors=['dodgerblue'], stroke_width=2)\n",
    "sol_path = plt.plot([], [], colors=['#ccc'], opacities=[.7])\n",
    "sol_points = plt.plot([], [], 'mo', default_size=20)\n",
    "\n",
    "def optimize():\n",
    "    f.marks = [curve]\n",
    "    gradient_descent(x0_box.value, f, df, eta=eta_box.value)\n",
    "\n",
    "def reset():\n",
    "    curve.scales['x'].min = .4\n",
    "    curve.scales['x'].max = 2.5\n",
    "    \n",
    "    curve.scales['y'].min = -.5\n",
    "    curve.scales['y'].max = .4\n",
    "    sol_path.x = sol_path.y = []\n",
    "    sol_points.x = sol_points.y = []\n",
    "    sol_lbl.value = ''\n",
    "    \n",
    "go_btn.on_click(lambda btn: optimize())\n",
    "reset_btn.on_click(lambda btn: reset())\n",
    "\n",
    "final_fig = VBox([fig, fig.pyplot], \n",
    "                 layout=Layout(overflow_x='hidden'))\n",
    "HBox([final_fig, VBox([x0_box, eta_box, go_btn, reset_btn, sol_lbl])])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}