Practice extra 2 - SGD based Learning for Factorization Machines.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recommender Systems 2017/18\n",
    "\n",
    "### Practice 7 - SGD-based Learning for Factorization Machines\n",
    "\n",
    "S. Rendle, \"**Factorization Machines with libFM**,\" ACM Trans. Intell. Syst. Technol., 3(3), May 2012.\n",
    "\n",
    "##### The original version of this notebook is available at [Takuti Github repo](https://github.com/takuti/factorization-machines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from SGD_FM import Regression, Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(l, xlabel, ylabel):\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.plot(range(len(l)), l)\n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    ax.grid(True)\n",
    "    fig.set_size_inches((5.5,3))\n",
    "    fig.patch.set_alpha(0.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression: Mock dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([\n",
    "    [1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1],\n",
    "    [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0],\n",
    "    [1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0.5, 0.5],\n",
    "    [0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0.5, 0, 0.5],\n",
    "    [0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0],\n",
    "    [0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0],\n",
    "    [0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0.5, 0.5, 0]\n",
    "    ])\n",
    "y = np.array([5, 3, 1, 4, 5, 1, 5])\n",
    "\n",
    "n_sample = y.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWkAAADTCAYAAABDXeLpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt8XHWd//HXZ2ZyT9u0TRtKbylQ\nLpVKIQVhYbVF8YfgT7zgAruL4uKyurC6+1sVcHf9uf78KTzc1V1WHyorqOxPrW5xlUsVXWwBK5c2\nCL1QWnql6S1tmjZJ01xm5vP745yEaZo2l2YyZzLv5+Mxj5xz5syZ96TTz5x853u+X3N3REQkmmK5\nDiAiIiemIi0iEmEq0iIiEaYiLSISYSrSIiIRpiItIhJhKtIiIhGmIi0iEmEq0iIiEZbIdYChqq6u\n9tra2iE/7siRI1RUVIx8oFGQr9nzNTfkb/Z8zQ2Fl72+vv6Au08ZcEd3z6tbXV2dD8fy5cuH9bgo\nyNfs+ZrbPX+z52tu98LLDqz2QdQ8NXeIiESYirSISISpSIuIRJiKtIhIhBVUkX7mtf2s2NiY6xgi\nIoOWd13whuO5rU0809DNA798AYDt91yb40QiIoNTEEX6kZd38+imrlzHEBEZsoJo7phYXsSR7lyn\nEBEZuoIo0lVlxaQ1laOI5KHCKNLlRbmOICIyLAVSpItzHUFEZFgKokhP7HMmvbS+geDSeRGRaCuI\nIt23ueNT//kyL2w7mKM0IiKDVxBFekpl6XHbulLpHCQRERmagijS48uO7w7e1pHMQRIRkaEpiCJt\nZsdtO3RUHadFJPoKokj3p7ldVyCKSPQVTJGuq4kfs364XWfSIhJ9BVOk/+rCUu79wPze9QNtOpMW\nkegrmCIN0NaZ6l1+dW9LDpOIiAxO1oq0mc00s+VmtsHM1pvZJ/vZx8zsPjPbbGZrzOyibOWBY3t0\nvLq3lZYONXmISLRl80w6Cfytu58HXArcbmbz+uzzLmBueLsN+GYW81A9Lrg8/M8un0Mq7fzw+dez\n+XQiIqcsa+NJu/seYE+43GpmG4DpwCsZu10HPBROb/6cmVWZ2bTwsSPupotncdr4Uq48dyr/Wb+T\nvYc7svE0IiIjxkZjDAszqwWeBs5395aM7Y8B97j7b8P1J4E73X11n8ffRnCmTU1NTd2SJUuGnKGt\nrY3Kysre9TuePMIl0xJ8aF7JkI812vpmzxf5mhvyN3u+5obCy7548eJ6d1844I7untUbUAnUA+/v\n577HgSsy1p8E6k52vLq6Oh+O5cuXH7Ne939+5Xf/dM2wjjXa+mbPF/ma2z1/s+drbvfCyw6s9kHU\n0Kz27jCzIuBh4Afu/tN+dmkAZmaszwB2ZzNTj3jMSKU0Ep6IRFs2e3cY8ACwwd2/eoLdHgE+FPby\nuBQ47Flqj+4rbkZKw5WKSMRlcyLay4GbgbVm9lK47bPALAB3/xawDLgG2Ay0Ax/JYp5jxONGSnNq\niUjEZbN3x2+B40c2OnYfB27PVoaTiZuRVJEWkYgrqCsOM8VjRlpFWkQirmCLdCIWI5nWwP8iEm0F\nW6RjMUOTs4hI1BVskU7EjJTOpEUk4gq2SMdihrpJi0jUFWyR1pm0iOSDgi3ScVM/aRGJvsIt0jEV\naRGJPhVpEZEIU5EWEYmwwi7SGmBJRCKuoIt0Un3wRCTiCrZIJ2JGWmfSIhJxBVukYzGNgici0Vew\nRTqhLw5FJA8UbJHWxSwikg8Kt0jrTFpE8oCKtIhIhKlIi4hEWGEXaXXBE5GIK+wirYtZRCTiCrdI\nm86kRST6CrdIx3Uxi4hEX8EW6UTMSKtIi0jEZa1Im9mDZtZoZutOcP8iMztsZi+Ft89lK0t/4qYz\naRGJvkQWj/094OvAQyfZ5xl3f3cWM5xQPBZ8PqXTTixmuYggIjKgrJ1Ju/vTwMFsHf9UJeJBYe5K\naTJaEYku8yz2cDCzWuAxdz+/n/sWAQ8DDcBu4FPuvv4Ex7kNuA2gpqambsmSJUPO0tbWRmVlZe/6\nip3dfG99F19bVMbE0mg3zffNni/yNTfkb/Z8zQ2Fl33x4sX17r5wwB3dPWs3oBZYd4L7xgOV4fI1\nwGuDOWZdXZ0Px/Lly49Z/8Xa3T77zsd83a5DwzreaOqbPV/ka273/M2er7ndCy87sNoHUfNydgrp\n7i3u3hYuLwOKzKx6tJ5/UkUJAM1HukfrKUVEhixnRdrMTjMzC5cvCbM0jdbzT6ooAqDpSOdoPaWI\nyJBlrXeHmf0IWARUm1kD8L+BIgB3/xZwPfBxM0sCR4Ebwz8BRsUbZ9Jdo/WUIiJDlrUi7e43DXD/\n1wm66OXEhLIiqsqLeGDlNv7w7CmcOSU/v7AQkbEt2t0asigeM75y/QXsPdzBnUvX5DqOiEi/CrZI\nA1w1r4abL63llT0tuY4iItKvgi7SAFPHl9DelaKtM5nrKCIix1GRHhd8gdjY0pHjJCIix1ORHlcK\nQGOruuKJSPQUfJGuGR+cSb/e1J7jJCIixztpkTazKzOW5/S57/3ZCjWazpxSyYyJZfz85V25jiIi\ncpyBzqT/KWP54T73/f0IZ8mJWMz4o4UzWbm5ieUbG3MdR0TkGAMVaTvBcn/reeumS2YxdVwJn1m6\nhq6khi4VkegYqEj7CZb7W89bU8aV8IXr3sT+1k7WNBzKdRwRkV4DXRZ+hpk9QnDW3LNMuD7nxA/L\nP7MnVwCwr0W9PEQkOgYq0tdlLP9Tn/v6rue1nv7S+1vVX1pEouOkRdrdn8pcN7Mi4Hxgl7uPqW/Z\nJpYXk4iZ+kuLSKQM1AXvW2b2pnB5AvAywcSyvzezk45yl29iMaO6skTNHSISKQN9cfiH/sa8gx8B\nNrn7fKAO+ExWk+XA/BkTePjFBpa88Hquo4iIAAMX6cwR8a8Cfgbg7nuzliiHvvjeYL7cJ9aPyZcn\nInlooCJ9yMzebWYXApcDvwQwswRQlu1wo61mfCnXzD+NHbpEXEQiYqAi/RfAHcB3gb/OOIN+O/B4\nNoPlyuzJFexsbieZ0kUtIpJ7A/Xu2ARc3c/2J4AnshUql849bRzdKWfd7hYWzKzKdRwRKXAnLdJm\ndt/J7nf3T4xsnNx729lTiBksf7VRRVpEcm6gi1k+BqwDfgLsZgyN13EiVeXFTB1Xyq5DR4FgMoDO\nZJqZk8pznExECtFARXoa8EHgBiAJ/Bh42N2bsx0slyZVFLO0voE51RV85YmNAGy/59ocpxKRQnTS\nLw7dvcndv+Xui4FbgCpgvZndPBrhcqUjmQLoLdAiIrky0Jk0AGZ2EXATQV/pXwD12QyVay1H+5+U\nNpV29rV0cHrVmOt9KCIRNdBl4f9oZvXA/wKeAha6+63u/spABzazB82s0czWneB+M7P7zGyzma0J\nPwgioaIkfty2ju4Udz68hsvv/Q3fWL6ZVHrMjNQqIhE2UD/pfwAmABcAXwZeDAvqWjNbM8Bjv0c/\n3fcyvAuYG95uA745qMSj4IEPX8zHF515zLY9hztYWt+Ae9AM8uNVO3OUTkQKyUDNHcMeM9rdnzaz\n2pPsch3wkLs78JyZVZnZNHffM9znHClnTa3kzqvP5QMXTWfFxv188fENXHvfMwB85fo38+mla9jb\noiFNRST7BrqYZUd/280sDtwI9Hv/IE0HMk9HG8JtxxVpM7uN4GybmpoaVqxYMeQna2trG9bj2vcH\n7dPtXSmmVxqVhzZTGoeNW7axomj3kI83HMPNnmv5mhvyN3u+5gZlPyF3P+ENGA/cDXwdeCdBP+m/\nIijOPz/ZY8PH1wLrTnDf48AVGetPAnUDHbOurs6HY/ny5cN6XEd30v/tyU1+sK2zd9vCL/7a73p4\nzbCONxzDzZ5r+ZrbPX+z52tu98LLDqz2Aeqduw/Y3PEfQDPwLPBR4NNAMXCdu790ip8PDcDMjPUZ\nBBfMREpJIs4dV849ZltFcZz2rv57gIiIjKQB5zj0YPxozOw7wAFglru3jsBzPwLcYWZLgLcAhz0C\n7dGDUV6cYPX2ZpZvbGTxOVNzHUdExrCBend09yy4ewrYNtgCbWY/IjgDP8fMGszsVjP7mJl9LNxl\nGbAV2Az8O/CXQ06fIxUlcXYdOspHvrsq11FEZIwb6Ez6AjNrCZcNKAvXDXB3H3+iB7r7SafXCttk\nbh9K2KhIxN74bHN3zMb8kCYikiMD9e44/qoO4fWDb0wKcLQ7RXnxoC7cFBEZsoGaO6QfPSPkATS1\ndZ1kTxGRU6MiPQx1syf2Lje3q0iLSPaoSA/Dg7dczFf/6AIAmo6oSItI9qhID8OEsiIumTMJgD2H\ndHm4iGSPivQwnT6hjOJEjO1NR3IdRUTGMBXpYYrFjNrJ5WzdryItItmjIn0KZk2qoKG5feAdRUSG\nSUX6FJw2oYTtTUd4atP+nkGiRERGlIr0KagZV0pHd5oPP/gC33pqa67jiMgYpCJ9CqrHlfQu3/vL\nV9mniQBEZISpSJ+C4njw65s1qRyAHU1qnxaRkaVBJ07BexacTntXkotmT+Ta+37LnsNHB36QiMgQ\n6Ez6FBTFY9x8WW3vmfQnl7xE7V2PayZxERkxKtIjYFxp0THrW/a35SiJiIw1KtIj5O+uOa93edna\nvJhgRkTygIr0CLnhkjema/zOM9tIq8lDREaAivQIGV9axJ1Xn8tNl8yirTPJnn664x1q7+ID3/wd\n63YdzkFCEclHKtIj6OOLzuS6BacD8NDvttPa0c3S+obes+ofr9pJ/Y5mvvOMLnwRkcFRF7wRNu/0\n8dSML+HbT2/l208HxbjnkvGedbWEiMhgqUiPsPGlRay880r++debWLn5AGsaDvPppWsAOPe0cVSU\nxNncqN4fIjI4KtJZkIjHuPPqc4Ggp8eytXu45Q9quXDWRO75xQYeenYHqbQTj2mWcRE5ORXpLLtm\n/jSumT+td33u1HF0JtM0NLcze3JFDpOJSD7I6heHZna1mW00s81mdlc/999iZvvN7KXw9tFs5omC\nM6dWAvDavqDJo70ryfrdh3ltXys/fP51OrpTxz2mM+V8edkGGls1gJNIocnambSZxYFvAFcBDcAq\nM3vE3V/ps+uP3f2ObOWImrN6inRjG++YV8MXHn2FJat2kogZybRTv6OZfw4nue3x2JZuHt26lYqS\nBLdcXstTG/fz7jdPw0zNJSJjXTabOy4BNrv7VgAzWwJcB/Qt0gVlQlkRNeNLeK2xFYAnX20EIBl2\n+Xh0zW7+7/vOp60zyU9W72TT3lYe3doNwFd/vYmv/noTABfMqGLW5PIcvAIRGU2WrRlFzOx64Gp3\n/2i4fjPwlsyzZjO7BfgysB/YBPyNu+/s51i3AbcB1NTU1C1ZsmTIedra2qisrBzGKxl5/7y6g7UH\nUpw/Oc66phRXzU4wb3KcuMFX6ztZWBOnIwXrDqQoT8CsSufVQ8eeNX/+slJqJ8Rz9AoGJ0q/86HK\n1+z5mhsKL/vixYvr3X3hQPtl80y6v7/F+34iPAr8yN07zexjwPeBK497kPv9wP0ACxcu9EWLFg05\nzIoVKxjO47JhT/nr3P3TtaxrCtqfP/vBK5hTXUEq7eyytfz85V10dKe5al4NX//jC3n2t8+w3mew\naV8rH6ybyZ8+8Dznzl/ApWdMzvErObko/c6HKl+z52tuUPYTyWaRbgBmZqzPAHZn7uDuTRmr/w7c\nm8U8kXF93QzKi+PMnz6Blo4kc6qDXh7xmHHv9W/my++fz96WDiZXFlOSCM6Wb198FgAv7zwEQFtH\nMjfhRWRUZbNIrwLmmtkcYBdwI/DHmTuY2TR37xky7j3AhizmiYyieIzrFkw/4f2xmHF6VVm/91WU\nBP9kR7pUpEUKQdaKtLsnzewO4AkgDjzo7uvN7AvAand/BPiEmb0HSAIHgVuylWesqAyLdFunirRI\nIcjqxSzuvgxY1mfb5zKW7wbuzmaGsaaiJGj+ONKZpL0rSf2OZhbOnkRZcbS/RBSR4dEVh3mmojj4\nJ/vxqp18admrAHz6f5zDZWdOZubEcqori1m1vZn6Hc1cMmcSdbMn5jKuiJwiFek8EwvH+9iy/wgz\nJpbR0HyUrzyxEQj6YM+dWsnqHc0ATKoo5sV/uCpnWUXk1Gk86Tw0qaKYKeNK+M3fLuKiWVUA3PuB\n+dRWV/D6wXb+5C2z+PBlszl4pIujXcdfZi4i+UNn0nlo+acWUV4cpyge4/4PLaQ17MZ3w8WzevdZ\nWt8A7GBfSwe11RrISSRf6Uw6D00oK6IoHvzTVVeW9PazznTa+FIA9vUzjZeI5A+dSY9RNeNLAPjl\n+r08v+0gZ06p5F3nn8bS+gZ2HTrK31x1do4TishgqEiPUWdOqeSd82r47srtvdvedPp41u9uAYL5\nGEuL1G1PJOrU3DFGxWLGt2+u42e3X86X3jef0qIYrR1JqiuLAdhzWM0gIvlAZ9JjmJmxYGYVC2ZW\ncdMlMzEzntvaxI33P8eu5qP9tmWLSLToTLpA9EwQMD0cE+RPH3ieJ9bv5bmtTRzpTNLRneKm+59j\n2do9JzuMiIwynUkXmOlVZdywcCZPbdrPX/xHPQC1k8uZMbGcZ7c28ezWJj7/P+fRnXLOqqlk8TlT\nc5xYpLCpSBeYWDgcancqzcrNB3h60wEeXLmN7U3tABQnYnz+0Tcmz7n2zdP4+NvO5PzpE3IVWaSg\nqUgXqKJ4jEXnTGXROVO59s2nsbmxjXecV0NJUZzV2w9y+Gg33//ddh5fs4dfrN3DyruuZNqE/odP\nFZHsUZEW6mZPom72pN71RWETx3ULpvPkhn3c+v3VPL1p/zFXNIrI6FCRlpO68typzKmu4M6H1/KN\n5Vs40NbJnOoKzjltHLuaj3LVvBpuvWKOZi4XyRIVaTkpM+NrNyzgn57YSFtnMEbI77YcYPuBIxzp\nSvH8toO8tPMQTW1dVJQkOLumkgNtnVx6xmRKurMzybFIIVGRlgEtmFnF//voW3rXO7pTlCRidKXS\n3PydF3h+20H2t3YC8N8b9gHwk9UNVJcZpTP28fbzanKSW2QsUJGWIeu5nLwkEecnH7sMCAp3UTxG\ne1eS/96wj9ebjvJvT27i1u+vZnpVGQtmVfHXb5/L3JpxAKTSzubGNjY3tjF1fAkX10464fOJFDIV\naRkRPYV7XGkR77twBgBneQNrk9NYsbGRx9fs4cUdzXzi7XPZ3nSEn/1+F/taOnsf/8X3ns+fXjo7\nJ9lFokxFWrKmosi466pzuetd51K/4yAfeuAF7v7pWuIxY9HZU7jz6mnMmFjOR777An//s3UAXH5W\nNRXFcZ7bdpC3nT2FCWVFOX4VIrmlIi2jom72JF783FUcaOtiQllR76znAD/480t57zdW9hbqHvOn\nT+CyMydTWhRnckUxp1eVEY/BrEnlpB1aO7ppPtLN4aPdPLVpP9fXzeCtZ08Z7ZcmklUq0jJqShLx\n3rFDMi2YWcXLn3snLzUcoqmtk+b2btY2HGLZur281thKR3d6UMd/5OXdfOl985lUUcz40gRnTa3k\n8NFuGpqP0plMsb+1k5JEnF2HjmIGf7noLDqSKfYe7uDssK1cJGpUpCUSJpQX8bY+Z8Ffc8fMSKed\nLfvbaO1Msr+1k85kmphBZUmCieXFTCgr4lev7OXLv3iVz/7X2kE/57/892u9y9+95WK6U2mOdqf4\n/euHeHVbB//wwm9YfM5U5k+fwNL6Btzh/g/VUVVePGKvW2QgKtISWT0XyMRi1tsr5ERue+uZfPSK\nM9jedITm9i5aOpJs2NNCdWUJMyaWMb60iLLi4MvN6VVlvPcbKzl4pAsz2NfSyUe+t+qY440rhtau\nozz07A4AiuNBl8N/ffI1LpgRTP47fWIZFcUJYjGImYU3qChJ0N6VojuVpmZcKRPK1a4uw5fVIm1m\nVwP/CsSB77j7PX3uLwEeAuqAJuAGd9+ezUwydsVixhlTKnvXTzaC38/vuJyiWIxYzFjbcJiN+1qZ\nVFFEzfhSxpcWsWXNC5y/8DJWbj7A5IoSLj1jEtfc98wxM90MhhmUJGJUliR469lTKC+Oc6QzRWtH\nN22dSTq603Qm03SnwlsyTVcqTVcyTXfKmVBWxHsvnE5pUYx508YzY2I5ibhh4bHBMIOe6z33tAXH\nSaWdVNpJhj9TaacobhQnYhTHYyTiQxulOJ324Hl0Zemoy1qRNrM48A3gKqABWGVmj7j7Kxm73Qo0\nu/tZZnYjcC9wQ7YyifQoSbwxddj8GROYP+PYUf62EEzye92C6b3bfvjnl7L70FEqShK4O7sOdXC0\nK0naIe1O2sHdaWzppKq8iJKiOBv3tpBMOet3t/DsliY6ulNUlCQYV1rEuJIE40oTVCfiFCeM4niM\noniMorCQxsx4fO1uvvXUliG9trt/+4sB90nEgoJ9elUZqbQTM5g9uYLG1g7isRhFMcOBaRNKOdTe\nTf2OZkrCD4rSojilRUHWmBmzJ5eTSjtdqTTptJNKB78PD38nqZ7lcHuwHnTbnFQR/JVhGDt2dFHf\ntbH3Ayf8FMr4QAr267nLMnZLpp2uZM+HW/BB15kMPqyK4zFSaac77aTSwYdfKu10p9JBnowPspQT\nvgYn7cGH3KSKYs6cUkFX0ulMpnCgK5nGgKPdKdLu/PlZQ/onGpJsnklfAmx2960AZrYEuA7ILNLX\nAZ8Pl5cCXzczc3ddTyyRU11ZQnVlSe/6WVMH8WXjBaef0nP+3bXnkUo7jrO24TD7Wzt7i5wTfCgA\n9PyP+f3aV6ieXks8biRiRjwWI27BXxndqaAwdSXTtHelaGztoLUjSWlRnEPtXew53MHUccHrS6bT\ndHanWbfrMJWlCa6vm8HB9i72Hu7g8NFuOrpTdKeclo5uDrV3YxaMrBg3Ix4Lmn1isTeagDKbgyzc\np6Wjm7aOJD3/2dNph22bj3k9Q9HzwVOcCD5Agr8YjM7uNPGYURQ3EvEYiZgFWWPWe0vEYpQkjFjM\niBu922NmvLq3la37j1CSCP76iMeM8uKeC7pinF0zjrQfHfa/8UAsW/XQzK4Hrnb3j4brNwNvcfc7\nMvZZF+7TEK5vCfc50OdYtwG3AdTU1NQtWbJkyHna2tqorKwceMcIytfs+Zob8jf7aOd2d5ygCJ+q\nE2Xv/SA6ZhvHbOv5IMiV4fzeFy9eXO/uCwfaL5tn0v39xvp+IgxmH9z9fuB+gIULF/qiRYuGHGbF\nihUM53FRkK/Z8zU35G/2fM0Nyn4i2ZzjsAGYmbE+A9h9on3MLAFMAA5mMZOISF7JZpFeBcw1szlm\nVgzcCDzSZ59HgA+Hy9cDv1F7tIjIG7LW3OHuSTO7A3iCoAveg+6+3sy+AKx290eAB4D/MLPNBGfQ\nN2Yrj4hIPspqP2l3XwYs67PtcxnLHcAHs5lBRCSfZbO5Q0RETlHWuuBli5ntB3YM46HVwIEB94qm\nfM2er7khf7Pna24ovOyz3X3AYRvzrkgPl5mtHkyfxCjK1+z5mhvyN3u+5gZlPxE1d4iIRJiKtIhI\nhBVSkb4/1wFOQb5mz9fckL/Z8zU3KHu/CqZNWkQkHxXSmbSISN5RkRYRibCCKNJmdrWZbTSzzWZ2\nV67zZDKzB82sMRy2tWfbJDP7tZm9Fv6cGG43M7svfB1rzOyi3CUHM5tpZsvNbIOZrTezT+ZDfjMr\nNbMXzOzlMPc/htvnmNnzYe4fh2POYGYl4frm8P7aXOTOZGZxM/u9mT0Wrkc+u5ltN7O1ZvaSma0O\nt0X6vZKRvcrMlprZq+H7/bLRyj7mi7S9MUPMu4B5wE1mNi+3qY7xPeDqPtvuAp5097nAk+E6BK9h\nbni7DfjmKGU8kSTwt+5+HnApcHv4u416/k7gSne/AFgAXG1mlxLMDPS1MHczwcxBkDGDEPC1cL9c\n+ySwIWM9X7IvdvcFGX2Ko/5e6fGvwC/d/VzgAoLf/ehk93Bqm7F6Ay4DnshYvxu4O9e5+mSsBdZl\nrG8EpoXL04CN4fK3gZv62y8KN+DnBNOl5U1+oBx4EXgLwRVjib7vG4JBwi4LlxPhfpbDzDPConAl\n8BjBuOyRzw5sB6r7bIv8ewUYD2zr+3sbrexj/kwamA7szFhvCLdFWY277wEIf/bMqBrZ1xL+GX0h\n8Dx5kD9sLngJaAR+TTCt4SF3T/aTrTd3eP9hYPLoJj7GvwCfAdLh+mTyI7sDvzKz+nC2JciD9wpw\nBrAf+G7YxPQdM6tglLIXQpEe1OwveSKSr8XMKoGHgb9295aT7drPtpzkd/eUuy8gOCu9BDivv93C\nn5HJbWbvBhrdvT5zcz+7Ri47cLm7X0TQHHC7mb31JPtGKXcCuAj4prtfCBzhjaaN/oxo9kIo0oOZ\nISZq9pnZNIDwZ2O4PXKvxcyKCAr0D9z9p+HmvMnv7oeAFQRt6lUWzBAEx2aL0gxClwPvMbPtwBKC\nJo9/IQ+yu/vu8Gcj8F8EH4758F5pABrc/flwfSlB0R6V7IVQpAczQ0zUZM5Y82GCtt6e7R8Kvz2+\nFDjc8+dWLpiZEUzcsMHdv5pxV6Tzm9kUM6sKl8uAdxB8EbScYIYgOD53JGYQcve73X2Gu9cSvJd/\n4+5/QsSzm1mFmY3rWQbeCawj4u8VAHffC+w0s3PCTW8HXmG0sueiIT4HDf/XAJsI2h3/Ltd5+mT7\nEbAH6Cb4BL6VoM3wSeC18OekcF8j6KmyBVgLLMxx9isI/oxbA7wU3q6Jen7gzcDvw9zrgM+F288A\nXgA2A/8JlITbS8P1zeH9Z+T6fRPmWgQ8lg/Zw3wvh7f1Pf8Po/5eyci/AFgdvmd+Bkwcrey6LFxE\nJMIKoblDRCRvqUiLiESYirSISISpSIuIRJiKtIhIhKlIy5hgZm3hz1oz++MRPvZn+6z/biSPL3Iy\nKtIy1tQCQyrS4UiJJ3NMkXb3PxhiJpFhU5GWseYe4A/DMYv/JhxI6Stmtioc2/cvAMxskQVjYf+Q\n4IIDzOxn4eA/63sGADKze4Cy8Hg/CLf1nLVbeOx14TjJN2Qce0XG+MM/CK/OFBmyxMC7iOSVu4BP\nufu7AcJie9jdLzazEmClmf0q3PcS4Hx33xau/5m7HwwvFV9lZg+7+11mdocHgzH19X6CK9EuAKrD\nxzwd3nch8CaCMRtWEoy58dsmH0XkAAABIElEQVSRf7ky1ulMWsa6dxKMo/ASwTCqkwkGYwd4IaNA\nA3zCzF4GniMYIGcuJ3cF8CMPRtTbBzwFXJxx7AZ3TxNcLl87Iq9GCo7OpGWsM+Cv3P2JYzaaLSIY\ncjJz/R0EA+S3m9kKgnEvBjr2iXRmLKfQ/zUZJp1Jy1jTCozLWH8C+Hg4pCpmdnY4CltfEwimmWo3\ns3MJhi7t0d3z+D6eBm4I272nAG8lGMRIZMTo013GmjVAMmy2+B7B3HS1wIvhl3f7gff287hfAh8z\nszUE0x09l3Hf/cAaM3vRg2FBe/wXwVRVLxOMBvgZd98bFnmREaFR8EREIkzNHSIiEaYiLSISYSrS\nIiIRpiItIhJhKtIiIhGmIi0iEmEq0iIiEfb/AZFbOyGJE3T+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xaf2dcf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# learn incrementally\n",
    "regressor = Regression(X, y)\n",
    "\n",
    "RMSEs = []\n",
    "for i in range(600):\n",
    "    # update with randomly sampled one\n",
    "    j = np.random.randint(n_sample)\n",
    "    RMSE = regressor.update(X[j], y[j])\n",
    "    RMSEs.append(RMSE)\n",
    "    \n",
    "plot(RMSEs, 'Iteration', 'RMSE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAADTCAYAAACY5kjLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VPW5x/HPM5NJQlYSEkLCFhAE\ng+yIVlBwgaJttYtWbevVim31XrW1tVXb2lrb3lpt7XVprUsRta6t1VpqVWRREZFFQdmEsCmyhD0k\nkG3muX+cExzjZCUnZyZ53q/Xec2Zs8x8OZonJ2fO/B5RVYwxxiSegN8BjDHGtI0VcGOMSVBWwI0x\nJkFZATfGmARlBdwYYxKUFXBjjElQVsCNMSZBWQE3xpgEZQXcGGMSVJLfAdpTXl6eFhcXt3q/yspK\n0tPT2z9QguYAy9IYyxKbZYmtLVmWLVu2W1XzW7SxqnaaaezYsdoW8+bNa9N+7S1ecqhalsZYltgs\nS2xtyQIs1RbWPLuEYowxCcoKuDHGJCgr4MYYk6CsgBtjTILqVHehtFZpWQV/mlfK+IyI31GMMabV\nuvQZeDii/OOdj9h4wAq4MSbxdOkCfkx+OilJAT4oD/sdxRhjWq1LF/CkYIChvTLZUm5n4MaYxONZ\nAReRGSJSJiIrG1n/QxFZ7k4rRSQsIrnuus0i8p67bqlXGQFKirL44GAEtd6gxpgE4+UZ+ExgWmMr\nVfV2VR2lqqOAG4FXVXVv1CanuevHeZiRkqJsKmvho/2HvXwbY4xpd54VcFV9Ddjb7IaOi4AnvMrS\nlGFFWQCs3lbux9sbY0ybiZeXDkSkGJilqsc3sU0asBUYVH8GLiKbgH2AAvep6v1N7P9t4NsABQUF\nY5988slWZawOK1fMruScY5L50uDkVu3b3ioqKsjIyPA1Qz3LEptlic2yxNaWLKeddtqyFl95aOmg\nKW2ZgGJgZTPbXAD8q8GyIvexJ7ACOLUl79fWwaxOuuXfOn3mkjbt254SfRAer1iW2CxLbImehQQb\nzOpCGlw+UdVt7mMZ8Cww3ssA/bMCrN52wMu3MMaYdudrAReRbGAS8M+oZekiklk/D0wFYt7J0l76\nZQXYdqCKfZU1Xr6NMca0K8++Si8iTwCTgTwR2Qr8HAgBqOqf3c2+BLysqpVRuxYAz4pIfb7HVfVF\nr3IC9MsMArWs3l7OhEF5Xr6VMca0G88KuKpe1IJtZuLcbhi9bCMw0ptUsfXPcv4QWbXtgBVwY0zC\niIdr4L7LTBYKs1NZZbcSGmMSiBVw17CiLLsX3BiTUKyAu0oKs9iwq4LDNTawlTEmMVgBd5UUZRNR\nWLvDzsKNMYnBCrir/iv1dh3cGJMorIC7+uR0Iys1idXbrYAbYxKDFXCXiFBSlGVn4MaYhGEFPMqw\nomzWbi+nLmwNHowx8c8KeJRhRVlU10XYuLuy+Y2NMcZnVsCjlBz5INMGtjLGxD8r4FGOyc8gOSlg\nX+gxxiQEP3tiThaRA1F9MX8WtW6aiLwvIqUicoNXGRsKuU2O7YNMY0wi8K0nput1dftiquotACIS\nBP4InAWUABeJSImHOT9hmHsnilqTY2NMnIuXnpjRxgOlqrpRVWuAJ4Fz2zVcE0oKszhwuNaaHBtj\n4p5vPTFFZDLwDE4/zG3Adaq6SkTOA6ap6uXudhcDJ6rqVY28x1H1xIRP9q0r3RfmV29Vcc3oFMYU\neDbabrM5/GZZYrMssVmW2DptT0wgC8hw588G1rvz5wMPRm13MXB3S96vrT0xo/vWVVbXavENs/SO\nl99v02sdjUTv5ecVyxKbZYkt0bOQCD0xVbVcVSvc+ReAkIjk4ZyR943atA/OGXqHSEtOYmBeun2Q\naYyJe74VcBHpJW7fNBEZ72bZAywBBovIABFJxml6/HxHZispyrYmx8aYuOdnT8zzgCtFpA44DFzo\n/vlQJyJXAS8BQWCGqq7yKmcsw4qy+NeKbeyrrCEnPbkj39oYY1rMt56YqnoPcE8j614AXvAiV0vU\nDy1rTY6NMfHMvokZQ0mhfaXeGBP/rIDH0CMjhV5Z1uTYGBPfrIA3wpocG2PinRXwRgwrsibHxpj4\nZgW8ESVFWUQUVm+36+DGmPhkBbwRJw7oQUBg3tpdfkcxxpiYrIA3Iic9mROKc5m9eqffUYwxJiYr\n4E2YUlLA+zsPsmWPtVgzxsQfK+BNmFrSC8DOwo0xcckKeBP69UhjaK9MXrYCboyJQ1bAmzGlpICl\nm/eyt7LG7yjGGPMJVsCbMaWkgIjCnDV2Fm6MiS9+NjX+uoi8604LRWRk1LrNIvKe2+x4qVcZW2J4\n72x6ZaXadXBjTNzxs6nxJmCSqo4Afgnc32D9aeo0O25ZayGPiAhTSgp4ff1uqmrtW5nGmPjhW1Nj\nVV2oqvvcp4twOu/EpSklBRyuDbNg/W6/oxhjzBG+NTVusN11wFD9uJHxJmAfoMB9qtrw7Dx633Zt\nahxLXUS5eu4hxhUkMX14Sqtfv71ydCTLEptlic2yxNZpmxpHbXMasAboEbWsyH3sCawATm3J+7VH\nU+PGXPX42zrmlpe1Lhxp03u0V46OYllisyyxWZbYOm1TYwARGQE8CJyrqnvql6vqNvexDHgWGO9P\nwo9NKSlgT2UN73ywr/mNjTGmA/jZ1Lgf8A/gYlVdF7U8XUQy6+eBqUDMO1k60uQh+YSCYl/qMcbE\nDS9vI3wCeBMYIiJbRWS6iFwhIle4m/wM6AH8qcHtggXAAhFZASwG/q2qL3qVs6WyUkOcNLAHs1fv\nrL/MY4wxvvKzqfHlwOUxlm8ERn56D/9NLSngpn+uYsOuCgb1zPQ7jjGmi7NvYrbCmSUFALy0yi6j\nGGP8ZwW8FQqzuzGiT7Z9K9MYExesgLfSlOMKWP7hfsrKq/yOYozp4qyAt9KUYc5llNk2uJUxxmdW\nwFtpSEEm/XLT7DKKMcZ3VsBbqX5wq4Wle6iorvM7jjGmC7MC3gZnD+9FTTjCM8u2+h3FGNOFWQFv\ngzH9chjXP4d752+gus6GmDXG+MMKeBuICN89czA7yqv421I7CzfG+MMKeBtNHJTH6H7duXf+Bmrq\nIn7HMcZ0QU0WcBE5PWp+QIN1X/YqVCIQEa45YzAf7T/MM2/bWbgxpuM1dwb+u6j5Zxqs+2lzL96C\nvpgiIneJSKnbG3NM1LpLRGS9O13S3Hv5YfKx+Yzsk80f55VSG7azcGNMx2qugEsj87GexzKTpvti\nngUMdqdvA/cCiEgu8HPgRJyxwH8uIjkteL8OVX8WvnXfYZ59+yO/4xhjupjmCrg2Mh/r+ad3bqYv\nJnAu8IjbiGIR0F1ECoHPArNVda86fTNn0/QvAt+cPrQnx/fO4p55pdTZWbgxpgM12RNTRPYDr+Gc\nbZ/izuM+n6iqzZ4VN9UXU0RmAbeq6gL3+RzgemAykKqqv3KX3wQcVtXfxXgNz3tiNuftnXXc9U41\n3xqezITeoTa/TqL38vOKZYnNssSW6Fla0xOzufHAz42ab1g8P1VM2yDWZRhtYvmnFzoNj+8HGDdu\nnE6ePLnVIebPn09b9qs3SZXZOxbwyvYwN1w0iWCgJVeX2j9He7IssVmW2CxLbF5nafISiqq+Gj0B\nC4FyYI37/GhtBfpGPe8DbGtieVwSEb57xiA27a7kXyviNqYxppNp7jbCP4vIMHc+G6dD/CPAOyLS\nZMedFnoe+C/3bpSTgAOquh14CZgqIjnuh5dT3WVxa2pJL4YUZHL33PWEI9ZyzRjjveY+xDxFVVe5\n898E1qnqcGAs8KPmXrwFfTFfADYCpcADwH8DqOpe4JfAEne6xV0WtwIB4eozBrFhVyUvvLfd7zjG\nmC6guWvgNVHzU4C/AajqDpHmr/O2oC+mAv/TyLoZwIxm3ySOnH18IYN7rufuuev53PBCAm28Fm6M\nMS3R3Bn4fhH5vIiMBiYALwKISBLQzetwicY5Cx/Mup0VzFy42e84xphOrrkC/h3gKuAh4HuqusNd\nfgbwby+DJaovjCjk9KE9ufXFtazdUe53HGNMJ9bcXSjrVHWaqo5S1ZlRy19S1R94ni4BiQi3nTeC\nrNQkvvfkcqpqbbhZY4w3mrwGLiJ3NbVeVa9p3zidQ15GCrefP5JvPrSE2158n599ocTvSMaYTqi5\nDzGvAFYCT+Pch22fyrXQaUN6cunJxcx4YxOThuQz6dh8vyMZYzqZ5q6BF+J8y/GzwMVACHheVR9W\n1Ye9DpfobjhrKMcWZHDd31awp6La7zjGmE6muWvge1T1z6p6GnAp0B1YJSIXd0S4RJcaCvJ/F4zm\nwKFarn/mPZoad8YYY1qrRR153HG6vwd8A/gPsMzLUJ1JSVEWP5o2hFfW7OTxxR/4HccY04k091X6\nX4jIMuD7wKvAOFWdrqqrOyRdJ3HZhAGcMjiPX85aTWlZhd9xjDGdRHNn4DcB2cBI4DfA227nnPdE\n5F3P03USgYDwu/NH0i0U5HtPvWO3Fhpj2kVzd6EMaGa9aaGCrFR++5URfPvRZVz71HLu+dqYNg87\na4wx0PyHmFtiTTjDvU7smIidx9Rhvfjp547jPyt3cNM/V9qHmsaYo9LcNfAsEblRRO4RkanusK9X\n44wg+NXmXlxEponI+27T4htirP+DiCx3p3VuB6D6deGodc+35R8Xjy4/ZSBXTDqGx9/6gD+8st7v\nOMaYBNbcJZRHgX04Q8JeDvwQSAbOVdXlTe0oIkHgjzijGG4FlojI89EfgKrqtVHbXw2MjnqJw6o6\nqhX/loRx/bQh7Kmo5q4568nPSObizxT7HckYk4CaK+AD3fG/EZEHgd1AP1U92ILXHg+UqupGd/8n\ncVq0NXYHy0U4neg7PRHhN18ezr5DNfzs+VXkpCfz+RFFfscyxiSY5poav62qYxp73uQLi5wHTFPV\ny93nFwMnqupVMbbtDywC+qhq2F1WBywH6nAaHz/XyPv43tS4rarDyu+XVrFhf4Tvj02lf+rhhG7G\n6hXLEptliS3Rs7SmqTGq2ugEhHF6YJYDB3GKaf18eTP7ng88GPX8YuDuRra9vuE6oMh9HAhsBo5p\n6v1UlbFjx2pbzJs3r037tYf9lTU69Y5XteSm/+iM517xLUdDfh6ThixLbJYltkTPAizVZmpd/dTc\nXShBVc1yp0xVTYqaz2rmd0NrGhNfCDzR4L23uY8bgfl88vp4p5GdFuKR6ePpnpbMHcuqWLPdxhA3\nxrRMi75K30ZLgMEiMkBEknGK9KfuJhGRIUAOzgel9ctyRCTFnc/D6QbUab/9WZCVyqPTx5Mkwlfv\ne5Mlm+O6/acxJk54VsBVtQ6nm89LwBrgaVVdJSK3iMg5UZteBDzp/ulQ7zhgqYisAObhXAPvtAUc\nYGB+Bj85KZX8jBQu/stbzFtb5nckY0yca+4ulKOiqi/gdJ6PXvazBs9vjrHfQmC4l9niUV63AE9f\n8RkufWgx33pkKb87fyRfHN3b71jGmDjl5SUU0wZ5GSk88a2TGNs/h+89tZyZb2zyO5IxJk5ZAY9D\nmakhHr5sPFNKCrj5X6v5w+x19rV7Y8ynWAGPU6mhIPd+fQznje3DnXPW8/PnVxGOWBE3xnzM02vg\n5ugkBQPc9pUR5KSFeOD1TWzZc4i7LhxNdlrI72jGmDhgZ+BxLhAQfvK5Ev73S8NZuGE35/5xAet3\ntmQkA2NMZ2cFPEF87cR+PP6tk6ioDvOlPy1k9uqdfkcyxvjMCngCOaE4l39dPYGB+el865Gl3DVn\nPRG7Lm5Ml2UFPMEUZnfj6e98hi+P7s0ds9fx34+9TWV1nd+xjDE+sAKegFJDQX7/1ZH89HPH8fLq\nHXz5TwspLbPr4sZ0NVbAE5SIcPkpA3n4svHsrqjm83cv4PG3PrD7xY3pQqyAJ7hTBufzn++ewgnF\nufz42fe48q9vs/9Qjd+xjDEdwNMC3oKemJeKyK6o3peXR627RETWu9MlXuZMdD2zUnn4m+P58dlD\nmbN2J2fd+TqLNu7xO5YxxmOeFfConphnASXARSJSEmPTp1R1lDs96O6bi9Ne7USc1mw/F5Ecr7J2\nBoGA8O1Tj+EfV04gNRTkogcW8fuX36c2HPE7mjHGI16egR/piamqNUB9T8yW+CwwW1X3quo+YDYw\nzaOcncrwPtnMunoi543pw91zS/nqfW9SWlbhdyxjjAea7Il5VC/cgp6YInIp8BtgF7AOuFZVPxSR\n64BUVf2Vu91NOF3qfxfjfRK2J6bXOd7aXscjq6upDsMXjwkxbUCIpID4kuVoWJbYLEtsiZ6l3Xpi\nHs1EC3piAj2AFHf+CmCuO/9D4KdR290E/KC590zEnpjRvMhRVl6lV/51qfa/fpaefedr+t7W/b5l\naSvLEptliS3Rs9BePTGPUrM9MVV1j6pWu08fAMa2dF/TMvmZKfzp62P58zfGsLO8mnP/+Aa3vbiW\nqtqw39GMMUfJ156YIlIY9fQcnNZr4LRhm+r2xswBprrLTBtNO76QOd+fxJdH9+ZP8zdw9l2vs9R6\nbxqT0PzuiXmNiKxye19eA1zq7rsX+CXOL4ElwC3uMnMUstNC3H7+SB65bDzVtRHO+/ObXPe3FZQd\nrPI7mjGmDXztiamqNwI3NrLvDGCGl/m6qlOPzefla0/l7rml/GXBRl5cuYPvnjGYS04uJjnJvttl\nTKKwn9YuKj0liRvOGsrL105i/IBcfv3CGqbd+RqvrtvldzRjTAtZAe/iBuSlM+PSE5hx6ThU4ZIZ\ni7n84aWUHbIvABkT76ylmgHg9KEFTBiUx0NvbObuOeuZtzbM6rpVXHX6IPIyUvyOZ4yJwc7AzREp\nSUGumHQMc6+bzMTeSTy6aAun3jaPO15+n/KqWr/jGWMasAJuPqUgK5VvHp/C7GtP5bShPblrbimn\n3jaP+1/bYPePGxNHrICbRg3Mz+CPXxvDrKsnMrJPd/73hbVMvn0+j721heo6K+TG+M0KuGnW8b2z\nefiy8Tz57ZMo6p7KT55dyaTb5vOXBZs4VGPt3IzxixVw02InDezBM1eezKPTx1Ocl8YvZ61mwq1z\nuWvOeg4csmvkxnQ0uwvFtIqIcMrgfE4ZnM+yLfu4d34pd8xex32vbuAbJ/Vn+sQB9MxK9TumMV2C\nFXDTZmP75/DgJSewdkc5987fwAOvb+ShNzbzuRGFXHJyMaP6dvc7ojGdmhVwc9SG9srizgtH8/0p\nx/LQG5v5+7KtPPvOR4zs251LT+7P2cMLSUkK+h3TmE7HroGbdtO/Rzo3nzOMRT8+g1+cM4yDVbVc\n+9QKJtzq3Eu+s9wGzTKmPfnd1Pj7IrJaRN4VkTki0j9qXTiq2fHzDfc18SsjJYlLTi7mlWsn8chl\n4xnZJ5u755Vy8q1zmT5zCS+u3EFNnX1V35ij5dkllKimxlNwGjQsEZHnVXV11GbvAONU9ZCIXAnc\nBlzgrjusqqO8yme8FwgIpx6bz6nH5rNlTyVPLvmQZ5ZtZc7aMnLTk/nS6N6cP64PQ3tl+R3VmITk\n5TXwI02NAUSkvqnxkQKuqvOitl8EfMPDPMZH/Xukc/20ofxgyrG8vn43Ty/9kEfe3MxfFmxiRJ9s\nvjKmD2cPLyQ/08ZdMaalfG1q3GD7e4Ad+nEj4zpgOVAH3KqqzzWynzU19kBHZDlYo7y5rY7XP6rj\nw4MRBBiaG+DEwiTGFiSRmSwdlqWlLEtsliU2r5sae3kGHqv9eczfFiLyDWAcMClqcT9V3SYiA4G5\nIvKeqm741Auq3g/cDzBu3DidPHlyq4POnz+ftuzX3uIlB3Rcli/gNNZ+f+dBZq3Yzqx3tzFz1SEe\nXVPLhEF5fGFEIWm1pV3uuLSEZYmtK2XxsoC3qDGxiJwJ/ASYFNXgGFXd5j5uFJH5wGjgUwXcJD4R\nYWivLIb2yuIHU49l1bZyZr3rFPMf/v1dggKPbV7EmccVcOZxBfTrkeZ3ZGPigpcF/EhTY+AjnKbG\nX4veQERGA/fhXGopi1qeAxxS1WoRyQMm4HzAaTo5EeH43tkc3zub66cNYcXWA9z/wmLWH6zmllmr\nuWXWagb3zODMkgLOPK4no/rmEAzE+mPPmM7PswKuqnUiUt/UOAjMqG9qDCxV1eeB24EM4G8iAvCB\nqp4DHAfcJyIRnFsdb21w94rpAkSEUX2789UhyUyePIkteyp5ZU0Zc9bs5IHXNnLv/A3kpIU4+Zg8\nJg7OY+KgPPrm2tm56Tr8bmp8ZiP7LQSGe5nNJJ7+PdKZPnEA0ycO4MDhWl5dt4tX39/FgtJd/Pu9\n7e42aUwc5BTzzxzTg+5pyT6nNsY79lV6k5Cyu4U4Z2QR54wsQlXZsKuC19fv5o3S3Tz3zkc89tYH\nAAwpyOSEATmcUJzLCcW5FHXv5nNyY9qPFXCT8ESEQT0zGdQzk29OGEBtOMLyD/fz1sY9LN68j+fe\n2cZfFzkFvXf3bpxQnMPY4lxG9enOkF6ZJCfZiBImMVkBN51OKBg4csYNUBeOsHbHQRZv2svSLXtZ\nULqH55Y7N0QlJwUoKcxiVN/ujOiTzci+3RnQI52AfTBqEoAVcNPpJQUDR+5suWziAFSVrfsOs2Lr\nflZ8uJ8VWw/w1JIPmblwM+CM5TK0VybHFWZxXGEWJUVZDCnIpFuyjaho4osVcNPliAh9c9Pom5vG\n50cUARCOKKVlFaz4cD8rtx1gzfZynn3nIx5dtAWAgEBxXjo9glW8XbuOwT0zGNQzgwF56aSGrLAb\nf1gBNwYIBoQhvTIZ0iuTr7rfP4tEnDP11dvLWeNOKzYfYtnc9UTc7xQHBPrlpjGoZybH5KdTnJdO\n/x5pDMhLpyAz1S7FGE9ZATemEYGA0K9HGv16pDHt+F6A89Xokyacwqbdlawvq6C0rILSsoOs31nB\na+t2URP+eJjc1FCA/rlOQe/fI40+OWn0yel25DE9xX78zNGx/4OMaaXUUPDI9fFo4Yiybf9htuw5\nxKY9lWzZXcnmPZVs2FXBq+t2Ud1gDPSctBB9ctIo6p5KYXY3emWnUpjtzBdmp9IzK8U6GZkmWQE3\npp0EAx9fW584OO8T61SV3RU1bN13iK37DrvTIT7cd5hNuytZWLqHg9V1n3rN3PRk8jNS6JmVQn5G\nCvnuY8+sVD7aE6Zwx0Fy05PJSQuRFLTbIbsaK+DGdAARIT8zhfzMFEb3y4m5zcGqWnaWV7H9gDvt\nr6LsYBVlB6vZdbCajbsq2XWw+hOXaX675DX39aF7txA9MlLITU8mNy2ZnPQQ2d2c4p6Tlkz3tBA5\n6clkdwuRlRoiu1uI1FAAdxgLk4CsgBsTJzJTQ2SmhhjUM7PRbVSVA4drKTtYzZwFi+k7+Dj2VNSw\np7KGvZXVR+Y37Kpg/we17D9UQ2248TH/Q0EhKzVEVrcQWalJboYkMlKSyEhNIrP+MTVEWnKQjJQk\n0pKTSE8Jkp6SRLo7H/Gor4BpmqcFXESmAXfiDGb1oKre2mB9CvAIMBbYA1ygqpvddTcC04EwcI2q\nvuRlVmMSgYjQPS2Z7mnJbOsRZLJ7G2RjVJXKmjD7Kms4cLiWfYecx/LDdc5jVS3lh2s54E4V1XXs\nLK+iorqOiqo6KmrqaGltTn7lP3RLDpKWHKRbcpBuIWc+NeTMp4aCpIYCUfPOlJIUICUUICXJnU8K\nkOxO9cuSkwIkBz9eHgo624WCgS49GqXfPTGnA/tUdZCIXAj8FrhAREpwhp8dBhQBr4jIsaoa9iqv\nMZ2RiDhn0ylJnxicv6UiEaWypo6DVXUcqqmjsjpMpft45Hl1HavXldKzd18O14Q5XBPmUK376O67\n62A1VbVhqmojVNU56xp+qNtWAXG+fZscDBBKChCpqyV90VxCQSEpGCApIISCAZKCQijgPAbrlwWE\npKCQFHDmg+7zYMBZ5jwKgfpH+eTzoLvMeXTuXAqKfPxY5W3zbl97YrrPb3bn/w7cI84FuXOBJ90G\nD5tEpNR9vTc9zGuMaSAQkCOXdpoyP/IBkycf16rXjkSUmnCE6roI1XVhqmudxyr3sbouQk39FP70\nfG1YqQ1HqA1HL4vwwdZt5PXMpS6s1EWc7erCEeoiH29/qMZZ52zjrA+rEnafhyMfP9aGI0TUmY+0\n8krRdeO87fHqZQHvDXwY9XwrcGJj27jjhx8AerjLFzXYt3esN2nQE5P58+e3OmhFRUWb9mtv8ZID\nLEtjLEtsXmZJdqeYgu4UnSWployM/a18FyF2F8hPUnWKeFhB3ceIQgTnecRdH1Gnf2RS7SFP/xv5\n3ROzsW1a3E/TemJ6w7LEZllisyyxeZ3FyxtHW9IT88g2IpIEZAN7W7ivMcZ0aV4W8CM9MUUkGedD\nyecbbPM8cIk7fx4wV1XVXX6hiKS4PTUHA4s9zGqMMQnH756YfwEedT+k3ItT5HG3exrnA8864H/s\nDhRjjPkkv3tiVgHnN7Lvr4Ffe5nPGGMSmQ2eYIwxCUq0E30FVkR2AVvasGsesLud47RFvOQAy9IY\nyxKbZYmtLVn6q2p+SzbsVAW8rURkqaqOsxwfsyyxWZbYLEtsXmexSyjGGJOgrIAbY0yCsgLuuN/v\nAK54yQGWpTGWJTbLEpunWewauDHGJCg7AzfGmARlBdwYYxJUly7gIjJNRN4XkVIRucHnLJtF5D0R\nWS4iSzv4vWeISJmIrIxalisis0VkvfsYu5Fjx2S5WUQ+co/NchE5uwNy9BWReSKyRkRWich33eUd\nflyayOLHcUkVkcUissLN8gt3+QARecs9Lk+54x/5lWWmiGyKOi6jvM4SlSkoIu+IyCz3ubfHRVW7\n5IQzPssGYCDOcMMrgBIf82wG8nx671OBMcDKqGW3ATe48zcAv/Uxy83AdR18TAqBMe58JrAOKPHj\nuDSRxY/jIkCGOx8C3gJOAp4GLnSX/xm40scsM4HzOvK4RGX6PvA4MMt97ulx6cpn4Ec6BqlqDVDf\nMajLUdXXcAYTi3Yu8LA7/zDwRR+zdDhV3a6qb7vzB4E1OE1FOvy4NJGlw6mjwn0acicFTsfpqgUd\nd1way+ILEekDfA540H0ueHxcunIBj9UxyJcfCpcCL4vIMrfLkN8KVHU7OAUE6OlznqtE5F33EkuH\nXM6pJyLFwGicMzxfj0uDLOBepEsHAAAEDElEQVTDcXEvEywHyoDZOH/J7lfVOneTDvtZaphFVeuP\ny6/d4/IHt3l6R/g/4Ec4DXrA6S7m6XHpygW8xV1/OsgEVR0DnAX8j4ic6mOWeHMvcAwwCtgO/L6j\n3lhEMoBngO+panlHvW8Ls/hyXFQ1rKqjcBqtjAdiNcPskJ+lhllE5HjgRmAocAKQC1zvdQ4R+TxQ\npqrLohfHitye79uVC3hcdf1R1W3uYxnwLM4Php92ikghgPtY5lcQVd3p/qBGgAfooGMjIiGcgvmY\nqv7DXezLcYmVxa/jUk9V9wPzca47d3e7aoEPP0tRWaa5l5xUnaboD9Exx2UCcI6IbMa5HHs6zhm5\np8elKxfwlnQM6hAiki4imfXzwFRgZdN7eS66W9IlwD/9ClJfMF1fogOOjXv98i/AGlW9I2pVhx+X\nxrL4dFzyRaS7O98NOBPnmvw8nK5a0HHHJVaWtVG/YAXnmrPnx0VVb1TVPqpajFNL5qrq1/H6uPjx\nSW28TMDZOJ/obwB+4mOOgTh3wawAVnV0FuAJnD/Ba3H+MpmOc/1uDrDefcz1McujwHvAuzgFtLAD\nckzE+XP3XWC5O53tx3FpIosfx2UE8I77niuBn0X9P7wYKAX+BqT4mGWue1xWAn/FvVOloyZgMh/f\nheLpcbGv0htjTILqypdQjDEmoVkBN8aYBGUF3BhjEpQVcGOMSVBWwI0xJkFZATedmohUuI/FIvK1\ndn7tHzd4vrA9X9+Y5lgBN11FMdCqAi4iwWY2+UQBV9WTW5nJmKNiBdx0FbcCp7jjQ1/rDoJ0u4gs\ncQc9+g6AiEx2x95+HOfLIIjIc+4gY6vqBxoTkVuBbu7rPeYuqz/bF/e1V4ozxvsFUa89X0T+LiJr\nReQx99uCxrRJUvObGNMp3IAzdvbnAdxCfEBVT3BHq3tDRF52tx0PHK+qm9znl6nqXvfr2ktE5BlV\nvUFErlJnIKWGvowzwNRIIM/d5zV33WhgGM6YGG/gjKGxoP3/uaYrsDNw01VNBf7LHYr0LZyvyA92\n1y2OKt4A14jICmARzgBog2naROAJdQaa2gm8ijMyXv1rb1VnAKrlOJd2jGkTOwM3XZUAV6vqS59Y\nKDIZqGzw/EzgM6p6SETmA6kteO3GVEfNh7GfQXMU7AzcdBUHcdqR1XsJuNIdphUROdYdCbKhbGCf\nW7yH4gydWq+2fv8GXgMucK+z5+O0iVvcLv8KY6LYb3/TVbwL1LmXQmYCd+Jcvnjb/SBxF7HbXb0I\nXCEi7wLv41xGqXc/8K6IvK3O0KH1ngU+gzO6pAI/UtUd7i8AY9qNjUZojDEJyi6hGGNMgrICbowx\nCcoKuDHGJCgr4MYYk6CsgBtjTIKyAm6MMQnKCrgxxiSo/wePxEEZkEUIIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xaf2dd30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# learn at once\n",
    "# 1 iteration = descent for all samples\n",
    "regressor = Regression(X, y)\n",
    "RMSEs = regressor.fit()\n",
    "plot(RMSEs, 'Iteration', 'RMSE')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification: Based on the mock dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1, -1, -1,  1,  1, -1,  1])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# give labels for the mock dataset\n",
    "y_labels = np.ones_like(y)\n",
    "y_labels[y < np.mean(y)] = -1\n",
    "y_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAADTCAYAAACY5kjLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGMxJREFUeJzt3X+UX3V95/HnKxMygUwigcgcmyCB\nblSorSAjkVK7g/wwstZsxVZoa7GlJ9s9YrusbjfseqDiekpP2XbtloNGm/XHcUlZtG7W5hQpZsQf\noEkwCSQIBBQzRDAQJN8vMJP5Tt77x71f+DKZGfL9cb937p3X45zvme/99b2fT26+r7nzufd+PooI\nzMyseObkXQAzM2uNA9zMrKAc4GZmBeUANzMrKAe4mVlBOcDNzArKAW5mVlAOcDOzgnKAm5kV1Ny8\nC9ApS5YsieXLlze93XPPPceCBQs6X6AZpOx1dP2Kr+x1bKZ+27ZteyoiXn0065YmwJcvX87WrVub\n3m5oaIjBwcHOF2gGKXsdXb/iK3sdm6mfpMeO9nPdhGJmVlAOcDOzgsoswCWtl/QzSfdPsVyS/lbS\nHkk7Jb25YdkVkh5OX1dkVUYzsyLL8gz8c8CqaZa/E1iRvtYANwNIOgG4DlgJnANcJ2lxhuU0Myuk\nzC5iRsRdkpZPs8pq4AuRdEh+j6TjJb0GGATuiIgDAJLuIPlFcEtWZbXO2bhjH996aH9X9/nEE6P8\n0/4dXd1nN5W9flDeOv6nVa/npIXzM/v8PO9CWQrsbZgeTudNNf8IktaQnL3T39/P0NBQ04WoVqst\nbVck3azjX377efY/H/TNU1f2BxBxmF1PP961/XVb2esH5a3jWxY8zUnHzcnsO5hngE/2DY9p5h85\nM2IdsA5gYGAgWrkNqey3L0F363j47jtZfdYS/uq33tSV/UH5j2HZ6wflr2NW9cvzLpRh4OSG6WXA\nvmnmWwFURmosnH9M3sUwmxXyDPCNwO+nd6O8FXg2In4K3A5cLGlxevHy4nSezXDjh4PqaI2++aV5\nPsxsRsvsmybpFpILkkskDZPcWXIMQER8CtgEXALsAZ4H/iBddkDSx4Et6UddX7+gaTPbc4dqACxy\ngJt1RZZ3oVz+CssD+OAUy9YD67Mol2WnMpIE+EIHuFlX+ElM65hqGuB9vW4DN+sGB7h1TGVkDPAZ\nuFm3OMCtY+pNKL6IadYdDnDrmMqoL2KadZMD3DrmpSYUt4GbdYMD3DrmpYuYPgM36wYHuHVMZaTG\nHMFx83ryLorZrOAAt46pjIzR1zsXqXsdWZnNZg5w65jKqPtBMesmB7h1TNKRldu/zbrFAW4dU3WA\nm3WVA9w6pjI65iYUsy5ygFvHVEZqvoXQrIsc4NYxbkIx6y4HuHWMR+Mx6y4HuHXEaG2cQ+OHfQZu\n1kUOcOsID+Zg1n0OcOuIivtBMes6B7h1RPXFM3C3gZt1iwPcOsKj8Zh1nwPcOqI+mIObUMy6J9MA\nl7RK0oOS9khaO8nyUyTdKWmnpCFJyxqWjUvanr42ZllOa1+9DXyRm1DMuiaz0yVJPcBNwEXAMLBF\n0saI2N2w2o3AFyLi85LeDvwF8P502QsRcWZW5bPOqqZNKB4P06x7sjwDPwfYExGPRsQhYAOwesI6\nZwB3pu83T7LcCsK3EZp1X5bftqXA3obpYWDlhHV2AJcCnwR+E1go6cSIeBqYL2krUANuiIivTtyB\npDXAGoD+/n6GhoaaLmS1Wm1puyLpRh13PXyIeXPgO9+6K9P9TKbsx7Ds9YPy1zGr+mUZ4JMNyxIT\npj8C/J2kDwB3AY+TBDbAayNin6TTgG9Iui8iHnnZh0WsA9YBDAwMxODgYNOFHBoaopXtiqQbdbz9\nwH0seurJXP4ty34My14/KH8ds6pflgE+DJzcML0M2Ne4QkTsA94DIKkPuDQinm1YRkQ8KmkIOAt4\nWYDbzFEZGWORm0/MuirLNvAtwApJp0qaB1wGvOxuEklLJNXLcA2wPp2/WFJvfR3gPKDx4qfNMNXR\nmi9gmnVZZgEeETXgKuB24AHg1ojYJel6Se9OVxsEHpT0ENAPfCKdfzqwVdIOkoubN0y4e8VmGA+n\nZtZ9mX7jImITsGnCvGsb3t8G3DbJdt8FfjnLsllnVUbGWNK3IO9imM0qfhLTOqLqvsDNus4Bbh3h\nJhSz7nOAW9sOHw6qh2osdD8oZl3lALe2PXeoRoS7kjXrNge4te3FwRzchGLWVQ5wa1t11P2gmOXB\nAW5te2kwBzehmHWTA9za5vEwzfLhALe2vTSYgwPcrJsc4NY2X8Q0y4cD3NpWHXUbuFkeHODWtspI\nDQkWzOvJuyhms4r/5i25Jw+O8ONnx7lv+NnM9vHY08/T1zsXabIxPMwsKw7wEosILvrrb3JwpAZ3\nfzvTfZ22xD0RmnWbA7zEXhgb5+BIjbctncsVF5yZ6b5+8aS+TD/fzI7kAC+x+t0hp71qDhee0Z9z\nacys03wRs8TqAX7sXLdNm5WRA7zE6o+4H+u7+8xKyQFeYj4DNys3B3iJ1XsJdICblVOmAS5plaQH\nJe2RtHaS5adIulPSTklDkpY1LLtC0sPp64osy1lW9SaU43yp2qyUMgtwST3ATcA7gTOAyyWdMWG1\nG4EvRMSvANcDf5FuewJwHbASOAe4TtLirMpaVvUmlPk+AzcrpSzPwM8B9kTEoxFxCNgArJ6wzhnA\nnen7zQ3L3wHcEREHIuIZ4A5gVYZlLaWX2sBzLoiZZSLLr/ZSYG/D9DDJGXWjHcClwCeB3wQWSjpx\nim2XTtyBpDXAGoD+/n6GhoaaLmS1Wm1puyJ4YM8o83vg+eeeK20dodzHEMpfPyh/HbOqX5YBPtnf\n7TFh+iPA30n6AHAX8DhQO8ptiYh1wDqAgYGBGBwcbLqQQ0NDtLJdEWx6agfHP/MUfX09pa0jlPsY\nQvnrB+WvY1b1yzLAh4GTG6aXAfsaV4iIfcB7ACT1AZdGxLOShoHBCdsOZVjWUqqM1NJxKo/43Wdm\nJZBlG/gWYIWkUyXNAy4DNjauIGmJpHoZrgHWp+9vBy6WtDi9eHlxOs+aUB2teZAFsxLLLMAjogZc\nRRK8DwC3RsQuSddLene62iDwoKSHgH7gE+m2B4CPk/wS2AJcn86zJhwcqXmQBbMSy/T0LCI2AZsm\nzLu24f1twG1TbLuel87IrQWVkTGWHX9s3sUws4z4ScwSq77YBm5mZeQAL7GKA9ys1KYMcEnvkPTe\nSeb/rqSLsi2Wtas2fpgXxsbp63UbuFlZTXcG/jHgm5PMv5PksXebweodWfkM3Ky8pgvw4yJi/8SZ\nEfEE4AEQZ7j6Y/S+jdCsvKYL8PmSjvj2SzoG8K0NM1w9wBc5wM1Ka7oA/wrwGUkvnm2n7z+VLrMZ\nrN6VrO8DNyuv6QL8o8CTwGOStkm6F/gxsD9dZjNYvQ28r9dn4GZlNeW3O32Scq2kjwH/Kp29JyJe\n6ErJrC31JpSF8+fyTM5lMbNsTBngkt4zYVYAx0vaHhGVbItl7ao3ofgipll5Tfft/o1J5p0A/Iqk\nKyPiGxmVyTqgMlq/iOk2cLOymq4J5Q8mmy/pFOBWjhycwWaQykiNY3pE71w/bGtWVk1/uyPiMcCn\ndTNcdaRGX+9cJI+HaVZWTQe4pDcAoxmUxTqoMjLmWwjNSm66i5j/jyOHcjkBeA3we1kWytpXSc/A\nzay8pvuG3zhhOoADJCH+e8DdWRXK2lcZdU+EZmU33UXMFzuyknQm8DvAbwM/Ar6cfdGsHZWRGks9\nmINZqU3XhPI6knEsLweeBv4BUESc36WyWRuqo2MsnL8w72KYWYam+xv7h8C3gN+IiD0Akq7uSqms\nbR7Mwaz8prsL5VLgCWCzpM9IugDwPWkFEBG+iGk2C0wZ4BHxjxHxPuANwBBwNdAv6WZJF3epfNaC\nkbHDjB8O30ZoVnKveB94RDwXEV+KiHcBy4DtwNqj+XBJqyQ9KGmPpCO2kfRaSZsl/UDSTkmXpPOX\nS3pB0vb09akm6zWrvdSVrM/AzcqsqW94RBwAPp2+piWpB7gJuAgYBrZI2hgRuxtW+yhwa0TcLOkM\nYBOwPF32SESc2Uz5LFHxcGpms0KWHWWcQ9L97KMRcQjYAKyesE4Ai9L3rwL2ZVieWaOxK1kzKy9F\nTHzYskMfnIxovyoi/iidfj+wMiKualjnNcDXgcUk42xeGBHbJC0HdgEPAQeBj0bEtybZxxpgDUB/\nf//ZGzZsaLqc1WqVvr6+prebye5/apwbt45wzTnzef0JPaWsYyPXr/jKXsdm6nf++edvi4iBo1o5\nIjJ5Ab8FfLZh+v3A/5ywzn8EPpy+PxfYTfJXQS9wYjr/bGAvsGi6/Z199tnRis2bN7e03Uy2aee+\nOOU/fy1273s2IspZx0auX/GVvY7N1A/YGkeZs1k2oQwDJzdML+PIJpIrSbqmJSLuBuYDSyJiNCKe\nTudvAx4BXpdhWUvFTShms0OWAb4FWCHpVEnzSJ7q3DhhnZ8AFwBIOp0kwPdLenV6ERRJpwErgEcz\nLGupvHgRs9e3EZqVWWanaBFRk3QVcDvQA6yPiF2Srif5E2Ej8GGSke+vJrmg+YGICEm/DlwvqQaM\nA38cyR0wdhQ8nJrZ7JDpNzwiNpHcGtg479qG97uB8ybZ7su4w6yWVUZqHDevh545fnDWrMw83lYJ\nVd0Pitms4AAvocqoR+Mxmw0c4CXkjqzMZgd/y9u0Y+/P+fjXdlM7nM0DUa14+MkKbz5lcd7FMLOM\nOcDb9O09T7H1sWd424olM2YE+LOXn8B7z16WdzHMLGMO8DYdHBlj3tw5fPHKlXkXxcxmGbeBt6k6\nUmOh25vNLAcO8DZ56DIzy4sDvE2VkTE/8WhmuXCAt6k6WnOfI2aWCwd4m9yEYmZ5cYC3qTJScxOK\nmeXCAd6mysgYi/zYupnlwAHehoigOurH1s0sHw7wNjx/aJzD4ZFvzCwfDvA2vDR0mZtQzKz7HOBt\nqI565Bszy48DvA0HPXiwmeXIAd6GF5tQfBHTzHLgAG9D1W3gZpYjB3gb6qO/uwnFzPKQaYBLWiXp\nQUl7JK2dZPlrJW2W9ANJOyVd0rDsmnS7ByW9I8tytqo6mpyB+yKmmeUhs+SR1APcBFwEDANbJG2M\niN0Nq30UuDUibpZ0BrAJWJ6+vwz4JeAXgH+R9LqIGM+qvK2oX8Tsm+cAN7Puy/IM/BxgT0Q8GhGH\ngA3A6gnrBLAoff8qYF/6fjWwISJGI+JHwJ7082aUysgYfb1zmTNnZgylZmazS5anjkuBvQ3Tw8DE\nccf+HPi6pA8BC4ALG7a9Z8K2SyfuQNIaYA1Af38/Q0NDTReyWq22tB3Anh+PMk/jLW/fLe3UsQhc\nv+Irex2zql+WAT7ZaenEodsvBz4XEf9d0rnAFyW98Si3JSLWAesABgYGYnBwsOlCDg0N0cp2ABv2\nbmNJrcrg4L9uaftuaaeOReD6FV/Z65hV/bIM8GHg5IbpZbzURFJ3JbAKICLuljQfWHKU2+bOHVmZ\nWZ6ybAPfAqyQdKqkeSQXJTdOWOcnwAUAkk4H5gP70/Uuk9Qr6VRgBfD9DMvaksrImO8BN7PcZHb6\nGBE1SVcBtwM9wPqI2CXpemBrRGwEPgx8RtLVJE0kH4iIAHZJuhXYDdSAD860O1AAKqM1lp1wXN7F\nMLNZKtO//yNiE8mtgY3zrm14vxs4b4ptPwF8IsvytasyUmOR7wE3s5z4Scw2uAnFzPLkAG/R2Phh\nRsYO+yKmmeXGAd6iqruSNbOcOcBb9GI/KD4DN7OcOMBbdPDFngjdBm5m+XCAt6jiJhQzy5kDvEVu\nAzezvDnAW1QZdROKmeXLAd6i+hm4L2KaWV4c4C3yiPRmljcHeIsqIzWO6RG9c/1PaGb5cPq0qDqa\nPEYveTQeM8uHA7xFlZGam0/MLFcO8BZVRzyYg5nlywHeIp+Bm1neHOAtOjgyRl+v7wE3s/w4wFtU\nHfVgDmaWLwd4i9yEYmZ5c4C3ICKSEekd4GaWIwd4C14YG2f8cLgfFDPLVaYBLmmVpAcl7ZG0dpLl\nfyNpe/p6SNLPG5aNNyzbmGU5m1VxPyhmNgNklkCSeoCbgIuAYWCLpI3pSPQARMTVDet/CDir4SNe\niIgzsypfO9wXuJnNBFkm0DnAnoh4FEDSBmA1sHuK9S8HrsuwPEfYe+B5vvP4GAfuHW5yuxcAWOQm\nFDPLkSIimw+W3gusiog/SqffD6yMiKsmWfcU4B5gWUSMp/NqwHagBtwQEV+dZLs1wBqA/v7+szds\n2NBUGe/eV+PTO0eb2qbRfzvvWJYtnPmXEarVKn19fXkXIzOuX/GVvY7N1O/888/fFhEDR7Nulmfg\nk/XyNNVvi8uA2+rhnXptROyTdBrwDUn3RcQjL/uwiHXAOoCBgYEYHBxsqoADozV+8fi7WLlyZVPb\nARw7r4eTFs5vers8DA0N0ey/TZG4fsVX9jpmVb8sA3wYOLlhehmwb4p1LwM+2DgjIvalPx+VNETS\nPv7IkZu2rq93LicdN4dTTlzQyY81M+uKLP/+3wKskHSqpHkkIX3E3SSSXg8sBu5umLdYUm/6fglw\nHlO3nZuZzUqZnYFHRE3SVcDtQA+wPiJ2Sboe2BoR9TC/HNgQL2+MPx34tKTDJL9kbmi8e8XMzLJt\nQiEiNgGbJsy7dsL0n0+y3XeBX86ybGZmRTfzb6EwM7NJOcDNzAoqs/vAu03SfuCxFjZdAjzV4eLM\nNGWvo+tXfGWvYzP1OyUiXn00K5YmwFslaevR3jRfVGWvo+tXfGWvY1b1cxOKmVlBOcDNzArKAZ4+\nil9yZa+j61d8Za9jJvWb9W3gZmZF5TNwM7OCcoCbmRXUrA7wVxryrWgknSxps6QHJO2S9Kfp/BMk\n3SHp4fTn4rzL2g5JPZJ+IOlr6fSpkr6X1u8f0s7TCkvS8ZJuk/TD9FieW6ZjKOnq9P/n/ZJukTS/\n6MdQ0npJP5N0f8O8SY+ZEn+b5s5OSW9udb+zNsAbhnx7J3AGcLmkM/ItVdtqwIcj4nTgrcAH0zqt\nBe6MiBXAnel0kf0p8EDD9F8Cf5PW7xngylxK1TmfBP45It4AvImkrqU4hpKWAn8CDETEG0k6uruM\n4h/DzwGrJsyb6pi9E1iRvtYAN7e601kb4DQM+RYRh4D6kG+FFRE/jYh70/cVki/+UpJ6fT5d7fPA\nv82nhO2TtAz4N8Bn02kBbwduS1cpev0WAb8O/D1ARByKiJ9TomNI0onesZLmAscBP6XgxzAi7gIO\nTJg91TFbDXwhEvcAx0t6TSv7nc0BvhTY2zA9nM4rBUnLSQbB+B7QHxE/hSTkgZPyK1nb/gfwZ8Dh\ndPpE4OcRUUuni34cTwP2A/8rbSb6rKQFlOQYRsTjwI3AT0iC+1lgG+U6hnVTHbOOZc9sDvBmhnwr\nFEl9wJeB/xARB/MuT6dIehfws4jY1jh7klWLfBznAm8Gbo6Is4DnKGhzyWTSduDVwKnALwALSJoU\nJiryMXwlHfs/O5sDvJkh3wpD0jEk4f2liPhKOvvJ+p9o6c+f5VW+Np0HvFvSj0mavN5OckZ+fPrn\nOBT/OA4DwxHxvXT6NpJAL8sxvBD4UUTsj4gx4CvAr1KuY1g31THrWPbM5gA/qiHfiiRtD/574IGI\n+OuGRRuBK9L3VwD/t9tl64SIuCYilkXEcpLj9Y2I+F1gM/DedLXC1g8gIp4A9qZDDQJcQDKcYCmO\nIUnTyVslHZf+f63XrzTHsMFUx2wj8Pvp3ShvBZ6tN7U0LSJm7Qu4BHiIZLDk/5p3eTpQn18j+VNs\nJ7A9fV1C0k58J/Bw+vOEvMvagboOAl9L358GfB/YA/wfoDfv8rVZtzOBrelx/CrJmLGlOYbAx4Af\nAvcDXwR6i34MgVtI2vTHSM6wr5zqmJE0odyU5s59JHfktLRfP0pvZlZQs7kJxcys0BzgZmYF5QA3\nMysoB7iZWUE5wM3MCsoBbqUmqZr+XC7pdzr82f9lwvR3O/n5Zq/EAW6zxXKgqQBPe6yczssCPCJ+\ntckymbXFAW6zxQ3A2yRtT/uj7pH0V5K2pH0y/zsASYNpn+r/m+QhCyR9VdK2tA/rNem8G0h61Nsu\n6UvpvPrZvtLPvl/SfZLe1/DZQw19fX8pfRrRrCVzX3kVs1JYC3wkIt4FkAbxsxHxFkm9wHckfT1d\n9xzgjRHxo3T6DyPigKRjgS2SvhwRayVdFRFnTrKv95A8TfkmYEm6zV3psrOAXyLp++I7JP27fLvz\n1bXZwGfgNltdTNIfxXaSLndPJOlgH+D7DeEN8CeSdgD3kHRCtILp/RpwS0SMR8STwDeBtzR89nBE\nHCbp6mB5R2pjs5LPwG22EvChiLj9ZTOlQZIuXBunLwTOjYjnJQ0B84/is6cy2vB+HH8HrQ0+A7fZ\nogIsbJi+Hfj3afe7SHpdOnDCRK8CnknD+w0kQ9XVjdW3n+Au4H1pO/urSUbY+X5HamHWwL/9bbbY\nCdTSppDPkYw7uRy4N72QuJ/Jh/H6Z+CPJe0EHiRpRqlbB+yUdG8k3drW/SNwLrCDpHfIP4uIJ9Jf\nAGYd494IzcwKyk0oZmYF5QA3MysoB7iZWUE5wM3MCsoBbmZWUA5wM7OCcoCbmRXU/wcNAqkPlRHO\nmwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb16e128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# learn incrementally\n",
    "classifier = Classification(X, y_labels)\n",
    "\n",
    "AUCs = []\n",
    "for i in range(100):\n",
    "    # update with random sample\n",
    "    j = np.random.randint(n_sample)\n",
    "    current = classifier.update(X[j], y_labels[j])\n",
    "    AUCs.append(current)\n",
    "\n",
    "plot(AUCs, 'Iteration', 'AUC')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Let's look at the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from abc import ABCMeta, abstractmethod\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "\n",
    "class SGD:\n",
    "    __metaclass__ = ABCMeta\n",
    "\n",
    "    def __init__(self, X, y, l2_reg_w0=0.01, l2_reg_w=0.01, l2_reg_V=0.01, learn_rate=0.01):\n",
    "        self.X, self.y = X, y\n",
    "        self.n, self.p = X.shape # number of samples and their dimension\n",
    "\n",
    "        # parameter settings\n",
    "        self.k = int(np.sqrt(self.p)) # for the low-rank assumption of pairwise interaction\n",
    "        self.l2_reg_w0 = l2_reg_w0\n",
    "        self.l2_reg_w = l2_reg_w\n",
    "        self.l2_reg_V = l2_reg_V\n",
    "        self.learn_rate = learn_rate\n",
    "\n",
    "        # initialize the model parameters\n",
    "        # we have the weight vector and the pairwise interaction matrix\n",
    "        self.w0 = 0.\n",
    "        self.w = np.zeros(self.p)\n",
    "        self.V = np.random.random((self.p, self.k))\n",
    "\n",
    "    def predict(self, x):\n",
    "        \"\"\"\n",
    "        Return a predicted value for an input vector x.\n",
    "        \"\"\"\n",
    "\n",
    "        # efficient vectorized implementation of the pairwise interaction term\n",
    "        interaction = float(np.sum(np.dot(self.V.T, np.array([x]).T) ** 2 - \n",
    "                                   np.dot(self.V.T ** 2, np.array([x]).T ** 2)) / 2.)\n",
    "\n",
    "        # Add the pairwise interaction to the linear weight\n",
    "        return self.w0 + np.inner(self.w, x) + interaction\n",
    "\n",
    "    def fit(self):\n",
    "        \"\"\"\n",
    "        Learn the model parameters with SGD. Iterate until convergence.\n",
    "        \"\"\"\n",
    "\n",
    "        prev = float('inf')\n",
    "        current = 0.\n",
    "        eps = 1e-3\n",
    "\n",
    "        history = []\n",
    "        while abs(prev - current) > eps:\n",
    "            prev = current\n",
    "            for x, y in zip(self.X, self.y): # for each (x, y) training sample\n",
    "                current = self.update(x, y)\n",
    "            history.append(current)\n",
    "        return history\n",
    "\n",
    "    def update(self, x, y):\n",
    "        \"\"\"\n",
    "        Update the model parameters based on the given vector-value pair.\n",
    "        \"\"\"\n",
    "\n",
    "        # common part of the gradient\n",
    "        grad_base = self._loss_derivative(x, y)\n",
    "\n",
    "        grad_w0 = grad_base * 1.\n",
    "        self.w0 = self.w0 - self.learn_rate * (grad_w0 + 2. * self.l2_reg_w0 * self.w0)\n",
    "\n",
    "        # Update every paramether\n",
    "        for i in range(self.p):\n",
    "            if x[i] == 0.: continue\n",
    "            grad_w = grad_base * x[i]\n",
    "            self.w[i] = self.w[i] - self.learn_rate * (grad_w + 2. * self.l2_reg_w * self.w[i])\n",
    "\n",
    "            for f in range(self.k):\n",
    "                grad_V = grad_base * x[i] * (sum(x * self.V[:, f]) - x[i] * self.V[i, f])\n",
    "                self.V[i, f] = self.V[i, f] - \\\n",
    "                                self.learn_rate * (grad_V + 2. * self.l2_reg_V * self.V[i, f])\n",
    "\n",
    "        return self._evaluate()\n",
    "\n",
    "    @abstractmethod\n",
    "    def _loss_derivative(self, x, y):\n",
    "        # for grad_base\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def _evaluate(self):\n",
    "        # RMSE for regression\n",
    "        # AUC for classification\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Regression(SGD):\n",
    "    def _loss_derivative(self, x, y):\n",
    "        return 2. * (self.predict(x) - y)\n",
    "\n",
    "    def _evaluate(self):\n",
    "        # Efficient vectorized RMSE computation\n",
    "        linear = np.dot(self.X, np.array([self.w]).T).T\n",
    "        interaction = np.array([np.sum(np.dot(self.X, self.V) ** 2 - np.dot(self.X ** 2, self.V ** 2), axis=1) / 2.])\n",
    "        y_pred = self.w0 + linear + interaction\n",
    "        y_pred = y_pred.reshape((self.n))\n",
    "\n",
    "        return metrics.mean_squared_error(self.y, y_pred) ** 0.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Classification(SGD):\n",
    "    def _loss_derivative(self, x, y):\n",
    "        return (1. / (1. + np.e ** (- self.predict(x) * y)) - 1) * y\n",
    "\n",
    "    def _evaluate(self):\n",
    "        linear = np.dot(self.X, np.array([self.w]).T).T\n",
    "        interaction = np.array([np.sum(np.dot(self.X, self.V) ** 2 - np.dot(self.X ** 2, self.V ** 2), axis=1) / 2.])\n",
    "        y_pred = self.w0 + linear + interaction\n",
    "        y_pred = y_pred.reshape((self.n))\n",
    "\n",
    "        fpr, tpr, thresholds = metrics.roc_curve(self.y, y_pred, pos_label=1)\n",
    "        return metrics.auc(fpr, tpr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}